diff options
author | Kent Overstreet <kent.overstreet@gmail.com> | 2019-06-30 16:35:37 -0400 |
---|---|---|
committer | Kent Overstreet <kent.overstreet@linux.dev> | 2022-10-14 04:59:38 -0400 |
commit | 29f20baecf850c6381a48ceb57dcefac8e15bd5b (patch) | |
tree | 0436e075bd8499e67ae70887e3b7b1ee6f823394 | |
parent | d91a7b6ebca99e6140c913038614f153bc4d4610 (diff) |
bcachefs: Fixes for 4.19bcachefs-v4.19-backport-2.0
44 files changed, 977 insertions, 426 deletions
diff --git a/fs/bcachefs/acl.c b/fs/bcachefs/acl.c index 5c6ccf685094..0ed933f82189 100644 --- a/fs/bcachefs/acl.c +++ b/fs/bcachefs/acl.c @@ -212,7 +212,7 @@ bch2_acl_to_xattr(struct btree_trans *trans, return xattr; } -struct posix_acl *bch2_get_acl(struct inode *vinode, int type, bool rcu) +struct posix_acl *bch2_get_acl(struct inode *vinode, int type) { struct bch_inode_info *inode = to_bch_ei(vinode); struct bch_fs *c = inode->v.i_sb->s_fs_info; @@ -224,9 +224,6 @@ struct posix_acl *bch2_get_acl(struct inode *vinode, int type, bool rcu) struct bkey_s_c k; int ret; - if (rcu) - return ERR_PTR(-ECHILD); - bch2_trans_init(&trans, c, 0, 0); retry: bch2_trans_begin(&trans); @@ -292,8 +289,7 @@ int bch2_set_acl_trans(struct btree_trans *trans, subvol_inum inum, return ret == -ENOENT ? 0 : ret; } -int bch2_set_acl(struct user_namespace *mnt_userns, - struct inode *vinode, struct posix_acl *_acl, int type) +int bch2_set_acl(struct inode *vinode, struct posix_acl *_acl, int type) { struct bch_inode_info *inode = to_bch_ei(vinode); struct bch_fs *c = inode->v.i_sb->s_fs_info; @@ -318,7 +314,7 @@ retry: mode = inode_u.bi_mode; if (type == ACL_TYPE_ACCESS) { - ret = posix_acl_update_mode(mnt_userns, &inode->v, &mode, &acl); + ret = posix_acl_update_mode(&inode->v, &mode, &acl); if (ret) goto btree_err; } diff --git a/fs/bcachefs/acl.h b/fs/bcachefs/acl.h index 2d76a4897ba8..14cabbc91808 100644 --- a/fs/bcachefs/acl.h +++ b/fs/bcachefs/acl.h @@ -26,12 +26,12 @@ typedef struct { __le32 a_version; } bch_acl_header; -struct posix_acl *bch2_get_acl(struct inode *, int, bool); +struct posix_acl *bch2_get_acl(struct inode *, int); int bch2_set_acl_trans(struct btree_trans *, subvol_inum, struct bch_inode_unpacked *, struct posix_acl *, int); -int bch2_set_acl(struct user_namespace *, struct inode *, struct posix_acl *, int); +int bch2_set_acl(struct inode *, struct posix_acl *, int); int bch2_acl_chmod(struct btree_trans *, subvol_inum, struct bch_inode_unpacked *, umode_t, struct posix_acl **); diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index d0d7690a4940..c2f97d03dad6 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -1072,7 +1072,7 @@ static int bch2_discard_one_bucket(struct btree_trans *trans, blkdev_issue_discard(ca->disk_sb.bdev, k.k->p.offset * ca->mi.bucket_size, ca->mi.bucket_size, - GFP_KERNEL); + GFP_KERNEL, 0); ret = bch2_trans_relock(trans); if (ret) @@ -1386,7 +1386,7 @@ void bch2_recalc_capacity(struct bch_fs *c) lockdep_assert_held(&c->state_lock); for_each_online_member(ca, c, i) { - struct backing_dev_info *bdi = ca->disk_sb.bdev->bd_disk->bdi; + struct backing_dev_info *bdi = ca->disk_sb.bdev->bd_bdi; ra_pages += bdi->ra_pages; } diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index ccac2a3fcdf7..38f5753107c6 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -204,6 +204,7 @@ #include <linux/zstd.h> #include "bcachefs_format.h" +#include "darray.h" #include "errcode.h" #include "fifo.h" #include "opts.h" @@ -211,6 +212,7 @@ #define dynamic_fault(...) 0 #define race_fault(...) 0 +#define fallthrough do {} while (0) /* fallthrough */ #define trace_and_count(_c, _name, ...) \ do { \ @@ -827,7 +829,7 @@ struct bch_fs { ZSTD_parameters zstd_params; struct crypto_shash *sha256; - struct crypto_sync_skcipher *chacha20; + struct crypto_skcipher *chacha20; struct crypto_shash *poly1305; atomic64_t key_version; diff --git a/fs/bcachefs/btree_cache.c b/fs/bcachefs/btree_cache.c index f84b50869de2..67661cbf0ae8 100644 --- a/fs/bcachefs/btree_cache.c +++ b/fs/bcachefs/btree_cache.c @@ -93,7 +93,8 @@ static int btree_node_data_alloc(struct bch_fs *c, struct btree *b, gfp_t gfp) if (!b->data) return -ENOMEM; #ifdef __KERNEL__ - b->aux_data = vmalloc_exec(btree_aux_data_bytes(b), gfp); + b->aux_data = __vmalloc(btree_aux_data_bytes(b), gfp, + PAGE_KERNEL_EXEC); #else b->aux_data = mmap(NULL, btree_aux_data_bytes(b), PROT_READ|PROT_WRITE|PROT_EXEC, @@ -118,9 +119,6 @@ static struct btree *__btree_node_mem_alloc(struct bch_fs *c, gfp_t gfp) bkey_btree_ptr_init(&b->key); __six_lock_init(&b->c.lock, "b->c.lock", &bch2_btree_node_lock_key); -#ifdef CONFIG_DEBUG_LOCK_ALLOC - lockdep_set_no_check_recursion(&b->c.lock.dep_map); -#endif INIT_LIST_HEAD(&b->list); INIT_LIST_HEAD(&b->write_blocked); b->byte_order = ilog2(btree_bytes(c)); @@ -385,14 +383,6 @@ static unsigned long bch2_btree_cache_count(struct shrinker *shrink, return btree_cache_can_free(bc); } -static void bch2_btree_cache_shrinker_to_text(struct printbuf *out, struct shrinker *shrink) -{ - struct bch_fs *c = container_of(shrink, struct bch_fs, - btree_cache.shrink); - - bch2_btree_cache_to_text(out, c); -} - void bch2_fs_btree_cache_exit(struct bch_fs *c) { struct btree_cache *bc = &c->btree_cache; @@ -476,9 +466,8 @@ int bch2_fs_btree_cache_init(struct bch_fs *c) bc->shrink.count_objects = bch2_btree_cache_count; bc->shrink.scan_objects = bch2_btree_cache_scan; - bc->shrink.to_text = bch2_btree_cache_shrinker_to_text; bc->shrink.seeks = 4; - ret = register_shrinker(&bc->shrink, "%s/btree_cache", c->name); + ret = register_shrinker(&bc->shrink); out: pr_verbose_init(c->opts, "ret %i", ret); return ret; diff --git a/fs/bcachefs/btree_io.c b/fs/bcachefs/btree_io.c index dd6b536ced6a..da55b8d9a173 100644 --- a/fs/bcachefs/btree_io.c +++ b/fs/bcachefs/btree_io.c @@ -33,8 +33,6 @@ void bch2_btree_node_io_unlock(struct btree *b) void bch2_btree_node_io_lock(struct btree *b) { - BUG_ON(lock_class_is_held(&bch2_btree_node_lock_key)); - wait_on_bit_lock_io(&b->flags, BTREE_NODE_write_in_flight, TASK_UNINTERRUPTIBLE); } @@ -53,16 +51,12 @@ void __bch2_btree_node_wait_on_write(struct btree *b) void bch2_btree_node_wait_on_read(struct btree *b) { - BUG_ON(lock_class_is_held(&bch2_btree_node_lock_key)); - wait_on_bit_io(&b->flags, BTREE_NODE_read_in_flight, TASK_UNINTERRUPTIBLE); } void bch2_btree_node_wait_on_write(struct btree *b) { - BUG_ON(lock_class_is_held(&bch2_btree_node_lock_key)); - wait_on_bit_io(&b->flags, BTREE_NODE_write_in_flight, TASK_UNINTERRUPTIBLE); } @@ -1156,7 +1150,8 @@ static void btree_node_read_work(struct work_struct *work) bch_info(c, "retrying read"); ca = bch_dev_bkey_exists(c, rb->pick.ptr.dev); rb->have_ioref = bch2_dev_get_ioref(ca, READ); - bio_reset(bio, NULL, REQ_OP_READ|REQ_SYNC|REQ_META); + bio_reset(bio); + bio->bi_opf = REQ_OP_READ|REQ_SYNC|REQ_META; bio->bi_iter.bi_sector = rb->pick.ptr.offset; bio->bi_iter.bi_size = btree_bytes(c); @@ -1434,10 +1429,8 @@ static int btree_node_read_all_replicas(struct bch_fs *c, struct btree *b, bool for (i = 0; i < ra->nr; i++) { ra->buf[i] = mempool_alloc(&c->btree_bounce_pool, GFP_NOFS); - ra->bio[i] = bio_alloc_bioset(NULL, - buf_pages(ra->buf[i], btree_bytes(c)), - REQ_OP_READ|REQ_SYNC|REQ_META, - GFP_NOFS, + ra->bio[i] = bio_alloc_bioset(GFP_NOFS, buf_pages(ra->buf[i], + btree_bytes(c)), &c->btree_bio); } @@ -1453,6 +1446,7 @@ static int btree_node_read_all_replicas(struct bch_fs *c, struct btree *b, bool rb->have_ioref = bch2_dev_get_ioref(ca, READ); rb->idx = i; rb->pick = pick; + rb->bio.bi_opf = REQ_OP_READ|REQ_SYNC|REQ_META; rb->bio.bi_iter.bi_sector = pick.ptr.offset; rb->bio.bi_end_io = btree_node_read_all_replicas_endio; bch2_bio_map(&rb->bio, ra->buf[i], btree_bytes(c)); @@ -1519,10 +1513,8 @@ void bch2_btree_node_read(struct bch_fs *c, struct btree *b, ca = bch_dev_bkey_exists(c, pick.ptr.dev); - bio = bio_alloc_bioset(NULL, - buf_pages(b->data, btree_bytes(c)), - REQ_OP_READ|REQ_SYNC|REQ_META, - GFP_NOIO, + bio = bio_alloc_bioset(GFP_NOIO, buf_pages(b->data, + btree_bytes(c)), &c->btree_bio); rb = container_of(bio, struct btree_read_bio, bio); rb->c = c; @@ -1532,6 +1524,7 @@ void bch2_btree_node_read(struct bch_fs *c, struct btree *b, rb->have_ioref = bch2_dev_get_ioref(ca, READ); rb->pick = pick; INIT_WORK(&rb->work, btree_node_read_work); + bio->bi_opf = REQ_OP_READ|REQ_SYNC|REQ_META; bio->bi_iter.bi_sector = pick.ptr.offset; bio->bi_end_io = btree_node_read_endio; bch2_bio_map(bio, b->data, btree_bytes(c)); @@ -1994,10 +1987,8 @@ do_write: trace_and_count(c, btree_node_write, b, bytes_to_write, sectors_to_write); - wbio = container_of(bio_alloc_bioset(NULL, + wbio = container_of(bio_alloc_bioset(GFP_NOIO, buf_pages(data, sectors_to_write << 9), - REQ_OP_WRITE|REQ_META, - GFP_NOIO, &c->btree_bio), struct btree_write_bio, wbio.bio); wbio_init(&wbio->wbio.bio); @@ -2007,6 +1998,7 @@ do_write: wbio->wbio.c = c; wbio->wbio.used_mempool = used_mempool; wbio->wbio.first_btree_write = !b->written; + wbio->wbio.bio.bi_opf = REQ_OP_WRITE|REQ_META; wbio->wbio.bio.bi_end_io = btree_node_write_endio; wbio->wbio.bio.bi_private = b; diff --git a/fs/bcachefs/btree_io.h b/fs/bcachefs/btree_io.h index 8af853642123..a24e19039ac0 100644 --- a/fs/bcachefs/btree_io.h +++ b/fs/bcachefs/btree_io.h @@ -114,7 +114,7 @@ static inline int bset_encrypt(struct bch_fs *c, struct bset *i, unsigned offset if (ret) return ret; - nonce = nonce_add(nonce, round_up(bytes, CHACHA_BLOCK_SIZE)); + nonce = nonce_add(nonce, round_up(bytes, CHACHA20_BLOCK_SIZE)); } return bch2_encrypt(c, BSET_CSUM_TYPE(i), nonce, i->_data, diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index af658390bd67..176c488a25ea 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -16,7 +16,7 @@ #include "replicas.h" #include "subvolume.h" -#include <linux/prandom.h> +#include <linux/random.h> #include <linux/prefetch.h> #include <trace/events/bcachefs.h> @@ -2845,8 +2845,6 @@ void __bch2_trans_init(struct btree_trans *trans, struct bch_fs *c, const char * struct btree_transaction_stats *s; struct btree_trans *pos; - BUG_ON(lock_class_is_held(&bch2_btree_node_lock_key)); - memset(trans, 0, sizeof(*trans)); trans->c = c; trans->fn = fn; diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index 910f6d7bc961..7bdedf786345 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -141,13 +141,13 @@ int __must_check bch2_btree_path_traverse(struct btree_trans *, struct btree_path *, unsigned); struct btree_path *bch2_path_get(struct btree_trans *, enum btree_id, struct bpos, unsigned, unsigned, unsigned, unsigned long); -inline struct bkey_s_c bch2_btree_path_peek_slot(struct btree_path *, struct bkey *); +struct bkey_s_c bch2_btree_path_peek_slot(struct btree_path *, struct bkey *); struct bkey_i *bch2_btree_journal_peek_slot(struct btree_trans *, struct btree_iter *, struct bpos); -inline void bch2_btree_path_level_init(struct btree_trans *, - struct btree_path *, struct btree *); +void bch2_btree_path_level_init(struct btree_trans *, + struct btree_path *, struct btree *); #ifdef CONFIG_BCACHEFS_DEBUG void bch2_trans_verify_paths(struct btree_trans *); diff --git a/fs/bcachefs/btree_key_cache.c b/fs/bcachefs/btree_key_cache.c index 35e941949f49..7dc68afdb96a 100644 --- a/fs/bcachefs/btree_key_cache.c +++ b/fs/bcachefs/btree_key_cache.c @@ -813,7 +813,7 @@ static unsigned long bch2_btree_key_cache_scan(struct shrinker *shrink, do { struct rhash_head *pos, *next; - pos = rht_ptr_rcu(rht_bucket(tbl, bc->shrink_iter)); + pos = *rht_bucket(tbl, bc->shrink_iter); while (!rht_is_a_nulls(pos)) { next = rht_dereference_bucket_rcu(pos->next, tbl, bc->shrink_iter); @@ -930,17 +930,8 @@ void bch2_fs_btree_key_cache_init_early(struct btree_key_cache *c) INIT_LIST_HEAD(&c->freed_nonpcpu); } -static void bch2_btree_key_cache_shrinker_to_text(struct printbuf *out, struct shrinker *shrink) -{ - struct btree_key_cache *bc = - container_of(shrink, struct btree_key_cache, shrink); - - bch2_btree_key_cache_to_text(out, bc); -} - int bch2_fs_btree_key_cache_init(struct btree_key_cache *bc) { - struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache); int ret; bc->pcpu_freed = alloc_percpu(struct btree_key_cache_freelist); @@ -956,8 +947,7 @@ int bch2_fs_btree_key_cache_init(struct btree_key_cache *bc) bc->shrink.seeks = 1; bc->shrink.count_objects = bch2_btree_key_cache_count; bc->shrink.scan_objects = bch2_btree_key_cache_scan; - bc->shrink.to_text = bch2_btree_key_cache_shrinker_to_text; - return register_shrinker(&bc->shrink, "%s/btree_key_cache", c->name); + return register_shrinker(&bc->shrink); } void bch2_btree_key_cache_to_text(struct printbuf *out, struct btree_key_cache *c) diff --git a/fs/bcachefs/btree_locking.c b/fs/bcachefs/btree_locking.c index 93a6ebed3aba..93f4eb782e7a 100644 --- a/fs/bcachefs/btree_locking.c +++ b/fs/bcachefs/btree_locking.c @@ -622,13 +622,6 @@ void bch2_trans_unlock(struct btree_trans *trans) trans_for_each_path(trans, path) __bch2_btree_path_unlock(trans, path); - - /* - * bch2_gc_btree_init_recurse() doesn't use btree iterators for walking - * btree nodes, it implements its own walking: - */ - EBUG_ON(!trans->is_initial_gc && - lock_class_is_held(&bch2_btree_node_lock_key)); } bool bch2_trans_locked(struct btree_trans *trans) diff --git a/fs/bcachefs/checksum.c b/fs/bcachefs/checksum.c index b5850a761b91..eab1e7b51ec8 100644 --- a/fs/bcachefs/checksum.c +++ b/fs/bcachefs/checksum.c @@ -12,7 +12,7 @@ #include <linux/random.h> #include <linux/scatterlist.h> #include <crypto/algapi.h> -#include <crypto/chacha.h> +#include <crypto/chacha20.h> #include <crypto/hash.h> #include <crypto/poly1305.h> #include <crypto/skcipher.h> @@ -94,14 +94,14 @@ static void bch2_checksum_update(struct bch2_checksum_state *state, const void * } } -static inline int do_encrypt_sg(struct crypto_sync_skcipher *tfm, +static inline int do_encrypt_sg(struct crypto_skcipher *tfm, struct nonce nonce, struct scatterlist *sg, size_t len) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm); + SKCIPHER_REQUEST_ON_STACK(req, tfm); int ret; - skcipher_request_set_sync_tfm(req, tfm); + skcipher_request_set_tfm(req, tfm); skcipher_request_set_crypt(req, sg, sg, len, nonce.d); ret = crypto_skcipher_encrypt(req); @@ -111,7 +111,7 @@ static inline int do_encrypt_sg(struct crypto_sync_skcipher *tfm, return ret; } -static inline int do_encrypt(struct crypto_sync_skcipher *tfm, +static inline int do_encrypt(struct crypto_skcipher *tfm, struct nonce nonce, void *buf, size_t len) { @@ -155,8 +155,8 @@ static inline int do_encrypt(struct crypto_sync_skcipher *tfm, int bch2_chacha_encrypt_key(struct bch_key *key, struct nonce nonce, void *buf, size_t len) { - struct crypto_sync_skcipher *chacha20 = - crypto_alloc_sync_skcipher("chacha20", 0, 0); + struct crypto_skcipher *chacha20 = + crypto_alloc_skcipher("chacha20", 0, 0); int ret; if (!chacha20) { @@ -164,8 +164,7 @@ int bch2_chacha_encrypt_key(struct bch_key *key, struct nonce nonce, return PTR_ERR(chacha20); } - ret = crypto_skcipher_setkey(&chacha20->base, - (void *) key, sizeof(*key)); + ret = crypto_skcipher_setkey(chacha20, (void *) key, sizeof(*key)); if (ret) { pr_err("crypto_skcipher_setkey() error: %i", ret); goto err; @@ -173,7 +172,7 @@ int bch2_chacha_encrypt_key(struct bch_key *key, struct nonce nonce, ret = do_encrypt(chacha20, nonce, buf, len); err: - crypto_free_sync_skcipher(chacha20); + crypto_free_skcipher(chacha20); return ret; } @@ -270,7 +269,7 @@ static struct bch_csum __bch2_checksum_bio(struct bch_fs *c, unsigned type, kunmap_atomic(p); } #else - __bio_for_each_bvec(bv, bio, *iter, *iter) + __bio_for_each_contig_segment(bv, bio, *iter, *iter) bch2_checksum_update(&state, page_address(bv.bv_page) + bv.bv_offset, bv.bv_len); #endif @@ -293,7 +292,7 @@ static struct bch_csum __bch2_checksum_bio(struct bch_fs *c, unsigned type, kunmap_atomic(p); } #else - __bio_for_each_bvec(bv, bio, *iter, *iter) + __bio_for_each_contig_segment(bv, bio, *iter, *iter) crypto_shash_update(desc, page_address(bv.bv_page) + bv.bv_offset, bv.bv_len); @@ -556,7 +555,7 @@ static int bch2_alloc_ciphers(struct bch_fs *c) int ret; if (!c->chacha20) - c->chacha20 = crypto_alloc_sync_skcipher("chacha20", 0, 0); + c->chacha20 = crypto_alloc_skcipher("chacha20", 0, 0); ret = PTR_ERR_OR_ZERO(c->chacha20); if (ret) { @@ -641,7 +640,7 @@ int bch2_enable_encryption(struct bch_fs *c, bool keyed) goto err; } - ret = crypto_skcipher_setkey(&c->chacha20->base, + ret = crypto_skcipher_setkey(c->chacha20, (void *) &key.key, sizeof(key.key)); if (ret) goto err; @@ -669,7 +668,7 @@ void bch2_fs_encryption_exit(struct bch_fs *c) if (!IS_ERR_OR_NULL(c->poly1305)) crypto_free_shash(c->poly1305); if (!IS_ERR_OR_NULL(c->chacha20)) - crypto_free_sync_skcipher(c->chacha20); + crypto_free_skcipher(c->chacha20); if (!IS_ERR_OR_NULL(c->sha256)) crypto_free_shash(c->sha256); } @@ -701,7 +700,7 @@ int bch2_fs_encryption_init(struct bch_fs *c) if (ret) goto out; - ret = crypto_skcipher_setkey(&c->chacha20->base, + ret = crypto_skcipher_setkey(c->chacha20, (void *) &key.key, sizeof(key.key)); if (ret) goto out; diff --git a/fs/bcachefs/checksum.h b/fs/bcachefs/checksum.h index c86c3c05d620..0fd4c19cd7b6 100644 --- a/fs/bcachefs/checksum.h +++ b/fs/bcachefs/checksum.h @@ -7,7 +7,7 @@ #include "super-io.h" #include <linux/crc64.h> -#include <crypto/chacha.h> +#include <crypto/chacha20.h> static inline bool bch2_checksum_mergeable(unsigned type) { @@ -140,9 +140,9 @@ static inline bool bch2_crc_cmp(struct bch_csum l, struct bch_csum r) /* for skipping ahead and encrypting/decrypting at an offset: */ static inline struct nonce nonce_add(struct nonce nonce, unsigned offset) { - EBUG_ON(offset & (CHACHA_BLOCK_SIZE - 1)); + EBUG_ON(offset & (CHACHA20_BLOCK_SIZE - 1)); - le32_add_cpu(&nonce.d[0], offset / CHACHA_BLOCK_SIZE); + le32_add_cpu(&nonce.d[0], offset / CHACHA20_BLOCK_SIZE); return nonce; } diff --git a/fs/bcachefs/compress.c b/fs/bcachefs/compress.c index f692f35a6a98..691009fc2431 100644 --- a/fs/bcachefs/compress.c +++ b/fs/bcachefs/compress.c @@ -45,7 +45,7 @@ static bool bio_phys_contig(struct bio *bio, struct bvec_iter start) struct bvec_iter iter; void *expected_start = NULL; - __bio_for_each_bvec(bv, bio, iter, start) { + __bio_for_each_segment(bv, bio, iter, start) { if (expected_start && expected_start != page_address(bv.bv_page) + bv.bv_offset) return false; @@ -197,9 +197,9 @@ static int __bio_uncompress(struct bch_fs *c, struct bio *src, goto err; workspace = mempool_alloc(&c->decompress_workspace, GFP_NOIO); - ctx = zstd_init_dctx(workspace, zstd_dctx_workspace_bound()); + ctx = ZSTD_initDCtx(workspace, ZSTD_DCtxWorkspaceBound()); - ret = zstd_decompress_dctx(ctx, + ret = ZSTD_decompressDCtx(ctx, dst_data, dst_len, src_data.b + 4, real_src_len); @@ -333,8 +333,8 @@ static int attempt_compress(struct bch_fs *c, return strm.total_out; } case BCH_COMPRESSION_TYPE_zstd: { - ZSTD_CCtx *ctx = zstd_init_cctx(workspace, - zstd_cctx_workspace_bound(&c->zstd_params.cParams)); + ZSTD_CCtx *ctx = ZSTD_initCCtx(workspace, + ZSTD_CCtxWorkspaceBound(c->zstd_params.cParams)); /* * ZSTD requires that when we decompress we pass in the exact @@ -347,11 +347,11 @@ static int attempt_compress(struct bch_fs *c, * factor (7 bytes) from the dst buffer size to account for * that. */ - size_t len = zstd_compress_cctx(ctx, + size_t len = ZSTD_compressCCtx(ctx, dst + 4, dst_len - 4 - 7, src, src_len, - &c->zstd_params); - if (zstd_is_error(len)) + c->zstd_params); + if (ZSTD_isError(len)) return 0; *((__le32 *) dst) = cpu_to_le32(len); @@ -546,7 +546,7 @@ static int __bch2_fs_compress_init(struct bch_fs *c, u64 features) { size_t decompress_workspace_size = 0; bool decompress_workspace_needed; - ZSTD_parameters params = zstd_get_params(0, c->opts.encoded_extent_max); + ZSTD_parameters params = ZSTD_getParams(0, c->opts.encoded_extent_max, 0); struct { unsigned feature; unsigned type; @@ -558,8 +558,8 @@ static int __bch2_fs_compress_init(struct bch_fs *c, u64 features) zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL), zlib_inflate_workspacesize(), }, { BCH_FEATURE_zstd, BCH_COMPRESSION_TYPE_zstd, - zstd_cctx_workspace_bound(¶ms.cParams), - zstd_dctx_workspace_bound() }, + ZSTD_CCtxWorkspaceBound(params.cParams), + ZSTD_DCtxWorkspaceBound() }, }, *i; int ret = 0; diff --git a/fs/bcachefs/darray.h b/fs/bcachefs/darray.h index 519ab9b96e67..6de0c1c8ae36 100644 --- a/fs/bcachefs/darray.h +++ b/fs/bcachefs/darray.h @@ -11,6 +11,19 @@ #include "util.h" #include <linux/slab.h> +static inline void * __must_check krealloc_array(void *p, + size_t new_n, + size_t new_size, + gfp_t flags) +{ + size_t bytes; + + if (unlikely(check_mul_overflow(new_n, new_size, &bytes))) + return NULL; + + return krealloc(p, bytes, flags); +} + #define DARRAY(type) \ struct { \ size_t nr, size; \ diff --git a/fs/bcachefs/debug.c b/fs/bcachefs/debug.c index d87131f58c80..485f8df64730 100644 --- a/fs/bcachefs/debug.c +++ b/fs/bcachefs/debug.c @@ -45,11 +45,11 @@ static bool bch2_btree_verify_replica(struct bch_fs *c, struct btree *b, if (!bch2_dev_get_ioref(ca, READ)) return false; - bio = bio_alloc_bioset(ca->disk_sb.bdev, - buf_pages(n_sorted, btree_bytes(c)), - REQ_OP_READ|REQ_META, - GFP_NOIO, - &c->btree_bio); + bio = bio_alloc_bioset(GFP_NOIO, + buf_pages(n_sorted, btree_bytes(c)), + &c->btree_bio); + bio_set_dev(bio, ca->disk_sb.bdev); + bio->bi_opf = REQ_OP_READ|REQ_META; bio->bi_iter.bi_sector = pick.ptr.offset; bch2_bio_map(bio, n_sorted, btree_bytes(c)); diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index d3fa2d7ae1a2..ab8fdbb61e1c 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -407,16 +407,13 @@ static void ec_block_io(struct bch_fs *c, struct ec_stripe_buf *buf, this_cpu_add(ca->io_done->sectors[rw][data_type], buf->size); while (offset < bytes) { - unsigned nr_iovecs = min_t(size_t, BIO_MAX_VECS, + unsigned nr_iovecs = min_t(size_t, BIO_MAX_PAGES, DIV_ROUND_UP(bytes, PAGE_SIZE)); unsigned b = min_t(size_t, bytes - offset, nr_iovecs << PAGE_SHIFT); struct ec_bio *ec_bio; - ec_bio = container_of(bio_alloc_bioset(ca->disk_sb.bdev, - nr_iovecs, - rw, - GFP_KERNEL, + ec_bio = container_of(bio_alloc_bioset(GFP_KERNEL, nr_iovecs, &c->ec_bioset), struct ec_bio, bio); @@ -424,6 +421,9 @@ static void ec_block_io(struct bch_fs *c, struct ec_stripe_buf *buf, ec_bio->buf = buf; ec_bio->idx = idx; + bio_set_dev(&ec_bio->bio, ca->disk_sb.bdev); + bio_set_op_attrs(&ec_bio->bio, rw, 0); + ec_bio->bio.bi_iter.bi_sector = ptr->offset + buf->offset + (offset >> 9); ec_bio->bio.bi_end_io = ec_block_endio; ec_bio->bio.bi_private = cl; diff --git a/fs/bcachefs/errcode.c b/fs/bcachefs/errcode.c index cc9ce0be356e..0db8aa75ee28 100644 --- a/fs/bcachefs/errcode.c +++ b/fs/bcachefs/errcode.c @@ -3,8 +3,6 @@ #include "bcachefs.h" #include "errcode.h" -#include <linux/errname.h> - static const char * const bch2_errcode_strs[] = { #define x(class, err) [BCH_ERR_##err - BCH_ERR_START] = #err, BCH_ERRCODES() @@ -20,6 +18,223 @@ static unsigned bch2_errcode_parents[] = { #undef x }; +/* + * Ensure these tables do not accidentally become gigantic if some + * huge errno makes it in. On most architectures, the first table will + * only have about 140 entries, but mips and parisc have more sparsely + * allocated errnos (with EHWPOISON = 257 on parisc, and EDQUOT = 1133 + * on mips), so this wastes a bit of space on those - though we + * special case the EDQUOT case. + */ +#define E(err) [err + BUILD_BUG_ON_ZERO(err <= 0 || err > 300)] = "-" #err +static const char *names_0[] = { + E(E2BIG), + E(EACCES), + E(EADDRINUSE), + E(EADDRNOTAVAIL), + E(EADV), + E(EAFNOSUPPORT), + E(EALREADY), + E(EBADE), + E(EBADF), + E(EBADFD), + E(EBADMSG), + E(EBADR), + E(EBADRQC), + E(EBADSLT), + E(EBFONT), + E(EBUSY), +#ifdef ECANCELLED + E(ECANCELLED), +#endif + E(ECHILD), + E(ECHRNG), + E(ECOMM), + E(ECONNABORTED), + E(ECONNRESET), + E(EDEADLOCK), + E(EDESTADDRREQ), + E(EDOM), + E(EDOTDOT), +#ifndef CONFIG_MIPS + E(EDQUOT), +#endif + E(EEXIST), + E(EFAULT), + E(EFBIG), + E(EHOSTDOWN), + E(EHOSTUNREACH), + E(EHWPOISON), + E(EIDRM), + E(EILSEQ), +#ifdef EINIT + E(EINIT), +#endif + E(EINPROGRESS), + E(EINTR), + E(EINVAL), + E(EIO), + E(EISCONN), + E(EISDIR), + E(EISNAM), + E(EKEYEXPIRED), + E(EKEYREJECTED), + E(EKEYREVOKED), + E(EL2HLT), + E(EL2NSYNC), + E(EL3HLT), + E(EL3RST), + E(ELIBACC), + E(ELIBBAD), + E(ELIBEXEC), + E(ELIBMAX), + E(ELIBSCN), + E(ELNRNG), + E(ELOOP), + E(EMEDIUMTYPE), + E(EMFILE), + E(EMLINK), + E(EMSGSIZE), + E(EMULTIHOP), + E(ENAMETOOLONG), + E(ENAVAIL), + E(ENETDOWN), + E(ENETRESET), + E(ENETUNREACH), + E(ENFILE), + E(ENOANO), + E(ENOBUFS), + E(ENOCSI), + E(ENODATA), + E(ENODEV), + E(ENOENT), + E(ENOEXEC), + E(ENOKEY), + E(ENOLCK), + E(ENOLINK), + E(ENOMEDIUM), + E(ENOMEM), + E(ENOMSG), + E(ENONET), + E(ENOPKG), + E(ENOPROTOOPT), + E(ENOSPC), + E(ENOSR), + E(ENOSTR), +#ifdef ENOSYM + E(ENOSYM), +#endif + E(ENOSYS), + E(ENOTBLK), + E(ENOTCONN), + E(ENOTDIR), + E(ENOTEMPTY), + E(ENOTNAM), + E(ENOTRECOVERABLE), + E(ENOTSOCK), + E(ENOTTY), + E(ENOTUNIQ), + E(ENXIO), + E(EOPNOTSUPP), + E(EOVERFLOW), + E(EOWNERDEAD), + E(EPERM), + E(EPFNOSUPPORT), + E(EPIPE), +#ifdef EPROCLIM + E(EPROCLIM), +#endif + E(EPROTO), + E(EPROTONOSUPPORT), + E(EPROTOTYPE), + E(ERANGE), + E(EREMCHG), +#ifdef EREMDEV + E(EREMDEV), +#endif + E(EREMOTE), + E(EREMOTEIO), +#ifdef EREMOTERELEASE + E(EREMOTERELEASE), +#endif + E(ERESTART), + E(ERFKILL), + E(EROFS), +#ifdef ERREMOTE + E(ERREMOTE), +#endif + E(ESHUTDOWN), + E(ESOCKTNOSUPPORT), + E(ESPIPE), + E(ESRCH), + E(ESRMNT), + E(ESTALE), + E(ESTRPIPE), + E(ETIME), + E(ETIMEDOUT), + E(ETOOMANYREFS), + E(ETXTBSY), + E(EUCLEAN), + E(EUNATCH), + E(EUSERS), + E(EXDEV), + E(EXFULL), + + E(ECANCELED), /* ECANCELLED */ + E(EAGAIN), /* EWOULDBLOCK */ + E(ECONNREFUSED), /* EREFUSED */ + E(EDEADLK), /* EDEADLOCK */ +}; +#undef E + +#define E(err) [err - 512 + BUILD_BUG_ON_ZERO(err < 512 || err > 550)] = "-" #err +static const char *names_512[] = { + E(ERESTARTSYS), + E(ERESTARTNOINTR), + E(ERESTARTNOHAND), + E(ENOIOCTLCMD), + E(ERESTART_RESTARTBLOCK), + E(EPROBE_DEFER), + E(EOPENSTALE), + + E(EBADHANDLE), + E(ENOTSYNC), + E(EBADCOOKIE), + E(ENOTSUPP), + E(ETOOSMALL), + E(ESERVERFAULT), + E(EBADTYPE), + E(EJUKEBOX), + E(EIOCBQUEUED), + E(ERECALLCONFLICT), +}; +#undef E + +static const char *__errname(unsigned err) +{ + if (err < ARRAY_SIZE(names_0)) + return names_0[err]; + if (err >= 512 && err - 512 < ARRAY_SIZE(names_512)) + return names_512[err - 512]; + /* But why? */ + if (IS_ENABLED(CONFIG_MIPS) && err == EDQUOT) /* 1133 */ + return "-EDQUOT"; + return NULL; +} + +/* + * errname(EIO) -> "EIO" + * errname(-EIO) -> "-EIO" + */ +static const char *errname(int err) +{ + const char *name = __errname(abs(err)); + if (!name) + return NULL; + + return err > 0 ? name + 1 : name; +} + const char *bch2_err_str(int err) { const char *errstr; diff --git a/fs/bcachefs/fs-io.c b/fs/bcachefs/fs-io.c index 02ef3430a30b..70374fa463b4 100644 --- a/fs/bcachefs/fs-io.c +++ b/fs/bcachefs/fs-io.c @@ -35,15 +35,6 @@ #include <trace/events/bcachefs.h> #include <trace/events/writeback.h> -static inline bool bio_full(struct bio *bio, unsigned len) -{ - if (bio->bi_vcnt >= bio->bi_max_vecs) - return true; - if (bio->bi_iter.bi_size > UINT_MAX - len) - return true; - return false; -} - static inline struct address_space *faults_disabled_mapping(void) { return (void *) (((unsigned long) current->faults_disabled_mapping) & ~1UL); @@ -297,13 +288,28 @@ static inline struct bch_page_state *bch2_page_state(struct page *page) /* for newly allocated pages: */ static void __bch2_page_state_release(struct page *page) { - kfree(detach_page_private(page)); + struct bch_page_state *s = __bch2_page_state(page); + + if (!s) + return; + + ClearPagePrivate(page); + set_page_private(page, 0); + put_page(page); + kfree(s); } static void bch2_page_state_release(struct page *page) { - EBUG_ON(!PageLocked(page)); - __bch2_page_state_release(page); + struct bch_page_state *s = bch2_page_state(page); + + if (!s) + return; + + ClearPagePrivate(page); + set_page_private(page, 0); + put_page(page); + kfree(s); } /* for newly allocated pages: */ @@ -317,7 +323,13 @@ static struct bch_page_state *__bch2_page_state_create(struct page *page, return NULL; spin_lock_init(&s->lock); - attach_page_private(page, s); + /* + * migrate_page_move_mapping() assumes that pages with private data + * have their count elevated by 1. + */ + get_page(page); + set_page_private(page, (unsigned long) s); + SetPagePrivate(page); return s; } @@ -434,20 +446,22 @@ static void mark_pagecache_unallocated(struct bch_inode_info *inode, { pgoff_t index = start >> PAGE_SECTORS_SHIFT; pgoff_t end_index = (end - 1) >> PAGE_SECTORS_SHIFT; - struct folio_batch fbatch; - unsigned i, j; + struct pagevec pvec; if (end <= start) return; - folio_batch_init(&fbatch); + pagevec_init(&pvec); - while (filemap_get_folios(inode->v.i_mapping, - &index, end_index, &fbatch)) { - for (i = 0; i < folio_batch_count(&fbatch); i++) { - struct folio *folio = fbatch.folios[i]; - u64 pg_start = folio->index << PAGE_SECTORS_SHIFT; - u64 pg_end = (folio->index + 1) << PAGE_SECTORS_SHIFT; + do { + unsigned nr_pages, i, j; + + nr_pages = pagevec_lookup_range(&pvec, inode->v.i_mapping, + &index, end_index); + for (i = 0; i < nr_pages; i++) { + struct page *page = pvec.pages[i]; + u64 pg_start = page->index << PAGE_SECTORS_SHIFT; + u64 pg_end = (page->index + 1) << PAGE_SECTORS_SHIFT; unsigned pg_offset = max(start, pg_start) - pg_start; unsigned pg_len = min(end, pg_end) - pg_offset - pg_start; struct bch_page_state *s; @@ -456,8 +470,8 @@ static void mark_pagecache_unallocated(struct bch_inode_info *inode, BUG_ON(pg_offset >= PAGE_SECTORS); BUG_ON(pg_offset + pg_len > PAGE_SECTORS); - folio_lock(folio); - s = bch2_page_state(&folio->page); + lock_page(page); + s = bch2_page_state(page); if (s) { spin_lock(&s->lock); @@ -466,11 +480,10 @@ static void mark_pagecache_unallocated(struct bch_inode_info *inode, spin_unlock(&s->lock); } - folio_unlock(folio); + unlock_page(page); } - folio_batch_release(&fbatch); - cond_resched(); - } + pagevec_release(&pvec); + } while (index <= end_index); } static void mark_pagecache_reserved(struct bch_inode_info *inode, @@ -479,21 +492,23 @@ static void mark_pagecache_reserved(struct bch_inode_info *inode, struct bch_fs *c = inode->v.i_sb->s_fs_info; pgoff_t index = start >> PAGE_SECTORS_SHIFT; pgoff_t end_index = (end - 1) >> PAGE_SECTORS_SHIFT; - struct folio_batch fbatch; + struct pagevec pvec; s64 i_sectors_delta = 0; - unsigned i, j; if (end <= start) return; - folio_batch_init(&fbatch); + pagevec_init(&pvec); + + do { + unsigned nr_pages, i, j; - while (filemap_get_folios(inode->v.i_mapping, - &index, end_index, &fbatch)) { - for (i = 0; i < folio_batch_count(&fbatch); i++) { - struct folio *folio = fbatch.folios[i]; - u64 pg_start = folio->index << PAGE_SECTORS_SHIFT; - u64 pg_end = (folio->index + 1) << PAGE_SECTORS_SHIFT; + nr_pages = pagevec_lookup_range(&pvec, inode->v.i_mapping, + &index, end_index); + for (i = 0; i < nr_pages; i++) { + struct page *page = pvec.pages[i]; + u64 pg_start = page->index << PAGE_SECTORS_SHIFT; + u64 pg_end = (page->index + 1) << PAGE_SECTORS_SHIFT; unsigned pg_offset = max(start, pg_start) - pg_start; unsigned pg_len = min(end, pg_end) - pg_offset - pg_start; struct bch_page_state *s; @@ -502,8 +517,8 @@ static void mark_pagecache_reserved(struct bch_inode_info *inode, BUG_ON(pg_offset >= PAGE_SECTORS); BUG_ON(pg_offset + pg_len > PAGE_SECTORS); - folio_lock(folio); - s = bch2_page_state(&folio->page); + lock_page(page); + s = bch2_page_state(page); if (s) { spin_lock(&s->lock); @@ -522,11 +537,10 @@ static void mark_pagecache_reserved(struct bch_inode_info *inode, spin_unlock(&s->lock); } - folio_unlock(folio); + unlock_page(page); } - folio_batch_release(&fbatch); - cond_resched(); - } + pagevec_release(&pvec); + } while (index <= end_index); i_sectors_acct(c, inode, NULL, i_sectors_delta); } @@ -840,31 +854,62 @@ out: return ret; } -void bch2_invalidate_folio(struct folio *folio, size_t offset, size_t length) +void bch2_invalidatepage(struct page *page, unsigned int offset, + unsigned int length) { - if (offset || length < folio_size(folio)) + if (offset || length < PAGE_SIZE) return; - bch2_clear_page_bits(&folio->page); + bch2_clear_page_bits(page); +} + +int bch2_releasepage(struct page *page, gfp_t gfp_mask) +{ + if (PageDirty(page)) + return 0; + + bch2_clear_page_bits(page); + return 1; } -bool bch2_release_folio(struct folio *folio, gfp_t gfp_mask) +#ifdef CONFIG_MIGRATION +int bch2_migrate_page(struct address_space *mapping, struct page *newpage, + struct page *page, enum migrate_mode mode) { - if (folio_test_dirty(folio) || folio_test_writeback(folio)) - return false; + int ret; + + EBUG_ON(!PageLocked(page)); + EBUG_ON(!PageLocked(newpage)); + + ret = migrate_page_move_mapping(mapping, newpage, page, NULL, mode, 0); + if (ret != MIGRATEPAGE_SUCCESS) + return ret; + + if (PagePrivate(page)) { + ClearPagePrivate(page); + get_page(newpage); + set_page_private(newpage, page_private(page)); + set_page_private(page, 0); + put_page(page); + SetPagePrivate(newpage); + } - bch2_clear_page_bits(&folio->page); - return true; + if (mode != MIGRATE_SYNC_NO_COPY) + migrate_page_copy(newpage, page); + else + migrate_page_states(newpage, page); + return MIGRATEPAGE_SUCCESS; } +#endif /* readpage(s): */ static void bch2_readpages_end_io(struct bio *bio) { - struct bvec_iter_all iter; struct bio_vec *bv; + unsigned i; - bio_for_each_segment_all(bv, bio, iter) { + bio_for_each_segment_all(bv, bio, i) { struct page *page = bv->bv_page; if (!bio->bi_status) { @@ -883,29 +928,31 @@ struct readpages_iter { struct address_space *mapping; struct page **pages; unsigned nr_pages; + unsigned nr_added; unsigned idx; pgoff_t offset; }; static int readpages_iter_init(struct readpages_iter *iter, - struct readahead_control *ractl) + struct address_space *mapping, + struct list_head *pages, unsigned nr_pages) { - unsigned i, nr_pages = readahead_count(ractl); - memset(iter, 0, sizeof(*iter)); - iter->mapping = ractl->mapping; - iter->offset = readahead_index(ractl); - iter->nr_pages = nr_pages; + iter->mapping = mapping; + iter->offset = list_last_entry(pages, struct page, lru)->index; iter->pages = kmalloc_array(nr_pages, sizeof(struct page *), GFP_NOFS); if (!iter->pages) return -ENOMEM; - nr_pages = __readahead_batch(ractl, iter->pages, nr_pages); - for (i = 0; i < nr_pages; i++) { - __bch2_page_state_create(iter->pages[i], __GFP_NOFAIL); - put_page(iter->pages[i]); + while (!list_empty(pages)) { + struct page *page = list_last_entry(pages, struct page, lru); + + __bch2_page_state_create(page, __GFP_NOFAIL); + + iter->pages[iter->nr_pages++] = page; + list_del(&page->lru); } return 0; @@ -913,9 +960,41 @@ static int readpages_iter_init(struct readpages_iter *iter, static inline struct page *readpage_iter_next(struct readpages_iter *iter) { - if (iter->idx >= iter->nr_pages) - return NULL; + struct page *page; + unsigned i; + int ret; + + BUG_ON(iter->idx > iter->nr_added); + BUG_ON(iter->nr_added > iter->nr_pages); + + if (iter->idx < iter->nr_added) + goto out; + + while (1) { + if (iter->idx == iter->nr_pages) + return NULL; + + ret = add_to_page_cache_lru_vec(iter->mapping, + iter->pages + iter->nr_added, + iter->nr_pages - iter->nr_added, + iter->offset + iter->nr_added, + GFP_NOFS); + if (ret > 0) + break; + + page = iter->pages[iter->nr_added]; + iter->idx++; + iter->nr_added++; + __bch2_page_state_release(page); + put_page(page); + } + + iter->nr_added += ret; + + for (i = iter->idx; i < iter->nr_added; i++) + put_page(iter->pages[i]); +out: EBUG_ON(iter->pages[iter->idx]->index != iter->offset + iter->idx); return iter->pages[iter->idx]; @@ -953,8 +1032,11 @@ static void readpage_bio_extend(struct readpages_iter *iter, if (!get_more) break; - page = xa_load(&iter->mapping->i_pages, page_offset); - if (page && !xa_is_value(page)) + rcu_read_lock(); + page = radix_tree_lookup(&iter->mapping->i_pages, page_offset); + rcu_read_unlock(); + + if (page && !radix_tree_exceptional_entry(page)) break; page = __page_cache_alloc(readahead_gfp_mask(iter->mapping)); @@ -1087,9 +1169,10 @@ err: bch2_bkey_buf_exit(&sk, c); } -void bch2_readahead(struct readahead_control *ractl) +int bch2_readpages(struct file *file, struct address_space *mapping, + struct list_head *pages, unsigned nr_pages) { - struct bch_inode_info *inode = to_bch_ei(ractl->mapping->host); + struct bch_inode_info *inode = to_bch_ei(mapping->host); struct bch_fs *c = inode->v.i_sb->s_fs_info; struct bch_io_opts opts = io_opts(c, &inode->ei_inode); struct btree_trans trans; @@ -1097,7 +1180,7 @@ void bch2_readahead(struct readahead_control *ractl) struct readpages_iter readpages_iter; int ret; - ret = readpages_iter_init(&readpages_iter, ractl); + ret = readpages_iter_init(&readpages_iter, mapping, pages, nr_pages); BUG_ON(ret); bch2_trans_init(&trans, c, 0, 0); @@ -1109,14 +1192,14 @@ void bch2_readahead(struct readahead_control *ractl) unsigned n = min_t(unsigned, readpages_iter.nr_pages - readpages_iter.idx, - BIO_MAX_VECS); + BIO_MAX_PAGES); struct bch_read_bio *rbio = - rbio_init(bio_alloc_bioset(NULL, n, REQ_OP_READ, - GFP_NOFS, &c->bio_read), + rbio_init(bio_alloc_bioset(GFP_NOFS, n, &c->bio_read), opts); readpages_iter.idx++; + bio_set_op_attrs(&rbio->bio, REQ_OP_READ, 0); rbio->bio.bi_iter.bi_sector = (sector_t) index << PAGE_SECTORS_SHIFT; rbio->bio.bi_end_io = bch2_readpages_end_io; BUG_ON(!bio_add_page(&rbio->bio, page, PAGE_SIZE, 0)); @@ -1129,6 +1212,8 @@ void bch2_readahead(struct readahead_control *ractl) bch2_trans_exit(&trans); kfree(readpages_iter.pages); + + return 0; } static void __bchfs_readpage(struct bch_fs *c, struct bch_read_bio *rbio, @@ -1148,6 +1233,20 @@ static void __bchfs_readpage(struct bch_fs *c, struct bch_read_bio *rbio, bch2_trans_exit(&trans); } +int bch2_readpage(struct file *file, struct page *page) +{ + struct bch_inode_info *inode = to_bch_ei(page->mapping->host); + struct bch_fs *c = inode->v.i_sb->s_fs_info; + struct bch_io_opts opts = io_opts(c, &inode->ei_inode); + struct bch_read_bio *rbio; + + rbio = rbio_init(bio_alloc_bioset(GFP_NOFS, 1, &c->bio_read), opts); + rbio->bio.bi_end_io = bch2_readpages_end_io; + + __bchfs_readpage(c, rbio, inode_inum(inode), page); + return 0; +} + static void bch2_read_single_page_end_io(struct bio *bio) { complete(bio->bi_private); @@ -1162,7 +1261,7 @@ static int bch2_read_single_page(struct page *page, int ret; DECLARE_COMPLETION_ONSTACK(done); - rbio = rbio_init(bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_NOFS, &c->bio_read), + rbio = rbio_init(bio_alloc_bioset(GFP_NOFS, 1, &c->bio_read), io_opts(c, &inode->ei_inode)); rbio->bio.bi_private = &done; rbio->bio.bi_end_io = bch2_read_single_page_end_io; @@ -1180,16 +1279,6 @@ static int bch2_read_single_page(struct page *page, return 0; } -int bch2_read_folio(struct file *file, struct folio *folio) -{ - struct page *page = &folio->page; - int ret; - - ret = bch2_read_single_page(page, page->mapping); - folio_unlock(folio); - return bch2_err_class(ret); -} - /* writepages: */ struct bch_writepage_state { @@ -1219,35 +1308,34 @@ static void bch2_writepage_io_done(struct closure *cl) struct bch_writepage_io, cl); struct bch_fs *c = io->op.c; struct bio *bio = &io->op.wbio.bio; - struct bvec_iter_all iter; struct bio_vec *bvec; - unsigned i; + unsigned i, j; if (io->op.error) { set_bit(EI_INODE_ERROR, &io->inode->ei_flags); - bio_for_each_segment_all(bvec, bio, iter) { + bio_for_each_segment_all(bvec, bio, i) { struct bch_page_state *s; SetPageError(bvec->bv_page); - mapping_set_error(bvec->bv_page->mapping, -EIO); + mapping_set_error(io->inode->v.i_mapping, -EIO); s = __bch2_page_state(bvec->bv_page); spin_lock(&s->lock); - for (i = 0; i < PAGE_SECTORS; i++) - s->s[i].nr_replicas = 0; + for (j = 0; j < PAGE_SECTORS; j++) + s->s[j].nr_replicas = 0; spin_unlock(&s->lock); } } if (io->op.flags & BCH_WRITE_WROTE_DATA_INLINE) { - bio_for_each_segment_all(bvec, bio, iter) { + bio_for_each_segment_all(bvec, bio, i) { struct bch_page_state *s; s = __bch2_page_state(bvec->bv_page); spin_lock(&s->lock); - for (i = 0; i < PAGE_SECTORS; i++) - s->s[i].nr_replicas = 0; + for (j = 0; j < PAGE_SECTORS; j++) + s->s[j].nr_replicas = 0; spin_unlock(&s->lock); } } @@ -1271,7 +1359,7 @@ static void bch2_writepage_io_done(struct closure *cl) */ i_sectors_acct(c, io->inode, NULL, io->op.i_sectors_delta); - bio_for_each_segment_all(bvec, bio, iter) { + bio_for_each_segment_all(bvec, bio, i) { struct bch_page_state *s = __bch2_page_state(bvec->bv_page); if (atomic_dec_and_test(&s->write_count)) @@ -1303,9 +1391,7 @@ static void bch2_writepage_io_alloc(struct bch_fs *c, { struct bch_write_op *op; - w->io = container_of(bio_alloc_bioset(NULL, BIO_MAX_VECS, - REQ_OP_WRITE, - GFP_NOFS, + w->io = container_of(bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, &c->writepage_bioset), struct bch_writepage_io, op.wbio.bio); @@ -1425,9 +1511,9 @@ do_io: if (w->io && (w->io->op.res.nr_replicas != nr_replicas_this_write || - bio_full(&w->io->op.wbio.bio, PAGE_SIZE) || + bio_full(&w->io->op.wbio.bio) || w->io->op.wbio.bio.bi_iter.bi_size + (sectors << 9) >= - (BIO_MAX_VECS * PAGE_SIZE) || + (BIO_MAX_PAGES * PAGE_SIZE) || bio_end_sector(&w->io->op.wbio.bio) != sector)) bch2_writepage_do_io(w); @@ -1477,7 +1563,7 @@ int bch2_writepages(struct address_space *mapping, struct writeback_control *wbc /* buffered writes: */ int bch2_write_begin(struct file *file, struct address_space *mapping, - loff_t pos, unsigned len, + loff_t pos, unsigned len, unsigned flags, struct page **pagep, void **fsdata) { struct bch_inode_info *inode = to_bch_ei(mapping->host); @@ -1497,7 +1583,7 @@ int bch2_write_begin(struct file *file, struct address_space *mapping, bch2_pagecache_add_get(&inode->ei_pagecache_lock); - page = grab_cache_page_write_begin(mapping, index); + page = grab_cache_page_write_begin(mapping, index, flags); if (!page) goto err_unlock; @@ -1628,7 +1714,7 @@ static int __bch2_buffered_write(struct bch_inode_info *inode, bch2_page_reservation_init(c, inode, &res); for (i = 0; i < nr_pages; i++) { - pages[i] = grab_cache_page_write_begin(mapping, index + i); + pages[i] = grab_cache_page_write_begin(mapping, index + i, 0); if (!pages[i]) { nr_pages = i; if (!i) { @@ -1689,8 +1775,8 @@ static int __bch2_buffered_write(struct bch_inode_info *inode, unsigned pg_offset = (offset + copied) & (PAGE_SIZE - 1); unsigned pg_len = min_t(unsigned, len - copied, PAGE_SIZE - pg_offset); - unsigned pg_copied = copy_page_from_iter_atomic(page, - pg_offset, pg_len,iter); + unsigned pg_copied = iov_iter_copy_from_user_atomic(page, + iter, pg_offset, pg_len); if (!pg_copied) break; @@ -1703,6 +1789,7 @@ static int __bch2_buffered_write(struct bch_inode_info *inode, } flush_dcache_page(page); + iov_iter_advance(iter, pg_copied); copied += pg_copied; if (pg_copied != pg_len) @@ -1772,11 +1859,11 @@ again: * to check that the address is actually valid, when atomic * usercopies are used, below. */ - if (unlikely(fault_in_iov_iter_readable(iter, bytes))) { + if (unlikely(iov_iter_fault_in_readable(iter, bytes))) { bytes = min_t(unsigned long, iov_iter_count(iter), PAGE_SIZE - offset); - if (unlikely(fault_in_iov_iter_readable(iter, bytes))) { + if (unlikely(iov_iter_fault_in_readable(iter, bytes))) { ret = -EFAULT; break; } @@ -1820,6 +1907,18 @@ again: /* O_DIRECT reads */ +static void bio_release_pages(struct bio *bio, bool mark_dirty) +{ + struct bio_vec *bvec; + unsigned i; + + bio_for_each_segment_all(bvec, bio, i) { + if (mark_dirty && !PageCompound(bvec->bv_page)) + set_page_dirty_lock(bvec->bv_page); + put_page(bvec->bv_page); + } +} + static void bio_check_or_release(struct bio *bio, bool check_dirty) { if (check_dirty) { @@ -1834,7 +1933,7 @@ static void bch2_dio_read_complete(struct closure *cl) { struct dio_read *dio = container_of(cl, struct dio_read, cl); - dio->req->ki_complete(dio->req, dio->ret); + dio->req->ki_complete(dio->req, dio->ret, 0); bio_check_or_release(&dio->rbio.bio, dio->should_dirty); } @@ -1882,10 +1981,8 @@ static int bch2_direct_IO_read(struct kiocb *req, struct iov_iter *iter) shorten = iov_iter_count(iter) - round_up(ret, block_bytes(c)); iter->count -= shorten; - bio = bio_alloc_bioset(NULL, - bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS), - REQ_OP_READ, - GFP_KERNEL, + bio = bio_alloc_bioset(GFP_KERNEL, + iov_iter_npages(iter, BIO_MAX_PAGES), &c->dio_read_bioset); bio->bi_end_io = bch2_direct_IO_read_endio; @@ -1919,10 +2016,8 @@ static int bch2_direct_IO_read(struct kiocb *req, struct iov_iter *iter) goto start; while (iter->count) { - bio = bio_alloc_bioset(NULL, - bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS), - REQ_OP_READ, - GFP_KERNEL, + bio = bio_alloc_bioset(GFP_KERNEL, + iov_iter_npages(iter, BIO_MAX_PAGES), &c->bio_read); bio->bi_end_io = bch2_direct_IO_read_split_endio; start: @@ -2055,9 +2150,8 @@ static long bch2_dio_write_loop(struct dio_write *dio) struct bch_inode_info *inode = file_bch_inode(req->ki_filp); struct bch_fs *c = inode->v.i_sb->s_fs_info; struct bio *bio = &dio->op.wbio.bio; - struct bvec_iter_all iter; struct bio_vec *bv; - unsigned unaligned, iter_count; + unsigned i, unaligned, iter_count; bool sync = dio->sync, dropped_locks; long ret; @@ -2068,7 +2162,7 @@ static long bch2_dio_write_loop(struct dio_write *dio) iter_count = dio->iter.count; if (kthread && dio->mm) - kthread_use_mm(dio->mm); + use_mm(dio->mm); BUG_ON(current->faults_disabled_mapping); current->faults_disabled_mapping = mapping; @@ -2078,7 +2172,7 @@ static long bch2_dio_write_loop(struct dio_write *dio) current->faults_disabled_mapping = NULL; if (kthread && dio->mm) - kthread_unuse_mm(dio->mm); + unuse_mm(dio->mm); /* * If the fault handler returned an error but also signalled @@ -2175,9 +2269,8 @@ loop: i_size_write(&inode->v, req->ki_pos); spin_unlock(&inode->v.i_lock); - if (likely(!bio_flagged(bio, BIO_NO_PAGE_REF))) - bio_for_each_segment_all(bv, bio, iter) - put_page(bv->bv_page); + bio_for_each_segment_all(bv, bio, i) + put_page(bv->bv_page); bio->bi_vcnt = 0; if (dio->op.error) { @@ -2188,7 +2281,7 @@ loop: if (!dio->iter.count) break; - bio_reset(bio, NULL, REQ_OP_WRITE); + bio_reset(bio); reinit_completion(&dio->done); } @@ -2200,9 +2293,8 @@ err: if (dio->free_iov) kfree(dio->iter.iov); - if (likely(!bio_flagged(bio, BIO_NO_PAGE_REF))) - bio_for_each_segment_all(bv, bio, iter) - put_page(bv->bv_page); + bio_for_each_segment_all(bv, bio, i) + put_page(bv->bv_page); bio_put(bio); /* inode->i_dio_count is our ref on inode and thus bch_fs */ @@ -2212,7 +2304,7 @@ err: ret = bch2_err_class(ret); if (!sync) { - req->ki_complete(req, ret); + req->ki_complete(req, ret, 0); ret = -EIOCBQUEUED; } return ret; @@ -2271,10 +2363,8 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter) locked = false; } - bio = bio_alloc_bioset(NULL, - bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS), - REQ_OP_WRITE, - GFP_KERNEL, + bio = bio_alloc_bioset(GFP_KERNEL, + iov_iter_npages(iter, BIO_MAX_PAGES), &c->dio_write_bioset); dio = container_of(bio, struct dio_write, op.wbio.bio); init_completion(&dio->done); @@ -2563,7 +2653,7 @@ static int bch2_extend(struct user_namespace *mnt_userns, truncate_setsize(&inode->v, iattr->ia_size); - return bch2_setattr_nonsize(mnt_userns, inode, iattr); + return bch2_setattr_nonsize(inode, iattr); } static int bch2_truncate_finish_fn(struct bch_inode_info *inode, @@ -2685,7 +2775,7 @@ int bch2_truncate(struct user_namespace *mnt_userns, ret = bch2_write_inode(c, inode, bch2_truncate_finish_fn, NULL, 0); mutex_unlock(&inode->ei_update_lock); - ret = bch2_setattr_nonsize(mnt_userns, inode, iattr); + ret = bch2_setattr_nonsize(inode, iattr); err: bch2_pagecache_block_put(&inode->ei_pagecache_lock); return bch2_err_class(ret); @@ -3102,10 +3192,6 @@ long bch2_fallocate_dispatch(struct file *file, int mode, inode_dio_wait(&inode->v); bch2_pagecache_block_get(&inode->ei_pagecache_lock); - ret = file_modified(file); - if (ret) - goto err; - if (!(mode & ~(FALLOC_FL_KEEP_SIZE|FALLOC_FL_ZERO_RANGE))) ret = bchfs_fallocate(inode, mode, offset, len); else if (mode == (FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE)) @@ -3116,7 +3202,7 @@ long bch2_fallocate_dispatch(struct file *file, int mode, ret = bchfs_fcollapse_finsert(inode, offset, len, false); else ret = -EOPNOTSUPP; -err: + bch2_pagecache_block_put(&inode->ei_pagecache_lock); inode_unlock(&inode->v); percpu_ref_put(&c->writes); @@ -3173,6 +3259,235 @@ err: return bch2_quota_reservation_add(c, inode, res, sectors, true); } +static int generic_access_check_limits(struct file *file, loff_t pos, + loff_t *count) +{ + struct inode *inode = file->f_mapping->host; + loff_t max_size = inode->i_sb->s_maxbytes; + + if (!(file->f_flags & O_LARGEFILE)) + max_size = MAX_NON_LFS; + + if (unlikely(pos >= max_size)) + return -EFBIG; + *count = min(*count, max_size - pos); + return 0; +} + +static int generic_write_check_limits(struct file *file, loff_t pos, + loff_t *count) +{ + loff_t limit = rlimit(RLIMIT_FSIZE); + + if (limit != RLIM_INFINITY) { + if (pos >= limit) { + send_sig(SIGXFSZ, current, 0); + return -EFBIG; + } + *count = min(*count, limit - pos); + } + + return generic_access_check_limits(file, pos, count); +} + +static int generic_remap_checks(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t *req_count, unsigned int remap_flags) +{ + struct inode *inode_in = file_in->f_mapping->host; + struct inode *inode_out = file_out->f_mapping->host; + uint64_t count = *req_count; + uint64_t bcount; + loff_t size_in, size_out; + loff_t bs = inode_out->i_sb->s_blocksize; + int ret; + + /* The start of both ranges must be aligned to an fs block. */ + if (!IS_ALIGNED(pos_in, bs) || !IS_ALIGNED(pos_out, bs)) + return -EINVAL; + + /* Ensure offsets don't wrap. */ + if (pos_in + count < pos_in || pos_out + count < pos_out) + return -EINVAL; + + size_in = i_size_read(inode_in); + size_out = i_size_read(inode_out); + + /* Dedupe requires both ranges to be within EOF. */ + if ((remap_flags & REMAP_FILE_DEDUP) && + (pos_in >= size_in || pos_in + count > size_in || + pos_out >= size_out || pos_out + count > size_out)) + return -EINVAL; + + /* Ensure the infile range is within the infile. */ + if (pos_in >= size_in) + return -EINVAL; + count = min(count, size_in - (uint64_t)pos_in); + + ret = generic_access_check_limits(file_in, pos_in, &count); + if (ret) + return ret; + + ret = generic_write_check_limits(file_out, pos_out, &count); + if (ret) + return ret; + + /* + * If the user wanted us to link to the infile's EOF, round up to the + * next block boundary for this check. + * + * Otherwise, make sure the count is also block-aligned, having + * already confirmed the starting offsets' block alignment. + */ + if (pos_in + count == size_in) { + bcount = ALIGN(size_in, bs) - pos_in; + } else { + if (!IS_ALIGNED(count, bs)) + count = ALIGN_DOWN(count, bs); + bcount = count; + } + + /* Don't allow overlapped cloning within the same file. */ + if (inode_in == inode_out && + pos_out + bcount > pos_in && + pos_out < pos_in + bcount) + return -EINVAL; + + /* + * We shortened the request but the caller can't deal with that, so + * bounce the request back to userspace. + */ + if (*req_count != count && !(remap_flags & REMAP_FILE_CAN_SHORTEN)) + return -EINVAL; + + *req_count = count; + return 0; +} + +static int generic_remap_check_len(struct inode *inode_in, + struct inode *inode_out, + loff_t pos_out, + loff_t *len, + unsigned int remap_flags) +{ + u64 blkmask = i_blocksize(inode_in) - 1; + loff_t new_len = *len; + + if ((*len & blkmask) == 0) + return 0; + + if ((remap_flags & REMAP_FILE_DEDUP) || + pos_out + *len < i_size_read(inode_out)) + new_len &= ~blkmask; + + if (new_len == *len) + return 0; + + if (remap_flags & REMAP_FILE_CAN_SHORTEN) { + *len = new_len; + return 0; + } + + return (remap_flags & REMAP_FILE_DEDUP) ? -EBADE : -EINVAL; +} + +static int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t *len, unsigned int remap_flags) +{ + struct inode *inode_in = file_inode(file_in); + struct inode *inode_out = file_inode(file_out); + bool same_inode = (inode_in == inode_out); + int ret; + + /* Don't touch certain kinds of inodes */ + if (IS_IMMUTABLE(inode_out)) + return -EPERM; + + if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out)) + return -ETXTBSY; + + /* Don't reflink dirs, pipes, sockets... */ + if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode)) + return -EISDIR; + if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode)) + return -EINVAL; + + /* Zero length dedupe exits immediately; reflink goes to EOF. */ + if (*len == 0) { + loff_t isize = i_size_read(inode_in); + + if ((remap_flags & REMAP_FILE_DEDUP) || pos_in == isize) + return 0; + if (pos_in > isize) + return -EINVAL; + *len = isize - pos_in; + if (*len == 0) + return 0; + } + + /* Check that we don't violate system file offset limits. */ + ret = generic_remap_checks(file_in, pos_in, file_out, pos_out, len, + remap_flags); + if (ret) + return ret; + + /* Wait for the completion of any pending IOs on both files */ + inode_dio_wait(inode_in); + if (!same_inode) + inode_dio_wait(inode_out); + + ret = filemap_write_and_wait_range(inode_in->i_mapping, + pos_in, pos_in + *len - 1); + if (ret) + return ret; + + ret = filemap_write_and_wait_range(inode_out->i_mapping, + pos_out, pos_out + *len - 1); + if (ret) + return ret; + + /* + * Check that the extents are the same. + */ + if (remap_flags & REMAP_FILE_DEDUP) { + bool is_same = false; + + ret = vfs_dedupe_file_range_compare(inode_in, pos_in, + inode_out, pos_out, *len, &is_same); + if (ret) + return ret; + if (!is_same) + return -EBADE; + } + + ret = generic_remap_check_len(inode_in, inode_out, pos_out, len, + remap_flags); + if (ret) + return ret; + + /* If can't alter the file contents, we're done. */ + if (!(remap_flags & REMAP_FILE_DEDUP)) { + /* Update the timestamps, since we can alter file contents. */ + if (!(file_out->f_mode & FMODE_NOCMTIME)) { + ret = file_update_time(file_out); + if (ret) + return ret; + } + + /* + * Clear the security bits if the process is not being run by + * root. This keeps people from modifying setuid and setgid + * binaries. + */ + ret = file_remove_privs(file_out); + if (ret) + return ret; + } + + return 0; +} + loff_t bch2_remap_file_range(struct file *file_src, loff_t pos_src, struct file *file_dst, loff_t pos_dst, loff_t len, unsigned remap_flags) @@ -3276,40 +3591,36 @@ static loff_t bch2_seek_pagecache_data(struct inode *vinode, loff_t start_offset, loff_t end_offset) { - struct folio_batch fbatch; + struct address_space *mapping = vinode->i_mapping; + struct page *page; pgoff_t start_index = start_offset >> PAGE_SHIFT; pgoff_t end_index = end_offset >> PAGE_SHIFT; pgoff_t index = start_index; - unsigned i; loff_t ret; int offset; - folio_batch_init(&fbatch); - - while (filemap_get_folios(vinode->i_mapping, - &index, end_index, &fbatch)) { - for (i = 0; i < folio_batch_count(&fbatch); i++) { - struct folio *folio = fbatch.folios[i]; + while (index <= end_index) { + if (find_get_pages_range(mapping, &index, end_index, 1, &page)) { + lock_page(page); - folio_lock(folio); - - offset = page_data_offset(&folio->page, - folio->index == start_index + offset = page_data_offset(page, + page->index == start_index ? start_offset & (PAGE_SIZE - 1) : 0); if (offset >= 0) { - ret = clamp(((loff_t) folio->index << PAGE_SHIFT) + + ret = clamp(((loff_t) page->index << PAGE_SHIFT) + offset, start_offset, end_offset); - folio_unlock(folio); - folio_batch_release(&fbatch); + unlock_page(page); + put_page(page); return ret; } - folio_unlock(folio); + unlock_page(page); + put_page(page); + } else { + break; } - folio_batch_release(&fbatch); - cond_resched(); } return end_offset; diff --git a/fs/bcachefs/fs-io.h b/fs/bcachefs/fs-io.h index a8835298613a..f9e7f49b13c7 100644 --- a/fs/bcachefs/fs-io.h +++ b/fs/bcachefs/fs-io.h @@ -15,13 +15,15 @@ int __must_check bch2_write_inode_size(struct bch_fs *, struct bch_inode_info *, loff_t, unsigned); -int bch2_read_folio(struct file *, struct folio *); +int bch2_writepage(struct page *, struct writeback_control *); +int bch2_readpage(struct file *, struct page *); int bch2_writepages(struct address_space *, struct writeback_control *); -void bch2_readahead(struct readahead_control *); +int bch2_readpages(struct file *, struct address_space *, + struct list_head *, unsigned); int bch2_write_begin(struct file *, struct address_space *, loff_t, - unsigned, struct page **, void **); + unsigned, unsigned, struct page **, void **); int bch2_write_end(struct file *, struct address_space *, loff_t, unsigned, unsigned, struct page *, void *); @@ -34,6 +36,10 @@ int bch2_truncate(struct user_namespace *, struct bch_inode_info *, struct iattr *); long bch2_fallocate_dispatch(struct file *, int, loff_t, loff_t); +#define REMAP_FILE_ADVISORY (0) +#define REMAP_FILE_DEDUP (1 << 0) +#define REMAP_FILE_CAN_SHORTEN (1 << 1) + loff_t bch2_remap_file_range(struct file *, loff_t, struct file *, loff_t, loff_t, unsigned); @@ -41,8 +47,10 @@ loff_t bch2_llseek(struct file *, loff_t, int); vm_fault_t bch2_page_fault(struct vm_fault *); vm_fault_t bch2_page_mkwrite(struct vm_fault *); -void bch2_invalidate_folio(struct folio *, size_t, size_t); -bool bch2_release_folio(struct folio *, gfp_t); +void bch2_invalidatepage(struct page *, unsigned int, unsigned int); +int bch2_releasepage(struct page *, gfp_t); +int bch2_migrate_page(struct address_space *, struct page *, + struct page *, enum migrate_mode); void bch2_fs_fsio_exit(struct bch_fs *); int bch2_fs_fsio_init(struct bch_fs *); diff --git a/fs/bcachefs/fs-ioctl.c b/fs/bcachefs/fs-ioctl.c index 2bb680827b44..2ff9e344e2d6 100644 --- a/fs/bcachefs/fs-ioctl.c +++ b/fs/bcachefs/fs-ioctl.c @@ -93,7 +93,7 @@ static int bch2_ioc_setflags(struct bch_fs *c, return ret; inode_lock(&inode->v); - if (!inode_owner_or_capable(file_mnt_user_ns(file), &inode->v)) { + if (!inode_owner_or_capable(&inode->v)) { ret = -EACCES; goto setflags_out; } @@ -172,7 +172,7 @@ static int bch2_ioc_fssetxattr(struct bch_fs *c, return ret; inode_lock(&inode->v); - if (!inode_owner_or_capable(file_mnt_user_ns(file), &inode->v)) { + if (!inode_owner_or_capable(&inode->v)) { ret = -EACCES; goto err; } @@ -284,20 +284,22 @@ static int bch2_ioc_goingdown(struct bch_fs *c, u32 __user *arg) down_write(&c->vfs_sb->s_umount); switch (flags) { - case FSOP_GOING_FLAGS_DEFAULT: - ret = freeze_bdev(c->vfs_sb->s_bdev); + case FSOP_GOING_FLAGS_DEFAULT: { + struct super_block *sb = freeze_bdev(c->vfs_sb->s_bdev); if (ret) goto err; - bch2_journal_flush(&c->journal); - c->vfs_sb->s_flags |= SB_RDONLY; - bch2_fs_emergency_read_only(c); - thaw_bdev(c->vfs_sb->s_bdev); + if (sb && !IS_ERR(sb)) { + bch2_journal_flush(&c->journal); + c->vfs_sb->s_flags |= SB_RDONLY; + bch2_fs_emergency_read_only(c); + thaw_bdev(c->vfs_sb->s_bdev, sb); + } break; + } case FSOP_GOING_FLAGS_LOGFLUSH: bch2_journal_flush(&c->journal); - fallthrough; case FSOP_GOING_FLAGS_NOLOGFLUSH: c->vfs_sb->s_flags |= SB_RDONLY; @@ -393,8 +395,7 @@ retry: goto err3; } - error = inode_permission(file_mnt_user_ns(filp), - dir, MAY_WRITE | MAY_EXEC); + error = inode_permission(dir, MAY_WRITE | MAY_EXEC); if (error) goto err3; @@ -409,7 +410,7 @@ retry: !arg.src_ptr) snapshot_src.subvol = to_bch_ei(dir)->ei_inode.bi_subvol; - inode = __bch2_create(file_mnt_user_ns(filp), to_bch_ei(dir), + inode = __bch2_create(NULL, to_bch_ei(dir), dst_dentry, arg.mode|S_IFDIR, 0, snapshot_src, create_flags); error = PTR_ERR_OR_ZERO(inode); @@ -458,10 +459,8 @@ static long bch2_ioctl_subvolume_destroy(struct bch_fs *c, struct file *filp, dir = path.dentry->d_parent->d_inode; ret = __bch2_unlink(dir, path.dentry, true); - if (!ret) { - fsnotify_rmdir(dir, path.dentry); + if (!ret) d_delete(path.dentry); - } path_put(&path); return ret; diff --git a/fs/bcachefs/fs.c b/fs/bcachefs/fs.c index 57e6e21896e1..6e539594e7e5 100644 --- a/fs/bcachefs/fs.c +++ b/fs/bcachefs/fs.c @@ -265,6 +265,7 @@ __bch2_create(struct user_namespace *mnt_userns, unsigned flags) { struct bch_fs *c = dir->v.i_sb->s_fs_info; + struct user_namespace *ns = dir->v.i_sb->s_user_ns; struct btree_trans trans; struct bch_inode_unpacked dir_u; struct bch_inode_info *inode, *old; @@ -305,8 +306,8 @@ retry: inode_inum(dir), &dir_u, &inode_u, !(flags & BCH_CREATE_TMPFILE) ? &dentry->d_name : NULL, - from_kuid(mnt_userns, current_fsuid()), - from_kgid(mnt_userns, current_fsgid()), + from_kuid(ns, current_fsuid()), + from_kgid(ns, current_fsgid()), mode, rdev, default_acl, acl, snapshot_src, flags) ?: bch2_quota_acct(c, bch_qid(&inode_u), Q_INO, 1, @@ -410,12 +411,11 @@ static struct dentry *bch2_lookup(struct inode *vdir, struct dentry *dentry, return d_splice_alias(vinode, dentry); } -static int bch2_mknod(struct user_namespace *mnt_userns, - struct inode *vdir, struct dentry *dentry, +static int bch2_mknod(struct inode *vdir, struct dentry *dentry, umode_t mode, dev_t rdev) { struct bch_inode_info *inode = - __bch2_create(mnt_userns, to_bch_ei(vdir), dentry, mode, rdev, + __bch2_create(NULL, to_bch_ei(vdir), dentry, mode, rdev, (subvol_inum) { 0 }, 0); if (IS_ERR(inode)) @@ -425,11 +425,10 @@ static int bch2_mknod(struct user_namespace *mnt_userns, return 0; } -static int bch2_create(struct user_namespace *mnt_userns, - struct inode *vdir, struct dentry *dentry, +static int bch2_create(struct inode *vdir, struct dentry *dentry, umode_t mode, bool excl) { - return bch2_mknod(mnt_userns, vdir, dentry, mode|S_IFREG, 0); + return bch2_mknod(vdir, dentry, mode|S_IFREG, 0); } static int __bch2_link(struct bch_fs *c, @@ -518,15 +517,14 @@ static int bch2_unlink(struct inode *vdir, struct dentry *dentry) return __bch2_unlink(vdir, dentry, false); } -static int bch2_symlink(struct user_namespace *mnt_userns, - struct inode *vdir, struct dentry *dentry, +static int bch2_symlink(struct inode *vdir, struct dentry *dentry, const char *symname) { struct bch_fs *c = vdir->i_sb->s_fs_info; struct bch_inode_info *dir = to_bch_ei(vdir), *inode; int ret; - inode = __bch2_create(mnt_userns, dir, dentry, S_IFLNK|S_IRWXUGO, 0, + inode = __bch2_create(NULL, dir, dentry, S_IFLNK|S_IRWXUGO, 0, (subvol_inum) { 0 }, BCH_CREATE_TMPFILE); if (unlikely(IS_ERR(inode))) return PTR_ERR(inode); @@ -553,14 +551,12 @@ err: return ret; } -static int bch2_mkdir(struct user_namespace *mnt_userns, - struct inode *vdir, struct dentry *dentry, umode_t mode) +static int bch2_mkdir(struct inode *vdir, struct dentry *dentry, umode_t mode) { - return bch2_mknod(mnt_userns, vdir, dentry, mode|S_IFDIR, 0); + return bch2_mknod(vdir, dentry, mode|S_IFDIR, 0); } -static int bch2_rename2(struct user_namespace *mnt_userns, - struct inode *src_vdir, struct dentry *src_dentry, +static int bch2_rename2(struct inode *src_vdir, struct dentry *src_dentry, struct inode *dst_vdir, struct dentry *dst_dentry, unsigned flags) { @@ -666,8 +662,7 @@ err: return ret; } -static void bch2_setattr_copy(struct user_namespace *mnt_userns, - struct bch_inode_info *inode, +static void bch2_setattr_copy(struct bch_inode_info *inode, struct bch_inode_unpacked *bi, struct iattr *attr) { @@ -675,9 +670,9 @@ static void bch2_setattr_copy(struct user_namespace *mnt_userns, unsigned int ia_valid = attr->ia_valid; if (ia_valid & ATTR_UID) - bi->bi_uid = from_kuid(mnt_userns, attr->ia_uid); + bi->bi_uid = from_kuid(c->vfs_sb->s_user_ns, attr->ia_uid); if (ia_valid & ATTR_GID) - bi->bi_gid = from_kgid(mnt_userns, attr->ia_gid); + bi->bi_gid = from_kgid(c->vfs_sb->s_user_ns, attr->ia_gid); if (ia_valid & ATTR_SIZE) bi->bi_size = attr->ia_size; @@ -696,14 +691,13 @@ static void bch2_setattr_copy(struct user_namespace *mnt_userns, : inode->v.i_gid; if (!in_group_p(gid) && - !capable_wrt_inode_uidgid(mnt_userns, &inode->v, CAP_FSETID)) + !capable_wrt_inode_uidgid(&inode->v, CAP_FSETID)) mode &= ~S_ISGID; bi->bi_mode = mode; } } -int bch2_setattr_nonsize(struct user_namespace *mnt_userns, - struct bch_inode_info *inode, +int bch2_setattr_nonsize(struct bch_inode_info *inode, struct iattr *attr) { struct bch_fs *c = inode->v.i_sb->s_fs_info; @@ -740,7 +734,7 @@ retry: if (ret) goto btree_err; - bch2_setattr_copy(mnt_userns, inode, &inode_u, attr); + bch2_setattr_copy(inode, &inode_u, attr); if (attr->ia_valid & ATTR_MODE) { ret = bch2_acl_chmod(&trans, inode_inum(inode), &inode_u, @@ -772,8 +766,7 @@ err: return bch2_err_class(ret); } -static int bch2_getattr(struct user_namespace *mnt_userns, - const struct path *path, struct kstat *stat, +static int bch2_getattr(const struct path *path, struct kstat *stat, u32 request_mask, unsigned query_flags) { struct bch_inode_info *inode = to_bch_ei(d_inode(path->dentry)); @@ -813,28 +806,26 @@ static int bch2_getattr(struct user_namespace *mnt_userns, return 0; } -static int bch2_setattr(struct user_namespace *mnt_userns, - struct dentry *dentry, struct iattr *iattr) +static int bch2_setattr(struct dentry *dentry, struct iattr *iattr) { struct bch_inode_info *inode = to_bch_ei(dentry->d_inode); int ret; lockdep_assert_held(&inode->v.i_rwsem); - ret = setattr_prepare(mnt_userns, dentry, iattr); + ret = setattr_prepare(dentry, iattr); if (ret) return ret; return iattr->ia_valid & ATTR_SIZE - ? bch2_truncate(mnt_userns, inode, iattr) - : bch2_setattr_nonsize(mnt_userns, inode, iattr); + ? bch2_truncate(NULL, inode, iattr) + : bch2_setattr_nonsize(inode, iattr); } -static int bch2_tmpfile(struct user_namespace *mnt_userns, - struct inode *vdir, struct dentry *dentry, umode_t mode) +static int bch2_tmpfile(struct inode *vdir, struct dentry *dentry, umode_t mode) { struct bch_inode_info *inode = - __bch2_create(mnt_userns, to_bch_ei(vdir), dentry, mode, 0, + __bch2_create(NULL, to_bch_ei(vdir), dentry, mode, 0, (subvol_inum) { 0 }, BCH_CREATE_TMPFILE); if (IS_ERR(inode)) @@ -913,10 +904,6 @@ static int bch2_fiemap(struct inode *vinode, struct fiemap_extent_info *info, u32 snapshot; int ret = 0; - ret = fiemap_prep(&ei->v, info, start, &len, FIEMAP_FLAG_SYNC); - if (ret) - return ret; - if (start + len < start) return -EINVAL; @@ -1032,6 +1019,15 @@ static int bch2_vfs_readdir(struct file *file, struct dir_context *ctx) return bch2_readdir(c, inode_inum(inode), ctx); } +static int bch2_clone_file_range(struct file *file_src, loff_t pos_src, + struct file *file_dst, loff_t pos_dst, + u64 len) +{ + return bch2_remap_file_range(file_src, pos_src, + file_dst, pos_dst, + len, 0); +} + static const struct file_operations bch_file_operations = { .llseek = bch2_llseek, .read_iter = bch2_read_iter, @@ -1046,7 +1042,7 @@ static const struct file_operations bch_file_operations = { #ifdef CONFIG_COMPAT .compat_ioctl = bch2_compat_fs_ioctl, #endif - .remap_file_range = bch2_remap_file_range, + .clone_file_range = bch2_clone_file_range, }; static const struct inode_operations bch_file_inode_operations = { @@ -1113,17 +1109,17 @@ static const struct inode_operations bch_special_inode_operations = { }; static const struct address_space_operations bch_address_space_operations = { - .read_folio = bch2_read_folio, + .readpage = bch2_readpage, .writepages = bch2_writepages, - .readahead = bch2_readahead, - .dirty_folio = filemap_dirty_folio, + .readpages = bch2_readpages, + .set_page_dirty = __set_page_dirty_nobuffers, .write_begin = bch2_write_begin, .write_end = bch2_write_end, - .invalidate_folio = bch2_invalidate_folio, - .release_folio = bch2_release_folio, + .invalidatepage = bch2_invalidatepage, + .releasepage = bch2_releasepage, .direct_IO = noop_direct_IO, #ifdef CONFIG_MIGRATION - .migrate_folio = filemap_migrate_folio, + .migratepage = bch2_migrate_page, #endif .error_remove_page = generic_error_remove_page, }; @@ -1574,14 +1570,13 @@ static int bch2_sync_fs(struct super_block *sb, int wait) static struct bch_fs *bch2_path_to_fs(const char *path) { struct bch_fs *c; - dev_t dev; - int ret; + struct block_device *bdev = lookup_bdev(path); - ret = lookup_bdev(path, &dev); - if (ret) - return ERR_PTR(ret); + if (IS_ERR(bdev)) + return ERR_CAST(bdev); - c = bch2_dev_to_fs(dev); + c = bch2_dev_to_fs(bdev->bd_dev); + bdput(bdev); if (c) closure_put(&c->cl); return c ?: ERR_PTR(-ENOENT); @@ -1843,8 +1838,6 @@ got_sb: sb->s_xattr = bch2_xattr_handlers; sb->s_magic = BCACHEFS_STATFS_MAGIC; sb->s_time_gran = c->sb.nsec_per_time_unit; - sb->s_time_min = div_s64(S64_MIN, c->sb.time_units_per_sec) + 1; - sb->s_time_max = div_s64(S64_MAX, c->sb.time_units_per_sec); c->vfs_sb = sb; strlcpy(sb->s_id, c->name, sizeof(sb->s_id)); @@ -1852,7 +1845,9 @@ got_sb: if (ret) goto err_put_super; - sb->s_bdi->ra_pages = VM_READAHEAD_PAGES; + sb->s_bdi->congested_fn = bch2_congested; + sb->s_bdi->congested_data = c; + sb->s_bdi->ra_pages = VM_MAX_READAHEAD * 1024 / PAGE_SIZE; for_each_online_member(ca, c, i) { struct block_device *bdev = ca->disk_sb.bdev; @@ -1871,7 +1866,7 @@ got_sb: sb->s_flags |= SB_POSIXACL; #endif - sb->s_shrink.seeks = 0; + sb->s_shrink.seeks = 1; vinode = bch2_vfs_inode_get(c, BCACHEFS_ROOT_SUBVOL_INUM); ret = PTR_ERR_OR_ZERO(vinode); diff --git a/fs/bcachefs/fs.h b/fs/bcachefs/fs.h index 9f4b57e30e2a..bb005f768a54 100644 --- a/fs/bcachefs/fs.h +++ b/fs/bcachefs/fs.h @@ -186,8 +186,7 @@ void bch2_inode_update_after_write(struct btree_trans *, int __must_check bch2_write_inode(struct bch_fs *, struct bch_inode_info *, inode_set_fn, void *, unsigned); -int bch2_setattr_nonsize(struct user_namespace *, - struct bch_inode_info *, +int bch2_setattr_nonsize(struct bch_inode_info *, struct iattr *); int __bch2_unlink(struct inode *, struct dentry *, bool); diff --git a/fs/bcachefs/fsck.c b/fs/bcachefs/fsck.c index 12f2ef4417cb..fbcbf2953230 100644 --- a/fs/bcachefs/fsck.c +++ b/fs/bcachefs/fsck.c @@ -2087,8 +2087,8 @@ static void inc_link(struct bch_fs *c, struct snapshots_seen *s, if (inum < range_start || inum >= range_end) return; - link = __inline_bsearch(&key, links->d, links->nr, - sizeof(links->d[0]), nlink_cmp); + link = bsearch(&key, links->d, links->nr, + sizeof(links->d[0]), nlink_cmp); if (!link) return; diff --git a/fs/bcachefs/io.c b/fs/bcachefs/io.c index 558d0c232816..02a7153adbba 100644 --- a/fs/bcachefs/io.c +++ b/fs/bcachefs/io.c @@ -136,10 +136,10 @@ void bch2_latency_acct(struct bch_dev *ca, u64 submit_time, int rw) void bch2_bio_free_pages_pool(struct bch_fs *c, struct bio *bio) { - struct bvec_iter_all iter; struct bio_vec *bv; + unsigned i; - bio_for_each_segment_all(bv, bio, iter) + bio_for_each_segment_all(bv, bio, i) if (bv->bv_page != ZERO_PAGE(0)) mempool_free(bv->bv_page, &c->bio_bounce_pages); bio->bi_vcnt = 0; @@ -470,8 +470,8 @@ void bch2_submit_wbio_replicas(struct bch_write_bio *wbio, struct bch_fs *c, ca = bch_dev_bkey_exists(c, ptr->dev); if (to_entry(ptr + 1) < ptrs.end) { - n = to_wbio(bio_alloc_clone(NULL, &wbio->bio, - GFP_NOIO, &ca->replica_set)); + n = to_wbio(bio_clone_fast(&wbio->bio, GFP_NOIO, + &ca->replica_set)); n->bio.bi_end_io = wbio->bio.bi_end_io; n->bio.bi_private = wbio->bio.bi_private; @@ -712,10 +712,9 @@ static struct bio *bch2_write_bio_alloc(struct bch_fs *c, ? ((unsigned long) buf & (PAGE_SIZE - 1)) : 0), PAGE_SIZE); - pages = min(pages, BIO_MAX_VECS); + pages = min_t(unsigned, pages, BIO_MAX_PAGES); - bio = bio_alloc_bioset(NULL, pages, 0, - GFP_NOIO, &c->bio_write); + bio = bio_alloc_bioset(GFP_NOIO, pages, &c->bio_write); wbio = wbio_init(bio); wbio->put_bio = true; /* copy WRITE_SYNC flag */ @@ -1446,7 +1445,7 @@ static struct promote_op *__promote_alloc(struct bch_fs *c, goto err; rbio_init(&(*rbio)->bio, opts); - bio_init(&(*rbio)->bio, NULL, (*rbio)->bio.bi_inline_vecs, pages, 0); + bio_init(&(*rbio)->bio, (*rbio)->bio.bi_inline_vecs, pages); if (bch2_bio_alloc_pages(&(*rbio)->bio, sectors << 9, GFP_NOIO)) @@ -1461,7 +1460,7 @@ static struct promote_op *__promote_alloc(struct bch_fs *c, goto err; bio = &op->write.op.wbio.bio; - bio_init(bio, NULL, bio->bi_inline_vecs, pages, 0); + bio_init(bio, bio->bi_inline_vecs, pages); ret = bch2_data_update_init(c, &op->write, writepoint_hashed((unsigned long) current), @@ -2144,10 +2143,8 @@ get_bio: } else if (bounce) { unsigned sectors = pick.crc.compressed_size; - rbio = rbio_init(bio_alloc_bioset(NULL, + rbio = rbio_init(bio_alloc_bioset(GFP_NOIO, DIV_ROUND_UP(sectors, PAGE_SECTORS), - 0, - GFP_NOIO, &c->bio_read_split), orig->opts); @@ -2163,8 +2160,8 @@ get_bio: * from the whole bio, in which case we don't want to retry and * lose the error) */ - rbio = rbio_init(bio_alloc_clone(NULL, &orig->bio, GFP_NOIO, - &c->bio_read_split), + rbio = rbio_init(bio_clone_fast(&orig->bio, GFP_NOIO, + &c->bio_read_split), orig->opts); rbio->bio.bi_iter = iter; rbio->split = true; diff --git a/fs/bcachefs/journal.c b/fs/bcachefs/journal.c index ab594623341f..1e53ba70a448 100644 --- a/fs/bcachefs/journal.c +++ b/fs/bcachefs/journal.c @@ -1168,12 +1168,10 @@ int bch2_dev_journal_init(struct bch_dev *ca, struct bch_sb *sb) nr_bvecs = DIV_ROUND_UP(JOURNAL_ENTRY_SIZE_MAX, PAGE_SIZE); - ca->journal.bio = bio_kmalloc(nr_bvecs, GFP_KERNEL); + ca->journal.bio = bio_kmalloc(GFP_KERNEL, nr_bvecs); if (!ca->journal.bio) return -ENOMEM; - bio_init(ca->journal.bio, NULL, ca->journal.bio->bi_inline_vecs, nr_bvecs, 0); - ja->buckets = kcalloc(ja->nr, sizeof(u64), GFP_KERNEL); if (!ja->buckets) return -ENOMEM; diff --git a/fs/bcachefs/journal.h b/fs/bcachefs/journal.h index d3caa7ea7ce9..5c31fe81db9d 100644 --- a/fs/bcachefs/journal.h +++ b/fs/bcachefs/journal.h @@ -279,7 +279,7 @@ static inline void bch2_journal_res_put(struct journal *j, if (!res->ref) return; - lock_release(&j->res_map, _THIS_IP_); + lock_release(&j->res_map, 0, _THIS_IP_); while (res->u64s) bch2_journal_add_entry(j, res, diff --git a/fs/bcachefs/journal_io.c b/fs/bcachefs/journal_io.c index 68113a08f1b9..047fcf341388 100644 --- a/fs/bcachefs/journal_io.c +++ b/fs/bcachefs/journal_io.c @@ -873,10 +873,11 @@ reread: end - offset, buf->size >> 9); nr_bvecs = buf_pages(buf->data, sectors_read << 9); - bio = bio_kmalloc(nr_bvecs, GFP_KERNEL); - bio_init(bio, ca->disk_sb.bdev, bio->bi_inline_vecs, nr_bvecs, REQ_OP_READ); + bio = bio_kmalloc(GFP_KERNEL, nr_bvecs); + bio_set_dev(bio, ca->disk_sb.bdev); bio->bi_iter.bi_sector = offset; + bio_set_op_attrs(bio, REQ_OP_READ, 0); bch2_bio_map(bio, buf->data, sectors_read << 9); ret = submit_bio_wait(bio); @@ -1613,10 +1614,12 @@ static void do_journal_write(struct closure *cl) sectors); bio = ca->journal.bio; - bio_reset(bio, ca->disk_sb.bdev, REQ_OP_WRITE|REQ_SYNC|REQ_META); + bio_reset(bio); + bio_set_dev(bio, ca->disk_sb.bdev); bio->bi_iter.bi_sector = ptr->offset; bio->bi_end_io = journal_write_endio; bio->bi_private = ca; + bio->bi_opf = REQ_OP_WRITE|REQ_SYNC|REQ_META; BUG_ON(bio->bi_iter.bi_sector == ca->prev_journal_sector); ca->prev_journal_sector = bio->bi_iter.bi_sector; @@ -1794,7 +1797,9 @@ retry_alloc: percpu_ref_get(&ca->io_ref); bio = ca->journal.bio; - bio_reset(bio, ca->disk_sb.bdev, REQ_OP_FLUSH); + bio_reset(bio); + bio_set_dev(bio, ca->disk_sb.bdev); + bio->bi_opf = REQ_OP_FLUSH; bio->bi_end_io = journal_write_endio; bio->bi_private = ca; closure_bio_submit(bio, cl); diff --git a/fs/bcachefs/journal_reclaim.c b/fs/bcachefs/journal_reclaim.c index e69595bd1359..b11e50759fa5 100644 --- a/fs/bcachefs/journal_reclaim.c +++ b/fs/bcachefs/journal_reclaim.c @@ -12,6 +12,7 @@ #include <linux/kthread.h> #include <linux/sched/mm.h> +#include <linux/sched/task.h> #include <trace/events/bcachefs.h> /* Free space calculations: */ @@ -283,11 +284,11 @@ void bch2_journal_do_discards(struct journal *j) while (should_discard_bucket(j, ja)) { if (!c->opts.nochanges && ca->mi.discard && - bdev_max_discard_sectors(ca->disk_sb.bdev)) + blk_queue_discard(bdev_get_queue(ca->disk_sb.bdev))) blkdev_issue_discard(ca->disk_sb.bdev, bucket_to_sector(ca, ja->buckets[ja->discard_idx]), - ca->mi.bucket_size, GFP_NOIO); + ca->mi.bucket_size, GFP_NOIO, 0); spin_lock(&j->lock); ja->discard_idx = (ja->discard_idx + 1) % ja->nr; diff --git a/fs/bcachefs/move.c b/fs/bcachefs/move.c index 4f4dfaa7bfb7..0ebb1c1773be 100644 --- a/fs/bcachefs/move.c +++ b/fs/bcachefs/move.c @@ -277,7 +277,7 @@ static int bch2_move_extent(struct btree_trans *trans, io->read_sectors = k.k->size; io->write_sectors = k.k->size; - bio_init(&io->write.op.wbio.bio, NULL, io->bi_inline_vecs, pages, 0); + bio_init(&io->write.op.wbio.bio, io->bi_inline_vecs, pages); bio_set_prio(&io->write.op.wbio.bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0)); @@ -287,7 +287,7 @@ static int bch2_move_extent(struct btree_trans *trans, io->rbio.c = c; io->rbio.opts = io_opts; - bio_init(&io->rbio.bio, NULL, io->bi_inline_vecs, pages, 0); + bio_init(&io->rbio.bio, io->bi_inline_vecs, pages); io->rbio.bio.bi_vcnt = pages; bio_set_prio(&io->rbio.bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0)); io->rbio.bio.bi_iter.bi_size = sectors << 9; diff --git a/fs/bcachefs/move.h b/fs/bcachefs/move.h index c0fec69bbb6a..16b2dad30e64 100644 --- a/fs/bcachefs/move.h +++ b/fs/bcachefs/move.h @@ -60,8 +60,6 @@ int bch2_data_job(struct bch_fs *, struct bch_move_stats *, struct bch_ioctl_data); -inline void bch_move_stats_init(struct bch_move_stats *stats, - char *name); - +void bch_move_stats_init(struct bch_move_stats *, char *); #endif /* _BCACHEFS_MOVE_H */ diff --git a/fs/bcachefs/str_hash.h b/fs/bcachefs/str_hash.h index 6178ae620ff1..fa411a3a10d5 100644 --- a/fs/bcachefs/str_hash.h +++ b/fs/bcachefs/str_hash.h @@ -13,7 +13,7 @@ #include <linux/crc32c.h> #include <crypto/hash.h> -#include <crypto/sha2.h> +#include <crypto/sha.h> static inline enum bch_str_hash_type bch2_str_hash_opt_to_type(struct bch_fs *c, enum bch_str_hash_opts opt) diff --git a/fs/bcachefs/super-io.c b/fs/bcachefs/super-io.c index cbc5979a5181..75770453ba08 100644 --- a/fs/bcachefs/super-io.c +++ b/fs/bcachefs/super-io.c @@ -143,12 +143,10 @@ int bch2_sb_realloc(struct bch_sb_handle *sb, unsigned u64s) if (sb->have_bio) { unsigned nr_bvecs = DIV_ROUND_UP(new_buffer_size, PAGE_SIZE); - bio = bio_kmalloc(nr_bvecs, GFP_KERNEL); + bio = bio_kmalloc(GFP_KERNEL, nr_bvecs); if (!bio) return -ENOMEM; - bio_init(bio, NULL, bio->bi_inline_vecs, nr_bvecs, 0); - if (sb->bio) kfree(sb->bio); sb->bio = bio; @@ -296,7 +294,7 @@ static int bch2_sb_validate(struct bch_sb_handle *disk_sb, struct printbuf *out, block_size = le16_to_cpu(sb->block_size); if (block_size > PAGE_SECTORS) { - prt_printf(out, "Block size too big (got %u, max %u)", + prt_printf(out, "Block size too big (got %u, max %lu)", block_size, PAGE_SECTORS); return -EINVAL; } @@ -530,8 +528,10 @@ static int read_one_super(struct bch_sb_handle *sb, u64 offset, struct printbuf size_t bytes; int ret; reread: - bio_reset(sb->bio, sb->bdev, REQ_OP_READ|REQ_SYNC|REQ_META); + bio_reset(sb->bio); + bio_set_dev(sb->bio, sb->bdev); sb->bio->bi_iter.bi_sector = offset; + bio_set_op_attrs(sb->bio, REQ_OP_READ, REQ_SYNC|REQ_META); bch2_bio_map(sb->bio, sb->sb, sb->buffer_size); ret = submit_bio_wait(sb->bio); @@ -659,8 +659,10 @@ int bch2_read_super(const char *path, struct bch_opts *opts, * Error reading primary superblock - read location of backup * superblocks: */ - bio_reset(sb->bio, sb->bdev, REQ_OP_READ|REQ_SYNC|REQ_META); + bio_reset(sb->bio); + bio_set_dev(sb->bio, sb->bdev); sb->bio->bi_iter.bi_sector = BCH_SB_LAYOUT_SECTOR; + bio_set_op_attrs(sb->bio, REQ_OP_READ, REQ_SYNC|REQ_META); /* * use sb buffer to read layout, since sb buffer is page aligned but * layout won't be: @@ -744,10 +746,12 @@ static void read_back_super(struct bch_fs *c, struct bch_dev *ca) struct bch_sb *sb = ca->disk_sb.sb; struct bio *bio = ca->disk_sb.bio; - bio_reset(bio, ca->disk_sb.bdev, REQ_OP_READ|REQ_SYNC|REQ_META); + bio_reset(bio); + bio_set_dev(bio, ca->disk_sb.bdev); bio->bi_iter.bi_sector = le64_to_cpu(sb->layout.sb_offset[0]); bio->bi_end_io = write_super_endio; bio->bi_private = ca; + bio_set_op_attrs(bio, REQ_OP_READ, REQ_SYNC|REQ_META); bch2_bio_map(bio, ca->sb_read_scratch, PAGE_SIZE); this_cpu_add(ca->io_done->sectors[READ][BCH_DATA_sb], @@ -768,10 +772,12 @@ static void write_one_super(struct bch_fs *c, struct bch_dev *ca, unsigned idx) sb->csum = csum_vstruct(c, BCH_SB_CSUM_TYPE(sb), null_nonce(), sb); - bio_reset(bio, ca->disk_sb.bdev, REQ_OP_WRITE|REQ_SYNC|REQ_META); + bio_reset(bio); + bio_set_dev(bio, ca->disk_sb.bdev); bio->bi_iter.bi_sector = le64_to_cpu(sb->offset); bio->bi_end_io = write_super_endio; bio->bi_private = ca; + bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_SYNC|REQ_META); bch2_bio_map(bio, sb, roundup((size_t) vstruct_bytes(sb), bdev_logical_block_size(ca->disk_sb.bdev))); diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index 3f674bf061ff..84fc22b3d2c8 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -65,19 +65,10 @@ MODULE_LICENSE("GPL"); MODULE_AUTHOR("Kent Overstreet <kent.overstreet@gmail.com>"); #define KTYPE(type) \ -static const struct attribute_group type ## _group = { \ - .attrs = type ## _files \ -}; \ - \ -static const struct attribute_group *type ## _groups[] = { \ - &type ## _group, \ - NULL \ -}; \ - \ -static const struct kobj_type type ## _ktype = { \ +struct kobj_type type ## _ktype = { \ .release = type ## _release, \ .sysfs_ops = &type ## _sysfs_ops, \ - .default_groups = type ## _groups \ + .default_attrs = type ## _files \ } static void bch2_fs_release(struct kobject *); @@ -182,6 +173,44 @@ static void bch2_dev_usage_journal_reserve(struct bch_fs *c) &c->dev_usage_journal_res, u64s * nr); } +int bch2_congested(void *data, int bdi_bits) +{ + struct bch_fs *c = data; + struct backing_dev_info *bdi; + struct bch_dev *ca; + unsigned i; + int ret = 0; + + rcu_read_lock(); + if (bdi_bits & (1 << WB_sync_congested)) { + /* Reads - check all devices: */ + for_each_readable_member(ca, c, i) { + bdi = ca->disk_sb.bdev->bd_bdi; + + if (bdi_congested(bdi, bdi_bits)) { + ret = 1; + break; + } + } + } else { + const struct bch_devs_mask *devs = + bch2_target_to_mask(c, c->opts.foreground_target) ?: + &c->rw_devs[BCH_DATA_user]; + + for_each_member_device_rcu(ca, c, i, devs) { + bdi = ca->disk_sb.bdev->bd_bdi; + + if (bdi_congested(bdi, bdi_bits)) { + ret = 1; + break; + } + } + } + rcu_read_unlock(); + + return ret; +} + /* Filesystem RO/RW: */ /* @@ -502,7 +531,8 @@ void __bch2_fs_stop(struct bch_fs *c) for_each_member_device(ca, c, i) if (ca->kobj.state_in_sysfs && ca->disk_sb.bdev) - sysfs_remove_link(bdev_kobj(ca->disk_sb.bdev), "bcachefs"); + sysfs_remove_link(&part_to_dev(ca->disk_sb.bdev->bd_part)->kobj, + "bcachefs"); if (c->kobj.state_in_sysfs) kobject_del(&c->kobj); @@ -993,7 +1023,8 @@ static void bch2_dev_free(struct bch_dev *ca) if (ca->kobj.state_in_sysfs && ca->disk_sb.bdev) - sysfs_remove_link(bdev_kobj(ca->disk_sb.bdev), "bcachefs"); + sysfs_remove_link(&part_to_dev(ca->disk_sb.bdev->bd_part)->kobj, + "bcachefs"); if (ca->kobj.state_in_sysfs) kobject_del(&ca->kobj); @@ -1029,7 +1060,10 @@ static void __bch2_dev_offline(struct bch_fs *c, struct bch_dev *ca) wait_for_completion(&ca->io_ref_completion); if (ca->kobj.state_in_sysfs) { - sysfs_remove_link(bdev_kobj(ca->disk_sb.bdev), "bcachefs"); + struct kobject *block = + &part_to_dev(ca->disk_sb.bdev->bd_part)->kobj; + + sysfs_remove_link(block, "bcachefs"); sysfs_remove_link(&ca->kobj, "block"); } @@ -1066,12 +1100,12 @@ static int bch2_dev_sysfs_online(struct bch_fs *c, struct bch_dev *ca) } if (ca->disk_sb.bdev) { - struct kobject *block = bdev_kobj(ca->disk_sb.bdev); + struct kobject *block = + &part_to_dev(ca->disk_sb.bdev->bd_part)->kobj; ret = sysfs_create_link(block, &ca->kobj, "bcachefs"); if (ret) return ret; - ret = sysfs_create_link(&ca->kobj, block, "block"); if (ret) return ret; diff --git a/fs/bcachefs/super.h b/fs/bcachefs/super.h index 8501adaff4c2..de7b40a886ec 100644 --- a/fs/bcachefs/super.h +++ b/fs/bcachefs/super.h @@ -223,6 +223,7 @@ static inline bool is_superblock_bucket(struct bch_dev *ca, u64 b) struct bch_fs *bch2_dev_to_fs(dev_t); struct bch_fs *bch2_uuid_to_fs(uuid_le); +int bch2_congested(void *, int); bool bch2_dev_state_allowed(struct bch_fs *, struct bch_dev *, enum bch_member_state, int); diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c index 103fde97231f..db7abe32517a 100644 --- a/fs/bcachefs/sysfs.c +++ b/fs/bcachefs/sysfs.c @@ -290,7 +290,7 @@ static int bch2_compression_stats_to_text(struct printbuf *out, struct bch_fs *c incompressible_sectors = 0, compressed_sectors_compressed = 0, compressed_sectors_uncompressed = 0; - int ret; + int ret = 0; if (!test_bit(BCH_FS_STARTED, &c->flags)) return -EPERM; diff --git a/fs/bcachefs/util.c b/fs/bcachefs/util.c index f08215af359f..04a8b4b188bd 100644 --- a/fs/bcachefs/util.c +++ b/fs/bcachefs/util.c @@ -299,21 +299,6 @@ void bch2_print_string_as_lines(const char *prefix, const char *lines) int bch2_prt_backtrace(struct printbuf *out, struct task_struct *task) { - unsigned long entries[32]; - unsigned i, nr_entries; - int ret; - - ret = down_read_killable(&task->signal->exec_update_lock); - if (ret) - return ret; - - nr_entries = stack_trace_save_tsk(task, entries, ARRAY_SIZE(entries), 0); - for (i = 0; i < nr_entries; i++) { - prt_printf(out, "[<0>] %pB", (void *)entries[i]); - prt_newline(out); - } - - up_read(&task->signal->exec_update_lock); return 0; } diff --git a/fs/bcachefs/util.h b/fs/bcachefs/util.h index 846e6024a80b..4a6968aa1cc1 100644 --- a/fs/bcachefs/util.h +++ b/fs/bcachefs/util.h @@ -20,6 +20,9 @@ #include <linux/workqueue.h> #include <linux/mean_and_variance.h> +#define PAGE_SECTORS_SHIFT (PAGE_SHIFT - 9) +#define PAGE_SECTORS (1UL << PAGE_SECTORS_SHIFT) + struct closure; #ifdef CONFIG_BCACHEFS_DEBUG @@ -87,7 +90,7 @@ static inline void *vpmalloc(size_t size, gfp_t gfp_mask) { return (void *) __get_free_pages(gfp_mask|__GFP_NOWARN, get_order(size)) ?: - __vmalloc(size, gfp_mask); + __vmalloc(size, gfp_mask, PAGE_KERNEL); } static inline void kvpfree(void *p, size_t size) @@ -669,6 +672,35 @@ static inline void memset_u64s_tail(void *s, int c, unsigned bytes) memset(s + bytes, c, rem); } +static inline struct bio_vec next_contig_bvec(struct bio *bio, + struct bvec_iter *iter) +{ + struct bio_vec bv = bio_iter_iovec(bio, *iter); + + bio_advance_iter(bio, iter, bv.bv_len); +#ifndef CONFIG_HIGHMEM + while (iter->bi_size) { + struct bio_vec next = bio_iter_iovec(bio, *iter); + + if (page_address(bv.bv_page) + bv.bv_offset + bv.bv_len != + page_address(next.bv_page) + next.bv_offset) + break; + + bv.bv_len += next.bv_len; + bio_advance_iter(bio, iter, next.bv_len); + } +#endif + return bv; +} + +#define __bio_for_each_contig_segment(bv, bio, iter, start) \ + for (iter = (start); \ + (iter).bi_size && \ + ((bv = next_contig_bvec((bio), &(iter))), 1);) + +#define bio_for_each_contig_segment(bv, bio, iter) \ + __bio_for_each_contig_segment(bv, bio, iter, (bio)->bi_iter) + void sort_cmp_size(void *base, size_t num, size_t size, int (*cmp_func)(const void *, const void *, size_t), void (*swap_func)(void *, void *, size_t)); diff --git a/fs/bcachefs/varint.c b/fs/bcachefs/varint.c index 5143b603bf67..a2d6bb7136c7 100644 --- a/fs/bcachefs/varint.c +++ b/fs/bcachefs/varint.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/bitops.h> -#include <linux/math.h> #include <linux/string.h> #include <asm/unaligned.h> diff --git a/fs/bcachefs/xattr.c b/fs/bcachefs/xattr.c index 4fc1c3afab69..d409596523d3 100644 --- a/fs/bcachefs/xattr.c +++ b/fs/bcachefs/xattr.c @@ -378,7 +378,6 @@ static int bch2_xattr_get_handler(const struct xattr_handler *handler, } static int bch2_xattr_set_handler(const struct xattr_handler *handler, - struct user_namespace *mnt_userns, struct dentry *dentry, struct inode *vinode, const char *name, const void *value, size_t size, int flags) @@ -517,7 +516,6 @@ static int inode_opt_set_fn(struct bch_inode_info *inode, } static int bch2_xattr_bcachefs_set(const struct xattr_handler *handler, - struct user_namespace *mnt_userns, struct dentry *dentry, struct inode *vinode, const char *name, const void *value, size_t size, int flags) diff --git a/include/linux/generic-radix-tree.h b/include/linux/generic-radix-tree.h index c74b7376990d..5aa1986128df 100644 --- a/include/linux/generic-radix-tree.h +++ b/include/linux/generic-radix-tree.h @@ -40,7 +40,6 @@ #include <linux/bug.h> #include <linux/limits.h> #include <linux/log2.h> -#include <linux/math.h> #include <linux/types.h> struct genradix_root; diff --git a/include/trace/events/bcachefs.h b/include/trace/events/bcachefs.h index d3d9e965e702..907649033621 100644 --- a/include/trace/events/bcachefs.h +++ b/include/trace/events/bcachefs.h @@ -104,10 +104,10 @@ DECLARE_EVENT_CLASS(bio, ), TP_fast_assign( - __entry->dev = bio->bi_bdev ? bio_dev(bio) : 0; + __entry->dev = bio->bi_disk ? bio_dev(bio) : 0; __entry->sector = bio->bi_iter.bi_sector; __entry->nr_sector = bio->bi_iter.bi_size >> 9; - blk_fill_rwbs(__entry->rwbs, bio->bi_opf); + blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size); ), TP_printk("%d,%d %s %llu + %u", diff --git a/kernel/locking/six.c b/kernel/locking/six.c index b11660af245b..e2e0acdaef5e 100644 --- a/kernel/locking/six.c +++ b/kernel/locking/six.c @@ -18,7 +18,7 @@ #endif #define six_acquire(l, t, r) lock_acquire(l, 0, t, r, 1, NULL, _RET_IP_) -#define six_release(l) lock_release(l, _RET_IP_) +#define six_release(l) lock_release(l, 0, _RET_IP_) static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type); diff --git a/lib/mean_and_variance.c b/lib/mean_and_variance.c index 643e3113500b..75fe68e13d97 100644 --- a/lib/mean_and_variance.c +++ b/lib/mean_and_variance.c @@ -38,7 +38,6 @@ #include <linux/compiler.h> #include <linux/export.h> #include <linux/limits.h> -#include <linux/math.h> #include <linux/math64.h> #include <linux/mean_and_variance.h> #include <linux/module.h> |