summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--fs/bcachefs/backpointers.c123
-rw-r--r--fs/bcachefs/backpointers.h4
-rw-r--r--fs/bcachefs/bcachefs.h9
-rw-r--r--fs/bcachefs/buckets.c22
-rw-r--r--fs/bcachefs/super.c7
5 files changed, 113 insertions, 52 deletions
diff --git a/fs/bcachefs/backpointers.c b/fs/bcachefs/backpointers.c
index e6178eb2c396..631d4d24d78f 100644
--- a/fs/bcachefs/backpointers.c
+++ b/fs/bcachefs/backpointers.c
@@ -15,6 +15,14 @@
#include <linux/mm.h>
+static inline struct bbpos bp_to_bbpos(struct bch_backpointer bp)
+{
+ return (struct bbpos) {
+ .btree = bp.btree_id,
+ .pos = bp.pos,
+ };
+}
+
int bch2_backpointer_validate(struct bch_fs *c, struct bkey_s_c k,
struct bkey_validate_context from)
{
@@ -671,8 +679,22 @@ static int check_extent_to_backpointers(struct btree_trans *trans,
rcu_read_lock();
struct bch_dev *ca = bch2_dev_rcu_noerror(c, p.ptr.dev);
- bool check = ca && test_bit(PTR_BUCKET_NR(ca, &p.ptr), ca->bucket_backpointer_mismatches);
- bool empty = ca && test_bit(PTR_BUCKET_NR(ca, &p.ptr), ca->bucket_backpointer_empty);
+ if (!ca) {
+ rcu_read_unlock();
+ continue;
+ }
+
+ u64 b = PTR_BUCKET_NR(ca, &p.ptr);
+ bool set[2];
+
+ for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++) {
+ unsigned long *bitmap =
+ READ_ONCE(ca->bucket_backpointer_mismatches[i].buckets);
+ set[i] = bitmap && test_bit(b, bitmap);
+ }
+
+ bool check = set[0];
+ bool empty = set[1];
bool stale = p.ptr.cached && (!ca || dev_ptr_stale_rcu(ca, &p.ptr));
rcu_read_unlock();
@@ -724,14 +746,6 @@ err:
return ret;
}
-static inline struct bbpos bp_to_bbpos(struct bch_backpointer bp)
-{
- return (struct bbpos) {
- .btree = bp.btree_id,
- .pos = bp.pos,
- };
-}
-
static u64 mem_may_pin_bytes(struct bch_fs *c)
{
struct sysinfo i;
@@ -933,12 +947,25 @@ static int check_bucket_backpointer_mismatch(struct btree_trans *trans, struct b
goto err;
}
- if (!sectors[ALLOC_dirty] &&
- !sectors[ALLOC_stripe] &&
- !sectors[ALLOC_cached])
- __set_bit(alloc_k.k->p.offset, ca->bucket_backpointer_empty);
- else
- __set_bit(alloc_k.k->p.offset, ca->bucket_backpointer_mismatches);
+ bool empty = (sectors[ALLOC_dirty] +
+ sectors[ALLOC_stripe] +
+ sectors[ALLOC_cached]) == 0;
+
+ struct bucket_bitmap *bitmap = &ca->bucket_backpointer_mismatches[empty];
+
+ mutex_lock(&bitmap->lock);
+ if (!bitmap->buckets) {
+ bitmap->buckets = kvcalloc(BITS_TO_LONGS(ca->mi.nbuckets),
+ sizeof(unsigned long), GFP_KERNEL);
+ if (!bitmap->buckets) {
+ mutex_unlock(&bitmap->lock);
+ ret = -BCH_ERR_ENOMEM_backpointer_mismatches_bitmap;
+ goto err;
+ }
+ }
+
+ bitmap->nr += !__test_and_set_bit(alloc_k.k->p.offset, bitmap->buckets);
+ mutex_unlock(&bitmap->lock);
}
err:
bch2_dev_put(ca);
@@ -962,8 +989,19 @@ static bool backpointer_node_has_missing(struct bch_fs *c, struct bkey_s_c k)
goto next;
struct bpos bucket = bp_pos_to_bucket(ca, pos);
- bucket.offset = find_next_bit(ca->bucket_backpointer_mismatches,
- ca->mi.nbuckets, bucket.offset);
+ u64 next = ca->mi.nbuckets;
+
+ for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++) {
+ unsigned long *bitmap =
+ READ_ONCE(ca->bucket_backpointer_mismatches[i].buckets);
+ if (bitmap)
+ next = min_t(u64, next,
+ find_next_bit(bitmap,
+ ca->mi.nbuckets,
+ bucket.offset));
+ }
+
+ bucket.offset = next;
if (bucket.offset == ca->mi.nbuckets)
goto next;
@@ -1072,28 +1110,6 @@ int bch2_check_extents_to_backpointers(struct bch_fs *c)
{
int ret = 0;
- /*
- * Can't allow devices to come/go/resize while we have bucket bitmaps
- * allocated
- */
- down_read(&c->state_lock);
-
- for_each_member_device(c, ca) {
- BUG_ON(ca->bucket_backpointer_mismatches);
- ca->bucket_backpointer_mismatches = kvcalloc(BITS_TO_LONGS(ca->mi.nbuckets),
- sizeof(unsigned long),
- GFP_KERNEL);
- ca->bucket_backpointer_empty = kvcalloc(BITS_TO_LONGS(ca->mi.nbuckets),
- sizeof(unsigned long),
- GFP_KERNEL);
- if (!ca->bucket_backpointer_mismatches ||
- !ca->bucket_backpointer_empty) {
- bch2_dev_put(ca);
- ret = -BCH_ERR_ENOMEM_backpointer_mismatches_bitmap;
- goto err_free_bitmaps;
- }
- }
-
struct btree_trans *trans = bch2_trans_get(c);
struct extents_to_bp_state s = { .bp_start = POS_MIN };
@@ -1110,8 +1126,8 @@ int bch2_check_extents_to_backpointers(struct bch_fs *c)
u64 nr_buckets = 0, nr_mismatches = 0, nr_empty = 0;
for_each_member_device(c, ca) {
nr_buckets += ca->mi.nbuckets;
- nr_mismatches += bitmap_weight(ca->bucket_backpointer_mismatches, ca->mi.nbuckets);
- nr_empty += bitmap_weight(ca->bucket_backpointer_empty, ca->mi.nbuckets);
+ nr_mismatches += ca->bucket_backpointer_mismatches[0].nr;
+ nr_empty += ca->bucket_backpointer_mismatches[1].nr;
}
if (!nr_mismatches && !nr_empty)
@@ -1153,19 +1169,17 @@ err:
bch2_trans_put(trans);
bch2_bkey_buf_exit(&s.last_flushed, c);
bch2_btree_cache_unpin(c);
-err_free_bitmaps:
- for_each_member_device(c, ca) {
- kvfree(ca->bucket_backpointer_empty);
- ca->bucket_backpointer_empty = NULL;
- kvfree(ca->bucket_backpointer_mismatches);
- ca->bucket_backpointer_mismatches = NULL;
- }
- up_read(&c->state_lock);
+ for_each_member_device(c, ca)
+ for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++)
+ bch2_bucket_bitmap_free(&ca->bucket_backpointer_mismatches[i]);
+
bch_err_fn(c, ret);
return ret;
}
+/* backpointers -> extents */
+
static int check_one_backpointer(struct btree_trans *trans,
struct bbpos start,
struct bbpos end,
@@ -1281,3 +1295,12 @@ int bch2_check_backpointers_to_extents(struct bch_fs *c)
bch_err_fn(c, ret);
return ret;
}
+
+void bch2_bucket_bitmap_free(struct bucket_bitmap *b)
+{
+ mutex_lock(&b->lock);
+ kvfree(b->buckets);
+ b->buckets = NULL;
+ b->nr = 0;
+ mutex_unlock(&b->lock);
+}
diff --git a/fs/bcachefs/backpointers.h b/fs/bcachefs/backpointers.h
index 16575dbc5736..c72707ee9d42 100644
--- a/fs/bcachefs/backpointers.h
+++ b/fs/bcachefs/backpointers.h
@@ -182,8 +182,12 @@ struct bkey_s_c bch2_backpointer_get_key(struct btree_trans *, struct bkey_s_c_b
struct btree *bch2_backpointer_get_node(struct btree_trans *, struct bkey_s_c_backpointer,
struct btree_iter *, struct bkey_buf *);
+int bch2_check_bucket_backpointer_mismatch(struct btree_trans *, struct bpos, struct bkey_buf *);
+
int bch2_check_btree_backpointers(struct bch_fs *);
int bch2_check_extents_to_backpointers(struct bch_fs *);
int bch2_check_backpointers_to_extents(struct bch_fs *);
+void bch2_bucket_bitmap_free(struct bucket_bitmap *);
+
#endif /* _BCACHEFS_BACKPOINTERS_BACKGROUND_H */
diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h
index 07a16c473af3..66659dade3f0 100644
--- a/fs/bcachefs/bcachefs.h
+++ b/fs/bcachefs/bcachefs.h
@@ -574,6 +574,12 @@ enum bch_dev_write_ref {
BCH_DEV_WRITE_REF_NR,
};
+struct bucket_bitmap {
+ unsigned long *buckets;
+ u64 nr;
+ struct mutex lock;
+};
+
struct bch_dev {
struct kobject kobj;
#ifdef CONFIG_BCACHEFS_DEBUG
@@ -618,8 +624,7 @@ struct bch_dev {
u8 *oldest_gen;
unsigned long *buckets_nouse;
- unsigned long *bucket_backpointer_mismatches;
- unsigned long *bucket_backpointer_empty;
+ struct bucket_bitmap bucket_backpointer_mismatches[2];
struct bch_dev_usage_full __percpu
*usage;
diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c
index 596edc7bba2f..8d6955ef631b 100644
--- a/fs/bcachefs/buckets.c
+++ b/fs/bcachefs/buckets.c
@@ -1324,6 +1324,28 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
sizeof(bucket_gens->b[0]) * copy);
}
+ for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++) {
+ struct bucket_bitmap *bitmap = &ca->bucket_backpointer_mismatches[i];
+
+ mutex_lock(&bitmap->lock);
+ if (bitmap->buckets) {
+ unsigned long *n = kvcalloc(BITS_TO_LONGS(nbuckets),
+ sizeof(unsigned long), GFP_KERNEL);
+ if (!n) {
+ mutex_unlock(&bitmap->lock);
+ ret = -BCH_ERR_ENOMEM_backpointer_mismatches_bitmap;
+ goto err;
+ }
+
+ memcpy(n, bitmap->buckets,
+ BITS_TO_LONGS(ca->mi.nbuckets) * sizeof(unsigned long));
+ kvfree(bitmap->buckets);
+ bitmap->buckets = n;
+
+ }
+ mutex_unlock(&bitmap->lock);
+ }
+
rcu_assign_pointer(ca->bucket_gens, bucket_gens);
bucket_gens = old_bucket_gens;
diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c
index dc8189f9d2f1..77b834cfe126 100644
--- a/fs/bcachefs/super.c
+++ b/fs/bcachefs/super.c
@@ -11,6 +11,7 @@
#include "alloc_background.h"
#include "alloc_foreground.h"
#include "async_objs.h"
+#include "backpointers.h"
#include "bkey_sort.h"
#include "btree_cache.h"
#include "btree_gc.h"
@@ -1341,6 +1342,9 @@ static void bch2_dev_free(struct bch_dev *ca)
if (ca->kobj.state_in_sysfs)
kobject_del(&ca->kobj);
+ for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++)
+ bch2_bucket_bitmap_free(&ca->bucket_backpointer_mismatches[i]);
+
bch2_free_super(&ca->disk_sb);
bch2_dev_allocator_background_exit(ca);
bch2_dev_journal_exit(ca);
@@ -1471,6 +1475,9 @@ static struct bch_dev *__bch2_dev_alloc(struct bch_fs *c,
atomic_long_set(&ca->ref, 1);
#endif
+ for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++)
+ mutex_init(&ca->bucket_backpointer_mismatches[i].lock);
+
bch2_dev_allocator_background_init(ca);
if (enumerated_ref_init(&ca->io_ref[READ], BCH_DEV_READ_REF_NR, NULL) ||
generated by cgit v1.2.3 (git 2.46.0) at 2025-07-01 08:34:40 +0000