bcachefs: break out filesystem level disk accounting by replicas entry

19 files changed, 537 insertions, 280 deletions

diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h
index a279e96472b1..a9d021df1366 100644
--- a/fs/bcachefs/bcachefs.h
+++ b/fs/bcachefs/bcachefs.h
@@ -621,10 +621,11 @@ struct bch_fs {
 
 	struct bch_fs_pcpu __percpu	*pcpu;
 
-	struct bch_fs_usage __percpu	*usage[2];
-
 	struct percpu_rw_semaphore	mark_lock;
 
+	struct bch_fs_usage __percpu	*usage[2];
+	struct bch_fs_usage __percpu	*usage_scratch;
+
 	/*
 	 * When we invalidate buckets, we use both the priority and the amount
 	 * of good data to determine which buckets to reuse first - to weight
diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c
index c43b2072349b..46cdd9687ce1 100644
--- a/fs/bcachefs/btree_gc.c
+++ b/fs/bcachefs/btree_gc.c
@@ -475,33 +475,12 @@ static void bch2_gc_free(struct bch_fs *c)
 		ca->usage[1] = NULL;
 	}
 
+	percpu_down_write(&c->mark_lock);
+
 	free_percpu(c->usage[1]);
 	c->usage[1] = NULL;
-}
-
-/*
- * Accumulate percpu counters onto one cpu's copy - only valid when access
- * against any percpu counter is guarded against
- */
-static u64 *acc_percpu_u64s(u64 __percpu *p, unsigned nr)
-{
-	u64 *ret;
-	int cpu;
-
-	preempt_disable();
-	ret = this_cpu_ptr(p);
-	preempt_enable();
-
-	for_each_possible_cpu(cpu) {
-		u64 *i = per_cpu_ptr(p, cpu);
-
-		if (i != ret) {
-			acc_u64s(ret, i, nr);
-			memset(i, 0, nr * sizeof(u64));
-		}
-	}
 
-	return ret;
+	percpu_up_write(&c->mark_lock);
 }
 
 static void bch2_gc_done_nocheck(struct bch_fs *c)
@@ -539,23 +518,24 @@ static void bch2_gc_done_nocheck(struct bch_fs *c)
 	for_each_member_device(ca, c, i) {
 		unsigned nr = sizeof(struct bch_dev_usage) / sizeof(u64);
 		struct bch_dev_usage *dst = (void *)
-			acc_percpu_u64s((void *) ca->usage[0], nr);
+			bch2_acc_percpu_u64s((void *) ca->usage[0], nr);
 		struct bch_dev_usage *src = (void *)
-			acc_percpu_u64s((void *) ca->usage[1], nr);
+			bch2_acc_percpu_u64s((void *) ca->usage[1], nr);
 
 		*dst = *src;
 	}
 
 	{
-		unsigned nr = sizeof(struct bch_fs_usage) / sizeof(u64);
+		unsigned nr = sizeof(struct bch_fs_usage) / sizeof(u64) +
+			c->replicas.nr;
 		struct bch_fs_usage *dst = (void *)
-			acc_percpu_u64s((void *) c->usage[0], nr);
+			bch2_acc_percpu_u64s((void *) c->usage[0], nr);
 		struct bch_fs_usage *src = (void *)
-			acc_percpu_u64s((void *) c->usage[1], nr);
+			bch2_acc_percpu_u64s((void *) c->usage[1], nr);
 
 		memcpy(&dst->s.gc_start[0],
 		       &src->s.gc_start[0],
-		       sizeof(*dst) - offsetof(typeof(*dst), s.gc_start));
+		       nr * sizeof(u64) - offsetof(typeof(*dst), s.gc_start));
 	}
 }
 
@@ -651,9 +631,9 @@ static void bch2_gc_done(struct bch_fs *c, bool initial)
 	for_each_member_device(ca, c, i) {
 		unsigned nr = sizeof(struct bch_dev_usage) / sizeof(u64);
 		struct bch_dev_usage *dst = (void *)
-			acc_percpu_u64s((void *) ca->usage[0], nr);
+			bch2_acc_percpu_u64s((void *) ca->usage[0], nr);
 		struct bch_dev_usage *src = (void *)
-			acc_percpu_u64s((void *) ca->usage[1], nr);
+			bch2_acc_percpu_u64s((void *) ca->usage[1], nr);
 		unsigned b;
 
 		for (b = 0; b < BCH_DATA_NR; b++)
@@ -670,12 +650,12 @@ static void bch2_gc_done(struct bch_fs *c, bool initial)
 	}
 
 	{
-		unsigned nr = sizeof(struct bch_fs_usage) / sizeof(u64);
+		unsigned nr = sizeof(struct bch_fs_usage) / sizeof(u64) +
+			c->replicas.nr;
 		struct bch_fs_usage *dst = (void *)
-			acc_percpu_u64s((void *) c->usage[0], nr);
+			bch2_acc_percpu_u64s((void *) c->usage[0], nr);
 		struct bch_fs_usage *src = (void *)
-			acc_percpu_u64s((void *) c->usage[1], nr);
-		unsigned r, b;
+			bch2_acc_percpu_u64s((void *) c->usage[1], nr);
 
 		copy_fs_field(s.hidden,		"hidden");
 		copy_fs_field(s.data,		"data");
@@ -683,20 +663,16 @@ static void bch2_gc_done(struct bch_fs *c, bool initial)
 		copy_fs_field(s.reserved,	"reserved");
 		copy_fs_field(s.nr_inodes,	"nr_inodes");
 
-		for (r = 0; r < BCH_REPLICAS_MAX; r++) {
-			for (b = 0; b < BCH_DATA_NR; b++)
-				copy_fs_field(replicas[r].data[b],
-					      "replicas[%i].data[%s]",
-					      r, bch2_data_types[b]);
-			copy_fs_field(replicas[r].ec_data,
-				      "replicas[%i].ec_data", r);
-			copy_fs_field(replicas[r].persistent_reserved,
-				      "replicas[%i].persistent_reserved", r);
-		}
+		for (i = 0; i < BCH_REPLICAS_MAX; i++)
+			copy_fs_field(persistent_reserved[i],
+				      "persistent_reserved[%i]", i);
 
-		for (b = 0; b < BCH_DATA_NR; b++)
-			copy_fs_field(buckets[b],
-				      "buckets[%s]", bch2_data_types[b]);
+		for (i = 0; i < c->replicas.nr; i++) {
+			/*
+			 * XXX: print out replicas entry
+			 */
+			copy_fs_field(data[i], "data[%i]", i);
+		}
 	}
 out:
 	percpu_up_write(&c->mark_lock);
@@ -719,9 +695,15 @@ static int bch2_gc_start(struct bch_fs *c)
 	 */
 	gc_pos_set(c, gc_phase(GC_PHASE_START));
 
+	percpu_down_write(&c->mark_lock);
 	BUG_ON(c->usage[1]);
 
-	c->usage[1] = alloc_percpu(struct bch_fs_usage);
+	c->usage[1] = __alloc_percpu_gfp(sizeof(struct bch_fs_usage) +
+					 sizeof(u64) * c->replicas.nr,
+					 sizeof(u64),
+					 GFP_KERNEL);
+	percpu_up_write(&c->mark_lock);
+
 	if (!c->usage[1])
 		return -ENOMEM;
 
diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c
index 6fab9996ff5a..0f2fa6f72619 100644
--- a/fs/bcachefs/btree_update_interior.c
+++ b/fs/bcachefs/btree_update_interior.c
@@ -1069,23 +1069,24 @@ static void bch2_btree_set_root_inmem(struct btree_update *as, struct btree *b)
 {
 	struct bch_fs *c = as->c;
 	struct btree *old = btree_node_root(c, b);
-	struct bch_fs_usage stats = { 0 };
+	struct bch_fs_usage *fs_usage;
 
 	__bch2_btree_set_root_inmem(c, b);
 
 	mutex_lock(&c->btree_interior_update_lock);
 	percpu_down_read_preempt_disable(&c->mark_lock);
+	fs_usage = bch2_fs_usage_get_scratch(c);
 
 	bch2_mark_key_locked(c, bkey_i_to_s_c(&b->key),
 		      true, 0,
 		      gc_pos_btree_root(b->btree_id),
-		      &stats, 0, 0);
+		      fs_usage, 0, 0);
 
 	if (old && !btree_node_fake(old))
 		bch2_btree_node_free_index(as, NULL,
 					   bkey_i_to_s_c(&old->key),
-					   &stats);
-	bch2_fs_usage_apply(c, &stats, &as->reserve->disk_res,
+					   fs_usage);
+	bch2_fs_usage_apply(c, fs_usage, &as->reserve->disk_res,
 			    gc_pos_btree_root(b->btree_id));
 
 	percpu_up_read_preempt_enable(&c->mark_lock);
@@ -1160,7 +1161,7 @@ static void bch2_insert_fixup_btree_ptr(struct btree_update *as, struct btree *b
 					struct btree_node_iter *node_iter)
 {
 	struct bch_fs *c = as->c;
-	struct bch_fs_usage stats = { 0 };
+	struct bch_fs_usage *fs_usage;
 	struct bkey_packed *k;
 	struct bkey tmp;
 
@@ -1168,10 +1169,11 @@ static void bch2_insert_fixup_btree_ptr(struct btree_update *as, struct btree *b
 
 	mutex_lock(&c->btree_interior_update_lock);
 	percpu_down_read_preempt_disable(&c->mark_lock);
+	fs_usage = bch2_fs_usage_get_scratch(c);
 
 	bch2_mark_key_locked(c, bkey_i_to_s_c(insert),
 			     true, 0,
-			     gc_pos_btree_node(b), &stats, 0, 0);
+			     gc_pos_btree_node(b), fs_usage, 0, 0);
 
 	while ((k = bch2_btree_node_iter_peek_all(node_iter, b)) &&
 	       bkey_iter_pos_cmp(b, &insert->k.p, k) > 0)
@@ -1184,9 +1186,9 @@ static void bch2_insert_fixup_btree_ptr(struct btree_update *as, struct btree *b
 	if (k && !bkey_cmp_packed(b, k, &insert->k))
 		bch2_btree_node_free_index(as, b,
 					   bkey_disassemble(b, k, &tmp),
-					   &stats);
+					   fs_usage);
 
-	bch2_fs_usage_apply(c, &stats, &as->reserve->disk_res,
+	bch2_fs_usage_apply(c, fs_usage, &as->reserve->disk_res,
 			    gc_pos_btree_node(b));
 
 	percpu_up_read_preempt_enable(&c->mark_lock);
@@ -1970,7 +1972,7 @@ static void __bch2_btree_node_update_key(struct bch_fs *c,
 			bkey_copy(&b->key, &new_key->k_i);
 		}
 	} else {
-		struct bch_fs_usage stats = { 0 };
+		struct bch_fs_usage *fs_usage;
 
 		BUG_ON(btree_node_root(c, b) != b);
 
@@ -1978,15 +1980,16 @@ static void __bch2_btree_node_update_key(struct bch_fs *c,
 
 		mutex_lock(&c->btree_interior_update_lock);
 		percpu_down_read_preempt_disable(&c->mark_lock);
+		fs_usage = bch2_fs_usage_get_scratch(c);
 
 		bch2_mark_key_locked(c, bkey_i_to_s_c(&new_key->k_i),
 			      true, 0,
 			      gc_pos_btree_root(b->btree_id),
-			      &stats, 0, 0);
+			      fs_usage, 0, 0);
 		bch2_btree_node_free_index(as, NULL,
 					   bkey_i_to_s_c(&b->key),
-					   &stats);
-		bch2_fs_usage_apply(c, &stats, &as->reserve->disk_res,
+					   fs_usage);
+		bch2_fs_usage_apply(c, fs_usage, &as->reserve->disk_res,
 				    gc_pos_btree_root(b->btree_id));
 
 		percpu_up_read_preempt_enable(&c->mark_lock);
diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c
index abc920b856cc..721d350f52e7 100644
--- a/fs/bcachefs/buckets.c
+++ b/fs/bcachefs/buckets.c
@@ -71,12 +71,11 @@
 #include "ec.h"
 #include "error.h"
 #include "movinggc.h"
+#include "replicas.h"
 
 #include <linux/preempt.h>
 #include <trace/events/bcachefs.h>
 
-static inline u64 __bch2_fs_sectors_used(struct bch_fs *, struct bch_fs_usage);
-
 /*
  * Clear journal_seq_valid for buckets for which it's not needed, to prevent
  * wraparound:
@@ -131,9 +130,29 @@ struct bch_dev_usage bch2_dev_usage_read(struct bch_fs *c, struct bch_dev *ca)
 	return bch2_usage_read_raw(ca->usage[0]);
 }
 
-struct bch_fs_usage bch2_fs_usage_read(struct bch_fs *c)
+struct bch_fs_usage *bch2_fs_usage_read(struct bch_fs *c)
 {
-	return bch2_usage_read_raw(c->usage[0]);
+	struct bch_fs_usage *ret;
+	unsigned nr = READ_ONCE(c->replicas.nr);
+retry:
+	ret = kzalloc(sizeof(*ret) + nr * sizeof(u64), GFP_NOFS);
+	if (unlikely(!ret))
+		return NULL;
+
+	percpu_down_read_preempt_disable(&c->mark_lock);
+
+	if (unlikely(nr < c->replicas.nr)) {
+		nr = c->replicas.nr;
+		percpu_up_read_preempt_enable(&c->mark_lock);
+		kfree(ret);
+		goto retry;
+	}
+
+	acc_u64s_percpu((u64 *) ret,
+			(u64 __percpu *) c->usage[0],
+			sizeof(*ret) / sizeof(u64) + nr);
+
+	return ret;
 }
 
 #define RESERVE_FACTOR	6
@@ -148,17 +167,13 @@ static u64 avail_factor(u64 r)
 	return (r << RESERVE_FACTOR) / ((1 << RESERVE_FACTOR) + 1);
 }
 
-static inline u64 __bch2_fs_sectors_used(struct bch_fs *c, struct bch_fs_usage fs_usage)
-{
-	return fs_usage.s.hidden +
-		fs_usage.s.data +
-		reserve_factor(fs_usage.s.reserved +
-			       fs_usage.s.online_reserved);
-}
-
 u64 bch2_fs_sectors_used(struct bch_fs *c, struct bch_fs_usage fs_usage)
 {
-	return min(c->capacity, __bch2_fs_sectors_used(c, fs_usage));
+	return min(fs_usage.s.hidden +
+		   fs_usage.s.data +
+		   reserve_factor(fs_usage.s.reserved +
+				  fs_usage.s.online_reserved),
+		   c->capacity);
 }
 
 struct bch_fs_usage_short
@@ -237,17 +252,15 @@ int bch2_fs_usage_apply(struct bch_fs *c,
 
 	acc_u64s((u64 *) this_cpu_ptr(c->usage[0]),
 		 (u64 *) fs_usage,
-		 sizeof(*fs_usage) / sizeof(u64));
+		 sizeof(*fs_usage) / sizeof(u64) + c->replicas.nr);
 
 	if (gc_visited(c, gc_pos)) {
 		BUG_ON(!c->usage[1]);
 		acc_u64s((u64 *) this_cpu_ptr(c->usage[1]),
 			 (u64 *) fs_usage,
-			 sizeof(*fs_usage) / sizeof(u64));
+			 sizeof(*fs_usage) / sizeof(u64) + c->replicas.nr);
 	}
 
-	memset(fs_usage, 0, sizeof(*fs_usage));
-
 	return ret;
 }
 
@@ -259,7 +272,6 @@ static inline void account_bucket(struct bch_fs_usage *fs_usage,
 	if (type == BCH_DATA_SB || type == BCH_DATA_JOURNAL)
 		fs_usage->s.hidden	+= size;
 
-	fs_usage->buckets[type]		+= size;
 	dev_usage->buckets[type]	+= nr;
 }
 
@@ -331,6 +343,34 @@ void bch2_dev_usage_from_buckets(struct bch_fs *c, struct bch_dev *ca)
 	_old;							\
 })
 
+static inline void update_replicas(struct bch_fs *c,
+				   struct bch_fs_usage *fs_usage,
+				   struct bch_replicas_entry *r,
+				   s64 sectors)
+{
+	int idx = bch2_replicas_entry_idx(c, r);
+
+	BUG_ON(idx < 0);
+	BUG_ON(!sectors);
+
+	if (r->data_type == BCH_DATA_CACHED)
+		fs_usage->s.cached	+= sectors;
+	else
+		fs_usage->s.data	+= sectors;
+	fs_usage->data[idx]		+= sectors;
+}
+
+static inline void update_cached_sectors(struct bch_fs *c,
+					 struct bch_fs_usage *fs_usage,
+					 unsigned dev, s64 sectors)
+{
+	struct bch_replicas_padded r;
+
+	bch2_replicas_entry_cached(&r.e, dev);
+
+	update_replicas(c, fs_usage, &r.e, sectors);
+}
+
 static void __bch2_invalidate_bucket(struct bch_fs *c, struct bch_dev *ca,
 				     size_t b, struct bucket_mark *old,
 				     bool gc)
@@ -349,8 +389,9 @@ static void __bch2_invalidate_bucket(struct bch_fs *c, struct bch_dev *ca,
 		new.gen++;
 	}));
 
-	fs_usage->replicas[0].data[BCH_DATA_CACHED]	-= old->cached_sectors;
-	fs_usage->s.cached				-= old->cached_sectors;
+	if (old->cached_sectors)
+		update_cached_sectors(c, fs_usage, ca->dev_idx,
+				      -old->cached_sectors);
 }
 
 void bch2_invalidate_bucket(struct bch_fs *c, struct bch_dev *ca,
@@ -417,11 +458,6 @@ static void __bch2_mark_metadata_bucket(struct bch_fs *c, struct bch_dev *ca,
 		new.data_type	= type;
 		checked_add(new.dirty_sectors, sectors);
 	}));
-
-	if (type == BCH_DATA_BTREE ||
-	    type == BCH_DATA_USER)
-		fs_usage->s.data		+= sectors;
-	fs_usage->replicas[0].data[type]	+= sectors;
 }
 
 void bch2_mark_metadata_bucket(struct bch_fs *c, struct bch_dev *ca,
@@ -545,9 +581,9 @@ static void bch2_mark_pointer(struct bch_fs *c,
 
 static int bch2_mark_stripe_ptr(struct bch_fs *c,
 				struct bch_extent_stripe_ptr p,
+				enum bch_data_type data_type,
+				struct bch_fs_usage *fs_usage,
 				s64 sectors, unsigned flags,
-				s64 *adjusted_disk_sectors,
-				unsigned *redundancy,
 				bool gc)
 {
 	struct stripe *m;
@@ -563,16 +599,15 @@ static int bch2_mark_stripe_ptr(struct bch_fs *c,
 		return -1;
 	}
 
+	BUG_ON(m->r.e.data_type != data_type);
+
 	nr_data = m->nr_blocks - m->nr_redundant;
 
 	parity_sectors = DIV_ROUND_UP(abs(sectors) * m->nr_redundant, nr_data);
 
 	if (sectors < 0)
 		parity_sectors = -parity_sectors;
-
-	*adjusted_disk_sectors += parity_sectors;
-
-	*redundancy = max_t(unsigned, *redundancy, m->nr_redundant + 1);
+	sectors += parity_sectors;
 
 	new = atomic_add_return(sectors, &m->block_sectors[p.block]);
 	old = new - sectors;
@@ -588,6 +623,8 @@ static int bch2_mark_stripe_ptr(struct bch_fs *c,
 	if (!gc)
 		bch2_stripes_heap_update(c, m, p.idx);
 
+	update_replicas(c, fs_usage, &m->r.e, sectors);
+
 	return 0;
 }
 
@@ -600,58 +637,46 @@ static int bch2_mark_extent(struct bch_fs *c, struct bkey_s_c k,
 	struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
 	const union bch_extent_entry *entry;
 	struct extent_ptr_decoded p;
-	s64 cached_sectors	= 0;
-	s64 dirty_sectors	= 0;
-	s64 ec_sectors		= 0;
-	unsigned replicas	= 0;
-	unsigned ec_redundancy	= 0;
+	struct bch_replicas_padded r;
+	s64 dirty_sectors = 0;
 	unsigned i;
 	int ret;
 
+	r.e.data_type	= data_type;
+	r.e.nr_devs	= 0;
+	r.e.nr_required	= 1;
+
 	BUG_ON(!sectors);
 
 	bkey_for_each_ptr_decode(k.k, ptrs, p, entry) {
 		s64 disk_sectors = data_type == BCH_DATA_BTREE
 			? sectors
 			: ptr_disk_sectors_delta(p, sectors);
-		s64 adjusted_disk_sectors = disk_sectors;
 
 		bch2_mark_pointer(c, p, disk_sectors, data_type,
 				  fs_usage, journal_seq, flags, gc);
 
-		if (!p.ptr.cached)
+		if (p.ptr.cached) {
+			update_cached_sectors(c, fs_usage, p.ptr.dev,
+					      disk_sectors);
+		} else if (!p.ec_nr) {
+			dirty_sectors	       += disk_sectors;
+			r.e.devs[r.e.nr_devs++]	= p.ptr.dev;
+		} else {
 			for (i = 0; i < p.ec_nr; i++) {
 				ret = bch2_mark_stripe_ptr(c, p.ec[i],
-						disk_sectors, flags,
-						&adjusted_disk_sectors,
-						&ec_redundancy, gc);
+						data_type, fs_usage,
+						disk_sectors, flags, gc);
 				if (ret)
 					return ret;
 			}
-		if (!p.ptr.cached)
-			replicas++;
 
-		if (p.ptr.cached)
-			cached_sectors	+= adjusted_disk_sectors;
-		else if (!p.ec_nr)
-			dirty_sectors	+= adjusted_disk_sectors;
-		else
-			ec_sectors	+= adjusted_disk_sectors;
+			r.e.nr_required = 0;
+		}
 	}
 
-	replicas	= clamp_t(unsigned,	replicas,
-				  1, ARRAY_SIZE(fs_usage->replicas));
-	ec_redundancy	= clamp_t(unsigned,	ec_redundancy,
-				  1, ARRAY_SIZE(fs_usage->replicas));
-
-	fs_usage->s.cached					+= cached_sectors;
-	fs_usage->replicas[0].data[BCH_DATA_CACHED]		+= cached_sectors;
-
-	fs_usage->s.data					+= dirty_sectors;
-	fs_usage->replicas[replicas - 1].data[data_type]	+= dirty_sectors;
-
-	fs_usage->s.data					+= ec_sectors;
-	fs_usage->replicas[ec_redundancy - 1].ec_data		+= ec_sectors;
+	if (dirty_sectors)
+		update_replicas(c, fs_usage, &r.e, dirty_sectors);
 
 	return 0;
 }
@@ -719,8 +744,24 @@ static int bch2_mark_stripe(struct bch_fs *c, struct bkey_s_c k,
 		m->algorithm	= s.v->algorithm;
 		m->nr_blocks	= s.v->nr_blocks;
 		m->nr_redundant	= s.v->nr_redundant;
+
+		memset(&m->r, 0, sizeof(m->r));
+
+		m->r.e.data_type	= BCH_DATA_USER;
+		m->r.e.nr_devs		= s.v->nr_blocks;
+		m->r.e.nr_required	= s.v->nr_blocks - s.v->nr_redundant;
+
+		for (i = 0; i < s.v->nr_blocks; i++)
+			m->r.e.devs[i] = s.v->ptrs[i].dev;
 	}
 
+	/*
+	 * XXX: account for stripes somehow here
+	 */
+#if 0
+	update_replicas(c, fs_usage, &m->r.e, stripe_sectors);
+#endif
+
 	if (!gc) {
 		if (inserting)
 			bch2_stripes_heap_insert(c, m, idx);
@@ -768,11 +809,11 @@ static int __bch2_mark_key(struct bch_fs *c, struct bkey_s_c k,
 		unsigned replicas = bkey_s_c_to_reservation(k).v->nr_replicas;
 
 		sectors *= replicas;
-		replicas = clamp_t(unsigned, replicas,
-				   1, ARRAY_SIZE(fs_usage->replicas));
+		replicas = clamp_t(unsigned, replicas, 1,
+				   ARRAY_SIZE(fs_usage->persistent_reserved));
 
-		fs_usage->s.reserved					+= sectors;
-		fs_usage->replicas[replicas - 1].persistent_reserved	+= sectors;
+		fs_usage->s.reserved				+= sectors;
+		fs_usage->persistent_reserved[replicas - 1]	+= sectors;
 		break;
 	}
 	default:
@@ -834,7 +875,7 @@ void bch2_mark_update(struct btree_insert *trans,
 	struct btree_iter	*iter = insert->iter;
 	struct btree		*b = iter->l[0].b;
 	struct btree_node_iter	node_iter = iter->l[0].iter;
-	struct bch_fs_usage	fs_usage = { 0 };
+	struct bch_fs_usage	*fs_usage;
 	struct gc_pos		pos = gc_pos_btree_node(b);
 	struct bkey_packed	*_k;
 	u64 disk_res_sectors = trans->disk_res ? trans->disk_res->sectors : 0;
@@ -844,12 +885,13 @@ void bch2_mark_update(struct btree_insert *trans,
 		return;
 
 	percpu_down_read_preempt_disable(&c->mark_lock);
+	fs_usage = bch2_fs_usage_get_scratch(c);
 
 	if (!(trans->flags & BTREE_INSERT_JOURNAL_REPLAY))
 		bch2_mark_key_locked(c, bkey_i_to_s_c(insert->k), true,
 			bpos_min(insert->k->k.p, b->key.k.p).offset -
 			bkey_start_offset(&insert->k->k),
-			pos, &fs_usage, trans->journal_res.seq, 0);
+			pos, fs_usage, trans->journal_res.seq, 0);
 
 	while ((_k = bch2_btree_node_iter_peek_filter(&node_iter, b,
 						      KEY_TYPE_discard))) {
@@ -882,7 +924,7 @@ void bch2_mark_update(struct btree_insert *trans,
 				BUG_ON(sectors <= 0);
 
 				bch2_mark_key_locked(c, k, true, sectors,
-					pos, &fs_usage, trans->journal_res.seq, 0);
+					pos, fs_usage, trans->journal_res.seq, 0);
 
 				sectors = bkey_start_offset(&insert->k->k) -
 					k.k->p.offset;
@@ -893,12 +935,12 @@ void bch2_mark_update(struct btree_insert *trans,
 		}
 
 		bch2_mark_key_locked(c, k, false, sectors,
-			pos, &fs_usage, trans->journal_res.seq, 0);
+			pos, fs_usage, trans->journal_res.seq, 0);
 
 		bch2_btree_node_iter_advance(&node_iter, b);
 	}
 
-	if (bch2_fs_usage_apply(c, &fs_usage, trans->disk_res, pos) &&
+	if (bch2_fs_usage_apply(c, fs_usage, trans->disk_res, pos) &&
 	    !warned_disk_usage &&
 	    !xchg(&warned_disk_usage, 1)) {
 		char buf[200];
diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h
index 10cacf36d238..fe93eb24dce9 100644
--- a/fs/bcachefs/buckets.h
+++ b/fs/bcachefs/buckets.h
@@ -217,7 +217,18 @@ static inline u64 dev_buckets_free(struct bch_fs *c, struct bch_dev *ca)
 
 /* Filesystem usage: */
 
-struct bch_fs_usage bch2_fs_usage_read(struct bch_fs *);
+static inline struct bch_fs_usage *bch2_fs_usage_get_scratch(struct bch_fs *c)
+{
+	struct bch_fs_usage *ret;
+
+	ret = this_cpu_ptr(c->usage_scratch);
+
+	memset(ret, 0, sizeof(*ret) + c->replicas.nr * sizeof(u64));
+
+	return ret;
+}
+
+struct bch_fs_usage *bch2_fs_usage_read(struct bch_fs *);
 
 u64 bch2_fs_sectors_used(struct bch_fs *, struct bch_fs_usage);
 
diff --git a/fs/bcachefs/buckets_types.h b/fs/bcachefs/buckets_types.h
index c5537a283b97..56863c237126 100644
--- a/fs/bcachefs/buckets_types.h
+++ b/fs/bcachefs/buckets_types.h
@@ -74,16 +74,18 @@ struct bch_fs_usage {
 		u64		cached;
 		u64		reserved;
 		u64		nr_inodes;
+
+		/* XXX: add stats for compression ratio */
+#if 0
+		u64		uncompressed;
+		u64		compressed;
+#endif
 	} s;
 
 	/* broken out: */
-	struct {
-		u64		data[BCH_DATA_NR];
-		u64		ec_data;
-		u64		persistent_reserved;
-	}			replicas[BCH_REPLICAS_MAX];
 
-	u64			buckets[BCH_DATA_NR];
+	u64			persistent_reserved[BCH_REPLICAS_MAX];
+	u64			data[];
 };
 
 struct bch_fs_usage_short {
diff --git a/fs/bcachefs/chardev.c b/fs/bcachefs/chardev.c
index ac1ec5f62cea..b84ae5c937e5 100644
--- a/fs/bcachefs/chardev.c
+++ b/fs/bcachefs/chardev.c
@@ -393,21 +393,31 @@ static long bch2_ioctl_usage(struct bch_fs *c,
 	}
 
 	{
-		struct bch_fs_usage src = bch2_fs_usage_read(c);
+		struct bch_fs_usage *src;
 		struct bch_ioctl_fs_usage dst = {
 			.capacity		= c->capacity,
-			.used			= bch2_fs_sectors_used(c, src),
-			.online_reserved	= src.s.online_reserved,
 		};
 
+		src = bch2_fs_usage_read(c);
+		if (!src)
+			return -ENOMEM;
+
+		percpu_up_read_preempt_enable(&c->mark_lock);
+
+		dst.used		= bch2_fs_sectors_used(c, *src);
+		dst.online_reserved	= src->s.online_reserved;
+
 		for (i = 0; i < BCH_REPLICAS_MAX; i++) {
 			dst.persistent_reserved[i] =
-				src.replicas[i].persistent_reserved;
-
+				src->persistent_reserved[i];
+#if 0
 			for (j = 0; j < BCH_DATA_NR; j++)
 				dst.sectors[j][i] = src.replicas[i].data[j];
+#endif
 		}
 
+		kfree(src);
+
 		ret = copy_to_user(&user_arg->fs, &dst, sizeof(dst));
 		if (ret)
 			return ret;
diff --git a/fs/bcachefs/ec_types.h b/fs/bcachefs/ec_types.h
index d04298106fa5..44c5d3821a38 100644
--- a/fs/bcachefs/ec_types.h
+++ b/fs/bcachefs/ec_types.h
@@ -5,6 +5,11 @@
 
 #define EC_STRIPE_MAX	16
 
+struct bch_replicas_padded {
+	struct bch_replicas_entry	e;
+	u8				pad[EC_STRIPE_MAX];
+};
+
 struct stripe {
 	size_t			heap_idx;
 
@@ -17,6 +22,8 @@ struct stripe {
 	u8			alive;
 	atomic_t		blocks_nonempty;
 	atomic_t		block_sectors[EC_STRIPE_MAX];
+
+	struct bch_replicas_padded r;
 };
 
 struct ec_stripe_heap_entry {
diff --git a/fs/bcachefs/eytzinger.h b/fs/bcachefs/eytzinger.h
index 66fa227c552d..d19d809c7580 100644
--- a/fs/bcachefs/eytzinger.h
+++ b/fs/bcachefs/eytzinger.h
@@ -262,18 +262,20 @@ static inline ssize_t eytzinger0_find_le(void *base, size_t nr, size_t size,
 	}
 }
 
-static inline size_t eytzinger0_find(void *base, size_t nr, size_t size,
-				     eytzinger_cmp_fn cmp, const void *search)
-{
-	size_t i = 0;
-	int res;
-
-	while (i < nr &&
-	       (res = cmp(search, base + i * size, size)))
-		i = eytzinger0_child(i, res > 0);
-
-	return i;
-}
+#define eytzinger0_find(base, nr, size, _cmp, search)			\
+({									\
+	void *_base	= (base);					\
+	void *_search	= (search);					\
+	size_t _nr	= (nr);						\
+	size_t _size	= (size);					\
+	size_t _i	= 0;						\
+	int _res;							\
+									\
+	while (_i < _nr &&						\
+	       (_res = _cmp(_search, _base + _i * _size, _size)))	\
+		_i = eytzinger0_child(_i, _res > 0);			\
+	_i;								\
+})
 
 void eytzinger0_sort(void *, size_t, size_t,
 		    int (*cmp_func)(const void *, const void *, size_t),
diff --git a/fs/bcachefs/journal_io.c b/fs/bcachefs/journal_io.c
index 52498627f70e..5cc0651c7449 100644
--- a/fs/bcachefs/journal_io.c
+++ b/fs/bcachefs/journal_io.c
@@ -694,6 +694,11 @@ int bch2_journal_read(struct bch_fs *c, struct list_head *list)
 	}
 
 	list_for_each_entry(i, list, list) {
+		struct bch_replicas_padded replicas;
+		char buf[80];
+
+		bch2_devlist_to_replicas(&replicas.e, BCH_DATA_JOURNAL, i->devs);
+
 		ret = jset_validate_entries(c, &i->j, READ);
 		if (ret)
 			goto fsck_err;
@@ -705,11 +710,11 @@ int bch2_journal_read(struct bch_fs *c, struct list_head *list)
 
 		if (!degraded &&
 		    (test_bit(BCH_FS_REBUILD_REPLICAS, &c->flags) ||
-		     fsck_err_on(!bch2_replicas_marked(c, BCH_DATA_JOURNAL,
-						       i->devs, false), c,
-				 "superblock not marked as containing replicas (type %u)",
-				 BCH_DATA_JOURNAL))) {
-			ret = bch2_mark_replicas(c, BCH_DATA_JOURNAL, i->devs);
+		     fsck_err_on(!bch2_replicas_marked(c, &replicas.e, false), c,
+				 "superblock not marked as containing replicas %s",
+				 (bch2_replicas_entry_to_text(&PBUF(buf),
+							      &replicas.e), buf)))) {
+			ret = bch2_mark_replicas(c, &replicas.e);
 			if (ret)
 				return ret;
 		}
@@ -1108,6 +1113,7 @@ static void journal_write_done(struct closure *cl)
 	struct journal_buf *w = journal_prev_buf(j);
 	struct bch_devs_list devs =
 		bch2_bkey_devs(bkey_i_to_s_c(&w->key));
+	struct bch_replicas_padded replicas;
 	u64 seq = le64_to_cpu(w->data->seq);
 	u64 last_seq = le64_to_cpu(w->data->last_seq);
 
@@ -1118,7 +1124,9 @@ static void journal_write_done(struct closure *cl)
 		goto err;
 	}
 
-	if (bch2_mark_replicas(c, BCH_DATA_JOURNAL, devs))
+	bch2_devlist_to_replicas(&replicas.e, BCH_DATA_JOURNAL, devs);
+
+	if (bch2_mark_replicas(c, &replicas.e))
 		goto err;
 
 	spin_lock(&j->lock);
diff --git a/fs/bcachefs/journal_reclaim.c b/fs/bcachefs/journal_reclaim.c
index 84be61f8ac48..a795e888c56b 100644
--- a/fs/bcachefs/journal_reclaim.c
+++ b/fs/bcachefs/journal_reclaim.c
@@ -387,7 +387,6 @@ int bch2_journal_flush_device_pins(struct journal *j, int dev_idx)
 {
 	struct bch_fs *c = container_of(j, struct bch_fs, journal);
 	struct journal_entry_pin_list *p;
-	struct bch_devs_list devs;
 	u64 iter, seq = 0;
 	int ret = 0;
 
@@ -412,12 +411,15 @@ int bch2_journal_flush_device_pins(struct journal *j, int dev_idx)
 
 	spin_lock(&j->lock);
 	while (!ret && seq < j->pin.back) {
+		struct bch_replicas_padded replicas;
+
 		seq = max(seq, journal_last_seq(j));
-		devs = journal_seq_pin(j, seq)->devs;
+		bch2_devlist_to_replicas(&replicas.e, BCH_DATA_JOURNAL,
+					 journal_seq_pin(j, seq)->devs);
 		seq++;
 
 		spin_unlock(&j->lock);
-		ret = bch2_mark_replicas(c, BCH_DATA_JOURNAL, devs);
+		ret = bch2_mark_replicas(c, &replicas.e);
 		spin_lock(&j->lock);
 	}
 	spin_unlock(&j->lock);
diff --git a/fs/bcachefs/migrate.c b/fs/bcachefs/migrate.c
index b21986514d66..bb425d88a84a 100644
--- a/fs/bcachefs/migrate.c
+++ b/fs/bcachefs/migrate.c
@@ -4,6 +4,7 @@
 
 #include "bcachefs.h"
 #include "btree_update.h"
+#include "btree_update_interior.h"
 #include "buckets.h"
 #include "extents.h"
 #include "io.h"
@@ -152,6 +153,16 @@ retry:
 		bch2_btree_iter_unlock(&iter);
 	}
 
+	/* flush relevant btree updates */
+	while (1) {
+		closure_wait_event(&c->btree_interior_update_wait,
+				   !bch2_btree_interior_updates_nr_pending(c) ||
+				   c->btree_roots_dirty);
+		if (!bch2_btree_interior_updates_nr_pending(c))
+			break;
+		bch2_journal_meta(&c->journal);
+	}
+
 	ret = 0;
 out:
 	ret = bch2_replicas_gc_end(c, ret);
diff --git a/fs/bcachefs/move.c b/fs/bcachefs/move.c
index 80909ae4fb6d..98cfcefd9cdf 100644
--- a/fs/bcachefs/move.c
+++ b/fs/bcachefs/move.c
@@ -3,6 +3,7 @@
 #include "alloc_foreground.h"
 #include "btree_gc.h"
 #include "btree_update.h"
+#include "btree_update_interior.h"
 #include "buckets.h"
 #include "disk_groups.h"
 #include "inode.h"
@@ -763,6 +764,16 @@ int bch2_data_job(struct bch_fs *c,
 		ret = bch2_journal_flush_device_pins(&c->journal, -1);
 
 		ret = bch2_move_btree(c, rereplicate_pred, c, stats) ?: ret;
+
+		while (1) {
+			closure_wait_event(&c->btree_interior_update_wait,
+					   !bch2_btree_interior_updates_nr_pending(c) ||
+					   c->btree_roots_dirty);
+			if (!bch2_btree_interior_updates_nr_pending(c))
+				break;
+			bch2_journal_meta(&c->journal);
+		}
+
 		ret = bch2_gc_btree_replicas(c) ?: ret;
 
 		ret = bch2_move_data(c, NULL,
diff --git a/fs/bcachefs/replicas.c b/fs/bcachefs/replicas.c
index 0ba817cf7ce1..230f807bdf10 100644
--- a/fs/bcachefs/replicas.c
+++ b/fs/bcachefs/replicas.c
@@ -3,11 +3,6 @@
 #include "replicas.h"
 #include "super-io.h"
 
-struct bch_replicas_padded {
-	struct bch_replicas_entry	e;
-	u8				pad[BCH_SB_MEMBERS_MAX];
-};
-
 static int bch2_cpu_replicas_to_sb_replicas(struct bch_fs *,
 					    struct bch_replicas_cpu *);
 
@@ -18,6 +13,16 @@ static inline int u8_cmp(u8 l, u8 r)
 	return (l > r) - (l < r);
 }
 
+static void verify_replicas_entry_sorted(struct bch_replicas_entry *e)
+{
+#ifdef CONFIG_BCACHES_DEBUG
+	unsigned i;
+
+	for (i = 0; i + 1 < e->nr_devs; i++)
+		BUG_ON(e->devs[i] >= e->devs[i + 1]);
+#endif
+}
+
 static void replicas_entry_sort(struct bch_replicas_entry *e)
 {
 	bubble_sort(e->devs, e->nr_devs, u8_cmp);
@@ -28,19 +33,13 @@ static void replicas_entry_sort(struct bch_replicas_entry *e)
 	     (void *) (_i) < (void *) (_r)->entries + (_r)->nr * (_r)->entry_size;\
 	     _i = (void *) (_i) + (_r)->entry_size)
 
-static inline struct bch_replicas_entry *
-cpu_replicas_entry(struct bch_replicas_cpu *r, unsigned i)
-{
-	return (void *) r->entries + r->entry_size * i;
-}
-
 static void bch2_cpu_replicas_sort(struct bch_replicas_cpu *r)
 {
 	eytzinger0_sort(r->entries, r->nr, r->entry_size, memcmp, NULL);
 }
 
-static void replicas_entry_to_text(struct printbuf *out,
-				  struct bch_replicas_entry *e)
+void bch2_replicas_entry_to_text(struct printbuf *out,
+				 struct bch_replicas_entry *e)
 {
 	unsigned i;
 
@@ -65,7 +64,7 @@ void bch2_cpu_replicas_to_text(struct printbuf *out,
 			pr_buf(out, " ");
 		first = false;
 
-		replicas_entry_to_text(out, e);
+		bch2_replicas_entry_to_text(out, e);
 	}
 }
 
@@ -105,8 +104,8 @@ static void stripe_to_replicas(struct bkey_s_c k,
 		r->devs[r->nr_devs++] = ptr->dev;
 }
 
-static void bkey_to_replicas(struct bkey_s_c k,
-			     struct bch_replicas_entry *e)
+static void bkey_to_replicas(struct bch_replicas_entry *e,
+			     struct bkey_s_c k)
 {
 	e->nr_devs = 0;
 
@@ -128,9 +127,9 @@ static void bkey_to_replicas(struct bkey_s_c k,
 	replicas_entry_sort(e);
 }
 
-static inline void devlist_to_replicas(struct bch_devs_list devs,
-				       enum bch_data_type data_type,
-				       struct bch_replicas_entry *e)
+void bch2_devlist_to_replicas(struct bch_replicas_entry *e,
+			      enum bch_data_type data_type,
+			      struct bch_devs_list devs)
 {
 	unsigned i;
 
@@ -159,6 +158,9 @@ cpu_replicas_add_entry(struct bch_replicas_cpu *old,
 					replicas_entry_bytes(new_entry)),
 	};
 
+	BUG_ON(!new_entry->data_type);
+	verify_replicas_entry_sorted(new_entry);
+
 	new.entries = kcalloc(new.nr, new.entry_size, GFP_NOIO);
 	if (!new.entries)
 		return new;
@@ -176,21 +178,49 @@ cpu_replicas_add_entry(struct bch_replicas_cpu *old,
 	return new;
 }
 
+static inline int __replicas_entry_idx(struct bch_replicas_cpu *r,
+				       struct bch_replicas_entry *search)
+{
+	int idx, entry_size = replicas_entry_bytes(search);
+
+	if (unlikely(entry_size > r->entry_size))
+		return -1;
+
+	verify_replicas_entry_sorted(search);
+
+#define entry_cmp(_l, _r, size)	memcmp(_l, _r, entry_size)
+	idx = eytzinger0_find(r->entries, r->nr, r->entry_size,
+			      entry_cmp, search);
+#undef entry_cmp
+
+	return idx < r->nr ? idx : -1;
+}
+
+int bch2_replicas_entry_idx(struct bch_fs *c,
+			    struct bch_replicas_entry *search)
+{
+	replicas_entry_sort(search);
+
+	return __replicas_entry_idx(&c->replicas, search);
+}
+
 static bool __replicas_has_entry(struct bch_replicas_cpu *r,
 				 struct bch_replicas_entry *search)
 {
-	return replicas_entry_bytes(search) <= r->entry_size &&
-		eytzinger0_find(r->entries, r->nr,
-				r->entry_size,
-				memcmp, search) < r->nr;
+	return __replicas_entry_idx(r, search) >= 0;
 }
 
-static bool replicas_has_entry(struct bch_fs *c,
-			       struct bch_replicas_entry *search,
-			       bool check_gc_replicas)
+bool bch2_replicas_marked(struct bch_fs *c,
+			  struct bch_replicas_entry *search,
+			  bool check_gc_replicas)
 {
 	bool marked;
 
+	if (!search->nr_devs)
+		return true;
+
+	verify_replicas_entry_sorted(search);
+
 	percpu_down_read_preempt_disable(&c->mark_lock);
 	marked = __replicas_has_entry(&c->replicas, search) &&
 		(!check_gc_replicas ||
@@ -201,6 +231,76 @@ static bool replicas_has_entry(struct bch_fs *c,
 	return marked;
 }
 
+static void __replicas_table_update(struct bch_fs_usage __percpu *dst_p,
+				    struct bch_replicas_cpu *dst_r,
+				    struct bch_fs_usage __percpu *src_p,
+				    struct bch_replicas_cpu *src_r)
+{
+	unsigned src_nr = sizeof(struct bch_fs_usage) / sizeof(u64) + src_r->nr;
+	struct bch_fs_usage *dst, *src = (void *)
+		bch2_acc_percpu_u64s((void *) src_p, src_nr);
+	int src_idx, dst_idx;
+
+	preempt_disable();
+	dst = this_cpu_ptr(dst_p);
+	preempt_enable();
+
+	*dst = *src;
+
+	for (src_idx = 0; src_idx < src_r->nr; src_idx++) {
+		if (!src->data[src_idx])
+			continue;
+
+		dst_idx = __replicas_entry_idx(dst_r,
+				cpu_replicas_entry(src_r, src_idx));
+		BUG_ON(dst_idx < 0);
+
+		dst->data[dst_idx] = src->data[src_idx];
+	}
+}
+
+/*
+ * Resize filesystem accounting:
+ */
+static int replicas_table_update(struct bch_fs *c,
+				 struct bch_replicas_cpu *new_r)
+{
+	struct bch_fs_usage __percpu *new_usage[3] = { NULL, NULL, NULL };
+	unsigned bytes = sizeof(struct bch_fs_usage) +
+		sizeof(u64) * new_r->nr;
+	unsigned i;
+	int ret = -ENOMEM;
+
+	for (i = 0; i < 3; i++) {
+		if (i < 2 && !c->usage[i])
+			continue;
+
+		new_usage[i] = __alloc_percpu_gfp(bytes, sizeof(u64),
+						  GFP_NOIO);
+		if (!new_usage[i])
+			goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (!c->usage[i])
+			continue;
+
+		__replicas_table_update(new_usage[i],	new_r,
+					c->usage[i],	&c->replicas);
+
+		swap(c->usage[i], new_usage[i]);
+	}
+
+	swap(c->usage_scratch, new_usage[2]);
+
+	swap(c->replicas, *new_r);
+	ret = 0;
+err:
+	for (i = 0; i < 3; i++)
+		free_percpu(new_usage[i]);
+	return ret;
+}
+
 noinline
 static int bch2_mark_replicas_slowpath(struct bch_fs *c,
 				struct bch_replicas_entry *new_entry)
@@ -242,7 +342,7 @@ static int bch2_mark_replicas_slowpath(struct bch_fs *c,
 	/* don't update in memory replicas until changes are persistent */
 	percpu_down_write(&c->mark_lock);
 	if (new_r.entries)
-		swap(new_r, c->replicas);
+		ret = replicas_table_update(c, &new_r);
 	if (new_gc.entries)
 		swap(new_gc, c->replicas_gc);
 	percpu_up_write(&c->mark_lock);
@@ -257,30 +357,32 @@ err:
 	return ret;
 }
 
-static int __bch2_mark_replicas(struct bch_fs *c,
-				struct bch_replicas_entry *devs)
+int bch2_mark_replicas(struct bch_fs *c,
+		       struct bch_replicas_entry *r)
 {
-	return likely(replicas_has_entry(c, devs, true))
+	return likely(bch2_replicas_marked(c, r, true))
 		? 0
-		: bch2_mark_replicas_slowpath(c, devs);
+		: bch2_mark_replicas_slowpath(c, r);
 }
 
-int bch2_mark_replicas(struct bch_fs *c,
-		       enum bch_data_type data_type,
-		       struct bch_devs_list devs)
+bool bch2_bkey_replicas_marked(struct bch_fs *c,
+			       struct bkey_s_c k,
+			       bool check_gc_replicas)
 {
 	struct bch_replicas_padded search;
+	struct bch_devs_list cached = bch2_bkey_cached_devs(k);
+	unsigned i;
 
-	if (!devs.nr)
-		return 0;
-
-	memset(&search, 0, sizeof(search));
+	for (i = 0; i < cached.nr; i++) {
+		bch2_replicas_entry_cached(&search.e, cached.devs[i]);
 
-	BUG_ON(devs.nr >= BCH_REPLICAS_MAX);
+		if (!bch2_replicas_marked(c, &search.e, check_gc_replicas))
+			return false;
+	}
 
-	devlist_to_replicas(devs, data_type, &search.e);
+	bkey_to_replicas(&search.e, k);
 
-	return __bch2_mark_replicas(c, &search.e);
+	return bch2_replicas_marked(c, &search.e, check_gc_replicas);
 }
 
 int bch2_mark_bkey_replicas(struct bch_fs *c, struct bkey_s_c k)
@@ -290,22 +392,23 @@ int bch2_mark_bkey_replicas(struct bch_fs *c, struct bkey_s_c k)
 	unsigned i;
 	int ret;
 
-	memset(&search, 0, sizeof(search));
+	for (i = 0; i < cached.nr; i++) {
+		bch2_replicas_entry_cached(&search.e, cached.devs[i]);
 
-	for (i = 0; i < cached.nr; i++)
-		if ((ret = bch2_mark_replicas(c, BCH_DATA_CACHED,
-					      bch2_dev_list_single(cached.devs[i]))))
+		ret = bch2_mark_replicas(c, &search.e);
+		if (ret)
 			return ret;
+	}
 
-	bkey_to_replicas(k, &search.e);
+	bkey_to_replicas(&search.e, k);
 
-	return search.e.nr_devs
-		? __bch2_mark_replicas(c, &search.e)
-		: 0;
+	return bch2_mark_replicas(c, &search.e);
 }
 
 int bch2_replicas_gc_end(struct bch_fs *c, int ret)
 {
+	unsigned i;
+
 	lockdep_assert_held(&c->replicas_gc_lock);
 
 	mutex_lock(&c->sb_lock);
@@ -313,6 +416,39 @@ int bch2_replicas_gc_end(struct bch_fs *c, int ret)
 	if (ret)
 		goto err;
 
+	/*
+	 * this is kind of crappy; the replicas gc mechanism needs to be ripped
+	 * out
+	 */
+
+	for (i = 0; i < c->replicas.nr; i++) {
+		struct bch_replicas_entry *e =
+			cpu_replicas_entry(&c->replicas, i);
+		struct bch_replicas_cpu n;
+		u64 v = 0;
+		int cpu;
+
+		if (__replicas_has_entry(&c->replicas_gc, e))
+			continue;
+
+		for_each_possible_cpu(cpu)
+			v += *per_cpu_ptr(&c->usage[0]->data[i], cpu);
+		if (!v)
+			continue;
+
+		n = cpu_replicas_add_entry(&c->replicas_gc, e);
+		if (!n.entries) {
+			ret = -ENOSPC;
+			goto err;
+		}
+
+		percpu_down_write(&c->mark_lock);
+		swap(n, c->replicas_gc);
+		percpu_up_write(&c->mark_lock);
+
+		kfree(n.entries);
+	}
+
 	if (bch2_cpu_replicas_to_sb_replicas(c, &c->replicas_gc)) {
 		ret = -ENOSPC;
 		goto err;
@@ -324,7 +460,7 @@ int bch2_replicas_gc_end(struct bch_fs *c, int ret)
 err:
 	percpu_down_write(&c->mark_lock);
 	if (!ret)
-		swap(c->replicas, c->replicas_gc);
+		ret = replicas_table_update(c, &c->replicas_gc);
 
 	kfree(c->replicas_gc.entries);
 	c->replicas_gc.entries = NULL;
@@ -460,7 +596,7 @@ int bch2_sb_replicas_to_cpu_replicas(struct bch_fs *c)
 	bch2_cpu_replicas_sort(&new_r);
 
 	percpu_down_write(&c->mark_lock);
-	swap(c->replicas, new_r);
+	ret = replicas_table_update(c, &new_r);
 	percpu_up_write(&c->mark_lock);
 
 	kfree(new_r.entries);
@@ -627,7 +763,7 @@ static void bch2_sb_replicas_to_text(struct printbuf *out,
 			pr_buf(out, " ");
 		first = false;
 
-		replicas_entry_to_text(out, e);
+		bch2_replicas_entry_to_text(out, e);
 	}
 }
 
@@ -676,46 +812,6 @@ const struct bch_sb_field_ops bch_sb_field_ops_replicas_v0 = {
 
 /* Query replicas: */
 
-bool bch2_replicas_marked(struct bch_fs *c,
-			  enum bch_data_type data_type,
-			  struct bch_devs_list devs,
-			  bool check_gc_replicas)
-{
-	struct bch_replicas_padded search;
-
-	if (!devs.nr)
-		return true;
-
-	memset(&search, 0, sizeof(search));
-
-	devlist_to_replicas(devs, data_type, &search.e);
-
-	return replicas_has_entry(c, &search.e, check_gc_replicas);
-}
-
-bool bch2_bkey_replicas_marked(struct bch_fs *c,
-			       struct bkey_s_c k,
-			       bool check_gc_replicas)
-{
-	struct bch_replicas_padded search;
-	struct bch_devs_list cached = bch2_bkey_cached_devs(k);
-	unsigned i;
-
-	memset(&search, 0, sizeof(search));
-
-	for (i = 0; i < cached.nr; i++)
-		if (!bch2_replicas_marked(c, BCH_DATA_CACHED,
-					  bch2_dev_list_single(cached.devs[i]),
-					  check_gc_replicas))
-			return false;
-
-	bkey_to_replicas(k, &search.e);
-
-	return search.e.nr_devs
-		? replicas_has_entry(c, &search.e, check_gc_replicas)
-		: true;
-}
-
 struct replicas_status __bch2_replicas_status(struct bch_fs *c,
 					      struct bch_devs_mask online_devs)
 {
diff --git a/fs/bcachefs/replicas.h b/fs/bcachefs/replicas.h
index 87246a04ccc4..0ac2b8e082cd 100644
--- a/fs/bcachefs/replicas.h
+++ b/fs/bcachefs/replicas.h
@@ -1,17 +1,42 @@
 #ifndef _BCACHEFS_REPLICAS_H
 #define _BCACHEFS_REPLICAS_H
 
+#include "eytzinger.h"
 #include "replicas_types.h"
 
-bool bch2_replicas_marked(struct bch_fs *, enum bch_data_type,
-			  struct bch_devs_list, bool);
+void bch2_replicas_entry_to_text(struct printbuf *,
+				 struct bch_replicas_entry *);
+void bch2_cpu_replicas_to_text(struct printbuf *, struct bch_replicas_cpu *);
+
+static inline struct bch_replicas_entry *
+cpu_replicas_entry(struct bch_replicas_cpu *r, unsigned i)
+{
+	return (void *) r->entries + r->entry_size * i;
+}
+
+int bch2_replicas_entry_idx(struct bch_fs *,
+			    struct bch_replicas_entry *);
+
+void bch2_devlist_to_replicas(struct bch_replicas_entry *,
+			      enum bch_data_type,
+			      struct bch_devs_list);
+bool bch2_replicas_marked(struct bch_fs *,
+			  struct bch_replicas_entry *, bool);
+int bch2_mark_replicas(struct bch_fs *,
+		       struct bch_replicas_entry *);
+
 bool bch2_bkey_replicas_marked(struct bch_fs *,
 			       struct bkey_s_c, bool);
-int bch2_mark_replicas(struct bch_fs *, enum bch_data_type,
-		       struct bch_devs_list);
 int bch2_mark_bkey_replicas(struct bch_fs *, struct bkey_s_c);
 
-void bch2_cpu_replicas_to_text(struct printbuf *, struct bch_replicas_cpu *);
+static inline void bch2_replicas_entry_cached(struct bch_replicas_entry *e,
+					      unsigned dev)
+{
+	e->data_type	= BCH_DATA_CACHED;
+	e->nr_devs	= 1;
+	e->nr_required	= 1;
+	e->devs[0]	= dev;
+}
 
 struct replicas_status {
 	struct {
diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c
index ee07c6735faa..b19ed44138d3 100644
--- a/fs/bcachefs/super.c
+++ b/fs/bcachefs/super.c
@@ -406,6 +406,7 @@ static void bch2_fs_free(struct bch_fs *c)
 	bch2_io_clock_exit(&c->io_clock[READ]);
 	bch2_fs_compress_exit(c);
 	percpu_free_rwsem(&c->mark_lock);
+	free_percpu(c->usage_scratch);
 	free_percpu(c->usage[0]);
 	free_percpu(c->pcpu);
 	mempool_exit(&c->btree_iters_pool);
@@ -538,7 +539,7 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts)
 {
 	struct bch_sb_field_members *mi;
 	struct bch_fs *c;
-	unsigned i, iter_size;
+	unsigned i, iter_size, fs_usage_size;
 	const char *err;
 
 	pr_verbose_init(opts, "");
@@ -632,6 +633,9 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts)
 		(btree_blocks(c) + 1) * 2 *
 		sizeof(struct btree_node_iter_set);
 
+	fs_usage_size = sizeof(struct bch_fs_usage) +
+		sizeof(u64) * c->replicas.nr;
+
 	if (!(c->wq = alloc_workqueue("bcachefs",
 				WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_HIGHPRI, 1)) ||
 	    !(c->copygc_wq = alloc_workqueue("bcache_copygc",
@@ -648,7 +652,8 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts)
 			max(offsetof(struct btree_read_bio, bio),
 			    offsetof(struct btree_write_bio, wbio.bio)),
 			BIOSET_NEED_BVECS) ||
-	    !(c->usage[0] = alloc_percpu(struct bch_fs_usage)) ||
+	    !(c->usage[0] = __alloc_percpu(fs_usage_size, sizeof(u64))) ||
+	    !(c->usage_scratch = __alloc_percpu(fs_usage_size, sizeof(u64))) ||
 	    !(c->pcpu = alloc_percpu(struct bch_fs_pcpu)) ||
 	    mempool_init_kvpmalloc_pool(&c->btree_bounce_pool, 1,
 					btree_bytes(c)) ||
diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c
index 3a25163e2de0..40384e7e5af8 100644
--- a/fs/bcachefs/sysfs.c
+++ b/fs/bcachefs/sysfs.c
@@ -233,33 +233,45 @@ static size_t bch2_btree_cache_size(struct bch_fs *c)
 static ssize_t show_fs_alloc_debug(struct bch_fs *c, char *buf)
 {
 	struct printbuf out = _PBUF(buf, PAGE_SIZE);
-	struct bch_fs_usage stats = bch2_fs_usage_read(c);
-	unsigned replicas, type;
+	struct bch_fs_usage *fs_usage = bch2_fs_usage_read(c);
+	unsigned i;
 
-	pr_buf(&out, "capacity:\t\t%llu\n", c->capacity);
+	if (!fs_usage)
+		return -ENOMEM;
 
-	for (replicas = 0; replicas < ARRAY_SIZE(stats.replicas); replicas++) {
-		pr_buf(&out, "%u replicas:\n", replicas + 1);
+	pr_buf(&out, "capacity:\t\t%llu\n", c->capacity);
 
+	for (i = 0;
+	     i < ARRAY_SIZE(fs_usage->persistent_reserved);
+	     i++) {
+		pr_buf(&out, "%u replicas:\n", i + 1);
+#if 0
 		for (type = BCH_DATA_SB; type < BCH_DATA_NR; type++)
 			pr_buf(&out, "\t%s:\t\t%llu\n",
 			       bch2_data_types[type],
 			       stats.replicas[replicas].data[type]);
 		pr_buf(&out, "\terasure coded:\t%llu\n",
 		       stats.replicas[replicas].ec_data);
+#endif
 		pr_buf(&out, "\treserved:\t%llu\n",
-		       stats.replicas[replicas].persistent_reserved);
+		       fs_usage->persistent_reserved[i]);
 	}
 
-	pr_buf(&out, "bucket usage\n");
+	pr_buf(&out, "online reserved:\t%llu\n",
+	       fs_usage->s.online_reserved);
 
-	for (type = BCH_DATA_SB; type < BCH_DATA_NR; type++)
-		pr_buf(&out, "\t%s:\t\t%llu\n",
-		       bch2_data_types[type],
-		       stats.buckets[type]);
+	for (i = 0; i < c->replicas.nr; i++) {
+		struct bch_replicas_entry *e =
+			cpu_replicas_entry(&c->replicas, i);
 
-	pr_buf(&out, "online reserved:\t%llu\n",
-	       stats.s.online_reserved);
+		pr_buf(&out, "\t");
+		bch2_replicas_entry_to_text(&out, e);
+		pr_buf(&out, ":\t%llu\n", fs_usage->data[i]);
+	}
+
+	percpu_up_read_preempt_enable(&c->mark_lock);
+
+	kfree(fs_usage);
 
 	return out.pos - buf;
 }
diff --git a/fs/bcachefs/util.c b/fs/bcachefs/util.c
index 528cb2fc4a69..fea80e248667 100644
--- a/fs/bcachefs/util.c
+++ b/fs/bcachefs/util.c
@@ -898,3 +898,28 @@ void eytzinger0_find_test(void)
 	kfree(test_array);
 }
 #endif
+
+/*
+ * Accumulate percpu counters onto one cpu's copy - only valid when access
+ * against any percpu counter is guarded against
+ */
+u64 *bch2_acc_percpu_u64s(u64 __percpu *p, unsigned nr)
+{
+	u64 *ret;
+	int cpu;
+
+	preempt_disable();
+	ret = this_cpu_ptr(p);
+	preempt_enable();
+
+	for_each_possible_cpu(cpu) {
+		u64 *i = per_cpu_ptr(p, cpu);
+
+		if (i != ret) {
+			acc_u64s(ret, i, nr);
+			memset(i, 0, nr * sizeof(u64));
+		}
+	}
+
+	return ret;
+}
diff --git a/fs/bcachefs/util.h b/fs/bcachefs/util.h
index 25d6750915e3..fbfb2085801c 100644
--- a/fs/bcachefs/util.h
+++ b/fs/bcachefs/util.h
@@ -715,4 +715,6 @@ static inline void acc_u64s_percpu(u64 *acc, const u64 __percpu *src,
 		acc_u64s(acc, per_cpu_ptr(src, cpu), nr);
 }
 
+u64 *bch2_acc_percpu_u64s(u64 __percpu *, unsigned);
+
 #endif /* _BCACHEFS_UTIL_H */