bcachefs: Optimize bch2_dev_usage_read()

 - add bch2_dev_usage_read_fast(), which doesn't return by value -
   bch_dev_usage is big enough that we don't want the silent memcpy
 - tweak the allocation path to only call bch2_dev_usage_read() once per
   bucket allocated

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

3 files changed, 37 insertions, 20 deletions

diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c
index 580a02750c81..a9e0c7397292 100644
--- a/fs/bcachefs/alloc_foreground.c
+++ b/fs/bcachefs/alloc_foreground.c
@@ -494,11 +494,11 @@ static struct open_bucket *bch2_bucket_alloc_trans(struct btree_trans *trans,
 				      struct bch_dev *ca,
 				      enum alloc_reserve reserve,
 				      bool may_alloc_partial,
-				      struct closure *cl)
+				      struct closure *cl,
+				      struct bch_dev_usage *usage)
 {
 	struct bch_fs *c = trans->c;
 	struct open_bucket *ob = NULL;
-	struct bch_dev_usage usage;
 	bool freespace_initialized = READ_ONCE(ca->mi.freespace_initialized);
 	u64 start = freespace_initialized ? 0 : ca->bucket_alloc_trans_early_cursor;
 	u64 avail;
@@ -509,16 +509,16 @@ static struct open_bucket *bch2_bucket_alloc_trans(struct btree_trans *trans,
 	u64 skipped_nouse = 0;
 	bool waiting = false;
 again:
-	usage = bch2_dev_usage_read(ca);
-	avail = dev_buckets_free(ca, usage, reserve);
+	bch2_dev_usage_read_fast(ca, usage);
+	avail = dev_buckets_free(ca, *usage, reserve);
 
-	if (usage.d[BCH_DATA_need_discard].buckets > avail)
+	if (usage->d[BCH_DATA_need_discard].buckets > avail)
 		bch2_do_discards(c);
 
-	if (usage.d[BCH_DATA_need_gc_gens].buckets > avail)
+	if (usage->d[BCH_DATA_need_gc_gens].buckets > avail)
 		bch2_do_gc_gens(c);
 
-	if (should_invalidate_buckets(ca, usage))
+	if (should_invalidate_buckets(ca, *usage))
 		bch2_do_invalidates(c);
 
 	if (!avail) {
@@ -580,7 +580,7 @@ err:
 	else if (!bch2_err_matches(PTR_ERR(ob), BCH_ERR_transaction_restart))
 		trace_and_count(c, bucket_alloc_fail,
 				ca, bch2_alloc_reserves[reserve],
-				usage.d[BCH_DATA_free].buckets,
+				usage->d[BCH_DATA_free].buckets,
 				avail,
 				bch2_copygc_wait_amount(c),
 				c->copygc_wait - atomic64_read(&c->io_clock[WRITE].now),
@@ -599,11 +599,12 @@ struct open_bucket *bch2_bucket_alloc(struct bch_fs *c, struct bch_dev *ca,
 				      bool may_alloc_partial,
 				      struct closure *cl)
 {
+	struct bch_dev_usage usage;
 	struct open_bucket *ob;
 
 	bch2_trans_do(c, NULL, NULL, 0,
 		      PTR_ERR_OR_ZERO(ob = bch2_bucket_alloc_trans(&trans, ca, reserve,
-								   may_alloc_partial, cl)));
+							may_alloc_partial, cl, &usage)));
 	return ob;
 }
 
@@ -630,8 +631,9 @@ struct dev_alloc_list bch2_dev_alloc_list(struct bch_fs *c,
 	return ret;
 }
 
-void bch2_dev_stripe_increment(struct bch_dev *ca,
-			       struct dev_stripe_state *stripe)
+static inline void bch2_dev_stripe_increment_inlined(struct bch_dev *ca,
+			       struct dev_stripe_state *stripe,
+			       struct bch_dev_usage *usage)
 {
 	u64 *v = stripe->next_alloc + ca->dev_idx;
 	u64 free_space = dev_buckets_available(ca, RESERVE_none);
@@ -650,6 +652,15 @@ void bch2_dev_stripe_increment(struct bch_dev *ca,
 		*v = *v < scale ? 0 : *v - scale;
 }
 
+void bch2_dev_stripe_increment(struct bch_dev *ca,
+			       struct dev_stripe_state *stripe)
+{
+	struct bch_dev_usage usage;
+
+	bch2_dev_usage_read_fast(ca, &usage);
+	bch2_dev_stripe_increment_inlined(ca, stripe, &usage);
+}
+
 #define BUCKET_MAY_ALLOC_PARTIAL	(1 << 0)
 #define BUCKET_ALLOC_USE_DURABILITY	(1 << 1)
 
@@ -694,6 +705,7 @@ static int bch2_bucket_alloc_set_trans(struct btree_trans *trans,
 	BUG_ON(*nr_effective >= nr_replicas);
 
 	for (i = 0; i < devs_sorted.nr; i++) {
+		struct bch_dev_usage usage;
 		struct open_bucket *ob;
 
 		dev = devs_sorted.devs[i];
@@ -713,9 +725,9 @@ static int bch2_bucket_alloc_set_trans(struct btree_trans *trans,
 		}
 
 		ob = bch2_bucket_alloc_trans(trans, ca, reserve,
-				flags & BUCKET_MAY_ALLOC_PARTIAL, cl);
+				flags & BUCKET_MAY_ALLOC_PARTIAL, cl, &usage);
 		if (!IS_ERR(ob))
-			bch2_dev_stripe_increment(ca, stripe);
+			bch2_dev_stripe_increment_inlined(ca, stripe, &usage);
 		percpu_ref_put(&ca->ref);
 
 		if (IS_ERR(ob)) {
diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c
index e0b8c50add61..9ae6e866b69a 100644
--- a/fs/bcachefs/buckets.c
+++ b/fs/bcachefs/buckets.c
@@ -89,20 +89,17 @@ static inline struct bch_dev_usage *dev_usage_ptr(struct bch_dev *ca,
 			    : ca->usage[journal_seq & JOURNAL_BUF_MASK]);
 }
 
-struct bch_dev_usage bch2_dev_usage_read(struct bch_dev *ca)
+void bch2_dev_usage_read_fast(struct bch_dev *ca, struct bch_dev_usage *usage)
 {
 	struct bch_fs *c = ca->fs;
-	struct bch_dev_usage ret;
 	unsigned seq, i, u64s = dev_usage_u64s();
 
 	do {
 		seq = read_seqcount_begin(&c->usage_lock);
-		memcpy(&ret, ca->usage_base, u64s * sizeof(u64));
+		memcpy(usage, ca->usage_base, u64s * sizeof(u64));
 		for (i = 0; i < ARRAY_SIZE(ca->usage); i++)
-			acc_u64s_percpu((u64 *) &ret, (u64 __percpu *) ca->usage[i], u64s);
+			acc_u64s_percpu((u64 *) usage, (u64 __percpu *) ca->usage[i], u64s);
 	} while (read_seqcount_retry(&c->usage_lock, seq));
-
-	return ret;
 }
 
 static inline struct bch_fs_usage *fs_usage_ptr(struct bch_fs *c,
diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h
index 0d0894b5ee7c..2b9fe3aa6ef6 100644
--- a/fs/bcachefs/buckets.h
+++ b/fs/bcachefs/buckets.h
@@ -139,7 +139,15 @@ static inline u8 ptr_stale(struct bch_dev *ca,
 
 /* Device usage: */
 
-struct bch_dev_usage bch2_dev_usage_read(struct bch_dev *);
+void bch2_dev_usage_read_fast(struct bch_dev *, struct bch_dev_usage *);
+static inline struct bch_dev_usage bch2_dev_usage_read(struct bch_dev *ca)
+{
+	struct bch_dev_usage ret;
+
+	bch2_dev_usage_read_fast(ca, &ret);
+	return ret;
+}
+
 void bch2_dev_usage_init(struct bch_dev *);
 
 static inline u64 bch2_dev_buckets_reserved(struct bch_dev *ca, enum alloc_reserve reserve)