bcachefs: Hold usage_lock over mark_key and fs_usage_apply

Fixes an inconsistency at the end of gc

5 files changed, 104 insertions, 60 deletions

diff --git a/fs/bcachefs/bkey_methods.h b/fs/bcachefs/bkey_methods.h
index 11ce12f54c5d..a124eb4635c8 100644
--- a/fs/bcachefs/bkey_methods.h
+++ b/fs/bcachefs/bkey_methods.h
@@ -53,6 +53,18 @@ struct bkey_ops {
 	bool		is_extents;
 };
 
+static inline bool bkey_type_needs_gc(enum bkey_type type)
+{
+	switch (type) {
+	case BKEY_TYPE_BTREE:
+	case BKEY_TYPE_EXTENTS:
+	case BKEY_TYPE_EC:
+		return true;
+	default:
+		return false;
+	}
+}
+
 const char *bch2_bkey_val_invalid(struct bch_fs *, enum bkey_type,
 				  struct bkey_s_c);
 const char *__bch2_bkey_invalid(struct bch_fs *, enum bkey_type, struct bkey_s_c);
diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c
index 7c0394fbf90f..cab8a5c1e2f1 100644
--- a/fs/bcachefs/btree_gc.c
+++ b/fs/bcachefs/btree_gc.c
@@ -109,18 +109,6 @@ static void btree_node_range_checks(struct bch_fs *c, struct btree *b,
 
 /* marking of btree keys/nodes: */
 
-static bool bkey_type_needs_gc(enum bkey_type type)
-{
-	switch (type) {
-	case BKEY_TYPE_BTREE:
-	case BKEY_TYPE_EXTENTS:
-	case BKEY_TYPE_EC:
-		return true;
-	default:
-		return false;
-	}
-}
-
 static void ptr_gen_recalc_oldest(struct bch_fs *c,
 				  const struct bch_extent_ptr *ptr,
 				  u8 *max_stale)
diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c
index 52d67bcbca8c..7dfc332c2938 100644
--- a/fs/bcachefs/btree_update_interior.c
+++ b/fs/bcachefs/btree_update_interior.c
@@ -159,12 +159,7 @@ static void bch2_btree_node_free_index(struct btree_update *as, struct btree *b,
 {
 	struct bch_fs *c = as->c;
 	struct pending_btree_node_free *d;
-
-	/*
-	 * btree_update lock is only needed here to avoid racing with
-	 * gc:
-	 */
-	mutex_lock(&c->btree_interior_update_lock);
+	struct gc_pos pos = { 0 };
 
 	for (d = as->pending; d < as->pending + as->nr_pending; d++)
 		if (!bkey_cmp(k.k->p, d->key.k.p) &&
@@ -200,20 +195,11 @@ found:
 	if (gc_pos_cmp(c->gc_pos, b
 		       ? gc_pos_btree_node(b)
 		       : gc_pos_btree_root(as->btree_id)) >= 0 &&
-	    gc_pos_cmp(c->gc_pos, gc_phase(GC_PHASE_PENDING_DELETE)) < 0) {
-		struct gc_pos pos = { 0 };
-
-		bch2_mark_key(c, BKEY_TYPE_BTREE,
+	    gc_pos_cmp(c->gc_pos, gc_phase(GC_PHASE_PENDING_DELETE)) < 0)
+		bch2_mark_key_locked(c, BKEY_TYPE_BTREE,
 			      bkey_i_to_s_c(&d->key),
 			      false, 0, pos,
 			      NULL, 0, BCH_BUCKET_MARK_GC);
-		/*
-		 * Don't apply tmp - pending deletes aren't tracked in
-		 * bch_alloc_stats:
-		 */
-	}
-
-	mutex_unlock(&c->btree_interior_update_lock);
 }
 
 static void __btree_node_free(struct bch_fs *c, struct btree *b)
@@ -1082,7 +1068,10 @@ static void bch2_btree_set_root_inmem(struct btree_update *as, struct btree *b)
 
 	__bch2_btree_set_root_inmem(c, b);
 
-	bch2_mark_key(c, BKEY_TYPE_BTREE,
+	mutex_lock(&c->btree_interior_update_lock);
+	percpu_down_read_preempt_disable(&c->usage_lock);
+
+	bch2_mark_key_locked(c, BKEY_TYPE_BTREE,
 		      bkey_i_to_s_c(&b->key),
 		      true, 0,
 		      gc_pos_btree_root(b->btree_id),
@@ -1094,6 +1083,9 @@ static void bch2_btree_set_root_inmem(struct btree_update *as, struct btree *b)
 					   &stats);
 	bch2_fs_usage_apply(c, &stats, &as->reserve->disk_res,
 			    gc_pos_btree_root(b->btree_id));
+
+	percpu_up_read_preempt_enable(&c->usage_lock);
+	mutex_unlock(&c->btree_interior_update_lock);
 }
 
 static void bch2_btree_set_root_ondisk(struct bch_fs *c, struct btree *b, int rw)
@@ -1170,8 +1162,11 @@ static void bch2_insert_fixup_btree_ptr(struct btree_update *as, struct btree *b
 
 	BUG_ON(insert->k.u64s > bch_btree_keys_u64s_remaining(c, b));
 
+	mutex_lock(&c->btree_interior_update_lock);
+	percpu_down_read_preempt_disable(&c->usage_lock);
+
 	if (bkey_extent_is_data(&insert->k))
-		bch2_mark_key(c, BKEY_TYPE_BTREE,
+		bch2_mark_key_locked(c, BKEY_TYPE_BTREE,
 			      bkey_i_to_s_c(insert),
 			      true, 0,
 			      gc_pos_btree_node(b), &stats, 0, 0);
@@ -1192,6 +1187,9 @@ static void bch2_insert_fixup_btree_ptr(struct btree_update *as, struct btree *b
 	bch2_fs_usage_apply(c, &stats, &as->reserve->disk_res,
 			    gc_pos_btree_node(b));
 
+	percpu_up_read_preempt_enable(&c->usage_lock);
+	mutex_unlock(&c->btree_interior_update_lock);
+
 	bch2_btree_bset_insert_key(iter, b, node_iter, insert);
 	set_btree_node_dirty(b);
 	set_btree_node_need_write(b);
@@ -1976,7 +1974,10 @@ static void __bch2_btree_node_update_key(struct bch_fs *c,
 
 		bch2_btree_node_lock_write(b, iter);
 
-		bch2_mark_key(c, BKEY_TYPE_BTREE,
+		mutex_lock(&c->btree_interior_update_lock);
+		percpu_down_read_preempt_disable(&c->usage_lock);
+
+		bch2_mark_key_locked(c, BKEY_TYPE_BTREE,
 			      bkey_i_to_s_c(&new_key->k_i),
 			      true, 0,
 			      gc_pos_btree_root(b->btree_id),
@@ -1987,6 +1988,9 @@ static void __bch2_btree_node_update_key(struct bch_fs *c,
 		bch2_fs_usage_apply(c, &stats, &as->reserve->disk_res,
 				    gc_pos_btree_root(b->btree_id));
 
+		percpu_up_read_preempt_enable(&c->usage_lock);
+		mutex_unlock(&c->btree_interior_update_lock);
+
 		if (PTR_HASH(&new_key->k_i) != PTR_HASH(&b->key)) {
 			mutex_lock(&c->btree_cache.lock);
 			bch2_btree_node_hash_remove(&c->btree_cache, b);
diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c
index 1a10df42c50c..1f6b7b5c59ce 100644
--- a/fs/bcachefs/buckets.c
+++ b/fs/bcachefs/buckets.c
@@ -322,6 +322,8 @@ void bch2_fs_usage_apply(struct bch_fs *c,
 	s64 added = sum.data + sum.reserved;
 	s64 should_not_have_added;
 
+	percpu_rwsem_assert_held(&c->usage_lock);
+
 	/*
 	 * Not allowed to reduce sectors_available except by getting a
 	 * reservation:
@@ -338,7 +340,6 @@ void bch2_fs_usage_apply(struct bch_fs *c,
 		stats->online_reserved	-= added;
 	}
 
-	percpu_down_read_preempt_disable(&c->usage_lock);
 	/* online_reserved not subject to gc: */
 	this_cpu_ptr(c->usage[0])->online_reserved +=
 		stats->online_reserved;
@@ -350,7 +351,6 @@ void bch2_fs_usage_apply(struct bch_fs *c,
 		bch2_usage_add(this_cpu_ptr(c->usage[1]), stats);
 
 	bch2_fs_stats_verify(c);
-	percpu_up_read_preempt_enable(&c->usage_lock);
 
 	memset(stats, 0, sizeof(*stats));
 }
@@ -402,7 +402,24 @@ static void bch2_dev_usage_update(struct bch_fs *c, struct bch_dev *ca,
 	bch2_dev_stats_verify(ca);
 }
 
-#define bucket_data_cmpxchg(c, ca, fs_usage, g, new, expr)		\
+void bch2_dev_usage_from_buckets(struct bch_fs *c, struct bch_dev *ca)
+{
+	struct bucket_mark old = { .v.counter = 0 };
+	struct bch_fs_usage *fs_usage;
+	struct bucket_array *buckets;
+	struct bucket *g;
+
+	percpu_down_read_preempt_disable(&c->usage_lock);
+	fs_usage = this_cpu_ptr(c->usage[0]);
+	buckets = bucket_array(ca);
+
+	for_each_bucket(g, buckets)
+		if (g->mark.data_type)
+			bch2_dev_usage_update(c, ca, fs_usage, old, g->mark, false);
+	percpu_up_read_preempt_enable(&c->usage_lock);
+}
+
+#define bucket_data_cmpxchg(c, ca, fs_usage, g, new, expr)	\
 ({								\
 	struct bucket_mark _old = bucket_cmpxchg(g, new, expr);	\
 								\
@@ -486,12 +503,12 @@ static void __bch2_mark_metadata_bucket(struct bch_fs *c, struct bch_dev *ca,
 {
 	struct bch_fs_usage *fs_usage = this_cpu_ptr(c->usage[gc]);
 	struct bucket *g = __bucket(ca, b, gc);
-	struct bucket_mark old, new;
+	struct bucket_mark new;
 
 	BUG_ON(type != BCH_DATA_SB &&
 	       type != BCH_DATA_JOURNAL);
 
-	old = bucket_data_cmpxchg(c, ca, fs_usage, g, new, ({
+	bucket_data_cmpxchg(c, ca, fs_usage, g, new, ({
 		new.data_type	= type;
 		checked_add(new.dirty_sectors, sectors);
 	}));
@@ -868,16 +885,14 @@ static int __bch2_mark_key(struct bch_fs *c,
 	return ret;
 }
 
-int bch2_mark_key(struct bch_fs *c,
-		  enum bkey_type type, struct bkey_s_c k,
-		  bool inserting, s64 sectors,
-		  struct gc_pos pos,
-		  struct bch_fs_usage *stats,
-		  u64 journal_seq, unsigned flags)
+int bch2_mark_key_locked(struct bch_fs *c,
+		   enum bkey_type type, struct bkey_s_c k,
+		   bool inserting, s64 sectors,
+		   struct gc_pos pos,
+		   struct bch_fs_usage *stats,
+		   u64 journal_seq, unsigned flags)
 {
-	int ret = 0;
-
-	percpu_down_read_preempt_disable(&c->usage_lock);
+	int ret;
 
 	if (!(flags & BCH_BUCKET_MARK_GC)) {
 		if (!stats)
@@ -886,7 +901,7 @@ int bch2_mark_key(struct bch_fs *c,
 		ret = __bch2_mark_key(c, type, k, inserting, sectors,
 				      stats, journal_seq, flags, false);
 		if (ret)
-			goto out;
+			return ret;
 	}
 
 	if ((flags & BCH_BUCKET_MARK_GC) ||
@@ -895,9 +910,24 @@ int bch2_mark_key(struct bch_fs *c,
 				      this_cpu_ptr(c->usage[1]),
 				      journal_seq, flags, true);
 		if (ret)
-			goto out;
+			return ret;
 	}
-out:
+
+	return 0;
+}
+
+int bch2_mark_key(struct bch_fs *c,
+		  enum bkey_type type, struct bkey_s_c k,
+		  bool inserting, s64 sectors,
+		  struct gc_pos pos,
+		  struct bch_fs_usage *stats,
+		  u64 journal_seq, unsigned flags)
+{
+	int ret;
+
+	percpu_down_read_preempt_disable(&c->usage_lock);
+	ret = bch2_mark_key_locked(c, type, k, inserting, sectors,
+				   pos, stats, journal_seq, flags);
 	percpu_up_read_preempt_enable(&c->usage_lock);
 
 	return ret;
@@ -914,12 +944,17 @@ void bch2_mark_update(struct btree_insert *trans,
 	struct gc_pos		pos = gc_pos_btree_node(b);
 	struct bkey_packed	*_k;
 
+	if (!bkey_type_needs_gc(iter->btree_id))
+		return;
+
+	percpu_down_read_preempt_disable(&c->usage_lock);
+
 	if (!(trans->flags & BTREE_INSERT_JOURNAL_REPLAY))
-		bch2_mark_key(c, btree_node_type(b), bkey_i_to_s_c(insert->k),
-			      true,
-			      bpos_min(insert->k->k.p, b->key.k.p).offset -
-			      bkey_start_offset(&insert->k->k),
-			      pos, &stats, trans->journal_res.seq, 0);
+		bch2_mark_key_locked(c, btree_node_type(b),
+			bkey_i_to_s_c(insert->k), true,
+			bpos_min(insert->k->k.p, b->key.k.p).offset -
+			bkey_start_offset(&insert->k->k),
+			pos, &stats, trans->journal_res.seq, 0);
 
 	while ((_k = bch2_btree_node_iter_peek_filter(&node_iter, b,
 						      KEY_TYPE_DISCARD))) {
@@ -951,9 +986,9 @@ void bch2_mark_update(struct btree_insert *trans,
 				sectors = k.k->p.offset - insert->k->k.p.offset;
 				BUG_ON(sectors <= 0);
 
-				bch2_mark_key(c, btree_node_type(b), k,
-					      true, sectors,
-					      pos, &stats, trans->journal_res.seq, 0);
+				bch2_mark_key_locked(c, btree_node_type(b),
+					k, true, sectors, pos, &stats,
+					trans->journal_res.seq, 0);
 
 				sectors = bkey_start_offset(&insert->k->k) -
 					k.k->p.offset;
@@ -963,14 +998,16 @@ void bch2_mark_update(struct btree_insert *trans,
 			BUG_ON(sectors >= 0);
 		}
 
-		bch2_mark_key(c, btree_node_type(b), k,
-			      false, sectors,
-			      pos, &stats, trans->journal_res.seq, 0);
+		bch2_mark_key_locked(c, btree_node_type(b),
+			k, false, sectors, pos, &stats,
+			trans->journal_res.seq, 0);
 
 		bch2_btree_node_iter_advance(&node_iter, b);
 	}
 
 	bch2_fs_usage_apply(c, &stats, trans->disk_res, pos);
+
+	percpu_up_read_preempt_enable(&c->usage_lock);
 }
 
 /* Disk reservations: */
diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h
index 7e1cafdc326f..d7b23108fa7e 100644
--- a/fs/bcachefs/buckets.h
+++ b/fs/bcachefs/buckets.h
@@ -219,6 +219,9 @@ void bch2_mark_metadata_bucket(struct bch_fs *, struct bch_dev *,
 #define BCH_BUCKET_MARK_NOATOMIC		(1 << 0)
 #define BCH_BUCKET_MARK_GC			(1 << 1)
 
+int bch2_mark_key_locked(struct bch_fs *, enum bkey_type, struct bkey_s_c,
+		  bool, s64, struct gc_pos,
+		  struct bch_fs_usage *, u64, unsigned);
 int bch2_mark_key(struct bch_fs *, enum bkey_type, struct bkey_s_c,
 		  bool, s64, struct gc_pos,
 		  struct bch_fs_usage *, u64, unsigned);