bcachefs: Introduce a separate journal watermark for copygc

Since journal reclaim -> btree key cache flushing may require the
allocation of new btree nodes, it has an implicit dependency on copygc
in order to make forward progress - so we should avoid blocking copygc
unless the journal is really close to full.

This introduces watermarks to replace our single MAY_GET_UNRESERVED bit
in the journal, and adds a watermark for copygc and plumbs it through.

Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>

10 files changed, 50 insertions, 53 deletions

diff --git a/fs/bcachefs/btree_key_cache.c b/fs/bcachefs/btree_key_cache.c
index b1b7a30417bc..e48bbc6af33f 100644
--- a/fs/bcachefs/btree_key_cache.c
+++ b/fs/bcachefs/btree_key_cache.c
@@ -421,7 +421,7 @@ static int btree_key_cache_flush_pos(struct btree_trans *trans,
 				  BTREE_INSERT_NOFAIL|
 				  BTREE_INSERT_USE_RESERVE|
 				  (ck->journal.seq == journal_last_seq(j)
-				   ? BTREE_INSERT_JOURNAL_RESERVED
+				   ? JOURNAL_WATERMARK_RESERVED
 				   : 0)|
 				  commit_flags);
 	if (ret) {
diff --git a/fs/bcachefs/btree_update.h b/fs/bcachefs/btree_update.h
index d9a406a28f47..ca142f955193 100644
--- a/fs/bcachefs/btree_update.h
+++ b/fs/bcachefs/btree_update.h
@@ -16,12 +16,12 @@ bool bch2_btree_bset_insert_key(struct btree_trans *, struct btree_path *,
 void bch2_btree_add_journal_pin(struct bch_fs *, struct btree *, u64);
 
 enum btree_insert_flags {
-	__BTREE_INSERT_NOFAIL,
+	/* First two bits for journal watermark: */
+	__BTREE_INSERT_NOFAIL = 2,
 	__BTREE_INSERT_NOCHECK_RW,
 	__BTREE_INSERT_LAZY_RW,
 	__BTREE_INSERT_USE_RESERVE,
 	__BTREE_INSERT_JOURNAL_REPLAY,
-	__BTREE_INSERT_JOURNAL_RESERVED,
 	__BTREE_INSERT_JOURNAL_RECLAIM,
 	__BTREE_INSERT_NOWAIT,
 	__BTREE_INSERT_GC_LOCK_HELD,
@@ -41,9 +41,6 @@ enum btree_insert_flags {
 /* Insert is for journal replay - don't get journal reservations: */
 #define BTREE_INSERT_JOURNAL_REPLAY	(1 << __BTREE_INSERT_JOURNAL_REPLAY)
 
-/* Indicates that we have pre-reserved space in the journal: */
-#define BTREE_INSERT_JOURNAL_RESERVED	(1 << __BTREE_INSERT_JOURNAL_RESERVED)
-
 /* Insert is being called from journal reclaim path: */
 #define BTREE_INSERT_JOURNAL_RECLAIM (1 << __BTREE_INSERT_JOURNAL_RECLAIM)
 
diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c
index 4ba229bfb0ee..832aed81cd39 100644
--- a/fs/bcachefs/btree_update_interior.c
+++ b/fs/bcachefs/btree_update_interior.c
@@ -606,7 +606,7 @@ static void btree_update_nodes_written(struct btree_update *as)
 			      BTREE_INSERT_NOFAIL|
 			      BTREE_INSERT_NOCHECK_RW|
 			      BTREE_INSERT_JOURNAL_RECLAIM|
-			      BTREE_INSERT_JOURNAL_RESERVED,
+			      JOURNAL_WATERMARK_RESERVED,
 			      btree_update_nodes_written_trans(&trans, as));
 	bch2_trans_exit(&trans);
 
@@ -970,13 +970,11 @@ bch2_btree_update_start(struct btree_trans *trans, struct btree_path *path,
 		? BCH_DISK_RESERVATION_NOFAIL : 0;
 	unsigned nr_nodes[2] = { 0, 0 };
 	unsigned update_level = level;
-	int journal_flags = 0;
+	int journal_flags = flags & JOURNAL_WATERMARK_MASK;
 	int ret = 0;
 
 	BUG_ON(!path->should_be_locked);
 
-	if (flags & BTREE_INSERT_JOURNAL_RESERVED)
-		journal_flags |= JOURNAL_RES_GET_RESERVED;
 	if (flags & BTREE_INSERT_JOURNAL_RECLAIM)
 		journal_flags |= JOURNAL_RES_GET_NONBLOCK;
 
@@ -1958,7 +1956,7 @@ static int __bch2_btree_node_update_key(struct btree_trans *trans,
 				BTREE_INSERT_NOCHECK_RW|
 				BTREE_INSERT_USE_RESERVE|
 				BTREE_INSERT_JOURNAL_RECLAIM|
-				BTREE_INSERT_JOURNAL_RESERVED);
+				JOURNAL_WATERMARK_RESERVED);
 	if (ret)
 		goto err;
 
diff --git a/fs/bcachefs/btree_update_leaf.c b/fs/bcachefs/btree_update_leaf.c
index 47623f3aa610..d2354c8fb1a5 100644
--- a/fs/bcachefs/btree_update_leaf.c
+++ b/fs/bcachefs/btree_update_leaf.c
@@ -295,11 +295,10 @@ static inline int bch2_trans_journal_res_get(struct btree_trans *trans,
 	struct bch_fs *c = trans->c;
 	int ret;
 
-	if (trans->flags & BTREE_INSERT_JOURNAL_RESERVED)
-		flags |= JOURNAL_RES_GET_RESERVED;
-
 	ret = bch2_journal_res_get(&c->journal, &trans->journal_res,
-				   trans->journal_u64s, flags);
+				   trans->journal_u64s,
+				   flags|
+				   (trans->flags & JOURNAL_WATERMARK_MASK));
 
 	return ret == -EAGAIN ? BTREE_INSERT_NEED_JOURNAL_RES : ret;
 }
@@ -900,8 +899,7 @@ static inline int do_bch2_trans_commit(struct btree_trans *trans,
 	ret = bch2_journal_preres_get(&c->journal,
 			&trans->journal_preres, trans->journal_preres_u64s,
 			JOURNAL_RES_GET_NONBLOCK|
-			((trans->flags & BTREE_INSERT_JOURNAL_RESERVED)
-			 ? JOURNAL_RES_GET_RESERVED : 0));
+			(trans->flags & JOURNAL_WATERMARK_MASK));
 	if (unlikely(ret == -EAGAIN))
 		ret = bch2_trans_journal_preres_get_cold(trans,
 						trans->journal_preres_u64s, trace_ip);
@@ -986,7 +984,7 @@ int bch2_trans_commit_error(struct btree_trans *trans,
 		bch2_trans_unlock(trans);
 
 		if ((trans->flags & BTREE_INSERT_JOURNAL_RECLAIM) &&
-		    !(trans->flags & BTREE_INSERT_JOURNAL_RESERVED)) {
+		    !(trans->flags & JOURNAL_WATERMARK_RESERVED)) {
 			trans->restarted = true;
 			ret = -EAGAIN;
 			break;
diff --git a/fs/bcachefs/journal.c b/fs/bcachefs/journal.c
index 6ea6810337db..a8d374144d53 100644
--- a/fs/bcachefs/journal.c
+++ b/fs/bcachefs/journal.c
@@ -355,8 +355,7 @@ retry:
 		return 0;
 	}
 
-	if (!(flags & JOURNAL_RES_GET_RESERVED) &&
-	    !test_bit(JOURNAL_MAY_GET_UNRESERVED, &j->flags)) {
+	if ((flags & JOURNAL_WATERMARK_MASK) < j->watermark) {
 		/*
 		 * Don't want to close current journal entry, just need to
 		 * invoke reclaim:
@@ -398,7 +397,7 @@ unlock:
 	     ret == cur_entry_journal_pin_full) &&
 	    !can_discard &&
 	    !nr_unwritten_journal_entries(j) &&
-	    (flags & JOURNAL_RES_GET_RESERVED)) {
+	    (flags & JOURNAL_WATERMARK_RESERVED)) {
 		struct printbuf buf = PRINTBUF;
 
 		bch_err(c, "Journal stuck! Hava a pre-reservation but journal full");
@@ -1234,6 +1233,7 @@ void __bch2_journal_debug_to_text(struct printbuf *out, struct journal *j)
 	pr_buf(out, "last_seq_ondisk:\t%llu\n",		j->last_seq_ondisk);
 	pr_buf(out, "flushed_seq_ondisk:\t%llu\n",	j->flushed_seq_ondisk);
 	pr_buf(out, "prereserved:\t\t%u/%u\n",		j->prereserved.reserved, j->prereserved.remaining);
+	pr_buf(out, "watermark:\t\t%u\n",		j->watermark);
 	pr_buf(out, "each entry reserved:\t%u\n",	j->entry_u64s_reserved);
 	pr_buf(out, "nr flush writes:\t%llu\n",		j->nr_flush_writes);
 	pr_buf(out, "nr noflush writes:\t%llu\n",	j->nr_noflush_writes);
diff --git a/fs/bcachefs/journal.h b/fs/bcachefs/journal.h
index 989c33157cd2..15e3075c5583 100644
--- a/fs/bcachefs/journal.h
+++ b/fs/bcachefs/journal.h
@@ -295,9 +295,8 @@ static inline void bch2_journal_res_put(struct journal *j,
 int bch2_journal_res_get_slowpath(struct journal *, struct journal_res *,
 				  unsigned);
 
-#define JOURNAL_RES_GET_NONBLOCK	(1 << 0)
-#define JOURNAL_RES_GET_CHECK		(1 << 1)
-#define JOURNAL_RES_GET_RESERVED	(1 << 2)
+#define JOURNAL_RES_GET_NONBLOCK	(1 << 2)
+#define JOURNAL_RES_GET_CHECK		(1 << 3)
 
 static inline int journal_res_get_fast(struct journal *j,
 				       struct journal_res *res,
@@ -318,8 +317,7 @@ static inline int journal_res_get_fast(struct journal *j,
 
 		EBUG_ON(!journal_state_count(new, new.idx));
 
-		if (!(flags & JOURNAL_RES_GET_RESERVED) &&
-		    !test_bit(JOURNAL_MAY_GET_UNRESERVED, &j->flags))
+		if ((flags & JOURNAL_WATERMARK_MASK) < j->watermark)
 			return 0;
 
 		new.cur_entry_offset += res->u64s;
@@ -372,23 +370,27 @@ out:
 
 /* journal_preres: */
 
-static inline bool journal_check_may_get_unreserved(struct journal *j)
+static inline void journal_set_watermark(struct journal *j)
 {
 	union journal_preres_state s = READ_ONCE(j->prereserved);
-	bool ret = s.reserved < s.remaining &&
-		fifo_free(&j->pin) > j->pin.size / 4;
-
-	lockdep_assert_held(&j->lock);
-
-	if (ret != test_bit(JOURNAL_MAY_GET_UNRESERVED, &j->flags)) {
-		if (ret) {
-			set_bit(JOURNAL_MAY_GET_UNRESERVED, &j->flags);
-			journal_wake(j);
-		} else {
-			clear_bit(JOURNAL_MAY_GET_UNRESERVED, &j->flags);
-		}
-	}
-	return ret;
+	unsigned watermark = JOURNAL_WATERMARK_ANY;
+
+	if (fifo_free(&j->pin) < j->pin.size / 4)
+		watermark = max_t(unsigned, watermark, JOURNAL_WATERMARK_COPYGC);
+	if (fifo_free(&j->pin) < j->pin.size / 8)
+		watermark = max_t(unsigned, watermark, JOURNAL_WATERMARK_RESERVED);
+
+	if (s.reserved > s.remaining)
+		watermark = max_t(unsigned, watermark, JOURNAL_WATERMARK_COPYGC);
+	if (!s.remaining)
+		watermark = max_t(unsigned, watermark, JOURNAL_WATERMARK_RESERVED);
+
+	if (watermark == j->watermark)
+		return;
+
+	swap(watermark, j->watermark);
+	if (watermark > j->watermark)
+		journal_wake(j);
 }
 
 static inline void bch2_journal_preres_put(struct journal *j,
@@ -408,12 +410,8 @@ static inline void bch2_journal_preres_put(struct journal *j,
 		closure_wake_up(&j->preres_wait);
 	}
 
-	if (s.reserved <= s.remaining &&
-	    !test_bit(JOURNAL_MAY_GET_UNRESERVED, &j->flags)) {
-		spin_lock(&j->lock);
-		journal_check_may_get_unreserved(j);
-		spin_unlock(&j->lock);
-	}
+	if (s.reserved <= s.remaining && j->watermark)
+		journal_set_watermark(j);
 }
 
 int __bch2_journal_preres_get(struct journal *,
@@ -434,7 +432,7 @@ static inline int bch2_journal_preres_get_fast(struct journal *j,
 		old.v = new.v = v;
 		ret = 0;
 
-		if ((flags & JOURNAL_RES_GET_RESERVED) ||
+		if ((flags & JOURNAL_WATERMARK_RESERVED) ||
 		    new.reserved + d < new.remaining) {
 			new.reserved += d;
 			ret = 1;
diff --git a/fs/bcachefs/journal_reclaim.c b/fs/bcachefs/journal_reclaim.c
index a920a111dad7..df28fe4a5757 100644
--- a/fs/bcachefs/journal_reclaim.c
+++ b/fs/bcachefs/journal_reclaim.c
@@ -238,7 +238,7 @@ out:
 	j->cur_entry_sectors	= !ret ? j->space[journal_space_discarded].next_entry : 0;
 	j->cur_entry_error	= ret;
 	journal_set_remaining(j, u64s_remaining);
-	journal_check_may_get_unreserved(j);
+	journal_set_watermark(j);
 
 	if (!ret)
 		journal_wake(j);
diff --git a/fs/bcachefs/journal_types.h b/fs/bcachefs/journal_types.h
index 071fcb4a8422..1609fdd670e3 100644
--- a/fs/bcachefs/journal_types.h
+++ b/fs/bcachefs/journal_types.h
@@ -144,7 +144,6 @@ enum journal_space_from {
 enum {
 	JOURNAL_REPLAY_DONE,
 	JOURNAL_STARTED,
-	JOURNAL_MAY_GET_UNRESERVED,
 	JOURNAL_MAY_SKIP_FLUSH,
 };
 
@@ -155,6 +154,12 @@ struct journal {
 	unsigned long		flags;
 
 	union journal_res_state reservations;
+	enum journal_watermark {
+#define JOURNAL_WATERMARK_MASK	3
+		JOURNAL_WATERMARK_ANY,
+		JOURNAL_WATERMARK_COPYGC,
+		JOURNAL_WATERMARK_RESERVED,
+	}			watermark;
 
 	/* Max size of current journal entry */
 	unsigned		cur_entry_u64s;
diff --git a/fs/bcachefs/movinggc.c b/fs/bcachefs/movinggc.c
index 1c92d5365958..897244ff6570 100644
--- a/fs/bcachefs/movinggc.c
+++ b/fs/bcachefs/movinggc.c
@@ -91,7 +91,7 @@ static enum data_cmd copygc_pred(struct bch_fs *c, void *arg,
 			data_opts->target		= io_opts->background_target;
 			data_opts->nr_replicas		= 1;
 			data_opts->btree_insert_flags	= BTREE_INSERT_USE_RESERVE|
-				BTREE_INSERT_JOURNAL_RESERVED;
+				JOURNAL_WATERMARK_COPYGC;
 			data_opts->rewrite_dev		= p.ptr.dev;
 
 			if (p.has_ec)
diff --git a/fs/bcachefs/recovery.c b/fs/bcachefs/recovery.c
index fe2c5cb6d430..e60af9b481d2 100644
--- a/fs/bcachefs/recovery.c
+++ b/fs/bcachefs/recovery.c
@@ -563,8 +563,9 @@ static int bch2_journal_replay(struct bch_fs *c)
 		ret = bch2_trans_do(c, NULL, NULL,
 				    BTREE_INSERT_LAZY_RW|
 				    BTREE_INSERT_NOFAIL|
-				    BTREE_INSERT_JOURNAL_RESERVED|
-				    (!k->allocated ? BTREE_INSERT_JOURNAL_REPLAY : 0),
+				    (!k->allocated
+				     ? BTREE_INSERT_JOURNAL_REPLAY|JOURNAL_WATERMARK_RESERVED
+				     : 0),
 			     bch2_journal_replay_key(&trans, k));
 		if (ret) {
 			bch_err(c, "journal replay: error %d while replaying key at btree %s level %u",