bcachefs: fix BTREE_INSERT_NOUNLOCK

btree_node_merge -> get_sibling -> btree_node_lock could drop locks,
when using linked iterators

7 files changed, 70 insertions, 30 deletions

diff --git a/fs/bcachefs/btree_cache.c b/fs/bcachefs/btree_cache.c
index e347368e7afc..f15a415e37e3 100644
--- a/fs/bcachefs/btree_cache.c
+++ b/fs/bcachefs/btree_cache.c
@@ -651,7 +651,8 @@ static noinline struct btree *bch2_btree_node_fill(struct bch_fs *c,
  */
 struct btree *bch2_btree_node_get(struct bch_fs *c, struct btree_iter *iter,
 				  const struct bkey_i *k, unsigned level,
-				  enum six_lock_type lock_type)
+				  enum six_lock_type lock_type,
+				  bool may_drop_locks)
 {
 	struct btree_cache *bc = &c->btree_cache;
 	struct btree *b;
@@ -718,7 +719,8 @@ retry:
 		if (btree_node_read_locked(iter, level + 1))
 			btree_node_unlock(iter, level + 1);
 
-		if (!btree_node_lock(b, k->k.p, level, iter, lock_type))
+		if (!btree_node_lock(b, k->k.p, level, iter,
+				     lock_type, may_drop_locks))
 			return ERR_PTR(-EINTR);
 
 		if (unlikely(PTR_HASH(&b->key) != PTR_HASH(k) ||
@@ -795,7 +797,8 @@ struct btree *bch2_btree_node_get_sibling(struct bch_fs *c,
 
 	bch2_bkey_unpack(parent, &tmp.k, k);
 
-	ret = bch2_btree_node_get(c, iter, &tmp.k, level, SIX_LOCK_intent);
+	ret = bch2_btree_node_get(c, iter, &tmp.k, level,
+				  SIX_LOCK_intent, may_drop_locks);
 
 	if (PTR_ERR_OR_ZERO(ret) == -EINTR && may_drop_locks) {
 		struct btree_iter *linked;
@@ -815,7 +818,7 @@ struct btree *bch2_btree_node_get_sibling(struct bch_fs *c,
 			btree_node_unlock(iter, level);
 
 		ret = bch2_btree_node_get(c, iter, &tmp.k, level,
-					  SIX_LOCK_intent);
+					  SIX_LOCK_intent, may_drop_locks);
 
 		/*
 		 * before btree_iter_relock() calls btree_iter_verify_locks():
@@ -858,7 +861,7 @@ out:
 	return ret;
 out_upgrade:
 	if (may_drop_locks)
-		bch2_btree_iter_upgrade(iter, level + 2);
+		bch2_btree_iter_upgrade(iter, level + 2, true);
 	ret = ERR_PTR(-EINTR);
 	goto out;
 }
diff --git a/fs/bcachefs/btree_cache.h b/fs/bcachefs/btree_cache.h
index a89183996a4f..96d134f4d0fc 100644
--- a/fs/bcachefs/btree_cache.h
+++ b/fs/bcachefs/btree_cache.h
@@ -23,7 +23,7 @@ struct btree *bch2_btree_node_mem_alloc(struct bch_fs *);
 
 struct btree *bch2_btree_node_get(struct bch_fs *, struct btree_iter *,
 				  const struct bkey_i *, unsigned,
-				  enum six_lock_type);
+				  enum six_lock_type, bool);
 
 struct btree *bch2_btree_node_get_sibling(struct bch_fs *, struct btree_iter *,
 					  struct btree *, bool,
diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c
index acdf1317ac62..097b68e07399 100644
--- a/fs/bcachefs/btree_iter.c
+++ b/fs/bcachefs/btree_iter.c
@@ -117,12 +117,12 @@ static bool bch2_btree_node_upgrade(struct btree_iter *iter, unsigned level)
 	if (!is_btree_node(iter, level))
 		return false;
 
-	if (race_fault())
-		return false;
-
 	if (btree_node_intent_locked(iter, level))
 		return true;
 
+	if (race_fault())
+		return false;
+
 	if (btree_node_locked(iter, level)
 	    ? six_lock_tryupgrade(&b->lock)
 	    : six_relock_type(&b->lock, SIX_LOCK_intent, iter->lock_seq[level]))
@@ -182,7 +182,8 @@ static inline bool btree_iter_get_locks(struct btree_iter *iter,
 bool __bch2_btree_node_lock(struct btree *b, struct bpos pos,
 			   unsigned level,
 			   struct btree_iter *iter,
-			   enum six_lock_type type)
+			   enum six_lock_type type,
+			   bool may_drop_locks)
 {
 	struct bch_fs *c = iter->c;
 	struct btree_iter *linked;
@@ -233,10 +234,12 @@ bool __bch2_btree_node_lock(struct btree *b, struct bpos pos,
 		 */
 		if (type == SIX_LOCK_intent &&
 		    linked->nodes_locked != linked->nodes_intent_locked) {
-			linked->locks_want = max_t(unsigned,
-					linked->locks_want,
-					__fls(linked->nodes_locked) + 1);
-			btree_iter_get_locks(linked, true);
+			if (may_drop_locks) {
+				linked->locks_want = max_t(unsigned,
+						linked->locks_want,
+						__fls(linked->nodes_locked) + 1);
+				btree_iter_get_locks(linked, true);
+			}
 			ret = false;
 		}
 
@@ -247,10 +250,12 @@ bool __bch2_btree_node_lock(struct btree *b, struct bpos pos,
 		 */
 		if (linked->btree_id == iter->btree_id &&
 		    level > __fls(linked->nodes_locked)) {
-			linked->locks_want = max_t(unsigned,
-						   linked->locks_want,
-						   iter->locks_want);
-			btree_iter_get_locks(linked, true);
+			if (may_drop_locks) {
+				linked->locks_want = max_t(unsigned,
+							   linked->locks_want,
+							   iter->locks_want);
+				btree_iter_get_locks(linked, true);
+			}
 			ret = false;
 		}
 	}
@@ -325,6 +330,30 @@ bool __bch2_btree_iter_upgrade(struct btree_iter *iter,
 	return false;
 }
 
+bool __bch2_btree_iter_upgrade_nounlock(struct btree_iter *iter,
+					unsigned new_locks_want)
+{
+	unsigned l = iter->level;
+
+	EBUG_ON(iter->locks_want >= new_locks_want);
+
+	iter->locks_want = new_locks_want;
+
+	do {
+		if (!btree_iter_node(iter, l))
+			break;
+
+		if (!bch2_btree_node_upgrade(iter, l)) {
+			iter->locks_want = l;
+			return false;
+		}
+
+		l++;
+	} while (l < iter->locks_want);
+
+	return true;
+}
+
 void __bch2_btree_iter_downgrade(struct btree_iter *iter,
 				 unsigned downgrade_to)
 {
@@ -792,7 +821,7 @@ static inline int btree_iter_lock_root(struct btree_iter *iter,
 
 		lock_type = __btree_lock_want(iter, iter->level);
 		if (unlikely(!btree_node_lock(b, POS_MAX, iter->level,
-					      iter, lock_type)))
+					      iter, lock_type, true)))
 			return -EINTR;
 
 		if (likely(b == c->btree_roots[iter->btree_id].b &&
@@ -855,7 +884,7 @@ static inline int btree_iter_down(struct btree_iter *iter)
 	bch2_bkey_unpack(l->b, &tmp.k,
 			 bch2_btree_node_iter_peek(&l->iter, l->b));
 
-	b = bch2_btree_node_get(iter->c, iter, &tmp.k, level, lock_type);
+	b = bch2_btree_node_get(iter->c, iter, &tmp.k, level, lock_type, true);
 	if (unlikely(IS_ERR(b)))
 		return PTR_ERR(b);
 
diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h
index f82210d4d468..5db1cc581f56 100644
--- a/fs/bcachefs/btree_iter.h
+++ b/fs/bcachefs/btree_iter.h
@@ -106,14 +106,18 @@ void bch2_btree_node_iter_fix(struct btree_iter *, struct btree *,
 int bch2_btree_iter_unlock(struct btree_iter *);
 
 bool __bch2_btree_iter_upgrade(struct btree_iter *, unsigned);
+bool __bch2_btree_iter_upgrade_nounlock(struct btree_iter *, unsigned);
 
 static inline bool bch2_btree_iter_upgrade(struct btree_iter *iter,
-					   unsigned new_locks_want)
+					   unsigned new_locks_want,
+					   bool may_drop_locks)
 {
 	new_locks_want = min(new_locks_want, BTREE_MAX_DEPTH);
 
 	return iter->locks_want < new_locks_want
-		?  __bch2_btree_iter_upgrade(iter, new_locks_want)
+		? (may_drop_locks
+		   ? __bch2_btree_iter_upgrade(iter, new_locks_want)
+		   : __bch2_btree_iter_upgrade_nounlock(iter, new_locks_want))
 		: iter->uptodate <= BTREE_ITER_NEED_PEEK;
 }
 
diff --git a/fs/bcachefs/btree_locking.h b/fs/bcachefs/btree_locking.h
index 1d975207a163..419d0e815a25 100644
--- a/fs/bcachefs/btree_locking.h
+++ b/fs/bcachefs/btree_locking.h
@@ -147,17 +147,19 @@ static inline void btree_node_lock_type(struct bch_fs *c, struct btree *b,
 }
 
 bool __bch2_btree_node_lock(struct btree *, struct bpos, unsigned,
-			   struct btree_iter *, enum six_lock_type);
+			    struct btree_iter *, enum six_lock_type, bool);
 
 static inline bool btree_node_lock(struct btree *b, struct bpos pos,
 				   unsigned level,
 				   struct btree_iter *iter,
-				   enum six_lock_type type)
+				   enum six_lock_type type,
+				   bool may_drop_locks)
 {
 	EBUG_ON(level >= BTREE_MAX_DEPTH);
 
 	return likely(six_trylock_type(&b->lock, type)) ||
-		__bch2_btree_node_lock(b, pos, level, iter, type);
+		__bch2_btree_node_lock(b, pos, level, iter,
+				       type, may_drop_locks);
 }
 
 bool __bch2_btree_node_relock(struct btree_iter *, unsigned);
diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c
index 3e13f78476a2..392ee0a06592 100644
--- a/fs/bcachefs/btree_update_interior.c
+++ b/fs/bcachefs/btree_update_interior.c
@@ -1586,7 +1586,8 @@ int bch2_btree_split_leaf(struct bch_fs *c, struct btree_iter *iter,
 	 * XXX: figure out how far we might need to split,
 	 * instead of locking/reserving all the way to the root:
 	 */
-	if (!bch2_btree_iter_upgrade(iter, U8_MAX)) {
+	if (!bch2_btree_iter_upgrade(iter, U8_MAX,
+			!(flags & BTREE_INSERT_NOUNLOCK))) {
 		ret = -EINTR;
 		goto out;
 	}
@@ -1694,7 +1695,8 @@ retry:
 	if (!down_read_trylock(&c->gc_lock))
 		goto err_cycle_gc_lock;
 
-	if (!bch2_btree_iter_upgrade(iter, U8_MAX)) {
+	if (!bch2_btree_iter_upgrade(iter, U8_MAX,
+			!(flags & BTREE_INSERT_NOUNLOCK))) {
 		ret = -EINTR;
 		goto err_unlock;
 	}
@@ -1857,7 +1859,7 @@ int bch2_btree_node_rewrite(struct bch_fs *c, struct btree_iter *iter,
 
 	closure_init_stack(&cl);
 
-	bch2_btree_iter_upgrade(iter, U8_MAX);
+	bch2_btree_iter_upgrade(iter, U8_MAX, true);
 
 	if (!(flags & BTREE_INSERT_GC_LOCK_HELD)) {
 		if (!down_read_trylock(&c->gc_lock)) {
@@ -2000,7 +2002,7 @@ int bch2_btree_node_update_key(struct bch_fs *c, struct btree_iter *iter,
 
 	closure_init_stack(&cl);
 
-	if (!bch2_btree_iter_upgrade(iter, U8_MAX))
+	if (!bch2_btree_iter_upgrade(iter, U8_MAX, true))
 		return -EINTR;
 
 	if (!down_read_trylock(&c->gc_lock)) {
diff --git a/fs/bcachefs/btree_update_leaf.c b/fs/bcachefs/btree_update_leaf.c
index ad3242efb42e..588a1997e5ee 100644
--- a/fs/bcachefs/btree_update_leaf.c
+++ b/fs/bcachefs/btree_update_leaf.c
@@ -442,7 +442,7 @@ retry:
 	cycle_gc_lock = false;
 
 	trans_for_each_entry(trans, i) {
-		if (!bch2_btree_iter_upgrade(i->iter, 1)) {
+		if (!bch2_btree_iter_upgrade(i->iter, 1, true)) {
 			ret = -EINTR;
 			goto err;
 		}