bcache: uninline bkey_cmp_left_packed()

this is generally a slowpath

7 files changed, 185 insertions, 85 deletions

diff --git a/drivers/md/bcache/bkey.c b/drivers/md/bcache/bkey.c
index 760f52e5e4dd..dfbfd767ae02 100644
--- a/drivers/md/bcache/bkey.c
+++ b/drivers/md/bcache/bkey.c
@@ -471,7 +471,7 @@ static bool bkey_packed_successor(struct bkey_packed *out,
 
 		if ((*p & mask) != mask) {
 			*p += 1ULL << offset;
-			EBUG_ON(__bkey_cmp_packed(out, &k, b) <= 0);
+			EBUG_ON(bkey_cmp_packed(b, out, &k) <= 0);
 			return true;
 		}
 
@@ -549,13 +549,13 @@ enum bkey_pack_pos_ret bkey_pack_pos_lossy(struct bkey_packed *out,
 
 #ifdef CONFIG_BCACHE_DEBUG
 	if (exact) {
-		BUG_ON(bkey_cmp_left_packed(b, out, orig));
+		BUG_ON(bkey_cmp_left_packed(b, out, &orig));
 	} else {
 		struct bkey_packed successor;
 
-		BUG_ON(bkey_cmp_left_packed(b, out, orig) >= 0);
+		BUG_ON(bkey_cmp_left_packed(b, out, &orig) >= 0);
 		BUG_ON(bkey_packed_successor(&successor, b, *out) &&
-		       bkey_cmp_left_packed(b, &successor, orig) < 0);
+		       bkey_cmp_left_packed(b, &successor, &orig) < 0);
 	}
 #endif
 
@@ -689,6 +689,7 @@ const char *bch_bkey_format_validate(struct bkey_format *f)
  * Most significant differing bit
  * Bits are indexed from 0 - return is [0, nr_key_bits)
  */
+__pure
 unsigned bkey_greatest_differing_bit(const struct btree_keys *b,
 				     const struct bkey_packed *l_k,
 				     const struct bkey_packed *r_k)
@@ -732,6 +733,7 @@ unsigned bkey_greatest_differing_bit(const struct btree_keys *b,
  * First set bit
  * Bits are indexed from 0 - return is [0, nr_key_bits)
  */
+__pure
 unsigned bkey_ffs(const struct btree_keys *b,
 		  const struct bkey_packed *k)
 {
@@ -1113,9 +1115,10 @@ int bkey_cmp(const struct bkey *l, const struct bkey *r)
 }
 #endif
 
-int __bkey_cmp_packed(const struct bkey_packed *l,
-		      const struct bkey_packed *r,
-		      const struct btree_keys *b)
+__pure
+int __bkey_cmp_packed_format_checked(const struct bkey_packed *l,
+				     const struct bkey_packed *r,
+				     const struct btree_keys *b)
 {
 	const struct bkey_format *f = &b->format;
 	int ret;
@@ -1127,22 +1130,61 @@ int __bkey_cmp_packed(const struct bkey_packed *l,
 			      high_word(f, r),
 			      b->nr_key_bits);
 
-	EBUG_ON(ret != bkey_cmp(bkey_unpack_key(b, l).p,
-				bkey_unpack_key(b, r).p));
+	EBUG_ON(ret != bkey_cmp(bkey_unpack_key_format_checked(b, l).p,
+				bkey_unpack_key_format_checked(b, r).p));
 	return ret;
 }
 
-__flatten
-int __bkey_cmp_left_packed(const struct btree_keys *b,
-			   const struct bkey_packed *l, struct bpos r)
+__pure __flatten
+int __bkey_cmp_left_packed_format_checked(const struct btree_keys *b,
+					  const struct bkey_packed *l,
+					  const struct bpos *r)
 {
 #ifdef HAVE_BCACHE_COMPILED_UNPACK
-	return bkey_cmp(bkey_unpack_key(b, l).p, r);
+	return bkey_cmp(bkey_unpack_key_format_checked(b, l).p, *r);
 #else
-	return bkey_cmp(__bkey_unpack_pos(&b->format, l), r);
+	return bkey_cmp(__bkey_unpack_pos(&b->format, l), *r);
 #endif
 }
 
+__pure __flatten
+int __bkey_cmp_packed(const struct bkey_packed *l,
+		      const struct bkey_packed *r,
+		      const struct btree_keys *b)
+{
+	int packed = bkey_lr_packed(l, r);
+
+	if (likely(packed == BKEY_PACKED_BOTH))
+		return __bkey_cmp_packed_format_checked(l, r, b);
+
+	switch (packed) {
+	case BKEY_PACKED_NONE:
+		return bkey_cmp(((struct bkey *) l)->p,
+				((struct bkey *) r)->p);
+	case BKEY_PACKED_LEFT:
+		return __bkey_cmp_left_packed_format_checked(b,
+				  (struct bkey_packed *) l,
+				  &((struct bkey *) r)->p);
+	case BKEY_PACKED_RIGHT:
+		return -__bkey_cmp_left_packed_format_checked(b,
+				  (struct bkey_packed *) r,
+				  &((struct bkey *) l)->p);
+	default:
+		unreachable();
+	}
+}
+
+__pure __flatten
+int bkey_cmp_left_packed(const struct btree_keys *b,
+			 const struct bkey_packed *l, const struct bpos *r)
+{
+	const struct bkey *l_unpacked;
+
+	return unlikely(l_unpacked = packed_to_bkey_c(l))
+		? bkey_cmp(l_unpacked->p, *r)
+		: __bkey_cmp_left_packed_format_checked(b, l, r);
+}
+
 void bch_bpos_swab(struct bpos *p)
 {
 	u8 *l = (u8 *) p;
diff --git a/drivers/md/bcache/bkey.h b/drivers/md/bcache/bkey.h
index 1a47c0417a3c..57e5b8d3fb74 100644
--- a/drivers/md/bcache/bkey.h
+++ b/drivers/md/bcache/bkey.h
@@ -76,6 +76,26 @@ static inline void set_bkey_deleted(struct bkey *k)
 #define bkey_whiteout(_k)				\
 	((_k)->type == KEY_TYPE_DELETED || (_k)->type == KEY_TYPE_DISCARD)
 
+#define bkey_packed_typecheck(_k)					\
+({									\
+	BUILD_BUG_ON(!type_is(_k, struct bkey *) &&			\
+		     !type_is(_k, struct bkey_packed *));		\
+	type_is(_k, struct bkey_packed *);				\
+})
+
+enum bkey_lr_packed {
+	BKEY_PACKED_BOTH,
+	BKEY_PACKED_RIGHT,
+	BKEY_PACKED_LEFT,
+	BKEY_PACKED_NONE,
+};
+
+#define bkey_lr_packed_typecheck(_l, _r)				\
+	(!bkey_packed_typecheck(_l) + ((!bkey_packed_typecheck(_r)) << 1))
+
+#define bkey_lr_packed(_l, _r)						\
+	((_l)->format + ((_r)->format << 1))
+
 #define bkey_copy(_dst, _src)					\
 do {								\
 	BUILD_BUG_ON(!type_is(_dst, struct bkey_i *) &&		\
@@ -103,28 +123,76 @@ void bch_bkey_format_add_pos(struct bkey_format_state *, struct bpos);
 struct bkey_format bch_bkey_format_done(struct bkey_format_state *);
 const char *bch_bkey_format_validate(struct bkey_format *);
 
+__pure
 unsigned bkey_greatest_differing_bit(const struct btree_keys *,
 				     const struct bkey_packed *,
 				     const struct bkey_packed *);
+__pure
 unsigned bkey_ffs(const struct btree_keys *, const struct bkey_packed *);
 
-int __bkey_cmp_left_packed(const struct btree_keys *,
-			   const struct bkey_packed *,
-			   struct bpos);
+__pure
+int __bkey_cmp_packed_format_checked(const struct bkey_packed *,
+				     const struct bkey_packed *,
+				     const struct btree_keys *);
 
-#define bkey_cmp_left_packed(_format, _l, _r)			\
-({								\
-	const struct bkey *_l_unpacked;				\
-								\
-	unlikely(_l_unpacked = packed_to_bkey_c(_l))		\
-		? bkey_cmp(_l_unpacked->p, _r)			\
-		: __bkey_cmp_left_packed(_format, _l, _r);	\
-})
+__pure
+int __bkey_cmp_left_packed_format_checked(const struct btree_keys *,
+					  const struct bkey_packed *,
+					  const struct bpos *);
 
+__pure
 int __bkey_cmp_packed(const struct bkey_packed *,
 		      const struct bkey_packed *,
 		      const struct btree_keys *);
 
+__pure
+int bkey_cmp_left_packed(const struct btree_keys *,
+			 const struct bkey_packed *,
+			 const struct bpos *);
+
+/*
+ * we prefer to pass bpos by ref, but it's often enough terribly convenient to
+ * pass it by by val... as much as I hate c++, const ref would be nice here:
+ */
+__pure __flatten
+static inline int bkey_cmp_left_packed_byval(const struct btree_keys *b,
+					     const struct bkey_packed *l,
+					     struct bpos r)
+{
+	return bkey_cmp_left_packed(b, l, &r);
+}
+
+/*
+ * If @_l or @_r are struct bkey * (not bkey_packed *), uses type information to
+ * skip dispatching on k->format:
+ */
+#define bkey_cmp_packed(_b, _l, _r)					\
+({									\
+	int _cmp;							\
+									\
+	switch (bkey_lr_packed_typecheck(_l, _r)) {			\
+	case BKEY_PACKED_NONE:						\
+		_cmp = bkey_cmp(((struct bkey *) (_l))->p,		\
+				((struct bkey *) (_r))->p);		\
+		break;							\
+	case BKEY_PACKED_LEFT:						\
+		_cmp = bkey_cmp_left_packed((_b),			\
+				  (struct bkey_packed *) (_l),		\
+				  &((struct bkey *) (_r))->p);		\
+		break;							\
+	case BKEY_PACKED_RIGHT:						\
+		_cmp = -bkey_cmp_left_packed((_b),			\
+				  (struct bkey_packed *) (_r),		\
+				  &((struct bkey *) (_l))->p);		\
+		break;							\
+	case BKEY_PACKED_BOTH:						\
+		_cmp = __bkey_cmp_packed((void *) (_l),			\
+					 (void *) (_r), (_b));		\
+		break;							\
+	}								\
+	_cmp;								\
+})
+
 #if 1
 static __always_inline int bkey_cmp(struct bpos l, struct bpos r)
 {
@@ -187,31 +255,6 @@ static inline unsigned bkey_format_key_bits(const struct bkey_format *format)
 		format->bits_per_field[BKEY_FIELD_SNAPSHOT];
 }
 
-#define bkey_packed_typecheck(_k)					\
-({									\
-	BUILD_BUG_ON(!type_is(_k, struct bkey *) &&			\
-		     !type_is(_k, struct bkey_packed *));		\
-	type_is(_k, struct bkey_packed *) && bkey_packed(_k);		\
-})
-
-/*
- * If @_l and @_r are in the same format, does the comparison without unpacking.
- * Otherwise, unpacks whichever one is packed.
- */
-#define bkey_cmp_packed(_b, _l, _r)					\
-	((bkey_packed_typecheck(_l) && bkey_packed_typecheck(_r))	\
-	 ? __bkey_cmp_packed((void *) (_l), (void *) (_r), (_b))	\
-	 : bkey_packed_typecheck(_l)					\
-	 ? __bkey_cmp_left_packed(_b,					\
-				  (struct bkey_packed *) (_l),		\
-				  ((struct bkey *) (_r))->p)		\
-	 : bkey_packed_typecheck(_r)					\
-	 ? -__bkey_cmp_left_packed((_b),				\
-				   (struct bkey_packed *) (_r),		\
-				   ((struct bkey *) (_l))->p)		\
-	 : bkey_cmp(((struct bkey *) (_l))->p,				\
-		    ((struct bkey *) (_r))->p))
-
 static inline struct bpos bkey_successor(struct bpos p)
 {
 	struct bpos ret = p;
diff --git a/drivers/md/bcache/bset.c b/drivers/md/bcache/bset.c
index 9927273d89da..04e3151933ba 100644
--- a/drivers/md/bcache/bset.c
+++ b/drivers/md/bcache/bset.c
@@ -127,7 +127,7 @@ static bool keys_out_of_order(struct btree_keys *b,
 {
 	struct bkey nextu = bkey_unpack_key(b, next);
 
-	return bkey_cmp_left_packed(b, prev, bkey_start_pos(&nextu)) > 0 ||
+	return bkey_cmp_left_packed_byval(b, prev, bkey_start_pos(&nextu)) > 0 ||
 		((is_extents
 		  ? !bkey_deleted(next)
 		  : !bkey_deleted(prev)) &&
@@ -242,7 +242,7 @@ void bch_verify_key_order(struct btree_keys *b,
 		if (k == bset_bkey_last(t->data))
 			k = bkey_prev_all(t, k);
 
-		while (bkey_cmp_left_packed(b, k, bkey_start_pos(&uw)) > 0 &&
+		while (bkey_cmp_left_packed_byval(b, k, bkey_start_pos(&uw)) > 0 &&
 		       (prev = bkey_prev_all(t, k)))
 			k = prev;
 
@@ -1192,7 +1192,7 @@ static struct bkey_packed *bset_search_write_set(const struct btree_keys *b,
 		unsigned m = (li + ri) >> 1;
 
 		if (bkey_cmp_p_or_unp(b, table_to_bkey(t, m),
-				      packed_search, search) >= 0)
+				      packed_search, &search) >= 0)
 			ri = m;
 		else
 			li = m;
@@ -1201,6 +1201,16 @@ static struct bkey_packed *bset_search_write_set(const struct btree_keys *b,
 	return table_to_bkey(t, li);
 }
 
+noinline
+static int bset_search_tree_slowpath(const struct btree_keys *b,
+				struct bset_tree *t, struct bpos *search,
+				const struct bkey_packed *packed_search,
+				unsigned n)
+{
+	return bkey_cmp_p_or_unp(b, tree_to_bkey(t, n),
+				 packed_search, search) < 0;
+}
+
 __flatten
 static struct bkey_packed *bset_search_tree(const struct btree_keys *b,
 				struct bset_tree *t,
@@ -1243,9 +1253,8 @@ static struct bkey_packed *bset_search_tree(const struct btree_keys *b,
 			n = n * 2 + (f->mantissa <
 				     bfloat_mantissa(packed_search, f));
 		else
-			n = n * 2 + (bkey_cmp_p_or_unp(b, tree_to_bkey(t, n),
-						       packed_search,
-						       search) < 0);
+			n = n * 2 + bset_search_tree_slowpath(b, t,
+						&search, packed_search, n);
 	} while (n < t->size);
 
 	inorder = to_inorder(n >> 1, t);
@@ -1309,11 +1318,11 @@ static struct bkey_packed *bch_bset_search(struct btree_keys *b,
 		 */
 
 		if (unlikely(bkey_cmp_p_or_unp(b, &t->end,
-					       packed_search, search) < 0))
+					       packed_search, &search) < 0))
 			return bset_bkey_last(t->data);
 
 		if (unlikely(bkey_cmp_p_or_unp(b, t->data->start,
-					       packed_search, search) >= 0))
+					       packed_search, &search) >= 0))
 			m = t->data->start;
 		else
 			m = bset_search_tree(b, t, search, lossy_packed_search);
@@ -1328,7 +1337,7 @@ static struct bkey_packed *bch_bset_search(struct btree_keys *b,
 
 	if (!packed_search)
 		while (m != bset_bkey_last(t->data) &&
-		       !btree_iter_pos_cmp_packed(b, search, m, strictly_greater))
+		       !btree_iter_pos_cmp_packed(b, &search, m, strictly_greater))
 			m = bkey_next(m);
 
 	if (IS_ENABLED(CONFIG_BCACHE_DEBUG)) {
diff --git a/drivers/md/bcache/bset.h b/drivers/md/bcache/bset.h
index 9375c0fb3bae..a73dedbc740b 100644
--- a/drivers/md/bcache/bset.h
+++ b/drivers/md/bcache/bset.h
@@ -261,17 +261,12 @@ static inline void btree_node_set_format(struct btree_keys *b,
 	BUG_ON(len < 0 || len > 200);
 }
 
-/**
- * bkey_unpack_key -- unpack just the key, not the value
- */
-static inline struct bkey bkey_unpack_key(const struct btree_keys *b,
-					  const struct bkey_packed *src)
+static inline struct bkey
+bkey_unpack_key_format_checked(const struct btree_keys *b,
+			       const struct bkey_packed *src)
 {
 	struct bkey dst;
 
-	if (unlikely(!bkey_packed(src)))
-		return *packed_to_bkey_c(src);
-
 #ifdef HAVE_BCACHE_COMPILED_UNPACK
 	b->unpack_fn(&dst, src);
 
@@ -286,6 +281,17 @@ static inline struct bkey bkey_unpack_key(const struct btree_keys *b,
 	return dst;
 }
 
+/**
+ * bkey_unpack_key -- unpack just the key, not the value
+ */
+static inline struct bkey bkey_unpack_key(const struct btree_keys *b,
+					  const struct bkey_packed *src)
+{
+	return likely(bkey_packed(src))
+		? bkey_unpack_key_format_checked(b, src)
+		: *packed_to_bkey_c(src);
+}
+
 /* Disassembled bkeys */
 
 static inline struct bkey_s_c bkey_disassemble(struct btree_keys *b,
@@ -388,17 +394,17 @@ void bch_bset_delete(struct btree_keys *, struct bkey_packed *, unsigned);
 static inline int bkey_cmp_p_or_unp(const struct btree_keys *b,
 				    const struct bkey_packed *l,
 				    const struct bkey_packed *r_packed,
-				    struct bpos r)
+				    struct bpos *r)
 {
 	EBUG_ON(r_packed && !bkey_packed(r_packed));
 
 	if (unlikely(!bkey_packed(l)))
-		return bkey_cmp(packed_to_bkey_c(l)->p, r);
+		return bkey_cmp(packed_to_bkey_c(l)->p, *r);
 
 	if (likely(r_packed))
-		return __bkey_cmp_packed(l, r_packed, b);
+		return __bkey_cmp_packed_format_checked(l, r_packed, b);
 
-	return __bkey_cmp_left_packed(b, l, r);
+	return __bkey_cmp_left_packed_format_checked(b, l, r);
 }
 
 /* Returns true if @k is after iterator position @pos */
@@ -412,7 +418,7 @@ static inline bool btree_iter_pos_cmp(struct bpos pos, const struct bkey *k,
 }
 
 static inline bool btree_iter_pos_cmp_packed(const struct btree_keys *b,
-					     struct bpos pos,
+					     struct bpos *pos,
 					     const struct bkey_packed *k,
 					     bool strictly_greater)
 {
@@ -428,7 +434,7 @@ static inline bool btree_iter_pos_cmp_p_or_unp(const struct btree_keys *b,
 					const struct bkey_packed *k,
 					bool strictly_greater)
 {
-	int cmp = bkey_cmp_p_or_unp(b, k, pos_packed, pos);
+	int cmp = bkey_cmp_p_or_unp(b, k, pos_packed, &pos);
 
 	return cmp > 0 ||
 		(cmp == 0 && !strictly_greater && !bkey_deleted(k));
diff --git a/drivers/md/bcache/btree_io.c b/drivers/md/bcache/btree_io.c
index 4c132f452774..97c7e1d325f2 100644
--- a/drivers/md/bcache/btree_io.c
+++ b/drivers/md/bcache/btree_io.c
@@ -958,7 +958,7 @@ static const char *validate_bset(struct cache_set *c, struct btree *b,
 
 		if (!seen_non_whiteout &&
 		    (!bkey_whiteout(k) ||
-		     (prev && bkey_cmp_left_packed(&b->keys, prev,
+		     (prev && bkey_cmp_left_packed_byval(&b->keys, prev,
 					bkey_start_pos(u.k)) > 0))) {
 			*whiteout_u64s = k->_data - i->_data;
 			seen_non_whiteout = true;
diff --git a/drivers/md/bcache/btree_iter.c b/drivers/md/bcache/btree_iter.c
index 02b456169d43..ef4677b32265 100644
--- a/drivers/md/bcache/btree_iter.c
+++ b/drivers/md/bcache/btree_iter.c
@@ -300,7 +300,7 @@ static void __bch_btree_iter_verify(struct btree_iter *iter,
 	k = b->level
 		? bch_btree_node_iter_prev(&tmp, &b->keys)
 		: bch_btree_node_iter_prev_all(&tmp, &b->keys);
-	if (k && btree_iter_pos_cmp_packed(&b->keys, iter->pos, k,
+	if (k && btree_iter_pos_cmp_packed(&b->keys, &iter->pos, k,
 					   iter->is_extents)) {
 		char buf[100];
 		struct bkey uk = bkey_unpack_key(&b->keys, k);
@@ -311,7 +311,7 @@ static void __bch_btree_iter_verify(struct btree_iter *iter,
 	}
 
 	k = bch_btree_node_iter_peek_all(node_iter, &b->keys);
-	if (k && !btree_iter_pos_cmp_packed(&b->keys, iter->pos, k,
+	if (k && !btree_iter_pos_cmp_packed(&b->keys, &iter->pos, k,
 					    iter->is_extents)) {
 		char buf[100];
 		struct bkey uk = bkey_unpack_key(&b->keys, k);
@@ -355,7 +355,7 @@ static void __bch_btree_node_iter_fix(struct btree_iter *iter,
 
 	/* didn't find the bset in the iterator - might have to readd it: */
 	if (new_u64s &&
-	    btree_iter_pos_cmp_packed(&b->keys, iter->pos, where,
+	    btree_iter_pos_cmp_packed(&b->keys, &iter->pos, where,
 				      iter->is_extents))
 		bch_btree_node_iter_push(node_iter, &b->keys, where, end);
 	return;
@@ -367,7 +367,7 @@ found:
 		return;
 
 	if (new_u64s &&
-	    btree_iter_pos_cmp_packed(&b->keys, iter->pos, where,
+	    btree_iter_pos_cmp_packed(&b->keys, &iter->pos, where,
 				      iter->is_extents)) {
 		set->k = offset;
 		bch_btree_node_iter_sort(node_iter, &b->keys);
@@ -403,7 +403,7 @@ found:
 	 * to.
 	 */
 	if (b->level && new_u64s && !bkey_deleted(where) &&
-	    btree_iter_pos_cmp_packed(&b->keys, iter->pos, where,
+	    btree_iter_pos_cmp_packed(&b->keys, &iter->pos, where,
 				      iter->is_extents)) {
 		struct bset_tree *t;
 		struct bkey_packed *k;
@@ -536,7 +536,7 @@ static void btree_iter_verify_new_node(struct btree_iter *iter, struct btree *b)
 	if (!k ||
 	    bkey_deleted(k) ||
 	    bkey_cmp_left_packed(&iter->nodes[b->level + 1]->keys,
-				 k, b->key.k.p)) {
+				 k, &b->key.k.p)) {
 		char buf[100];
 		struct bkey uk = bkey_unpack_key(&b->keys, k);
 
@@ -974,7 +974,7 @@ void bch_btree_iter_set_pos_same_leaf(struct btree_iter *iter, struct bpos new_p
 	EBUG_ON(bkey_cmp(new_pos, iter->nodes[0]->key.k.p) > 0);
 
 	while ((k = bch_btree_node_iter_peek_all(node_iter, b)) &&
-	       !btree_iter_pos_cmp_packed(b, new_pos, k,
+	       !btree_iter_pos_cmp_packed(b, &new_pos, k,
 					  iter->is_extents))
 		bch_btree_node_iter_advance(node_iter, b);
 
diff --git a/drivers/md/bcache/btree_update.c b/drivers/md/bcache/btree_update.c
index 6a116051f678..c53e938515fb 100644
--- a/drivers/md/bcache/btree_update.c
+++ b/drivers/md/bcache/btree_update.c
@@ -636,7 +636,7 @@ static void bch_insert_fixup_btree_ptr(struct btree_iter *iter,
 			     gc_pos_btree_node(b), &stats);
 
 	while ((k = bch_btree_node_iter_peek_all(node_iter, &b->keys)) &&
-	       !btree_iter_pos_cmp_packed(&b->keys, insert->k.p, k, false))
+	       !btree_iter_pos_cmp_packed(&b->keys, &insert->k.p, k, false))
 		bch_btree_node_iter_advance(node_iter, &b->keys);
 
 	/*
@@ -1153,7 +1153,7 @@ static void btree_node_interior_verify(struct btree *b)
 	bch_btree_node_iter_init(&iter, &b->keys, b->key.k.p, false, false);
 #if 1
 	BUG_ON(!(k = bch_btree_node_iter_peek(&iter, &b->keys)) ||
-	       bkey_cmp_left_packed(&b->keys, k, b->key.k.p));
+	       bkey_cmp_left_packed(&b->keys, k, &b->key.k.p));
 
 	BUG_ON((bch_btree_node_iter_advance(&iter, &b->keys),
 		!bch_btree_node_iter_end(&iter)));
@@ -1166,7 +1166,7 @@ static void btree_node_interior_verify(struct btree *b)
 		goto err;
 
 	msg = "isn't what it should be";
-	if (bkey_cmp_left_packed(&b->keys, k, b->key.k.p))
+	if (bkey_cmp_left_packed(&b->keys, k, &b->key.k.p))
 		goto err;
 
 	bch_btree_node_iter_advance(&iter, &b->keys);
@@ -1605,7 +1605,7 @@ static struct btree *btree_node_get_sibling(struct btree_iter *iter,
 	node_iter = iter->node_iters[parent->level];
 
 	k = bch_btree_node_iter_peek_all(&node_iter, &parent->keys);
-	BUG_ON(bkey_cmp_left_packed(&parent->keys, k, b->key.k.p));
+	BUG_ON(bkey_cmp_left_packed(&parent->keys, k, &b->key.k.p));
 
 	do {
 		k = sib == btree_prev_sib