bcachefs: avoid cache misses in io path

5 files changed, 30 insertions, 25 deletions

diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c
index 905a77973872..ce69c411c34c 100644
--- a/fs/bcachefs/extents.c
+++ b/fs/bcachefs/extents.c
@@ -322,9 +322,7 @@ static bool should_drop_ptr(const struct bch_fs *c,
 			    struct bkey_s_c_extent e,
 			    const struct bch_extent_ptr *ptr)
 {
-	struct bch_dev *ca = c->devs[ptr->dev];
-
-	return ptr_stale(ca, ptr);
+	return ptr->cached && ptr_stale(c->devs[ptr->dev], ptr);
 }
 
 static void bch_extent_drop_stale(struct bch_fs *c, struct bkey_s_extent e)
@@ -2153,7 +2151,7 @@ void bch_extent_pick_ptr_avoiding(struct bch_fs *c, struct bkey_s_c k,
 		extent_for_each_ptr_crc(e, ptr, crc) {
 			struct bch_dev *ca = c->devs[ptr->dev];
 
-			if (ptr_stale(ca, ptr))
+			if (ptr->cached && ptr_stale(ca, ptr))
 				continue;
 
 			if (ca->mi.state == BCH_MEMBER_STATE_FAILED)
diff --git a/fs/bcachefs/io.c b/fs/bcachefs/io.c
index 206cc6c9601a..3541bc880e7d 100644
--- a/fs/bcachefs/io.c
+++ b/fs/bcachefs/io.c
@@ -1101,12 +1101,15 @@ static void bch_read_endio(struct bio *bio)
 {
 	struct bch_read_bio *rbio =
 		container_of(bio, struct bch_read_bio, bio);
-	struct bch_fs *c = rbio->ca->fs;
-	int stale = ((rbio->flags & BCH_READ_RETRY_IF_STALE) && race_fault()) ||
-		ptr_stale(rbio->ca, &rbio->ptr) ? -EINTR : 0;
+	struct bch_fs *c = rbio->c;
+	int stale = rbio->ptr.cached &&
+		(((rbio->flags & BCH_READ_RETRY_IF_STALE) && race_fault()) ||
+		 ptr_stale(rbio->ca, &rbio->ptr) ? -EINTR : 0);
 	int error = bio->bi_error ?: stale;
 
-	bch_account_io_completion_time(rbio->ca, rbio->submit_time_us, REQ_OP_READ);
+	if (rbio->flags & BCH_READ_ACCOUNT_TIMES)
+		bch_account_io_completion_time(rbio->ca, rbio->submit_time_us,
+					       REQ_OP_READ);
 
 	bch_dev_nonfatal_io_err_on(bio->bi_error, rbio->ca, "data read");
 
@@ -1238,11 +1241,12 @@ void bch_read_extent_iter(struct bch_fs *c, struct bch_read_bio *orig,
 		rbio->orig_bi_end_io = orig->bio.bi_end_io;
 	rbio->parent_iter	= iter;
 
-	rbio->inode		= k.k->p.inode;
 	rbio->flags		= flags;
 	rbio->bounce		= bounce;
 	rbio->split		= split;
-	rbio->version		= k.k->version;
+	rbio->c			= c;
+	rbio->ca		= pick->ca;
+	rbio->ptr		= pick->ptr;
 	rbio->crc		= pick->crc;
 	/*
 	 * crc.compressed_size will be 0 if there wasn't any checksum
@@ -1251,9 +1255,9 @@ void bch_read_extent_iter(struct bch_fs *c, struct bch_read_bio *orig,
 	 * only for promoting)
 	 */
 	rbio->crc._compressed_size = bio_sectors(&rbio->bio) - 1;
-	rbio->ptr		= pick->ptr;
-	rbio->ca		= pick->ca;
+	rbio->version		= k.k->version;
 	rbio->promote		= promote_op;
+	rbio->inode		= k.k->p.inode;
 
 	rbio->bio.bi_bdev	= pick->ca->disk_sb.bdev;
 	rbio->bio.bi_opf	= orig->bio.bi_opf;
diff --git a/fs/bcachefs/io.h b/fs/bcachefs/io.h
index 302ed2e0944f..975951a84dbc 100644
--- a/fs/bcachefs/io.h
+++ b/fs/bcachefs/io.h
@@ -69,6 +69,7 @@ enum bch_read_flags {
 	BCH_READ_PROMOTE		= 1 << 2,
 	BCH_READ_IS_LAST		= 1 << 3,
 	BCH_READ_MAY_REUSE_BIO		= 1 << 4,
+	BCH_READ_ACCOUNT_TIMES		= 1 << 5,
 };
 
 void bch_read(struct bch_fs *, struct bch_read_bio *, u64);
diff --git a/fs/bcachefs/io_types.h b/fs/bcachefs/io_types.h
index 3d096876208f..73d18e361ecc 100644
--- a/fs/bcachefs/io_types.h
+++ b/fs/bcachefs/io_types.h
@@ -29,27 +29,28 @@ struct bch_read_bio {
 	 */
 	struct bvec_iter	parent_iter;
 
-	/*
-	 * If we have to retry the read (IO error, checksum failure, read stale
-	 * data (raced with allocator), we retry the portion of the parent bio
-	 * that failed (i.e. this bio's portion, parent_iter).
-	 *
-	 * But we need to stash the inode somewhere:
-	 */
-	u64			inode;
-
 	unsigned		submit_time_us;
 	u16			flags;
 	u8			bounce:1,
 				split:1;
 
-	struct bversion		version;
-	struct bch_extent_crc128 crc;
-	struct bch_extent_ptr	ptr;
+	struct bch_fs		*c;
 	struct bch_dev		*ca;
+	struct bch_extent_ptr	ptr;
+	struct bch_extent_crc128 crc;
+	struct bversion		version;
 
 	struct cache_promote_op *promote;
 
+	/*
+	 * If we have to retry the read (IO error, checksum failure, read stale
+	 * data (raced with allocator), we retry the portion of the parent bio
+	 * that failed (i.e. this bio's portion, parent_iter).
+	 *
+	 * But we need to stash the inode somewhere:
+	 */
+	u64			inode;
+
 	/* bio_decompress_worker list */
 	struct llist_node	list;
 
@@ -63,7 +64,7 @@ bch_rbio_parent(struct bch_read_bio *rbio)
 }
 
 struct bch_write_bio {
-	struct bch_fs	*c;
+	struct bch_fs		*c;
 	struct bch_dev		*ca;
 	union {
 		struct bio	*orig;
diff --git a/fs/bcachefs/request.c b/fs/bcachefs/request.c
index 0646346e4667..9a5d3593402c 100644
--- a/fs/bcachefs/request.c
+++ b/fs/bcachefs/request.c
@@ -500,6 +500,7 @@ retry:
 				s->read_dirty_data = true;
 
 			bch_read_extent(c, &s->rbio, k, &pick,
+					BCH_READ_ACCOUNT_TIMES|
 					BCH_READ_FORCE_BOUNCE|
 					BCH_READ_RETRY_IF_STALE|
 					(!s->bypass ? BCH_READ_PROMOTE : 0)|