summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKent Overstreet <kent.overstreet@gmail.com>2018-04-15 18:31:49 -0400
committerKent Overstreet <kent.overstreet@gmail.com>2018-05-01 13:38:07 -0400
commitdbcdd83722691e284d877a55d4d97ca0b68bce7c (patch)
tree7395323a728acdf10f70cfa64521c07967bafd6b
parent911d41600157ea45c5e2616ad19f9ece4024a091 (diff)
bcachefs: Keep track of latency devices are capable of
-rw-r--r--fs/bcachefs/bcachefs.h3
-rw-r--r--fs/bcachefs/btree_io.c4
-rw-r--r--fs/bcachefs/extents.c4
-rw-r--r--fs/bcachefs/io.c28
-rw-r--r--fs/bcachefs/io.h2
-rw-r--r--fs/bcachefs/io_types.h6
-rw-r--r--fs/bcachefs/super.c6
-rw-r--r--fs/bcachefs/sysfs.c55
-rw-r--r--fs/bcachefs/util.c4
-rw-r--r--fs/bcachefs/util.h5
10 files changed, 74 insertions, 43 deletions
diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h
index f2cf7b95c081..61865240133f 100644
--- a/fs/bcachefs/bcachefs.h
+++ b/fs/bcachefs/bcachefs.h
@@ -417,7 +417,8 @@ struct bch_dev {
struct work_struct io_error_work;
/* The rest of this all shows up in sysfs */
- atomic_t latency[2];
+ atomic64_t cur_latency[2];
+ struct time_stats io_latency[2];
struct io_count __percpu *io_done;
};
diff --git a/fs/bcachefs/btree_io.c b/fs/bcachefs/btree_io.c
index 42190f05a0c1..a103e6a50a32 100644
--- a/fs/bcachefs/btree_io.c
+++ b/fs/bcachefs/btree_io.c
@@ -1380,7 +1380,7 @@ static void btree_node_read_endio(struct bio *bio)
if (rb->have_ioref) {
struct bch_dev *ca = bch_dev_bkey_exists(c, rb->pick.ptr.dev);
- bch2_latency_acct(ca, rb->start_time >> 10, READ);
+ bch2_latency_acct(ca, rb->start_time, READ);
}
queue_work(system_unbound_wq, &rb->work);
@@ -1628,7 +1628,7 @@ static void btree_node_write_endio(struct bio *bio)
unsigned long flags;
if (wbio->have_ioref)
- bch2_latency_acct(ca, wbio->submit_time_us, WRITE);
+ bch2_latency_acct(ca, wbio->submit_time, WRITE);
if (bio->bi_status == BLK_STS_REMOVED ||
bch2_dev_io_err_on(bio->bi_status, ca, "btree write") ||
diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c
index c6f969c4cfae..9efaa1ffa92f 100644
--- a/fs/bcachefs/extents.c
+++ b/fs/bcachefs/extents.c
@@ -594,8 +594,8 @@ static inline bool dev_latency_better(struct bch_fs *c,
{
struct bch_dev *dev1 = bch_dev_bkey_exists(c, ptr1->dev);
struct bch_dev *dev2 = bch_dev_bkey_exists(c, ptr2->dev);
- unsigned l1 = atomic_read(&dev1->latency[READ]);
- unsigned l2 = atomic_read(&dev2->latency[READ]);
+ u64 l1 = atomic64_read(&dev1->cur_latency[READ]);
+ u64 l2 = atomic64_read(&dev2->cur_latency[READ]);
/* Pick at random, biased in favor of the faster device: */
diff --git a/fs/bcachefs/io.c b/fs/bcachefs/io.c
index 258da9f73cca..80f2b3f5256a 100644
--- a/fs/bcachefs/io.c
+++ b/fs/bcachefs/io.c
@@ -30,14 +30,14 @@
#include <trace/events/bcachefs.h>
-/* Allocate, free from mempool: */
-
-void bch2_latency_acct(struct bch_dev *ca, unsigned submit_time_us, int rw)
+void bch2_latency_acct(struct bch_dev *ca, u64 submit_time, int rw)
{
+ atomic64_t *latency = &ca->cur_latency[rw];
u64 now = local_clock();
- unsigned io_latency = (now >> 10) - submit_time_us;
- atomic_t *latency = &ca->latency[rw];
- unsigned old, new, v = atomic_read(latency);
+ u64 io_latency = time_after64(now, submit_time)
+ ? now - submit_time
+ : 0;
+ u64 old, new, v = atomic64_read(latency);
do {
old = v;
@@ -51,10 +51,14 @@ void bch2_latency_acct(struct bch_dev *ca, unsigned submit_time_us, int rw)
now & ~(~0 << 5))
break;
- new = ewma_add((u64) old, io_latency, 6);
- } while ((v = atomic_cmpxchg(latency, old, new)) != old);
+ new = ewma_add(old, io_latency, 6);
+ } while ((v = atomic64_cmpxchg(latency, old, new)) != old);
+
+ bch2_time_stats_update(&ca->io_latency[rw], submit_time);
}
+/* Allocate, free from mempool: */
+
void bch2_bio_free_pages_pool(struct bch_fs *c, struct bio *bio)
{
struct bio_vec *bv;
@@ -171,7 +175,7 @@ void bch2_submit_wbio_replicas(struct bch_write_bio *wbio, struct bch_fs *c,
n->c = c;
n->dev = ptr->dev;
n->have_ioref = bch2_dev_get_ioref(ca, WRITE);
- n->submit_time_us = local_clock_us();
+ n->submit_time = local_clock();
n->bio.bi_iter.bi_sector = ptr->offset;
if (!journal_flushes_device(ca))
@@ -326,7 +330,7 @@ static void bch2_write_endio(struct bio *bio)
set_bit(wbio->dev, op->failed.d);
if (wbio->have_ioref) {
- bch2_latency_acct(ca, wbio->submit_time_us, WRITE);
+ bch2_latency_acct(ca, wbio->submit_time, WRITE);
percpu_ref_put(&ca->io_ref);
}
@@ -1369,7 +1373,7 @@ static void bch2_read_endio(struct bio *bio)
enum rbio_context context = RBIO_CONTEXT_NULL;
if (rbio->have_ioref) {
- bch2_latency_acct(ca, rbio->submit_time_us, READ);
+ bch2_latency_acct(ca, rbio->submit_time, READ);
percpu_ref_put(&ca->io_ref);
}
@@ -1520,12 +1524,12 @@ noclone:
BUG_ON(bio_sectors(&rbio->bio) != pick.crc.compressed_size);
rbio->c = c;
+ rbio->submit_time = local_clock();
if (split)
rbio->parent = orig;
else
rbio->end_io = orig->bio.bi_end_io;
rbio->bvec_iter = iter;
- rbio->submit_time_us = local_clock_us();
rbio->flags = flags;
rbio->bounce = bounce;
rbio->split = split;
diff --git a/fs/bcachefs/io.h b/fs/bcachefs/io.h
index 8562a5d01e37..8b1411c652c7 100644
--- a/fs/bcachefs/io.h
+++ b/fs/bcachefs/io.h
@@ -16,7 +16,7 @@ void bch2_bio_free_pages_pool(struct bch_fs *, struct bio *);
void bch2_bio_alloc_pages_pool(struct bch_fs *, struct bio *, size_t);
void bch2_bio_alloc_more_pages_pool(struct bch_fs *, struct bio *, size_t);
-void bch2_latency_acct(struct bch_dev *, unsigned, int);
+void bch2_latency_acct(struct bch_dev *, u64, int);
void bch2_submit_wbio_replicas(struct bch_write_bio *, struct bch_fs *,
enum bch_data_type, const struct bkey_i *);
diff --git a/fs/bcachefs/io_types.h b/fs/bcachefs/io_types.h
index 93e9a0e28eab..c3c33f6e0f73 100644
--- a/fs/bcachefs/io_types.h
+++ b/fs/bcachefs/io_types.h
@@ -15,6 +15,7 @@
struct bch_read_bio {
struct bch_fs *c;
u64 start_time;
+ u64 submit_time;
/*
* Reads will often have to be split, and if the extent being read from
@@ -36,7 +37,6 @@ struct bch_read_bio {
*/
struct bvec_iter bvec_iter;
- unsigned submit_time_us;
u16 flags;
union {
struct {
@@ -71,6 +71,8 @@ struct bch_write_bio {
struct bch_fs *c;
struct bch_write_bio *parent;
+ u64 submit_time;
+
struct bch_devs_list failed;
u8 order;
u8 dev;
@@ -81,8 +83,6 @@ struct bch_write_bio {
have_ioref:1,
used_mempool:1;
- unsigned submit_time_us;
-
struct bio bio;
};
diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c
index 5f869946680f..b6336eb1a86a 100644
--- a/fs/bcachefs/super.c
+++ b/fs/bcachefs/super.c
@@ -985,6 +985,9 @@ static void bch2_dev_free(struct bch_dev *ca)
bioset_exit(&ca->replica_set);
bch2_dev_buckets_free(ca);
+ bch2_time_stats_exit(&ca->io_latency[WRITE]);
+ bch2_time_stats_exit(&ca->io_latency[READ]);
+
percpu_ref_exit(&ca->io_ref);
percpu_ref_exit(&ca->ref);
kobject_put(&ca->kobj);
@@ -1081,6 +1084,9 @@ static struct bch_dev *__bch2_dev_alloc(struct bch_fs *c,
INIT_WORK(&ca->io_error_work, bch2_io_error_work);
+ bch2_time_stats_init(&ca->io_latency[READ]);
+ bch2_time_stats_init(&ca->io_latency[WRITE]);
+
ca->mi = bch2_mi_to_cpu(member);
ca->uuid = member->uuid;
diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c
index 8c7f62efecc1..d4e02f2aa63b 100644
--- a/fs/bcachefs/sysfs.c
+++ b/fs/bcachefs/sysfs.c
@@ -141,11 +141,18 @@ read_attribute(btree_node_size);
read_attribute(first_bucket);
read_attribute(nbuckets);
read_attribute(durability);
-read_attribute(iostats);
-read_attribute(last_read_quantiles);
-read_attribute(last_write_quantiles);
-read_attribute(fragmentation_quantiles);
-read_attribute(oldest_gen_quantiles);
+read_attribute(iodone);
+
+read_attribute(io_latency_read);
+read_attribute(io_latency_write);
+read_attribute(io_latency_stats_read);
+read_attribute(io_latency_stats_write);
+
+read_attribute(bucket_quantiles_last_read);
+read_attribute(bucket_quantiles_last_write);
+read_attribute(bucket_quantiles_fragmentation);
+read_attribute(bucket_quantiles_oldest_gen);
+
read_attribute(reserve_stats);
read_attribute(btree_cache_size);
read_attribute(compression_stats);
@@ -774,7 +781,7 @@ static const char * const bch2_rw[] = {
NULL
};
-static ssize_t show_dev_iostats(struct bch_dev *ca, char *buf)
+static ssize_t show_dev_iodone(struct bch_dev *ca, char *buf)
{
char *out = buf, *end = buf + PAGE_SIZE;
int rw, i, cpu;
@@ -851,16 +858,24 @@ SHOW(bch2_dev)
return out - buf;
}
- if (attr == &sysfs_iostats)
- return show_dev_iostats(ca, buf);
+ if (attr == &sysfs_iodone)
+ return show_dev_iodone(ca, buf);
+
+ sysfs_print(io_latency_read, atomic64_read(&ca->cur_latency[READ]));
+ sysfs_print(io_latency_write, atomic64_read(&ca->cur_latency[WRITE]));
- if (attr == &sysfs_last_read_quantiles)
+ if (attr == &sysfs_io_latency_stats_read)
+ return bch2_time_stats_print(&ca->io_latency[READ], buf, PAGE_SIZE);
+ if (attr == &sysfs_io_latency_stats_write)
+ return bch2_time_stats_print(&ca->io_latency[WRITE], buf, PAGE_SIZE);
+
+ if (attr == &sysfs_bucket_quantiles_last_read)
return show_quantiles(c, ca, buf, bucket_last_io_fn, (void *) 0);
- if (attr == &sysfs_last_write_quantiles)
+ if (attr == &sysfs_bucket_quantiles_last_write)
return show_quantiles(c, ca, buf, bucket_last_io_fn, (void *) 1);
- if (attr == &sysfs_fragmentation_quantiles)
+ if (attr == &sysfs_bucket_quantiles_fragmentation)
return show_quantiles(c, ca, buf, bucket_sectors_used_fn, NULL);
- if (attr == &sysfs_oldest_gen_quantiles)
+ if (attr == &sysfs_bucket_quantiles_oldest_gen)
return show_quantiles(c, ca, buf, bucket_oldest_gen_fn, NULL);
if (attr == &sysfs_reserve_stats)
@@ -944,13 +959,19 @@ struct attribute *bch2_dev_files[] = {
&sysfs_label,
&sysfs_has_data,
- &sysfs_iostats,
+ &sysfs_iodone,
+
+ &sysfs_io_latency_read,
+ &sysfs_io_latency_write,
+ &sysfs_io_latency_stats_read,
+ &sysfs_io_latency_stats_write,
/* alloc info - other stats: */
- &sysfs_last_read_quantiles,
- &sysfs_last_write_quantiles,
- &sysfs_fragmentation_quantiles,
- &sysfs_oldest_gen_quantiles,
+ &sysfs_bucket_quantiles_last_read,
+ &sysfs_bucket_quantiles_last_write,
+ &sysfs_bucket_quantiles_fragmentation,
+ &sysfs_bucket_quantiles_oldest_gen,
+
&sysfs_reserve_stats,
/* debug: */
diff --git a/fs/bcachefs/util.c b/fs/bcachefs/util.c
index 67bc3dfd4688..33e2d1fa795d 100644
--- a/fs/bcachefs/util.c
+++ b/fs/bcachefs/util.c
@@ -442,6 +442,8 @@ int bch2_ratelimit_wait_freezable_stoppable(struct bch_ratelimit *d)
}
}
+/* pd controller: */
+
/*
* Updates pd_controller. Attempts to scale inputed values to units per second.
* @target: desired value
@@ -536,6 +538,8 @@ size_t bch2_pd_controller_print_debug(struct bch_pd_controller *pd, char *buf)
derivative, change, next_io);
}
+/* misc: */
+
void bch2_bio_map(struct bio *bio, void *base)
{
size_t size = bio->bi_iter.bi_size;
diff --git a/fs/bcachefs/util.h b/fs/bcachefs/util.h
index 3ffeda08858d..97e4e6d00a37 100644
--- a/fs/bcachefs/util.h
+++ b/fs/bcachefs/util.h
@@ -407,11 +407,6 @@ size_t bch2_time_stats_print(struct time_stats *, char *, size_t);
void bch2_time_stats_exit(struct time_stats *);
void bch2_time_stats_init(struct time_stats *);
-static inline unsigned local_clock_us(void)
-{
- return local_clock() >> 10;
-}
-
#define ewma_add(ewma, val, weight) \
({ \
typeof(ewma) _ewma = (ewma); \