diff options
author | Kent Overstreet <kent.overstreet@gmail.com> | 2018-04-15 18:31:49 -0400 |
---|---|---|
committer | Kent Overstreet <kent.overstreet@gmail.com> | 2018-05-01 13:38:07 -0400 |
commit | dbcdd83722691e284d877a55d4d97ca0b68bce7c (patch) | |
tree | 7395323a728acdf10f70cfa64521c07967bafd6b | |
parent | 911d41600157ea45c5e2616ad19f9ece4024a091 (diff) |
bcachefs: Keep track of latency devices are capable of
-rw-r--r-- | fs/bcachefs/bcachefs.h | 3 | ||||
-rw-r--r-- | fs/bcachefs/btree_io.c | 4 | ||||
-rw-r--r-- | fs/bcachefs/extents.c | 4 | ||||
-rw-r--r-- | fs/bcachefs/io.c | 28 | ||||
-rw-r--r-- | fs/bcachefs/io.h | 2 | ||||
-rw-r--r-- | fs/bcachefs/io_types.h | 6 | ||||
-rw-r--r-- | fs/bcachefs/super.c | 6 | ||||
-rw-r--r-- | fs/bcachefs/sysfs.c | 55 | ||||
-rw-r--r-- | fs/bcachefs/util.c | 4 | ||||
-rw-r--r-- | fs/bcachefs/util.h | 5 |
10 files changed, 74 insertions, 43 deletions
diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index f2cf7b95c081..61865240133f 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -417,7 +417,8 @@ struct bch_dev { struct work_struct io_error_work; /* The rest of this all shows up in sysfs */ - atomic_t latency[2]; + atomic64_t cur_latency[2]; + struct time_stats io_latency[2]; struct io_count __percpu *io_done; }; diff --git a/fs/bcachefs/btree_io.c b/fs/bcachefs/btree_io.c index 42190f05a0c1..a103e6a50a32 100644 --- a/fs/bcachefs/btree_io.c +++ b/fs/bcachefs/btree_io.c @@ -1380,7 +1380,7 @@ static void btree_node_read_endio(struct bio *bio) if (rb->have_ioref) { struct bch_dev *ca = bch_dev_bkey_exists(c, rb->pick.ptr.dev); - bch2_latency_acct(ca, rb->start_time >> 10, READ); + bch2_latency_acct(ca, rb->start_time, READ); } queue_work(system_unbound_wq, &rb->work); @@ -1628,7 +1628,7 @@ static void btree_node_write_endio(struct bio *bio) unsigned long flags; if (wbio->have_ioref) - bch2_latency_acct(ca, wbio->submit_time_us, WRITE); + bch2_latency_acct(ca, wbio->submit_time, WRITE); if (bio->bi_status == BLK_STS_REMOVED || bch2_dev_io_err_on(bio->bi_status, ca, "btree write") || diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index c6f969c4cfae..9efaa1ffa92f 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -594,8 +594,8 @@ static inline bool dev_latency_better(struct bch_fs *c, { struct bch_dev *dev1 = bch_dev_bkey_exists(c, ptr1->dev); struct bch_dev *dev2 = bch_dev_bkey_exists(c, ptr2->dev); - unsigned l1 = atomic_read(&dev1->latency[READ]); - unsigned l2 = atomic_read(&dev2->latency[READ]); + u64 l1 = atomic64_read(&dev1->cur_latency[READ]); + u64 l2 = atomic64_read(&dev2->cur_latency[READ]); /* Pick at random, biased in favor of the faster device: */ diff --git a/fs/bcachefs/io.c b/fs/bcachefs/io.c index 258da9f73cca..80f2b3f5256a 100644 --- a/fs/bcachefs/io.c +++ b/fs/bcachefs/io.c @@ -30,14 +30,14 @@ #include <trace/events/bcachefs.h> -/* Allocate, free from mempool: */ - -void bch2_latency_acct(struct bch_dev *ca, unsigned submit_time_us, int rw) +void bch2_latency_acct(struct bch_dev *ca, u64 submit_time, int rw) { + atomic64_t *latency = &ca->cur_latency[rw]; u64 now = local_clock(); - unsigned io_latency = (now >> 10) - submit_time_us; - atomic_t *latency = &ca->latency[rw]; - unsigned old, new, v = atomic_read(latency); + u64 io_latency = time_after64(now, submit_time) + ? now - submit_time + : 0; + u64 old, new, v = atomic64_read(latency); do { old = v; @@ -51,10 +51,14 @@ void bch2_latency_acct(struct bch_dev *ca, unsigned submit_time_us, int rw) now & ~(~0 << 5)) break; - new = ewma_add((u64) old, io_latency, 6); - } while ((v = atomic_cmpxchg(latency, old, new)) != old); + new = ewma_add(old, io_latency, 6); + } while ((v = atomic64_cmpxchg(latency, old, new)) != old); + + bch2_time_stats_update(&ca->io_latency[rw], submit_time); } +/* Allocate, free from mempool: */ + void bch2_bio_free_pages_pool(struct bch_fs *c, struct bio *bio) { struct bio_vec *bv; @@ -171,7 +175,7 @@ void bch2_submit_wbio_replicas(struct bch_write_bio *wbio, struct bch_fs *c, n->c = c; n->dev = ptr->dev; n->have_ioref = bch2_dev_get_ioref(ca, WRITE); - n->submit_time_us = local_clock_us(); + n->submit_time = local_clock(); n->bio.bi_iter.bi_sector = ptr->offset; if (!journal_flushes_device(ca)) @@ -326,7 +330,7 @@ static void bch2_write_endio(struct bio *bio) set_bit(wbio->dev, op->failed.d); if (wbio->have_ioref) { - bch2_latency_acct(ca, wbio->submit_time_us, WRITE); + bch2_latency_acct(ca, wbio->submit_time, WRITE); percpu_ref_put(&ca->io_ref); } @@ -1369,7 +1373,7 @@ static void bch2_read_endio(struct bio *bio) enum rbio_context context = RBIO_CONTEXT_NULL; if (rbio->have_ioref) { - bch2_latency_acct(ca, rbio->submit_time_us, READ); + bch2_latency_acct(ca, rbio->submit_time, READ); percpu_ref_put(&ca->io_ref); } @@ -1520,12 +1524,12 @@ noclone: BUG_ON(bio_sectors(&rbio->bio) != pick.crc.compressed_size); rbio->c = c; + rbio->submit_time = local_clock(); if (split) rbio->parent = orig; else rbio->end_io = orig->bio.bi_end_io; rbio->bvec_iter = iter; - rbio->submit_time_us = local_clock_us(); rbio->flags = flags; rbio->bounce = bounce; rbio->split = split; diff --git a/fs/bcachefs/io.h b/fs/bcachefs/io.h index 8562a5d01e37..8b1411c652c7 100644 --- a/fs/bcachefs/io.h +++ b/fs/bcachefs/io.h @@ -16,7 +16,7 @@ void bch2_bio_free_pages_pool(struct bch_fs *, struct bio *); void bch2_bio_alloc_pages_pool(struct bch_fs *, struct bio *, size_t); void bch2_bio_alloc_more_pages_pool(struct bch_fs *, struct bio *, size_t); -void bch2_latency_acct(struct bch_dev *, unsigned, int); +void bch2_latency_acct(struct bch_dev *, u64, int); void bch2_submit_wbio_replicas(struct bch_write_bio *, struct bch_fs *, enum bch_data_type, const struct bkey_i *); diff --git a/fs/bcachefs/io_types.h b/fs/bcachefs/io_types.h index 93e9a0e28eab..c3c33f6e0f73 100644 --- a/fs/bcachefs/io_types.h +++ b/fs/bcachefs/io_types.h @@ -15,6 +15,7 @@ struct bch_read_bio { struct bch_fs *c; u64 start_time; + u64 submit_time; /* * Reads will often have to be split, and if the extent being read from @@ -36,7 +37,6 @@ struct bch_read_bio { */ struct bvec_iter bvec_iter; - unsigned submit_time_us; u16 flags; union { struct { @@ -71,6 +71,8 @@ struct bch_write_bio { struct bch_fs *c; struct bch_write_bio *parent; + u64 submit_time; + struct bch_devs_list failed; u8 order; u8 dev; @@ -81,8 +83,6 @@ struct bch_write_bio { have_ioref:1, used_mempool:1; - unsigned submit_time_us; - struct bio bio; }; diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index 5f869946680f..b6336eb1a86a 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -985,6 +985,9 @@ static void bch2_dev_free(struct bch_dev *ca) bioset_exit(&ca->replica_set); bch2_dev_buckets_free(ca); + bch2_time_stats_exit(&ca->io_latency[WRITE]); + bch2_time_stats_exit(&ca->io_latency[READ]); + percpu_ref_exit(&ca->io_ref); percpu_ref_exit(&ca->ref); kobject_put(&ca->kobj); @@ -1081,6 +1084,9 @@ static struct bch_dev *__bch2_dev_alloc(struct bch_fs *c, INIT_WORK(&ca->io_error_work, bch2_io_error_work); + bch2_time_stats_init(&ca->io_latency[READ]); + bch2_time_stats_init(&ca->io_latency[WRITE]); + ca->mi = bch2_mi_to_cpu(member); ca->uuid = member->uuid; diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c index 8c7f62efecc1..d4e02f2aa63b 100644 --- a/fs/bcachefs/sysfs.c +++ b/fs/bcachefs/sysfs.c @@ -141,11 +141,18 @@ read_attribute(btree_node_size); read_attribute(first_bucket); read_attribute(nbuckets); read_attribute(durability); -read_attribute(iostats); -read_attribute(last_read_quantiles); -read_attribute(last_write_quantiles); -read_attribute(fragmentation_quantiles); -read_attribute(oldest_gen_quantiles); +read_attribute(iodone); + +read_attribute(io_latency_read); +read_attribute(io_latency_write); +read_attribute(io_latency_stats_read); +read_attribute(io_latency_stats_write); + +read_attribute(bucket_quantiles_last_read); +read_attribute(bucket_quantiles_last_write); +read_attribute(bucket_quantiles_fragmentation); +read_attribute(bucket_quantiles_oldest_gen); + read_attribute(reserve_stats); read_attribute(btree_cache_size); read_attribute(compression_stats); @@ -774,7 +781,7 @@ static const char * const bch2_rw[] = { NULL }; -static ssize_t show_dev_iostats(struct bch_dev *ca, char *buf) +static ssize_t show_dev_iodone(struct bch_dev *ca, char *buf) { char *out = buf, *end = buf + PAGE_SIZE; int rw, i, cpu; @@ -851,16 +858,24 @@ SHOW(bch2_dev) return out - buf; } - if (attr == &sysfs_iostats) - return show_dev_iostats(ca, buf); + if (attr == &sysfs_iodone) + return show_dev_iodone(ca, buf); + + sysfs_print(io_latency_read, atomic64_read(&ca->cur_latency[READ])); + sysfs_print(io_latency_write, atomic64_read(&ca->cur_latency[WRITE])); - if (attr == &sysfs_last_read_quantiles) + if (attr == &sysfs_io_latency_stats_read) + return bch2_time_stats_print(&ca->io_latency[READ], buf, PAGE_SIZE); + if (attr == &sysfs_io_latency_stats_write) + return bch2_time_stats_print(&ca->io_latency[WRITE], buf, PAGE_SIZE); + + if (attr == &sysfs_bucket_quantiles_last_read) return show_quantiles(c, ca, buf, bucket_last_io_fn, (void *) 0); - if (attr == &sysfs_last_write_quantiles) + if (attr == &sysfs_bucket_quantiles_last_write) return show_quantiles(c, ca, buf, bucket_last_io_fn, (void *) 1); - if (attr == &sysfs_fragmentation_quantiles) + if (attr == &sysfs_bucket_quantiles_fragmentation) return show_quantiles(c, ca, buf, bucket_sectors_used_fn, NULL); - if (attr == &sysfs_oldest_gen_quantiles) + if (attr == &sysfs_bucket_quantiles_oldest_gen) return show_quantiles(c, ca, buf, bucket_oldest_gen_fn, NULL); if (attr == &sysfs_reserve_stats) @@ -944,13 +959,19 @@ struct attribute *bch2_dev_files[] = { &sysfs_label, &sysfs_has_data, - &sysfs_iostats, + &sysfs_iodone, + + &sysfs_io_latency_read, + &sysfs_io_latency_write, + &sysfs_io_latency_stats_read, + &sysfs_io_latency_stats_write, /* alloc info - other stats: */ - &sysfs_last_read_quantiles, - &sysfs_last_write_quantiles, - &sysfs_fragmentation_quantiles, - &sysfs_oldest_gen_quantiles, + &sysfs_bucket_quantiles_last_read, + &sysfs_bucket_quantiles_last_write, + &sysfs_bucket_quantiles_fragmentation, + &sysfs_bucket_quantiles_oldest_gen, + &sysfs_reserve_stats, /* debug: */ diff --git a/fs/bcachefs/util.c b/fs/bcachefs/util.c index 67bc3dfd4688..33e2d1fa795d 100644 --- a/fs/bcachefs/util.c +++ b/fs/bcachefs/util.c @@ -442,6 +442,8 @@ int bch2_ratelimit_wait_freezable_stoppable(struct bch_ratelimit *d) } } +/* pd controller: */ + /* * Updates pd_controller. Attempts to scale inputed values to units per second. * @target: desired value @@ -536,6 +538,8 @@ size_t bch2_pd_controller_print_debug(struct bch_pd_controller *pd, char *buf) derivative, change, next_io); } +/* misc: */ + void bch2_bio_map(struct bio *bio, void *base) { size_t size = bio->bi_iter.bi_size; diff --git a/fs/bcachefs/util.h b/fs/bcachefs/util.h index 3ffeda08858d..97e4e6d00a37 100644 --- a/fs/bcachefs/util.h +++ b/fs/bcachefs/util.h @@ -407,11 +407,6 @@ size_t bch2_time_stats_print(struct time_stats *, char *, size_t); void bch2_time_stats_exit(struct time_stats *); void bch2_time_stats_init(struct time_stats *); -static inline unsigned local_clock_us(void) -{ - return local_clock() >> 10; -} - #define ewma_add(ewma, val, weight) \ ({ \ typeof(ewma) _ewma = (ewma); \ |