diff options
| -rw-r--r-- | drivers/md/bcache/btree_cache.c | 8 | ||||
| -rw-r--r-- | drivers/md/bcache/btree_iter.c | 42 | ||||
| -rw-r--r-- | drivers/md/bcache/btree_locking.h | 46 |
3 files changed, 54 insertions, 42 deletions
diff --git a/drivers/md/bcache/btree_cache.c b/drivers/md/bcache/btree_cache.c index 14387879435e..56eab3e0b424 100644 --- a/drivers/md/bcache/btree_cache.c +++ b/drivers/md/bcache/btree_cache.c @@ -601,14 +601,12 @@ static noinline struct btree *bch_btree_node_fill(struct btree_iter *iter, bch_btree_node_read(c, b); six_unlock_write(&b->lock); - if (btree_want_intent(iter, level)) { - mark_btree_node_intent_locked(iter, level); - } else { - mark_btree_node_read_locked(iter, level); + mark_btree_node_locked(iter, level, btree_lock_want(iter, level)); + + if (btree_lock_want(iter, level) == SIX_LOCK_read) BUG_ON(!six_trylock_convert(&b->lock, SIX_LOCK_intent, SIX_LOCK_read)); - } return b; } diff --git a/drivers/md/bcache/btree_iter.c b/drivers/md/bcache/btree_iter.c index d5da861e6514..2187846894f8 100644 --- a/drivers/md/bcache/btree_iter.c +++ b/drivers/md/bcache/btree_iter.c @@ -81,17 +81,18 @@ static bool btree_lock_upgrade(struct btree_iter *iter, unsigned level) if (btree_node_intent_locked(iter, level)) return true; - if (is_btree_node(iter, level) && - (btree_node_locked(iter, level) - ? six_trylock_convert(&b->lock, SIX_LOCK_read, SIX_LOCK_intent) - : six_relock_intent(&b->lock, iter->lock_seq[level]))) { + if (!is_btree_node(iter, level)) + return false; + + if (btree_node_locked(iter, level) + ? six_trylock_convert(&b->lock, SIX_LOCK_read, SIX_LOCK_intent) + : six_relock_intent(&b->lock, iter->lock_seq[level])) { mark_btree_node_intent_locked(iter, level); trace_bcache_btree_upgrade_lock(b, iter); return true; } - if (is_btree_node(iter, level)) - trace_bcache_btree_upgrade_lock_fail(b, iter); + trace_bcache_btree_upgrade_lock_fail(b, iter); return false; } @@ -135,27 +136,22 @@ int bch_btree_iter_unlock(struct btree_iter *iter) return iter->error; } -#define __btree_node_relock(b, iter, _level, type) \ -({ \ - bool _locked = six_relock_##type(&(b)->lock, \ - (iter)->lock_seq[_level]); \ - \ - if (_locked) \ - mark_btree_node_##type##_locked((iter), (_level)); \ - \ - _locked; \ -}) - bool btree_node_relock(struct btree_iter *iter, unsigned level) { struct btree *b = iter->nodes[level]; + enum six_lock_type type = btree_lock_want(iter, level); + + if (btree_node_locked(iter, level)) + return true; - return btree_node_locked(iter, level) || - (!race_fault() && - is_btree_node(iter, level) && - (btree_want_intent(iter, level) - ? __btree_node_relock(b, iter, level, intent) - : __btree_node_relock(b, iter, level, read))); + if (!race_fault() && + is_btree_node(iter, level) && + six_relock_type(&b->lock, iter->lock_seq[level], type)) { + mark_btree_node_locked(iter, level, type); + return true; + } + + return false; } /* Btree iterator: */ diff --git a/drivers/md/bcache/btree_locking.h b/drivers/md/bcache/btree_locking.h index d128bbd89f3a..7337248cded5 100644 --- a/drivers/md/bcache/btree_locking.h +++ b/drivers/md/bcache/btree_locking.h @@ -1,7 +1,13 @@ #ifndef _BCACHE_BTREE_LOCKING_H #define _BCACHE_BTREE_LOCKING_H -/* Only for internal btree use: */ +/* + * Only for internal btree use: + * + * The btree iterator tracks what locks it wants to take, and what locks it + * currently has - here we have wrappers for locking/unlocking btree nodes and + * updating the iterator state + */ #include "btree_iter.h" #include "six.h" @@ -50,22 +56,35 @@ static inline void mark_btree_node_unlocked(struct btree_iter *iter, iter->nodes_intent_locked &= ~(1 << level); } +static inline void mark_btree_node_locked(struct btree_iter *iter, + unsigned level, + enum six_lock_type type) +{ + /* relying on this to avoid a branch */ + BUILD_BUG_ON(SIX_LOCK_read != 0); + BUILD_BUG_ON(SIX_LOCK_intent != 1); + + iter->nodes_locked |= 1 << level; + iter->nodes_intent_locked |= type << level; +} + static inline void mark_btree_node_intent_locked(struct btree_iter *iter, unsigned level) { - iter->nodes_locked |= 1 << level; - iter->nodes_intent_locked |= 1 << level; + mark_btree_node_locked(iter, level, SIX_LOCK_intent); } -static inline void mark_btree_node_read_locked(struct btree_iter *iter, - unsigned level) +static inline enum six_lock_type +btree_lock_want(struct btree_iter *iter, int level) { - iter->nodes_locked |= 1 << level; + return level > iter->locks_want + ? SIX_LOCK_read + : SIX_LOCK_intent; } static inline bool btree_want_intent(struct btree_iter *iter, int level) { - return level <= iter->locks_want; + return btree_lock_want(iter, level) == SIX_LOCK_intent; } static inline void __btree_node_unlock(struct btree_iter *iter, unsigned level, @@ -106,24 +125,23 @@ static inline void btree_node_lock_type(struct btree *b, struct btree_iter *iter six_lock_type(&b->lock, type); } -#define __btree_node_lock(b, iter, _level, check_if_raced, type) \ +#define __btree_node_lock(b, _iter, _level, check_if_raced) \ ({ \ + enum six_lock_type _type = btree_lock_want(_iter, _level); \ bool _raced; \ \ - btree_node_lock_type(b, iter, SIX_LOCK_##type); \ + btree_node_lock_type(b, _iter, _type); \ if ((_raced = ((check_if_raced) || ((b)->level != _level)))) \ - six_unlock_##type(&(b)->lock); \ + six_unlock_type(&(b)->lock, _type); \ else \ - mark_btree_node_##type##_locked((iter), (_level)); \ + mark_btree_node_locked(_iter, _level, _type); \ \ !_raced; \ }) #define btree_node_lock(b, iter, level, check_if_raced) \ (!race_fault() && \ - (btree_want_intent(iter, level) \ - ? __btree_node_lock(b, iter, level, check_if_raced, intent) \ - : __btree_node_lock(b, iter, level, check_if_raced, read))) + __btree_node_lock(b, iter, level, check_if_raced)) bool btree_node_relock(struct btree_iter *, unsigned); |