bcache: memcpy_u64s()

we do a lot of memcpy()s in bcache that we know are 8 byte aligned: on
x86-86, this reduces the size of .text by 3kb.

13 files changed, 149 insertions, 80 deletions

diff --git a/drivers/md/bcache/bkey.c b/drivers/md/bcache/bkey.c
index 1c48ea3e8a6c..4f1f5317ce43 100644
--- a/drivers/md/bcache/bkey.c
+++ b/drivers/md/bcache/bkey.c
@@ -247,9 +247,9 @@ bool bch_bkey_transform(const struct bkey_format *out_f,
 	if (!bch_bkey_transform_key(out_f, out, in_f, in))
 		return false;
 
-	memcpy((u64 *) out + out_f->key_u64s,
-	       (u64 *) in + in_f->key_u64s,
-	       (in->u64s - in_f->key_u64s) * sizeof(u64));
+	memcpy_u64s((u64 *) out + out_f->key_u64s,
+		    (u64 *) in + in_f->key_u64s,
+		    (in->u64s - in_f->key_u64s));
 	return true;
 }
 
@@ -391,9 +391,9 @@ void bkey_unpack(struct bkey_i *dst,
 {
 	dst->k = bkey_unpack_key(format, src);
 
-	memcpy(&dst->v,
-	       bkeyp_val(format, src),
-	       bkeyp_val_bytes(format, src));
+	memcpy_u64s(&dst->v,
+		    bkeyp_val(format, src),
+		    bkeyp_val_u64s(format, src));
 }
 
 /**
@@ -407,11 +407,10 @@ bool bkey_pack(struct bkey_packed *out, const struct bkey_i *in,
 	if (!bkey_pack_key(&tmp, &in->k, format))
 		return false;
 
-	memmove((u64 *) out + format->key_u64s,
-		&in->v,
-		bkey_val_bytes(&in->k));
-	memcpy(out, &tmp,
-	       format->key_u64s * sizeof(u64));
+	memmove_u64s((u64 *) out + format->key_u64s,
+		     &in->v,
+		     bkey_val_u64s(&in->k));
+	memcpy_u64s(out, &tmp, format->key_u64s);
 
 	return true;
 }
diff --git a/drivers/md/bcache/bkey.h b/drivers/md/bcache/bkey.h
index f0bd3faf0d76..6bdc54b9baa4 100644
--- a/drivers/md/bcache/bkey.h
+++ b/drivers/md/bcache/bkey.h
@@ -73,6 +73,82 @@ static inline void set_bkey_deleted(struct bkey *k)
 
 #define bkey_deleted(_k)	((_k)->type == KEY_TYPE_DELETED)
 
+static inline void __memcpy_u64s(void *dst, const void *src,
+				 unsigned u64s)
+{
+#ifdef CONFIG_X86_64
+	long d0, d1, d2;
+	asm volatile("rep ; movsq"
+		     : "=&c" (d0), "=&D" (d1), "=&S" (d2)
+		     : "0" (u64s), "1" (dst), "2" (src)
+		     : "memory");
+#else
+	u64 *d = dst;
+	const u64 *s = src;
+
+	while (u64s--)
+		*d++ = *s++;
+#endif
+}
+
+static inline void memcpy_u64s(void *dst, const void *src,
+			       unsigned u64s)
+{
+	EBUG_ON(!(dst >= src + u64s * sizeof(u64) ||
+		 dst + u64s * sizeof(u64) <= src));
+
+	__memcpy_u64s(dst, src, u64s);
+}
+
+static inline void __memmove_u64s_down(void *dst, const void *src,
+				       unsigned u64s)
+{
+	__memcpy_u64s(dst, src, u64s);
+}
+
+static inline void memmove_u64s_down(void *dst, const void *src,
+				     unsigned u64s)
+{
+	EBUG_ON(dst > src);
+
+	__memmove_u64s_down(dst, src, u64s);
+}
+
+static inline void memmove_u64s_up(void *dst, const void *src,
+				   unsigned u64s)
+{
+	u64 *d = dst;
+	const u64 *s = src;
+
+	EBUG_ON(dst < src);
+
+	while (u64s--)
+		d[u64s] = s[u64s];
+}
+
+static inline void memmove_u64s(void *dst, const void *src,
+				unsigned u64s)
+{
+	if (dst < src)
+		memmove_u64s_down(dst, src, u64s);
+	else
+		memmove_u64s_up(dst, src, u64s);
+}
+
+#define bkey_copy(_dst, _src)					\
+do {								\
+	BUILD_BUG_ON(!type_is(_dst, struct bkey_i *) &&		\
+		     !type_is(_dst, struct bkey_packed *));	\
+	BUILD_BUG_ON(!type_is(_src, struct bkey_i *) &&		\
+		     !type_is(_src, struct bkey_packed *));	\
+	EBUG_ON((u64 *) (_dst) > (u64 *) (_src) &&		\
+		(u64 *) (_dst) < (u64 *) (_src) +		\
+		((struct bkey *) (_src))->u64s);		\
+								\
+	__memmove_u64s_down((_dst), (_src),			\
+			    ((struct bkey *) (_src))->u64s);	\
+} while (0)
+
 struct btree_keys;
 
 struct bkey_format_state {
@@ -323,7 +399,7 @@ static inline void bkey_reassemble(struct bkey_i *dst,
 {
 	BUG_ON(bkey_packed(src.k));
 	dst->k = *src.k;
-	memcpy(&dst->v, src.v, bkey_val_bytes(src.k));
+	memcpy_u64s(&dst->v, src.v, bkey_val_u64s(src.k));
 }
 
 #define bkey_s_null		((struct bkey_s)   { .k = NULL })
diff --git a/drivers/md/bcache/bset.c b/drivers/md/bcache/bset.c
index 3cba1381d58f..83d3347c2159 100644
--- a/drivers/md/bcache/bset.c
+++ b/drivers/md/bcache/bset.c
@@ -1077,17 +1077,17 @@ void bch_bset_insert(struct btree_keys *b,
 		btree_keys_account_key_add(&b->nr, b->nsets, src);
 
 	if (src->u64s != clobber_u64s) {
-		void *src_p = where->_data + clobber_u64s;
-		void *dst_p = where->_data + src->u64s;
+		u64 *src_p = where->_data + clobber_u64s;
+		u64 *dst_p = where->_data + src->u64s;
 
-		memmove(dst_p, src_p, (void *) bset_bkey_last(i) - src_p);
+		memmove_u64s(dst_p, src_p, bset_bkey_last(i)->_data - src_p);
 		le16_add_cpu(&i->u64s, src->u64s - clobber_u64s);
 	}
 
-	memcpy(where, src,
-	       bkeyp_key_bytes(f, src));
-	memcpy(bkeyp_val(f, where), &insert->v,
-	       bkeyp_val_bytes(f, src));
+	memcpy_u64s(where, src,
+		    bkeyp_key_u64s(f, src));
+	memcpy_u64s(bkeyp_val(f, where), &insert->v,
+		    bkeyp_val_u64s(f, src));
 
 	bch_bset_fix_lookup_table(b, t, where, clobber_u64s, src->u64s);
 	bch_bset_verify_lookup_table(b, t);
@@ -1102,10 +1102,10 @@ void bch_bset_delete(struct btree_keys *b,
 {
 	struct bset_tree *t = bset_tree_last(b);
 	struct bset *i = t->data;
-	void *src_p = where->_data + clobber_u64s;
-	void *dst_p = where->_data;
+	u64 *src_p = where->_data + clobber_u64s;
+	u64 *dst_p = where->_data;
 
-	memmove(dst_p, src_p, (void *) bset_bkey_last(i) - src_p);
+	memmove_u64s_down(dst_p, src_p, bset_bkey_last(i)->_data - src_p);
 	le16_add_cpu(&i->u64s, -clobber_u64s);
 
 	bch_bset_fix_lookup_table(b, t, where, clobber_u64s, 0);
@@ -1602,8 +1602,7 @@ static struct btree_nr_keys btree_mergesort_simple(struct btree_keys *b,
 
 	while ((in = bch_btree_node_iter_next_all(iter, b))) {
 		if (!bkey_packed_is_whiteout(b, in)) {
-			/* XXX: need better bkey_copy */
-			memcpy(out, in, bkey_bytes(in));
+			bkey_copy(out, in);
 			out = bkey_next(out);
 		}
 	}
@@ -1706,7 +1705,7 @@ static struct btree_nr_keys btree_mergesort_extents(struct btree_keys *dst,
 			prev = dst_set->start;
 		}
 
-		bkey_copy((void *) prev, &tmp.k);
+		bkey_copy(prev, &tmp.k);
 
 		BUG_ON((void *) bkey_next(prev) >
 		       (void *) dst_set + (PAGE_SIZE << dst->page_order));
@@ -1829,7 +1828,7 @@ bool bch_maybe_compact_deleted_keys(struct btree_keys *b)
 			n = bkey_next(k);
 
 			if (!bkey_packed_is_whiteout(b, k)) {
-				memmove(out, k, bkey_bytes(k));
+				bkey_copy(out, k);
 				out = bkey_next(out);
 			}
 		}
diff --git a/drivers/md/bcache/btree_gc.c b/drivers/md/bcache/btree_gc.c
index 1edca3e37b5d..4055264b6bb4 100644
--- a/drivers/md/bcache/btree_gc.c
+++ b/drivers/md/bcache/btree_gc.c
@@ -544,9 +544,9 @@ static void bch_coalesce_nodes(struct btree *old_nodes[GC_MERGE_NODES],
 			/* n2 fits entirely in n1 */
 			n1->key.k.p = n1->data->max_key = n2->data->max_key;
 
-			memcpy(bset_bkey_last(s1),
-			       s2->start,
-			       le16_to_cpu(s2->u64s) * sizeof(u64));
+			memcpy_u64s(bset_bkey_last(s1),
+				    s2->start,
+				    le16_to_cpu(s2->u64s));
 			le16_add_cpu(&s1->u64s, le16_to_cpu(s2->u64s));
 
 			six_unlock_write(&n2->lock);
@@ -566,9 +566,8 @@ static void bch_coalesce_nodes(struct btree *old_nodes[GC_MERGE_NODES],
 				btree_type_successor(iter->btree_id,
 						     n1->data->max_key);
 
-			memcpy(bset_bkey_last(s1),
-			       s2->start,
-			       u64s * sizeof(u64));
+			memcpy_u64s(bset_bkey_last(s1),
+				    s2->start, u64s);
 			le16_add_cpu(&s1->u64s, u64s);
 
 			memmove(s2->start,
diff --git a/drivers/md/bcache/btree_io.c b/drivers/md/bcache/btree_io.c
index d28c768a1d8b..423fcc5869a8 100644
--- a/drivers/md/bcache/btree_io.c
+++ b/drivers/md/bcache/btree_io.c
@@ -233,8 +233,8 @@ static const char *validate_bset(struct cache_set *c, struct btree *b,
 					 "invalid bkey format %u", k->format);
 
 			i->u64s = cpu_to_le16(le16_to_cpu(i->u64s) - k->u64s);
-			memmove(k, bkey_next(k),
-				(void *) bset_bkey_last(i) - (void *) k);
+			memmove_u64s_down(k, bkey_next(k),
+					  (u64 *) bset_bkey_last(i) - (u64 *) k);
 			continue;
 		}
 
@@ -253,8 +253,8 @@ static const char *validate_bset(struct cache_set *c, struct btree *b,
 					 "invalid bkey %s", buf);
 
 			i->u64s = cpu_to_le16(le16_to_cpu(i->u64s) - k->u64s);
-			memmove(k, bkey_next(k),
-				(void *) bset_bkey_last(i) - (void *) k);
+			memmove_u64s_down(k, bkey_next(k),
+					  (u64 *) bset_bkey_last(i) - (u64 *) k);
 			continue;
 		}
 
diff --git a/drivers/md/bcache/btree_update.c b/drivers/md/bcache/btree_update.c
index 932940c69bbb..41884f8fc83e 100644
--- a/drivers/md/bcache/btree_update.c
+++ b/drivers/md/bcache/btree_update.c
@@ -1209,9 +1209,9 @@ static struct btree *__btree_split_node(struct btree_iter *iter, struct btree *n
 	BUG_ON(!set1->u64s);
 	BUG_ON(!set2->u64s);
 
-	memcpy(set2->start,
-	       bset_bkey_last(set1),
-	       le16_to_cpu(set2->u64s) * sizeof(u64));
+	memcpy_u64s(set2->start,
+		    bset_bkey_last(set1),
+		    le16_to_cpu(set2->u64s));
 
 	n1->keys.set->size = 0;
 	n1->keys.set->extra = BSET_AUX_TREE_NONE_VAL;
@@ -1323,9 +1323,9 @@ static void btree_split(struct btree *b, struct btree_iter *iter,
 		while (k != bset_bkey_last(i))
 			if (bkey_deleted(k)) {
 				i->u64s = cpu_to_le16(le16_to_cpu(i->u64s) - k->u64s);
-				memmove(k, bkey_next(k),
-					(void *) bset_bkey_last(i) -
-					(void *) k);
+				memmove_u64s_down(k, bkey_next(k),
+						  (u64 *) bset_bkey_last(i) -
+						  (u64 *) k);
 			} else
 				k = bkey_next(k);
 
diff --git a/drivers/md/bcache/extents.c b/drivers/md/bcache/extents.c
index af9a12c0c88e..a9a39ea67cf3 100644
--- a/drivers/md/bcache/extents.c
+++ b/drivers/md/bcache/extents.c
@@ -92,8 +92,7 @@ struct btree_nr_keys bch_key_sort_fix_overlapping(struct btree_keys *b,
 			struct bkey_packed *k =
 				__btree_node_offset_to_key(b, iter->data->k);
 
-			/* XXX: need better bkey_copy */
-			memcpy(out, k, bkey_bytes(k));
+			bkey_copy(out, k);
 			btree_keys_account_key_add(&nr, 0, out);
 			out = bkey_next(out);
 		}
@@ -298,8 +297,8 @@ next:
 		entry = next;
 		continue;
 drop:
-		memmove(crc, next,
-			(void *) extent_entry_last(e) - (void *) next);
+		memmove_u64s_down(crc, next,
+				  (u64 *) extent_entry_last(e) - (u64 *) next);
 		e.k->u64s -= crc_u64s;
 	}
 
@@ -782,7 +781,7 @@ static void extent_sort_append(struct btree_keys *b,
 		*prev = start;
 	}
 
-	bkey_copy((void *) *prev, &tmp.k);
+	bkey_copy(*prev, &tmp.k);
 }
 
 struct btree_nr_keys bch_extent_sort_fix_overlapping(struct btree_keys *b,
@@ -1795,9 +1794,9 @@ void bch_extent_entry_append(struct bkey_i_extent *e,
 	BUG_ON(bkey_val_u64s(&e->k) + extent_entry_u64s(entry) >
 	       BKEY_EXTENT_VAL_U64s_MAX);
 
-	memcpy(extent_entry_last(extent_i_to_s(e)),
-	       entry,
-	       extent_entry_bytes(entry));
+	memcpy_u64s(extent_entry_last(extent_i_to_s(e)),
+		    entry,
+		    extent_entry_u64s(entry));
 	e->k.u64s += extent_entry_u64s(entry);
 }
 
@@ -1938,18 +1937,19 @@ found:
 		crc_u64s = extent_entry_u64s(to_entry(src_crc));
 		u64s = crc_u64s + sizeof(*dst_ptr) / sizeof(u64);
 
-		memmove(p + u64s, p,
-			(void *) extent_entry_last(dst) - (void *) p);
+		memmove_u64s_up(p + u64s, p,
+				(u64 *) extent_entry_last(dst) - (u64 *) p);
 		set_bkey_val_u64s(dst.k, bkey_val_u64s(dst.k) + u64s);
 
-		memcpy(p, src_crc, crc_u64s * sizeof(u64));
-		memcpy(p + crc_u64s, src_ptr, sizeof(*src_ptr));
+		memcpy_u64s(p, src_crc, crc_u64s);
+		memcpy_u64s(p + crc_u64s, src_ptr,
+			    sizeof(*src_ptr) / sizeof(u64));
 	}
 
 	/* Sort done - now drop redundant crc entries: */
 	bch_extent_drop_redundant_crcs(dst);
 
-	memcpy(src.v, dst.v, bkey_val_bytes(dst.k));
+	memcpy_u64s(src.v, dst.v, bkey_val_u64s(dst.k));
 	set_bkey_val_u64s(src.k, bkey_val_u64s(dst.k));
 }
 
diff --git a/drivers/md/bcache/extents.h b/drivers/md/bcache/extents.h
index 451ae3df98da..ec8ff0078f08 100644
--- a/drivers/md/bcache/extents.h
+++ b/drivers/md/bcache/extents.h
@@ -481,7 +481,8 @@ static inline void __bch_extent_drop_ptr(struct bkey_s_extent e,
 	EBUG_ON(ptr < &e.v->start->ptr ||
 		ptr >= &extent_entry_last(e)->ptr);
 	EBUG_ON(ptr->type != 1 << BCH_EXTENT_ENTRY_ptr);
-	memmove(ptr, ptr + 1, (void *) extent_entry_last(e) - (void *) (ptr + 1));
+	memmove_u64s_down(ptr, ptr + 1,
+			  (u64 *) extent_entry_last(e) - (u64 *) (ptr + 1));
 	e.k->u64s -= sizeof(*ptr) / sizeof(u64);
 }
 
diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c
index e42483c11134..b3e2a762d365 100644
--- a/drivers/md/bcache/io.c
+++ b/drivers/md/bcache/io.c
@@ -331,7 +331,7 @@ static void bch_write_io_error(struct closure *cl)
 
 			set_bkey_val_u64s(&src->k, 0);
 			src->k.type = KEY_TYPE_DISCARD;
-			memmove(dst, src, bkey_bytes(&src->k));
+			bkey_copy(dst, src);
 
 			dst = bkey_next(dst);
 			src = n;
diff --git a/drivers/md/bcache/journal.c b/drivers/md/bcache/journal.c
index 35f55cfa6ef2..6c3cc983dd79 100644
--- a/drivers/md/bcache/journal.c
+++ b/drivers/md/bcache/journal.c
@@ -95,7 +95,7 @@ static inline void bch_journal_add_entry_at(struct journal_buf *buf,
 	entry->flags = 0;
 	SET_JOURNAL_ENTRY_TYPE(entry, type);
 
-	memcpy(entry->_data, data, u64s * sizeof(u64));
+	memcpy_u64s(entry->_data, data, u64s);
 }
 
 static inline void bch_journal_add_entry(struct journal_buf *buf,
@@ -903,9 +903,9 @@ const char *bch_journal_read(struct cache_set *c, struct list_head *list)
 		return NULL;
 	}
 
-	memcpy(c->journal.prio_buckets,
-	       prio_ptrs->_data,
-	       le16_to_cpu(prio_ptrs->u64s) * sizeof(u64));
+	memcpy_u64s(c->journal.prio_buckets,
+		    prio_ptrs->_data,
+		    le16_to_cpu(prio_ptrs->u64s));
 	c->journal.nr_prio_buckets = le16_to_cpu(prio_ptrs->u64s);
 
 	return NULL;
@@ -1815,9 +1815,9 @@ static void journal_write_compact(struct jset *jset)
 		    i->level	== prev->level &&
 		    JOURNAL_ENTRY_TYPE(i) == JOURNAL_ENTRY_TYPE(prev) &&
 		    JOURNAL_ENTRY_TYPE(i) == JOURNAL_ENTRY_BTREE_KEYS) {
-			memmove(jset_keys_next(prev),
-				i->_data,
-				u64s * sizeof(u64));
+			memmove_u64s_down(jset_keys_next(prev),
+					  i->_data,
+					  u64s);
 			le16_add_cpu(&prev->u64s, u64s);
 			continue;
 		}
@@ -1825,7 +1825,7 @@ static void journal_write_compact(struct jset *jset)
 		/* Couldn't merge, move i into new position (after prev): */
 		prev = prev ? jset_keys_next(prev) : jset->start;
 		if (i != prev)
-			memmove(prev, i, jset_u64s(u64s) * sizeof(u64));
+			memmove_u64s_down(prev, i, jset_u64s(u64s));
 	}
 
 	prev = prev ? jset_keys_next(prev) : jset->start;
diff --git a/drivers/md/bcache/keylist.c b/drivers/md/bcache/keylist.c
index 789e51ef6cee..adf5eebae0bb 100644
--- a/drivers/md/bcache/keylist.c
+++ b/drivers/md/bcache/keylist.c
@@ -21,7 +21,7 @@ int bch_keylist_realloc(struct keylist *l, u64 *inline_u64s,
 		return -ENOMEM;
 
 	if (!old_buf)
-		memcpy(new_keys, inline_u64s, sizeof(u64) * oldsize);
+		memcpy_u64s(new_keys, inline_u64s, oldsize);
 
 	l->keys_p = new_keys;
 	l->top_p = new_keys + oldsize;
@@ -37,9 +37,9 @@ void bch_keylist_add_in_order(struct keylist *l, struct bkey_i *insert)
 		if (bkey_cmp(insert->k.p, where->k.p) < 0)
 			break;
 
-	memmove((u64 *) where + insert->k.u64s,
-		where,
-		((void *) l->top) - ((void *) where));
+	memmove_u64s_up((u64 *) where + insert->k.u64s,
+			where,
+			((u64 *) l->top) - ((u64 *) where));
 
 	l->top_p += insert->k.u64s;
 	bkey_copy(where, insert);
@@ -49,7 +49,7 @@ void bch_keylist_pop_front(struct keylist *l)
 {
 	l->top_p -= bch_keylist_front(l)->k.u64s;
 
-	memmove(l->keys,
-		bkey_next(l->keys),
-		bch_keylist_bytes(l));
+	memmove_u64s_down(l->keys,
+			  bkey_next(l->keys),
+			  bch_keylist_u64s(l));
 }
diff --git a/drivers/md/bcache/move.c b/drivers/md/bcache/move.c
index 19308ff3d05a..9afc1d3271fb 100644
--- a/drivers/md/bcache/move.c
+++ b/drivers/md/bcache/move.c
@@ -91,9 +91,9 @@ static int bch_migrate_index_update(struct bch_write_op *op)
 			if (m->move)
 				__bch_extent_drop_ptr(e, ptr);
 
-			memcpy(extent_entry_last(e),
-			       &insert->v,
-			       bkey_val_bytes(&insert->k));
+			memcpy_u64s(extent_entry_last(e),
+				    &insert->v,
+				    bkey_val_u64s(&insert->k));
 			e.k->u64s += bkey_val_u64s(&insert->k);
 
 			bch_extent_narrow_crcs(e);
diff --git a/include/uapi/linux/bcache.h b/include/uapi/linux/bcache.h
index 01323c95024d..53b2889259a3 100644
--- a/include/uapi/linux/bcache.h
+++ b/include/uapi/linux/bcache.h
@@ -225,11 +225,6 @@ static inline void bkey_init(struct bkey *k)
 
 #define bkey_bytes(_k)		((_k)->u64s * sizeof(__u64))
 
-static inline void bkey_copy(struct bkey_i *dst, const struct bkey_i *src)
-{
-	memcpy(dst, src, bkey_bytes(&src->k));
-}
-
 #define __BKEY_PADDED(key, pad)					\
 	struct { struct bkey_i key; __u64 key ## _pad[pad]; }