s390/uaccess: Separate key uaccess functions

Implement separate raw_copy_to_user_key() and raw_copy_from_user_key()
functions, which allows to remove the open-coded operand access control
handling from the normal raw_copy_to_user() / raw_copy_from_user()
functions - they are simplified to use immediate instructions to load
hard-coded operand access control values into register zero, which saves
one instruction.

Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>

2 files changed, 92 insertions, 52 deletions

diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index f9ace938743c..87833f50fa52 100644
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -22,28 +22,6 @@
 
 void debug_user_asce(int exit);
 
-union oac {
-	unsigned int val;
-	struct {
-		struct {
-			unsigned short key : 4;
-			unsigned short	   : 4;
-			unsigned short as  : 2;
-			unsigned short	   : 4;
-			unsigned short k   : 1;
-			unsigned short a   : 1;
-		} oac1;
-		struct {
-			unsigned short key : 4;
-			unsigned short	   : 4;
-			unsigned short as  : 2;
-			unsigned short	   : 4;
-			unsigned short k   : 1;
-			unsigned short a   : 1;
-		} oac2;
-	};
-};
-
 #ifdef CONFIG_KMSAN
 #define uaccess_kmsan_or_inline noinline __maybe_unused __no_sanitize_memory
 #else
@@ -51,28 +29,22 @@ union oac {
 #endif
 
 static uaccess_kmsan_or_inline __must_check unsigned long
-raw_copy_from_user_key(void *to, const void __user *from, unsigned long size, unsigned long key)
+raw_copy_from_user(void *to, const void __user *from, unsigned long size)
 {
 	unsigned long osize;
-	union oac spec = {
-		.oac2.key = key,
-		.oac2.as = PSW_BITS_AS_SECONDARY,
-		.oac2.k = 1,
-		.oac2.a = 1,
-	};
 	int cc;
 
 	while (1) {
 		osize = size;
 		asm_inline volatile(
-			"	lr	%%r0,%[spec]\n"
+			"	lhi	%%r0,%[spec]\n"
 			"0:	mvcos	%[to],%[from],%[size]\n"
 			"1:	nopr	%%r7\n"
 			CC_IPM(cc)
 			EX_TABLE_UA_MVCOS_FROM(0b, 0b)
 			EX_TABLE_UA_MVCOS_FROM(1b, 0b)
 			: CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char *)to)
-			: [spec] "d" (spec.val), [from] "Q" (*(const char __user *)from)
+			: [spec] "I" (0x81), [from] "Q" (*(const char __user *)from)
 			: CC_CLOBBER_LIST("memory", "0"));
 		if (likely(CC_TRANSFORM(cc) == 0))
 			return osize - size;
@@ -82,35 +54,23 @@ raw_copy_from_user_key(void *to, const void __user *from, unsigned long size, un
 	}
 }
 
-static __always_inline __must_check unsigned long
-raw_copy_from_user(void *to, const void __user *from, unsigned long n)
-{
-	return raw_copy_from_user_key(to, from, n, 0);
-}
-
 static uaccess_kmsan_or_inline __must_check unsigned long
-raw_copy_to_user_key(void __user *to, const void *from, unsigned long size, unsigned long key)
+raw_copy_to_user(void __user *to, const void *from, unsigned long size)
 {
 	unsigned long osize;
-	union oac spec = {
-		.oac1.key = key,
-		.oac1.as = PSW_BITS_AS_SECONDARY,
-		.oac1.k = 1,
-		.oac1.a = 1,
-	};
 	int cc;
 
 	while (1) {
 		osize = size;
 		asm_inline volatile(
-			"	lr	%%r0,%[spec]\n"
+			"	llilh	%%r0,%[spec]\n"
 			"0:	mvcos	%[to],%[from],%[size]\n"
 			"1:	nopr	%%r7\n"
 			CC_IPM(cc)
 			EX_TABLE_UA_MVCOS_TO(0b, 0b)
 			EX_TABLE_UA_MVCOS_TO(1b, 0b)
 			: CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to)
-			: [spec] "d" (spec.val), [from] "Q" (*(const char *)from)
+			: [spec] "I" (0x81), [from] "Q" (*(const char *)from)
 			: CC_CLOBBER_LIST("memory", "0"));
 		if (likely(CC_TRANSFORM(cc) == 0))
 			return osize - size;
@@ -120,12 +80,6 @@ raw_copy_to_user_key(void __user *to, const void *from, unsigned long size, unsi
 	}
 }
 
-static __always_inline __must_check unsigned long
-raw_copy_to_user(void __user *to, const void *from, unsigned long n)
-{
-	return raw_copy_to_user_key(to, from, n, 0);
-}
-
 unsigned long __must_check
 _copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned long key);
 
diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c
index f977b7c37efc..2c85068aebfe 100644
--- a/arch/s390/lib/uaccess.c
+++ b/arch/s390/lib/uaccess.c
@@ -31,6 +31,60 @@ void debug_user_asce(int exit)
 }
 #endif /*CONFIG_DEBUG_ENTRY */
 
+union oac {
+	unsigned int val;
+	struct {
+		struct {
+			unsigned short key : 4;
+			unsigned short	   : 4;
+			unsigned short as  : 2;
+			unsigned short	   : 4;
+			unsigned short k   : 1;
+			unsigned short a   : 1;
+		} oac1;
+		struct {
+			unsigned short key : 4;
+			unsigned short	   : 4;
+			unsigned short as  : 2;
+			unsigned short	   : 4;
+			unsigned short k   : 1;
+			unsigned short a   : 1;
+		} oac2;
+	};
+};
+
+static uaccess_kmsan_or_inline __must_check unsigned long
+raw_copy_from_user_key(void *to, const void __user *from, unsigned long size, unsigned long key)
+{
+	unsigned long osize;
+	union oac spec = {
+		.oac2.key = key,
+		.oac2.as = PSW_BITS_AS_SECONDARY,
+		.oac2.k = 1,
+		.oac2.a = 1,
+	};
+	int cc;
+
+	while (1) {
+		osize = size;
+		asm_inline volatile(
+			"	lr	%%r0,%[spec]\n"
+			"0:	mvcos	%[to],%[from],%[size]\n"
+			"1:	nopr	%%r7\n"
+			CC_IPM(cc)
+			EX_TABLE_UA_MVCOS_FROM(0b, 0b)
+			EX_TABLE_UA_MVCOS_FROM(1b, 0b)
+			: CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char *)to)
+			: [spec] "d" (spec.val), [from] "Q" (*(const char __user *)from)
+			: CC_CLOBBER_LIST("memory", "0"));
+		if (CC_TRANSFORM(cc) == 0)
+			return osize - size;
+		size -= 4096;
+		to += 4096;
+		from += 4096;
+	}
+}
+
 unsigned long _copy_from_user_key(void *to, const void __user *from,
 				  unsigned long n, unsigned long key)
 {
@@ -48,6 +102,38 @@ unsigned long _copy_from_user_key(void *to, const void __user *from,
 }
 EXPORT_SYMBOL(_copy_from_user_key);
 
+static uaccess_kmsan_or_inline __must_check unsigned long
+raw_copy_to_user_key(void __user *to, const void *from, unsigned long size, unsigned long key)
+{
+	unsigned long osize;
+	union oac spec = {
+		.oac1.key = key,
+		.oac1.as = PSW_BITS_AS_SECONDARY,
+		.oac1.k = 1,
+		.oac1.a = 1,
+	};
+	int cc;
+
+	while (1) {
+		osize = size;
+		asm_inline volatile(
+			"	lr	%%r0,%[spec]\n"
+			"0:	mvcos	%[to],%[from],%[size]\n"
+			"1:	nopr	%%r7\n"
+			CC_IPM(cc)
+			EX_TABLE_UA_MVCOS_TO(0b, 0b)
+			EX_TABLE_UA_MVCOS_TO(1b, 0b)
+			: CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to)
+			: [spec] "d" (spec.val), [from] "Q" (*(const char *)from)
+			: CC_CLOBBER_LIST("memory", "0"));
+		if (CC_TRANSFORM(cc) == 0)
+			return osize - size;
+		size -= 4096;
+		to += 4096;
+		from += 4096;
+	}
+}
+
 unsigned long _copy_to_user_key(void __user *to, const void *from,
 				unsigned long n, unsigned long key)
 {