From 6cc25e4b7c819c183a21d6b9a4bcec84229131d1 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 23 Apr 2025 17:20:32 -0700 Subject: arm/crc: drop "glue" from filenames The use of the term "glue" in filenames is a Crypto API-ism that rarely shows up elsewhere in lib/ or arch/*/lib/. I think adopting it there was a mistake. The library just uses standard functions, so the amount of code that could be considered "glue" is quite small. And while often the C functions just wrap the assembly functions, there are also cases like crc32c_arch() in arch/x86/lib/crc32-glue.c that blur the line by in-lining the actual implementation into the C function. That's not "glue code", but rather the actual code. Therefore, let's drop "glue" from the filenames and instead use e.g. crc32.c instead of crc32-glue.c. Reviewed-by: "Martin K. Petersen" Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250424002038.179114-2-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm/lib/Makefile | 4 +- arch/arm/lib/crc-t10dif-glue.c | 72 ------------------------ arch/arm/lib/crc-t10dif.c | 72 ++++++++++++++++++++++++ arch/arm/lib/crc32-glue.c | 123 ----------------------------------------- arch/arm/lib/crc32.c | 123 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 197 insertions(+), 197 deletions(-) delete mode 100644 arch/arm/lib/crc-t10dif-glue.c create mode 100644 arch/arm/lib/crc-t10dif.c delete mode 100644 arch/arm/lib/crc32-glue.c create mode 100644 arch/arm/lib/crc32.c diff --git a/arch/arm/lib/Makefile b/arch/arm/lib/Makefile index 007874320937..d05dd672bcd9 100644 --- a/arch/arm/lib/Makefile +++ b/arch/arm/lib/Makefile @@ -47,7 +47,7 @@ endif obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o obj-$(CONFIG_CRC32_ARCH) += crc32-arm.o -crc32-arm-y := crc32-glue.o crc32-core.o +crc32-arm-y := crc32.o crc32-core.o obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-arm.o -crc-t10dif-arm-y := crc-t10dif-glue.o crc-t10dif-core.o +crc-t10dif-arm-y := crc-t10dif.o crc-t10dif-core.o diff --git a/arch/arm/lib/crc-t10dif-glue.c b/arch/arm/lib/crc-t10dif-glue.c deleted file mode 100644 index 382437094bdd..000000000000 --- a/arch/arm/lib/crc-t10dif-glue.c +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions - * - * Copyright (C) 2016 Linaro Ltd - */ - -#include -#include -#include -#include -#include - -#include - -#include -#include - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); -static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); - -#define CRC_T10DIF_PMULL_CHUNK_SIZE 16U - -asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len); -asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len, - u8 out[16]); - -u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length) -{ - if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) { - if (static_branch_likely(&have_pmull)) { - if (crypto_simd_usable()) { - kernel_neon_begin(); - crc = crc_t10dif_pmull64(crc, data, length); - kernel_neon_end(); - return crc; - } - } else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && - static_branch_likely(&have_neon) && - crypto_simd_usable()) { - u8 buf[16] __aligned(16); - - kernel_neon_begin(); - crc_t10dif_pmull8(crc, data, length, buf); - kernel_neon_end(); - - return crc_t10dif_generic(0, buf, sizeof(buf)); - } - } - return crc_t10dif_generic(crc, data, length); -} -EXPORT_SYMBOL(crc_t10dif_arch); - -static int __init crc_t10dif_arm_init(void) -{ - if (elf_hwcap & HWCAP_NEON) { - static_branch_enable(&have_neon); - if (elf_hwcap2 & HWCAP2_PMULL) - static_branch_enable(&have_pmull); - } - return 0; -} -arch_initcall(crc_t10dif_arm_init); - -static void __exit crc_t10dif_arm_exit(void) -{ -} -module_exit(crc_t10dif_arm_exit); - -MODULE_AUTHOR("Ard Biesheuvel "); -MODULE_DESCRIPTION("Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions"); -MODULE_LICENSE("GPL v2"); diff --git a/arch/arm/lib/crc-t10dif.c b/arch/arm/lib/crc-t10dif.c new file mode 100644 index 000000000000..382437094bdd --- /dev/null +++ b/arch/arm/lib/crc-t10dif.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions + * + * Copyright (C) 2016 Linaro Ltd + */ + +#include +#include +#include +#include +#include + +#include + +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); + +#define CRC_T10DIF_PMULL_CHUNK_SIZE 16U + +asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len); +asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len, + u8 out[16]); + +u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length) +{ + if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) { + if (static_branch_likely(&have_pmull)) { + if (crypto_simd_usable()) { + kernel_neon_begin(); + crc = crc_t10dif_pmull64(crc, data, length); + kernel_neon_end(); + return crc; + } + } else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && + static_branch_likely(&have_neon) && + crypto_simd_usable()) { + u8 buf[16] __aligned(16); + + kernel_neon_begin(); + crc_t10dif_pmull8(crc, data, length, buf); + kernel_neon_end(); + + return crc_t10dif_generic(0, buf, sizeof(buf)); + } + } + return crc_t10dif_generic(crc, data, length); +} +EXPORT_SYMBOL(crc_t10dif_arch); + +static int __init crc_t10dif_arm_init(void) +{ + if (elf_hwcap & HWCAP_NEON) { + static_branch_enable(&have_neon); + if (elf_hwcap2 & HWCAP2_PMULL) + static_branch_enable(&have_pmull); + } + return 0; +} +arch_initcall(crc_t10dif_arm_init); + +static void __exit crc_t10dif_arm_exit(void) +{ +} +module_exit(crc_t10dif_arm_exit); + +MODULE_AUTHOR("Ard Biesheuvel "); +MODULE_DESCRIPTION("Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions"); +MODULE_LICENSE("GPL v2"); diff --git a/arch/arm/lib/crc32-glue.c b/arch/arm/lib/crc32-glue.c deleted file mode 100644 index 7ef7db9c0de7..000000000000 --- a/arch/arm/lib/crc32-glue.c +++ /dev/null @@ -1,123 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions - * - * Copyright (C) 2016 Linaro Ltd - */ - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32); -static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); - -#define PMULL_MIN_LEN 64 /* min size of buffer for pmull functions */ - -asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc); -asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len); - -asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc); -asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len); - -static u32 crc32_le_scalar(u32 crc, const u8 *p, size_t len) -{ - if (static_branch_likely(&have_crc32)) - return crc32_armv8_le(crc, p, len); - return crc32_le_base(crc, p, len); -} - -u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) -{ - if (len >= PMULL_MIN_LEN + 15 && - static_branch_likely(&have_pmull) && crypto_simd_usable()) { - size_t n = -(uintptr_t)p & 15; - - /* align p to 16-byte boundary */ - if (n) { - crc = crc32_le_scalar(crc, p, n); - p += n; - len -= n; - } - n = round_down(len, 16); - kernel_neon_begin(); - crc = crc32_pmull_le(p, n, crc); - kernel_neon_end(); - p += n; - len -= n; - } - return crc32_le_scalar(crc, p, len); -} -EXPORT_SYMBOL(crc32_le_arch); - -static u32 crc32c_scalar(u32 crc, const u8 *p, size_t len) -{ - if (static_branch_likely(&have_crc32)) - return crc32c_armv8_le(crc, p, len); - return crc32c_base(crc, p, len); -} - -u32 crc32c_arch(u32 crc, const u8 *p, size_t len) -{ - if (len >= PMULL_MIN_LEN + 15 && - static_branch_likely(&have_pmull) && crypto_simd_usable()) { - size_t n = -(uintptr_t)p & 15; - - /* align p to 16-byte boundary */ - if (n) { - crc = crc32c_scalar(crc, p, n); - p += n; - len -= n; - } - n = round_down(len, 16); - kernel_neon_begin(); - crc = crc32c_pmull_le(p, n, crc); - kernel_neon_end(); - p += n; - len -= n; - } - return crc32c_scalar(crc, p, len); -} -EXPORT_SYMBOL(crc32c_arch); - -u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) -{ - return crc32_be_base(crc, p, len); -} -EXPORT_SYMBOL(crc32_be_arch); - -static int __init crc32_arm_init(void) -{ - if (elf_hwcap2 & HWCAP2_CRC32) - static_branch_enable(&have_crc32); - if (elf_hwcap2 & HWCAP2_PMULL) - static_branch_enable(&have_pmull); - return 0; -} -arch_initcall(crc32_arm_init); - -static void __exit crc32_arm_exit(void) -{ -} -module_exit(crc32_arm_exit); - -u32 crc32_optimizations(void) -{ - if (elf_hwcap2 & (HWCAP2_CRC32 | HWCAP2_PMULL)) - return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION; - return 0; -} -EXPORT_SYMBOL(crc32_optimizations); - -MODULE_AUTHOR("Ard Biesheuvel "); -MODULE_DESCRIPTION("Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions"); -MODULE_LICENSE("GPL v2"); diff --git a/arch/arm/lib/crc32.c b/arch/arm/lib/crc32.c new file mode 100644 index 000000000000..7ef7db9c0de7 --- /dev/null +++ b/arch/arm/lib/crc32.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions + * + * Copyright (C) 2016 Linaro Ltd + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); + +#define PMULL_MIN_LEN 64 /* min size of buffer for pmull functions */ + +asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc); +asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len); + +asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc); +asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len); + +static u32 crc32_le_scalar(u32 crc, const u8 *p, size_t len) +{ + if (static_branch_likely(&have_crc32)) + return crc32_armv8_le(crc, p, len); + return crc32_le_base(crc, p, len); +} + +u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) +{ + if (len >= PMULL_MIN_LEN + 15 && + static_branch_likely(&have_pmull) && crypto_simd_usable()) { + size_t n = -(uintptr_t)p & 15; + + /* align p to 16-byte boundary */ + if (n) { + crc = crc32_le_scalar(crc, p, n); + p += n; + len -= n; + } + n = round_down(len, 16); + kernel_neon_begin(); + crc = crc32_pmull_le(p, n, crc); + kernel_neon_end(); + p += n; + len -= n; + } + return crc32_le_scalar(crc, p, len); +} +EXPORT_SYMBOL(crc32_le_arch); + +static u32 crc32c_scalar(u32 crc, const u8 *p, size_t len) +{ + if (static_branch_likely(&have_crc32)) + return crc32c_armv8_le(crc, p, len); + return crc32c_base(crc, p, len); +} + +u32 crc32c_arch(u32 crc, const u8 *p, size_t len) +{ + if (len >= PMULL_MIN_LEN + 15 && + static_branch_likely(&have_pmull) && crypto_simd_usable()) { + size_t n = -(uintptr_t)p & 15; + + /* align p to 16-byte boundary */ + if (n) { + crc = crc32c_scalar(crc, p, n); + p += n; + len -= n; + } + n = round_down(len, 16); + kernel_neon_begin(); + crc = crc32c_pmull_le(p, n, crc); + kernel_neon_end(); + p += n; + len -= n; + } + return crc32c_scalar(crc, p, len); +} +EXPORT_SYMBOL(crc32c_arch); + +u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) +{ + return crc32_be_base(crc, p, len); +} +EXPORT_SYMBOL(crc32_be_arch); + +static int __init crc32_arm_init(void) +{ + if (elf_hwcap2 & HWCAP2_CRC32) + static_branch_enable(&have_crc32); + if (elf_hwcap2 & HWCAP2_PMULL) + static_branch_enable(&have_pmull); + return 0; +} +arch_initcall(crc32_arm_init); + +static void __exit crc32_arm_exit(void) +{ +} +module_exit(crc32_arm_exit); + +u32 crc32_optimizations(void) +{ + if (elf_hwcap2 & (HWCAP2_CRC32 | HWCAP2_PMULL)) + return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION; + return 0; +} +EXPORT_SYMBOL(crc32_optimizations); + +MODULE_AUTHOR("Ard Biesheuvel "); +MODULE_DESCRIPTION("Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions"); +MODULE_LICENSE("GPL v2"); -- cgit v1.2.3