diff options
265 files changed, 5263 insertions, 12701 deletions
diff --git a/Documentation/devicetree/bindings/crypto/qcom-qce.yaml b/Documentation/devicetree/bindings/crypto/qcom-qce.yaml index 3f35122f7873..e009cb712fb8 100644 --- a/Documentation/devicetree/bindings/crypto/qcom-qce.yaml +++ b/Documentation/devicetree/bindings/crypto/qcom-qce.yaml @@ -45,6 +45,7 @@ properties: - items: - enum: + - qcom,qcs615-qce - qcom,qcs8300-qce - qcom,sa8775p-qce - qcom,sc7280-qce diff --git a/MAINTAINERS b/MAINTAINERS index 96b827049501..a2604c13f11b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14267,8 +14267,6 @@ F: drivers/gpu/drm/armada/ F: include/uapi/drm/armada_drm.h MARVELL CRYPTO DRIVER -M: Boris Brezillon <bbrezillon@kernel.org> -M: Arnaud Ebalard <arno@natisbad.org> M: Srujana Challa <schalla@marvell.com> L: linux-crypto@vger.kernel.org S: Maintained diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index 23e4ea067ddb..3530e7c80793 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -48,14 +48,8 @@ config CRYPTO_NHPOLY1305_NEON config CRYPTO_POLY1305_ARM tristate - select CRYPTO_HASH select CRYPTO_ARCH_HAVE_LIB_POLY1305 default CRYPTO_LIB_POLY1305_INTERNAL - help - Poly1305 authenticator algorithm (RFC7539) - - Architecture: arm optionally using - - NEON (Advanced SIMD) extensions config CRYPTO_BLAKE2S_ARM bool "Hash functions: BLAKE2s" @@ -172,7 +166,6 @@ config CRYPTO_AES_ARM_BS select CRYPTO_AES_ARM select CRYPTO_SKCIPHER select CRYPTO_LIB_AES - select CRYPTO_SIMD help Length-preserving ciphers: AES cipher algorithms (FIPS-197) with block cipher modes: @@ -200,7 +193,6 @@ config CRYPTO_AES_ARM_CE depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_LIB_AES - select CRYPTO_SIMD help Length-preserving ciphers: AES cipher algorithms (FIPS-197) with block cipher modes: @@ -216,15 +208,8 @@ config CRYPTO_AES_ARM_CE config CRYPTO_CHACHA20_NEON tristate - select CRYPTO_SKCIPHER select CRYPTO_ARCH_HAVE_LIB_CHACHA default CRYPTO_LIB_CHACHA_INTERNAL - help - Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 - stream cipher algorithms - - Architecture: arm using: - - NEON (Advanced SIMD) extensions endmenu diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c index 1cf61f51e766..00591895d540 100644 --- a/arch/arm/crypto/aes-ce-glue.c +++ b/arch/arm/crypto/aes-ce-glue.c @@ -10,8 +10,6 @@ #include <asm/simd.h> #include <linux/unaligned.h> #include <crypto/aes.h> -#include <crypto/ctr.h> -#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <linux/cpufeature.h> @@ -418,29 +416,6 @@ static int ctr_encrypt(struct skcipher_request *req) return err; } -static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) -{ - struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); - unsigned long flags; - - /* - * Temporarily disable interrupts to avoid races where - * cachelines are evicted when the CPU is interrupted - * to do something else. - */ - local_irq_save(flags); - aes_encrypt(ctx, dst, src); - local_irq_restore(flags); -} - -static int ctr_encrypt_sync(struct skcipher_request *req) -{ - if (!crypto_simd_usable()) - return crypto_ctr_encrypt_walk(req, ctr_encrypt_one); - - return ctr_encrypt(req); -} - static int xts_encrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -586,10 +561,9 @@ static int xts_decrypt(struct skcipher_request *req) } static struct skcipher_alg aes_algs[] = { { - .base.cra_name = "__ecb(aes)", - .base.cra_driver_name = "__ecb-aes-ce", + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-ce", .base.cra_priority = 300, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct crypto_aes_ctx), .base.cra_module = THIS_MODULE, @@ -600,10 +574,9 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(aes)", - .base.cra_driver_name = "__cbc-aes-ce", + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-ce", .base.cra_priority = 300, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct crypto_aes_ctx), .base.cra_module = THIS_MODULE, @@ -615,10 +588,9 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, }, { - .base.cra_name = "__cts(cbc(aes))", - .base.cra_driver_name = "__cts-cbc-aes-ce", + .base.cra_name = "cts(cbc(aes))", + .base.cra_driver_name = "cts-cbc-aes-ce", .base.cra_priority = 300, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct crypto_aes_ctx), .base.cra_module = THIS_MODULE, @@ -631,10 +603,9 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = cts_cbc_encrypt, .decrypt = cts_cbc_decrypt, }, { - .base.cra_name = "__ctr(aes)", - .base.cra_driver_name = "__ctr-aes-ce", + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-ce", .base.cra_priority = 300, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct crypto_aes_ctx), .base.cra_module = THIS_MODULE, @@ -647,25 +618,9 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = ctr_encrypt, .decrypt = ctr_encrypt, }, { - .base.cra_name = "ctr(aes)", - .base.cra_driver_name = "ctr-aes-ce-sync", - .base.cra_priority = 300 - 1, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct crypto_aes_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .chunksize = AES_BLOCK_SIZE, - .setkey = ce_aes_setkey, - .encrypt = ctr_encrypt_sync, - .decrypt = ctr_encrypt_sync, -}, { - .base.cra_name = "__xts(aes)", - .base.cra_driver_name = "__xts-aes-ce", + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "xts-aes-ce", .base.cra_priority = 300, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), .base.cra_module = THIS_MODULE, @@ -679,51 +634,14 @@ static struct skcipher_alg aes_algs[] = { { .decrypt = xts_decrypt, } }; -static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; - static void aes_exit(void) { - int i; - - for (i = 0; i < ARRAY_SIZE(aes_simd_algs) && aes_simd_algs[i]; i++) - simd_skcipher_free(aes_simd_algs[i]); - crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } static int __init aes_init(void) { - struct simd_skcipher_alg *simd; - const char *basename; - const char *algname; - const char *drvname; - int err; - int i; - - err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); - if (err) - return err; - - for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { - if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL)) - continue; - - algname = aes_algs[i].base.cra_name + 2; - drvname = aes_algs[i].base.cra_driver_name + 2; - basename = aes_algs[i].base.cra_driver_name; - simd = simd_skcipher_create_compat(aes_algs + i, algname, drvname, basename); - err = PTR_ERR(simd); - if (IS_ERR(simd)) - goto unregister_simds; - - aes_simd_algs[i] = simd; - } - - return 0; - -unregister_simds: - aes_exit(); - return err; + return crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } module_cpu_feature_match(AES, aes_init); diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c index f6be80b5938b..95418df97fb4 100644 --- a/arch/arm/crypto/aes-neonbs-glue.c +++ b/arch/arm/crypto/aes-neonbs-glue.c @@ -8,8 +8,6 @@ #include <asm/neon.h> #include <asm/simd.h> #include <crypto/aes.h> -#include <crypto/ctr.h> -#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <crypto/xts.h> @@ -59,11 +57,6 @@ struct aesbs_xts_ctx { struct crypto_aes_ctx tweak_key; }; -struct aesbs_ctr_ctx { - struct aesbs_ctx key; /* must be first member */ - struct crypto_aes_ctx fallback; -}; - static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key, unsigned int key_len) { @@ -200,25 +193,6 @@ static int cbc_decrypt(struct skcipher_request *req) return err; } -static int aesbs_ctr_setkey_sync(struct crypto_skcipher *tfm, const u8 *in_key, - unsigned int key_len) -{ - struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); - int err; - - err = aes_expandkey(&ctx->fallback, in_key, key_len); - if (err) - return err; - - ctx->key.rounds = 6 + key_len / 4; - - kernel_neon_begin(); - aesbs_convert_key(ctx->key.rk, ctx->fallback.key_enc, ctx->key.rounds); - kernel_neon_end(); - - return 0; -} - static int ctr_encrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -254,21 +228,6 @@ static int ctr_encrypt(struct skcipher_request *req) return err; } -static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) -{ - struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); - - __aes_arm_encrypt(ctx->fallback.key_enc, ctx->key.rounds, src, dst); -} - -static int ctr_encrypt_sync(struct skcipher_request *req) -{ - if (!crypto_simd_usable()) - return crypto_ctr_encrypt_walk(req, ctr_encrypt_one); - - return ctr_encrypt(req); -} - static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key, unsigned int key_len) { @@ -374,13 +333,12 @@ static int xts_decrypt(struct skcipher_request *req) } static struct skcipher_alg aes_algs[] = { { - .base.cra_name = "__ecb(aes)", - .base.cra_driver_name = "__ecb-aes-neonbs", + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aesbs_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -389,13 +347,12 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(aes)", - .base.cra_driver_name = "__cbc-aes-neonbs", + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -405,13 +362,12 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, }, { - .base.cra_name = "__ctr(aes)", - .base.cra_driver_name = "__ctr-aes-neonbs", + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct aesbs_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -422,29 +378,12 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = ctr_encrypt, .decrypt = ctr_encrypt, }, { - .base.cra_name = "ctr(aes)", - .base.cra_driver_name = "ctr-aes-neonbs-sync", - .base.cra_priority = 250 - 1, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct aesbs_ctr_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .chunksize = AES_BLOCK_SIZE, - .walksize = 8 * AES_BLOCK_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = aesbs_ctr_setkey_sync, - .encrypt = ctr_encrypt_sync, - .decrypt = ctr_encrypt_sync, -}, { - .base.cra_name = "__xts(aes)", - .base.cra_driver_name = "__xts-aes-neonbs", + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "xts-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, @@ -455,54 +394,17 @@ static struct skcipher_alg aes_algs[] = { { .decrypt = xts_decrypt, } }; -static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; - static void aes_exit(void) { - int i; - - for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++) - if (aes_simd_algs[i]) - simd_skcipher_free(aes_simd_algs[i]); - crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } static int __init aes_init(void) { - struct simd_skcipher_alg *simd; - const char *basename; - const char *algname; - const char *drvname; - int err; - int i; - if (!(elf_hwcap & HWCAP_NEON)) return -ENODEV; - err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); - if (err) - return err; - - for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { - if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL)) - continue; - - algname = aes_algs[i].base.cra_name + 2; - drvname = aes_algs[i].base.cra_driver_name + 2; - basename = aes_algs[i].base.cra_driver_name; - simd = simd_skcipher_create_compat(aes_algs + i, algname, drvname, basename); - err = PTR_ERR(simd); - if (IS_ERR(simd)) - goto unregister_simds; - - aes_simd_algs[i] = simd; - } - return 0; - -unregister_simds: - aes_exit(); - return err; + return crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } late_initcall(aes_init); diff --git a/arch/arm/crypto/blake2b-neon-glue.c b/arch/arm/crypto/blake2b-neon-glue.c index 4b59d027ba4a..7ae4ba0afe06 100644 --- a/arch/arm/crypto/blake2b-neon-glue.c +++ b/arch/arm/crypto/blake2b-neon-glue.c @@ -7,7 +7,6 @@ #include <crypto/internal/blake2b.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> #include <linux/module.h> #include <linux/sizes.h> @@ -21,11 +20,6 @@ asmlinkage void blake2b_compress_neon(struct blake2b_state *state, static void blake2b_compress_arch(struct blake2b_state *state, const u8 *block, size_t nblocks, u32 inc) { - if (!crypto_simd_usable()) { - blake2b_compress_generic(state, block, nblocks, inc); - return; - } - do { const size_t blocks = min_t(size_t, nblocks, SZ_4K / BLAKE2B_BLOCK_SIZE); @@ -42,12 +36,14 @@ static void blake2b_compress_arch(struct blake2b_state *state, static int crypto_blake2b_update_neon(struct shash_desc *desc, const u8 *in, unsigned int inlen) { - return crypto_blake2b_update(desc, in, inlen, blake2b_compress_arch); + return crypto_blake2b_update_bo(desc, in, inlen, blake2b_compress_arch); } -static int crypto_blake2b_final_neon(struct shash_desc *desc, u8 *out) +static int crypto_blake2b_finup_neon(struct shash_desc *desc, const u8 *in, + unsigned int inlen, u8 *out) { - return crypto_blake2b_final(desc, out, blake2b_compress_arch); + return crypto_blake2b_finup(desc, in, inlen, out, + blake2b_compress_arch); } #define BLAKE2B_ALG(name, driver_name, digest_size) \ @@ -55,7 +51,8 @@ static int crypto_blake2b_final_neon(struct shash_desc *desc, u8 *out) .base.cra_name = name, \ .base.cra_driver_name = driver_name, \ .base.cra_priority = 200, \ - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY | \ + CRYPTO_AHASH_ALG_BLOCK_ONLY, \ .base.cra_blocksize = BLAKE2B_BLOCK_SIZE, \ .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), \ .base.cra_module = THIS_MODULE, \ @@ -63,8 +60,9 @@ static int crypto_blake2b_final_neon(struct shash_desc *desc, u8 *out) .setkey = crypto_blake2b_setkey, \ .init = crypto_blake2b_init, \ .update = crypto_blake2b_update_neon, \ - .final = crypto_blake2b_final_neon, \ + .finup = crypto_blake2b_finup_neon, \ .descsize = sizeof(struct blake2b_state), \ + .statesize = BLAKE2B_STATE_SIZE, \ } static struct shash_alg blake2b_neon_algs[] = { diff --git a/arch/arm/crypto/chacha-glue.c b/arch/arm/crypto/chacha-glue.c index 50e635512046..12afb40cf1ff 100644 --- a/arch/arm/crypto/chacha-glue.c +++ b/arch/arm/crypto/chacha-glue.c @@ -1,16 +1,13 @@ // SPDX-License-Identifier: GPL-2.0 /* - * ARM NEON accelerated ChaCha and XChaCha stream ciphers, - * including ChaCha20 (RFC7539) + * ChaCha and HChaCha functions (ARM optimized) * * Copyright (C) 2016-2019 Linaro, Ltd. <ard.biesheuvel@linaro.org> * Copyright (C) 2015 Martin Willi */ -#include <crypto/algapi.h> -#include <crypto/internal/chacha.h> +#include <crypto/chacha.h> #include <crypto/internal/simd.h> -#include <crypto/internal/skcipher.h> #include <linux/jump_label.h> #include <linux/kernel.h> #include <linux/module.h> @@ -100,206 +97,16 @@ void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, } EXPORT_SYMBOL(chacha_crypt_arch); -static int chacha_stream_xor(struct skcipher_request *req, - const struct chacha_ctx *ctx, const u8 *iv, - bool neon) +bool chacha_is_arch_optimized(void) { - struct skcipher_walk walk; - u32 state[16]; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - chacha_init(state, ctx->key, iv); - - while (walk.nbytes > 0) { - unsigned int nbytes = walk.nbytes; - - if (nbytes < walk.total) - nbytes = round_down(nbytes, walk.stride); - - if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon) { - chacha_doarm(walk.dst.virt.addr, walk.src.virt.addr, - nbytes, state, ctx->nrounds); - state[12] += DIV_ROUND_UP(nbytes, CHACHA_BLOCK_SIZE); - } else { - kernel_neon_begin(); - chacha_doneon(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, ctx->nrounds); - kernel_neon_end(); - } - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } - - return err; -} - -static int do_chacha(struct skcipher_request *req, bool neon) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - - return chacha_stream_xor(req, ctx, req->iv, neon); -} - -static int chacha_arm(struct skcipher_request *req) -{ - return do_chacha(req, false); + /* We always can use at least the ARM scalar implementation. */ + return true; } +EXPORT_SYMBOL(chacha_is_arch_optimized); -static int chacha_neon(struct skcipher_request *req) +static int __init chacha_arm_mod_init(void) { - return do_chacha(req, neon_usable()); -} - -static int do_xchacha(struct skcipher_request *req, bool neon) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - struct chacha_ctx subctx; - u32 state[16]; - u8 real_iv[16]; - - chacha_init(state, ctx->key, req->iv); - - if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon) { - hchacha_block_arm(state, subctx.key, ctx->nrounds); - } else { - kernel_neon_begin(); - hchacha_block_neon(state, subctx.key, ctx->nrounds); - kernel_neon_end(); - } - subctx.nrounds = ctx->nrounds; - - memcpy(&real_iv[0], req->iv + 24, 8); - memcpy(&real_iv[8], req->iv + 16, 8); - return chacha_stream_xor(req, &subctx, real_iv, neon); -} - -static int xchacha_arm(struct skcipher_request *req) -{ - return do_xchacha(req, false); -} - -static int xchacha_neon(struct skcipher_request *req) -{ - return do_xchacha(req, neon_usable()); -} - -static struct skcipher_alg arm_algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-arm", - .base.cra_priority = 200, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = chacha_arm, - .decrypt = chacha_arm, - }, { - .base.cra_name = "xchacha20", - .base.cra_driver_name = "xchacha20-arm", - .base.cra_priority = 200, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = xchacha_arm, - .decrypt = xchacha_arm, - }, { - .base.cra_name = "xchacha12", - .base.cra_driver_name = "xchacha12-arm", - .base.cra_priority = 200, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha12_setkey, - .encrypt = xchacha_arm, - .decrypt = xchacha_arm, - }, -}; - -static struct skcipher_alg neon_algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-neon", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .walksize = 4 * CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = chacha_neon, - .decrypt = chacha_neon, - }, { - .base.cra_name = "xchacha20", - .base.cra_driver_name = "xchacha20-neon", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .walksize = 4 * CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = xchacha_neon, - .decrypt = xchacha_neon, - }, { - .base.cra_name = "xchacha12", - .base.cra_driver_name = "xchacha12-neon", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .walksize = 4 * CHACHA_BLOCK_SIZE, - .setkey = chacha12_setkey, - .encrypt = xchacha_neon, - .decrypt = xchacha_neon, - } -}; - -static int __init chacha_simd_mod_init(void) -{ - int err = 0; - - if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) { - err = crypto_register_skciphers(arm_algs, ARRAY_SIZE(arm_algs)); - if (err) - return err; - } - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) { - int i; - switch (read_cpuid_part()) { case ARM_CPU_PART_CORTEX_A7: case ARM_CPU_PART_CORTEX_A5: @@ -308,45 +115,20 @@ static int __init chacha_simd_mod_init(void) * the NEON implementation but do incredibly with the * scalar one and use less power. */ - for (i = 0; i < ARRAY_SIZE(neon_algs); i++) - neon_algs[i].base.cra_priority = 0; break; default: static_branch_enable(&use_neon); } - - if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) { - err = crypto_register_skciphers(neon_algs, ARRAY_SIZE(neon_algs)); - if (err) - crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs)); - } } - return err; + return 0; } +arch_initcall(chacha_arm_mod_init); -static void __exit chacha_simd_mod_fini(void) +static void __exit chacha_arm_mod_exit(void) { - if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) { - crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs)); - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) - crypto_unregister_skciphers(neon_algs, ARRAY_SIZE(neon_algs)); - } } +module_exit(chacha_arm_mod_exit); -module_init(chacha_simd_mod_init); -module_exit(chacha_simd_mod_fini); - -MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (scalar and NEON accelerated)"); +MODULE_DESCRIPTION("ChaCha and HChaCha functions (ARM optimized)"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("chacha20"); -MODULE_ALIAS_CRYPTO("chacha20-arm"); -MODULE_ALIAS_CRYPTO("xchacha20"); -MODULE_ALIAS_CRYPTO("xchacha20-arm"); -MODULE_ALIAS_CRYPTO("xchacha12"); -MODULE_ALIAS_CRYPTO("xchacha12-arm"); -#ifdef CONFIG_KERNEL_MODE_NEON -MODULE_ALIAS_CRYPTO("chacha20-neon"); -MODULE_ALIAS_CRYPTO("xchacha20-neon"); -MODULE_ALIAS_CRYPTO("xchacha12-neon"); -#endif diff --git a/arch/arm/crypto/chacha-neon-core.S b/arch/arm/crypto/chacha-neon-core.S index 13d12f672656..ddd62b6294a5 100644 --- a/arch/arm/crypto/chacha-neon-core.S +++ b/arch/arm/crypto/chacha-neon-core.S @@ -1,5 +1,5 @@ /* - * ChaCha/XChaCha NEON helper functions + * ChaCha/HChaCha NEON helper functions * * Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org> * diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c index aabfcf522a2c..a52dcc8c1e33 100644 --- a/arch/arm/crypto/ghash-ce-glue.c +++ b/arch/arm/crypto/ghash-ce-glue.c @@ -8,22 +8,22 @@ #include <asm/hwcap.h> #include <asm/neon.h> -#include <asm/simd.h> -#include <linux/unaligned.h> #include <crypto/aes.h> -#include <crypto/gcm.h> #include <crypto/b128ops.h> -#include <crypto/cryptd.h> +#include <crypto/gcm.h> +#include <crypto/gf128mul.h> +#include <crypto/ghash.h> #include <crypto/internal/aead.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> -#include <crypto/gf128mul.h> #include <crypto/scatterwalk.h> #include <linux/cpufeature.h> -#include <linux/crypto.h> +#include <linux/errno.h> #include <linux/jump_label.h> +#include <linux/kernel.h> #include <linux/module.h> +#include <linux/string.h> +#include <linux/unaligned.h> MODULE_DESCRIPTION("GHASH hash function using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>"); @@ -32,9 +32,6 @@ MODULE_ALIAS_CRYPTO("ghash"); MODULE_ALIAS_CRYPTO("gcm(aes)"); MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); -#define GHASH_BLOCK_SIZE 16 -#define GHASH_DIGEST_SIZE 16 - #define RFC4106_NONCE_SIZE 4 struct ghash_key { @@ -49,10 +46,8 @@ struct gcm_key { u8 nonce[]; // for RFC4106 nonce }; -struct ghash_desc_ctx { +struct arm_ghash_desc_ctx { u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; - u8 buf[GHASH_BLOCK_SIZE]; - u32 count; }; asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, @@ -65,9 +60,9 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_p64); static int ghash_init(struct shash_desc *desc) { - struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - *ctx = (struct ghash_desc_ctx){}; + *ctx = (struct arm_ghash_desc_ctx){}; return 0; } @@ -85,52 +80,49 @@ static void ghash_do_update(int blocks, u64 dg[], const char *src, static int ghash_update(struct shash_desc *desc, const u8 *src, unsigned int len) { - struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); - unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; - - ctx->count += len; + struct ghash_key *key = crypto_shash_ctx(desc->tfm); + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); + int blocks; - if ((partial + len) >= GHASH_BLOCK_SIZE) { - struct ghash_key *key = crypto_shash_ctx(desc->tfm); - int blocks; + blocks = len / GHASH_BLOCK_SIZE; + ghash_do_update(blocks, ctx->digest, src, key, NULL); + return len - blocks * GHASH_BLOCK_SIZE; +} - if (partial) { - int p = GHASH_BLOCK_SIZE - partial; +static int ghash_export(struct shash_desc *desc, void *out) +{ + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); + u8 *dst = out; - memcpy(ctx->buf + partial, src, p); - src += p; - len -= p; - } + put_unaligned_be64(ctx->digest[1], dst); + put_unaligned_be64(ctx->digest[0], dst + 8); + return 0; +} - blocks = len / GHASH_BLOCK_SIZE; - len %= GHASH_BLOCK_SIZE; +static int ghash_import(struct shash_desc *desc, const void *in) +{ + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); + const u8 *src = in; - ghash_do_update(blocks, ctx->digest, src, key, - partial ? ctx->buf : NULL); - src += blocks * GHASH_BLOCK_SIZE; - partial = 0; - } - if (len) - memcpy(ctx->buf + partial, src, len); + ctx->digest[1] = get_unaligned_be64(src); + ctx->digest[0] = get_unaligned_be64(src + 8); return 0; } -static int ghash_final(struct shash_desc *desc, u8 *dst) +static int ghash_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *dst) { - struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); - unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; + struct ghash_key *key = crypto_shash_ctx(desc->tfm); + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - if (partial) { - struct ghash_key *key = crypto_shash_ctx(desc->tfm); + if (len) { + u8 buf[GHASH_BLOCK_SIZE] = {}; - memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial); - ghash_do_update(1, ctx->digest, ctx->buf, key, NULL); + memcpy(buf, src, len); + ghash_do_update(1, ctx->digest, buf, key, NULL); + memzero_explicit(buf, sizeof(buf)); } - put_unaligned_be64(ctx->digest[1], dst); - put_unaligned_be64(ctx->digest[0], dst + 8); - - *ctx = (struct ghash_desc_ctx){}; - return 0; + return ghash_export(desc, dst); } static void ghash_reflect(u64 h[], const be128 *k) @@ -175,13 +167,17 @@ static struct shash_alg ghash_alg = { .digestsize = GHASH_DIGEST_SIZE, .init = ghash_init, .update = ghash_update, - .final = ghash_final, + .finup = ghash_finup, .setkey = ghash_setkey, - .descsize = sizeof(struct ghash_desc_ctx), + .export = ghash_export, + .import = ghash_import, + .descsize = sizeof(struct arm_ghash_desc_ctx), + .statesize = sizeof(struct ghash_desc_ctx), .base.cra_name = "ghash", .base.cra_driver_name = "ghash-ce", .base.cra_priority = 300, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = GHASH_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]), .base.cra_module = THIS_MODULE, @@ -317,9 +313,6 @@ static int gcm_encrypt(struct aead_request *req, const u8 *iv, u32 assoclen) u8 *tag, *dst; int tail, err; - if (WARN_ON_ONCE(!may_use_simd())) - return -EBUSY; - err = skcipher_walk_aead_encrypt(&walk, req, false); kernel_neon_begin(); @@ -409,9 +402,6 @@ static int gcm_decrypt(struct aead_request *req, const u8 *iv, u32 assoclen) u8 *tag, *dst; int tail, err, ret; - if (WARN_ON_ONCE(!may_use_simd())) - return -EBUSY; - scatterwalk_map_and_copy(otag, req->src, req->assoclen + req->cryptlen - authsize, authsize, 0); diff --git a/arch/arm/crypto/poly1305-glue.c b/arch/arm/crypto/poly1305-glue.c index 4464ffbf8fd1..42d0ebde1ae1 100644 --- a/arch/arm/crypto/poly1305-glue.c +++ b/arch/arm/crypto/poly1305-glue.c @@ -8,15 +8,12 @@ #include <asm/hwcap.h> #include <asm/neon.h> #include <asm/simd.h> -#include <linux/unaligned.h> -#include <crypto/algapi.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/poly1305.h> +#include <crypto/poly1305.h> #include <crypto/internal/simd.h> #include <linux/cpufeature.h> -#include <linux/crypto.h> #include <linux/jump_label.h> #include <linux/module.h> +#include <linux/unaligned.h> void poly1305_init_arm(void *state, const u8 *key); void poly1305_blocks_arm(void *state, const u8 *src, u32 len, u32 hibit); @@ -40,102 +37,6 @@ void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KE } EXPORT_SYMBOL(poly1305_init_arch); -static int arm_poly1305_init(struct shash_desc *desc) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - dctx->buflen = 0; - dctx->rset = 0; - dctx->sset = false; - - return 0; -} - -static void arm_poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, - u32 len, u32 hibit, bool do_neon) -{ - if (unlikely(!dctx->sset)) { - if (!dctx->rset) { - poly1305_init_arm(&dctx->h, src); - src += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - dctx->rset = 1; - } - if (len >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(src + 0); - dctx->s[1] = get_unaligned_le32(src + 4); - dctx->s[2] = get_unaligned_le32(src + 8); - dctx->s[3] = get_unaligned_le32(src + 12); - src += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - if (len < POLY1305_BLOCK_SIZE) - return; - } - - len &= ~(POLY1305_BLOCK_SIZE - 1); - - if (static_branch_likely(&have_neon) && likely(do_neon)) - poly1305_blocks_neon(&dctx->h, src, len, hibit); - else - poly1305_blocks_arm(&dctx->h, src, len, hibit); -} - -static void arm_poly1305_do_update(struct poly1305_desc_ctx *dctx, - const u8 *src, u32 len, bool do_neon) -{ - if (unlikely(dctx->buflen)) { - u32 bytes = min(len, POLY1305_BLOCK_SIZE - dctx->buflen); - - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - len -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - arm_poly1305_blocks(dctx, dctx->buf, - POLY1305_BLOCK_SIZE, 1, false); - dctx->buflen = 0; - } - } - - if (likely(len >= POLY1305_BLOCK_SIZE)) { - arm_poly1305_blocks(dctx, src, len, 1, do_neon); - src += round_down(len, POLY1305_BLOCK_SIZE); - len %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(len)) { - dctx->buflen = len; - memcpy(dctx->buf, src, len); - } -} - -static int arm_poly1305_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - arm_poly1305_do_update(dctx, src, srclen, false); - return 0; -} - -static int __maybe_unused arm_poly1305_update_neon(struct shash_desc *desc, - const u8 *src, - unsigned int srclen) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - bool do_neon = crypto_simd_usable() && srclen > 128; - - if (static_branch_likely(&have_neon) && do_neon) - kernel_neon_begin(); - arm_poly1305_do_update(dctx, src, srclen, do_neon); - if (static_branch_likely(&have_neon) && do_neon) - kernel_neon_end(); - return 0; -} - void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int nbytes) { @@ -199,76 +100,26 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) } EXPORT_SYMBOL(poly1305_final_arch); -static int arm_poly1305_final(struct shash_desc *desc, u8 *dst) +bool poly1305_is_arch_optimized(void) { - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (unlikely(!dctx->sset)) - return -ENOKEY; - - poly1305_final_arch(dctx, dst); - return 0; + /* We always can use at least the ARM scalar implementation. */ + return true; } - -static struct shash_alg arm_poly1305_algs[] = {{ - .init = arm_poly1305_init, - .update = arm_poly1305_update, - .final = arm_poly1305_final, - .digestsize = POLY1305_DIGEST_SIZE, - .descsize = sizeof(struct poly1305_desc_ctx), - - .base.cra_name = "poly1305", - .base.cra_driver_name = "poly1305-arm", - .base.cra_priority = 150, - .base.cra_blocksize = POLY1305_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -#ifdef CONFIG_KERNEL_MODE_NEON -}, { - .init = arm_poly1305_init, - .update = arm_poly1305_update_neon, - .final = arm_poly1305_final, - .digestsize = POLY1305_DIGEST_SIZE, - .descsize = sizeof(struct poly1305_desc_ctx), - - .base.cra_name = "poly1305", - .base.cra_driver_name = "poly1305-neon", - .base.cra_priority = 200, - .base.cra_blocksize = POLY1305_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -#endif -}}; +EXPORT_SYMBOL(poly1305_is_arch_optimized); static int __init arm_poly1305_mod_init(void) { if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) static_branch_enable(&have_neon); - else if (IS_REACHABLE(CONFIG_CRYPTO_HASH)) - /* register only the first entry */ - return crypto_register_shash(&arm_poly1305_algs[0]); - - return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? - crypto_register_shashes(arm_poly1305_algs, - ARRAY_SIZE(arm_poly1305_algs)) : 0; + return 0; } +arch_initcall(arm_poly1305_mod_init); static void __exit arm_poly1305_mod_exit(void) { - if (!IS_REACHABLE(CONFIG_CRYPTO_HASH)) - return; - if (!static_branch_likely(&have_neon)) { - crypto_unregister_shash(&arm_poly1305_algs[0]); - return; - } - crypto_unregister_shashes(arm_poly1305_algs, - ARRAY_SIZE(arm_poly1305_algs)); } - -module_init(arm_poly1305_mod_init); module_exit(arm_poly1305_mod_exit); MODULE_DESCRIPTION("Accelerated Poly1305 transform for ARM"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("poly1305"); -MODULE_ALIAS_CRYPTO("poly1305-arm"); -MODULE_ALIAS_CRYPTO("poly1305-neon"); diff --git a/arch/arm/crypto/sha1-ce-glue.c b/arch/arm/crypto/sha1-ce-glue.c index de9100c67b37..fac07a4799de 100644 --- a/arch/arm/crypto/sha1-ce-glue.c +++ b/arch/arm/crypto/sha1-ce-glue.c @@ -5,20 +5,14 @@ * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org> */ +#include <asm/neon.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> #include <crypto/sha1.h> #include <crypto/sha1_base.h> #include <linux/cpufeature.h> -#include <linux/crypto.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <asm/hwcap.h> -#include <asm/neon.h> -#include <asm/simd.h> - -#include "sha1.h" - MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); MODULE_LICENSE("GPL v2"); @@ -29,50 +23,36 @@ asmlinkage void sha1_ce_transform(struct sha1_state *sst, u8 const *src, static int sha1_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha1_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) - return sha1_update_arm(desc, data, len); + int remain; kernel_neon_begin(); - sha1_base_do_update(desc, data, len, sha1_ce_transform); + remain = sha1_base_do_update_blocks(desc, data, len, sha1_ce_transform); kernel_neon_end(); - return 0; + return remain; } static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) - return sha1_finup_arm(desc, data, len, out); - kernel_neon_begin(); - if (len) - sha1_base_do_update(desc, data, len, sha1_ce_transform); - sha1_base_do_finalize(desc, sha1_ce_transform); + sha1_base_do_finup(desc, data, len, sha1_ce_transform); kernel_neon_end(); return sha1_base_finish(desc, out); } -static int sha1_ce_final(struct shash_desc *desc, u8 *out) -{ - return sha1_ce_finup(desc, NULL, 0, out); -} - static struct shash_alg alg = { .init = sha1_base_init, .update = sha1_ce_update, - .final = sha1_ce_final, .finup = sha1_ce_finup, - .descsize = sizeof(struct sha1_state), + .descsize = SHA1_STATE_SIZE, .digestsize = SHA1_DIGEST_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-ce", .cra_priority = 200, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm/crypto/sha1.h b/arch/arm/crypto/sha1.h deleted file mode 100644 index b1b7e21da2c3..000000000000 --- a/arch/arm/crypto/sha1.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef ASM_ARM_CRYPTO_SHA1_H -#define ASM_ARM_CRYPTO_SHA1_H - -#include <linux/crypto.h> -#include <crypto/sha1.h> - -extern int sha1_update_arm(struct shash_desc *desc, const u8 *data, - unsigned int len); - -extern int sha1_finup_arm(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out); - -#endif diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c index 95a727bcd664..255da00c7d98 100644 --- a/arch/arm/crypto/sha1_glue.c +++ b/arch/arm/crypto/sha1_glue.c @@ -12,53 +12,42 @@ */ #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/types.h> #include <crypto/sha1.h> #include <crypto/sha1_base.h> -#include <asm/byteorder.h> - -#include "sha1.h" +#include <linux/kernel.h> +#include <linux/module.h> asmlinkage void sha1_block_data_order(struct sha1_state *digest, const u8 *data, int rounds); -int sha1_update_arm(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int sha1_update_arm(struct shash_desc *desc, const u8 *data, + unsigned int len) { /* make sure signature matches sha1_block_fn() */ BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0); - return sha1_base_do_update(desc, data, len, sha1_block_data_order); + return sha1_base_do_update_blocks(desc, data, len, + sha1_block_data_order); } -EXPORT_SYMBOL_GPL(sha1_update_arm); -static int sha1_final(struct shash_desc *desc, u8 *out) +static int sha1_finup_arm(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - sha1_base_do_finalize(desc, sha1_block_data_order); + sha1_base_do_finup(desc, data, len, sha1_block_data_order); return sha1_base_finish(desc, out); } -int sha1_finup_arm(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha1_base_do_update(desc, data, len, sha1_block_data_order); - return sha1_final(desc, out); -} -EXPORT_SYMBOL_GPL(sha1_finup_arm); - static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = sha1_update_arm, - .final = sha1_final, .finup = sha1_finup_arm, - .descsize = sizeof(struct sha1_state), + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name= "sha1-asm", .cra_priority = 150, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm/crypto/sha1_neon_glue.c b/arch/arm/crypto/sha1_neon_glue.c index 9c70b87e69f7..d321850f22a6 100644 --- a/arch/arm/crypto/sha1_neon_glue.c +++ b/arch/arm/crypto/sha1_neon_glue.c @@ -13,18 +13,12 @@ * Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com> */ +#include <asm/neon.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> #include <crypto/sha1.h> #include <crypto/sha1_base.h> -#include <asm/neon.h> -#include <asm/simd.h> - -#include "sha1.h" +#include <linux/kernel.h> +#include <linux/module.h> asmlinkage void sha1_transform_neon(struct sha1_state *state_h, const u8 *data, int rounds); @@ -32,50 +26,37 @@ asmlinkage void sha1_transform_neon(struct sha1_state *state_h, static int sha1_neon_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha1_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) - return sha1_update_arm(desc, data, len); + int remain; kernel_neon_begin(); - sha1_base_do_update(desc, data, len, sha1_transform_neon); + remain = sha1_base_do_update_blocks(desc, data, len, + sha1_transform_neon); kernel_neon_end(); - return 0; + return remain; } static int sha1_neon_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) - return sha1_finup_arm(desc, data, len, out); - kernel_neon_begin(); - if (len) - sha1_base_do_update(desc, data, len, sha1_transform_neon); - sha1_base_do_finalize(desc, sha1_transform_neon); + sha1_base_do_finup(desc, data, len, sha1_transform_neon); kernel_neon_end(); return sha1_base_finish(desc, out); } -static int sha1_neon_final(struct shash_desc *desc, u8 *out) -{ - return sha1_neon_finup(desc, NULL, 0, out); -} - static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = sha1_neon_update, - .final = sha1_neon_final, .finup = sha1_neon_finup, - .descsize = sizeof(struct sha1_state), + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-neon", .cra_priority = 250, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm/crypto/sha2-ce-glue.c b/arch/arm/crypto/sha2-ce-glue.c index aeac45bfbf9f..1e9d16f79678 100644 --- a/arch/arm/crypto/sha2-ce-glue.c +++ b/arch/arm/crypto/sha2-ce-glue.c @@ -5,91 +5,69 @@ * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org> */ +#include <asm/neon.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> #include <crypto/sha2.h> #include <crypto/sha256_base.h> #include <linux/cpufeature.h> -#include <linux/crypto.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <asm/hwcap.h> -#include <asm/simd.h> -#include <asm/neon.h> -#include <linux/unaligned.h> - -#include "sha256_glue.h" - MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); MODULE_LICENSE("GPL v2"); -asmlinkage void sha2_ce_transform(struct sha256_state *sst, u8 const *src, - int blocks); +asmlinkage void sha2_ce_transform(struct crypto_sha256_state *sst, + u8 const *src, int blocks); static int sha2_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha256_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) - return crypto_sha256_arm_update(desc, data, len); + int remain; kernel_neon_begin(); - sha256_base_do_update(desc, data, len, - (sha256_block_fn *)sha2_ce_transform); + remain = sha256_base_do_update_blocks(desc, data, len, + sha2_ce_transform); kernel_neon_end(); - - return 0; + return remain; } static int sha2_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) - return crypto_sha256_arm_finup(desc, data, len, out); - kernel_neon_begin(); - if (len) - sha256_base_do_update(desc, data, len, - (sha256_block_fn *)sha2_ce_transform); - sha256_base_do_finalize(desc, (sha256_block_fn *)sha2_ce_transform); + sha256_base_do_finup(desc, data, len, sha2_ce_transform); kernel_neon_end(); - return sha256_base_finish(desc, out); } -static int sha2_ce_final(struct shash_desc *desc, u8 *out) -{ - return sha2_ce_finup(desc, NULL, 0, out); -} - static struct shash_alg algs[] = { { .init = sha224_base_init, .update = sha2_ce_update, - .final = sha2_ce_final, .finup = sha2_ce_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .digestsize = SHA224_DIGEST_SIZE, .base = { .cra_name = "sha224", .cra_driver_name = "sha224-ce", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } }, { .init = sha256_base_init, .update = sha2_ce_update, - .final = sha2_ce_final, .finup = sha2_ce_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .digestsize = SHA256_DIGEST_SIZE, .base = { .cra_name = "sha256", .cra_driver_name = "sha256-ce", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm/crypto/sha256_glue.c b/arch/arm/crypto/sha256_glue.c index f85933fdec75..d04c4e6bae6d 100644 --- a/arch/arm/crypto/sha256_glue.c +++ b/arch/arm/crypto/sha256_glue.c @@ -10,58 +10,47 @@ * Author: Tim Chen <tim.c.chen@linux.intel.com> */ +#include <asm/neon.h> #include <crypto/internal/hash.h> -#include <linux/crypto.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> -#include <linux/string.h> #include <crypto/sha2.h> #include <crypto/sha256_base.h> -#include <asm/simd.h> -#include <asm/neon.h> +#include <linux/kernel.h> +#include <linux/module.h> #include "sha256_glue.h" -asmlinkage void sha256_block_data_order(struct sha256_state *state, +asmlinkage void sha256_block_data_order(struct crypto_sha256_state *state, const u8 *data, int num_blks); -int crypto_sha256_arm_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int crypto_sha256_arm_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { /* make sure casting to sha256_block_fn() is safe */ - BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0); + BUILD_BUG_ON(offsetof(struct crypto_sha256_state, state) != 0); - return sha256_base_do_update(desc, data, len, sha256_block_data_order); + return sha256_base_do_update_blocks(desc, data, len, + sha256_block_data_order); } -EXPORT_SYMBOL(crypto_sha256_arm_update); -static int crypto_sha256_arm_final(struct shash_desc *desc, u8 *out) +static int crypto_sha256_arm_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - sha256_base_do_finalize(desc, sha256_block_data_order); + sha256_base_do_finup(desc, data, len, sha256_block_data_order); return sha256_base_finish(desc, out); } -int crypto_sha256_arm_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha256_base_do_update(desc, data, len, sha256_block_data_order); - return crypto_sha256_arm_final(desc, out); -} -EXPORT_SYMBOL(crypto_sha256_arm_finup); - static struct shash_alg algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = crypto_sha256_arm_update, - .final = crypto_sha256_arm_final, .finup = crypto_sha256_arm_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-asm", .cra_priority = 150, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -69,13 +58,14 @@ static struct shash_alg algs[] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = crypto_sha256_arm_update, - .final = crypto_sha256_arm_final, .finup = crypto_sha256_arm_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-asm", .cra_priority = 150, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm/crypto/sha256_glue.h b/arch/arm/crypto/sha256_glue.h index 9f0d578bab5f..9881c9a115d1 100644 --- a/arch/arm/crypto/sha256_glue.h +++ b/arch/arm/crypto/sha256_glue.h @@ -2,14 +2,8 @@ #ifndef _CRYPTO_SHA256_GLUE_H #define _CRYPTO_SHA256_GLUE_H -#include <linux/crypto.h> +#include <crypto/hash.h> extern struct shash_alg sha256_neon_algs[2]; -int crypto_sha256_arm_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -int crypto_sha256_arm_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); - #endif /* _CRYPTO_SHA256_GLUE_H */ diff --git a/arch/arm/crypto/sha256_neon_glue.c b/arch/arm/crypto/sha256_neon_glue.c index ccdcfff71910..76eb3cdc21c9 100644 --- a/arch/arm/crypto/sha256_neon_glue.c +++ b/arch/arm/crypto/sha256_neon_glue.c @@ -9,69 +9,51 @@ * Copyright © 2014 Jussi Kivilinna <jussi.kivilinna@iki.fi> */ +#include <asm/neon.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> -#include <linux/types.h> -#include <linux/string.h> #include <crypto/sha2.h> #include <crypto/sha256_base.h> -#include <asm/byteorder.h> -#include <asm/simd.h> -#include <asm/neon.h> +#include <linux/kernel.h> +#include <linux/module.h> #include "sha256_glue.h" -asmlinkage void sha256_block_data_order_neon(struct sha256_state *digest, - const u8 *data, int num_blks); +asmlinkage void sha256_block_data_order_neon( + struct crypto_sha256_state *digest, const u8 *data, int num_blks); static int crypto_sha256_neon_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha256_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) - return crypto_sha256_arm_update(desc, data, len); + int remain; kernel_neon_begin(); - sha256_base_do_update(desc, data, len, sha256_block_data_order_neon); + remain = sha256_base_do_update_blocks(desc, data, len, + sha256_block_data_order_neon); kernel_neon_end(); - - return 0; + return remain; } static int crypto_sha256_neon_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) - return crypto_sha256_arm_finup(desc, data, len, out); - kernel_neon_begin(); - if (len) - sha256_base_do_update(desc, data, len, - sha256_block_data_order_neon); - sha256_base_do_finalize(desc, sha256_block_data_order_neon); + sha256_base_do_finup(desc, data, len, sha256_block_data_order_neon); kernel_neon_end(); - return sha256_base_finish(desc, out); } -static int crypto_sha256_neon_final(struct shash_desc *desc, u8 *out) -{ - return crypto_sha256_neon_finup(desc, NULL, 0, out); -} - struct shash_alg sha256_neon_algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = crypto_sha256_neon_update, - .final = crypto_sha256_neon_final, .finup = crypto_sha256_neon_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-neon", .cra_priority = 250, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -79,13 +61,14 @@ struct shash_alg sha256_neon_algs[] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = crypto_sha256_neon_update, - .final = crypto_sha256_neon_final, .finup = crypto_sha256_neon_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-neon", .cra_priority = 250, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm/crypto/sha512-glue.c b/arch/arm/crypto/sha512-glue.c index 1be5bd498af3..f8a6480889b1 100644 --- a/arch/arm/crypto/sha512-glue.c +++ b/arch/arm/crypto/sha512-glue.c @@ -5,15 +5,14 @@ * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org> */ +#include <asm/hwcap.h> +#include <asm/neon.h> #include <crypto/internal/hash.h> #include <crypto/sha2.h> #include <crypto/sha512_base.h> -#include <linux/crypto.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <asm/hwcap.h> -#include <asm/neon.h> - #include "sha512.h" MODULE_DESCRIPTION("Accelerated SHA-384/SHA-512 secure hash for ARM"); @@ -28,50 +27,47 @@ MODULE_ALIAS_CRYPTO("sha512-arm"); asmlinkage void sha512_block_data_order(struct sha512_state *state, u8 const *src, int blocks); -int sha512_arm_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int sha512_arm_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - return sha512_base_do_update(desc, data, len, sha512_block_data_order); + return sha512_base_do_update_blocks(desc, data, len, + sha512_block_data_order); } -static int sha512_arm_final(struct shash_desc *desc, u8 *out) +static int sha512_arm_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - sha512_base_do_finalize(desc, sha512_block_data_order); + sha512_base_do_finup(desc, data, len, sha512_block_data_order); return sha512_base_finish(desc, out); } -int sha512_arm_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha512_base_do_update(desc, data, len, sha512_block_data_order); - return sha512_arm_final(desc, out); -} - static struct shash_alg sha512_arm_algs[] = { { .init = sha384_base_init, .update = sha512_arm_update, - .final = sha512_arm_final, .finup = sha512_arm_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .digestsize = SHA384_DIGEST_SIZE, .base = { .cra_name = "sha384", .cra_driver_name = "sha384-arm", .cra_priority = 250, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } }, { .init = sha512_base_init, .update = sha512_arm_update, - .final = sha512_arm_final, .finup = sha512_arm_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .digestsize = SHA512_DIGEST_SIZE, .base = { .cra_name = "sha512", .cra_driver_name = "sha512-arm", .cra_priority = 250, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm/crypto/sha512-neon-glue.c b/arch/arm/crypto/sha512-neon-glue.c index c6e58fe475ac..bd528077fefb 100644 --- a/arch/arm/crypto/sha512-neon-glue.c +++ b/arch/arm/crypto/sha512-neon-glue.c @@ -5,16 +5,13 @@ * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org> */ +#include <asm/neon.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> #include <crypto/sha2.h> #include <crypto/sha512_base.h> -#include <linux/crypto.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <asm/simd.h> -#include <asm/neon.h> - #include "sha512.h" MODULE_ALIAS_CRYPTO("sha384-neon"); @@ -26,51 +23,36 @@ asmlinkage void sha512_block_data_order_neon(struct sha512_state *state, static int sha512_neon_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha512_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) - return sha512_arm_update(desc, data, len); + int remain; kernel_neon_begin(); - sha512_base_do_update(desc, data, len, sha512_block_data_order_neon); + remain = sha512_base_do_update_blocks(desc, data, len, + sha512_block_data_order_neon); kernel_neon_end(); - - return 0; + return remain; } static int sha512_neon_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) - return sha512_arm_finup(desc, data, len, out); - kernel_neon_begin(); - if (len) - sha512_base_do_update(desc, data, len, - sha512_block_data_order_neon); - sha512_base_do_finalize(desc, sha512_block_data_order_neon); + sha512_base_do_finup(desc, data, len, sha512_block_data_order_neon); kernel_neon_end(); - return sha512_base_finish(desc, out); } -static int sha512_neon_final(struct shash_desc *desc, u8 *out) -{ - return sha512_neon_finup(desc, NULL, 0, out); -} - struct shash_alg sha512_neon_algs[] = { { .init = sha384_base_init, .update = sha512_neon_update, - .final = sha512_neon_final, .finup = sha512_neon_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .digestsize = SHA384_DIGEST_SIZE, .base = { .cra_name = "sha384", .cra_driver_name = "sha384-neon", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_module = THIS_MODULE, @@ -78,14 +60,15 @@ struct shash_alg sha512_neon_algs[] = { { }, { .init = sha512_base_init, .update = sha512_neon_update, - .final = sha512_neon_final, .finup = sha512_neon_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .digestsize = SHA512_DIGEST_SIZE, .base = { .cra_name = "sha512", .cra_driver_name = "sha512-neon", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm/crypto/sha512.h b/arch/arm/crypto/sha512.h index e14572be76d1..eeaee52cda69 100644 --- a/arch/arm/crypto/sha512.h +++ b/arch/arm/crypto/sha512.h @@ -1,9 +1,3 @@ /* SPDX-License-Identifier: GPL-2.0 */ -int sha512_arm_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -int sha512_arm_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out); - extern struct shash_alg sha512_neon_algs[2]; diff --git a/arch/arm/include/asm/simd.h b/arch/arm/include/asm/simd.h index 82191dbd7e78..d37559762180 100644 --- a/arch/arm/include/asm/simd.h +++ b/arch/arm/include/asm/simd.h @@ -1,8 +1,14 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SIMD_H +#define _ASM_SIMD_H -#include <linux/hardirq.h> +#include <linux/compiler_attributes.h> +#include <linux/preempt.h> +#include <linux/types.h> static __must_check inline bool may_use_simd(void) { return IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && !in_hardirq(); } + +#endif /* _ASM_SIMD_H */ diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 3418c8d3c78d..7c2f63f2e307 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -28,14 +28,8 @@ config CRYPTO_NHPOLY1305_NEON config CRYPTO_POLY1305_NEON tristate depends on KERNEL_MODE_NEON - select CRYPTO_HASH select CRYPTO_ARCH_HAVE_LIB_POLY1305 default CRYPTO_LIB_POLY1305_INTERNAL - help - Poly1305 authenticator algorithm (RFC7539) - - Architecture: arm64 using: - - NEON (Advanced SIMD) extensions config CRYPTO_SHA1_ARM64_CE tristate "Hash functions: SHA-1 (ARMv8 Crypto Extensions)" @@ -101,7 +95,7 @@ config CRYPTO_SM3_NEON tristate "Hash functions: SM3 (NEON)" depends on KERNEL_MODE_NEON select CRYPTO_HASH - select CRYPTO_SM3 + select CRYPTO_LIB_SM3 help SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) @@ -112,7 +106,7 @@ config CRYPTO_SM3_ARM64_CE tristate "Hash functions: SM3 (ARMv8.2 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH - select CRYPTO_SM3 + select CRYPTO_LIB_SM3 help SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) @@ -189,16 +183,9 @@ config CRYPTO_AES_ARM64_NEON_BLK config CRYPTO_CHACHA20_NEON tristate depends on KERNEL_MODE_NEON - select CRYPTO_SKCIPHER select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA default CRYPTO_LIB_CHACHA_INTERNAL - help - Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 - stream cipher algorithms - - Architecture: arm64 using: - - NEON (Advanced SIMD) extensions config CRYPTO_AES_ARM64_BS tristate "Ciphers: AES, modes: ECB/CBC/CTR/XCTR/XTS modes (bit-sliced NEON)" diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index b0150999743f..81560f722b9d 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -5,19 +5,20 @@ * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> */ -#include <asm/neon.h> #include <asm/hwcap.h> -#include <asm/simd.h> +#include <asm/neon.h> #include <crypto/aes.h> #include <crypto/ctr.h> -#include <crypto/sha2.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> -#include <linux/module.h> -#include <linux/cpufeature.h> +#include <crypto/sha2.h> +#include <crypto/utils.h> #include <crypto/xts.h> +#include <linux/cpufeature.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> #include "aes-ce-setkey.h" @@ -130,7 +131,6 @@ struct mac_tfm_ctx { }; struct mac_desc_ctx { - unsigned int len; u8 dg[AES_BLOCK_SIZE]; }; @@ -869,109 +869,64 @@ static int mac_init(struct shash_desc *desc) struct mac_desc_ctx *ctx = shash_desc_ctx(desc); memset(ctx->dg, 0, AES_BLOCK_SIZE); - ctx->len = 0; - return 0; } static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks, - u8 dg[], int enc_before, int enc_after) + u8 dg[], int enc_before) { int rounds = 6 + ctx->key_length / 4; + int rem; - if (crypto_simd_usable()) { - int rem; - - do { - kernel_neon_begin(); - rem = aes_mac_update(in, ctx->key_enc, rounds, blocks, - dg, enc_before, enc_after); - kernel_neon_end(); - in += (blocks - rem) * AES_BLOCK_SIZE; - blocks = rem; - enc_before = 0; - } while (blocks); - } else { - if (enc_before) - aes_encrypt(ctx, dg, dg); - - while (blocks--) { - crypto_xor(dg, in, AES_BLOCK_SIZE); - in += AES_BLOCK_SIZE; - - if (blocks || enc_after) - aes_encrypt(ctx, dg, dg); - } - } + do { + kernel_neon_begin(); + rem = aes_mac_update(in, ctx->key_enc, rounds, blocks, + dg, enc_before, !enc_before); + kernel_neon_end(); + in += (blocks - rem) * AES_BLOCK_SIZE; + blocks = rem; + } while (blocks); } static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len) { struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct mac_desc_ctx *ctx = shash_desc_ctx(desc); + int blocks = len / AES_BLOCK_SIZE; - while (len > 0) { - unsigned int l; - - if ((ctx->len % AES_BLOCK_SIZE) == 0 && - (ctx->len + len) > AES_BLOCK_SIZE) { - - int blocks = len / AES_BLOCK_SIZE; - - len %= AES_BLOCK_SIZE; - - mac_do_update(&tctx->key, p, blocks, ctx->dg, - (ctx->len != 0), (len != 0)); - - p += blocks * AES_BLOCK_SIZE; - - if (!len) { - ctx->len = AES_BLOCK_SIZE; - break; - } - ctx->len = 0; - } - - l = min(len, AES_BLOCK_SIZE - ctx->len); - - if (l <= AES_BLOCK_SIZE) { - crypto_xor(ctx->dg + ctx->len, p, l); - ctx->len += l; - len -= l; - p += l; - } - } - - return 0; + len %= AES_BLOCK_SIZE; + mac_do_update(&tctx->key, p, blocks, ctx->dg, 0); + return len; } -static int cbcmac_final(struct shash_desc *desc, u8 *out) +static int cbcmac_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct mac_desc_ctx *ctx = shash_desc_ctx(desc); - mac_do_update(&tctx->key, NULL, 0, ctx->dg, (ctx->len != 0), 0); - + if (len) { + crypto_xor(ctx->dg, src, len); + mac_do_update(&tctx->key, NULL, 0, ctx->dg, 1); + } memcpy(out, ctx->dg, AES_BLOCK_SIZE); - return 0; } -static int cmac_final(struct shash_desc *desc, u8 *out) +static int cmac_finup(struct shash_desc *desc, const u8 *src, unsigned int len, + u8 *out) { struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct mac_desc_ctx *ctx = shash_desc_ctx(desc); u8 *consts = tctx->consts; - if (ctx->len != AES_BLOCK_SIZE) { - ctx->dg[ctx->len] ^= 0x80; + crypto_xor(ctx->dg, src, len); + if (len != AES_BLOCK_SIZE) { + ctx->dg[len] ^= 0x80; consts += AES_BLOCK_SIZE; } - - mac_do_update(&tctx->key, consts, 1, ctx->dg, 0, 1); - + mac_do_update(&tctx->key, consts, 1, ctx->dg, 0); memcpy(out, ctx->dg, AES_BLOCK_SIZE); - return 0; } @@ -979,6 +934,8 @@ static struct shash_alg mac_algs[] = { { .base.cra_name = "cmac(aes)", .base.cra_driver_name = "cmac-aes-" MODE, .base.cra_priority = PRIO, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINAL_NONZERO, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + 2 * AES_BLOCK_SIZE, @@ -987,13 +944,15 @@ static struct shash_alg mac_algs[] = { { .digestsize = AES_BLOCK_SIZE, .init = mac_init, .update = mac_update, - .final = cmac_final, + .finup = cmac_finup, .setkey = cmac_setkey, .descsize = sizeof(struct mac_desc_ctx), }, { .base.cra_name = "xcbc(aes)", .base.cra_driver_name = "xcbc-aes-" MODE, .base.cra_priority = PRIO, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINAL_NONZERO, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + 2 * AES_BLOCK_SIZE, @@ -1002,21 +961,22 @@ static struct shash_alg mac_algs[] = { { .digestsize = AES_BLOCK_SIZE, .init = mac_init, .update = mac_update, - .final = cmac_final, + .finup = cmac_finup, .setkey = xcbc_setkey, .descsize = sizeof(struct mac_desc_ctx), }, { .base.cra_name = "cbcmac(aes)", .base.cra_driver_name = "cbcmac-aes-" MODE, .base.cra_priority = PRIO, - .base.cra_blocksize = 1, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, + .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct mac_tfm_ctx), .base.cra_module = THIS_MODULE, .digestsize = AES_BLOCK_SIZE, .init = mac_init, .update = mac_update, - .final = cbcmac_final, + .finup = cbcmac_finup, .setkey = cbcmac_setkey, .descsize = sizeof(struct mac_desc_ctx), } }; diff --git a/arch/arm64/crypto/chacha-neon-core.S b/arch/arm64/crypto/chacha-neon-core.S index b70ac76f2610..80079586ecc7 100644 --- a/arch/arm64/crypto/chacha-neon-core.S +++ b/arch/arm64/crypto/chacha-neon-core.S @@ -1,5 +1,5 @@ /* - * ChaCha/XChaCha NEON helper functions + * ChaCha/HChaCha NEON helper functions * * Copyright (C) 2016-2018 Linaro, Ltd. <ard.biesheuvel@linaro.org> * diff --git a/arch/arm64/crypto/chacha-neon-glue.c b/arch/arm64/crypto/chacha-neon-glue.c index 229876acfc58..14a2836eff61 100644 --- a/arch/arm64/crypto/chacha-neon-glue.c +++ b/arch/arm64/crypto/chacha-neon-glue.c @@ -1,6 +1,5 @@ /* - * ARM NEON and scalar accelerated ChaCha and XChaCha stream ciphers, - * including ChaCha20 (RFC7539) + * ChaCha and HChaCha functions (ARM64 optimized) * * Copyright (C) 2016 - 2017 Linaro, Ltd. <ard.biesheuvel@linaro.org> * @@ -19,10 +18,8 @@ * (at your option) any later version. */ -#include <crypto/algapi.h> -#include <crypto/internal/chacha.h> +#include <crypto/chacha.h> #include <crypto/internal/simd.h> -#include <crypto/internal/skcipher.h> #include <linux/jump_label.h> #include <linux/kernel.h> #include <linux/module.h> @@ -95,143 +92,25 @@ void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, } EXPORT_SYMBOL(chacha_crypt_arch); -static int chacha_neon_stream_xor(struct skcipher_request *req, - const struct chacha_ctx *ctx, const u8 *iv) +bool chacha_is_arch_optimized(void) { - struct skcipher_walk walk; - u32 state[16]; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - chacha_init(state, ctx->key, iv); - - while (walk.nbytes > 0) { - unsigned int nbytes = walk.nbytes; - - if (nbytes < walk.total) - nbytes = rounddown(nbytes, walk.stride); - - if (!static_branch_likely(&have_neon) || - !crypto_simd_usable()) { - chacha_crypt_generic(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, - ctx->nrounds); - } else { - kernel_neon_begin(); - chacha_doneon(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, ctx->nrounds); - kernel_neon_end(); - } - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } - - return err; -} - -static int chacha_neon(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - - return chacha_neon_stream_xor(req, ctx, req->iv); + return static_key_enabled(&have_neon); } - -static int xchacha_neon(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - struct chacha_ctx subctx; - u32 state[16]; - u8 real_iv[16]; - - chacha_init(state, ctx->key, req->iv); - hchacha_block_arch(state, subctx.key, ctx->nrounds); - subctx.nrounds = ctx->nrounds; - - memcpy(&real_iv[0], req->iv + 24, 8); - memcpy(&real_iv[8], req->iv + 16, 8); - return chacha_neon_stream_xor(req, &subctx, real_iv); -} - -static struct skcipher_alg algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-neon", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .walksize = 5 * CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = chacha_neon, - .decrypt = chacha_neon, - }, { - .base.cra_name = "xchacha20", - .base.cra_driver_name = "xchacha20-neon", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .walksize = 5 * CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = xchacha_neon, - .decrypt = xchacha_neon, - }, { - .base.cra_name = "xchacha12", - .base.cra_driver_name = "xchacha12-neon", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .walksize = 5 * CHACHA_BLOCK_SIZE, - .setkey = chacha12_setkey, - .encrypt = xchacha_neon, - .decrypt = xchacha_neon, - } -}; +EXPORT_SYMBOL(chacha_is_arch_optimized); static int __init chacha_simd_mod_init(void) { - if (!cpu_have_named_feature(ASIMD)) - return 0; - - static_branch_enable(&have_neon); - - return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ? - crypto_register_skciphers(algs, ARRAY_SIZE(algs)) : 0; + if (cpu_have_named_feature(ASIMD)) + static_branch_enable(&have_neon); + return 0; } +arch_initcall(chacha_simd_mod_init); -static void __exit chacha_simd_mod_fini(void) +static void __exit chacha_simd_mod_exit(void) { - if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) && cpu_have_named_feature(ASIMD)) - crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); } +module_exit(chacha_simd_mod_exit); -module_init(chacha_simd_mod_init); -module_exit(chacha_simd_mod_fini); - -MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (NEON accelerated)"); +MODULE_DESCRIPTION("ChaCha and HChaCha functions (ARM64 optimized)"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("chacha20"); -MODULE_ALIAS_CRYPTO("chacha20-neon"); -MODULE_ALIAS_CRYPTO("xchacha20"); -MODULE_ALIAS_CRYPTO("xchacha20-neon"); -MODULE_ALIAS_CRYPTO("xchacha12"); -MODULE_ALIAS_CRYPTO("xchacha12-neon"); diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 071e122f9c37..4995b6e22335 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -6,30 +6,27 @@ */ #include <asm/neon.h> -#include <asm/simd.h> -#include <linux/unaligned.h> #include <crypto/aes.h> -#include <crypto/gcm.h> -#include <crypto/algapi.h> #include <crypto/b128ops.h> +#include <crypto/gcm.h> +#include <crypto/ghash.h> #include <crypto/gf128mul.h> #include <crypto/internal/aead.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <linux/cpufeature.h> -#include <linux/crypto.h> +#include <linux/errno.h> +#include <linux/kernel.h> #include <linux/module.h> +#include <linux/string.h> +#include <linux/unaligned.h> MODULE_DESCRIPTION("GHASH and AES-GCM using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); MODULE_LICENSE("GPL v2"); MODULE_ALIAS_CRYPTO("ghash"); -#define GHASH_BLOCK_SIZE 16 -#define GHASH_DIGEST_SIZE 16 - #define RFC4106_NONCE_SIZE 4 struct ghash_key { @@ -37,10 +34,8 @@ struct ghash_key { u64 h[][2]; }; -struct ghash_desc_ctx { +struct arm_ghash_desc_ctx { u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; - u8 buf[GHASH_BLOCK_SIZE]; - u32 count; }; struct gcm_aes_ctx { @@ -65,36 +60,12 @@ asmlinkage int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[], static int ghash_init(struct shash_desc *desc) { - struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - *ctx = (struct ghash_desc_ctx){}; + *ctx = (struct arm_ghash_desc_ctx){}; return 0; } -static void ghash_do_update(int blocks, u64 dg[], const char *src, - struct ghash_key *key, const char *head) -{ - be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) }; - - do { - const u8 *in = src; - - if (head) { - in = head; - blocks++; - head = NULL; - } else { - src += GHASH_BLOCK_SIZE; - } - - crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE); - gf128mul_lle(&dst, &key->k); - } while (--blocks); - - dg[0] = be64_to_cpu(dst.b); - dg[1] = be64_to_cpu(dst.a); -} - static __always_inline void ghash_do_simd_update(int blocks, u64 dg[], const char *src, struct ghash_key *key, const char *head, @@ -103,13 +74,9 @@ void ghash_do_simd_update(int blocks, u64 dg[], const char *src, u64 const h[][2], const char *head)) { - if (likely(crypto_simd_usable())) { - kernel_neon_begin(); - simd_update(blocks, dg, src, key->h, head); - kernel_neon_end(); - } else { - ghash_do_update(blocks, dg, src, key, head); - } + kernel_neon_begin(); + simd_update(blocks, dg, src, key->h, head); + kernel_neon_end(); } /* avoid hogging the CPU for too long */ @@ -118,61 +85,59 @@ void ghash_do_simd_update(int blocks, u64 dg[], const char *src, static int ghash_update(struct shash_desc *desc, const u8 *src, unsigned int len) { - struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); - unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); + struct ghash_key *key = crypto_shash_ctx(desc->tfm); + int blocks; - ctx->count += len; + blocks = len / GHASH_BLOCK_SIZE; + len -= blocks * GHASH_BLOCK_SIZE; - if ((partial + len) >= GHASH_BLOCK_SIZE) { - struct ghash_key *key = crypto_shash_ctx(desc->tfm); - int blocks; - - if (partial) { - int p = GHASH_BLOCK_SIZE - partial; + do { + int chunk = min(blocks, MAX_BLOCKS); - memcpy(ctx->buf + partial, src, p); - src += p; - len -= p; - } + ghash_do_simd_update(chunk, ctx->digest, src, key, NULL, + pmull_ghash_update_p8); + blocks -= chunk; + src += chunk * GHASH_BLOCK_SIZE; + } while (unlikely(blocks > 0)); + return len; +} - blocks = len / GHASH_BLOCK_SIZE; - len %= GHASH_BLOCK_SIZE; +static int ghash_export(struct shash_desc *desc, void *out) +{ + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); + u8 *dst = out; - do { - int chunk = min(blocks, MAX_BLOCKS); + put_unaligned_be64(ctx->digest[1], dst); + put_unaligned_be64(ctx->digest[0], dst + 8); + return 0; +} - ghash_do_simd_update(chunk, ctx->digest, src, key, - partial ? ctx->buf : NULL, - pmull_ghash_update_p8); +static int ghash_import(struct shash_desc *desc, const void *in) +{ + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); + const u8 *src = in; - blocks -= chunk; - src += chunk * GHASH_BLOCK_SIZE; - partial = 0; - } while (unlikely(blocks > 0)); - } - if (len) - memcpy(ctx->buf + partial, src, len); + ctx->digest[1] = get_unaligned_be64(src); + ctx->digest[0] = get_unaligned_be64(src + 8); return 0; } -static int ghash_final(struct shash_desc *desc, u8 *dst) +static int ghash_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *dst) { - struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); - unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; - - if (partial) { - struct ghash_key *key = crypto_shash_ctx(desc->tfm); + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); + struct ghash_key *key = crypto_shash_ctx(desc->tfm); - memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial); + if (len) { + u8 buf[GHASH_BLOCK_SIZE] = {}; - ghash_do_simd_update(1, ctx->digest, ctx->buf, key, NULL, + memcpy(buf, src, len); + ghash_do_simd_update(1, ctx->digest, src, key, NULL, pmull_ghash_update_p8); + memzero_explicit(buf, sizeof(buf)); } - put_unaligned_be64(ctx->digest[1], dst); - put_unaligned_be64(ctx->digest[0], dst + 8); - - memzero_explicit(ctx, sizeof(*ctx)); - return 0; + return ghash_export(desc, dst); } static void ghash_reflect(u64 h[], const be128 *k) @@ -205,6 +170,7 @@ static struct shash_alg ghash_alg = { .base.cra_name = "ghash", .base.cra_driver_name = "ghash-neon", .base.cra_priority = 150, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = GHASH_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]), .base.cra_module = THIS_MODULE, @@ -212,9 +178,12 @@ static struct shash_alg ghash_alg = { .digestsize = GHASH_DIGEST_SIZE, .init = ghash_init, .update = ghash_update, - .final = ghash_final, + .finup = ghash_finup, .setkey = ghash_setkey, - .descsize = sizeof(struct ghash_desc_ctx), + .export = ghash_export, + .import = ghash_import, + .descsize = sizeof(struct arm_ghash_desc_ctx), + .statesize = sizeof(struct ghash_desc_ctx), }; static int num_rounds(struct crypto_aes_ctx *ctx) diff --git a/arch/arm64/crypto/poly1305-glue.c b/arch/arm64/crypto/poly1305-glue.c index 18883ea438f3..906970dd5373 100644 --- a/arch/arm64/crypto/poly1305-glue.c +++ b/arch/arm64/crypto/poly1305-glue.c @@ -8,15 +8,12 @@ #include <asm/hwcap.h> #include <asm/neon.h> #include <asm/simd.h> -#include <linux/unaligned.h> -#include <crypto/algapi.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/poly1305.h> +#include <crypto/poly1305.h> #include <crypto/internal/simd.h> #include <linux/cpufeature.h> -#include <linux/crypto.h> #include <linux/jump_label.h> #include <linux/module.h> +#include <linux/unaligned.h> asmlinkage void poly1305_init_arm64(void *state, const u8 *key); asmlinkage void poly1305_blocks(void *state, const u8 *src, u32 len, u32 hibit); @@ -36,92 +33,6 @@ void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KE } EXPORT_SYMBOL(poly1305_init_arch); -static int neon_poly1305_init(struct shash_desc *desc) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - dctx->buflen = 0; - dctx->rset = 0; - dctx->sset = false; - - return 0; -} - -static void neon_poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, - u32 len, u32 hibit, bool do_neon) -{ - if (unlikely(!dctx->sset)) { - if (!dctx->rset) { - poly1305_init_arm64(&dctx->h, src); - src += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - dctx->rset = 1; - } - if (len >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(src + 0); - dctx->s[1] = get_unaligned_le32(src + 4); - dctx->s[2] = get_unaligned_le32(src + 8); - dctx->s[3] = get_unaligned_le32(src + 12); - src += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - if (len < POLY1305_BLOCK_SIZE) - return; - } - - len &= ~(POLY1305_BLOCK_SIZE - 1); - - if (static_branch_likely(&have_neon) && likely(do_neon)) - poly1305_blocks_neon(&dctx->h, src, len, hibit); - else - poly1305_blocks(&dctx->h, src, len, hibit); -} - -static void neon_poly1305_do_update(struct poly1305_desc_ctx *dctx, - const u8 *src, u32 len, bool do_neon) -{ - if (unlikely(dctx->buflen)) { - u32 bytes = min(len, POLY1305_BLOCK_SIZE - dctx->buflen); - - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - len -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - neon_poly1305_blocks(dctx, dctx->buf, - POLY1305_BLOCK_SIZE, 1, false); - dctx->buflen = 0; - } - } - - if (likely(len >= POLY1305_BLOCK_SIZE)) { - neon_poly1305_blocks(dctx, src, len, 1, do_neon); - src += round_down(len, POLY1305_BLOCK_SIZE); - len %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(len)) { - dctx->buflen = len; - memcpy(dctx->buf, src, len); - } -} - -static int neon_poly1305_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - bool do_neon = crypto_simd_usable() && srclen > 128; - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (static_branch_likely(&have_neon) && do_neon) - kernel_neon_begin(); - neon_poly1305_do_update(dctx, src, srclen, do_neon); - if (static_branch_likely(&have_neon) && do_neon) - kernel_neon_end(); - return 0; -} - void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int nbytes) { @@ -181,52 +92,25 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) } EXPORT_SYMBOL(poly1305_final_arch); -static int neon_poly1305_final(struct shash_desc *desc, u8 *dst) +bool poly1305_is_arch_optimized(void) { - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (unlikely(!dctx->sset)) - return -ENOKEY; - - poly1305_final_arch(dctx, dst); - return 0; + /* We always can use at least the ARM64 scalar implementation. */ + return true; } - -static struct shash_alg neon_poly1305_alg = { - .init = neon_poly1305_init, - .update = neon_poly1305_update, - .final = neon_poly1305_final, - .digestsize = POLY1305_DIGEST_SIZE, - .descsize = sizeof(struct poly1305_desc_ctx), - - .base.cra_name = "poly1305", - .base.cra_driver_name = "poly1305-neon", - .base.cra_priority = 200, - .base.cra_blocksize = POLY1305_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -}; +EXPORT_SYMBOL(poly1305_is_arch_optimized); static int __init neon_poly1305_mod_init(void) { - if (!cpu_have_named_feature(ASIMD)) - return 0; - - static_branch_enable(&have_neon); - - return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? - crypto_register_shash(&neon_poly1305_alg) : 0; + if (cpu_have_named_feature(ASIMD)) + static_branch_enable(&have_neon); + return 0; } +arch_initcall(neon_poly1305_mod_init); static void __exit neon_poly1305_mod_exit(void) { - if (IS_REACHABLE(CONFIG_CRYPTO_HASH) && cpu_have_named_feature(ASIMD)) - crypto_unregister_shash(&neon_poly1305_alg); } - -module_init(neon_poly1305_mod_init); module_exit(neon_poly1305_mod_exit); -MODULE_DESCRIPTION("Poly1305 transform using NEON instructions"); +MODULE_DESCRIPTION("Poly1305 authenticator (ARM64 optimized)"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("poly1305"); -MODULE_ALIAS_CRYPTO("poly1305-neon"); diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index cbd14f208f83..1f8c93fe1e64 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -7,14 +7,14 @@ #include <asm/neon.h> #include <asm/simd.h> -#include <linux/unaligned.h> #include <crypto/internal/hash.h> #include <crypto/internal/simd.h> #include <crypto/sha1.h> #include <crypto/sha1_base.h> #include <linux/cpufeature.h> -#include <linux/crypto.h> +#include <linux/kernel.h> #include <linux/module.h> +#include <linux/string.h> MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); @@ -56,79 +56,47 @@ static int sha1_ce_update(struct shash_desc *desc, const u8 *data, { struct sha1_ce_state *sctx = shash_desc_ctx(desc); - if (!crypto_simd_usable()) - return crypto_sha1_update(desc, data, len); - sctx->finalize = 0; - sha1_base_do_update(desc, data, len, sha1_ce_transform); - - return 0; + return sha1_base_do_update_blocks(desc, data, len, sha1_ce_transform); } static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { struct sha1_ce_state *sctx = shash_desc_ctx(desc); - bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE) && len; - - if (!crypto_simd_usable()) - return crypto_sha1_finup(desc, data, len, out); + bool finalized = false; /* * Allow the asm code to perform the finalization if there is no * partial data and the input is a round multiple of the block size. */ - sctx->finalize = finalize; - - sha1_base_do_update(desc, data, len, sha1_ce_transform); - if (!finalize) - sha1_base_do_finalize(desc, sha1_ce_transform); - return sha1_base_finish(desc, out); -} - -static int sha1_ce_final(struct shash_desc *desc, u8 *out) -{ - struct sha1_ce_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable()) - return crypto_sha1_finup(desc, NULL, 0, out); - - sctx->finalize = 0; - sha1_base_do_finalize(desc, sha1_ce_transform); + if (len >= SHA1_BLOCK_SIZE) { + unsigned int remain = len - round_down(len, SHA1_BLOCK_SIZE); + + finalized = !remain; + sctx->finalize = finalized; + sha1_base_do_update_blocks(desc, data, len, sha1_ce_transform); + data += len - remain; + len = remain; + } + if (!finalized) + sha1_base_do_finup(desc, data, len, sha1_ce_transform); return sha1_base_finish(desc, out); } -static int sha1_ce_export(struct shash_desc *desc, void *out) -{ - struct sha1_ce_state *sctx = shash_desc_ctx(desc); - - memcpy(out, &sctx->sst, sizeof(struct sha1_state)); - return 0; -} - -static int sha1_ce_import(struct shash_desc *desc, const void *in) -{ - struct sha1_ce_state *sctx = shash_desc_ctx(desc); - - memcpy(&sctx->sst, in, sizeof(struct sha1_state)); - sctx->finalize = 0; - return 0; -} - static struct shash_alg alg = { .init = sha1_base_init, .update = sha1_ce_update, - .final = sha1_ce_final, .finup = sha1_ce_finup, - .import = sha1_ce_import, - .export = sha1_ce_export, .descsize = sizeof(struct sha1_ce_state), - .statesize = sizeof(struct sha1_state), + .statesize = SHA1_STATE_SIZE, .digestsize = SHA1_DIGEST_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-ce", .cra_priority = 200, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index 6b4866a88ded..912c215101eb 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -6,15 +6,13 @@ */ #include <asm/neon.h> -#include <asm/simd.h> -#include <linux/unaligned.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> #include <crypto/sha2.h> #include <crypto/sha256_base.h> #include <linux/cpufeature.h> -#include <linux/crypto.h> +#include <linux/kernel.h> #include <linux/module.h> +#include <linux/string.h> MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); @@ -23,7 +21,7 @@ MODULE_ALIAS_CRYPTO("sha224"); MODULE_ALIAS_CRYPTO("sha256"); struct sha256_ce_state { - struct sha256_state sst; + struct crypto_sha256_state sst; u32 finalize; }; @@ -33,7 +31,7 @@ extern const u32 sha256_ce_offsetof_finalize; asmlinkage int __sha256_ce_transform(struct sha256_ce_state *sst, u8 const *src, int blocks); -static void sha256_ce_transform(struct sha256_state *sst, u8 const *src, +static void sha256_ce_transform(struct crypto_sha256_state *sst, u8 const *src, int blocks) { while (blocks) { @@ -54,42 +52,21 @@ const u32 sha256_ce_offsetof_count = offsetof(struct sha256_ce_state, const u32 sha256_ce_offsetof_finalize = offsetof(struct sha256_ce_state, finalize); -asmlinkage void sha256_block_data_order(u32 *digest, u8 const *src, int blocks); - -static void sha256_arm64_transform(struct sha256_state *sst, u8 const *src, - int blocks) -{ - sha256_block_data_order(sst->state, src, blocks); -} - static int sha256_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct sha256_ce_state *sctx = shash_desc_ctx(desc); - if (!crypto_simd_usable()) - return sha256_base_do_update(desc, data, len, - sha256_arm64_transform); - sctx->finalize = 0; - sha256_base_do_update(desc, data, len, sha256_ce_transform); - - return 0; + return sha256_base_do_update_blocks(desc, data, len, + sha256_ce_transform); } static int sha256_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { struct sha256_ce_state *sctx = shash_desc_ctx(desc); - bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE) && len; - - if (!crypto_simd_usable()) { - if (len) - sha256_base_do_update(desc, data, len, - sha256_arm64_transform); - sha256_base_do_finalize(desc, sha256_arm64_transform); - return sha256_base_finish(desc, out); - } + bool finalize = !(len % SHA256_BLOCK_SIZE) && len; /* * Allow the asm code to perform the finalization if there is no @@ -97,23 +74,11 @@ static int sha256_ce_finup(struct shash_desc *desc, const u8 *data, */ sctx->finalize = finalize; - sha256_base_do_update(desc, data, len, sha256_ce_transform); - if (!finalize) - sha256_base_do_finalize(desc, sha256_ce_transform); - return sha256_base_finish(desc, out); -} - -static int sha256_ce_final(struct shash_desc *desc, u8 *out) -{ - struct sha256_ce_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable()) { - sha256_base_do_finalize(desc, sha256_arm64_transform); - return sha256_base_finish(desc, out); - } - - sctx->finalize = 0; - sha256_base_do_finalize(desc, sha256_ce_transform); + if (finalize) + sha256_base_do_update_blocks(desc, data, len, + sha256_ce_transform); + else + sha256_base_do_finup(desc, data, len, sha256_ce_transform); return sha256_base_finish(desc, out); } @@ -124,55 +89,36 @@ static int sha256_ce_digest(struct shash_desc *desc, const u8 *data, return sha256_ce_finup(desc, data, len, out); } -static int sha256_ce_export(struct shash_desc *desc, void *out) -{ - struct sha256_ce_state *sctx = shash_desc_ctx(desc); - - memcpy(out, &sctx->sst, sizeof(struct sha256_state)); - return 0; -} - -static int sha256_ce_import(struct shash_desc *desc, const void *in) -{ - struct sha256_ce_state *sctx = shash_desc_ctx(desc); - - memcpy(&sctx->sst, in, sizeof(struct sha256_state)); - sctx->finalize = 0; - return 0; -} - static struct shash_alg algs[] = { { .init = sha224_base_init, .update = sha256_ce_update, - .final = sha256_ce_final, .finup = sha256_ce_finup, - .export = sha256_ce_export, - .import = sha256_ce_import, .descsize = sizeof(struct sha256_ce_state), - .statesize = sizeof(struct sha256_state), + .statesize = sizeof(struct crypto_sha256_state), .digestsize = SHA224_DIGEST_SIZE, .base = { .cra_name = "sha224", .cra_driver_name = "sha224-ce", .cra_priority = 200, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } }, { .init = sha256_base_init, .update = sha256_ce_update, - .final = sha256_ce_final, .finup = sha256_ce_finup, .digest = sha256_ce_digest, - .export = sha256_ce_export, - .import = sha256_ce_import, .descsize = sizeof(struct sha256_ce_state), - .statesize = sizeof(struct sha256_state), + .statesize = sizeof(struct crypto_sha256_state), .digestsize = SHA256_DIGEST_SIZE, .base = { .cra_name = "sha256", .cra_driver_name = "sha256-ce", .cra_priority = 200, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm64/crypto/sha256-glue.c b/arch/arm64/crypto/sha256-glue.c index 35356987cc1e..26f9fdfae87b 100644 --- a/arch/arm64/crypto/sha256-glue.c +++ b/arch/arm64/crypto/sha256-glue.c @@ -5,16 +5,13 @@ * Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org> */ -#include <asm/hwcap.h> #include <asm/neon.h> -#include <asm/simd.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> #include <crypto/sha2.h> #include <crypto/sha256_base.h> +#include <linux/cpufeature.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <linux/string.h> -#include <linux/types.h> MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64"); MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>"); @@ -27,8 +24,8 @@ asmlinkage void sha256_block_data_order(u32 *digest, const void *data, unsigned int num_blks); EXPORT_SYMBOL(sha256_block_data_order); -static void sha256_arm64_transform(struct sha256_state *sst, u8 const *src, - int blocks) +static void sha256_arm64_transform(struct crypto_sha256_state *sst, + u8 const *src, int blocks) { sha256_block_data_order(sst->state, src, blocks); } @@ -36,55 +33,52 @@ static void sha256_arm64_transform(struct sha256_state *sst, u8 const *src, asmlinkage void sha256_block_neon(u32 *digest, const void *data, unsigned int num_blks); -static void sha256_neon_transform(struct sha256_state *sst, u8 const *src, - int blocks) +static void sha256_neon_transform(struct crypto_sha256_state *sst, + u8 const *src, int blocks) { + kernel_neon_begin(); sha256_block_neon(sst->state, src, blocks); + kernel_neon_end(); } static int crypto_sha256_arm64_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - return sha256_base_do_update(desc, data, len, sha256_arm64_transform); + return sha256_base_do_update_blocks(desc, data, len, + sha256_arm64_transform); } static int crypto_sha256_arm64_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (len) - sha256_base_do_update(desc, data, len, sha256_arm64_transform); - sha256_base_do_finalize(desc, sha256_arm64_transform); - + sha256_base_do_finup(desc, data, len, sha256_arm64_transform); return sha256_base_finish(desc, out); } -static int crypto_sha256_arm64_final(struct shash_desc *desc, u8 *out) -{ - return crypto_sha256_arm64_finup(desc, NULL, 0, out); -} - static struct shash_alg algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = crypto_sha256_arm64_update, - .final = crypto_sha256_arm64_final, .finup = crypto_sha256_arm64_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base.cra_name = "sha256", .base.cra_driver_name = "sha256-arm64", .base.cra_priority = 125, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA256_BLOCK_SIZE, .base.cra_module = THIS_MODULE, }, { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = crypto_sha256_arm64_update, - .final = crypto_sha256_arm64_final, .finup = crypto_sha256_arm64_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base.cra_name = "sha224", .base.cra_driver_name = "sha224-arm64", .base.cra_priority = 125, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA224_BLOCK_SIZE, .base.cra_module = THIS_MODULE, } }; @@ -92,13 +86,7 @@ static struct shash_alg algs[] = { { static int sha256_update_neon(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha256_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable()) - return sha256_base_do_update(desc, data, len, - sha256_arm64_transform); - - while (len > 0) { + do { unsigned int chunk = len; /* @@ -106,65 +94,54 @@ static int sha256_update_neon(struct shash_desc *desc, const u8 *data, * input when running on a preemptible kernel, but process the * data block by block instead. */ - if (IS_ENABLED(CONFIG_PREEMPTION) && - chunk + sctx->count % SHA256_BLOCK_SIZE > SHA256_BLOCK_SIZE) - chunk = SHA256_BLOCK_SIZE - - sctx->count % SHA256_BLOCK_SIZE; - - kernel_neon_begin(); - sha256_base_do_update(desc, data, chunk, sha256_neon_transform); - kernel_neon_end(); + if (IS_ENABLED(CONFIG_PREEMPTION)) + chunk = SHA256_BLOCK_SIZE; + + chunk -= sha256_base_do_update_blocks(desc, data, chunk, + sha256_neon_transform); data += chunk; len -= chunk; - } - return 0; + } while (len >= SHA256_BLOCK_SIZE); + return len; } static int sha256_finup_neon(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) { - if (len) - sha256_base_do_update(desc, data, len, - sha256_arm64_transform); - sha256_base_do_finalize(desc, sha256_arm64_transform); - } else { - if (len) - sha256_update_neon(desc, data, len); - kernel_neon_begin(); - sha256_base_do_finalize(desc, sha256_neon_transform); - kernel_neon_end(); + if (len >= SHA256_BLOCK_SIZE) { + int remain = sha256_update_neon(desc, data, len); + + data += len - remain; + len = remain; } + sha256_base_do_finup(desc, data, len, sha256_neon_transform); return sha256_base_finish(desc, out); } -static int sha256_final_neon(struct shash_desc *desc, u8 *out) -{ - return sha256_finup_neon(desc, NULL, 0, out); -} - static struct shash_alg neon_algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = sha256_update_neon, - .final = sha256_final_neon, .finup = sha256_finup_neon, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base.cra_name = "sha256", .base.cra_driver_name = "sha256-arm64-neon", .base.cra_priority = 150, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA256_BLOCK_SIZE, .base.cra_module = THIS_MODULE, }, { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = sha256_update_neon, - .final = sha256_final_neon, .finup = sha256_finup_neon, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base.cra_name = "sha224", .base.cra_driver_name = "sha224-arm64-neon", .base.cra_priority = 150, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA224_BLOCK_SIZE, .base.cra_module = THIS_MODULE, } }; diff --git a/arch/arm64/crypto/sha3-ce-glue.c b/arch/arm64/crypto/sha3-ce-glue.c index 5662c3ac49e9..b4f1001046c9 100644 --- a/arch/arm64/crypto/sha3-ce-glue.c +++ b/arch/arm64/crypto/sha3-ce-glue.c @@ -12,13 +12,13 @@ #include <asm/hwcap.h> #include <asm/neon.h> #include <asm/simd.h> -#include <linux/unaligned.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> #include <crypto/sha3.h> #include <linux/cpufeature.h> -#include <linux/crypto.h> +#include <linux/kernel.h> #include <linux/module.h> +#include <linux/string.h> +#include <linux/unaligned.h> MODULE_DESCRIPTION("SHA3 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); @@ -35,74 +35,55 @@ static int sha3_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct sha3_state *sctx = shash_desc_ctx(desc); - unsigned int digest_size = crypto_shash_digestsize(desc->tfm); - - if (!crypto_simd_usable()) - return crypto_sha3_update(desc, data, len); - - if ((sctx->partial + len) >= sctx->rsiz) { - int blocks; - - if (sctx->partial) { - int p = sctx->rsiz - sctx->partial; - - memcpy(sctx->buf + sctx->partial, data, p); - kernel_neon_begin(); - sha3_ce_transform(sctx->st, sctx->buf, 1, digest_size); - kernel_neon_end(); - - data += p; - len -= p; - sctx->partial = 0; - } - - blocks = len / sctx->rsiz; - len %= sctx->rsiz; - - while (blocks) { - int rem; - - kernel_neon_begin(); - rem = sha3_ce_transform(sctx->st, data, blocks, - digest_size); - kernel_neon_end(); - data += (blocks - rem) * sctx->rsiz; - blocks = rem; - } - } - - if (len) { - memcpy(sctx->buf + sctx->partial, data, len); - sctx->partial += len; - } - return 0; + struct crypto_shash *tfm = desc->tfm; + unsigned int bs, ds; + int blocks; + + ds = crypto_shash_digestsize(tfm); + bs = crypto_shash_blocksize(tfm); + blocks = len / bs; + len -= blocks * bs; + do { + int rem; + + kernel_neon_begin(); + rem = sha3_ce_transform(sctx->st, data, blocks, ds); + kernel_neon_end(); + data += (blocks - rem) * bs; + blocks = rem; + } while (blocks); + return len; } -static int sha3_final(struct shash_desc *desc, u8 *out) +static int sha3_finup(struct shash_desc *desc, const u8 *src, unsigned int len, + u8 *out) { struct sha3_state *sctx = shash_desc_ctx(desc); - unsigned int digest_size = crypto_shash_digestsize(desc->tfm); + struct crypto_shash *tfm = desc->tfm; __le64 *digest = (__le64 *)out; + u8 block[SHA3_224_BLOCK_SIZE]; + unsigned int bs, ds; int i; - if (!crypto_simd_usable()) - return crypto_sha3_final(desc, out); + ds = crypto_shash_digestsize(tfm); + bs = crypto_shash_blocksize(tfm); + memcpy(block, src, len); - sctx->buf[sctx->partial++] = 0x06; - memset(sctx->buf + sctx->partial, 0, sctx->rsiz - sctx->partial); - sctx->buf[sctx->rsiz - 1] |= 0x80; + block[len++] = 0x06; + memset(block + len, 0, bs - len); + block[bs - 1] |= 0x80; kernel_neon_begin(); - sha3_ce_transform(sctx->st, sctx->buf, 1, digest_size); + sha3_ce_transform(sctx->st, block, 1, ds); kernel_neon_end(); + memzero_explicit(block , sizeof(block)); - for (i = 0; i < digest_size / 8; i++) + for (i = 0; i < ds / 8; i++) put_unaligned_le64(sctx->st[i], digest++); - if (digest_size & 4) + if (ds & 4) put_unaligned_le32(sctx->st[i], (__le32 *)digest); - memzero_explicit(sctx, sizeof(*sctx)); return 0; } @@ -110,10 +91,11 @@ static struct shash_alg algs[] = { { .digestsize = SHA3_224_DIGEST_SIZE, .init = crypto_sha3_init, .update = sha3_update, - .final = sha3_final, - .descsize = sizeof(struct sha3_state), + .finup = sha3_finup, + .descsize = SHA3_STATE_SIZE, .base.cra_name = "sha3-224", .base.cra_driver_name = "sha3-224-ce", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = SHA3_224_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .base.cra_priority = 200, @@ -121,10 +103,11 @@ static struct shash_alg algs[] = { { .digestsize = SHA3_256_DIGEST_SIZE, .init = crypto_sha3_init, .update = sha3_update, - .final = sha3_final, - .descsize = sizeof(struct sha3_state), + .finup = sha3_finup, + .descsize = SHA3_STATE_SIZE, .base.cra_name = "sha3-256", .base.cra_driver_name = "sha3-256-ce", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = SHA3_256_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .base.cra_priority = 200, @@ -132,10 +115,11 @@ static struct shash_alg algs[] = { { .digestsize = SHA3_384_DIGEST_SIZE, .init = crypto_sha3_init, .update = sha3_update, - .final = sha3_final, - .descsize = sizeof(struct sha3_state), + .finup = sha3_finup, + .descsize = SHA3_STATE_SIZE, .base.cra_name = "sha3-384", .base.cra_driver_name = "sha3-384-ce", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = SHA3_384_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .base.cra_priority = 200, @@ -143,10 +127,11 @@ static struct shash_alg algs[] = { { .digestsize = SHA3_512_DIGEST_SIZE, .init = crypto_sha3_init, .update = sha3_update, - .final = sha3_final, - .descsize = sizeof(struct sha3_state), + .finup = sha3_finup, + .descsize = SHA3_STATE_SIZE, .base.cra_name = "sha3-512", .base.cra_driver_name = "sha3-512-ce", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = SHA3_512_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .base.cra_priority = 200, diff --git a/arch/arm64/crypto/sha512-ce-glue.c b/arch/arm64/crypto/sha512-ce-glue.c index 071f64293227..6fb3001fa2c9 100644 --- a/arch/arm64/crypto/sha512-ce-glue.c +++ b/arch/arm64/crypto/sha512-ce-glue.c @@ -10,14 +10,11 @@ */ #include <asm/neon.h> -#include <asm/simd.h> -#include <linux/unaligned.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> #include <crypto/sha2.h> #include <crypto/sha512_base.h> #include <linux/cpufeature.h> -#include <linux/crypto.h> +#include <linux/kernel.h> #include <linux/module.h> MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash using ARMv8 Crypto Extensions"); @@ -29,12 +26,10 @@ MODULE_ALIAS_CRYPTO("sha512"); asmlinkage int __sha512_ce_transform(struct sha512_state *sst, u8 const *src, int blocks); -asmlinkage void sha512_block_data_order(u64 *digest, u8 const *src, int blocks); - static void sha512_ce_transform(struct sha512_state *sst, u8 const *src, int blocks) { - while (blocks) { + do { int rem; kernel_neon_begin(); @@ -42,67 +37,47 @@ static void sha512_ce_transform(struct sha512_state *sst, u8 const *src, kernel_neon_end(); src += (blocks - rem) * SHA512_BLOCK_SIZE; blocks = rem; - } -} - -static void sha512_arm64_transform(struct sha512_state *sst, u8 const *src, - int blocks) -{ - sha512_block_data_order(sst->state, src, blocks); + } while (blocks); } static int sha512_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - sha512_block_fn *fn = crypto_simd_usable() ? sha512_ce_transform - : sha512_arm64_transform; - - sha512_base_do_update(desc, data, len, fn); - return 0; + return sha512_base_do_update_blocks(desc, data, len, + sha512_ce_transform); } static int sha512_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - sha512_block_fn *fn = crypto_simd_usable() ? sha512_ce_transform - : sha512_arm64_transform; - - sha512_base_do_update(desc, data, len, fn); - sha512_base_do_finalize(desc, fn); - return sha512_base_finish(desc, out); -} - -static int sha512_ce_final(struct shash_desc *desc, u8 *out) -{ - sha512_block_fn *fn = crypto_simd_usable() ? sha512_ce_transform - : sha512_arm64_transform; - - sha512_base_do_finalize(desc, fn); + sha512_base_do_finup(desc, data, len, sha512_ce_transform); return sha512_base_finish(desc, out); } static struct shash_alg algs[] = { { .init = sha384_base_init, .update = sha512_ce_update, - .final = sha512_ce_final, .finup = sha512_ce_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .digestsize = SHA384_DIGEST_SIZE, .base.cra_name = "sha384", .base.cra_driver_name = "sha384-ce", .base.cra_priority = 200, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA512_BLOCK_SIZE, .base.cra_module = THIS_MODULE, }, { .init = sha512_base_init, .update = sha512_ce_update, - .final = sha512_ce_final, .finup = sha512_ce_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .digestsize = SHA512_DIGEST_SIZE, .base.cra_name = "sha512", .base.cra_driver_name = "sha512-ce", .base.cra_priority = 200, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA512_BLOCK_SIZE, .base.cra_module = THIS_MODULE, } }; diff --git a/arch/arm64/crypto/sha512-glue.c b/arch/arm64/crypto/sha512-glue.c index 62f129dea83d..ab2e1c13dfad 100644 --- a/arch/arm64/crypto/sha512-glue.c +++ b/arch/arm64/crypto/sha512-glue.c @@ -6,11 +6,10 @@ */ #include <crypto/internal/hash.h> -#include <linux/types.h> -#include <linux/string.h> #include <crypto/sha2.h> #include <crypto/sha512_base.h> -#include <asm/neon.h> +#include <linux/kernel.h> +#include <linux/module.h> MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash for arm64"); MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>"); @@ -21,7 +20,6 @@ MODULE_ALIAS_CRYPTO("sha512"); asmlinkage void sha512_block_data_order(u64 *digest, const void *data, unsigned int num_blks); -EXPORT_SYMBOL(sha512_block_data_order); static void sha512_arm64_transform(struct sha512_state *sst, u8 const *src, int blocks) @@ -32,46 +30,41 @@ static void sha512_arm64_transform(struct sha512_state *sst, u8 const *src, static int sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - return sha512_base_do_update(desc, data, len, sha512_arm64_transform); + return sha512_base_do_update_blocks(desc, data, len, + sha512_arm64_transform); } static int sha512_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (len) - sha512_base_do_update(desc, data, len, sha512_arm64_transform); - sha512_base_do_finalize(desc, sha512_arm64_transform); - + sha512_base_do_finup(desc, data, len, sha512_arm64_transform); return sha512_base_finish(desc, out); } -static int sha512_final(struct shash_desc *desc, u8 *out) -{ - return sha512_finup(desc, NULL, 0, out); -} - static struct shash_alg algs[] = { { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, .update = sha512_update, - .final = sha512_final, .finup = sha512_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base.cra_name = "sha512", .base.cra_driver_name = "sha512-arm64", .base.cra_priority = 150, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA512_BLOCK_SIZE, .base.cra_module = THIS_MODULE, }, { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, .update = sha512_update, - .final = sha512_final, .finup = sha512_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base.cra_name = "sha384", .base.cra_driver_name = "sha384-arm64", .base.cra_priority = 150, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA384_BLOCK_SIZE, .base.cra_module = THIS_MODULE, } }; diff --git a/arch/arm64/crypto/sm3-ce-glue.c b/arch/arm64/crypto/sm3-ce-glue.c index 1a71788c4cda..eac6f5fa0abe 100644 --- a/arch/arm64/crypto/sm3-ce-glue.c +++ b/arch/arm64/crypto/sm3-ce-glue.c @@ -6,14 +6,11 @@ */ #include <asm/neon.h> -#include <asm/simd.h> -#include <linux/unaligned.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> #include <crypto/sm3.h> #include <crypto/sm3_base.h> #include <linux/cpufeature.h> -#include <linux/crypto.h> +#include <linux/kernel.h> #include <linux/module.h> MODULE_DESCRIPTION("SM3 secure hash using ARMv8 Crypto Extensions"); @@ -26,50 +23,20 @@ asmlinkage void sm3_ce_transform(struct sm3_state *sst, u8 const *src, static int sm3_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - if (!crypto_simd_usable()) { - sm3_update(shash_desc_ctx(desc), data, len); - return 0; - } + int remain; kernel_neon_begin(); - sm3_base_do_update(desc, data, len, sm3_ce_transform); + remain = sm3_base_do_update_blocks(desc, data, len, sm3_ce_transform); kernel_neon_end(); - - return 0; -} - -static int sm3_ce_final(struct shash_desc *desc, u8 *out) -{ - if (!crypto_simd_usable()) { - sm3_final(shash_desc_ctx(desc), out); - return 0; - } - - kernel_neon_begin(); - sm3_base_do_finalize(desc, sm3_ce_transform); - kernel_neon_end(); - - return sm3_base_finish(desc, out); + return remain; } static int sm3_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) { - struct sm3_state *sctx = shash_desc_ctx(desc); - - if (len) - sm3_update(sctx, data, len); - sm3_final(sctx, out); - return 0; - } - kernel_neon_begin(); - if (len) - sm3_base_do_update(desc, data, len, sm3_ce_transform); - sm3_base_do_finalize(desc, sm3_ce_transform); + sm3_base_do_finup(desc, data, len, sm3_ce_transform); kernel_neon_end(); - return sm3_base_finish(desc, out); } @@ -77,11 +44,12 @@ static struct shash_alg sm3_alg = { .digestsize = SM3_DIGEST_SIZE, .init = sm3_base_init, .update = sm3_ce_update, - .final = sm3_ce_final, .finup = sm3_ce_finup, - .descsize = sizeof(struct sm3_state), + .descsize = SM3_STATE_SIZE, .base.cra_name = "sm3", .base.cra_driver_name = "sm3-ce", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SM3_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .base.cra_priority = 400, diff --git a/arch/arm64/crypto/sm3-neon-glue.c b/arch/arm64/crypto/sm3-neon-glue.c index 8dd71ce79b69..6c4611a503a3 100644 --- a/arch/arm64/crypto/sm3-neon-glue.c +++ b/arch/arm64/crypto/sm3-neon-glue.c @@ -6,14 +6,11 @@ */ #include <asm/neon.h> -#include <asm/simd.h> -#include <linux/unaligned.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> #include <crypto/sm3.h> #include <crypto/sm3_base.h> #include <linux/cpufeature.h> -#include <linux/crypto.h> +#include <linux/kernel.h> #include <linux/module.h> @@ -23,50 +20,20 @@ asmlinkage void sm3_neon_transform(struct sm3_state *sst, u8 const *src, static int sm3_neon_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - if (!crypto_simd_usable()) { - sm3_update(shash_desc_ctx(desc), data, len); - return 0; - } + int remain; kernel_neon_begin(); - sm3_base_do_update(desc, data, len, sm3_neon_transform); + remain = sm3_base_do_update_blocks(desc, data, len, sm3_neon_transform); kernel_neon_end(); - - return 0; -} - -static int sm3_neon_final(struct shash_desc *desc, u8 *out) -{ - if (!crypto_simd_usable()) { - sm3_final(shash_desc_ctx(desc), out); - return 0; - } - - kernel_neon_begin(); - sm3_base_do_finalize(desc, sm3_neon_transform); - kernel_neon_end(); - - return sm3_base_finish(desc, out); + return remain; } static int sm3_neon_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) { - struct sm3_state *sctx = shash_desc_ctx(desc); - - if (len) - sm3_update(sctx, data, len); - sm3_final(sctx, out); - return 0; - } - kernel_neon_begin(); - if (len) - sm3_base_do_update(desc, data, len, sm3_neon_transform); - sm3_base_do_finalize(desc, sm3_neon_transform); + sm3_base_do_finup(desc, data, len, sm3_neon_transform); kernel_neon_end(); - return sm3_base_finish(desc, out); } @@ -74,11 +41,12 @@ static struct shash_alg sm3_alg = { .digestsize = SM3_DIGEST_SIZE, .init = sm3_base_init, .update = sm3_neon_update, - .final = sm3_neon_final, .finup = sm3_neon_finup, - .descsize = sizeof(struct sm3_state), + .descsize = SM3_STATE_SIZE, .base.cra_name = "sm3", .base.cra_driver_name = "sm3-neon", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SM3_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .base.cra_priority = 200, diff --git a/arch/arm64/crypto/sm4-ce-glue.c b/arch/arm64/crypto/sm4-ce-glue.c index 43741bed874e..7a60e7b559dc 100644 --- a/arch/arm64/crypto/sm4-ce-glue.c +++ b/arch/arm64/crypto/sm4-ce-glue.c @@ -8,19 +8,18 @@ * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> */ -#include <linux/module.h> -#include <linux/crypto.h> -#include <linux/kernel.h> -#include <linux/cpufeature.h> #include <asm/neon.h> -#include <asm/simd.h> #include <crypto/b128ops.h> -#include <crypto/internal/simd.h> -#include <crypto/internal/skcipher.h> #include <crypto/internal/hash.h> +#include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> -#include <crypto/xts.h> #include <crypto/sm4.h> +#include <crypto/utils.h> +#include <crypto/xts.h> +#include <linux/cpufeature.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> #define BYTES2BLKS(nbytes) ((nbytes) >> 4) @@ -64,7 +63,6 @@ struct sm4_mac_tfm_ctx { }; struct sm4_mac_desc_ctx { - unsigned int len; u8 digest[SM4_BLOCK_SIZE]; }; @@ -591,8 +589,6 @@ static int sm4_mac_init(struct shash_desc *desc) struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc); memset(ctx->digest, 0, SM4_BLOCK_SIZE); - ctx->len = 0; - return 0; } @@ -601,87 +597,50 @@ static int sm4_mac_update(struct shash_desc *desc, const u8 *p, { struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc); - unsigned int l, nblocks; - - if (len == 0) - return 0; - - if (ctx->len || ctx->len + len < SM4_BLOCK_SIZE) { - l = min(len, SM4_BLOCK_SIZE - ctx->len); - - crypto_xor(ctx->digest + ctx->len, p, l); - ctx->len += l; - len -= l; - p += l; - } - - if (len && (ctx->len % SM4_BLOCK_SIZE) == 0) { - kernel_neon_begin(); - - if (len < SM4_BLOCK_SIZE && ctx->len == SM4_BLOCK_SIZE) { - sm4_ce_crypt_block(tctx->key.rkey_enc, - ctx->digest, ctx->digest); - ctx->len = 0; - } else { - nblocks = len / SM4_BLOCK_SIZE; - len %= SM4_BLOCK_SIZE; + unsigned int nblocks = len / SM4_BLOCK_SIZE; - sm4_ce_mac_update(tctx->key.rkey_enc, ctx->digest, p, - nblocks, (ctx->len == SM4_BLOCK_SIZE), - (len != 0)); - - p += nblocks * SM4_BLOCK_SIZE; - - if (len == 0) - ctx->len = SM4_BLOCK_SIZE; - } - - kernel_neon_end(); - - if (len) { - crypto_xor(ctx->digest, p, len); - ctx->len = len; - } - } - - return 0; + len %= SM4_BLOCK_SIZE; + kernel_neon_begin(); + sm4_ce_mac_update(tctx->key.rkey_enc, ctx->digest, p, + nblocks, false, true); + kernel_neon_end(); + return len; } -static int sm4_cmac_final(struct shash_desc *desc, u8 *out) +static int sm4_cmac_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc); const u8 *consts = tctx->consts; - if (ctx->len != SM4_BLOCK_SIZE) { - ctx->digest[ctx->len] ^= 0x80; + crypto_xor(ctx->digest, src, len); + if (len != SM4_BLOCK_SIZE) { + ctx->digest[len] ^= 0x80; consts += SM4_BLOCK_SIZE; } - kernel_neon_begin(); sm4_ce_mac_update(tctx->key.rkey_enc, ctx->digest, consts, 1, false, true); kernel_neon_end(); - memcpy(out, ctx->digest, SM4_BLOCK_SIZE); - return 0; } -static int sm4_cbcmac_final(struct shash_desc *desc, u8 *out) +static int sm4_cbcmac_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc); - if (ctx->len) { + if (len) { + crypto_xor(ctx->digest, src, len); kernel_neon_begin(); sm4_ce_crypt_block(tctx->key.rkey_enc, ctx->digest, ctx->digest); kernel_neon_end(); } - memcpy(out, ctx->digest, SM4_BLOCK_SIZE); - return 0; } @@ -691,6 +650,8 @@ static struct shash_alg sm4_mac_algs[] = { .cra_name = "cmac(sm4)", .cra_driver_name = "cmac-sm4-ce", .cra_priority = 400, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINAL_NONZERO, .cra_blocksize = SM4_BLOCK_SIZE, .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx) + SM4_BLOCK_SIZE * 2, @@ -699,7 +660,7 @@ static struct shash_alg sm4_mac_algs[] = { .digestsize = SM4_BLOCK_SIZE, .init = sm4_mac_init, .update = sm4_mac_update, - .final = sm4_cmac_final, + .finup = sm4_cmac_finup, .setkey = sm4_cmac_setkey, .descsize = sizeof(struct sm4_mac_desc_ctx), }, { @@ -707,6 +668,8 @@ static struct shash_alg sm4_mac_algs[] = { .cra_name = "xcbc(sm4)", .cra_driver_name = "xcbc-sm4-ce", .cra_priority = 400, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINAL_NONZERO, .cra_blocksize = SM4_BLOCK_SIZE, .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx) + SM4_BLOCK_SIZE * 2, @@ -715,7 +678,7 @@ static struct shash_alg sm4_mac_algs[] = { .digestsize = SM4_BLOCK_SIZE, .init = sm4_mac_init, .update = sm4_mac_update, - .final = sm4_cmac_final, + .finup = sm4_cmac_finup, .setkey = sm4_xcbc_setkey, .descsize = sizeof(struct sm4_mac_desc_ctx), }, { @@ -723,14 +686,15 @@ static struct shash_alg sm4_mac_algs[] = { .cra_name = "cbcmac(sm4)", .cra_driver_name = "cbcmac-sm4-ce", .cra_priority = 400, - .cra_blocksize = 1, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, + .cra_blocksize = SM4_BLOCK_SIZE, .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx), .cra_module = THIS_MODULE, }, .digestsize = SM4_BLOCK_SIZE, .init = sm4_mac_init, .update = sm4_mac_update, - .final = sm4_cbcmac_final, + .finup = sm4_cbcmac_finup, .setkey = sm4_cbcmac_setkey, .descsize = sizeof(struct sm4_mac_desc_ctx), } diff --git a/arch/mips/cavium-octeon/crypto/octeon-md5.c b/arch/mips/cavium-octeon/crypto/octeon-md5.c index 5ee4ade99b99..fbc84eb7fedf 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-md5.c +++ b/arch/mips/cavium-octeon/crypto/octeon-md5.c @@ -19,22 +19,26 @@ * any later version. */ +#include <asm/octeon/octeon.h> +#include <crypto/internal/hash.h> #include <crypto/md5.h> -#include <linux/init.h> -#include <linux/types.h> +#include <linux/kernel.h> #include <linux/module.h> #include <linux/string.h> -#include <asm/byteorder.h> -#include <asm/octeon/octeon.h> -#include <crypto/internal/hash.h> +#include <linux/unaligned.h> #include "octeon-crypto.h" +struct octeon_md5_state { + __le32 hash[MD5_HASH_WORDS]; + u64 byte_count; +}; + /* * We pass everything as 64-bit. OCTEON can handle misaligned data. */ -static void octeon_md5_store_hash(struct md5_state *ctx) +static void octeon_md5_store_hash(struct octeon_md5_state *ctx) { u64 *hash = (u64 *)ctx->hash; @@ -42,7 +46,7 @@ static void octeon_md5_store_hash(struct md5_state *ctx) write_octeon_64bit_hash_dword(hash[1], 1); } -static void octeon_md5_read_hash(struct md5_state *ctx) +static void octeon_md5_read_hash(struct octeon_md5_state *ctx) { u64 *hash = (u64 *)ctx->hash; @@ -66,13 +70,12 @@ static void octeon_md5_transform(const void *_block) static int octeon_md5_init(struct shash_desc *desc) { - struct md5_state *mctx = shash_desc_ctx(desc); + struct octeon_md5_state *mctx = shash_desc_ctx(desc); - mctx->hash[0] = MD5_H0; - mctx->hash[1] = MD5_H1; - mctx->hash[2] = MD5_H2; - mctx->hash[3] = MD5_H3; - cpu_to_le32_array(mctx->hash, 4); + mctx->hash[0] = cpu_to_le32(MD5_H0); + mctx->hash[1] = cpu_to_le32(MD5_H1); + mctx->hash[2] = cpu_to_le32(MD5_H2); + mctx->hash[3] = cpu_to_le32(MD5_H3); mctx->byte_count = 0; return 0; @@ -81,52 +84,38 @@ static int octeon_md5_init(struct shash_desc *desc) static int octeon_md5_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct md5_state *mctx = shash_desc_ctx(desc); - const u32 avail = sizeof(mctx->block) - (mctx->byte_count & 0x3f); + struct octeon_md5_state *mctx = shash_desc_ctx(desc); struct octeon_cop2_state state; unsigned long flags; mctx->byte_count += len; - - if (avail > len) { - memcpy((char *)mctx->block + (sizeof(mctx->block) - avail), - data, len); - return 0; - } - - memcpy((char *)mctx->block + (sizeof(mctx->block) - avail), data, - avail); - flags = octeon_crypto_enable(&state); octeon_md5_store_hash(mctx); - octeon_md5_transform(mctx->block); - data += avail; - len -= avail; - - while (len >= sizeof(mctx->block)) { + do { octeon_md5_transform(data); - data += sizeof(mctx->block); - len -= sizeof(mctx->block); - } + data += MD5_HMAC_BLOCK_SIZE; + len -= MD5_HMAC_BLOCK_SIZE; + } while (len >= MD5_HMAC_BLOCK_SIZE); octeon_md5_read_hash(mctx); octeon_crypto_disable(&state, flags); - - memcpy(mctx->block, data, len); - - return 0; + mctx->byte_count -= len; + return len; } -static int octeon_md5_final(struct shash_desc *desc, u8 *out) +static int octeon_md5_finup(struct shash_desc *desc, const u8 *src, + unsigned int offset, u8 *out) { - struct md5_state *mctx = shash_desc_ctx(desc); - const unsigned int offset = mctx->byte_count & 0x3f; - char *p = (char *)mctx->block + offset; + struct octeon_md5_state *mctx = shash_desc_ctx(desc); int padding = 56 - (offset + 1); struct octeon_cop2_state state; + u32 block[MD5_BLOCK_WORDS]; unsigned long flags; + char *p; + p = memcpy(block, src, offset); + p += offset; *p++ = 0x80; flags = octeon_crypto_enable(&state); @@ -134,39 +123,56 @@ static int octeon_md5_final(struct shash_desc *desc, u8 *out) if (padding < 0) { memset(p, 0x00, padding + sizeof(u64)); - octeon_md5_transform(mctx->block); - p = (char *)mctx->block; + octeon_md5_transform(block); + p = (char *)block; padding = 56; } memset(p, 0, padding); - mctx->block[14] = mctx->byte_count << 3; - mctx->block[15] = mctx->byte_count >> 29; - cpu_to_le32_array(mctx->block + 14, 2); - octeon_md5_transform(mctx->block); + mctx->byte_count += offset; + block[14] = mctx->byte_count << 3; + block[15] = mctx->byte_count >> 29; + cpu_to_le32_array(block + 14, 2); + octeon_md5_transform(block); octeon_md5_read_hash(mctx); octeon_crypto_disable(&state, flags); + memzero_explicit(block, sizeof(block)); memcpy(out, mctx->hash, sizeof(mctx->hash)); - memset(mctx, 0, sizeof(*mctx)); return 0; } static int octeon_md5_export(struct shash_desc *desc, void *out) { - struct md5_state *ctx = shash_desc_ctx(desc); - - memcpy(out, ctx, sizeof(*ctx)); + struct octeon_md5_state *ctx = shash_desc_ctx(desc); + union { + u8 *u8; + u32 *u32; + u64 *u64; + } p = { .u8 = out }; + int i; + + for (i = 0; i < MD5_HASH_WORDS; i++) + put_unaligned(le32_to_cpu(ctx->hash[i]), p.u32++); + put_unaligned(ctx->byte_count, p.u64); return 0; } static int octeon_md5_import(struct shash_desc *desc, const void *in) { - struct md5_state *ctx = shash_desc_ctx(desc); - - memcpy(ctx, in, sizeof(*ctx)); + struct octeon_md5_state *ctx = shash_desc_ctx(desc); + union { + const u8 *u8; + const u32 *u32; + const u64 *u64; + } p = { .u8 = in }; + int i; + + for (i = 0; i < MD5_HASH_WORDS; i++) + ctx->hash[i] = cpu_to_le32(get_unaligned(p.u32++)); + ctx->byte_count = get_unaligned(p.u64); return 0; } @@ -174,15 +180,16 @@ static struct shash_alg alg = { .digestsize = MD5_DIGEST_SIZE, .init = octeon_md5_init, .update = octeon_md5_update, - .final = octeon_md5_final, + .finup = octeon_md5_finup, .export = octeon_md5_export, .import = octeon_md5_import, - .descsize = sizeof(struct md5_state), - .statesize = sizeof(struct md5_state), + .statesize = MD5_STATE_SIZE, + .descsize = sizeof(struct octeon_md5_state), .base = { .cra_name = "md5", .cra_driver_name= "octeon-md5", .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = MD5_HMAC_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha1.c b/arch/mips/cavium-octeon/crypto/octeon-sha1.c index 37a07b3c4568..e70f21a473da 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha1.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha1.c @@ -13,15 +13,13 @@ * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> */ -#include <linux/mm.h> +#include <asm/octeon/octeon.h> +#include <crypto/internal/hash.h> #include <crypto/sha1.h> #include <crypto/sha1_base.h> -#include <linux/init.h> -#include <linux/types.h> +#include <linux/errno.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <asm/byteorder.h> -#include <asm/octeon/octeon.h> -#include <crypto/internal/hash.h> #include "octeon-crypto.h" @@ -58,49 +56,23 @@ static void octeon_sha1_read_hash(struct sha1_state *sctx) memzero_explicit(&hash_tail.dword, sizeof(hash_tail.dword)); } -static void octeon_sha1_transform(const void *_block) +static void octeon_sha1_transform(struct sha1_state *sctx, const u8 *src, + int blocks) { - const u64 *block = _block; - - write_octeon_64bit_block_dword(block[0], 0); - write_octeon_64bit_block_dword(block[1], 1); - write_octeon_64bit_block_dword(block[2], 2); - write_octeon_64bit_block_dword(block[3], 3); - write_octeon_64bit_block_dword(block[4], 4); - write_octeon_64bit_block_dword(block[5], 5); - write_octeon_64bit_block_dword(block[6], 6); - octeon_sha1_start(block[7]); -} - -static void __octeon_sha1_update(struct sha1_state *sctx, const u8 *data, - unsigned int len) -{ - unsigned int partial; - unsigned int done; - const u8 *src; - - partial = sctx->count % SHA1_BLOCK_SIZE; - sctx->count += len; - done = 0; - src = data; - - if ((partial + len) >= SHA1_BLOCK_SIZE) { - if (partial) { - done = -partial; - memcpy(sctx->buffer + partial, data, - done + SHA1_BLOCK_SIZE); - src = sctx->buffer; - } - - do { - octeon_sha1_transform(src); - done += SHA1_BLOCK_SIZE; - src = data + done; - } while (done + SHA1_BLOCK_SIZE <= len); - - partial = 0; - } - memcpy(sctx->buffer + partial, src, len - done); + do { + const u64 *block = (const u64 *)src; + + write_octeon_64bit_block_dword(block[0], 0); + write_octeon_64bit_block_dword(block[1], 1); + write_octeon_64bit_block_dword(block[2], 2); + write_octeon_64bit_block_dword(block[3], 3); + write_octeon_64bit_block_dword(block[4], 4); + write_octeon_64bit_block_dword(block[5], 5); + write_octeon_64bit_block_dword(block[6], 6); + octeon_sha1_start(block[7]); + + src += SHA1_BLOCK_SIZE; + } while (--blocks); } static int octeon_sha1_update(struct shash_desc *desc, const u8 *data, @@ -109,95 +81,47 @@ static int octeon_sha1_update(struct shash_desc *desc, const u8 *data, struct sha1_state *sctx = shash_desc_ctx(desc); struct octeon_cop2_state state; unsigned long flags; - - /* - * Small updates never reach the crypto engine, so the generic sha1 is - * faster because of the heavyweight octeon_crypto_enable() / - * octeon_crypto_disable(). - */ - if ((sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) - return crypto_sha1_update(desc, data, len); + int remain; flags = octeon_crypto_enable(&state); octeon_sha1_store_hash(sctx); - __octeon_sha1_update(sctx, data, len); + remain = sha1_base_do_update_blocks(desc, data, len, + octeon_sha1_transform); octeon_sha1_read_hash(sctx); octeon_crypto_disable(&state, flags); - - return 0; + return remain; } -static int octeon_sha1_final(struct shash_desc *desc, u8 *out) +static int octeon_sha1_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { struct sha1_state *sctx = shash_desc_ctx(desc); - static const u8 padding[64] = { 0x80, }; struct octeon_cop2_state state; - __be32 *dst = (__be32 *)out; - unsigned int pad_len; unsigned long flags; - unsigned int index; - __be64 bits; - int i; - - /* Save number of bits. */ - bits = cpu_to_be64(sctx->count << 3); - - /* Pad out to 56 mod 64. */ - index = sctx->count & 0x3f; - pad_len = (index < 56) ? (56 - index) : ((64+56) - index); flags = octeon_crypto_enable(&state); octeon_sha1_store_hash(sctx); - __octeon_sha1_update(sctx, padding, pad_len); - - /* Append length (before padding). */ - __octeon_sha1_update(sctx, (const u8 *)&bits, sizeof(bits)); + sha1_base_do_finup(desc, src, len, octeon_sha1_transform); octeon_sha1_read_hash(sctx); octeon_crypto_disable(&state, flags); - - /* Store state in digest */ - for (i = 0; i < 5; i++) - dst[i] = cpu_to_be32(sctx->state[i]); - - /* Zeroize sensitive information. */ - memset(sctx, 0, sizeof(*sctx)); - - return 0; -} - -static int octeon_sha1_export(struct shash_desc *desc, void *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; -} - -static int octeon_sha1_import(struct shash_desc *desc, const void *in) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - return 0; + return sha1_base_finish(desc, out); } static struct shash_alg octeon_sha1_alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = octeon_sha1_update, - .final = octeon_sha1_final, - .export = octeon_sha1_export, - .import = octeon_sha1_import, - .descsize = sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), + .finup = octeon_sha1_finup, + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name= "octeon-sha1", .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha256.c b/arch/mips/cavium-octeon/crypto/octeon-sha256.c index 435e4a6e7f13..8e85ea65387c 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha256.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha256.c @@ -14,15 +14,12 @@ * SHA224 Support Copyright 2007 Intel Corporation <jonathan.lynch@intel.com> */ -#include <linux/mm.h> +#include <asm/octeon/octeon.h> +#include <crypto/internal/hash.h> #include <crypto/sha2.h> #include <crypto/sha256_base.h> -#include <linux/init.h> -#include <linux/types.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <asm/byteorder.h> -#include <asm/octeon/octeon.h> -#include <crypto/internal/hash.h> #include "octeon-crypto.h" @@ -30,7 +27,7 @@ * We pass everything as 64-bit. OCTEON can handle misaligned data. */ -static void octeon_sha256_store_hash(struct sha256_state *sctx) +static void octeon_sha256_store_hash(struct crypto_sha256_state *sctx) { u64 *hash = (u64 *)sctx->state; @@ -40,7 +37,7 @@ static void octeon_sha256_store_hash(struct sha256_state *sctx) write_octeon_64bit_hash_dword(hash[3], 3); } -static void octeon_sha256_read_hash(struct sha256_state *sctx) +static void octeon_sha256_read_hash(struct crypto_sha256_state *sctx) { u64 *hash = (u64 *)sctx->state; @@ -50,158 +47,72 @@ static void octeon_sha256_read_hash(struct sha256_state *sctx) hash[3] = read_octeon_64bit_hash_dword(3); } -static void octeon_sha256_transform(const void *_block) +static void octeon_sha256_transform(struct crypto_sha256_state *sctx, + const u8 *src, int blocks) { - const u64 *block = _block; - - write_octeon_64bit_block_dword(block[0], 0); - write_octeon_64bit_block_dword(block[1], 1); - write_octeon_64bit_block_dword(block[2], 2); - write_octeon_64bit_block_dword(block[3], 3); - write_octeon_64bit_block_dword(block[4], 4); - write_octeon_64bit_block_dword(block[5], 5); - write_octeon_64bit_block_dword(block[6], 6); - octeon_sha256_start(block[7]); -} - -static void __octeon_sha256_update(struct sha256_state *sctx, const u8 *data, - unsigned int len) -{ - unsigned int partial; - unsigned int done; - const u8 *src; - - partial = sctx->count % SHA256_BLOCK_SIZE; - sctx->count += len; - done = 0; - src = data; - - if ((partial + len) >= SHA256_BLOCK_SIZE) { - if (partial) { - done = -partial; - memcpy(sctx->buf + partial, data, - done + SHA256_BLOCK_SIZE); - src = sctx->buf; - } - - do { - octeon_sha256_transform(src); - done += SHA256_BLOCK_SIZE; - src = data + done; - } while (done + SHA256_BLOCK_SIZE <= len); - - partial = 0; - } - memcpy(sctx->buf + partial, src, len - done); + do { + const u64 *block = (const u64 *)src; + + write_octeon_64bit_block_dword(block[0], 0); + write_octeon_64bit_block_dword(block[1], 1); + write_octeon_64bit_block_dword(block[2], 2); + write_octeon_64bit_block_dword(block[3], 3); + write_octeon_64bit_block_dword(block[4], 4); + write_octeon_64bit_block_dword(block[5], 5); + write_octeon_64bit_block_dword(block[6], 6); + octeon_sha256_start(block[7]); + + src += SHA256_BLOCK_SIZE; + } while (--blocks); } static int octeon_sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha256_state *sctx = shash_desc_ctx(desc); + struct crypto_sha256_state *sctx = shash_desc_ctx(desc); struct octeon_cop2_state state; unsigned long flags; - - /* - * Small updates never reach the crypto engine, so the generic sha256 is - * faster because of the heavyweight octeon_crypto_enable() / - * octeon_crypto_disable(). - */ - if ((sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) - return crypto_sha256_update(desc, data, len); + int remain; flags = octeon_crypto_enable(&state); octeon_sha256_store_hash(sctx); - __octeon_sha256_update(sctx, data, len); + remain = sha256_base_do_update_blocks(desc, data, len, + octeon_sha256_transform); octeon_sha256_read_hash(sctx); octeon_crypto_disable(&state, flags); - - return 0; + return remain; } -static int octeon_sha256_final(struct shash_desc *desc, u8 *out) +static int octeon_sha256_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - struct sha256_state *sctx = shash_desc_ctx(desc); - static const u8 padding[64] = { 0x80, }; + struct crypto_sha256_state *sctx = shash_desc_ctx(desc); struct octeon_cop2_state state; - __be32 *dst = (__be32 *)out; - unsigned int pad_len; unsigned long flags; - unsigned int index; - __be64 bits; - int i; - - /* Save number of bits. */ - bits = cpu_to_be64(sctx->count << 3); - - /* Pad out to 56 mod 64. */ - index = sctx->count & 0x3f; - pad_len = (index < 56) ? (56 - index) : ((64+56) - index); flags = octeon_crypto_enable(&state); octeon_sha256_store_hash(sctx); - __octeon_sha256_update(sctx, padding, pad_len); - - /* Append length (before padding). */ - __octeon_sha256_update(sctx, (const u8 *)&bits, sizeof(bits)); + sha256_base_do_finup(desc, src, len, octeon_sha256_transform); octeon_sha256_read_hash(sctx); octeon_crypto_disable(&state, flags); - - /* Store state in digest */ - for (i = 0; i < 8; i++) - dst[i] = cpu_to_be32(sctx->state[i]); - - /* Zeroize sensitive information. */ - memset(sctx, 0, sizeof(*sctx)); - - return 0; -} - -static int octeon_sha224_final(struct shash_desc *desc, u8 *hash) -{ - u8 D[SHA256_DIGEST_SIZE]; - - octeon_sha256_final(desc, D); - - memcpy(hash, D, SHA224_DIGEST_SIZE); - memzero_explicit(D, SHA256_DIGEST_SIZE); - - return 0; -} - -static int octeon_sha256_export(struct shash_desc *desc, void *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; -} - -static int octeon_sha256_import(struct shash_desc *desc, const void *in) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - return 0; + return sha256_base_finish(desc, out); } static struct shash_alg octeon_sha256_algs[2] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = octeon_sha256_update, - .final = octeon_sha256_final, - .export = octeon_sha256_export, - .import = octeon_sha256_import, - .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), + .finup = octeon_sha256_finup, + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name= "octeon-sha256", .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -209,11 +120,13 @@ static struct shash_alg octeon_sha256_algs[2] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = octeon_sha256_update, - .final = octeon_sha224_final, - .descsize = sizeof(struct sha256_state), + .finup = octeon_sha256_finup, + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name= "octeon-sha224", + .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha512.c b/arch/mips/cavium-octeon/crypto/octeon-sha512.c index 2dee9354e33f..215311053db3 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha512.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha512.c @@ -13,15 +13,12 @@ * Copyright (c) 2003 Kyle McMartin <kyle@debian.org> */ -#include <linux/mm.h> +#include <asm/octeon/octeon.h> +#include <crypto/internal/hash.h> #include <crypto/sha2.h> #include <crypto/sha512_base.h> -#include <linux/init.h> -#include <linux/types.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <asm/byteorder.h> -#include <asm/octeon/octeon.h> -#include <crypto/internal/hash.h> #include "octeon-crypto.h" @@ -53,60 +50,31 @@ static void octeon_sha512_read_hash(struct sha512_state *sctx) sctx->state[7] = read_octeon_64bit_hash_sha512(7); } -static void octeon_sha512_transform(const void *_block) +static void octeon_sha512_transform(struct sha512_state *sctx, + const u8 *src, int blocks) { - const u64 *block = _block; - - write_octeon_64bit_block_sha512(block[0], 0); - write_octeon_64bit_block_sha512(block[1], 1); - write_octeon_64bit_block_sha512(block[2], 2); - write_octeon_64bit_block_sha512(block[3], 3); - write_octeon_64bit_block_sha512(block[4], 4); - write_octeon_64bit_block_sha512(block[5], 5); - write_octeon_64bit_block_sha512(block[6], 6); - write_octeon_64bit_block_sha512(block[7], 7); - write_octeon_64bit_block_sha512(block[8], 8); - write_octeon_64bit_block_sha512(block[9], 9); - write_octeon_64bit_block_sha512(block[10], 10); - write_octeon_64bit_block_sha512(block[11], 11); - write_octeon_64bit_block_sha512(block[12], 12); - write_octeon_64bit_block_sha512(block[13], 13); - write_octeon_64bit_block_sha512(block[14], 14); - octeon_sha512_start(block[15]); -} - -static void __octeon_sha512_update(struct sha512_state *sctx, const u8 *data, - unsigned int len) -{ - unsigned int part_len; - unsigned int index; - unsigned int i; - - /* Compute number of bytes mod 128. */ - index = sctx->count[0] % SHA512_BLOCK_SIZE; - - /* Update number of bytes. */ - if ((sctx->count[0] += len) < len) - sctx->count[1]++; - - part_len = SHA512_BLOCK_SIZE - index; - - /* Transform as many times as possible. */ - if (len >= part_len) { - memcpy(&sctx->buf[index], data, part_len); - octeon_sha512_transform(sctx->buf); - - for (i = part_len; i + SHA512_BLOCK_SIZE <= len; - i += SHA512_BLOCK_SIZE) - octeon_sha512_transform(&data[i]); - - index = 0; - } else { - i = 0; - } - - /* Buffer remaining input. */ - memcpy(&sctx->buf[index], &data[i], len - i); + do { + const u64 *block = (const u64 *)src; + + write_octeon_64bit_block_sha512(block[0], 0); + write_octeon_64bit_block_sha512(block[1], 1); + write_octeon_64bit_block_sha512(block[2], 2); + write_octeon_64bit_block_sha512(block[3], 3); + write_octeon_64bit_block_sha512(block[4], 4); + write_octeon_64bit_block_sha512(block[5], 5); + write_octeon_64bit_block_sha512(block[6], 6); + write_octeon_64bit_block_sha512(block[7], 7); + write_octeon_64bit_block_sha512(block[8], 8); + write_octeon_64bit_block_sha512(block[9], 9); + write_octeon_64bit_block_sha512(block[10], 10); + write_octeon_64bit_block_sha512(block[11], 11); + write_octeon_64bit_block_sha512(block[12], 12); + write_octeon_64bit_block_sha512(block[13], 13); + write_octeon_64bit_block_sha512(block[14], 14); + octeon_sha512_start(block[15]); + + src += SHA512_BLOCK_SIZE; + } while (--blocks); } static int octeon_sha512_update(struct shash_desc *desc, const u8 *data, @@ -115,89 +83,48 @@ static int octeon_sha512_update(struct shash_desc *desc, const u8 *data, struct sha512_state *sctx = shash_desc_ctx(desc); struct octeon_cop2_state state; unsigned long flags; - - /* - * Small updates never reach the crypto engine, so the generic sha512 is - * faster because of the heavyweight octeon_crypto_enable() / - * octeon_crypto_disable(). - */ - if ((sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) - return crypto_sha512_update(desc, data, len); + int remain; flags = octeon_crypto_enable(&state); octeon_sha512_store_hash(sctx); - __octeon_sha512_update(sctx, data, len); + remain = sha512_base_do_update_blocks(desc, data, len, + octeon_sha512_transform); octeon_sha512_read_hash(sctx); octeon_crypto_disable(&state, flags); - - return 0; + return remain; } -static int octeon_sha512_final(struct shash_desc *desc, u8 *hash) +static int octeon_sha512_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *hash) { struct sha512_state *sctx = shash_desc_ctx(desc); - static u8 padding[128] = { 0x80, }; struct octeon_cop2_state state; - __be64 *dst = (__be64 *)hash; - unsigned int pad_len; unsigned long flags; - unsigned int index; - __be64 bits[2]; - int i; - - /* Save number of bits. */ - bits[1] = cpu_to_be64(sctx->count[0] << 3); - bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61); - - /* Pad out to 112 mod 128. */ - index = sctx->count[0] & 0x7f; - pad_len = (index < 112) ? (112 - index) : ((128+112) - index); flags = octeon_crypto_enable(&state); octeon_sha512_store_hash(sctx); - __octeon_sha512_update(sctx, padding, pad_len); - - /* Append length (before padding). */ - __octeon_sha512_update(sctx, (const u8 *)bits, sizeof(bits)); + sha512_base_do_finup(desc, src, len, octeon_sha512_transform); octeon_sha512_read_hash(sctx); octeon_crypto_disable(&state, flags); - - /* Store state in digest. */ - for (i = 0; i < 8; i++) - dst[i] = cpu_to_be64(sctx->state[i]); - - /* Zeroize sensitive information. */ - memset(sctx, 0, sizeof(struct sha512_state)); - - return 0; -} - -static int octeon_sha384_final(struct shash_desc *desc, u8 *hash) -{ - u8 D[64]; - - octeon_sha512_final(desc, D); - - memcpy(hash, D, 48); - memzero_explicit(D, 64); - - return 0; + return sha512_base_finish(desc, hash); } static struct shash_alg octeon_sha512_algs[2] = { { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, .update = octeon_sha512_update, - .final = octeon_sha512_final, - .descsize = sizeof(struct sha512_state), + .finup = octeon_sha512_finup, + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha512", .cra_driver_name= "octeon-sha512", .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -205,12 +132,14 @@ static struct shash_alg octeon_sha512_algs[2] = { { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, .update = octeon_sha512_update, - .final = octeon_sha384_final, - .descsize = sizeof(struct sha512_state), + .finup = octeon_sha512_finup, + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha384", .cra_driver_name= "octeon-sha384", .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/mips/crypto/Kconfig b/arch/mips/crypto/Kconfig index 545fc0e12422..8283664a1f24 100644 --- a/arch/mips/crypto/Kconfig +++ b/arch/mips/crypto/Kconfig @@ -4,14 +4,8 @@ menu "Accelerated Cryptographic Algorithms for CPU (mips)" config CRYPTO_POLY1305_MIPS tristate - depends on MIPS - select CRYPTO_HASH select CRYPTO_ARCH_HAVE_LIB_POLY1305 default CRYPTO_LIB_POLY1305_INTERNAL - help - Poly1305 authenticator algorithm (RFC7539) - - Architecture: mips config CRYPTO_MD5_OCTEON tristate "Digests: MD5 (OCTEON)" @@ -56,13 +50,7 @@ config CRYPTO_SHA512_OCTEON config CRYPTO_CHACHA_MIPS tristate depends on CPU_MIPS32_R2 - select CRYPTO_SKCIPHER select CRYPTO_ARCH_HAVE_LIB_CHACHA default CRYPTO_LIB_CHACHA_INTERNAL - help - Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 - stream cipher algorithms - - Architecture: MIPS32r2 endmenu diff --git a/arch/mips/crypto/chacha-glue.c b/arch/mips/crypto/chacha-glue.c index f6fc2e1079a1..334ecb29fb8f 100644 --- a/arch/mips/crypto/chacha-glue.c +++ b/arch/mips/crypto/chacha-glue.c @@ -1,15 +1,11 @@ // SPDX-License-Identifier: GPL-2.0 /* - * MIPS accelerated ChaCha and XChaCha stream ciphers, - * including ChaCha20 (RFC7539) + * ChaCha and HChaCha functions (MIPS optimized) * * Copyright (C) 2019 Linaro, Ltd. <ard.biesheuvel@linaro.org> */ -#include <asm/byteorder.h> -#include <crypto/algapi.h> -#include <crypto/internal/chacha.h> -#include <crypto/internal/skcipher.h> +#include <crypto/chacha.h> #include <linux/kernel.h> #include <linux/module.h> @@ -20,127 +16,12 @@ EXPORT_SYMBOL(chacha_crypt_arch); asmlinkage void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds); EXPORT_SYMBOL(hchacha_block_arch); -static int chacha_mips_stream_xor(struct skcipher_request *req, - const struct chacha_ctx *ctx, const u8 *iv) +bool chacha_is_arch_optimized(void) { - struct skcipher_walk walk; - u32 state[16]; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - chacha_init(state, ctx->key, iv); - - while (walk.nbytes > 0) { - unsigned int nbytes = walk.nbytes; - - if (nbytes < walk.total) - nbytes = round_down(nbytes, walk.stride); - - chacha_crypt(state, walk.dst.virt.addr, walk.src.virt.addr, - nbytes, ctx->nrounds); - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } - - return err; -} - -static int chacha_mips(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - - return chacha_mips_stream_xor(req, ctx, req->iv); + return true; } +EXPORT_SYMBOL(chacha_is_arch_optimized); -static int xchacha_mips(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - struct chacha_ctx subctx; - u32 state[16]; - u8 real_iv[16]; - - chacha_init(state, ctx->key, req->iv); - - hchacha_block(state, subctx.key, ctx->nrounds); - subctx.nrounds = ctx->nrounds; - - memcpy(&real_iv[0], req->iv + 24, 8); - memcpy(&real_iv[8], req->iv + 16, 8); - return chacha_mips_stream_xor(req, &subctx, real_iv); -} - -static struct skcipher_alg algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-mips", - .base.cra_priority = 200, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = chacha_mips, - .decrypt = chacha_mips, - }, { - .base.cra_name = "xchacha20", - .base.cra_driver_name = "xchacha20-mips", - .base.cra_priority = 200, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = xchacha_mips, - .decrypt = xchacha_mips, - }, { - .base.cra_name = "xchacha12", - .base.cra_driver_name = "xchacha12-mips", - .base.cra_priority = 200, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha12_setkey, - .encrypt = xchacha_mips, - .decrypt = xchacha_mips, - } -}; - -static int __init chacha_simd_mod_init(void) -{ - return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ? - crypto_register_skciphers(algs, ARRAY_SIZE(algs)) : 0; -} - -static void __exit chacha_simd_mod_fini(void) -{ - if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) - crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); -} - -module_init(chacha_simd_mod_init); -module_exit(chacha_simd_mod_fini); - -MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (MIPS accelerated)"); +MODULE_DESCRIPTION("ChaCha and HChaCha functions (MIPS optimized)"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("chacha20"); -MODULE_ALIAS_CRYPTO("chacha20-mips"); -MODULE_ALIAS_CRYPTO("xchacha20"); -MODULE_ALIAS_CRYPTO("xchacha20-mips"); -MODULE_ALIAS_CRYPTO("xchacha12"); -MODULE_ALIAS_CRYPTO("xchacha12-mips"); diff --git a/arch/mips/crypto/poly1305-glue.c b/arch/mips/crypto/poly1305-glue.c index c03ad0bbe69c..576e7a58e0b1 100644 --- a/arch/mips/crypto/poly1305-glue.c +++ b/arch/mips/crypto/poly1305-glue.c @@ -5,13 +5,10 @@ * Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org> */ -#include <linux/unaligned.h> -#include <crypto/algapi.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/poly1305.h> +#include <crypto/poly1305.h> #include <linux/cpufeature.h> -#include <linux/crypto.h> #include <linux/module.h> +#include <linux/unaligned.h> asmlinkage void poly1305_init_mips(void *state, const u8 *key); asmlinkage void poly1305_blocks_mips(void *state, const u8 *src, u32 len, u32 hibit); @@ -28,77 +25,6 @@ void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KE } EXPORT_SYMBOL(poly1305_init_arch); -static int mips_poly1305_init(struct shash_desc *desc) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - dctx->buflen = 0; - dctx->rset = 0; - dctx->sset = false; - - return 0; -} - -static void mips_poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, - u32 len, u32 hibit) -{ - if (unlikely(!dctx->sset)) { - if (!dctx->rset) { - poly1305_init_mips(&dctx->h, src); - src += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - dctx->rset = 1; - } - if (len >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(src + 0); - dctx->s[1] = get_unaligned_le32(src + 4); - dctx->s[2] = get_unaligned_le32(src + 8); - dctx->s[3] = get_unaligned_le32(src + 12); - src += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - if (len < POLY1305_BLOCK_SIZE) - return; - } - - len &= ~(POLY1305_BLOCK_SIZE - 1); - - poly1305_blocks_mips(&dctx->h, src, len, hibit); -} - -static int mips_poly1305_update(struct shash_desc *desc, const u8 *src, - unsigned int len) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (unlikely(dctx->buflen)) { - u32 bytes = min(len, POLY1305_BLOCK_SIZE - dctx->buflen); - - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - len -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - mips_poly1305_blocks(dctx, dctx->buf, POLY1305_BLOCK_SIZE, 1); - dctx->buflen = 0; - } - } - - if (likely(len >= POLY1305_BLOCK_SIZE)) { - mips_poly1305_blocks(dctx, src, len, 1); - src += round_down(len, POLY1305_BLOCK_SIZE); - len %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(len)) { - dctx->buflen = len; - memcpy(dctx->buf, src, len); - } - return 0; -} - void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int nbytes) { @@ -146,47 +72,11 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) } EXPORT_SYMBOL(poly1305_final_arch); -static int mips_poly1305_final(struct shash_desc *desc, u8 *dst) +bool poly1305_is_arch_optimized(void) { - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (unlikely(!dctx->sset)) - return -ENOKEY; - - poly1305_final_arch(dctx, dst); - return 0; + return true; } - -static struct shash_alg mips_poly1305_alg = { - .init = mips_poly1305_init, - .update = mips_poly1305_update, - .final = mips_poly1305_final, - .digestsize = POLY1305_DIGEST_SIZE, - .descsize = sizeof(struct poly1305_desc_ctx), - - .base.cra_name = "poly1305", - .base.cra_driver_name = "poly1305-mips", - .base.cra_priority = 200, - .base.cra_blocksize = POLY1305_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -}; - -static int __init mips_poly1305_mod_init(void) -{ - return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? - crypto_register_shash(&mips_poly1305_alg) : 0; -} - -static void __exit mips_poly1305_mod_exit(void) -{ - if (IS_REACHABLE(CONFIG_CRYPTO_HASH)) - crypto_unregister_shash(&mips_poly1305_alg); -} - -module_init(mips_poly1305_mod_init); -module_exit(mips_poly1305_mod_exit); +EXPORT_SYMBOL(poly1305_is_arch_optimized); MODULE_DESCRIPTION("Poly1305 transform (MIPS accelerated"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("poly1305"); -MODULE_ALIAS_CRYPTO("poly1305-mips"); diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index 370db8192ce6..cbcf76953d83 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -95,29 +95,16 @@ config CRYPTO_AES_GCM_P10 config CRYPTO_CHACHA20_P10 tristate depends on PPC64 && CPU_LITTLE_ENDIAN && VSX - select CRYPTO_SKCIPHER select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA default CRYPTO_LIB_CHACHA_INTERNAL - help - Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 - stream cipher algorithms - - Architecture: PowerPC64 - - Power10 or later - - Little-endian config CRYPTO_POLY1305_P10 - tristate "Hash functions: Poly1305 (P10 or later)" + tristate depends on PPC64 && CPU_LITTLE_ENDIAN && VSX - select CRYPTO_HASH + select CRYPTO_ARCH_HAVE_LIB_POLY1305 select CRYPTO_LIB_POLY1305_GENERIC - help - Poly1305 authenticator algorithm (RFC7539) - - Architecture: PowerPC64 - - Power10 or later - - Little-endian + default CRYPTO_LIB_POLY1305_INTERNAL config CRYPTO_DEV_VMX bool "Support for VMX cryptographic acceleration instructions" diff --git a/arch/powerpc/crypto/aes.c b/arch/powerpc/crypto/aes.c index ec06189fbf99..3f1e5e894902 100644 --- a/arch/powerpc/crypto/aes.c +++ b/arch/powerpc/crypto/aes.c @@ -7,15 +7,15 @@ * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com> */ -#include <linux/types.h> -#include <linux/err.h> -#include <linux/crypto.h> -#include <linux/delay.h> #include <asm/simd.h> #include <asm/switch_to.h> #include <crypto/aes.h> #include <crypto/internal/cipher.h> #include <crypto/internal/simd.h> +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/uaccess.h> #include "aesp8-ppc.h" diff --git a/arch/powerpc/crypto/aes_cbc.c b/arch/powerpc/crypto/aes_cbc.c index ed0debc7acb5..5f2a4f375eef 100644 --- a/arch/powerpc/crypto/aes_cbc.c +++ b/arch/powerpc/crypto/aes_cbc.c @@ -12,6 +12,10 @@ #include <crypto/aes.h> #include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/uaccess.h> #include "aesp8-ppc.h" diff --git a/arch/powerpc/crypto/aes_ctr.c b/arch/powerpc/crypto/aes_ctr.c index 3da75f42529a..e27c4036e711 100644 --- a/arch/powerpc/crypto/aes_ctr.c +++ b/arch/powerpc/crypto/aes_ctr.c @@ -12,6 +12,10 @@ #include <crypto/aes.h> #include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/uaccess.h> #include "aesp8-ppc.h" diff --git a/arch/powerpc/crypto/aes_xts.c b/arch/powerpc/crypto/aes_xts.c index dabbccb41550..9440e771cede 100644 --- a/arch/powerpc/crypto/aes_xts.c +++ b/arch/powerpc/crypto/aes_xts.c @@ -13,6 +13,10 @@ #include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/xts.h> +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/uaccess.h> #include "aesp8-ppc.h" diff --git a/arch/powerpc/crypto/chacha-p10-glue.c b/arch/powerpc/crypto/chacha-p10-glue.c index d8796decc1fb..351ed409f9b2 100644 --- a/arch/powerpc/crypto/chacha-p10-glue.c +++ b/arch/powerpc/crypto/chacha-p10-glue.c @@ -1,15 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * PowerPC P10 (ppc64le) accelerated ChaCha and XChaCha stream ciphers, - * including ChaCha20 (RFC7539) + * ChaCha stream cipher (P10 accelerated) * * Copyright 2023- IBM Corp. All rights reserved. */ -#include <crypto/algapi.h> -#include <crypto/internal/chacha.h> +#include <crypto/chacha.h> #include <crypto/internal/simd.h> -#include <crypto/internal/skcipher.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/cpufeature.h> @@ -78,144 +75,25 @@ void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, } EXPORT_SYMBOL(chacha_crypt_arch); -static int chacha_p10_stream_xor(struct skcipher_request *req, - const struct chacha_ctx *ctx, const u8 *iv) +bool chacha_is_arch_optimized(void) { - struct skcipher_walk walk; - u32 state[16]; - int err; - - err = skcipher_walk_virt(&walk, req, false); - if (err) - return err; - - chacha_init(state, ctx->key, iv); - - while (walk.nbytes > 0) { - unsigned int nbytes = walk.nbytes; - - if (nbytes < walk.total) - nbytes = rounddown(nbytes, walk.stride); - - if (!crypto_simd_usable()) { - chacha_crypt_generic(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, - ctx->nrounds); - } else { - vsx_begin(); - chacha_p10_do_8x(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, ctx->nrounds); - vsx_end(); - } - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - if (err) - break; - } - - return err; -} - -static int chacha_p10(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - - return chacha_p10_stream_xor(req, ctx, req->iv); + return static_key_enabled(&have_p10); } - -static int xchacha_p10(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - struct chacha_ctx subctx; - u32 state[16]; - u8 real_iv[16]; - - chacha_init(state, ctx->key, req->iv); - hchacha_block_arch(state, subctx.key, ctx->nrounds); - subctx.nrounds = ctx->nrounds; - - memcpy(&real_iv[0], req->iv + 24, 8); - memcpy(&real_iv[8], req->iv + 16, 8); - return chacha_p10_stream_xor(req, &subctx, real_iv); -} - -static struct skcipher_alg algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-p10", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = chacha_p10, - .decrypt = chacha_p10, - }, { - .base.cra_name = "xchacha20", - .base.cra_driver_name = "xchacha20-p10", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = xchacha_p10, - .decrypt = xchacha_p10, - }, { - .base.cra_name = "xchacha12", - .base.cra_driver_name = "xchacha12-p10", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha12_setkey, - .encrypt = xchacha_p10, - .decrypt = xchacha_p10, - } -}; +EXPORT_SYMBOL(chacha_is_arch_optimized); static int __init chacha_p10_init(void) { - if (!cpu_has_feature(CPU_FTR_ARCH_31)) - return 0; - - static_branch_enable(&have_p10); - - return crypto_register_skciphers(algs, ARRAY_SIZE(algs)); + if (cpu_has_feature(CPU_FTR_ARCH_31)) + static_branch_enable(&have_p10); + return 0; } +arch_initcall(chacha_p10_init); static void __exit chacha_p10_exit(void) { - if (!static_branch_likely(&have_p10)) - return; - - crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); } - -module_init(chacha_p10_init); module_exit(chacha_p10_exit); -MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (P10 accelerated)"); +MODULE_DESCRIPTION("ChaCha stream cipher (P10 accelerated)"); MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com>"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("chacha20"); -MODULE_ALIAS_CRYPTO("chacha20-p10"); -MODULE_ALIAS_CRYPTO("xchacha20"); -MODULE_ALIAS_CRYPTO("xchacha20-p10"); -MODULE_ALIAS_CRYPTO("xchacha12"); -MODULE_ALIAS_CRYPTO("xchacha12-p10"); diff --git a/arch/powerpc/crypto/ghash.c b/arch/powerpc/crypto/ghash.c index 77eca20bc7ac..7308735bdb33 100644 --- a/arch/powerpc/crypto/ghash.c +++ b/arch/powerpc/crypto/ghash.c @@ -11,19 +11,18 @@ * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> */ -#include <linux/types.h> -#include <linux/err.h> -#include <linux/crypto.h> -#include <linux/delay.h> -#include <asm/simd.h> +#include "aesp8-ppc.h" #include <asm/switch_to.h> #include <crypto/aes.h> +#include <crypto/gf128mul.h> #include <crypto/ghash.h> -#include <crypto/scatterwalk.h> #include <crypto/internal/hash.h> #include <crypto/internal/simd.h> -#include <crypto/b128ops.h> -#include "aesp8-ppc.h" +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> +#include <linux/uaccess.h> void gcm_init_p8(u128 htable[16], const u64 Xi[2]); void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]); @@ -39,15 +38,12 @@ struct p8_ghash_ctx { struct p8_ghash_desc_ctx { u64 shash[2]; - u8 buffer[GHASH_DIGEST_SIZE]; - int bytes; }; static int p8_ghash_init(struct shash_desc *desc) { struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - dctx->bytes = 0; memset(dctx->shash, 0, GHASH_DIGEST_SIZE); return 0; } @@ -74,27 +70,30 @@ static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key, } static inline void __ghash_block(struct p8_ghash_ctx *ctx, - struct p8_ghash_desc_ctx *dctx) + struct p8_ghash_desc_ctx *dctx, + const u8 *src) { if (crypto_simd_usable()) { preempt_disable(); pagefault_disable(); enable_kernel_vsx(); - gcm_ghash_p8(dctx->shash, ctx->htable, - dctx->buffer, GHASH_DIGEST_SIZE); + gcm_ghash_p8(dctx->shash, ctx->htable, src, GHASH_BLOCK_SIZE); disable_kernel_vsx(); pagefault_enable(); preempt_enable(); } else { - crypto_xor((u8 *)dctx->shash, dctx->buffer, GHASH_BLOCK_SIZE); + crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE); gf128mul_lle((be128 *)dctx->shash, &ctx->key); } } -static inline void __ghash_blocks(struct p8_ghash_ctx *ctx, - struct p8_ghash_desc_ctx *dctx, - const u8 *src, unsigned int srclen) +static inline int __ghash_blocks(struct p8_ghash_ctx *ctx, + struct p8_ghash_desc_ctx *dctx, + const u8 *src, unsigned int srclen) { + int remain = srclen - round_down(srclen, GHASH_BLOCK_SIZE); + + srclen -= remain; if (crypto_simd_usable()) { preempt_disable(); pagefault_disable(); @@ -105,62 +104,38 @@ static inline void __ghash_blocks(struct p8_ghash_ctx *ctx, pagefault_enable(); preempt_enable(); } else { - while (srclen >= GHASH_BLOCK_SIZE) { + do { crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE); gf128mul_lle((be128 *)dctx->shash, &ctx->key); srclen -= GHASH_BLOCK_SIZE; src += GHASH_BLOCK_SIZE; - } + } while (srclen); } + + return remain; } static int p8_ghash_update(struct shash_desc *desc, const u8 *src, unsigned int srclen) { - unsigned int len; struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - if (dctx->bytes) { - if (dctx->bytes + srclen < GHASH_DIGEST_SIZE) { - memcpy(dctx->buffer + dctx->bytes, src, - srclen); - dctx->bytes += srclen; - return 0; - } - memcpy(dctx->buffer + dctx->bytes, src, - GHASH_DIGEST_SIZE - dctx->bytes); - - __ghash_block(ctx, dctx); - - src += GHASH_DIGEST_SIZE - dctx->bytes; - srclen -= GHASH_DIGEST_SIZE - dctx->bytes; - dctx->bytes = 0; - } - len = srclen & ~(GHASH_DIGEST_SIZE - 1); - if (len) { - __ghash_blocks(ctx, dctx, src, len); - src += len; - srclen -= len; - } - if (srclen) { - memcpy(dctx->buffer, src, srclen); - dctx->bytes = srclen; - } - return 0; + return __ghash_blocks(ctx, dctx, src, srclen); } -static int p8_ghash_final(struct shash_desc *desc, u8 *out) +static int p8_ghash_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - int i; struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - if (dctx->bytes) { - for (i = dctx->bytes; i < GHASH_DIGEST_SIZE; i++) - dctx->buffer[i] = 0; - __ghash_block(ctx, dctx); - dctx->bytes = 0; + if (len) { + u8 buf[GHASH_BLOCK_SIZE] = {}; + + memcpy(buf, src, len); + __ghash_block(ctx, dctx, buf); + memzero_explicit(buf, sizeof(buf)); } memcpy(out, dctx->shash, GHASH_DIGEST_SIZE); return 0; @@ -170,14 +145,14 @@ struct shash_alg p8_ghash_alg = { .digestsize = GHASH_DIGEST_SIZE, .init = p8_ghash_init, .update = p8_ghash_update, - .final = p8_ghash_final, + .finup = p8_ghash_finup, .setkey = p8_ghash_setkey, - .descsize = sizeof(struct p8_ghash_desc_ctx) - + sizeof(struct ghash_desc_ctx), + .descsize = sizeof(struct p8_ghash_desc_ctx), .base = { .cra_name = "ghash", .cra_driver_name = "p8_ghash", .cra_priority = 1000, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = GHASH_BLOCK_SIZE, .cra_ctxsize = sizeof(struct p8_ghash_ctx), .cra_module = THIS_MODULE, diff --git a/arch/powerpc/crypto/md5-glue.c b/arch/powerpc/crypto/md5-glue.c index c24f605033bd..204440a90cd8 100644 --- a/arch/powerpc/crypto/md5-glue.c +++ b/arch/powerpc/crypto/md5-glue.c @@ -8,25 +8,13 @@ */ #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> #include <crypto/md5.h> -#include <asm/byteorder.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> extern void ppc_md5_transform(u32 *state, const u8 *src, u32 blocks); -static inline void ppc_md5_clear_context(struct md5_state *sctx) -{ - int count = sizeof(struct md5_state) >> 2; - u32 *ptr = (u32 *)sctx; - - /* make sure we can clear the fast way */ - BUILD_BUG_ON(sizeof(struct md5_state) % 4); - do { *ptr++ = 0; } while (--count); -} - static int ppc_md5_init(struct shash_desc *desc) { struct md5_state *sctx = shash_desc_ctx(desc); @@ -44,79 +32,34 @@ static int ppc_md5_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct md5_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->byte_count & 0x3f; - unsigned int avail = 64 - offset; - const u8 *src = data; - sctx->byte_count += len; - - if (avail > len) { - memcpy((char *)sctx->block + offset, src, len); - return 0; - } - - if (offset) { - memcpy((char *)sctx->block + offset, src, avail); - ppc_md5_transform(sctx->hash, (const u8 *)sctx->block, 1); - len -= avail; - src += avail; - } - - if (len > 63) { - ppc_md5_transform(sctx->hash, src, len >> 6); - src += len & ~0x3f; - len &= 0x3f; - } - - memcpy((char *)sctx->block, src, len); - return 0; + sctx->byte_count += round_down(len, MD5_HMAC_BLOCK_SIZE); + ppc_md5_transform(sctx->hash, data, len >> 6); + return len - round_down(len, MD5_HMAC_BLOCK_SIZE); } -static int ppc_md5_final(struct shash_desc *desc, u8 *out) +static int ppc_md5_finup(struct shash_desc *desc, const u8 *src, + unsigned int offset, u8 *out) { struct md5_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->byte_count & 0x3f; - const u8 *src = (const u8 *)sctx->block; - u8 *p = (u8 *)src + offset; - int padlen = 55 - offset; - __le64 *pbits = (__le64 *)((char *)sctx->block + 56); + __le64 block[MD5_BLOCK_WORDS] = {}; + u8 *p = memcpy(block, src, offset); __le32 *dst = (__le32 *)out; + __le64 *pbits; + src = p; + p += offset; *p++ = 0x80; - - if (padlen < 0) { - memset(p, 0x00, padlen + sizeof (u64)); - ppc_md5_transform(sctx->hash, src, 1); - p = (char *)sctx->block; - padlen = 56; - } - - memset(p, 0, padlen); + sctx->byte_count += offset; + pbits = &block[(MD5_BLOCK_WORDS / (offset > 55 ? 1 : 2)) - 1]; *pbits = cpu_to_le64(sctx->byte_count << 3); - ppc_md5_transform(sctx->hash, src, 1); + ppc_md5_transform(sctx->hash, src, (pbits - block + 1) / 8); + memzero_explicit(block, sizeof(block)); dst[0] = cpu_to_le32(sctx->hash[0]); dst[1] = cpu_to_le32(sctx->hash[1]); dst[2] = cpu_to_le32(sctx->hash[2]); dst[3] = cpu_to_le32(sctx->hash[3]); - - ppc_md5_clear_context(sctx); - return 0; -} - -static int ppc_md5_export(struct shash_desc *desc, void *out) -{ - struct md5_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; -} - -static int ppc_md5_import(struct shash_desc *desc, const void *in) -{ - struct md5_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); return 0; } @@ -124,15 +67,13 @@ static struct shash_alg alg = { .digestsize = MD5_DIGEST_SIZE, .init = ppc_md5_init, .update = ppc_md5_update, - .final = ppc_md5_final, - .export = ppc_md5_export, - .import = ppc_md5_import, - .descsize = sizeof(struct md5_state), - .statesize = sizeof(struct md5_state), + .finup = ppc_md5_finup, + .descsize = MD5_STATE_SIZE, .base = { .cra_name = "md5", .cra_driver_name= "md5-ppc", .cra_priority = 200, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = MD5_HMAC_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/powerpc/crypto/poly1305-p10-glue.c b/arch/powerpc/crypto/poly1305-p10-glue.c index 369686e9370b..00617f4c58e6 100644 --- a/arch/powerpc/crypto/poly1305-p10-glue.c +++ b/arch/powerpc/crypto/poly1305-p10-glue.c @@ -4,15 +4,11 @@ * * Copyright 2023- IBM Corp. All rights reserved. */ - -#include <crypto/algapi.h> -#include <linux/crypto.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/jump_label.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/poly1305.h> #include <crypto/internal/simd.h> +#include <crypto/poly1305.h> #include <linux/cpufeature.h> #include <linux/unaligned.h> #include <asm/simd.h> @@ -22,6 +18,8 @@ asmlinkage void poly1305_p10le_4blocks(void *h, const u8 *m, u32 mlen); asmlinkage void poly1305_64s(void *h, const u8 *m, u32 mlen, int highbit); asmlinkage void poly1305_emit_64(void *h, void *s, u8 *dst); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10); + static void vsx_begin(void) { preempt_disable(); @@ -34,51 +32,29 @@ static void vsx_end(void) preempt_enable(); } -static int crypto_poly1305_p10_init(struct shash_desc *desc) +void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) { - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - poly1305_core_init(&dctx->h); + if (!static_key_enabled(&have_p10)) + return poly1305_init_generic(dctx, key); + + dctx->h = (struct poly1305_state){}; + dctx->core_r.key.r64[0] = get_unaligned_le64(key + 0); + dctx->core_r.key.r64[1] = get_unaligned_le64(key + 8); + dctx->s[0] = get_unaligned_le32(key + 16); + dctx->s[1] = get_unaligned_le32(key + 20); + dctx->s[2] = get_unaligned_le32(key + 24); + dctx->s[3] = get_unaligned_le32(key + 28); dctx->buflen = 0; - dctx->rset = 0; - dctx->sset = false; - - return 0; } +EXPORT_SYMBOL(poly1305_init_arch); -static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx, - const u8 *inp, unsigned int len) +void poly1305_update_arch(struct poly1305_desc_ctx *dctx, + const u8 *src, unsigned int srclen) { - unsigned int acc = 0; - - if (unlikely(!dctx->sset)) { - if (!dctx->rset && len >= POLY1305_BLOCK_SIZE) { - struct poly1305_core_key *key = &dctx->core_r; - - key->key.r64[0] = get_unaligned_le64(&inp[0]); - key->key.r64[1] = get_unaligned_le64(&inp[8]); - inp += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - acc += POLY1305_BLOCK_SIZE; - dctx->rset = 1; - } - if (len >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(&inp[0]); - dctx->s[1] = get_unaligned_le32(&inp[4]); - dctx->s[2] = get_unaligned_le32(&inp[8]); - dctx->s[3] = get_unaligned_le32(&inp[12]); - acc += POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - } - return acc; -} + unsigned int bytes; -static int crypto_poly1305_p10_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - unsigned int bytes, used; + if (!static_key_enabled(&have_p10)) + return poly1305_update_generic(dctx, src, srclen); if (unlikely(dctx->buflen)) { bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); @@ -86,26 +62,16 @@ static int crypto_poly1305_p10_update(struct shash_desc *desc, src += bytes; srclen -= bytes; dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - if (likely(!crypto_poly1305_setdctxkey(dctx, dctx->buf, - POLY1305_BLOCK_SIZE))) { - vsx_begin(); - poly1305_64s(&dctx->h, dctx->buf, - POLY1305_BLOCK_SIZE, 1); - vsx_end(); - } - dctx->buflen = 0; - } + if (dctx->buflen < POLY1305_BLOCK_SIZE) + return; + vsx_begin(); + poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1); + vsx_end(); + dctx->buflen = 0; } if (likely(srclen >= POLY1305_BLOCK_SIZE)) { bytes = round_down(srclen, POLY1305_BLOCK_SIZE); - used = crypto_poly1305_setdctxkey(dctx, src, bytes); - if (likely(used)) { - srclen -= used; - src += used; - } if (crypto_simd_usable() && (srclen >= POLY1305_BLOCK_SIZE*4)) { vsx_begin(); poly1305_p10le_4blocks(&dctx->h, src, srclen); @@ -126,61 +92,46 @@ static int crypto_poly1305_p10_update(struct shash_desc *desc, dctx->buflen = srclen; memcpy(dctx->buf, src, srclen); } - - return 0; } +EXPORT_SYMBOL(poly1305_update_arch); -static int crypto_poly1305_p10_final(struct shash_desc *desc, u8 *dst) +void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) { - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (unlikely(!dctx->sset)) - return -ENOKEY; + if (!static_key_enabled(&have_p10)) + return poly1305_final_generic(dctx, dst); - if ((dctx->buflen)) { + if (dctx->buflen) { dctx->buf[dctx->buflen++] = 1; memset(dctx->buf + dctx->buflen, 0, POLY1305_BLOCK_SIZE - dctx->buflen); vsx_begin(); poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); vsx_end(); - dctx->buflen = 0; } poly1305_emit_64(&dctx->h, &dctx->s, dst); - return 0; } +EXPORT_SYMBOL(poly1305_final_arch); -static struct shash_alg poly1305_alg = { - .digestsize = POLY1305_DIGEST_SIZE, - .init = crypto_poly1305_p10_init, - .update = crypto_poly1305_p10_update, - .final = crypto_poly1305_p10_final, - .descsize = sizeof(struct poly1305_desc_ctx), - .base = { - .cra_name = "poly1305", - .cra_driver_name = "poly1305-p10", - .cra_priority = 300, - .cra_blocksize = POLY1305_BLOCK_SIZE, - .cra_module = THIS_MODULE, - }, -}; +bool poly1305_is_arch_optimized(void) +{ + return static_key_enabled(&have_p10); +} +EXPORT_SYMBOL(poly1305_is_arch_optimized); static int __init poly1305_p10_init(void) { - return crypto_register_shash(&poly1305_alg); + if (cpu_has_feature(CPU_FTR_ARCH_31)) + static_branch_enable(&have_p10); + return 0; } +arch_initcall(poly1305_p10_init); static void __exit poly1305_p10_exit(void) { - crypto_unregister_shash(&poly1305_alg); } - -module_cpu_feature_match(PPC_MODULE_FEATURE_P10, poly1305_p10_init); module_exit(poly1305_p10_exit); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com>"); MODULE_DESCRIPTION("Optimized Poly1305 for P10"); -MODULE_ALIAS_CRYPTO("poly1305"); -MODULE_ALIAS_CRYPTO("poly1305-p10"); diff --git a/arch/powerpc/crypto/sha1-spe-glue.c b/arch/powerpc/crypto/sha1-spe-glue.c index 9170892a8557..04c88e173ce1 100644 --- a/arch/powerpc/crypto/sha1-spe-glue.c +++ b/arch/powerpc/crypto/sha1-spe-glue.c @@ -7,16 +7,13 @@ * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> */ +#include <asm/switch_to.h> #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> #include <crypto/sha1.h> #include <crypto/sha1_base.h> -#include <asm/byteorder.h> -#include <asm/switch_to.h> -#include <linux/hardirq.h> +#include <linux/kernel.h> +#include <linux/preempt.h> +#include <linux/module.h> /* * MAX_BYTES defines the number of bytes that are allowed to be processed @@ -30,7 +27,7 @@ */ #define MAX_BYTES 2048 -extern void ppc_spe_sha1_transform(u32 *state, const u8 *src, u32 blocks); +asmlinkage void ppc_spe_sha1_transform(u32 *state, const u8 *src, u32 blocks); static void spe_begin(void) { @@ -46,126 +43,45 @@ static void spe_end(void) preempt_enable(); } -static inline void ppc_sha1_clear_context(struct sha1_state *sctx) +static void ppc_spe_sha1_block(struct sha1_state *sctx, const u8 *src, + int blocks) { - int count = sizeof(struct sha1_state) >> 2; - u32 *ptr = (u32 *)sctx; - - /* make sure we can clear the fast way */ - BUILD_BUG_ON(sizeof(struct sha1_state) % 4); - do { *ptr++ = 0; } while (--count); -} - -static int ppc_spe_sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->count & 0x3f; - const unsigned int avail = 64 - offset; - unsigned int bytes; - const u8 *src = data; - - if (avail > len) { - sctx->count += len; - memcpy((char *)sctx->buffer + offset, src, len); - return 0; - } - - sctx->count += len; - - if (offset) { - memcpy((char *)sctx->buffer + offset, src, avail); + do { + int unit = min(blocks, MAX_BYTES / SHA1_BLOCK_SIZE); spe_begin(); - ppc_spe_sha1_transform(sctx->state, (const u8 *)sctx->buffer, 1); + ppc_spe_sha1_transform(sctx->state, src, unit); spe_end(); - len -= avail; - src += avail; - } - - while (len > 63) { - bytes = (len > MAX_BYTES) ? MAX_BYTES : len; - bytes = bytes & ~0x3f; - - spe_begin(); - ppc_spe_sha1_transform(sctx->state, src, bytes >> 6); - spe_end(); - - src += bytes; - len -= bytes; - } - - memcpy((char *)sctx->buffer, src, len); - return 0; -} - -static int ppc_spe_sha1_final(struct shash_desc *desc, u8 *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->count & 0x3f; - char *p = (char *)sctx->buffer + offset; - int padlen; - __be64 *pbits = (__be64 *)(((char *)&sctx->buffer) + 56); - __be32 *dst = (__be32 *)out; - - padlen = 55 - offset; - *p++ = 0x80; - - spe_begin(); - - if (padlen < 0) { - memset(p, 0x00, padlen + sizeof (u64)); - ppc_spe_sha1_transform(sctx->state, sctx->buffer, 1); - p = (char *)sctx->buffer; - padlen = 56; - } - - memset(p, 0, padlen); - *pbits = cpu_to_be64(sctx->count << 3); - ppc_spe_sha1_transform(sctx->state, sctx->buffer, 1); - - spe_end(); - - dst[0] = cpu_to_be32(sctx->state[0]); - dst[1] = cpu_to_be32(sctx->state[1]); - dst[2] = cpu_to_be32(sctx->state[2]); - dst[3] = cpu_to_be32(sctx->state[3]); - dst[4] = cpu_to_be32(sctx->state[4]); - - ppc_sha1_clear_context(sctx); - return 0; + src += unit * SHA1_BLOCK_SIZE; + blocks -= unit; + } while (blocks); } -static int ppc_spe_sha1_export(struct shash_desc *desc, void *out) +static int ppc_spe_sha1_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; + return sha1_base_do_update_blocks(desc, data, len, ppc_spe_sha1_block); } -static int ppc_spe_sha1_import(struct shash_desc *desc, const void *in) +static int ppc_spe_sha1_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - return 0; + sha1_base_do_finup(desc, src, len, ppc_spe_sha1_block); + return sha1_base_finish(desc, out); } static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = ppc_spe_sha1_update, - .final = ppc_spe_sha1_final, - .export = ppc_spe_sha1_export, - .import = ppc_spe_sha1_import, - .descsize = sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), + .finup = ppc_spe_sha1_finup, + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name= "sha1-ppc-spe", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/powerpc/crypto/sha1.c b/arch/powerpc/crypto/sha1.c index f283bbd3f121..4593946aa9b3 100644 --- a/arch/powerpc/crypto/sha1.c +++ b/arch/powerpc/crypto/sha1.c @@ -13,107 +13,46 @@ * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> */ #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> #include <crypto/sha1.h> #include <crypto/sha1_base.h> -#include <asm/byteorder.h> - -void powerpc_sha_transform(u32 *state, const u8 *src); - -static int powerpc_sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int partial, done; - const u8 *src; - - partial = sctx->count & 0x3f; - sctx->count += len; - done = 0; - src = data; - - if ((partial + len) > 63) { - - if (partial) { - done = -partial; - memcpy(sctx->buffer + partial, data, done + 64); - src = sctx->buffer; - } - - do { - powerpc_sha_transform(sctx->state, src); - done += 64; - src = data + done; - } while (done + 63 < len); - - partial = 0; - } - memcpy(sctx->buffer + partial, src, len - done); - - return 0; -} +#include <linux/kernel.h> +#include <linux/module.h> +asmlinkage void powerpc_sha_transform(u32 *state, const u8 *src); -/* Add padding and return the message digest. */ -static int powerpc_sha1_final(struct shash_desc *desc, u8 *out) +static void powerpc_sha_block(struct sha1_state *sctx, const u8 *data, + int blocks) { - struct sha1_state *sctx = shash_desc_ctx(desc); - __be32 *dst = (__be32 *)out; - u32 i, index, padlen; - __be64 bits; - static const u8 padding[64] = { 0x80, }; - - bits = cpu_to_be64(sctx->count << 3); - - /* Pad out to 56 mod 64 */ - index = sctx->count & 0x3f; - padlen = (index < 56) ? (56 - index) : ((64+56) - index); - powerpc_sha1_update(desc, padding, padlen); - - /* Append length */ - powerpc_sha1_update(desc, (const u8 *)&bits, sizeof(bits)); - - /* Store state in digest */ - for (i = 0; i < 5; i++) - dst[i] = cpu_to_be32(sctx->state[i]); - - /* Wipe context */ - memset(sctx, 0, sizeof *sctx); - - return 0; + do { + powerpc_sha_transform(sctx->state, data); + data += 64; + } while (--blocks); } -static int powerpc_sha1_export(struct shash_desc *desc, void *out) +static int powerpc_sha1_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; + return sha1_base_do_update_blocks(desc, data, len, powerpc_sha_block); } -static int powerpc_sha1_import(struct shash_desc *desc, const void *in) +/* Add padding and return the message digest. */ +static int powerpc_sha1_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - return 0; + sha1_base_do_finup(desc, src, len, powerpc_sha_block); + return sha1_base_finish(desc, out); } static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = powerpc_sha1_update, - .final = powerpc_sha1_final, - .export = powerpc_sha1_export, - .import = powerpc_sha1_import, - .descsize = sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), + .finup = powerpc_sha1_finup, + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name= "sha1-powerpc", + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/powerpc/crypto/sha256-spe-glue.c b/arch/powerpc/crypto/sha256-spe-glue.c index 2997d13236e0..42c76bf8062d 100644 --- a/arch/powerpc/crypto/sha256-spe-glue.c +++ b/arch/powerpc/crypto/sha256-spe-glue.c @@ -8,16 +8,13 @@ * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> */ +#include <asm/switch_to.h> #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> #include <crypto/sha2.h> #include <crypto/sha256_base.h> -#include <asm/byteorder.h> -#include <asm/switch_to.h> -#include <linux/hardirq.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/preempt.h> /* * MAX_BYTES defines the number of bytes that are allowed to be processed @@ -47,151 +44,48 @@ static void spe_end(void) preempt_enable(); } -static inline void ppc_sha256_clear_context(struct sha256_state *sctx) -{ - int count = sizeof(struct sha256_state) >> 2; - u32 *ptr = (u32 *)sctx; - - /* make sure we can clear the fast way */ - BUILD_BUG_ON(sizeof(struct sha256_state) % 4); - do { *ptr++ = 0; } while (--count); -} - -static int ppc_spe_sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static void ppc_spe_sha256_block(struct crypto_sha256_state *sctx, + const u8 *src, int blocks) { - struct sha256_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->count & 0x3f; - const unsigned int avail = 64 - offset; - unsigned int bytes; - const u8 *src = data; - - if (avail > len) { - sctx->count += len; - memcpy((char *)sctx->buf + offset, src, len); - return 0; - } - - sctx->count += len; - - if (offset) { - memcpy((char *)sctx->buf + offset, src, avail); - - spe_begin(); - ppc_spe_sha256_transform(sctx->state, (const u8 *)sctx->buf, 1); - spe_end(); - - len -= avail; - src += avail; - } - - while (len > 63) { + do { /* cut input data into smaller blocks */ - bytes = (len > MAX_BYTES) ? MAX_BYTES : len; - bytes = bytes & ~0x3f; + int unit = min(blocks, MAX_BYTES / SHA256_BLOCK_SIZE); spe_begin(); - ppc_spe_sha256_transform(sctx->state, src, bytes >> 6); + ppc_spe_sha256_transform(sctx->state, src, unit); spe_end(); - src += bytes; - len -= bytes; - } - - memcpy((char *)sctx->buf, src, len); - return 0; -} - -static int ppc_spe_sha256_final(struct shash_desc *desc, u8 *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->count & 0x3f; - char *p = (char *)sctx->buf + offset; - int padlen; - __be64 *pbits = (__be64 *)(((char *)&sctx->buf) + 56); - __be32 *dst = (__be32 *)out; - - padlen = 55 - offset; - *p++ = 0x80; - - spe_begin(); - - if (padlen < 0) { - memset(p, 0x00, padlen + sizeof (u64)); - ppc_spe_sha256_transform(sctx->state, sctx->buf, 1); - p = (char *)sctx->buf; - padlen = 56; - } - - memset(p, 0, padlen); - *pbits = cpu_to_be64(sctx->count << 3); - ppc_spe_sha256_transform(sctx->state, sctx->buf, 1); - - spe_end(); - - dst[0] = cpu_to_be32(sctx->state[0]); - dst[1] = cpu_to_be32(sctx->state[1]); - dst[2] = cpu_to_be32(sctx->state[2]); - dst[3] = cpu_to_be32(sctx->state[3]); - dst[4] = cpu_to_be32(sctx->state[4]); - dst[5] = cpu_to_be32(sctx->state[5]); - dst[6] = cpu_to_be32(sctx->state[6]); - dst[7] = cpu_to_be32(sctx->state[7]); - - ppc_sha256_clear_context(sctx); - return 0; -} - -static int ppc_spe_sha224_final(struct shash_desc *desc, u8 *out) -{ - __be32 D[SHA256_DIGEST_SIZE >> 2]; - __be32 *dst = (__be32 *)out; - - ppc_spe_sha256_final(desc, (u8 *)D); - - /* avoid bytewise memcpy */ - dst[0] = D[0]; - dst[1] = D[1]; - dst[2] = D[2]; - dst[3] = D[3]; - dst[4] = D[4]; - dst[5] = D[5]; - dst[6] = D[6]; - - /* clear sensitive data */ - memzero_explicit(D, SHA256_DIGEST_SIZE); - return 0; + src += unit * SHA256_BLOCK_SIZE; + blocks -= unit; + } while (blocks); } -static int ppc_spe_sha256_export(struct shash_desc *desc, void *out) +static int ppc_spe_sha256_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - struct sha256_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; + return sha256_base_do_update_blocks(desc, data, len, + ppc_spe_sha256_block); } -static int ppc_spe_sha256_import(struct shash_desc *desc, const void *in) +static int ppc_spe_sha256_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - struct sha256_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - return 0; + sha256_base_do_finup(desc, src, len, ppc_spe_sha256_block); + return sha256_base_finish(desc, out); } static struct shash_alg algs[2] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = ppc_spe_sha256_update, - .final = ppc_spe_sha256_final, - .export = ppc_spe_sha256_export, - .import = ppc_spe_sha256_import, - .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), + .finup = ppc_spe_sha256_finup, + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name= "sha256-ppc-spe", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -199,15 +93,14 @@ static struct shash_alg algs[2] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = ppc_spe_sha256_update, - .final = ppc_spe_sha224_final, - .export = ppc_spe_sha256_export, - .import = ppc_spe_sha256_import, - .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), + .finup = ppc_spe_sha256_finup, + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name= "sha224-ppc-spe", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/powerpc/lib/crc-t10dif-glue.c b/arch/powerpc/lib/crc-t10dif-glue.c index f411b0120cc5..fa988e249f01 100644 --- a/arch/powerpc/lib/crc-t10dif-glue.c +++ b/arch/powerpc/lib/crc-t10dif-glue.c @@ -6,15 +6,15 @@ * [based on crc32c-vpmsum_glue.c] */ -#include <linux/crc-t10dif.h> +#include <asm/switch_to.h> #include <crypto/internal/simd.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/string.h> -#include <linux/kernel.h> #include <linux/cpufeature.h> -#include <asm/simd.h> -#include <asm/switch_to.h> +#include <linux/crc-t10dif.h> +#include <linux/jump_label.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/preempt.h> +#include <linux/uaccess.h> #define VMX_ALIGN 16 #define VMX_ALIGN_MASK (VMX_ALIGN-1) diff --git a/arch/powerpc/lib/crc32-glue.c b/arch/powerpc/lib/crc32-glue.c index dbd10f339183..28450fe04e86 100644 --- a/arch/powerpc/lib/crc32-glue.c +++ b/arch/powerpc/lib/crc32-glue.c @@ -1,12 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only -#include <linux/crc32.h> +#include <asm/switch_to.h> #include <crypto/internal/simd.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/kernel.h> #include <linux/cpufeature.h> -#include <asm/simd.h> -#include <asm/switch_to.h> +#include <linux/crc32.h> +#include <linux/jump_label.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/preempt.h> +#include <linux/uaccess.h> #define VMX_ALIGN 16 #define VMX_ALIGN_MASK (VMX_ALIGN-1) diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index c67095a3d669..27a1f26d41bd 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -19,14 +19,11 @@ config CRYPTO_AES_RISCV64 - Zvkg vector crypto extension (XTS) config CRYPTO_CHACHA_RISCV64 - tristate "Ciphers: ChaCha" + tristate depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO - select CRYPTO_SKCIPHER - help - Length-preserving ciphers: ChaCha20 stream cipher algorithm - - Architecture: riscv64 using: - - Zvkb vector crypto extension + select CRYPTO_ARCH_HAVE_LIB_CHACHA + select CRYPTO_LIB_CHACHA_GENERIC + default CRYPTO_LIB_CHACHA_INTERNAL config CRYPTO_GHASH_RISCV64 tristate "Hash functions: GHASH" @@ -64,7 +61,7 @@ config CRYPTO_SM3_RISCV64 tristate "Hash functions: SM3 (ShangMi 3)" depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO select CRYPTO_HASH - select CRYPTO_SM3 + select CRYPTO_LIB_SM3 help SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) diff --git a/arch/riscv/crypto/chacha-riscv64-glue.c b/arch/riscv/crypto/chacha-riscv64-glue.c index 10b46f36375a..afc4e3be3cac 100644 --- a/arch/riscv/crypto/chacha-riscv64-glue.c +++ b/arch/riscv/crypto/chacha-riscv64-glue.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * ChaCha20 using the RISC-V vector crypto extensions + * ChaCha stream cipher (RISC-V optimized) * * Copyright (C) 2023 SiFive, Inc. * Author: Jerry Shih <jerry.shih@sifive.com> @@ -8,94 +8,67 @@ #include <asm/simd.h> #include <asm/vector.h> -#include <crypto/internal/chacha.h> -#include <crypto/internal/skcipher.h> +#include <crypto/chacha.h> +#include <crypto/internal/simd.h> #include <linux/linkage.h> #include <linux/module.h> -asmlinkage void chacha20_zvkb(const u32 key[8], const u8 *in, u8 *out, - size_t len, const u32 iv[4]); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_zvkb); -static int riscv64_chacha20_crypt(struct skcipher_request *req) +asmlinkage void chacha_zvkb(u32 state[16], const u8 *in, u8 *out, + size_t nblocks, int nrounds); + +void hchacha_block_arch(const u32 *state, u32 *out, int nrounds) { - u32 iv[CHACHA_IV_SIZE / sizeof(u32)]; - u8 block_buffer[CHACHA_BLOCK_SIZE]; - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - const struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - unsigned int nbytes; - unsigned int tail_bytes; - int err; + hchacha_block_generic(state, out, nrounds); +} +EXPORT_SYMBOL(hchacha_block_arch); - iv[0] = get_unaligned_le32(req->iv); - iv[1] = get_unaligned_le32(req->iv + 4); - iv[2] = get_unaligned_le32(req->iv + 8); - iv[3] = get_unaligned_le32(req->iv + 12); +void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, + int nrounds) +{ + u8 block_buffer[CHACHA_BLOCK_SIZE]; + unsigned int full_blocks = bytes / CHACHA_BLOCK_SIZE; + unsigned int tail_bytes = bytes % CHACHA_BLOCK_SIZE; - err = skcipher_walk_virt(&walk, req, false); - while (walk.nbytes) { - nbytes = walk.nbytes & ~(CHACHA_BLOCK_SIZE - 1); - tail_bytes = walk.nbytes & (CHACHA_BLOCK_SIZE - 1); - kernel_vector_begin(); - if (nbytes) { - chacha20_zvkb(ctx->key, walk.src.virt.addr, - walk.dst.virt.addr, nbytes, iv); - iv[0] += nbytes / CHACHA_BLOCK_SIZE; - } - if (walk.nbytes == walk.total && tail_bytes > 0) { - memcpy(block_buffer, walk.src.virt.addr + nbytes, - tail_bytes); - chacha20_zvkb(ctx->key, block_buffer, block_buffer, - CHACHA_BLOCK_SIZE, iv); - memcpy(walk.dst.virt.addr + nbytes, block_buffer, - tail_bytes); - tail_bytes = 0; - } - kernel_vector_end(); + if (!static_branch_likely(&use_zvkb) || !crypto_simd_usable()) + return chacha_crypt_generic(state, dst, src, bytes, nrounds); - err = skcipher_walk_done(&walk, tail_bytes); + kernel_vector_begin(); + if (full_blocks) { + chacha_zvkb(state, src, dst, full_blocks, nrounds); + src += full_blocks * CHACHA_BLOCK_SIZE; + dst += full_blocks * CHACHA_BLOCK_SIZE; } - - return err; + if (tail_bytes) { + memcpy(block_buffer, src, tail_bytes); + chacha_zvkb(state, block_buffer, block_buffer, 1, nrounds); + memcpy(dst, block_buffer, tail_bytes); + } + kernel_vector_end(); } +EXPORT_SYMBOL(chacha_crypt_arch); -static struct skcipher_alg riscv64_chacha_alg = { - .setkey = chacha20_setkey, - .encrypt = riscv64_chacha20_crypt, - .decrypt = riscv64_chacha20_crypt, - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .walksize = 4 * CHACHA_BLOCK_SIZE, - .base = { - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct chacha_ctx), - .cra_priority = 300, - .cra_name = "chacha20", - .cra_driver_name = "chacha20-riscv64-zvkb", - .cra_module = THIS_MODULE, - }, -}; +bool chacha_is_arch_optimized(void) +{ + return static_key_enabled(&use_zvkb); +} +EXPORT_SYMBOL(chacha_is_arch_optimized); static int __init riscv64_chacha_mod_init(void) { if (riscv_isa_extension_available(NULL, ZVKB) && riscv_vector_vlen() >= 128) - return crypto_register_skcipher(&riscv64_chacha_alg); - - return -ENODEV; + static_branch_enable(&use_zvkb); + return 0; } +arch_initcall(riscv64_chacha_mod_init); static void __exit riscv64_chacha_mod_exit(void) { - crypto_unregister_skcipher(&riscv64_chacha_alg); } - -module_init(riscv64_chacha_mod_init); module_exit(riscv64_chacha_mod_exit); -MODULE_DESCRIPTION("ChaCha20 (RISC-V accelerated)"); +MODULE_DESCRIPTION("ChaCha stream cipher (RISC-V optimized)"); MODULE_AUTHOR("Jerry Shih <jerry.shih@sifive.com>"); MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("chacha20"); diff --git a/arch/riscv/crypto/chacha-riscv64-zvkb.S b/arch/riscv/crypto/chacha-riscv64-zvkb.S index bf057737ac69..ab4423b3880e 100644 --- a/arch/riscv/crypto/chacha-riscv64-zvkb.S +++ b/arch/riscv/crypto/chacha-riscv64-zvkb.S @@ -46,11 +46,11 @@ .text .option arch, +zvkb -#define KEYP a0 +#define STATEP a0 #define INP a1 #define OUTP a2 -#define LEN a3 -#define IVP a4 +#define NBLOCKS a3 +#define NROUNDS a4 #define CONSTS0 a5 #define CONSTS1 a6 @@ -59,7 +59,7 @@ #define TMP t1 #define VL t2 #define STRIDE t3 -#define NROUNDS t4 +#define ROUND_CTR t4 #define KEY0 s0 #define KEY1 s1 #define KEY2 s2 @@ -132,14 +132,16 @@ vror.vi \b3, \b3, 32 - 7 .endm -// void chacha20_zvkb(const u32 key[8], const u8 *in, u8 *out, size_t len, -// const u32 iv[4]); +// void chacha_zvkb(u32 state[16], const u8 *in, u8 *out, size_t nblocks, +// int nrounds); // -// |len| must be nonzero and a multiple of 64 (CHACHA_BLOCK_SIZE). -// The counter is treated as 32-bit, following the RFC7539 convention. -SYM_FUNC_START(chacha20_zvkb) - srli LEN, LEN, 6 // Bytes to blocks - +// |nblocks| is the number of 64-byte blocks to process, and must be nonzero. +// +// |state| gives the ChaCha state matrix, including the 32-bit counter in +// state[12] following the RFC7539 convention; note that this differs from the +// original Salsa20 paper which uses a 64-bit counter in state[12..13]. The +// updated 32-bit counter is written back to state[12] before returning. +SYM_FUNC_START(chacha_zvkb) addi sp, sp, -96 sd s0, 0(sp) sd s1, 8(sp) @@ -157,26 +159,26 @@ SYM_FUNC_START(chacha20_zvkb) li STRIDE, 64 // Set up the initial state matrix in scalar registers. - li CONSTS0, 0x61707865 // "expa" little endian - li CONSTS1, 0x3320646e // "nd 3" little endian - li CONSTS2, 0x79622d32 // "2-by" little endian - li CONSTS3, 0x6b206574 // "te k" little endian - lw KEY0, 0(KEYP) - lw KEY1, 4(KEYP) - lw KEY2, 8(KEYP) - lw KEY3, 12(KEYP) - lw KEY4, 16(KEYP) - lw KEY5, 20(KEYP) - lw KEY6, 24(KEYP) - lw KEY7, 28(KEYP) - lw COUNTER, 0(IVP) - lw NONCE0, 4(IVP) - lw NONCE1, 8(IVP) - lw NONCE2, 12(IVP) + lw CONSTS0, 0(STATEP) + lw CONSTS1, 4(STATEP) + lw CONSTS2, 8(STATEP) + lw CONSTS3, 12(STATEP) + lw KEY0, 16(STATEP) + lw KEY1, 20(STATEP) + lw KEY2, 24(STATEP) + lw KEY3, 28(STATEP) + lw KEY4, 32(STATEP) + lw KEY5, 36(STATEP) + lw KEY6, 40(STATEP) + lw KEY7, 44(STATEP) + lw COUNTER, 48(STATEP) + lw NONCE0, 52(STATEP) + lw NONCE1, 56(STATEP) + lw NONCE2, 60(STATEP) .Lblock_loop: // Set vl to the number of blocks to process in this iteration. - vsetvli VL, LEN, e32, m1, ta, ma + vsetvli VL, NBLOCKS, e32, m1, ta, ma // Set up the initial state matrix for the next VL blocks in v0-v15. // v{i} holds the i'th 32-bit word of the state matrix for all blocks. @@ -203,16 +205,16 @@ SYM_FUNC_START(chacha20_zvkb) // v{16+i} holds the i'th 32-bit word for all blocks. vlsseg8e32.v v16, (INP), STRIDE - li NROUNDS, 20 + mv ROUND_CTR, NROUNDS .Lnext_doubleround: - addi NROUNDS, NROUNDS, -2 + addi ROUND_CTR, ROUND_CTR, -2 // column round chacha_round v0, v4, v8, v12, v1, v5, v9, v13, \ v2, v6, v10, v14, v3, v7, v11, v15 // diagonal round chacha_round v0, v5, v10, v15, v1, v6, v11, v12, \ v2, v7, v8, v13, v3, v4, v9, v14 - bnez NROUNDS, .Lnext_doubleround + bnez ROUND_CTR, .Lnext_doubleround // Load the second half of the input data for each block into v24-v31. // v{24+i} holds the {8+i}'th 32-bit word for all blocks. @@ -271,12 +273,13 @@ SYM_FUNC_START(chacha20_zvkb) // Update the counter, the remaining number of blocks, and the input and // output pointers according to the number of blocks processed (VL). add COUNTER, COUNTER, VL - sub LEN, LEN, VL + sub NBLOCKS, NBLOCKS, VL slli TMP, VL, 6 add OUTP, OUTP, TMP add INP, INP, TMP - bnez LEN, .Lblock_loop + bnez NBLOCKS, .Lblock_loop + sw COUNTER, 48(STATEP) ld s0, 0(sp) ld s1, 8(sp) ld s2, 16(sp) @@ -291,4 +294,4 @@ SYM_FUNC_START(chacha20_zvkb) ld s11, 88(sp) addi sp, sp, 96 ret -SYM_FUNC_END(chacha20_zvkb) +SYM_FUNC_END(chacha_zvkb) diff --git a/arch/riscv/crypto/ghash-riscv64-glue.c b/arch/riscv/crypto/ghash-riscv64-glue.c index 312e7891fd0a..d86073d25387 100644 --- a/arch/riscv/crypto/ghash-riscv64-glue.c +++ b/arch/riscv/crypto/ghash-riscv64-glue.c @@ -11,11 +11,16 @@ #include <asm/simd.h> #include <asm/vector.h> +#include <crypto/b128ops.h> +#include <crypto/gf128mul.h> #include <crypto/ghash.h> #include <crypto/internal/hash.h> #include <crypto/internal/simd.h> -#include <linux/linkage.h> +#include <crypto/utils.h> +#include <linux/errno.h> +#include <linux/kernel.h> #include <linux/module.h> +#include <linux/string.h> asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data, size_t len); @@ -26,8 +31,6 @@ struct riscv64_ghash_tfm_ctx { struct riscv64_ghash_desc_ctx { be128 accumulator; - u8 buffer[GHASH_BLOCK_SIZE]; - u32 bytes; }; static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key, @@ -78,50 +81,24 @@ static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src, { const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - unsigned int len; - - if (dctx->bytes) { - if (dctx->bytes + srclen < GHASH_BLOCK_SIZE) { - memcpy(dctx->buffer + dctx->bytes, src, srclen); - dctx->bytes += srclen; - return 0; - } - memcpy(dctx->buffer + dctx->bytes, src, - GHASH_BLOCK_SIZE - dctx->bytes); - riscv64_ghash_blocks(tctx, dctx, dctx->buffer, - GHASH_BLOCK_SIZE); - src += GHASH_BLOCK_SIZE - dctx->bytes; - srclen -= GHASH_BLOCK_SIZE - dctx->bytes; - dctx->bytes = 0; - } - - len = round_down(srclen, GHASH_BLOCK_SIZE); - if (len) { - riscv64_ghash_blocks(tctx, dctx, src, len); - src += len; - srclen -= len; - } - if (srclen) { - memcpy(dctx->buffer, src, srclen); - dctx->bytes = srclen; - } - - return 0; + riscv64_ghash_blocks(tctx, dctx, src, + round_down(srclen, GHASH_BLOCK_SIZE)); + return srclen - round_down(srclen, GHASH_BLOCK_SIZE); } -static int riscv64_ghash_final(struct shash_desc *desc, u8 *out) +static int riscv64_ghash_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - int i; - if (dctx->bytes) { - for (i = dctx->bytes; i < GHASH_BLOCK_SIZE; i++) - dctx->buffer[i] = 0; + if (len) { + u8 buf[GHASH_BLOCK_SIZE] = {}; - riscv64_ghash_blocks(tctx, dctx, dctx->buffer, - GHASH_BLOCK_SIZE); + memcpy(buf, src, len); + riscv64_ghash_blocks(tctx, dctx, buf, GHASH_BLOCK_SIZE); + memzero_explicit(buf, sizeof(buf)); } memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE); @@ -131,7 +108,7 @@ static int riscv64_ghash_final(struct shash_desc *desc, u8 *out) static struct shash_alg riscv64_ghash_alg = { .init = riscv64_ghash_init, .update = riscv64_ghash_update, - .final = riscv64_ghash_final, + .finup = riscv64_ghash_finup, .setkey = riscv64_ghash_setkey, .descsize = sizeof(struct riscv64_ghash_desc_ctx), .digestsize = GHASH_DIGEST_SIZE, @@ -139,6 +116,7 @@ static struct shash_alg riscv64_ghash_alg = { .cra_blocksize = GHASH_BLOCK_SIZE, .cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx), .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_name = "ghash", .cra_driver_name = "ghash-riscv64-zvkg", .cra_module = THIS_MODULE, diff --git a/arch/riscv/crypto/sha256-riscv64-glue.c b/arch/riscv/crypto/sha256-riscv64-glue.c index 71e051e40a64..c998300ab843 100644 --- a/arch/riscv/crypto/sha256-riscv64-glue.c +++ b/arch/riscv/crypto/sha256-riscv64-glue.c @@ -14,7 +14,7 @@ #include <crypto/internal/hash.h> #include <crypto/internal/simd.h> #include <crypto/sha256_base.h> -#include <linux/linkage.h> +#include <linux/kernel.h> #include <linux/module.h> /* @@ -22,50 +22,36 @@ * It is assumed to be the first field. */ asmlinkage void sha256_transform_zvknha_or_zvknhb_zvkb( - struct sha256_state *state, const u8 *data, int num_blocks); + struct crypto_sha256_state *state, const u8 *data, int num_blocks); -static int riscv64_sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static void sha256_block(struct crypto_sha256_state *state, const u8 *data, + int num_blocks) { /* - * Ensure struct sha256_state begins directly with the SHA-256 + * Ensure struct crypto_sha256_state begins directly with the SHA-256 * 256-bit internal state, as this is what the asm function expects. */ - BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0); + BUILD_BUG_ON(offsetof(struct crypto_sha256_state, state) != 0); if (crypto_simd_usable()) { kernel_vector_begin(); - sha256_base_do_update(desc, data, len, - sha256_transform_zvknha_or_zvknhb_zvkb); + sha256_transform_zvknha_or_zvknhb_zvkb(state, data, num_blocks); kernel_vector_end(); - } else { - crypto_sha256_update(desc, data, len); - } - return 0; + } else + sha256_transform_blocks(state, data, num_blocks); } -static int riscv64_sha256_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) +static int riscv64_sha256_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - if (crypto_simd_usable()) { - kernel_vector_begin(); - if (len) - sha256_base_do_update( - desc, data, len, - sha256_transform_zvknha_or_zvknhb_zvkb); - sha256_base_do_finalize( - desc, sha256_transform_zvknha_or_zvknhb_zvkb); - kernel_vector_end(); - - return sha256_base_finish(desc, out); - } - - return crypto_sha256_finup(desc, data, len, out); + return sha256_base_do_update_blocks(desc, data, len, sha256_block); } -static int riscv64_sha256_final(struct shash_desc *desc, u8 *out) +static int riscv64_sha256_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - return riscv64_sha256_finup(desc, NULL, 0, out); + sha256_base_do_finup(desc, data, len, sha256_block); + return sha256_base_finish(desc, out); } static int riscv64_sha256_digest(struct shash_desc *desc, const u8 *data, @@ -79,13 +65,14 @@ static struct shash_alg riscv64_sha256_algs[] = { { .init = sha256_base_init, .update = riscv64_sha256_update, - .final = riscv64_sha256_final, .finup = riscv64_sha256_finup, .digest = riscv64_sha256_digest, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .digestsize = SHA256_DIGEST_SIZE, .base = { .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_priority = 300, .cra_name = "sha256", .cra_driver_name = "sha256-riscv64-zvknha_or_zvknhb-zvkb", @@ -94,12 +81,13 @@ static struct shash_alg riscv64_sha256_algs[] = { }, { .init = sha224_base_init, .update = riscv64_sha256_update, - .final = riscv64_sha256_final, .finup = riscv64_sha256_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .digestsize = SHA224_DIGEST_SIZE, .base = { .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_priority = 300, .cra_name = "sha224", .cra_driver_name = "sha224-riscv64-zvknha_or_zvknhb-zvkb", diff --git a/arch/riscv/crypto/sha512-riscv64-glue.c b/arch/riscv/crypto/sha512-riscv64-glue.c index 43b56a08aeb5..4634fca78ae2 100644 --- a/arch/riscv/crypto/sha512-riscv64-glue.c +++ b/arch/riscv/crypto/sha512-riscv64-glue.c @@ -14,7 +14,7 @@ #include <crypto/internal/hash.h> #include <crypto/internal/simd.h> #include <crypto/sha512_base.h> -#include <linux/linkage.h> +#include <linux/kernel.h> #include <linux/module.h> /* @@ -24,8 +24,8 @@ asmlinkage void sha512_transform_zvknhb_zvkb( struct sha512_state *state, const u8 *data, int num_blocks); -static int riscv64_sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static void sha512_block(struct sha512_state *state, const u8 *data, + int num_blocks) { /* * Ensure struct sha512_state begins directly with the SHA-512 @@ -35,35 +35,24 @@ static int riscv64_sha512_update(struct shash_desc *desc, const u8 *data, if (crypto_simd_usable()) { kernel_vector_begin(); - sha512_base_do_update(desc, data, len, - sha512_transform_zvknhb_zvkb); + sha512_transform_zvknhb_zvkb(state, data, num_blocks); kernel_vector_end(); } else { - crypto_sha512_update(desc, data, len); + sha512_generic_block_fn(state, data, num_blocks); } - return 0; } -static int riscv64_sha512_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) +static int riscv64_sha512_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - if (crypto_simd_usable()) { - kernel_vector_begin(); - if (len) - sha512_base_do_update(desc, data, len, - sha512_transform_zvknhb_zvkb); - sha512_base_do_finalize(desc, sha512_transform_zvknhb_zvkb); - kernel_vector_end(); - - return sha512_base_finish(desc, out); - } - - return crypto_sha512_finup(desc, data, len, out); + return sha512_base_do_update_blocks(desc, data, len, sha512_block); } -static int riscv64_sha512_final(struct shash_desc *desc, u8 *out) +static int riscv64_sha512_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - return riscv64_sha512_finup(desc, NULL, 0, out); + sha512_base_do_finup(desc, data, len, sha512_block); + return sha512_base_finish(desc, out); } static int riscv64_sha512_digest(struct shash_desc *desc, const u8 *data, @@ -77,14 +66,15 @@ static struct shash_alg riscv64_sha512_algs[] = { { .init = sha512_base_init, .update = riscv64_sha512_update, - .final = riscv64_sha512_final, .finup = riscv64_sha512_finup, .digest = riscv64_sha512_digest, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .digestsize = SHA512_DIGEST_SIZE, .base = { .cra_blocksize = SHA512_BLOCK_SIZE, .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_name = "sha512", .cra_driver_name = "sha512-riscv64-zvknhb-zvkb", .cra_module = THIS_MODULE, @@ -92,13 +82,14 @@ static struct shash_alg riscv64_sha512_algs[] = { }, { .init = sha384_base_init, .update = riscv64_sha512_update, - .final = riscv64_sha512_final, .finup = riscv64_sha512_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .digestsize = SHA384_DIGEST_SIZE, .base = { .cra_blocksize = SHA384_BLOCK_SIZE, .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_name = "sha384", .cra_driver_name = "sha384-riscv64-zvknhb-zvkb", .cra_module = THIS_MODULE, diff --git a/arch/riscv/crypto/sm3-riscv64-glue.c b/arch/riscv/crypto/sm3-riscv64-glue.c index e1737a970c7c..abdfe4a63a27 100644 --- a/arch/riscv/crypto/sm3-riscv64-glue.c +++ b/arch/riscv/crypto/sm3-riscv64-glue.c @@ -13,8 +13,9 @@ #include <asm/vector.h> #include <crypto/internal/hash.h> #include <crypto/internal/simd.h> +#include <crypto/sm3.h> #include <crypto/sm3_base.h> -#include <linux/linkage.h> +#include <linux/kernel.h> #include <linux/module.h> /* @@ -24,8 +25,8 @@ asmlinkage void sm3_transform_zvksh_zvkb( struct sm3_state *state, const u8 *data, int num_blocks); -static int riscv64_sm3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static void sm3_block(struct sm3_state *state, const u8 *data, + int num_blocks) { /* * Ensure struct sm3_state begins directly with the SM3 @@ -35,52 +36,36 @@ static int riscv64_sm3_update(struct shash_desc *desc, const u8 *data, if (crypto_simd_usable()) { kernel_vector_begin(); - sm3_base_do_update(desc, data, len, sm3_transform_zvksh_zvkb); + sm3_transform_zvksh_zvkb(state, data, num_blocks); kernel_vector_end(); } else { - sm3_update(shash_desc_ctx(desc), data, len); + sm3_block_generic(state, data, num_blocks); } - return 0; } -static int riscv64_sm3_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) +static int riscv64_sm3_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - struct sm3_state *ctx; - - if (crypto_simd_usable()) { - kernel_vector_begin(); - if (len) - sm3_base_do_update(desc, data, len, - sm3_transform_zvksh_zvkb); - sm3_base_do_finalize(desc, sm3_transform_zvksh_zvkb); - kernel_vector_end(); - - return sm3_base_finish(desc, out); - } - - ctx = shash_desc_ctx(desc); - if (len) - sm3_update(ctx, data, len); - sm3_final(ctx, out); - - return 0; + return sm3_base_do_update_blocks(desc, data, len, sm3_block); } -static int riscv64_sm3_final(struct shash_desc *desc, u8 *out) +static int riscv64_sm3_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - return riscv64_sm3_finup(desc, NULL, 0, out); + sm3_base_do_finup(desc, data, len, sm3_block); + return sm3_base_finish(desc, out); } static struct shash_alg riscv64_sm3_alg = { .init = sm3_base_init, .update = riscv64_sm3_update, - .final = riscv64_sm3_final, .finup = riscv64_sm3_finup, - .descsize = sizeof(struct sm3_state), + .descsize = SM3_STATE_SIZE, .digestsize = SM3_DIGEST_SIZE, .base = { .cra_blocksize = SM3_BLOCK_SIZE, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_priority = 300, .cra_name = "sm3", .cra_driver_name = "sm3-riscv64-zvksh-zvkb", diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig index 8c4db8b64fa2..055b08f259ab 100644 --- a/arch/s390/crypto/Kconfig +++ b/arch/s390/crypto/Kconfig @@ -110,16 +110,9 @@ config CRYPTO_DES_S390 config CRYPTO_CHACHA_S390 tristate depends on S390 - select CRYPTO_SKCIPHER select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA default CRYPTO_LIB_CHACHA_INTERNAL - help - Length-preserving cipher: ChaCha20 stream cipher (RFC 7539) - - Architecture: s390 - - It is available as of z13. config CRYPTO_HMAC_S390 tristate "Keyed-hash message authentication code: HMAC" diff --git a/arch/s390/crypto/chacha-glue.c b/arch/s390/crypto/chacha-glue.c index 920e9f0941e7..b3ffaa555385 100644 --- a/arch/s390/crypto/chacha-glue.c +++ b/arch/s390/crypto/chacha-glue.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * s390 ChaCha stream cipher. + * ChaCha stream cipher (s390 optimized) * * Copyright IBM Corp. 2021 */ @@ -8,9 +8,7 @@ #define KMSG_COMPONENT "chacha_s390" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt -#include <crypto/internal/chacha.h> -#include <crypto/internal/skcipher.h> -#include <crypto/algapi.h> +#include <crypto/chacha.h> #include <linux/cpufeature.h> #include <linux/kernel.h> #include <linux/module.h> @@ -18,50 +16,6 @@ #include <asm/fpu.h> #include "chacha-s390.h" -static void chacha20_crypt_s390(u32 *state, u8 *dst, const u8 *src, - unsigned int nbytes, const u32 *key, - u32 *counter) -{ - DECLARE_KERNEL_FPU_ONSTACK32(vxstate); - - kernel_fpu_begin(&vxstate, KERNEL_VXR); - chacha20_vx(dst, src, nbytes, key, counter); - kernel_fpu_end(&vxstate, KERNEL_VXR); - - *counter += round_up(nbytes, CHACHA_BLOCK_SIZE) / CHACHA_BLOCK_SIZE; -} - -static int chacha20_s390(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - u32 state[CHACHA_STATE_WORDS] __aligned(16); - struct skcipher_walk walk; - unsigned int nbytes; - int rc; - - rc = skcipher_walk_virt(&walk, req, false); - chacha_init(state, ctx->key, req->iv); - - while (walk.nbytes > 0) { - nbytes = walk.nbytes; - if (nbytes < walk.total) - nbytes = round_down(nbytes, walk.stride); - - if (nbytes <= CHACHA_BLOCK_SIZE) { - chacha_crypt_generic(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, - ctx->nrounds); - } else { - chacha20_crypt_s390(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, - &state[4], &state[12]); - } - rc = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } - return rc; -} - void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) { /* TODO: implement hchacha_block_arch() in assembly */ @@ -76,49 +30,26 @@ void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, * it cannot handle a block of data or less, but otherwise * it can handle data of arbitrary size */ - if (bytes <= CHACHA_BLOCK_SIZE || nrounds != 20 || !cpu_has_vx()) + if (bytes <= CHACHA_BLOCK_SIZE || nrounds != 20 || !cpu_has_vx()) { chacha_crypt_generic(state, dst, src, bytes, nrounds); - else - chacha20_crypt_s390(state, dst, src, bytes, - &state[4], &state[12]); -} -EXPORT_SYMBOL(chacha_crypt_arch); + } else { + DECLARE_KERNEL_FPU_ONSTACK32(vxstate); -static struct skcipher_alg chacha_algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-s390", - .base.cra_priority = 900, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, + kernel_fpu_begin(&vxstate, KERNEL_VXR); + chacha20_vx(dst, src, bytes, &state[4], &state[12]); + kernel_fpu_end(&vxstate, KERNEL_VXR); - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = chacha20_s390, - .decrypt = chacha20_s390, + state[12] += round_up(bytes, CHACHA_BLOCK_SIZE) / + CHACHA_BLOCK_SIZE; } -}; - -static int __init chacha_mod_init(void) -{ - return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ? - crypto_register_skciphers(chacha_algs, ARRAY_SIZE(chacha_algs)) : 0; } +EXPORT_SYMBOL(chacha_crypt_arch); -static void __exit chacha_mod_fini(void) +bool chacha_is_arch_optimized(void) { - if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) - crypto_unregister_skciphers(chacha_algs, ARRAY_SIZE(chacha_algs)); + return cpu_has_vx(); } +EXPORT_SYMBOL(chacha_is_arch_optimized); -module_cpu_feature_match(S390_CPU_FEATURE_VXRS, chacha_mod_init); -module_exit(chacha_mod_fini); - -MODULE_DESCRIPTION("ChaCha20 stream cipher"); +MODULE_DESCRIPTION("ChaCha stream cipher (s390 optimized)"); MODULE_LICENSE("GPL v2"); - -MODULE_ALIAS_CRYPTO("chacha20"); diff --git a/arch/s390/crypto/ghash_s390.c b/arch/s390/crypto/ghash_s390.c index 0800a2a5799f..dcbcee37cb63 100644 --- a/arch/s390/crypto/ghash_s390.c +++ b/arch/s390/crypto/ghash_s390.c @@ -8,29 +8,28 @@ * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com> */ +#include <asm/cpacf.h> +#include <crypto/ghash.h> #include <crypto/internal/hash.h> -#include <linux/module.h> #include <linux/cpufeature.h> -#include <asm/cpacf.h> - -#define GHASH_BLOCK_SIZE 16 -#define GHASH_DIGEST_SIZE 16 +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> -struct ghash_ctx { +struct s390_ghash_ctx { u8 key[GHASH_BLOCK_SIZE]; }; -struct ghash_desc_ctx { +struct s390_ghash_desc_ctx { u8 icv[GHASH_BLOCK_SIZE]; u8 key[GHASH_BLOCK_SIZE]; - u8 buffer[GHASH_BLOCK_SIZE]; - u32 bytes; }; static int ghash_init(struct shash_desc *desc) { - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); + struct s390_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); + struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); memset(dctx, 0, sizeof(*dctx)); memcpy(dctx->key, ctx->key, GHASH_BLOCK_SIZE); @@ -41,7 +40,7 @@ static int ghash_init(struct shash_desc *desc) static int ghash_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) { - struct ghash_ctx *ctx = crypto_shash_ctx(tfm); + struct s390_ghash_ctx *ctx = crypto_shash_ctx(tfm); if (keylen != GHASH_BLOCK_SIZE) return -EINVAL; @@ -54,80 +53,71 @@ static int ghash_setkey(struct crypto_shash *tfm, static int ghash_update(struct shash_desc *desc, const u8 *src, unsigned int srclen) { - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); + struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); unsigned int n; - u8 *buf = dctx->buffer; - - if (dctx->bytes) { - u8 *pos = buf + (GHASH_BLOCK_SIZE - dctx->bytes); - n = min(srclen, dctx->bytes); - dctx->bytes -= n; - srclen -= n; - - memcpy(pos, src, n); - src += n; + n = srclen & ~(GHASH_BLOCK_SIZE - 1); + cpacf_kimd(CPACF_KIMD_GHASH, dctx, src, n); + return srclen - n; +} - if (!dctx->bytes) { - cpacf_kimd(CPACF_KIMD_GHASH, dctx, buf, - GHASH_BLOCK_SIZE); - } - } +static void ghash_flush(struct s390_ghash_desc_ctx *dctx, const u8 *src, + unsigned int len) +{ + if (len) { + u8 buf[GHASH_BLOCK_SIZE] = {}; - n = srclen & ~(GHASH_BLOCK_SIZE - 1); - if (n) { - cpacf_kimd(CPACF_KIMD_GHASH, dctx, src, n); - src += n; - srclen -= n; + memcpy(buf, src, len); + cpacf_kimd(CPACF_KIMD_GHASH, dctx, buf, GHASH_BLOCK_SIZE); + memzero_explicit(buf, sizeof(buf)); } +} - if (srclen) { - dctx->bytes = GHASH_BLOCK_SIZE - srclen; - memcpy(buf, src, srclen); - } +static int ghash_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *dst) +{ + struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); + ghash_flush(dctx, src, len); + memcpy(dst, dctx->icv, GHASH_BLOCK_SIZE); return 0; } -static int ghash_flush(struct ghash_desc_ctx *dctx) +static int ghash_export(struct shash_desc *desc, void *out) { - u8 *buf = dctx->buffer; - - if (dctx->bytes) { - u8 *pos = buf + (GHASH_BLOCK_SIZE - dctx->bytes); - - memset(pos, 0, dctx->bytes); - cpacf_kimd(CPACF_KIMD_GHASH, dctx, buf, GHASH_BLOCK_SIZE); - dctx->bytes = 0; - } + struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); + memcpy(out, dctx->icv, GHASH_DIGEST_SIZE); return 0; } -static int ghash_final(struct shash_desc *desc, u8 *dst) +static int ghash_import(struct shash_desc *desc, const void *in) { - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - int ret; + struct s390_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); + struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - ret = ghash_flush(dctx); - if (!ret) - memcpy(dst, dctx->icv, GHASH_BLOCK_SIZE); - return ret; + memcpy(dctx->icv, in, GHASH_DIGEST_SIZE); + memcpy(dctx->key, ctx->key, GHASH_BLOCK_SIZE); + return 0; } static struct shash_alg ghash_alg = { .digestsize = GHASH_DIGEST_SIZE, .init = ghash_init, .update = ghash_update, - .final = ghash_final, + .finup = ghash_finup, .setkey = ghash_setkey, - .descsize = sizeof(struct ghash_desc_ctx), + .export = ghash_export, + .import = ghash_import, + .statesize = sizeof(struct ghash_desc_ctx), + .descsize = sizeof(struct s390_ghash_desc_ctx), .base = { .cra_name = "ghash", .cra_driver_name = "ghash-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ghash_ctx), + .cra_ctxsize = sizeof(struct s390_ghash_ctx), .cra_module = THIS_MODULE, }, }; diff --git a/arch/s390/crypto/sha.h b/arch/s390/crypto/sha.h index 2bb22db54c31..0a3cc1739144 100644 --- a/arch/s390/crypto/sha.h +++ b/arch/s390/crypto/sha.h @@ -10,27 +10,33 @@ #ifndef _CRYPTO_ARCH_S390_SHA_H #define _CRYPTO_ARCH_S390_SHA_H -#include <linux/crypto.h> -#include <crypto/sha1.h> #include <crypto/sha2.h> #include <crypto/sha3.h> +#include <linux/types.h> /* must be big enough for the largest SHA variant */ -#define SHA3_STATE_SIZE 200 #define CPACF_MAX_PARMBLOCK_SIZE SHA3_STATE_SIZE #define SHA_MAX_BLOCK_SIZE SHA3_224_BLOCK_SIZE +#define S390_SHA_CTX_SIZE sizeof(struct s390_sha_ctx) struct s390_sha_ctx { u64 count; /* message length in bytes */ - u32 state[CPACF_MAX_PARMBLOCK_SIZE / sizeof(u32)]; - u8 buf[SHA_MAX_BLOCK_SIZE]; + union { + u32 state[CPACF_MAX_PARMBLOCK_SIZE / sizeof(u32)]; + struct { + u64 state[SHA512_DIGEST_SIZE]; + u64 count_hi; + } sha512; + }; int func; /* KIMD function to use */ - int first_message_part; + bool first_message_part; }; struct shash_desc; -int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len); -int s390_sha_final(struct shash_desc *desc, u8 *out); +int s390_sha_update_blocks(struct shash_desc *desc, const u8 *data, + unsigned int len); +int s390_sha_finup(struct shash_desc *desc, const u8 *src, unsigned int len, + u8 *out); #endif diff --git a/arch/s390/crypto/sha1_s390.c b/arch/s390/crypto/sha1_s390.c index bc3a22704e09..d229cbd2ba22 100644 --- a/arch/s390/crypto/sha1_s390.c +++ b/arch/s390/crypto/sha1_s390.c @@ -18,12 +18,12 @@ * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> */ +#include <asm/cpacf.h> #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/cpufeature.h> #include <crypto/sha1.h> -#include <asm/cpacf.h> +#include <linux/cpufeature.h> +#include <linux/kernel.h> +#include <linux/module.h> #include "sha.h" @@ -49,7 +49,6 @@ static int s390_sha1_export(struct shash_desc *desc, void *out) octx->count = sctx->count; memcpy(octx->state, sctx->state, sizeof(octx->state)); - memcpy(octx->buffer, sctx->buf, sizeof(octx->buffer)); return 0; } @@ -60,7 +59,6 @@ static int s390_sha1_import(struct shash_desc *desc, const void *in) sctx->count = ictx->count; memcpy(sctx->state, ictx->state, sizeof(ictx->state)); - memcpy(sctx->buf, ictx->buffer, sizeof(ictx->buffer)); sctx->func = CPACF_KIMD_SHA_1; return 0; } @@ -68,16 +66,18 @@ static int s390_sha1_import(struct shash_desc *desc, const void *in) static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = s390_sha1_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = s390_sha1_export, .import = s390_sha1_import, - .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha1_state), + .descsize = S390_SHA_CTX_SIZE, + .statesize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name= "sha1-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/s390/crypto/sha256_s390.c b/arch/s390/crypto/sha256_s390.c index 6f1ccdf93d3e..e6876c49414d 100644 --- a/arch/s390/crypto/sha256_s390.c +++ b/arch/s390/crypto/sha256_s390.c @@ -8,12 +8,13 @@ * Copyright IBM Corp. 2005, 2011 * Author(s): Jan Glauber (jang@de.ibm.com) */ +#include <asm/cpacf.h> #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/cpufeature.h> #include <crypto/sha2.h> -#include <asm/cpacf.h> +#include <linux/cpufeature.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> #include "sha.h" @@ -38,22 +39,20 @@ static int s390_sha256_init(struct shash_desc *desc) static int sha256_export(struct shash_desc *desc, void *out) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - struct sha256_state *octx = out; + struct crypto_sha256_state *octx = out; octx->count = sctx->count; memcpy(octx->state, sctx->state, sizeof(octx->state)); - memcpy(octx->buf, sctx->buf, sizeof(octx->buf)); return 0; } static int sha256_import(struct shash_desc *desc, const void *in) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - const struct sha256_state *ictx = in; + const struct crypto_sha256_state *ictx = in; sctx->count = ictx->count; memcpy(sctx->state, ictx->state, sizeof(ictx->state)); - memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); sctx->func = CPACF_KIMD_SHA_256; return 0; } @@ -61,16 +60,17 @@ static int sha256_import(struct shash_desc *desc, const void *in) static struct shash_alg sha256_alg = { .digestsize = SHA256_DIGEST_SIZE, .init = s390_sha256_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = sha256_export, .import = sha256_import, - .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha256_state), + .descsize = S390_SHA_CTX_SIZE, + .statesize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name= "sha256-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -97,16 +97,17 @@ static int s390_sha224_init(struct shash_desc *desc) static struct shash_alg sha224_alg = { .digestsize = SHA224_DIGEST_SIZE, .init = s390_sha224_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = sha256_export, .import = sha256_import, - .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha256_state), + .descsize = S390_SHA_CTX_SIZE, + .statesize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name= "sha224-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/s390/crypto/sha3_256_s390.c b/arch/s390/crypto/sha3_256_s390.c index a84ef692f572..4a7731ac6bcd 100644 --- a/arch/s390/crypto/sha3_256_s390.c +++ b/arch/s390/crypto/sha3_256_s390.c @@ -8,12 +8,14 @@ * Copyright IBM Corp. 2019 * Author(s): Joerg Schmidbauer (jschmidb@de.ibm.com) */ +#include <asm/cpacf.h> #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/cpufeature.h> #include <crypto/sha3.h> -#include <asm/cpacf.h> +#include <linux/cpufeature.h> +#include <linux/errno.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> #include "sha.h" @@ -21,11 +23,11 @@ static int sha3_256_init(struct shash_desc *desc) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - if (!test_facility(86)) /* msa 12 */ + sctx->first_message_part = test_facility(86); + if (!sctx->first_message_part) memset(sctx->state, 0, sizeof(sctx->state)); sctx->count = 0; sctx->func = CPACF_KIMD_SHA3_256; - sctx->first_message_part = 1; return 0; } @@ -35,11 +37,11 @@ static int sha3_256_export(struct shash_desc *desc, void *out) struct s390_sha_ctx *sctx = shash_desc_ctx(desc); struct sha3_state *octx = out; - octx->rsiz = sctx->count; + if (sctx->first_message_part) { + memset(sctx->state, 0, sizeof(sctx->state)); + sctx->first_message_part = 0; + } memcpy(octx->st, sctx->state, sizeof(octx->st)); - memcpy(octx->buf, sctx->buf, sizeof(octx->buf)); - octx->partial = sctx->first_message_part; - return 0; } @@ -48,10 +50,9 @@ static int sha3_256_import(struct shash_desc *desc, const void *in) struct s390_sha_ctx *sctx = shash_desc_ctx(desc); const struct sha3_state *ictx = in; - sctx->count = ictx->rsiz; + sctx->count = 0; memcpy(sctx->state, ictx->st, sizeof(ictx->st)); - memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); - sctx->first_message_part = ictx->partial; + sctx->first_message_part = 0; sctx->func = CPACF_KIMD_SHA3_256; return 0; @@ -60,30 +61,26 @@ static int sha3_256_import(struct shash_desc *desc, const void *in) static int sha3_224_import(struct shash_desc *desc, const void *in) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - const struct sha3_state *ictx = in; - sctx->count = ictx->rsiz; - memcpy(sctx->state, ictx->st, sizeof(ictx->st)); - memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); - sctx->first_message_part = ictx->partial; + sha3_256_import(desc, in); sctx->func = CPACF_KIMD_SHA3_224; - return 0; } static struct shash_alg sha3_256_alg = { .digestsize = SHA3_256_DIGEST_SIZE, /* = 32 */ .init = sha3_256_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = sha3_256_export, .import = sha3_256_import, - .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha3_state), + .descsize = S390_SHA_CTX_SIZE, + .statesize = SHA3_STATE_SIZE, .base = { .cra_name = "sha3-256", .cra_driver_name = "sha3-256-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA3_256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -93,28 +90,25 @@ static int sha3_224_init(struct shash_desc *desc) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - if (!test_facility(86)) /* msa 12 */ - memset(sctx->state, 0, sizeof(sctx->state)); - sctx->count = 0; + sha3_256_init(desc); sctx->func = CPACF_KIMD_SHA3_224; - sctx->first_message_part = 1; - return 0; } static struct shash_alg sha3_224_alg = { .digestsize = SHA3_224_DIGEST_SIZE, .init = sha3_224_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = sha3_256_export, /* same as for 256 */ .import = sha3_224_import, /* function code different! */ - .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha3_state), + .descsize = S390_SHA_CTX_SIZE, + .statesize = SHA3_STATE_SIZE, .base = { .cra_name = "sha3-224", .cra_driver_name = "sha3-224-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA3_224_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/s390/crypto/sha3_512_s390.c b/arch/s390/crypto/sha3_512_s390.c index 07528fc98ff7..018f02fff444 100644 --- a/arch/s390/crypto/sha3_512_s390.c +++ b/arch/s390/crypto/sha3_512_s390.c @@ -7,12 +7,14 @@ * Copyright IBM Corp. 2019 * Author(s): Joerg Schmidbauer (jschmidb@de.ibm.com) */ +#include <asm/cpacf.h> #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/cpufeature.h> #include <crypto/sha3.h> -#include <asm/cpacf.h> +#include <linux/cpufeature.h> +#include <linux/errno.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> #include "sha.h" @@ -20,11 +22,11 @@ static int sha3_512_init(struct shash_desc *desc) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - if (!test_facility(86)) /* msa 12 */ + sctx->first_message_part = test_facility(86); + if (!sctx->first_message_part) memset(sctx->state, 0, sizeof(sctx->state)); sctx->count = 0; sctx->func = CPACF_KIMD_SHA3_512; - sctx->first_message_part = 1; return 0; } @@ -34,13 +36,12 @@ static int sha3_512_export(struct shash_desc *desc, void *out) struct s390_sha_ctx *sctx = shash_desc_ctx(desc); struct sha3_state *octx = out; - octx->rsiz = sctx->count; - octx->rsizw = sctx->count >> 32; + if (sctx->first_message_part) { + memset(sctx->state, 0, sizeof(sctx->state)); + sctx->first_message_part = 0; + } memcpy(octx->st, sctx->state, sizeof(octx->st)); - memcpy(octx->buf, sctx->buf, sizeof(octx->buf)); - octx->partial = sctx->first_message_part; - return 0; } @@ -49,13 +50,9 @@ static int sha3_512_import(struct shash_desc *desc, const void *in) struct s390_sha_ctx *sctx = shash_desc_ctx(desc); const struct sha3_state *ictx = in; - if (unlikely(ictx->rsizw)) - return -ERANGE; - sctx->count = ictx->rsiz; - + sctx->count = 0; memcpy(sctx->state, ictx->st, sizeof(ictx->st)); - memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); - sctx->first_message_part = ictx->partial; + sctx->first_message_part = 0; sctx->func = CPACF_KIMD_SHA3_512; return 0; @@ -64,33 +61,26 @@ static int sha3_512_import(struct shash_desc *desc, const void *in) static int sha3_384_import(struct shash_desc *desc, const void *in) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - const struct sha3_state *ictx = in; - if (unlikely(ictx->rsizw)) - return -ERANGE; - sctx->count = ictx->rsiz; - - memcpy(sctx->state, ictx->st, sizeof(ictx->st)); - memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); - sctx->first_message_part = ictx->partial; + sha3_512_import(desc, in); sctx->func = CPACF_KIMD_SHA3_384; - return 0; } static struct shash_alg sha3_512_alg = { .digestsize = SHA3_512_DIGEST_SIZE, .init = sha3_512_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = sha3_512_export, .import = sha3_512_import, - .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha3_state), + .descsize = S390_SHA_CTX_SIZE, + .statesize = SHA3_STATE_SIZE, .base = { .cra_name = "sha3-512", .cra_driver_name = "sha3-512-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA3_512_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -102,28 +92,25 @@ static int sha3_384_init(struct shash_desc *desc) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - if (!test_facility(86)) /* msa 12 */ - memset(sctx->state, 0, sizeof(sctx->state)); - sctx->count = 0; + sha3_512_init(desc); sctx->func = CPACF_KIMD_SHA3_384; - sctx->first_message_part = 1; - return 0; } static struct shash_alg sha3_384_alg = { .digestsize = SHA3_384_DIGEST_SIZE, .init = sha3_384_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = sha3_512_export, /* same as for 512 */ .import = sha3_384_import, /* function code different! */ - .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha3_state), + .descsize = S390_SHA_CTX_SIZE, + .statesize = SHA3_STATE_SIZE, .base = { .cra_name = "sha3-384", .cra_driver_name = "sha3-384-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA3_384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct s390_sha_ctx), .cra_module = THIS_MODULE, diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c index 04f11c407763..14818fcc9cd4 100644 --- a/arch/s390/crypto/sha512_s390.c +++ b/arch/s390/crypto/sha512_s390.c @@ -7,14 +7,13 @@ * Copyright IBM Corp. 2007 * Author(s): Jan Glauber (jang@de.ibm.com) */ +#include <asm/cpacf.h> #include <crypto/internal/hash.h> #include <crypto/sha2.h> +#include <linux/cpufeature.h> #include <linux/errno.h> -#include <linux/init.h> #include <linux/kernel.h> #include <linux/module.h> -#include <linux/cpufeature.h> -#include <asm/cpacf.h> #include "sha.h" @@ -22,15 +21,16 @@ static int sha512_init(struct shash_desc *desc) { struct s390_sha_ctx *ctx = shash_desc_ctx(desc); - *(__u64 *)&ctx->state[0] = SHA512_H0; - *(__u64 *)&ctx->state[2] = SHA512_H1; - *(__u64 *)&ctx->state[4] = SHA512_H2; - *(__u64 *)&ctx->state[6] = SHA512_H3; - *(__u64 *)&ctx->state[8] = SHA512_H4; - *(__u64 *)&ctx->state[10] = SHA512_H5; - *(__u64 *)&ctx->state[12] = SHA512_H6; - *(__u64 *)&ctx->state[14] = SHA512_H7; + ctx->sha512.state[0] = SHA512_H0; + ctx->sha512.state[2] = SHA512_H1; + ctx->sha512.state[4] = SHA512_H2; + ctx->sha512.state[6] = SHA512_H3; + ctx->sha512.state[8] = SHA512_H4; + ctx->sha512.state[10] = SHA512_H5; + ctx->sha512.state[12] = SHA512_H6; + ctx->sha512.state[14] = SHA512_H7; ctx->count = 0; + ctx->sha512.count_hi = 0; ctx->func = CPACF_KIMD_SHA_512; return 0; @@ -42,9 +42,8 @@ static int sha512_export(struct shash_desc *desc, void *out) struct sha512_state *octx = out; octx->count[0] = sctx->count; - octx->count[1] = 0; + octx->count[1] = sctx->sha512.count_hi; memcpy(octx->state, sctx->state, sizeof(octx->state)); - memcpy(octx->buf, sctx->buf, sizeof(octx->buf)); return 0; } @@ -53,12 +52,10 @@ static int sha512_import(struct shash_desc *desc, const void *in) struct s390_sha_ctx *sctx = shash_desc_ctx(desc); const struct sha512_state *ictx = in; - if (unlikely(ictx->count[1])) - return -ERANGE; sctx->count = ictx->count[0]; + sctx->sha512.count_hi = ictx->count[1]; memcpy(sctx->state, ictx->state, sizeof(ictx->state)); - memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); sctx->func = CPACF_KIMD_SHA_512; return 0; } @@ -66,16 +63,18 @@ static int sha512_import(struct shash_desc *desc, const void *in) static struct shash_alg sha512_alg = { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = sha512_export, .import = sha512_import, .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha512_state), + .statesize = SHA512_STATE_SIZE, .base = { .cra_name = "sha512", .cra_driver_name= "sha512-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -104,17 +103,19 @@ static int sha384_init(struct shash_desc *desc) static struct shash_alg sha384_alg = { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = sha512_export, .import = sha512_import, .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha512_state), + .statesize = SHA512_STATE_SIZE, .base = { .cra_name = "sha384", .cra_driver_name= "sha384-s390", .cra_priority = 300, .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_ctxsize = sizeof(struct s390_sha_ctx), .cra_module = THIS_MODULE, } diff --git a/arch/s390/crypto/sha_common.c b/arch/s390/crypto/sha_common.c index 961d7d522af1..b5e2c365ea05 100644 --- a/arch/s390/crypto/sha_common.c +++ b/arch/s390/crypto/sha_common.c @@ -13,50 +13,33 @@ #include <asm/cpacf.h> #include "sha.h" -int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len) +int s390_sha_update_blocks(struct shash_desc *desc, const u8 *data, + unsigned int len) { - struct s390_sha_ctx *ctx = shash_desc_ctx(desc); unsigned int bsize = crypto_shash_blocksize(desc->tfm); - unsigned int index, n; + struct s390_sha_ctx *ctx = shash_desc_ctx(desc); + unsigned int n; int fc; - /* how much is already in the buffer? */ - index = ctx->count % bsize; - ctx->count += len; - - if ((index + len) < bsize) - goto store; - fc = ctx->func; if (ctx->first_message_part) - fc |= test_facility(86) ? CPACF_KIMD_NIP : 0; - - /* process one stored block */ - if (index) { - memcpy(ctx->buf + index, data, bsize - index); - cpacf_kimd(fc, ctx->state, ctx->buf, bsize); - ctx->first_message_part = 0; - fc &= ~CPACF_KIMD_NIP; - data += bsize - index; - len -= bsize - index; - index = 0; - } + fc |= CPACF_KIMD_NIP; /* process as many blocks as possible */ - if (len >= bsize) { - n = (len / bsize) * bsize; - cpacf_kimd(fc, ctx->state, data, n); - ctx->first_message_part = 0; - data += n; - len -= n; + n = (len / bsize) * bsize; + ctx->count += n; + switch (ctx->func) { + case CPACF_KLMD_SHA_512: + case CPACF_KLMD_SHA3_384: + if (ctx->count < n) + ctx->sha512.count_hi++; + break; } -store: - if (len) - memcpy(ctx->buf + index , data, len); - - return 0; + cpacf_kimd(fc, ctx->state, data, n); + ctx->first_message_part = 0; + return len - n; } -EXPORT_SYMBOL_GPL(s390_sha_update); +EXPORT_SYMBOL_GPL(s390_sha_update_blocks); static int s390_crypto_shash_parmsize(int func) { @@ -77,15 +60,15 @@ static int s390_crypto_shash_parmsize(int func) } } -int s390_sha_final(struct shash_desc *desc, u8 *out) +int s390_sha_finup(struct shash_desc *desc, const u8 *src, unsigned int len, + u8 *out) { struct s390_sha_ctx *ctx = shash_desc_ctx(desc); - unsigned int bsize = crypto_shash_blocksize(desc->tfm); - u64 bits; - unsigned int n; int mbl_offset, fc; + u64 bits; + + ctx->count += len; - n = ctx->count % bsize; bits = ctx->count * 8; mbl_offset = s390_crypto_shash_parmsize(ctx->func); if (mbl_offset < 0) @@ -95,17 +78,16 @@ int s390_sha_final(struct shash_desc *desc, u8 *out) /* set total msg bit length (mbl) in CPACF parmblock */ switch (ctx->func) { - case CPACF_KLMD_SHA_1: - case CPACF_KLMD_SHA_256: - memcpy(ctx->state + mbl_offset, &bits, sizeof(bits)); - break; case CPACF_KLMD_SHA_512: - /* - * the SHA512 parmblock has a 128-bit mbl field, clear - * high-order u64 field, copy bits to low-order u64 field - */ - memset(ctx->state + mbl_offset, 0x00, sizeof(bits)); + /* The SHA512 parmblock has a 128-bit mbl field. */ + if (ctx->count < len) + ctx->sha512.count_hi++; + ctx->sha512.count_hi <<= 3; + ctx->sha512.count_hi |= ctx->count >> 61; mbl_offset += sizeof(u64) / sizeof(u32); + fallthrough; + case CPACF_KLMD_SHA_1: + case CPACF_KLMD_SHA_256: memcpy(ctx->state + mbl_offset, &bits, sizeof(bits)); break; case CPACF_KLMD_SHA3_224: @@ -121,16 +103,14 @@ int s390_sha_final(struct shash_desc *desc, u8 *out) fc |= test_facility(86) ? CPACF_KLMD_DUFOP : 0; if (ctx->first_message_part) fc |= CPACF_KLMD_NIP; - cpacf_klmd(fc, ctx->state, ctx->buf, n); + cpacf_klmd(fc, ctx->state, src, len); /* copy digest to out */ memcpy(out, ctx->state, crypto_shash_digestsize(desc->tfm)); - /* wipe context */ - memset(ctx, 0, sizeof *ctx); return 0; } -EXPORT_SYMBOL_GPL(s390_sha_final); +EXPORT_SYMBOL_GPL(s390_sha_finup); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("s390 SHA cipher common functions"); diff --git a/arch/sparc/crypto/md5_glue.c b/arch/sparc/crypto/md5_glue.c index 511db98d590a..5b018c6a376c 100644 --- a/arch/sparc/crypto/md5_glue.c +++ b/arch/sparc/crypto/md5_glue.c @@ -14,121 +14,105 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include <asm/elf.h> +#include <asm/pstate.h> #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> #include <crypto/md5.h> - -#include <asm/pstate.h> -#include <asm/elf.h> +#include <linux/errno.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> +#include <linux/unaligned.h> #include "opcodes.h" -asmlinkage void md5_sparc64_transform(u32 *digest, const char *data, +struct sparc_md5_state { + __le32 hash[MD5_HASH_WORDS]; + u64 byte_count; +}; + +asmlinkage void md5_sparc64_transform(__le32 *digest, const char *data, unsigned int rounds); static int md5_sparc64_init(struct shash_desc *desc) { - struct md5_state *mctx = shash_desc_ctx(desc); + struct sparc_md5_state *mctx = shash_desc_ctx(desc); - mctx->hash[0] = MD5_H0; - mctx->hash[1] = MD5_H1; - mctx->hash[2] = MD5_H2; - mctx->hash[3] = MD5_H3; - le32_to_cpu_array(mctx->hash, 4); + mctx->hash[0] = cpu_to_le32(MD5_H0); + mctx->hash[1] = cpu_to_le32(MD5_H1); + mctx->hash[2] = cpu_to_le32(MD5_H2); + mctx->hash[3] = cpu_to_le32(MD5_H3); mctx->byte_count = 0; return 0; } -static void __md5_sparc64_update(struct md5_state *sctx, const u8 *data, - unsigned int len, unsigned int partial) -{ - unsigned int done = 0; - - sctx->byte_count += len; - if (partial) { - done = MD5_HMAC_BLOCK_SIZE - partial; - memcpy((u8 *)sctx->block + partial, data, done); - md5_sparc64_transform(sctx->hash, (u8 *)sctx->block, 1); - } - if (len - done >= MD5_HMAC_BLOCK_SIZE) { - const unsigned int rounds = (len - done) / MD5_HMAC_BLOCK_SIZE; - - md5_sparc64_transform(sctx->hash, data + done, rounds); - done += rounds * MD5_HMAC_BLOCK_SIZE; - } - - memcpy(sctx->block, data + done, len - done); -} - static int md5_sparc64_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct md5_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->byte_count % MD5_HMAC_BLOCK_SIZE; - - /* Handle the fast case right here */ - if (partial + len < MD5_HMAC_BLOCK_SIZE) { - sctx->byte_count += len; - memcpy((u8 *)sctx->block + partial, data, len); - } else - __md5_sparc64_update(sctx, data, len, partial); + struct sparc_md5_state *sctx = shash_desc_ctx(desc); - return 0; + sctx->byte_count += round_down(len, MD5_HMAC_BLOCK_SIZE); + md5_sparc64_transform(sctx->hash, data, len / MD5_HMAC_BLOCK_SIZE); + return len - round_down(len, MD5_HMAC_BLOCK_SIZE); } /* Add padding and return the message digest. */ -static int md5_sparc64_final(struct shash_desc *desc, u8 *out) +static int md5_sparc64_finup(struct shash_desc *desc, const u8 *src, + unsigned int offset, u8 *out) { - struct md5_state *sctx = shash_desc_ctx(desc); - unsigned int i, index, padlen; - u32 *dst = (u32 *)out; - __le64 bits; - static const u8 padding[MD5_HMAC_BLOCK_SIZE] = { 0x80, }; - - bits = cpu_to_le64(sctx->byte_count << 3); - - /* Pad out to 56 mod 64 and append length */ - index = sctx->byte_count % MD5_HMAC_BLOCK_SIZE; - padlen = (index < 56) ? (56 - index) : ((MD5_HMAC_BLOCK_SIZE+56) - index); - - /* We need to fill a whole block for __md5_sparc64_update() */ - if (padlen <= 56) { - sctx->byte_count += padlen; - memcpy((u8 *)sctx->block + index, padding, padlen); - } else { - __md5_sparc64_update(sctx, padding, padlen, index); - } - __md5_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 56); + struct sparc_md5_state *sctx = shash_desc_ctx(desc); + __le64 block[MD5_BLOCK_WORDS] = {}; + u8 *p = memcpy(block, src, offset); + __le32 *dst = (__le32 *)out; + __le64 *pbits; + int i; + + src = p; + p += offset; + *p++ = 0x80; + sctx->byte_count += offset; + pbits = &block[(MD5_BLOCK_WORDS / (offset > 55 ? 1 : 2)) - 1]; + *pbits = cpu_to_le64(sctx->byte_count << 3); + md5_sparc64_transform(sctx->hash, src, (pbits - block + 1) / 8); + memzero_explicit(block, sizeof(block)); /* Store state in digest */ for (i = 0; i < MD5_HASH_WORDS; i++) dst[i] = sctx->hash[i]; - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); - return 0; } static int md5_sparc64_export(struct shash_desc *desc, void *out) { - struct md5_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); + struct sparc_md5_state *sctx = shash_desc_ctx(desc); + union { + u8 *u8; + u32 *u32; + u64 *u64; + } p = { .u8 = out }; + int i; + for (i = 0; i < MD5_HASH_WORDS; i++) + put_unaligned(le32_to_cpu(sctx->hash[i]), p.u32++); + put_unaligned(sctx->byte_count, p.u64); return 0; } static int md5_sparc64_import(struct shash_desc *desc, const void *in) { - struct md5_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); + struct sparc_md5_state *sctx = shash_desc_ctx(desc); + union { + const u8 *u8; + const u32 *u32; + const u64 *u64; + } p = { .u8 = in }; + int i; + for (i = 0; i < MD5_HASH_WORDS; i++) + sctx->hash[i] = cpu_to_le32(get_unaligned(p.u32++)); + sctx->byte_count = get_unaligned(p.u64); return 0; } @@ -136,15 +120,16 @@ static struct shash_alg alg = { .digestsize = MD5_DIGEST_SIZE, .init = md5_sparc64_init, .update = md5_sparc64_update, - .final = md5_sparc64_final, + .finup = md5_sparc64_finup, .export = md5_sparc64_export, .import = md5_sparc64_import, - .descsize = sizeof(struct md5_state), - .statesize = sizeof(struct md5_state), + .descsize = sizeof(struct sparc_md5_state), + .statesize = sizeof(struct sparc_md5_state), .base = { .cra_name = "md5", .cra_driver_name= "md5-sparc64", .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = MD5_HMAC_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/sparc/crypto/sha1_glue.c b/arch/sparc/crypto/sha1_glue.c index 06b7becfcb21..ec5a06948e0d 100644 --- a/arch/sparc/crypto/sha1_glue.c +++ b/arch/sparc/crypto/sha1_glue.c @@ -11,124 +11,45 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include <asm/elf.h> +#include <asm/pstate.h> #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> #include <crypto/sha1.h> #include <crypto/sha1_base.h> - -#include <asm/pstate.h> -#include <asm/elf.h> +#include <linux/kernel.h> +#include <linux/module.h> #include "opcodes.h" -asmlinkage void sha1_sparc64_transform(u32 *digest, const char *data, - unsigned int rounds); - -static void __sha1_sparc64_update(struct sha1_state *sctx, const u8 *data, - unsigned int len, unsigned int partial) -{ - unsigned int done = 0; - - sctx->count += len; - if (partial) { - done = SHA1_BLOCK_SIZE - partial; - memcpy(sctx->buffer + partial, data, done); - sha1_sparc64_transform(sctx->state, sctx->buffer, 1); - } - if (len - done >= SHA1_BLOCK_SIZE) { - const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE; - - sha1_sparc64_transform(sctx->state, data + done, rounds); - done += rounds * SHA1_BLOCK_SIZE; - } - - memcpy(sctx->buffer, data + done, len - done); -} +asmlinkage void sha1_sparc64_transform(struct sha1_state *digest, + const u8 *data, int rounds); static int sha1_sparc64_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; - - /* Handle the fast case right here */ - if (partial + len < SHA1_BLOCK_SIZE) { - sctx->count += len; - memcpy(sctx->buffer + partial, data, len); - } else - __sha1_sparc64_update(sctx, data, len, partial); - - return 0; + return sha1_base_do_update_blocks(desc, data, len, + sha1_sparc64_transform); } /* Add padding and return the message digest. */ -static int sha1_sparc64_final(struct shash_desc *desc, u8 *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int i, index, padlen; - __be32 *dst = (__be32 *)out; - __be64 bits; - static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, }; - - bits = cpu_to_be64(sctx->count << 3); - - /* Pad out to 56 mod 64 and append length */ - index = sctx->count % SHA1_BLOCK_SIZE; - padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index); - - /* We need to fill a whole block for __sha1_sparc64_update() */ - if (padlen <= 56) { - sctx->count += padlen; - memcpy(sctx->buffer + index, padding, padlen); - } else { - __sha1_sparc64_update(sctx, padding, padlen, index); - } - __sha1_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 56); - - /* Store state in digest */ - for (i = 0; i < 5; i++) - dst[i] = cpu_to_be32(sctx->state[i]); - - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); - - return 0; -} - -static int sha1_sparc64_export(struct shash_desc *desc, void *out) +static int sha1_sparc64_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - - return 0; -} - -static int sha1_sparc64_import(struct shash_desc *desc, const void *in) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - - return 0; + sha1_base_do_finup(desc, src, len, sha1_sparc64_transform); + return sha1_base_finish(desc, out); } static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = sha1_sparc64_update, - .final = sha1_sparc64_final, - .export = sha1_sparc64_export, - .import = sha1_sparc64_import, - .descsize = sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), + .finup = sha1_sparc64_finup, + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name= "sha1-sparc64", .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/sparc/crypto/sha256_glue.c b/arch/sparc/crypto/sha256_glue.c index 285561a1cde5..ddb250242faf 100644 --- a/arch/sparc/crypto/sha256_glue.c +++ b/arch/sparc/crypto/sha256_glue.c @@ -11,133 +11,50 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include <asm/elf.h> +#include <asm/pstate.h> #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> #include <crypto/sha2.h> #include <crypto/sha256_base.h> - -#include <asm/pstate.h> -#include <asm/elf.h> +#include <linux/kernel.h> +#include <linux/module.h> #include "opcodes.h" asmlinkage void sha256_sparc64_transform(u32 *digest, const char *data, unsigned int rounds); -static void __sha256_sparc64_update(struct sha256_state *sctx, const u8 *data, - unsigned int len, unsigned int partial) +static void sha256_block(struct crypto_sha256_state *sctx, const u8 *src, + int blocks) { - unsigned int done = 0; - - sctx->count += len; - if (partial) { - done = SHA256_BLOCK_SIZE - partial; - memcpy(sctx->buf + partial, data, done); - sha256_sparc64_transform(sctx->state, sctx->buf, 1); - } - if (len - done >= SHA256_BLOCK_SIZE) { - const unsigned int rounds = (len - done) / SHA256_BLOCK_SIZE; - - sha256_sparc64_transform(sctx->state, data + done, rounds); - done += rounds * SHA256_BLOCK_SIZE; - } - - memcpy(sctx->buf, data + done, len - done); + sha256_sparc64_transform(sctx->state, src, blocks); } static int sha256_sparc64_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha256_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; - - /* Handle the fast case right here */ - if (partial + len < SHA256_BLOCK_SIZE) { - sctx->count += len; - memcpy(sctx->buf + partial, data, len); - } else - __sha256_sparc64_update(sctx, data, len, partial); - - return 0; -} - -static int sha256_sparc64_final(struct shash_desc *desc, u8 *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - unsigned int i, index, padlen; - __be32 *dst = (__be32 *)out; - __be64 bits; - static const u8 padding[SHA256_BLOCK_SIZE] = { 0x80, }; - - bits = cpu_to_be64(sctx->count << 3); - - /* Pad out to 56 mod 64 and append length */ - index = sctx->count % SHA256_BLOCK_SIZE; - padlen = (index < 56) ? (56 - index) : ((SHA256_BLOCK_SIZE+56) - index); - - /* We need to fill a whole block for __sha256_sparc64_update() */ - if (padlen <= 56) { - sctx->count += padlen; - memcpy(sctx->buf + index, padding, padlen); - } else { - __sha256_sparc64_update(sctx, padding, padlen, index); - } - __sha256_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 56); - - /* Store state in digest */ - for (i = 0; i < 8; i++) - dst[i] = cpu_to_be32(sctx->state[i]); - - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); - - return 0; -} - -static int sha224_sparc64_final(struct shash_desc *desc, u8 *hash) -{ - u8 D[SHA256_DIGEST_SIZE]; - - sha256_sparc64_final(desc, D); - - memcpy(hash, D, SHA224_DIGEST_SIZE); - memzero_explicit(D, SHA256_DIGEST_SIZE); - - return 0; -} - -static int sha256_sparc64_export(struct shash_desc *desc, void *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; + return sha256_base_do_update_blocks(desc, data, len, sha256_block); } -static int sha256_sparc64_import(struct shash_desc *desc, const void *in) +static int sha256_sparc64_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - struct sha256_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - return 0; + sha256_base_do_finup(desc, src, len, sha256_block); + return sha256_base_finish(desc, out); } static struct shash_alg sha256_alg = { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = sha256_sparc64_update, - .final = sha256_sparc64_final, - .export = sha256_sparc64_export, - .import = sha256_sparc64_import, - .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), + .finup = sha256_sparc64_finup, + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name= "sha256-sparc64", .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -147,12 +64,14 @@ static struct shash_alg sha224_alg = { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = sha256_sparc64_update, - .final = sha224_sparc64_final, - .descsize = sizeof(struct sha256_state), + .finup = sha256_sparc64_finup, + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name= "sha224-sparc64", .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/sparc/crypto/sha512_glue.c b/arch/sparc/crypto/sha512_glue.c index d66efa4ec59a..1d0e1f98ca46 100644 --- a/arch/sparc/crypto/sha512_glue.c +++ b/arch/sparc/crypto/sha512_glue.c @@ -10,115 +10,43 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include <asm/elf.h> +#include <asm/pstate.h> #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> #include <crypto/sha2.h> #include <crypto/sha512_base.h> - -#include <asm/pstate.h> -#include <asm/elf.h> +#include <linux/kernel.h> +#include <linux/module.h> #include "opcodes.h" asmlinkage void sha512_sparc64_transform(u64 *digest, const char *data, unsigned int rounds); -static void __sha512_sparc64_update(struct sha512_state *sctx, const u8 *data, - unsigned int len, unsigned int partial) +static void sha512_block(struct sha512_state *sctx, const u8 *src, int blocks) { - unsigned int done = 0; - - if ((sctx->count[0] += len) < len) - sctx->count[1]++; - if (partial) { - done = SHA512_BLOCK_SIZE - partial; - memcpy(sctx->buf + partial, data, done); - sha512_sparc64_transform(sctx->state, sctx->buf, 1); - } - if (len - done >= SHA512_BLOCK_SIZE) { - const unsigned int rounds = (len - done) / SHA512_BLOCK_SIZE; - - sha512_sparc64_transform(sctx->state, data + done, rounds); - done += rounds * SHA512_BLOCK_SIZE; - } - - memcpy(sctx->buf, data + done, len - done); + sha512_sparc64_transform(sctx->state, src, blocks); } static int sha512_sparc64_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha512_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE; - - /* Handle the fast case right here */ - if (partial + len < SHA512_BLOCK_SIZE) { - if ((sctx->count[0] += len) < len) - sctx->count[1]++; - memcpy(sctx->buf + partial, data, len); - } else - __sha512_sparc64_update(sctx, data, len, partial); - - return 0; + return sha512_base_do_update_blocks(desc, data, len, sha512_block); } -static int sha512_sparc64_final(struct shash_desc *desc, u8 *out) +static int sha512_sparc64_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - struct sha512_state *sctx = shash_desc_ctx(desc); - unsigned int i, index, padlen; - __be64 *dst = (__be64 *)out; - __be64 bits[2]; - static const u8 padding[SHA512_BLOCK_SIZE] = { 0x80, }; - - /* Save number of bits */ - bits[1] = cpu_to_be64(sctx->count[0] << 3); - bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61); - - /* Pad out to 112 mod 128 and append length */ - index = sctx->count[0] % SHA512_BLOCK_SIZE; - padlen = (index < 112) ? (112 - index) : ((SHA512_BLOCK_SIZE+112) - index); - - /* We need to fill a whole block for __sha512_sparc64_update() */ - if (padlen <= 112) { - if ((sctx->count[0] += padlen) < padlen) - sctx->count[1]++; - memcpy(sctx->buf + index, padding, padlen); - } else { - __sha512_sparc64_update(sctx, padding, padlen, index); - } - __sha512_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 112); - - /* Store state in digest */ - for (i = 0; i < 8; i++) - dst[i] = cpu_to_be64(sctx->state[i]); - - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); - - return 0; -} - -static int sha384_sparc64_final(struct shash_desc *desc, u8 *hash) -{ - u8 D[64]; - - sha512_sparc64_final(desc, D); - - memcpy(hash, D, 48); - memzero_explicit(D, 64); - - return 0; + sha512_base_do_finup(desc, src, len, sha512_block); + return sha512_base_finish(desc, out); } static struct shash_alg sha512 = { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, .update = sha512_sparc64_update, - .final = sha512_sparc64_final, - .descsize = sizeof(struct sha512_state), + .finup = sha512_sparc64_finup, + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha512", .cra_driver_name= "sha512-sparc64", @@ -132,8 +60,8 @@ static struct shash_alg sha384 = { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, .update = sha512_sparc64_update, - .final = sha384_sparc64_final, - .descsize = sizeof(struct sha512_state), + .finup = sha512_sparc64_finup, + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha384", .cra_driver_name= "sha384-sparc64", diff --git a/arch/x86/Kconfig.assembler b/arch/x86/Kconfig.assembler index 6d20a6ce0507..4d06fd3c8dfe 100644 --- a/arch/x86/Kconfig.assembler +++ b/arch/x86/Kconfig.assembler @@ -6,15 +6,6 @@ config AS_AVX512 help Supported by binutils >= 2.25 and LLVM integrated assembler -config AS_SHA1_NI - def_bool $(as-instr,sha1msg1 %xmm0$(comma)%xmm1) - help - Supported by binutils >= 2.24 and LLVM integrated assembler - -config AS_SHA256_NI - def_bool $(as-instr,sha256msg1 %xmm0$(comma)%xmm1) - help - Supported by binutils >= 2.24 and LLVM integrated assembler config AS_TPAUSE def_bool $(as-instr,tpause %ecx) help diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 3d948f10c94c..9a3cade49f63 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -23,7 +23,6 @@ config CRYPTO_AES_NI_INTEL select CRYPTO_LIB_GF128MUL select CRYPTO_ALGAPI select CRYPTO_SKCIPHER - select CRYPTO_SIMD help Block cipher: AES cipher algorithms AEAD cipher: AES with GCM @@ -64,7 +63,6 @@ config CRYPTO_CAMELLIA_AESNI_AVX_X86_64 depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_CAMELLIA_X86_64 - select CRYPTO_SIMD imply CRYPTO_XTS help Length-preserving ciphers: Camellia with ECB and CBC modes @@ -90,7 +88,6 @@ config CRYPTO_CAST5_AVX_X86_64 select CRYPTO_SKCIPHER select CRYPTO_CAST5 select CRYPTO_CAST_COMMON - select CRYPTO_SIMD imply CRYPTO_CTR help Length-preserving ciphers: CAST5 (CAST-128) cipher algorithm @@ -107,7 +104,6 @@ config CRYPTO_CAST6_AVX_X86_64 select CRYPTO_SKCIPHER select CRYPTO_CAST6 select CRYPTO_CAST_COMMON - select CRYPTO_SIMD imply CRYPTO_XTS imply CRYPTO_CTR help @@ -138,7 +134,6 @@ config CRYPTO_SERPENT_SSE2_X86_64 depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_SERPENT - select CRYPTO_SIMD imply CRYPTO_CTR help Length-preserving ciphers: Serpent cipher algorithm @@ -154,7 +149,6 @@ config CRYPTO_SERPENT_SSE2_586 depends on X86 && !64BIT select CRYPTO_SKCIPHER select CRYPTO_SERPENT - select CRYPTO_SIMD imply CRYPTO_CTR help Length-preserving ciphers: Serpent cipher algorithm @@ -170,7 +164,6 @@ config CRYPTO_SERPENT_AVX_X86_64 depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_SERPENT - select CRYPTO_SIMD imply CRYPTO_XTS imply CRYPTO_CTR help @@ -199,7 +192,6 @@ config CRYPTO_SM4_AESNI_AVX_X86_64 tristate "Ciphers: SM4 with modes: ECB, CBC, CTR (AES-NI/AVX)" depends on X86 && 64BIT select CRYPTO_SKCIPHER - select CRYPTO_SIMD select CRYPTO_ALGAPI select CRYPTO_SM4 help @@ -220,7 +212,6 @@ config CRYPTO_SM4_AESNI_AVX2_X86_64 tristate "Ciphers: SM4 with modes: ECB, CBC, CTR (AES-NI/AVX2)" depends on X86 && 64BIT select CRYPTO_SKCIPHER - select CRYPTO_SIMD select CRYPTO_ALGAPI select CRYPTO_SM4 select CRYPTO_SM4_AESNI_AVX_X86_64 @@ -279,7 +270,6 @@ config CRYPTO_TWOFISH_AVX_X86_64 tristate "Ciphers: Twofish with modes: ECB, CBC (AVX)" depends on X86 && 64BIT select CRYPTO_SKCIPHER - select CRYPTO_SIMD select CRYPTO_TWOFISH_COMMON select CRYPTO_TWOFISH_X86_64 select CRYPTO_TWOFISH_X86_64_3WAY @@ -297,7 +287,6 @@ config CRYPTO_ARIA_AESNI_AVX_X86_64 tristate "Ciphers: ARIA with modes: ECB, CTR (AES-NI/AVX/GFNI)" depends on X86 && 64BIT select CRYPTO_SKCIPHER - select CRYPTO_SIMD select CRYPTO_ALGAPI select CRYPTO_ARIA help @@ -315,7 +304,6 @@ config CRYPTO_ARIA_AESNI_AVX2_X86_64 tristate "Ciphers: ARIA with modes: ECB, CTR (AES-NI/AVX2/GFNI)" depends on X86 && 64BIT select CRYPTO_SKCIPHER - select CRYPTO_SIMD select CRYPTO_ALGAPI select CRYPTO_ARIA select CRYPTO_ARIA_AESNI_AVX_X86_64 @@ -332,9 +320,8 @@ config CRYPTO_ARIA_AESNI_AVX2_X86_64 config CRYPTO_ARIA_GFNI_AVX512_X86_64 tristate "Ciphers: ARIA with modes: ECB, CTR (AVX512/GFNI)" - depends on X86 && 64BIT && AS_AVX512 && AS_GFNI + depends on X86 && 64BIT && AS_GFNI select CRYPTO_SKCIPHER - select CRYPTO_SIMD select CRYPTO_ALGAPI select CRYPTO_ARIA select CRYPTO_ARIA_AESNI_AVX_X86_64 @@ -352,24 +339,14 @@ config CRYPTO_ARIA_GFNI_AVX512_X86_64 config CRYPTO_CHACHA20_X86_64 tristate depends on X86 && 64BIT - select CRYPTO_SKCIPHER select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA default CRYPTO_LIB_CHACHA_INTERNAL - help - Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 - stream cipher algorithms - - Architecture: x86_64 using: - - SSSE3 (Supplemental SSE3) - - AVX2 (Advanced Vector Extensions 2) - - AVX-512VL (Advanced Vector Extensions-512VL) config CRYPTO_AEGIS128_AESNI_SSE2 tristate "AEAD ciphers: AEGIS-128 (AES-NI/SSE4.1)" depends on X86 && 64BIT select CRYPTO_AEAD - select CRYPTO_SIMD help AEGIS-128 AEAD algorithm @@ -422,16 +399,8 @@ config CRYPTO_POLYVAL_CLMUL_NI config CRYPTO_POLY1305_X86_64 tristate depends on X86 && 64BIT - select CRYPTO_HASH - select CRYPTO_LIB_POLY1305_GENERIC select CRYPTO_ARCH_HAVE_LIB_POLY1305 default CRYPTO_LIB_POLY1305_INTERNAL - help - Poly1305 authenticator algorithm (RFC7539) - - Architecture: x86_64 using: - - SSE2 (Streaming SIMD Extensions 2) - - AVX2 (Advanced Vector Extensions 2) config CRYPTO_SHA1_SSSE3 tristate "Hash functions: SHA-1 (SSSE3/AVX/AVX2/SHA-NI)" @@ -478,7 +447,7 @@ config CRYPTO_SM3_AVX_X86_64 tristate "Hash functions: SM3 (AVX)" depends on X86 && 64BIT select CRYPTO_HASH - select CRYPTO_SM3 + select CRYPTO_LIB_SM3 help SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3 diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 5d19f41bde58..2f22b381f244 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -43,8 +43,7 @@ obj-$(CONFIG_CRYPTO_AEGIS128_AESNI_SSE2) += aegis128-aesni.o aegis128-aesni-y := aegis128-aesni-asm.o aegis128-aesni-glue.o obj-$(CONFIG_CRYPTO_CHACHA20_X86_64) += chacha-x86_64.o -chacha-x86_64-y := chacha-avx2-x86_64.o chacha-ssse3-x86_64.o chacha_glue.o -chacha-x86_64-$(CONFIG_AS_AVX512) += chacha-avx512vl-x86_64.o +chacha-x86_64-y := chacha-avx2-x86_64.o chacha-ssse3-x86_64.o chacha-avx512vl-x86_64.o chacha_glue.o obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o @@ -56,12 +55,10 @@ aesni-intel-$(CONFIG_64BIT) += aes-gcm-avx10-x86_64.o endif obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o -sha1-ssse3-y := sha1_avx2_x86_64_asm.o sha1_ssse3_asm.o sha1_ssse3_glue.o -sha1-ssse3-$(CONFIG_AS_SHA1_NI) += sha1_ni_asm.o +sha1-ssse3-y := sha1_avx2_x86_64_asm.o sha1_ssse3_asm.o sha1_ni_asm.o sha1_ssse3_glue.o obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o -sha256-ssse3-y := sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256_ssse3_glue.o -sha256-ssse3-$(CONFIG_AS_SHA256_NI) += sha256_ni_asm.o +sha256-ssse3-y := sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256_ni_asm.o sha256_ssse3_glue.o obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o diff --git a/arch/x86/crypto/aegis128-aesni-glue.c b/arch/x86/crypto/aegis128-aesni-glue.c index 26786e15abac..f1b6d40154e3 100644 --- a/arch/x86/crypto/aegis128-aesni-glue.c +++ b/arch/x86/crypto/aegis128-aesni-glue.c @@ -8,7 +8,6 @@ */ #include <crypto/internal/aead.h> -#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <linux/module.h> @@ -233,21 +232,18 @@ static struct aead_alg crypto_aegis128_aesni_alg = { .chunksize = AEGIS128_BLOCK_SIZE, .base = { - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct aegis_ctx) + __alignof__(struct aegis_ctx), .cra_priority = 400, - .cra_name = "__aegis128", - .cra_driver_name = "__aegis128-aesni", + .cra_name = "aegis128", + .cra_driver_name = "aegis128-aesni", .cra_module = THIS_MODULE, } }; -static struct simd_aead_alg *simd_alg; - static int __init crypto_aegis128_aesni_module_init(void) { if (!boot_cpu_has(X86_FEATURE_XMM4_1) || @@ -255,13 +251,12 @@ static int __init crypto_aegis128_aesni_module_init(void) !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) return -ENODEV; - return simd_register_aeads_compat(&crypto_aegis128_aesni_alg, 1, - &simd_alg); + return crypto_register_aead(&crypto_aegis128_aesni_alg); } static void __exit crypto_aegis128_aesni_module_exit(void) { - simd_unregister_aeads(&crypto_aegis128_aesni_alg, 1, &simd_alg); + crypto_unregister_aead(&crypto_aegis128_aesni_alg); } module_init(crypto_aegis128_aesni_module_init); diff --git a/arch/x86/crypto/aes-ctr-avx-x86_64.S b/arch/x86/crypto/aes-ctr-avx-x86_64.S index 1685d8b24b2c..bbbfd80f5a50 100644 --- a/arch/x86/crypto/aes-ctr-avx-x86_64.S +++ b/arch/x86/crypto/aes-ctr-avx-x86_64.S @@ -48,8 +48,7 @@ // using the following sets of CPU features: // - AES-NI && AVX // - VAES && AVX2 -// - VAES && (AVX10/256 || (AVX512BW && AVX512VL)) && BMI2 -// - VAES && (AVX10/512 || (AVX512BW && AVX512VL)) && BMI2 +// - VAES && AVX512BW && AVX512VL && BMI2 // // See the function definitions at the bottom of the file for more information. @@ -76,7 +75,6 @@ .text // Move a vector between memory and a register. -// The register operand must be in the first 16 vector registers. .macro _vmovdqu src, dst .if VL < 64 vmovdqu \src, \dst @@ -86,7 +84,6 @@ .endm // Move a vector between registers. -// The registers must be in the first 16 vector registers. .macro _vmovdqa src, dst .if VL < 64 vmovdqa \src, \dst @@ -96,7 +93,7 @@ .endm // Broadcast a 128-bit value from memory to all 128-bit lanes of a vector -// register. The register operand must be in the first 16 vector registers. +// register. .macro _vbroadcast128 src, dst .if VL == 16 vmovdqu \src, \dst @@ -108,7 +105,6 @@ .endm // XOR two vectors together. -// Any register operands must be in the first 16 vector registers. .macro _vpxor src1, src2, dst .if VL < 64 vpxor \src1, \src2, \dst @@ -199,8 +195,8 @@ // XOR each with the zero-th round key. Also update LE_CTR if !\final. .macro _prepare_2_ctr_vecs is_xctr, i0, i1, final=0 .if \is_xctr - .if USE_AVX10 - _vmovdqa LE_CTR, AESDATA\i0 + .if USE_AVX512 + vmovdqa64 LE_CTR, AESDATA\i0 vpternlogd $0x96, XCTR_IV, RNDKEY0, AESDATA\i0 .else vpxor XCTR_IV, LE_CTR, AESDATA\i0 @@ -208,7 +204,7 @@ .endif vpaddq LE_CTR_INC1, LE_CTR, AESDATA\i1 - .if USE_AVX10 + .if USE_AVX512 vpternlogd $0x96, XCTR_IV, RNDKEY0, AESDATA\i1 .else vpxor XCTR_IV, AESDATA\i1, AESDATA\i1 @@ -481,18 +477,12 @@ .Lxor_tail_partial_vec_0\@: // XOR the remaining 1 <= LEN < VL bytes. It's easy if masked // loads/stores are available; otherwise it's a bit harder... -.if USE_AVX10 - .if VL <= 32 - mov $-1, %eax - bzhi LEN, %eax, %eax - kmovd %eax, %k1 - .else +.if USE_AVX512 mov $-1, %rax bzhi LEN64, %rax, %rax kmovq %rax, %k1 - .endif vmovdqu8 (SRC), AESDATA1{%k1}{z} - _vpxor AESDATA1, AESDATA0, AESDATA0 + vpxord AESDATA1, AESDATA0, AESDATA0 vmovdqu8 AESDATA0, (DST){%k1} .else .if VL == 32 @@ -554,7 +544,7 @@ // eliminates carries. |ctr| is the per-message block counter starting at 1. .set VL, 16 -.set USE_AVX10, 0 +.set USE_AVX512, 0 SYM_TYPED_FUNC_START(aes_ctr64_crypt_aesni_avx) _aes_ctr_crypt 0 SYM_FUNC_END(aes_ctr64_crypt_aesni_avx) @@ -564,7 +554,7 @@ SYM_FUNC_END(aes_xctr_crypt_aesni_avx) #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) .set VL, 32 -.set USE_AVX10, 0 +.set USE_AVX512, 0 SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx2) _aes_ctr_crypt 0 SYM_FUNC_END(aes_ctr64_crypt_vaes_avx2) @@ -572,21 +562,12 @@ SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx2) _aes_ctr_crypt 1 SYM_FUNC_END(aes_xctr_crypt_vaes_avx2) -.set VL, 32 -.set USE_AVX10, 1 -SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx10_256) - _aes_ctr_crypt 0 -SYM_FUNC_END(aes_ctr64_crypt_vaes_avx10_256) -SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx10_256) - _aes_ctr_crypt 1 -SYM_FUNC_END(aes_xctr_crypt_vaes_avx10_256) - .set VL, 64 -.set USE_AVX10, 1 -SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx10_512) +.set USE_AVX512, 1 +SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx512) _aes_ctr_crypt 0 -SYM_FUNC_END(aes_ctr64_crypt_vaes_avx10_512) -SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx10_512) +SYM_FUNC_END(aes_ctr64_crypt_vaes_avx512) +SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx512) _aes_ctr_crypt 1 -SYM_FUNC_END(aes_xctr_crypt_vaes_avx10_512) +SYM_FUNC_END(aes_xctr_crypt_vaes_avx512) #endif // CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ diff --git a/arch/x86/crypto/aes-xts-avx-x86_64.S b/arch/x86/crypto/aes-xts-avx-x86_64.S index 93ba0ddbe009..db79cdf81588 100644 --- a/arch/x86/crypto/aes-xts-avx-x86_64.S +++ b/arch/x86/crypto/aes-xts-avx-x86_64.S @@ -52,32 +52,25 @@ * different code, it uses a macro to generate several implementations that * share similar source code but are targeted at different CPUs, listed below: * - * AES-NI + AVX + * AES-NI && AVX * - 128-bit vectors (1 AES block per vector) * - VEX-coded instructions * - xmm0-xmm15 * - This is for older CPUs that lack VAES but do have AVX. * - * VAES + VPCLMULQDQ + AVX2 + * VAES && VPCLMULQDQ && AVX2 * - 256-bit vectors (2 AES blocks per vector) * - VEX-coded instructions * - ymm0-ymm15 - * - This is for CPUs that have VAES but lack AVX512 or AVX10, - * e.g. Intel's Alder Lake and AMD's Zen 3. + * - This is for CPUs that have VAES but either lack AVX512 (e.g. Intel's + * Alder Lake and AMD's Zen 3) or downclock too eagerly when using zmm + * registers (e.g. Intel's Ice Lake). * - * VAES + VPCLMULQDQ + AVX10/256 + BMI2 - * - 256-bit vectors (2 AES blocks per vector) + * VAES && VPCLMULQDQ && AVX512BW && AVX512VL && BMI2 + * - 512-bit vectors (4 AES blocks per vector) * - EVEX-coded instructions - * - ymm0-ymm31 - * - This is for CPUs that have AVX512 but where using zmm registers causes - * downclocking, and for CPUs that have AVX10/256 but not AVX10/512. - * - By "AVX10/256" we really mean (AVX512BW + AVX512VL) || AVX10/256. - * To avoid confusion with 512-bit, we just write AVX10/256. - * - * VAES + VPCLMULQDQ + AVX10/512 + BMI2 - * - Same as the previous one, but upgrades to 512-bit vectors - * (4 AES blocks per vector) in zmm0-zmm31. - * - This is for CPUs that have good AVX512 or AVX10/512 support. + * - zmm0-zmm31 + * - This is for CPUs that have good AVX512 support. * * This file doesn't have an implementation for AES-NI alone (without AVX), as * the lack of VEX would make all the assembly code different. @@ -107,9 +100,20 @@ // exists when there's a carry out of the low 64 bits of the tweak. .quad 0x87, 1 + // These are the shift amounts that are needed when multiplying by [x^0, + // x^1, x^2, x^3] to compute the first vector of tweaks when VL=64. + // + // The right shifts by 64 are expected to zeroize the destination. + // 'vpsrlvq' is indeed defined to do that; i.e. it doesn't truncate the + // amount to 64 & 63 = 0 like the 'shr' scalar shift instruction would. +.Lrshift_amounts: + .byte 64, 64, 63, 63, 62, 62, 61, 61 +.Llshift_amounts: + .byte 0, 0, 1, 1, 2, 2, 3, 3 + // This table contains constants for vpshufb and vpblendvb, used to // handle variable byte shifts and blending during ciphertext stealing - // on CPUs that don't support AVX10-style masking. + // on CPUs that don't support AVX512-style masking. .Lcts_permute_table: .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 @@ -138,7 +142,7 @@ .irp i, 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 _define_Vi \i .endr -.if USE_AVX10 +.if USE_AVX512 .irp i, 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 _define_Vi \i .endr @@ -193,7 +197,7 @@ // keys to the *end* of this register range. I.e., AES-128 uses // KEY5-KEY14, AES-192 uses KEY3-KEY14, and AES-256 uses KEY1-KEY14. // (All also use KEY0 for the XOR-only "round" at the beginning.) -.if USE_AVX10 +.if USE_AVX512 .set KEY1_XMM, %xmm16 .set KEY1, V16 .set KEY2_XMM, %xmm17 @@ -227,7 +231,6 @@ .endm // Move a vector between memory and a register. -// The register operand must be in the first 16 vector registers. .macro _vmovdqu src, dst .if VL < 64 vmovdqu \src, \dst @@ -238,9 +241,9 @@ // Broadcast a 128-bit value into a vector. .macro _vbroadcast128 src, dst -.if VL == 16 && !USE_AVX10 +.if VL == 16 vmovdqu \src, \dst -.elseif VL == 32 && !USE_AVX10 +.elseif VL == 32 vbroadcasti128 \src, \dst .else vbroadcasti32x4 \src, \dst @@ -248,7 +251,6 @@ .endm // XOR two vectors together. -// Any register operands must be in the first 16 vector registers. .macro _vpxor src1, src2, dst .if VL < 64 vpxor \src1, \src2, \dst @@ -259,7 +261,7 @@ // XOR three vectors together. .macro _xor3 src1, src2, src3_and_dst -.if USE_AVX10 +.if USE_AVX512 // vpternlogd with immediate 0x96 is a three-argument XOR. vpternlogd $0x96, \src1, \src2, \src3_and_dst .else @@ -274,7 +276,7 @@ vpshufd $0x13, \src, \tmp vpaddq \src, \src, \dst vpsrad $31, \tmp, \tmp -.if USE_AVX10 +.if USE_AVX512 vpternlogd $0x78, GF_POLY_XMM, \tmp, \dst .else vpand GF_POLY_XMM, \tmp, \tmp @@ -303,52 +305,75 @@ // Given the first XTS tweak at (TWEAK), compute the first set of tweaks and // store them in the vector registers TWEAK0-TWEAK3. Clobbers V0-V5. .macro _compute_first_set_of_tweaks - vmovdqu (TWEAK), TWEAK0_XMM - _vbroadcast128 .Lgf_poly(%rip), GF_POLY .if VL == 16 - // With VL=16, multiplying by x serially is fastest. + vmovdqu (TWEAK), TWEAK0_XMM + vmovdqu .Lgf_poly(%rip), GF_POLY _next_tweak TWEAK0, %xmm0, TWEAK1 _next_tweak TWEAK1, %xmm0, TWEAK2 _next_tweak TWEAK2, %xmm0, TWEAK3 -.else -.if VL == 32 - // Compute the second block of TWEAK0. +.elseif VL == 32 + vmovdqu (TWEAK), TWEAK0_XMM + vbroadcasti128 .Lgf_poly(%rip), GF_POLY + + // Compute the first vector of tweaks. _next_tweak TWEAK0_XMM, %xmm0, %xmm1 vinserti128 $1, %xmm1, TWEAK0, TWEAK0 -.elseif VL == 64 - // Compute the remaining blocks of TWEAK0. - _next_tweak TWEAK0_XMM, %xmm0, %xmm1 - _next_tweak %xmm1, %xmm0, %xmm2 - _next_tweak %xmm2, %xmm0, %xmm3 - vinserti32x4 $1, %xmm1, TWEAK0, TWEAK0 - vinserti32x4 $2, %xmm2, TWEAK0, TWEAK0 - vinserti32x4 $3, %xmm3, TWEAK0, TWEAK0 -.endif - // Compute TWEAK[1-3] from TWEAK0. - vpsrlq $64 - 1*VL/16, TWEAK0, V0 - vpsrlq $64 - 2*VL/16, TWEAK0, V2 - vpsrlq $64 - 3*VL/16, TWEAK0, V4 + + // Compute the next three vectors of tweaks: + // TWEAK1 = TWEAK0 * [x^2, x^2] + // TWEAK2 = TWEAK0 * [x^4, x^4] + // TWEAK3 = TWEAK0 * [x^6, x^6] + vpsrlq $64 - 2, TWEAK0, V0 + vpsrlq $64 - 4, TWEAK0, V2 + vpsrlq $64 - 6, TWEAK0, V4 vpclmulqdq $0x01, GF_POLY, V0, V1 vpclmulqdq $0x01, GF_POLY, V2, V3 vpclmulqdq $0x01, GF_POLY, V4, V5 vpslldq $8, V0, V0 vpslldq $8, V2, V2 vpslldq $8, V4, V4 - vpsllq $1*VL/16, TWEAK0, TWEAK1 - vpsllq $2*VL/16, TWEAK0, TWEAK2 - vpsllq $3*VL/16, TWEAK0, TWEAK3 -.if USE_AVX10 - vpternlogd $0x96, V0, V1, TWEAK1 - vpternlogd $0x96, V2, V3, TWEAK2 - vpternlogd $0x96, V4, V5, TWEAK3 -.else + vpsllq $2, TWEAK0, TWEAK1 + vpsllq $4, TWEAK0, TWEAK2 + vpsllq $6, TWEAK0, TWEAK3 vpxor V0, TWEAK1, TWEAK1 vpxor V2, TWEAK2, TWEAK2 vpxor V4, TWEAK3, TWEAK3 vpxor V1, TWEAK1, TWEAK1 vpxor V3, TWEAK2, TWEAK2 vpxor V5, TWEAK3, TWEAK3 -.endif +.else + vbroadcasti32x4 (TWEAK), TWEAK0 + vbroadcasti32x4 .Lgf_poly(%rip), GF_POLY + + // Compute the first vector of tweaks: + // TWEAK0 = broadcast128(TWEAK) * [x^0, x^1, x^2, x^3] + vpmovzxbq .Lrshift_amounts(%rip), V4 + vpsrlvq V4, TWEAK0, V0 + vpclmulqdq $0x01, GF_POLY, V0, V1 + vpmovzxbq .Llshift_amounts(%rip), V4 + vpslldq $8, V0, V0 + vpsllvq V4, TWEAK0, TWEAK0 + vpternlogd $0x96, V0, V1, TWEAK0 + + // Compute the next three vectors of tweaks: + // TWEAK1 = TWEAK0 * [x^4, x^4, x^4, x^4] + // TWEAK2 = TWEAK0 * [x^8, x^8, x^8, x^8] + // TWEAK3 = TWEAK0 * [x^12, x^12, x^12, x^12] + // x^8 only needs byte-aligned shifts, so optimize accordingly. + vpsrlq $64 - 4, TWEAK0, V0 + vpsrldq $(64 - 8) / 8, TWEAK0, V2 + vpsrlq $64 - 12, TWEAK0, V4 + vpclmulqdq $0x01, GF_POLY, V0, V1 + vpclmulqdq $0x01, GF_POLY, V2, V3 + vpclmulqdq $0x01, GF_POLY, V4, V5 + vpslldq $8, V0, V0 + vpslldq $8, V4, V4 + vpsllq $4, TWEAK0, TWEAK1 + vpslldq $8 / 8, TWEAK0, TWEAK2 + vpsllq $12, TWEAK0, TWEAK3 + vpternlogd $0x96, V0, V1, TWEAK1 + vpxord V3, TWEAK2, TWEAK2 + vpternlogd $0x96, V4, V5, TWEAK3 .endif .endm @@ -474,26 +499,26 @@ lea OFFS-16(KEY, KEYLEN64, 4), KEY // If all 32 SIMD registers are available, cache all the round keys. -.if USE_AVX10 +.if USE_AVX512 cmp $24, KEYLEN jl .Laes128\@ je .Laes192\@ - _vbroadcast128 -6*16(KEY), KEY1 - _vbroadcast128 -5*16(KEY), KEY2 + vbroadcasti32x4 -6*16(KEY), KEY1 + vbroadcasti32x4 -5*16(KEY), KEY2 .Laes192\@: - _vbroadcast128 -4*16(KEY), KEY3 - _vbroadcast128 -3*16(KEY), KEY4 + vbroadcasti32x4 -4*16(KEY), KEY3 + vbroadcasti32x4 -3*16(KEY), KEY4 .Laes128\@: - _vbroadcast128 -2*16(KEY), KEY5 - _vbroadcast128 -1*16(KEY), KEY6 - _vbroadcast128 0*16(KEY), KEY7 - _vbroadcast128 1*16(KEY), KEY8 - _vbroadcast128 2*16(KEY), KEY9 - _vbroadcast128 3*16(KEY), KEY10 - _vbroadcast128 4*16(KEY), KEY11 - _vbroadcast128 5*16(KEY), KEY12 - _vbroadcast128 6*16(KEY), KEY13 - _vbroadcast128 7*16(KEY), KEY14 + vbroadcasti32x4 -2*16(KEY), KEY5 + vbroadcasti32x4 -1*16(KEY), KEY6 + vbroadcasti32x4 0*16(KEY), KEY7 + vbroadcasti32x4 1*16(KEY), KEY8 + vbroadcasti32x4 2*16(KEY), KEY9 + vbroadcasti32x4 3*16(KEY), KEY10 + vbroadcasti32x4 4*16(KEY), KEY11 + vbroadcasti32x4 5*16(KEY), KEY12 + vbroadcasti32x4 6*16(KEY), KEY13 + vbroadcasti32x4 7*16(KEY), KEY14 .endif .endm @@ -521,7 +546,7 @@ // using the same key for all block(s). The round key is loaded from the // appropriate register or memory location for round \i. May clobber \tmp. .macro _vaes_1x enc, i, xmm_suffix, data, tmp -.if USE_AVX10 +.if USE_AVX512 _vaes \enc, KEY\i\xmm_suffix, \data .else .ifnb \xmm_suffix @@ -538,7 +563,7 @@ // appropriate register or memory location for round \i. In addition, does two // steps of the computation of the next set of tweaks. May clobber V4 and V5. .macro _vaes_4x enc, i -.if USE_AVX10 +.if USE_AVX512 _tweak_step (2*(\i-5)) _vaes \enc, KEY\i, V0 _vaes \enc, KEY\i, V1 @@ -574,7 +599,7 @@ .irp i, 5,6,7,8,9,10,11,12,13 _vaes_1x \enc, \i, \xmm_suffix, \data, tmp=\tmp .endr -.if USE_AVX10 +.if USE_AVX512 vpxord KEY14\xmm_suffix, \tweak, \tmp .else .ifnb \xmm_suffix @@ -617,11 +642,11 @@ // This is the main loop, en/decrypting 4*VL bytes per iteration. // XOR each source block with its tweak and the zero-th round key. -.if USE_AVX10 - _vmovdqu 0*VL(SRC), V0 - _vmovdqu 1*VL(SRC), V1 - _vmovdqu 2*VL(SRC), V2 - _vmovdqu 3*VL(SRC), V3 +.if USE_AVX512 + vmovdqu8 0*VL(SRC), V0 + vmovdqu8 1*VL(SRC), V1 + vmovdqu8 2*VL(SRC), V2 + vmovdqu8 3*VL(SRC), V3 vpternlogd $0x96, TWEAK0, KEY0, V0 vpternlogd $0x96, TWEAK1, KEY0, V1 vpternlogd $0x96, TWEAK2, KEY0, V2 @@ -654,7 +679,7 @@ // Reduce latency by doing the XOR before the vaesenclast, utilizing the // property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a) // (and likewise for vaesdeclast). -.if USE_AVX10 +.if USE_AVX512 _tweak_step 18 _tweak_step 19 vpxord TWEAK0, KEY14, V4 @@ -762,7 +787,7 @@ _aes_crypt \enc, _XMM, TWEAK1_XMM, %xmm0, tmp=%xmm1 .endif -.if USE_AVX10 +.if USE_AVX512 // Create a mask that has the first LEN bits set. mov $-1, %r9d bzhi LEN, %r9d, %r9d @@ -811,7 +836,7 @@ // u8 iv[AES_BLOCK_SIZE]); // // Encrypt |iv| using the AES key |tweak_key| to get the first tweak. Assumes -// that the CPU supports AES-NI and AVX, but not necessarily VAES or AVX10. +// that the CPU supports AES-NI and AVX, but not necessarily VAES or AVX512. SYM_TYPED_FUNC_START(aes_xts_encrypt_iv) .set TWEAK_KEY, %rdi .set IV, %rsi @@ -853,7 +878,7 @@ SYM_FUNC_END(aes_xts_encrypt_iv) // multiple of 16, then this function updates |tweak| to contain the next tweak. .set VL, 16 -.set USE_AVX10, 0 +.set USE_AVX512, 0 SYM_TYPED_FUNC_START(aes_xts_encrypt_aesni_avx) _aes_xts_crypt 1 SYM_FUNC_END(aes_xts_encrypt_aesni_avx) @@ -863,7 +888,7 @@ SYM_FUNC_END(aes_xts_decrypt_aesni_avx) #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) .set VL, 32 -.set USE_AVX10, 0 +.set USE_AVX512, 0 SYM_TYPED_FUNC_START(aes_xts_encrypt_vaes_avx2) _aes_xts_crypt 1 SYM_FUNC_END(aes_xts_encrypt_vaes_avx2) @@ -871,21 +896,12 @@ SYM_TYPED_FUNC_START(aes_xts_decrypt_vaes_avx2) _aes_xts_crypt 0 SYM_FUNC_END(aes_xts_decrypt_vaes_avx2) -.set VL, 32 -.set USE_AVX10, 1 -SYM_TYPED_FUNC_START(aes_xts_encrypt_vaes_avx10_256) - _aes_xts_crypt 1 -SYM_FUNC_END(aes_xts_encrypt_vaes_avx10_256) -SYM_TYPED_FUNC_START(aes_xts_decrypt_vaes_avx10_256) - _aes_xts_crypt 0 -SYM_FUNC_END(aes_xts_decrypt_vaes_avx10_256) - .set VL, 64 -.set USE_AVX10, 1 -SYM_TYPED_FUNC_START(aes_xts_encrypt_vaes_avx10_512) +.set USE_AVX512, 1 +SYM_TYPED_FUNC_START(aes_xts_encrypt_vaes_avx512) _aes_xts_crypt 1 -SYM_FUNC_END(aes_xts_encrypt_vaes_avx10_512) -SYM_TYPED_FUNC_START(aes_xts_decrypt_vaes_avx10_512) +SYM_FUNC_END(aes_xts_encrypt_vaes_avx512) +SYM_TYPED_FUNC_START(aes_xts_decrypt_vaes_avx512) _aes_xts_crypt 0 -SYM_FUNC_END(aes_xts_decrypt_vaes_avx10_512) +SYM_FUNC_END(aes_xts_decrypt_vaes_avx512) #endif /* CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ */ diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index bc655d794a95..061b1ced93c5 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -566,10 +566,9 @@ static struct crypto_alg aesni_cipher_alg = { static struct skcipher_alg aesni_skciphers[] = { { .base = { - .cra_name = "__ecb(aes)", - .cra_driver_name = "__ecb-aes-aesni", + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-aesni", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = CRYPTO_AES_CTX_SIZE, .cra_module = THIS_MODULE, @@ -581,10 +580,9 @@ static struct skcipher_alg aesni_skciphers[] = { .decrypt = ecb_decrypt, }, { .base = { - .cra_name = "__cbc(aes)", - .cra_driver_name = "__cbc-aes-aesni", + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-aesni", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = CRYPTO_AES_CTX_SIZE, .cra_module = THIS_MODULE, @@ -597,10 +595,9 @@ static struct skcipher_alg aesni_skciphers[] = { .decrypt = cbc_decrypt, }, { .base = { - .cra_name = "__cts(cbc(aes))", - .cra_driver_name = "__cts-cbc-aes-aesni", + .cra_name = "cts(cbc(aes))", + .cra_driver_name = "cts-cbc-aes-aesni", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = CRYPTO_AES_CTX_SIZE, .cra_module = THIS_MODULE, @@ -615,10 +612,9 @@ static struct skcipher_alg aesni_skciphers[] = { #ifdef CONFIG_X86_64 }, { .base = { - .cra_name = "__ctr(aes)", - .cra_driver_name = "__ctr-aes-aesni", + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-aesni", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = CRYPTO_AES_CTX_SIZE, .cra_module = THIS_MODULE, @@ -633,10 +629,9 @@ static struct skcipher_alg aesni_skciphers[] = { #endif }, { .base = { - .cra_name = "__xts(aes)", - .cra_driver_name = "__xts-aes-aesni", + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-aesni", .cra_priority = 401, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = XTS_AES_CTX_SIZE, .cra_module = THIS_MODULE, @@ -651,9 +646,6 @@ static struct skcipher_alg aesni_skciphers[] = { } }; -static -struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)]; - #ifdef CONFIG_X86_64 asmlinkage void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key, u8 iv[AES_BLOCK_SIZE]); @@ -792,10 +784,9 @@ static int xctr_crypt_##suffix(struct skcipher_request *req) \ } \ \ static struct skcipher_alg skcipher_algs_##suffix[] = {{ \ - .base.cra_name = "__xts(aes)", \ - .base.cra_driver_name = "__xts-aes-" driver_name_suffix, \ + .base.cra_name = "xts(aes)", \ + .base.cra_driver_name = "xts-aes-" driver_name_suffix, \ .base.cra_priority = priority, \ - .base.cra_flags = CRYPTO_ALG_INTERNAL, \ .base.cra_blocksize = AES_BLOCK_SIZE, \ .base.cra_ctxsize = XTS_AES_CTX_SIZE, \ .base.cra_module = THIS_MODULE, \ @@ -807,10 +798,9 @@ static struct skcipher_alg skcipher_algs_##suffix[] = {{ \ .encrypt = xts_encrypt_##suffix, \ .decrypt = xts_decrypt_##suffix, \ }, { \ - .base.cra_name = "__ctr(aes)", \ - .base.cra_driver_name = "__ctr-aes-" driver_name_suffix, \ + .base.cra_name = "ctr(aes)", \ + .base.cra_driver_name = "ctr-aes-" driver_name_suffix, \ .base.cra_priority = priority, \ - .base.cra_flags = CRYPTO_ALG_INTERNAL, \ .base.cra_blocksize = 1, \ .base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \ .base.cra_module = THIS_MODULE, \ @@ -822,10 +812,9 @@ static struct skcipher_alg skcipher_algs_##suffix[] = {{ \ .encrypt = ctr_crypt_##suffix, \ .decrypt = ctr_crypt_##suffix, \ }, { \ - .base.cra_name = "__xctr(aes)", \ - .base.cra_driver_name = "__xctr-aes-" driver_name_suffix, \ + .base.cra_name = "xctr(aes)", \ + .base.cra_driver_name = "xctr-aes-" driver_name_suffix, \ .base.cra_priority = priority, \ - .base.cra_flags = CRYPTO_ALG_INTERNAL, \ .base.cra_blocksize = 1, \ .base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \ .base.cra_module = THIS_MODULE, \ @@ -836,16 +825,12 @@ static struct skcipher_alg skcipher_algs_##suffix[] = {{ \ .setkey = aesni_skcipher_setkey, \ .encrypt = xctr_crypt_##suffix, \ .decrypt = xctr_crypt_##suffix, \ -}}; \ - \ -static struct simd_skcipher_alg * \ -simd_skcipher_algs_##suffix[ARRAY_SIZE(skcipher_algs_##suffix)] +}} DEFINE_AVX_SKCIPHER_ALGS(aesni_avx, "aesni-avx", 500); #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) DEFINE_AVX_SKCIPHER_ALGS(vaes_avx2, "vaes-avx2", 600); -DEFINE_AVX_SKCIPHER_ALGS(vaes_avx10_256, "vaes-avx10_256", 700); -DEFINE_AVX_SKCIPHER_ALGS(vaes_avx10_512, "vaes-avx10_512", 800); +DEFINE_AVX_SKCIPHER_ALGS(vaes_avx512, "vaes-avx512", 800); #endif /* The common part of the x86_64 AES-GCM key struct */ @@ -1499,10 +1484,9 @@ static struct aead_alg aes_gcm_algs_##suffix[] = { { \ .chunksize = AES_BLOCK_SIZE, \ .maxauthsize = 16, \ .base = { \ - .cra_name = "__gcm(aes)", \ - .cra_driver_name = "__" generic_driver_name, \ + .cra_name = "gcm(aes)", \ + .cra_driver_name = generic_driver_name, \ .cra_priority = (priority), \ - .cra_flags = CRYPTO_ALG_INTERNAL, \ .cra_blocksize = 1, \ .cra_ctxsize = (ctxsize), \ .cra_module = THIS_MODULE, \ @@ -1516,17 +1500,14 @@ static struct aead_alg aes_gcm_algs_##suffix[] = { { \ .chunksize = AES_BLOCK_SIZE, \ .maxauthsize = 16, \ .base = { \ - .cra_name = "__rfc4106(gcm(aes))", \ - .cra_driver_name = "__" rfc_driver_name, \ + .cra_name = "rfc4106(gcm(aes))", \ + .cra_driver_name = rfc_driver_name, \ .cra_priority = (priority), \ - .cra_flags = CRYPTO_ALG_INTERNAL, \ .cra_blocksize = 1, \ .cra_ctxsize = (ctxsize), \ .cra_module = THIS_MODULE, \ }, \ -} }; \ - \ -static struct simd_aead_alg *aes_gcm_simdalgs_##suffix[2] \ +} } /* aes_gcm_algs_aesni */ DEFINE_GCM_ALGS(aesni, /* no flags */ 0, @@ -1556,14 +1537,12 @@ static int __init register_avx_algs(void) if (!boot_cpu_has(X86_FEATURE_AVX)) return 0; - err = simd_register_skciphers_compat(skcipher_algs_aesni_avx, - ARRAY_SIZE(skcipher_algs_aesni_avx), - simd_skcipher_algs_aesni_avx); + err = crypto_register_skciphers(skcipher_algs_aesni_avx, + ARRAY_SIZE(skcipher_algs_aesni_avx)); if (err) return err; - err = simd_register_aeads_compat(aes_gcm_algs_aesni_avx, - ARRAY_SIZE(aes_gcm_algs_aesni_avx), - aes_gcm_simdalgs_aesni_avx); + err = crypto_register_aeads(aes_gcm_algs_aesni_avx, + ARRAY_SIZE(aes_gcm_algs_aesni_avx)); if (err) return err; /* @@ -1579,9 +1558,8 @@ static int __init register_avx_algs(void) !boot_cpu_has(X86_FEATURE_PCLMULQDQ) || !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) return 0; - err = simd_register_skciphers_compat(skcipher_algs_vaes_avx2, - ARRAY_SIZE(skcipher_algs_vaes_avx2), - simd_skcipher_algs_vaes_avx2); + err = crypto_register_skciphers(skcipher_algs_vaes_avx2, + ARRAY_SIZE(skcipher_algs_vaes_avx2)); if (err) return err; @@ -1592,76 +1570,52 @@ static int __init register_avx_algs(void) XFEATURE_MASK_AVX512, NULL)) return 0; - err = simd_register_skciphers_compat(skcipher_algs_vaes_avx10_256, - ARRAY_SIZE(skcipher_algs_vaes_avx10_256), - simd_skcipher_algs_vaes_avx10_256); - if (err) - return err; - err = simd_register_aeads_compat(aes_gcm_algs_vaes_avx10_256, - ARRAY_SIZE(aes_gcm_algs_vaes_avx10_256), - aes_gcm_simdalgs_vaes_avx10_256); + err = crypto_register_aeads(aes_gcm_algs_vaes_avx10_256, + ARRAY_SIZE(aes_gcm_algs_vaes_avx10_256)); if (err) return err; if (boot_cpu_has(X86_FEATURE_PREFER_YMM)) { int i; - for (i = 0; i < ARRAY_SIZE(skcipher_algs_vaes_avx10_512); i++) - skcipher_algs_vaes_avx10_512[i].base.cra_priority = 1; + for (i = 0; i < ARRAY_SIZE(skcipher_algs_vaes_avx512); i++) + skcipher_algs_vaes_avx512[i].base.cra_priority = 1; for (i = 0; i < ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512); i++) aes_gcm_algs_vaes_avx10_512[i].base.cra_priority = 1; } - err = simd_register_skciphers_compat(skcipher_algs_vaes_avx10_512, - ARRAY_SIZE(skcipher_algs_vaes_avx10_512), - simd_skcipher_algs_vaes_avx10_512); + err = crypto_register_skciphers(skcipher_algs_vaes_avx512, + ARRAY_SIZE(skcipher_algs_vaes_avx512)); if (err) return err; - err = simd_register_aeads_compat(aes_gcm_algs_vaes_avx10_512, - ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512), - aes_gcm_simdalgs_vaes_avx10_512); + err = crypto_register_aeads(aes_gcm_algs_vaes_avx10_512, + ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512)); if (err) return err; #endif /* CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ */ return 0; } +#define unregister_skciphers(A) \ + if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \ + crypto_unregister_skciphers((A), ARRAY_SIZE(A)) +#define unregister_aeads(A) \ + if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \ + crypto_unregister_aeads((A), ARRAY_SIZE(A)) + static void unregister_avx_algs(void) { - if (simd_skcipher_algs_aesni_avx[0]) - simd_unregister_skciphers(skcipher_algs_aesni_avx, - ARRAY_SIZE(skcipher_algs_aesni_avx), - simd_skcipher_algs_aesni_avx); - if (aes_gcm_simdalgs_aesni_avx[0]) - simd_unregister_aeads(aes_gcm_algs_aesni_avx, - ARRAY_SIZE(aes_gcm_algs_aesni_avx), - aes_gcm_simdalgs_aesni_avx); + unregister_skciphers(skcipher_algs_aesni_avx); + unregister_aeads(aes_gcm_algs_aesni_avx); #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) - if (simd_skcipher_algs_vaes_avx2[0]) - simd_unregister_skciphers(skcipher_algs_vaes_avx2, - ARRAY_SIZE(skcipher_algs_vaes_avx2), - simd_skcipher_algs_vaes_avx2); - if (simd_skcipher_algs_vaes_avx10_256[0]) - simd_unregister_skciphers(skcipher_algs_vaes_avx10_256, - ARRAY_SIZE(skcipher_algs_vaes_avx10_256), - simd_skcipher_algs_vaes_avx10_256); - if (aes_gcm_simdalgs_vaes_avx10_256[0]) - simd_unregister_aeads(aes_gcm_algs_vaes_avx10_256, - ARRAY_SIZE(aes_gcm_algs_vaes_avx10_256), - aes_gcm_simdalgs_vaes_avx10_256); - if (simd_skcipher_algs_vaes_avx10_512[0]) - simd_unregister_skciphers(skcipher_algs_vaes_avx10_512, - ARRAY_SIZE(skcipher_algs_vaes_avx10_512), - simd_skcipher_algs_vaes_avx10_512); - if (aes_gcm_simdalgs_vaes_avx10_512[0]) - simd_unregister_aeads(aes_gcm_algs_vaes_avx10_512, - ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512), - aes_gcm_simdalgs_vaes_avx10_512); + unregister_skciphers(skcipher_algs_vaes_avx2); + unregister_skciphers(skcipher_algs_vaes_avx512); + unregister_aeads(aes_gcm_algs_vaes_avx10_256); + unregister_aeads(aes_gcm_algs_vaes_avx10_512); #endif } #else /* CONFIG_X86_64 */ static struct aead_alg aes_gcm_algs_aesni[0]; -static struct simd_aead_alg *aes_gcm_simdalgs_aesni[0]; static int __init register_avx_algs(void) { @@ -1690,15 +1644,13 @@ static int __init aesni_init(void) if (err) return err; - err = simd_register_skciphers_compat(aesni_skciphers, - ARRAY_SIZE(aesni_skciphers), - aesni_simd_skciphers); + err = crypto_register_skciphers(aesni_skciphers, + ARRAY_SIZE(aesni_skciphers)); if (err) goto unregister_cipher; - err = simd_register_aeads_compat(aes_gcm_algs_aesni, - ARRAY_SIZE(aes_gcm_algs_aesni), - aes_gcm_simdalgs_aesni); + err = crypto_register_aeads(aes_gcm_algs_aesni, + ARRAY_SIZE(aes_gcm_algs_aesni)); if (err) goto unregister_skciphers; @@ -1710,12 +1662,11 @@ static int __init aesni_init(void) unregister_avx: unregister_avx_algs(); - simd_unregister_aeads(aes_gcm_algs_aesni, - ARRAY_SIZE(aes_gcm_algs_aesni), - aes_gcm_simdalgs_aesni); + crypto_unregister_aeads(aes_gcm_algs_aesni, + ARRAY_SIZE(aes_gcm_algs_aesni)); unregister_skciphers: - simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), - aesni_simd_skciphers); + crypto_unregister_skciphers(aesni_skciphers, + ARRAY_SIZE(aesni_skciphers)); unregister_cipher: crypto_unregister_alg(&aesni_cipher_alg); return err; @@ -1723,11 +1674,10 @@ unregister_cipher: static void __exit aesni_exit(void) { - simd_unregister_aeads(aes_gcm_algs_aesni, - ARRAY_SIZE(aes_gcm_algs_aesni), - aes_gcm_simdalgs_aesni); - simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), - aesni_simd_skciphers); + crypto_unregister_aeads(aes_gcm_algs_aesni, + ARRAY_SIZE(aes_gcm_algs_aesni)); + crypto_unregister_skciphers(aesni_skciphers, + ARRAY_SIZE(aesni_skciphers)); crypto_unregister_alg(&aesni_cipher_alg); unregister_avx_algs(); } diff --git a/arch/x86/crypto/aria_aesni_avx2_glue.c b/arch/x86/crypto/aria_aesni_avx2_glue.c index 87a11804fc77..b4bddcd58457 100644 --- a/arch/x86/crypto/aria_aesni_avx2_glue.c +++ b/arch/x86/crypto/aria_aesni_avx2_glue.c @@ -6,7 +6,6 @@ */ #include <crypto/algapi.h> -#include <crypto/internal/simd.h> #include <crypto/aria.h> #include <linux/crypto.h> #include <linux/err.h> @@ -165,10 +164,9 @@ static int aria_avx2_init_tfm(struct crypto_skcipher *tfm) static struct skcipher_alg aria_algs[] = { { - .base.cra_name = "__ecb(aria)", - .base.cra_driver_name = "__ecb-aria-avx2", + .base.cra_name = "ecb(aria)", + .base.cra_driver_name = "ecb-aria-avx2", .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = ARIA_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aria_ctx), .base.cra_module = THIS_MODULE, @@ -178,11 +176,10 @@ static struct skcipher_alg aria_algs[] = { .encrypt = aria_avx2_ecb_encrypt, .decrypt = aria_avx2_ecb_decrypt, }, { - .base.cra_name = "__ctr(aria)", - .base.cra_driver_name = "__ctr-aria-avx2", + .base.cra_name = "ctr(aria)", + .base.cra_driver_name = "ctr-aria-avx2", .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL | - CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE, + .base.cra_flags = CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct aria_ctx), .base.cra_module = THIS_MODULE, @@ -197,8 +194,6 @@ static struct skcipher_alg aria_algs[] = { } }; -static struct simd_skcipher_alg *aria_simd_algs[ARRAY_SIZE(aria_algs)]; - static int __init aria_avx2_init(void) { const char *feature_name; @@ -233,15 +228,12 @@ static int __init aria_avx2_init(void) aria_ops.aria_ctr_crypt_32way = aria_aesni_avx2_ctr_crypt_32way; } - return simd_register_skciphers_compat(aria_algs, - ARRAY_SIZE(aria_algs), - aria_simd_algs); + return crypto_register_skciphers(aria_algs, ARRAY_SIZE(aria_algs)); } static void __exit aria_avx2_exit(void) { - simd_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs), - aria_simd_algs); + crypto_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs)); } module_init(aria_avx2_init); diff --git a/arch/x86/crypto/aria_aesni_avx_glue.c b/arch/x86/crypto/aria_aesni_avx_glue.c index 4e1516b76669..ab9b38d05332 100644 --- a/arch/x86/crypto/aria_aesni_avx_glue.c +++ b/arch/x86/crypto/aria_aesni_avx_glue.c @@ -6,7 +6,6 @@ */ #include <crypto/algapi.h> -#include <crypto/internal/simd.h> #include <crypto/aria.h> #include <linux/crypto.h> #include <linux/err.h> @@ -152,10 +151,9 @@ static int aria_avx_init_tfm(struct crypto_skcipher *tfm) static struct skcipher_alg aria_algs[] = { { - .base.cra_name = "__ecb(aria)", - .base.cra_driver_name = "__ecb-aria-avx", + .base.cra_name = "ecb(aria)", + .base.cra_driver_name = "ecb-aria-avx", .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = ARIA_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aria_ctx), .base.cra_module = THIS_MODULE, @@ -165,10 +163,9 @@ static struct skcipher_alg aria_algs[] = { .encrypt = aria_avx_ecb_encrypt, .decrypt = aria_avx_ecb_decrypt, }, { - .base.cra_name = "__ctr(aria)", - .base.cra_driver_name = "__ctr-aria-avx", + .base.cra_name = "ctr(aria)", + .base.cra_driver_name = "ctr-aria-avx", .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct aria_ctx), .base.cra_module = THIS_MODULE, @@ -184,8 +181,6 @@ static struct skcipher_alg aria_algs[] = { } }; -static struct simd_skcipher_alg *aria_simd_algs[ARRAY_SIZE(aria_algs)]; - static int __init aria_avx_init(void) { const char *feature_name; @@ -213,15 +208,12 @@ static int __init aria_avx_init(void) aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_ctr_crypt_16way; } - return simd_register_skciphers_compat(aria_algs, - ARRAY_SIZE(aria_algs), - aria_simd_algs); + return crypto_register_skciphers(aria_algs, ARRAY_SIZE(aria_algs)); } static void __exit aria_avx_exit(void) { - simd_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs), - aria_simd_algs); + crypto_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs)); } module_init(aria_avx_init); diff --git a/arch/x86/crypto/aria_gfni_avx512_glue.c b/arch/x86/crypto/aria_gfni_avx512_glue.c index f4a2208d2638..363cbf4399cc 100644 --- a/arch/x86/crypto/aria_gfni_avx512_glue.c +++ b/arch/x86/crypto/aria_gfni_avx512_glue.c @@ -6,7 +6,6 @@ */ #include <crypto/algapi.h> -#include <crypto/internal/simd.h> #include <crypto/aria.h> #include <linux/crypto.h> #include <linux/err.h> @@ -165,10 +164,9 @@ static int aria_avx512_init_tfm(struct crypto_skcipher *tfm) static struct skcipher_alg aria_algs[] = { { - .base.cra_name = "__ecb(aria)", - .base.cra_driver_name = "__ecb-aria-avx512", + .base.cra_name = "ecb(aria)", + .base.cra_driver_name = "ecb-aria-avx512", .base.cra_priority = 600, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = ARIA_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aria_ctx), .base.cra_module = THIS_MODULE, @@ -178,11 +176,10 @@ static struct skcipher_alg aria_algs[] = { .encrypt = aria_avx512_ecb_encrypt, .decrypt = aria_avx512_ecb_decrypt, }, { - .base.cra_name = "__ctr(aria)", - .base.cra_driver_name = "__ctr-aria-avx512", + .base.cra_name = "ctr(aria)", + .base.cra_driver_name = "ctr-aria-avx512", .base.cra_priority = 600, - .base.cra_flags = CRYPTO_ALG_INTERNAL | - CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE, + .base.cra_flags = CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct aria_ctx), .base.cra_module = THIS_MODULE, @@ -197,8 +194,6 @@ static struct skcipher_alg aria_algs[] = { } }; -static struct simd_skcipher_alg *aria_simd_algs[ARRAY_SIZE(aria_algs)]; - static int __init aria_avx512_init(void) { const char *feature_name; @@ -229,15 +224,12 @@ static int __init aria_avx512_init(void) aria_ops.aria_decrypt_64way = aria_gfni_avx512_decrypt_64way; aria_ops.aria_ctr_crypt_64way = aria_gfni_avx512_ctr_crypt_64way; - return simd_register_skciphers_compat(aria_algs, - ARRAY_SIZE(aria_algs), - aria_simd_algs); + return crypto_register_skciphers(aria_algs, ARRAY_SIZE(aria_algs)); } static void __exit aria_avx512_exit(void) { - simd_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs), - aria_simd_algs); + crypto_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs)); } module_init(aria_avx512_init); diff --git a/arch/x86/crypto/blake2s-core.S b/arch/x86/crypto/blake2s-core.S index b50b35ff1fdb..ac1c845445a4 100644 --- a/arch/x86/crypto/blake2s-core.S +++ b/arch/x86/crypto/blake2s-core.S @@ -29,7 +29,6 @@ SIGMA: .byte 13, 7, 12, 3, 11, 14, 1, 9, 2, 5, 15, 8, 10, 0, 4, 6 .byte 6, 14, 11, 0, 15, 9, 3, 8, 10, 12, 13, 1, 5, 2, 7, 4 .byte 10, 8, 7, 1, 2, 4, 6, 5, 13, 15, 9, 3, 0, 11, 14, 12 -#ifdef CONFIG_AS_AVX512 .section .rodata.cst64.BLAKE2S_SIGMA2, "aM", @progbits, 640 .align 64 SIGMA2: @@ -43,7 +42,6 @@ SIGMA2: .long 6, 13, 0, 14, 12, 2, 1, 11, 15, 4, 5, 8, 7, 9, 3, 10 .long 15, 5, 4, 13, 10, 7, 3, 11, 12, 2, 0, 6, 9, 8, 1, 14 .long 8, 7, 14, 11, 13, 15, 0, 12, 10, 4, 5, 6, 3, 2, 1, 9 -#endif /* CONFIG_AS_AVX512 */ .text SYM_FUNC_START(blake2s_compress_ssse3) @@ -174,7 +172,6 @@ SYM_FUNC_START(blake2s_compress_ssse3) RET SYM_FUNC_END(blake2s_compress_ssse3) -#ifdef CONFIG_AS_AVX512 SYM_FUNC_START(blake2s_compress_avx512) vmovdqu (%rdi),%xmm0 vmovdqu 0x10(%rdi),%xmm1 @@ -253,4 +250,3 @@ SYM_FUNC_START(blake2s_compress_avx512) vzeroupper RET SYM_FUNC_END(blake2s_compress_avx512) -#endif /* CONFIG_AS_AVX512 */ diff --git a/arch/x86/crypto/blake2s-glue.c b/arch/x86/crypto/blake2s-glue.c index 0313f9673f56..00f84f29cc8c 100644 --- a/arch/x86/crypto/blake2s-glue.c +++ b/arch/x86/crypto/blake2s-glue.c @@ -41,8 +41,7 @@ void blake2s_compress(struct blake2s_state *state, const u8 *block, SZ_4K / BLAKE2S_BLOCK_SIZE); kernel_fpu_begin(); - if (IS_ENABLED(CONFIG_AS_AVX512) && - static_branch_likely(&blake2s_use_avx512)) + if (static_branch_likely(&blake2s_use_avx512)) blake2s_compress_avx512(state, block, blocks, inc); else blake2s_compress_ssse3(state, block, blocks, inc); @@ -59,8 +58,7 @@ static int __init blake2s_mod_init(void) if (boot_cpu_has(X86_FEATURE_SSSE3)) static_branch_enable(&blake2s_use_ssse3); - if (IS_ENABLED(CONFIG_AS_AVX512) && - boot_cpu_has(X86_FEATURE_AVX) && + if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX512F) && boot_cpu_has(X86_FEATURE_AVX512VL) && diff --git a/arch/x86/crypto/camellia_aesni_avx2_glue.c b/arch/x86/crypto/camellia_aesni_avx2_glue.c index e7e4d64e9577..2d2f4e16537c 100644 --- a/arch/x86/crypto/camellia_aesni_avx2_glue.c +++ b/arch/x86/crypto/camellia_aesni_avx2_glue.c @@ -6,7 +6,6 @@ */ #include <crypto/algapi.h> -#include <crypto/internal/simd.h> #include <linux/crypto.h> #include <linux/err.h> #include <linux/module.h> @@ -69,10 +68,9 @@ static int cbc_decrypt(struct skcipher_request *req) static struct skcipher_alg camellia_algs[] = { { - .base.cra_name = "__ecb(camellia)", - .base.cra_driver_name = "__ecb-camellia-aesni-avx2", + .base.cra_name = "ecb(camellia)", + .base.cra_driver_name = "ecb-camellia-aesni-avx2", .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct camellia_ctx), .base.cra_module = THIS_MODULE, @@ -82,10 +80,9 @@ static struct skcipher_alg camellia_algs[] = { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(camellia)", - .base.cra_driver_name = "__cbc-camellia-aesni-avx2", + .base.cra_name = "cbc(camellia)", + .base.cra_driver_name = "cbc-camellia-aesni-avx2", .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct camellia_ctx), .base.cra_module = THIS_MODULE, @@ -98,8 +95,6 @@ static struct skcipher_alg camellia_algs[] = { }, }; -static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)]; - static int __init camellia_aesni_init(void) { const char *feature_name; @@ -118,15 +113,13 @@ static int __init camellia_aesni_init(void) return -ENODEV; } - return simd_register_skciphers_compat(camellia_algs, - ARRAY_SIZE(camellia_algs), - camellia_simd_algs); + return crypto_register_skciphers(camellia_algs, + ARRAY_SIZE(camellia_algs)); } static void __exit camellia_aesni_fini(void) { - simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs), - camellia_simd_algs); + crypto_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs)); } module_init(camellia_aesni_init); diff --git a/arch/x86/crypto/camellia_aesni_avx_glue.c b/arch/x86/crypto/camellia_aesni_avx_glue.c index c7ccf63e741e..a7d162388142 100644 --- a/arch/x86/crypto/camellia_aesni_avx_glue.c +++ b/arch/x86/crypto/camellia_aesni_avx_glue.c @@ -6,7 +6,6 @@ */ #include <crypto/algapi.h> -#include <crypto/internal/simd.h> #include <linux/crypto.h> #include <linux/err.h> #include <linux/module.h> @@ -69,10 +68,9 @@ static int cbc_decrypt(struct skcipher_request *req) static struct skcipher_alg camellia_algs[] = { { - .base.cra_name = "__ecb(camellia)", - .base.cra_driver_name = "__ecb-camellia-aesni", + .base.cra_name = "ecb(camellia)", + .base.cra_driver_name = "ecb-camellia-aesni", .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct camellia_ctx), .base.cra_module = THIS_MODULE, @@ -82,10 +80,9 @@ static struct skcipher_alg camellia_algs[] = { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(camellia)", - .base.cra_driver_name = "__cbc-camellia-aesni", + .base.cra_name = "cbc(camellia)", + .base.cra_driver_name = "cbc-camellia-aesni", .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct camellia_ctx), .base.cra_module = THIS_MODULE, @@ -98,8 +95,6 @@ static struct skcipher_alg camellia_algs[] = { } }; -static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)]; - static int __init camellia_aesni_init(void) { const char *feature_name; @@ -117,15 +112,13 @@ static int __init camellia_aesni_init(void) return -ENODEV; } - return simd_register_skciphers_compat(camellia_algs, - ARRAY_SIZE(camellia_algs), - camellia_simd_algs); + return crypto_register_skciphers(camellia_algs, + ARRAY_SIZE(camellia_algs)); } static void __exit camellia_aesni_fini(void) { - simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs), - camellia_simd_algs); + crypto_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs)); } module_init(camellia_aesni_init); diff --git a/arch/x86/crypto/cast5_avx_glue.c b/arch/x86/crypto/cast5_avx_glue.c index 3976a87f92ad..3aca04d43b34 100644 --- a/arch/x86/crypto/cast5_avx_glue.c +++ b/arch/x86/crypto/cast5_avx_glue.c @@ -8,7 +8,6 @@ #include <crypto/algapi.h> #include <crypto/cast5.h> -#include <crypto/internal/simd.h> #include <linux/crypto.h> #include <linux/err.h> #include <linux/module.h> @@ -64,10 +63,9 @@ static int cbc_decrypt(struct skcipher_request *req) static struct skcipher_alg cast5_algs[] = { { - .base.cra_name = "__ecb(cast5)", - .base.cra_driver_name = "__ecb-cast5-avx", + .base.cra_name = "ecb(cast5)", + .base.cra_driver_name = "ecb-cast5-avx", .base.cra_priority = 200, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = CAST5_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct cast5_ctx), .base.cra_module = THIS_MODULE, @@ -77,10 +75,9 @@ static struct skcipher_alg cast5_algs[] = { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(cast5)", - .base.cra_driver_name = "__cbc-cast5-avx", + .base.cra_name = "cbc(cast5)", + .base.cra_driver_name = "cbc-cast5-avx", .base.cra_priority = 200, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = CAST5_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct cast5_ctx), .base.cra_module = THIS_MODULE, @@ -93,8 +90,6 @@ static struct skcipher_alg cast5_algs[] = { } }; -static struct simd_skcipher_alg *cast5_simd_algs[ARRAY_SIZE(cast5_algs)]; - static int __init cast5_init(void) { const char *feature_name; @@ -105,15 +100,13 @@ static int __init cast5_init(void) return -ENODEV; } - return simd_register_skciphers_compat(cast5_algs, - ARRAY_SIZE(cast5_algs), - cast5_simd_algs); + return crypto_register_skciphers(cast5_algs, + ARRAY_SIZE(cast5_algs)); } static void __exit cast5_exit(void) { - simd_unregister_skciphers(cast5_algs, ARRAY_SIZE(cast5_algs), - cast5_simd_algs); + crypto_unregister_skciphers(cast5_algs, ARRAY_SIZE(cast5_algs)); } module_init(cast5_init); diff --git a/arch/x86/crypto/cast6_avx_glue.c b/arch/x86/crypto/cast6_avx_glue.c index 7e2aea372349..c4dd28c30303 100644 --- a/arch/x86/crypto/cast6_avx_glue.c +++ b/arch/x86/crypto/cast6_avx_glue.c @@ -14,7 +14,6 @@ #include <linux/err.h> #include <crypto/algapi.h> #include <crypto/cast6.h> -#include <crypto/internal/simd.h> #include "ecb_cbc_helpers.h" @@ -64,10 +63,9 @@ static int cbc_decrypt(struct skcipher_request *req) static struct skcipher_alg cast6_algs[] = { { - .base.cra_name = "__ecb(cast6)", - .base.cra_driver_name = "__ecb-cast6-avx", + .base.cra_name = "ecb(cast6)", + .base.cra_driver_name = "ecb-cast6-avx", .base.cra_priority = 200, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = CAST6_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct cast6_ctx), .base.cra_module = THIS_MODULE, @@ -77,10 +75,9 @@ static struct skcipher_alg cast6_algs[] = { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(cast6)", - .base.cra_driver_name = "__cbc-cast6-avx", + .base.cra_name = "cbc(cast6)", + .base.cra_driver_name = "cbc-cast6-avx", .base.cra_priority = 200, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = CAST6_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct cast6_ctx), .base.cra_module = THIS_MODULE, @@ -93,8 +90,6 @@ static struct skcipher_alg cast6_algs[] = { }, }; -static struct simd_skcipher_alg *cast6_simd_algs[ARRAY_SIZE(cast6_algs)]; - static int __init cast6_init(void) { const char *feature_name; @@ -105,15 +100,12 @@ static int __init cast6_init(void) return -ENODEV; } - return simd_register_skciphers_compat(cast6_algs, - ARRAY_SIZE(cast6_algs), - cast6_simd_algs); + return crypto_register_skciphers(cast6_algs, ARRAY_SIZE(cast6_algs)); } static void __exit cast6_exit(void) { - simd_unregister_skciphers(cast6_algs, ARRAY_SIZE(cast6_algs), - cast6_simd_algs); + crypto_unregister_skciphers(cast6_algs, ARRAY_SIZE(cast6_algs)); } module_init(cast6_init); diff --git a/arch/x86/crypto/chacha_glue.c b/arch/x86/crypto/chacha_glue.c index 8bb74a272879..59bf63c00072 100644 --- a/arch/x86/crypto/chacha_glue.c +++ b/arch/x86/crypto/chacha_glue.c @@ -1,19 +1,16 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * x64 SIMD accelerated ChaCha and XChaCha stream ciphers, - * including ChaCha20 (RFC7539) + * ChaCha and HChaCha functions (x86_64 optimized) * * Copyright (C) 2015 Martin Willi */ -#include <crypto/algapi.h> -#include <crypto/internal/chacha.h> -#include <crypto/internal/simd.h> -#include <crypto/internal/skcipher.h> +#include <asm/simd.h> +#include <crypto/chacha.h> +#include <linux/jump_label.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/sizes.h> -#include <asm/simd.h> asmlinkage void chacha_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src, unsigned int len, int nrounds); @@ -48,8 +45,7 @@ static unsigned int chacha_advance(unsigned int len, unsigned int maxblocks) static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { - if (IS_ENABLED(CONFIG_AS_AVX512) && - static_branch_likely(&chacha_use_avx512vl)) { + if (static_branch_likely(&chacha_use_avx512vl)) { while (bytes >= CHACHA_BLOCK_SIZE * 8) { chacha_8block_xor_avx512vl(state, dst, src, bytes, nrounds); @@ -123,7 +119,7 @@ static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src, void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) { - if (!static_branch_likely(&chacha_use_simd) || !crypto_simd_usable()) { + if (!static_branch_likely(&chacha_use_simd)) { hchacha_block_generic(state, stream, nrounds); } else { kernel_fpu_begin(); @@ -136,7 +132,7 @@ EXPORT_SYMBOL(hchacha_block_arch); void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { - if (!static_branch_likely(&chacha_use_simd) || !crypto_simd_usable() || + if (!static_branch_likely(&chacha_use_simd) || bytes <= CHACHA_BLOCK_SIZE) return chacha_crypt_generic(state, dst, src, bytes, nrounds); @@ -154,121 +150,11 @@ void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, } EXPORT_SYMBOL(chacha_crypt_arch); -static int chacha_simd_stream_xor(struct skcipher_request *req, - const struct chacha_ctx *ctx, const u8 *iv) +bool chacha_is_arch_optimized(void) { - u32 state[CHACHA_STATE_WORDS] __aligned(8); - struct skcipher_walk walk; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - chacha_init(state, ctx->key, iv); - - while (walk.nbytes > 0) { - unsigned int nbytes = walk.nbytes; - - if (nbytes < walk.total) - nbytes = round_down(nbytes, walk.stride); - - if (!static_branch_likely(&chacha_use_simd) || - !crypto_simd_usable()) { - chacha_crypt_generic(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, - ctx->nrounds); - } else { - kernel_fpu_begin(); - chacha_dosimd(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, - ctx->nrounds); - kernel_fpu_end(); - } - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } - - return err; + return static_key_enabled(&chacha_use_simd); } - -static int chacha_simd(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - - return chacha_simd_stream_xor(req, ctx, req->iv); -} - -static int xchacha_simd(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - u32 state[CHACHA_STATE_WORDS] __aligned(8); - struct chacha_ctx subctx; - u8 real_iv[16]; - - chacha_init(state, ctx->key, req->iv); - - if (req->cryptlen > CHACHA_BLOCK_SIZE && crypto_simd_usable()) { - kernel_fpu_begin(); - hchacha_block_ssse3(state, subctx.key, ctx->nrounds); - kernel_fpu_end(); - } else { - hchacha_block_generic(state, subctx.key, ctx->nrounds); - } - subctx.nrounds = ctx->nrounds; - - memcpy(&real_iv[0], req->iv + 24, 8); - memcpy(&real_iv[8], req->iv + 16, 8); - return chacha_simd_stream_xor(req, &subctx, real_iv); -} - -static struct skcipher_alg algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-simd", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = chacha_simd, - .decrypt = chacha_simd, - }, { - .base.cra_name = "xchacha20", - .base.cra_driver_name = "xchacha20-simd", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = xchacha_simd, - .decrypt = xchacha_simd, - }, { - .base.cra_name = "xchacha12", - .base.cra_driver_name = "xchacha12-simd", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha12_setkey, - .encrypt = xchacha_simd, - .decrypt = xchacha_simd, - }, -}; +EXPORT_SYMBOL(chacha_is_arch_optimized); static int __init chacha_simd_mod_init(void) { @@ -282,30 +168,19 @@ static int __init chacha_simd_mod_init(void) cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { static_branch_enable(&chacha_use_avx2); - if (IS_ENABLED(CONFIG_AS_AVX512) && - boot_cpu_has(X86_FEATURE_AVX512VL) && + if (boot_cpu_has(X86_FEATURE_AVX512VL) && boot_cpu_has(X86_FEATURE_AVX512BW)) /* kmovq */ static_branch_enable(&chacha_use_avx512vl); } - return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ? - crypto_register_skciphers(algs, ARRAY_SIZE(algs)) : 0; + return 0; } +arch_initcall(chacha_simd_mod_init); -static void __exit chacha_simd_mod_fini(void) +static void __exit chacha_simd_mod_exit(void) { - if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) && boot_cpu_has(X86_FEATURE_SSSE3)) - crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); } - -module_init(chacha_simd_mod_init); -module_exit(chacha_simd_mod_fini); +module_exit(chacha_simd_mod_exit); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Martin Willi <martin@strongswan.org>"); -MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (x64 SIMD accelerated)"); -MODULE_ALIAS_CRYPTO("chacha20"); -MODULE_ALIAS_CRYPTO("chacha20-simd"); -MODULE_ALIAS_CRYPTO("xchacha20"); -MODULE_ALIAS_CRYPTO("xchacha20-simd"); -MODULE_ALIAS_CRYPTO("xchacha12"); -MODULE_ALIAS_CRYPTO("xchacha12-simd"); +MODULE_DESCRIPTION("ChaCha and HChaCha functions (x86_64 optimized)"); diff --git a/arch/x86/crypto/ghash-clmulni-intel_asm.S b/arch/x86/crypto/ghash-clmulni-intel_asm.S index 99cb983ded9e..c4fbaa82ed7a 100644 --- a/arch/x86/crypto/ghash-clmulni-intel_asm.S +++ b/arch/x86/crypto/ghash-clmulni-intel_asm.S @@ -103,8 +103,8 @@ SYM_FUNC_START(clmul_ghash_mul) SYM_FUNC_END(clmul_ghash_mul) /* - * void clmul_ghash_update(char *dst, const char *src, unsigned int srclen, - * const le128 *shash); + * int clmul_ghash_update(char *dst, const char *src, unsigned int srclen, + * const le128 *shash); */ SYM_FUNC_START(clmul_ghash_update) FRAME_BEGIN @@ -127,6 +127,7 @@ SYM_FUNC_START(clmul_ghash_update) pshufb BSWAP, DATA movups DATA, (%rdi) .Lupdate_just_ret: + mov %rdx, %rax FRAME_END RET SYM_FUNC_END(clmul_ghash_update) diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c index c759ec808bf1..aea5d4d06be7 100644 --- a/arch/x86/crypto/ghash-clmulni-intel_glue.c +++ b/arch/x86/crypto/ghash-clmulni-intel_glue.c @@ -7,41 +7,27 @@ * Author: Huang Ying <ying.huang@intel.com> */ -#include <linux/err.h> -#include <linux/module.h> -#include <linux/init.h> -#include <linux/kernel.h> -#include <linux/crypto.h> -#include <crypto/algapi.h> -#include <crypto/cryptd.h> -#include <crypto/gf128mul.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> #include <asm/cpu_device_id.h> #include <asm/simd.h> +#include <crypto/b128ops.h> +#include <crypto/ghash.h> +#include <crypto/internal/hash.h> +#include <crypto/utils.h> +#include <linux/errno.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> #include <linux/unaligned.h> -#define GHASH_BLOCK_SIZE 16 -#define GHASH_DIGEST_SIZE 16 +asmlinkage void clmul_ghash_mul(char *dst, const le128 *shash); -void clmul_ghash_mul(char *dst, const le128 *shash); +asmlinkage int clmul_ghash_update(char *dst, const char *src, + unsigned int srclen, const le128 *shash); -void clmul_ghash_update(char *dst, const char *src, unsigned int srclen, - const le128 *shash); - -struct ghash_async_ctx { - struct cryptd_ahash *cryptd_tfm; -}; - -struct ghash_ctx { +struct x86_ghash_ctx { le128 shash; }; -struct ghash_desc_ctx { - u8 buffer[GHASH_BLOCK_SIZE]; - u32 bytes; -}; - static int ghash_init(struct shash_desc *desc) { struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); @@ -54,7 +40,7 @@ static int ghash_init(struct shash_desc *desc) static int ghash_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) { - struct ghash_ctx *ctx = crypto_shash_ctx(tfm); + struct x86_ghash_ctx *ctx = crypto_shash_ctx(tfm); u64 a, b; if (keylen != GHASH_BLOCK_SIZE) @@ -95,64 +81,38 @@ static int ghash_setkey(struct crypto_shash *tfm, static int ghash_update(struct shash_desc *desc, const u8 *src, unsigned int srclen) { + struct x86_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); u8 *dst = dctx->buffer; + int remain; kernel_fpu_begin(); - if (dctx->bytes) { - int n = min(srclen, dctx->bytes); - u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes); - - dctx->bytes -= n; - srclen -= n; - - while (n--) - *pos++ ^= *src++; - - if (!dctx->bytes) - clmul_ghash_mul(dst, &ctx->shash); - } - - clmul_ghash_update(dst, src, srclen, &ctx->shash); + remain = clmul_ghash_update(dst, src, srclen, &ctx->shash); kernel_fpu_end(); - - if (srclen & 0xf) { - src += srclen - (srclen & 0xf); - srclen &= 0xf; - dctx->bytes = GHASH_BLOCK_SIZE - srclen; - while (srclen--) - *dst++ ^= *src++; - } - - return 0; + return remain; } -static void ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx) +static void ghash_flush(struct x86_ghash_ctx *ctx, struct ghash_desc_ctx *dctx, + const u8 *src, unsigned int len) { u8 *dst = dctx->buffer; - if (dctx->bytes) { - u8 *tmp = dst + (GHASH_BLOCK_SIZE - dctx->bytes); - - while (dctx->bytes--) - *tmp++ ^= 0; - - kernel_fpu_begin(); + kernel_fpu_begin(); + if (len) { + crypto_xor(dst, src, len); clmul_ghash_mul(dst, &ctx->shash); - kernel_fpu_end(); } - - dctx->bytes = 0; + kernel_fpu_end(); } -static int ghash_final(struct shash_desc *desc, u8 *dst) +static int ghash_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *dst) { + struct x86_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); u8 *buf = dctx->buffer; - ghash_flush(ctx, dctx); + ghash_flush(ctx, dctx, src, len); memcpy(dst, buf, GHASH_BLOCK_SIZE); return 0; @@ -162,186 +122,20 @@ static struct shash_alg ghash_alg = { .digestsize = GHASH_DIGEST_SIZE, .init = ghash_init, .update = ghash_update, - .final = ghash_final, + .finup = ghash_finup, .setkey = ghash_setkey, .descsize = sizeof(struct ghash_desc_ctx), .base = { - .cra_name = "__ghash", - .cra_driver_name = "__ghash-pclmulqdqni", - .cra_priority = 0, - .cra_flags = CRYPTO_ALG_INTERNAL, + .cra_name = "ghash", + .cra_driver_name = "ghash-pclmulqdqni", + .cra_priority = 400, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ghash_ctx), + .cra_ctxsize = sizeof(struct x86_ghash_ctx), .cra_module = THIS_MODULE, }, }; -static int ghash_async_init(struct ahash_request *req) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); - struct ahash_request *cryptd_req = ahash_request_ctx(req); - struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - struct shash_desc *desc = cryptd_shash_desc(cryptd_req); - struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); - - desc->tfm = child; - return crypto_shash_init(desc); -} - -static void ghash_init_cryptd_req(struct ahash_request *req) -{ - struct ahash_request *cryptd_req = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); - struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - - ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); - ahash_request_set_callback(cryptd_req, req->base.flags, - req->base.complete, req->base.data); - ahash_request_set_crypt(cryptd_req, req->src, req->result, - req->nbytes); -} - -static int ghash_async_update(struct ahash_request *req) -{ - struct ahash_request *cryptd_req = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); - struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - - if (!crypto_simd_usable() || - (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { - ghash_init_cryptd_req(req); - return crypto_ahash_update(cryptd_req); - } else { - struct shash_desc *desc = cryptd_shash_desc(cryptd_req); - return shash_ahash_update(req, desc); - } -} - -static int ghash_async_final(struct ahash_request *req) -{ - struct ahash_request *cryptd_req = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); - struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - - if (!crypto_simd_usable() || - (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { - ghash_init_cryptd_req(req); - return crypto_ahash_final(cryptd_req); - } else { - struct shash_desc *desc = cryptd_shash_desc(cryptd_req); - return crypto_shash_final(desc, req->result); - } -} - -static int ghash_async_import(struct ahash_request *req, const void *in) -{ - struct ahash_request *cryptd_req = ahash_request_ctx(req); - struct shash_desc *desc = cryptd_shash_desc(cryptd_req); - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - ghash_async_init(req); - memcpy(dctx, in, sizeof(*dctx)); - return 0; - -} - -static int ghash_async_export(struct ahash_request *req, void *out) -{ - struct ahash_request *cryptd_req = ahash_request_ctx(req); - struct shash_desc *desc = cryptd_shash_desc(cryptd_req); - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memcpy(out, dctx, sizeof(*dctx)); - return 0; - -} - -static int ghash_async_digest(struct ahash_request *req) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); - struct ahash_request *cryptd_req = ahash_request_ctx(req); - struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - - if (!crypto_simd_usable() || - (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { - ghash_init_cryptd_req(req); - return crypto_ahash_digest(cryptd_req); - } else { - struct shash_desc *desc = cryptd_shash_desc(cryptd_req); - struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); - - desc->tfm = child; - return shash_ahash_digest(req, desc); - } -} - -static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key, - unsigned int keylen) -{ - struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); - struct crypto_ahash *child = &ctx->cryptd_tfm->base; - - crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK); - crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm) - & CRYPTO_TFM_REQ_MASK); - return crypto_ahash_setkey(child, key, keylen); -} - -static int ghash_async_init_tfm(struct crypto_tfm *tfm) -{ - struct cryptd_ahash *cryptd_tfm; - struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); - - cryptd_tfm = cryptd_alloc_ahash("__ghash-pclmulqdqni", - CRYPTO_ALG_INTERNAL, - CRYPTO_ALG_INTERNAL); - if (IS_ERR(cryptd_tfm)) - return PTR_ERR(cryptd_tfm); - ctx->cryptd_tfm = cryptd_tfm; - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), - sizeof(struct ahash_request) + - crypto_ahash_reqsize(&cryptd_tfm->base)); - - return 0; -} - -static void ghash_async_exit_tfm(struct crypto_tfm *tfm) -{ - struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); - - cryptd_free_ahash(ctx->cryptd_tfm); -} - -static struct ahash_alg ghash_async_alg = { - .init = ghash_async_init, - .update = ghash_async_update, - .final = ghash_async_final, - .setkey = ghash_async_setkey, - .digest = ghash_async_digest, - .export = ghash_async_export, - .import = ghash_async_import, - .halg = { - .digestsize = GHASH_DIGEST_SIZE, - .statesize = sizeof(struct ghash_desc_ctx), - .base = { - .cra_name = "ghash", - .cra_driver_name = "ghash-clmulni", - .cra_priority = 400, - .cra_ctxsize = sizeof(struct ghash_async_ctx), - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_module = THIS_MODULE, - .cra_init = ghash_async_init_tfm, - .cra_exit = ghash_async_exit_tfm, - }, - }, -}; - static const struct x86_cpu_id pcmul_cpu_id[] = { X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL), /* Pickle-Mickle-Duck */ {} @@ -350,29 +144,14 @@ MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id); static int __init ghash_pclmulqdqni_mod_init(void) { - int err; - if (!x86_match_cpu(pcmul_cpu_id)) return -ENODEV; - err = crypto_register_shash(&ghash_alg); - if (err) - goto err_out; - err = crypto_register_ahash(&ghash_async_alg); - if (err) - goto err_shash; - - return 0; - -err_shash: - crypto_unregister_shash(&ghash_alg); -err_out: - return err; + return crypto_register_shash(&ghash_alg); } static void __exit ghash_pclmulqdqni_mod_exit(void) { - crypto_unregister_ahash(&ghash_async_alg); crypto_unregister_shash(&ghash_alg); } diff --git a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl index b9abcd79c1f4..409ec6955733 100644 --- a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl +++ b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl @@ -2811,18 +2811,10 @@ if ($avx>2) { # reason stack layout is kept identical to poly1305_blocks_avx2. If not # for this tail, we wouldn't have to even allocate stack frame... -if($kernel) { - $code .= "#ifdef CONFIG_AS_AVX512\n"; -} - &declare_function("poly1305_blocks_avx512", 32, 4); poly1305_blocks_avxN(1); &end_function("poly1305_blocks_avx512"); -if ($kernel) { - $code .= "#endif\n"; -} - if (!$kernel && $avx>3) { ######################################################################## # VPMADD52 version using 2^44 radix. diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index 08ff4b489f7e..cff35ca5822a 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -3,15 +3,13 @@ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. */ -#include <crypto/algapi.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/poly1305.h> #include <crypto/internal/simd.h> -#include <linux/crypto.h> +#include <crypto/poly1305.h> #include <linux/jump_label.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/sizes.h> +#include <linux/unaligned.h> #include <asm/cpu_device_id.h> #include <asm/simd.h> @@ -107,7 +105,7 @@ static void poly1305_simd_blocks(void *ctx, const u8 *inp, size_t len, const size_t bytes = min_t(size_t, len, SZ_4K); kernel_fpu_begin(); - if (IS_ENABLED(CONFIG_AS_AVX512) && static_branch_likely(&poly1305_use_avx512)) + if (static_branch_likely(&poly1305_use_avx512)) poly1305_blocks_avx512(ctx, inp, bytes, padbit); else if (static_branch_likely(&poly1305_use_avx2)) poly1305_blocks_avx2(ctx, inp, bytes, padbit); @@ -137,38 +135,13 @@ void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KE dctx->s[2] = get_unaligned_le32(&key[24]); dctx->s[3] = get_unaligned_le32(&key[28]); dctx->buflen = 0; - dctx->sset = true; } EXPORT_SYMBOL(poly1305_init_arch); -static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx, - const u8 *inp, unsigned int len) -{ - unsigned int acc = 0; - if (unlikely(!dctx->sset)) { - if (!dctx->rset && len >= POLY1305_BLOCK_SIZE) { - poly1305_simd_init(&dctx->h, inp); - inp += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - acc += POLY1305_BLOCK_SIZE; - dctx->rset = 1; - } - if (len >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(&inp[0]); - dctx->s[1] = get_unaligned_le32(&inp[4]); - dctx->s[2] = get_unaligned_le32(&inp[8]); - dctx->s[3] = get_unaligned_le32(&inp[12]); - acc += POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - } - return acc; -} - void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int srclen) { - unsigned int bytes, used; + unsigned int bytes; if (unlikely(dctx->buflen)) { bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); @@ -178,19 +151,16 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, dctx->buflen += bytes; if (dctx->buflen == POLY1305_BLOCK_SIZE) { - if (likely(!crypto_poly1305_setdctxkey(dctx, dctx->buf, POLY1305_BLOCK_SIZE))) - poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1); + poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1); dctx->buflen = 0; } } if (likely(srclen >= POLY1305_BLOCK_SIZE)) { bytes = round_down(srclen, POLY1305_BLOCK_SIZE); - srclen -= bytes; - used = crypto_poly1305_setdctxkey(dctx, src, bytes); - if (likely(bytes - used)) - poly1305_simd_blocks(&dctx->h, src + used, bytes - used, 1); + poly1305_simd_blocks(&dctx->h, src, bytes, 1); src += bytes; + srclen -= bytes; } if (unlikely(srclen)) { @@ -214,48 +184,11 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) } EXPORT_SYMBOL(poly1305_final_arch); -static int crypto_poly1305_init(struct shash_desc *desc) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - *dctx = (struct poly1305_desc_ctx){}; - return 0; -} - -static int crypto_poly1305_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) +bool poly1305_is_arch_optimized(void) { - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - poly1305_update_arch(dctx, src, srclen); - return 0; + return static_key_enabled(&poly1305_use_avx); } - -static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (unlikely(!dctx->sset)) - return -ENOKEY; - - poly1305_final_arch(dctx, dst); - return 0; -} - -static struct shash_alg alg = { - .digestsize = POLY1305_DIGEST_SIZE, - .init = crypto_poly1305_init, - .update = crypto_poly1305_update, - .final = crypto_poly1305_final, - .descsize = sizeof(struct poly1305_desc_ctx), - .base = { - .cra_name = "poly1305", - .cra_driver_name = "poly1305-simd", - .cra_priority = 300, - .cra_blocksize = POLY1305_BLOCK_SIZE, - .cra_module = THIS_MODULE, - }, -}; +EXPORT_SYMBOL(poly1305_is_arch_optimized); static int __init poly1305_simd_mod_init(void) { @@ -265,26 +198,21 @@ static int __init poly1305_simd_mod_init(void) if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) static_branch_enable(&poly1305_use_avx2); - if (IS_ENABLED(CONFIG_AS_AVX512) && boot_cpu_has(X86_FEATURE_AVX) && - boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX512F) && + if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX512F) && cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | XFEATURE_MASK_AVX512, NULL) && /* Skylake downclocks unacceptably much when using zmm, but later generations are fast. */ boot_cpu_data.x86_vfm != INTEL_SKYLAKE_X) static_branch_enable(&poly1305_use_avx512); - return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? crypto_register_shash(&alg) : 0; + return 0; } +arch_initcall(poly1305_simd_mod_init); static void __exit poly1305_simd_mod_exit(void) { - if (IS_REACHABLE(CONFIG_CRYPTO_HASH)) - crypto_unregister_shash(&alg); } - -module_init(poly1305_simd_mod_init); module_exit(poly1305_simd_mod_exit); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); MODULE_DESCRIPTION("Poly1305 authenticator"); -MODULE_ALIAS_CRYPTO("poly1305"); -MODULE_ALIAS_CRYPTO("poly1305-simd"); diff --git a/arch/x86/crypto/serpent_avx2_glue.c b/arch/x86/crypto/serpent_avx2_glue.c index 347e97f4b713..f5f2121b7956 100644 --- a/arch/x86/crypto/serpent_avx2_glue.c +++ b/arch/x86/crypto/serpent_avx2_glue.c @@ -10,7 +10,6 @@ #include <linux/crypto.h> #include <linux/err.h> #include <crypto/algapi.h> -#include <crypto/internal/simd.h> #include <crypto/serpent.h> #include "serpent-avx.h" @@ -65,10 +64,9 @@ static int cbc_decrypt(struct skcipher_request *req) static struct skcipher_alg serpent_algs[] = { { - .base.cra_name = "__ecb(serpent)", - .base.cra_driver_name = "__ecb-serpent-avx2", + .base.cra_name = "ecb(serpent)", + .base.cra_driver_name = "ecb-serpent-avx2", .base.cra_priority = 600, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = SERPENT_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct serpent_ctx), .base.cra_module = THIS_MODULE, @@ -78,10 +76,9 @@ static struct skcipher_alg serpent_algs[] = { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(serpent)", - .base.cra_driver_name = "__cbc-serpent-avx2", + .base.cra_name = "cbc(serpent)", + .base.cra_driver_name = "cbc-serpent-avx2", .base.cra_priority = 600, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = SERPENT_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct serpent_ctx), .base.cra_module = THIS_MODULE, @@ -94,8 +91,6 @@ static struct skcipher_alg serpent_algs[] = { }, }; -static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)]; - static int __init serpent_avx2_init(void) { const char *feature_name; @@ -110,15 +105,13 @@ static int __init serpent_avx2_init(void) return -ENODEV; } - return simd_register_skciphers_compat(serpent_algs, - ARRAY_SIZE(serpent_algs), - serpent_simd_algs); + return crypto_register_skciphers(serpent_algs, + ARRAY_SIZE(serpent_algs)); } static void __exit serpent_avx2_fini(void) { - simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs), - serpent_simd_algs); + crypto_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs)); } module_init(serpent_avx2_init); diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c index 6c248e1ea4ef..e640abc1cb8a 100644 --- a/arch/x86/crypto/serpent_avx_glue.c +++ b/arch/x86/crypto/serpent_avx_glue.c @@ -13,7 +13,6 @@ #include <linux/crypto.h> #include <linux/err.h> #include <crypto/algapi.h> -#include <crypto/internal/simd.h> #include <crypto/serpent.h> #include "serpent-avx.h" @@ -71,10 +70,9 @@ static int cbc_decrypt(struct skcipher_request *req) static struct skcipher_alg serpent_algs[] = { { - .base.cra_name = "__ecb(serpent)", - .base.cra_driver_name = "__ecb-serpent-avx", + .base.cra_name = "ecb(serpent)", + .base.cra_driver_name = "ecb-serpent-avx", .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = SERPENT_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct serpent_ctx), .base.cra_module = THIS_MODULE, @@ -84,10 +82,9 @@ static struct skcipher_alg serpent_algs[] = { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(serpent)", - .base.cra_driver_name = "__cbc-serpent-avx", + .base.cra_name = "cbc(serpent)", + .base.cra_driver_name = "cbc-serpent-avx", .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = SERPENT_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct serpent_ctx), .base.cra_module = THIS_MODULE, @@ -100,8 +97,6 @@ static struct skcipher_alg serpent_algs[] = { }, }; -static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)]; - static int __init serpent_init(void) { const char *feature_name; @@ -112,15 +107,13 @@ static int __init serpent_init(void) return -ENODEV; } - return simd_register_skciphers_compat(serpent_algs, - ARRAY_SIZE(serpent_algs), - serpent_simd_algs); + return crypto_register_skciphers(serpent_algs, + ARRAY_SIZE(serpent_algs)); } static void __exit serpent_exit(void) { - simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs), - serpent_simd_algs); + crypto_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs)); } module_init(serpent_init); diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c index d78f37e9b2cf..80ee17ec21b4 100644 --- a/arch/x86/crypto/serpent_sse2_glue.c +++ b/arch/x86/crypto/serpent_sse2_glue.c @@ -18,7 +18,6 @@ #include <linux/err.h> #include <crypto/algapi.h> #include <crypto/b128ops.h> -#include <crypto/internal/simd.h> #include <crypto/serpent.h> #include "serpent-sse2.h" @@ -74,10 +73,9 @@ static int cbc_decrypt(struct skcipher_request *req) static struct skcipher_alg serpent_algs[] = { { - .base.cra_name = "__ecb(serpent)", - .base.cra_driver_name = "__ecb-serpent-sse2", + .base.cra_name = "ecb(serpent)", + .base.cra_driver_name = "ecb-serpent-sse2", .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = SERPENT_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct serpent_ctx), .base.cra_module = THIS_MODULE, @@ -87,10 +85,9 @@ static struct skcipher_alg serpent_algs[] = { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(serpent)", - .base.cra_driver_name = "__cbc-serpent-sse2", + .base.cra_name = "cbc(serpent)", + .base.cra_driver_name = "cbc-serpent-sse2", .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = SERPENT_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct serpent_ctx), .base.cra_module = THIS_MODULE, @@ -103,8 +100,6 @@ static struct skcipher_alg serpent_algs[] = { }, }; -static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)]; - static int __init serpent_sse2_init(void) { if (!boot_cpu_has(X86_FEATURE_XMM2)) { @@ -112,15 +107,13 @@ static int __init serpent_sse2_init(void) return -ENODEV; } - return simd_register_skciphers_compat(serpent_algs, - ARRAY_SIZE(serpent_algs), - serpent_simd_algs); + return crypto_register_skciphers(serpent_algs, + ARRAY_SIZE(serpent_algs)); } static void __exit serpent_sse2_exit(void) { - simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs), - serpent_simd_algs); + crypto_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs)); } module_init(serpent_sse2_init); diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c index ab8bc54f254d..0a912bfc86c5 100644 --- a/arch/x86/crypto/sha1_ssse3_glue.c +++ b/arch/x86/crypto/sha1_ssse3_glue.c @@ -16,21 +16,17 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include <asm/cpu_device_id.h> +#include <asm/simd.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> #include <crypto/sha1.h> #include <crypto/sha1_base.h> -#include <asm/cpu_device_id.h> -#include <asm/simd.h> +#include <linux/errno.h> +#include <linux/kernel.h> +#include <linux/module.h> static const struct x86_cpu_id module_cpu_ids[] = { -#ifdef CONFIG_AS_SHA1_NI X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL), -#endif X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL), X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL), X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL), @@ -38,14 +34,10 @@ static const struct x86_cpu_id module_cpu_ids[] = { }; MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids); -static int sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len, sha1_block_fn *sha1_xform) +static inline int sha1_update(struct shash_desc *desc, const u8 *data, + unsigned int len, sha1_block_fn *sha1_xform) { - struct sha1_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) - return crypto_sha1_update(desc, data, len); + int remain; /* * Make sure struct sha1_state begins directly with the SHA1 @@ -54,22 +46,18 @@ static int sha1_update(struct shash_desc *desc, const u8 *data, BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0); kernel_fpu_begin(); - sha1_base_do_update(desc, data, len, sha1_xform); + remain = sha1_base_do_update_blocks(desc, data, len, sha1_xform); kernel_fpu_end(); - return 0; + return remain; } -static int sha1_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out, sha1_block_fn *sha1_xform) +static inline int sha1_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out, + sha1_block_fn *sha1_xform) { - if (!crypto_simd_usable()) - return crypto_sha1_finup(desc, data, len, out); - kernel_fpu_begin(); - if (len) - sha1_base_do_update(desc, data, len, sha1_xform); - sha1_base_do_finalize(desc, sha1_xform); + sha1_base_do_finup(desc, data, len, sha1_xform); kernel_fpu_end(); return sha1_base_finish(desc, out); @@ -90,23 +78,17 @@ static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data, return sha1_finup(desc, data, len, out, sha1_transform_ssse3); } -/* Add padding and return the message digest. */ -static int sha1_ssse3_final(struct shash_desc *desc, u8 *out) -{ - return sha1_ssse3_finup(desc, NULL, 0, out); -} - static struct shash_alg sha1_ssse3_alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = sha1_ssse3_update, - .final = sha1_ssse3_final, .finup = sha1_ssse3_finup, - .descsize = sizeof(struct sha1_state), + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-ssse3", .cra_priority = 150, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -140,22 +122,17 @@ static int sha1_avx_finup(struct shash_desc *desc, const u8 *data, return sha1_finup(desc, data, len, out, sha1_transform_avx); } -static int sha1_avx_final(struct shash_desc *desc, u8 *out) -{ - return sha1_avx_finup(desc, NULL, 0, out); -} - static struct shash_alg sha1_avx_alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = sha1_avx_update, - .final = sha1_avx_final, .finup = sha1_avx_finup, - .descsize = sizeof(struct sha1_state), + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-avx", .cra_priority = 160, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -200,8 +177,8 @@ static bool avx2_usable(void) return false; } -static void sha1_apply_transform_avx2(struct sha1_state *state, - const u8 *data, int blocks) +static inline void sha1_apply_transform_avx2(struct sha1_state *state, + const u8 *data, int blocks) { /* Select the optimal transform based on data block size */ if (blocks >= SHA1_AVX2_BLOCK_OPTSIZE) @@ -222,22 +199,17 @@ static int sha1_avx2_finup(struct shash_desc *desc, const u8 *data, return sha1_finup(desc, data, len, out, sha1_apply_transform_avx2); } -static int sha1_avx2_final(struct shash_desc *desc, u8 *out) -{ - return sha1_avx2_finup(desc, NULL, 0, out); -} - static struct shash_alg sha1_avx2_alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = sha1_avx2_update, - .final = sha1_avx2_final, .finup = sha1_avx2_finup, - .descsize = sizeof(struct sha1_state), + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-avx2", .cra_priority = 170, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -256,7 +228,6 @@ static void unregister_sha1_avx2(void) crypto_unregister_shash(&sha1_avx2_alg); } -#ifdef CONFIG_AS_SHA1_NI asmlinkage void sha1_ni_transform(struct sha1_state *digest, const u8 *data, int rounds); @@ -272,22 +243,17 @@ static int sha1_ni_finup(struct shash_desc *desc, const u8 *data, return sha1_finup(desc, data, len, out, sha1_ni_transform); } -static int sha1_ni_final(struct shash_desc *desc, u8 *out) -{ - return sha1_ni_finup(desc, NULL, 0, out); -} - static struct shash_alg sha1_ni_alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = sha1_ni_update, - .final = sha1_ni_final, .finup = sha1_ni_finup, - .descsize = sizeof(struct sha1_state), + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-ni", .cra_priority = 250, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -306,11 +272,6 @@ static void unregister_sha1_ni(void) crypto_unregister_shash(&sha1_ni_alg); } -#else -static inline int register_sha1_ni(void) { return 0; } -static inline void unregister_sha1_ni(void) { } -#endif - static int __init sha1_ssse3_mod_init(void) { if (!x86_match_cpu(module_cpu_ids)) @@ -360,6 +321,4 @@ MODULE_ALIAS_CRYPTO("sha1"); MODULE_ALIAS_CRYPTO("sha1-ssse3"); MODULE_ALIAS_CRYPTO("sha1-avx"); MODULE_ALIAS_CRYPTO("sha1-avx2"); -#ifdef CONFIG_AS_SHA1_NI MODULE_ALIAS_CRYPTO("sha1-ni"); -#endif diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c index e04a43d9f7d5..a5d3be00550b 100644 --- a/arch/x86/crypto/sha256_ssse3_glue.c +++ b/arch/x86/crypto/sha256_ssse3_glue.c @@ -29,25 +29,19 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include <asm/cpu_device_id.h> +#include <asm/fpu/api.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> #include <crypto/sha2.h> #include <crypto/sha256_base.h> -#include <linux/string.h> -#include <asm/cpu_device_id.h> -#include <asm/simd.h> +#include <linux/kernel.h> +#include <linux/module.h> -asmlinkage void sha256_transform_ssse3(struct sha256_state *state, +asmlinkage void sha256_transform_ssse3(struct crypto_sha256_state *state, const u8 *data, int blocks); static const struct x86_cpu_id module_cpu_ids[] = { -#ifdef CONFIG_AS_SHA256_NI X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL), -#endif X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL), X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL), X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL), @@ -56,37 +50,29 @@ static const struct x86_cpu_id module_cpu_ids[] = { MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids); static int _sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len, sha256_block_fn *sha256_xform) + unsigned int len, + sha256_block_fn *sha256_xform) { - struct sha256_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) - return crypto_sha256_update(desc, data, len); + int remain; /* - * Make sure struct sha256_state begins directly with the SHA256 + * Make sure struct crypto_sha256_state begins directly with the SHA256 * 256-bit internal state, as this is what the asm functions expect. */ - BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0); + BUILD_BUG_ON(offsetof(struct crypto_sha256_state, state) != 0); kernel_fpu_begin(); - sha256_base_do_update(desc, data, len, sha256_xform); + remain = sha256_base_do_update_blocks(desc, data, len, sha256_xform); kernel_fpu_end(); - return 0; + return remain; } static int sha256_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out, sha256_block_fn *sha256_xform) { - if (!crypto_simd_usable()) - return crypto_sha256_finup(desc, data, len, out); - kernel_fpu_begin(); - if (len) - sha256_base_do_update(desc, data, len, sha256_xform); - sha256_base_do_finalize(desc, sha256_xform); + sha256_base_do_finup(desc, data, len, sha256_xform); kernel_fpu_end(); return sha256_base_finish(desc, out); @@ -104,12 +90,6 @@ static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data, return sha256_finup(desc, data, len, out, sha256_transform_ssse3); } -/* Add padding and return the message digest. */ -static int sha256_ssse3_final(struct shash_desc *desc, u8 *out) -{ - return sha256_ssse3_finup(desc, NULL, 0, out); -} - static int sha256_ssse3_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { @@ -121,14 +101,15 @@ static struct shash_alg sha256_ssse3_algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = sha256_ssse3_update, - .final = sha256_ssse3_final, .finup = sha256_ssse3_finup, .digest = sha256_ssse3_digest, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-ssse3", .cra_priority = 150, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -136,13 +117,14 @@ static struct shash_alg sha256_ssse3_algs[] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = sha256_ssse3_update, - .final = sha256_ssse3_final, .finup = sha256_ssse3_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-ssse3", .cra_priority = 150, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -163,7 +145,7 @@ static void unregister_sha256_ssse3(void) ARRAY_SIZE(sha256_ssse3_algs)); } -asmlinkage void sha256_transform_avx(struct sha256_state *state, +asmlinkage void sha256_transform_avx(struct crypto_sha256_state *state, const u8 *data, int blocks); static int sha256_avx_update(struct shash_desc *desc, const u8 *data, @@ -178,11 +160,6 @@ static int sha256_avx_finup(struct shash_desc *desc, const u8 *data, return sha256_finup(desc, data, len, out, sha256_transform_avx); } -static int sha256_avx_final(struct shash_desc *desc, u8 *out) -{ - return sha256_avx_finup(desc, NULL, 0, out); -} - static int sha256_avx_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { @@ -194,14 +171,15 @@ static struct shash_alg sha256_avx_algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = sha256_avx_update, - .final = sha256_avx_final, .finup = sha256_avx_finup, .digest = sha256_avx_digest, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-avx", .cra_priority = 160, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -209,13 +187,14 @@ static struct shash_alg sha256_avx_algs[] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = sha256_avx_update, - .final = sha256_avx_final, .finup = sha256_avx_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-avx", .cra_priority = 160, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -247,7 +226,7 @@ static void unregister_sha256_avx(void) ARRAY_SIZE(sha256_avx_algs)); } -asmlinkage void sha256_transform_rorx(struct sha256_state *state, +asmlinkage void sha256_transform_rorx(struct crypto_sha256_state *state, const u8 *data, int blocks); static int sha256_avx2_update(struct shash_desc *desc, const u8 *data, @@ -262,11 +241,6 @@ static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data, return sha256_finup(desc, data, len, out, sha256_transform_rorx); } -static int sha256_avx2_final(struct shash_desc *desc, u8 *out) -{ - return sha256_avx2_finup(desc, NULL, 0, out); -} - static int sha256_avx2_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { @@ -278,14 +252,15 @@ static struct shash_alg sha256_avx2_algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = sha256_avx2_update, - .final = sha256_avx2_final, .finup = sha256_avx2_finup, .digest = sha256_avx2_digest, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-avx2", .cra_priority = 170, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -293,13 +268,14 @@ static struct shash_alg sha256_avx2_algs[] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = sha256_avx2_update, - .final = sha256_avx2_final, .finup = sha256_avx2_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-avx2", .cra_priority = 170, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -329,8 +305,7 @@ static void unregister_sha256_avx2(void) ARRAY_SIZE(sha256_avx2_algs)); } -#ifdef CONFIG_AS_SHA256_NI -asmlinkage void sha256_ni_transform(struct sha256_state *digest, +asmlinkage void sha256_ni_transform(struct crypto_sha256_state *digest, const u8 *data, int rounds); static int sha256_ni_update(struct shash_desc *desc, const u8 *data, @@ -345,11 +320,6 @@ static int sha256_ni_finup(struct shash_desc *desc, const u8 *data, return sha256_finup(desc, data, len, out, sha256_ni_transform); } -static int sha256_ni_final(struct shash_desc *desc, u8 *out) -{ - return sha256_ni_finup(desc, NULL, 0, out); -} - static int sha256_ni_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { @@ -361,14 +331,15 @@ static struct shash_alg sha256_ni_algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = sha256_ni_update, - .final = sha256_ni_final, .finup = sha256_ni_finup, .digest = sha256_ni_digest, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-ni", .cra_priority = 250, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -376,13 +347,14 @@ static struct shash_alg sha256_ni_algs[] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = sha256_ni_update, - .final = sha256_ni_final, .finup = sha256_ni_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-ni", .cra_priority = 250, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -403,11 +375,6 @@ static void unregister_sha256_ni(void) ARRAY_SIZE(sha256_ni_algs)); } -#else -static inline int register_sha256_ni(void) { return 0; } -static inline void unregister_sha256_ni(void) { } -#endif - static int __init sha256_ssse3_mod_init(void) { if (!x86_match_cpu(module_cpu_ids)) @@ -461,7 +428,5 @@ MODULE_ALIAS_CRYPTO("sha224"); MODULE_ALIAS_CRYPTO("sha224-ssse3"); MODULE_ALIAS_CRYPTO("sha224-avx"); MODULE_ALIAS_CRYPTO("sha224-avx2"); -#ifdef CONFIG_AS_SHA256_NI MODULE_ALIAS_CRYPTO("sha256-ni"); MODULE_ALIAS_CRYPTO("sha224-ni"); -#endif diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_ssse3_glue.c index 6d3b85e53d0e..067684c54395 100644 --- a/arch/x86/crypto/sha512_ssse3_glue.c +++ b/arch/x86/crypto/sha512_ssse3_glue.c @@ -27,17 +27,13 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include <asm/cpu_device_id.h> +#include <asm/simd.h> #include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> -#include <linux/init.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <linux/mm.h> -#include <linux/string.h> -#include <linux/types.h> #include <crypto/sha2.h> #include <crypto/sha512_base.h> -#include <asm/cpu_device_id.h> -#include <asm/simd.h> asmlinkage void sha512_transform_ssse3(struct sha512_state *state, const u8 *data, int blocks); @@ -45,11 +41,7 @@ asmlinkage void sha512_transform_ssse3(struct sha512_state *state, static int sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len, sha512_block_fn *sha512_xform) { - struct sha512_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) - return crypto_sha512_update(desc, data, len); + int remain; /* * Make sure struct sha512_state begins directly with the SHA512 @@ -58,22 +50,17 @@ static int sha512_update(struct shash_desc *desc, const u8 *data, BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0); kernel_fpu_begin(); - sha512_base_do_update(desc, data, len, sha512_xform); + remain = sha512_base_do_update_blocks(desc, data, len, sha512_xform); kernel_fpu_end(); - return 0; + return remain; } static int sha512_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out, sha512_block_fn *sha512_xform) { - if (!crypto_simd_usable()) - return crypto_sha512_finup(desc, data, len, out); - kernel_fpu_begin(); - if (len) - sha512_base_do_update(desc, data, len, sha512_xform); - sha512_base_do_finalize(desc, sha512_xform); + sha512_base_do_finup(desc, data, len, sha512_xform); kernel_fpu_end(); return sha512_base_finish(desc, out); @@ -91,23 +78,18 @@ static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data, return sha512_finup(desc, data, len, out, sha512_transform_ssse3); } -/* Add padding and return the message digest. */ -static int sha512_ssse3_final(struct shash_desc *desc, u8 *out) -{ - return sha512_ssse3_finup(desc, NULL, 0, out); -} - static struct shash_alg sha512_ssse3_algs[] = { { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, .update = sha512_ssse3_update, - .final = sha512_ssse3_final, .finup = sha512_ssse3_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha512", .cra_driver_name = "sha512-ssse3", .cra_priority = 150, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -115,13 +97,14 @@ static struct shash_alg sha512_ssse3_algs[] = { { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, .update = sha512_ssse3_update, - .final = sha512_ssse3_final, .finup = sha512_ssse3_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha384", .cra_driver_name = "sha384-ssse3", .cra_priority = 150, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -167,23 +150,18 @@ static int sha512_avx_finup(struct shash_desc *desc, const u8 *data, return sha512_finup(desc, data, len, out, sha512_transform_avx); } -/* Add padding and return the message digest. */ -static int sha512_avx_final(struct shash_desc *desc, u8 *out) -{ - return sha512_avx_finup(desc, NULL, 0, out); -} - static struct shash_alg sha512_avx_algs[] = { { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, .update = sha512_avx_update, - .final = sha512_avx_final, .finup = sha512_avx_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha512", .cra_driver_name = "sha512-avx", .cra_priority = 160, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -191,13 +169,14 @@ static struct shash_alg sha512_avx_algs[] = { { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, .update = sha512_avx_update, - .final = sha512_avx_final, .finup = sha512_avx_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha384", .cra_driver_name = "sha384-avx", .cra_priority = 160, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -233,23 +212,18 @@ static int sha512_avx2_finup(struct shash_desc *desc, const u8 *data, return sha512_finup(desc, data, len, out, sha512_transform_rorx); } -/* Add padding and return the message digest. */ -static int sha512_avx2_final(struct shash_desc *desc, u8 *out) -{ - return sha512_avx2_finup(desc, NULL, 0, out); -} - static struct shash_alg sha512_avx2_algs[] = { { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, .update = sha512_avx2_update, - .final = sha512_avx2_final, .finup = sha512_avx2_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha512", .cra_driver_name = "sha512-avx2", .cra_priority = 170, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -257,13 +231,14 @@ static struct shash_alg sha512_avx2_algs[] = { { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, .update = sha512_avx2_update, - .final = sha512_avx2_final, .finup = sha512_avx2_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha384", .cra_driver_name = "sha384-avx2", .cra_priority = 170, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/x86/crypto/sm3_avx_glue.c b/arch/x86/crypto/sm3_avx_glue.c index 661b6f22ffcd..6e8c42b9dc8e 100644 --- a/arch/x86/crypto/sm3_avx_glue.c +++ b/arch/x86/crypto/sm3_avx_glue.c @@ -10,12 +10,11 @@ #include <crypto/internal/hash.h> #include <crypto/internal/simd.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/types.h> #include <crypto/sm3.h> #include <crypto/sm3_base.h> -#include <asm/simd.h> +#include <linux/cpufeature.h> +#include <linux/kernel.h> +#include <linux/module.h> asmlinkage void sm3_transform_avx(struct sm3_state *state, const u8 *data, int nblocks); @@ -23,13 +22,7 @@ asmlinkage void sm3_transform_avx(struct sm3_state *state, static int sm3_avx_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sm3_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count % SM3_BLOCK_SIZE) + len < SM3_BLOCK_SIZE) { - sm3_update(sctx, data, len); - return 0; - } + int remain; /* * Make sure struct sm3_state begins directly with the SM3 @@ -38,45 +31,17 @@ static int sm3_avx_update(struct shash_desc *desc, const u8 *data, BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0); kernel_fpu_begin(); - sm3_base_do_update(desc, data, len, sm3_transform_avx); + remain = sm3_base_do_update_blocks(desc, data, len, sm3_transform_avx); kernel_fpu_end(); - - return 0; + return remain; } static int sm3_avx_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) { - struct sm3_state *sctx = shash_desc_ctx(desc); - - if (len) - sm3_update(sctx, data, len); - - sm3_final(sctx, out); - return 0; - } - kernel_fpu_begin(); - if (len) - sm3_base_do_update(desc, data, len, sm3_transform_avx); - sm3_base_do_finalize(desc, sm3_transform_avx); + sm3_base_do_finup(desc, data, len, sm3_transform_avx); kernel_fpu_end(); - - return sm3_base_finish(desc, out); -} - -static int sm3_avx_final(struct shash_desc *desc, u8 *out) -{ - if (!crypto_simd_usable()) { - sm3_final(shash_desc_ctx(desc), out); - return 0; - } - - kernel_fpu_begin(); - sm3_base_do_finalize(desc, sm3_transform_avx); - kernel_fpu_end(); - return sm3_base_finish(desc, out); } @@ -84,13 +49,14 @@ static struct shash_alg sm3_avx_alg = { .digestsize = SM3_DIGEST_SIZE, .init = sm3_base_init, .update = sm3_avx_update, - .final = sm3_avx_final, .finup = sm3_avx_finup, - .descsize = sizeof(struct sm3_state), + .descsize = SM3_STATE_SIZE, .base = { .cra_name = "sm3", .cra_driver_name = "sm3-avx", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SM3_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/x86/crypto/sm4_aesni_avx2_glue.c b/arch/x86/crypto/sm4_aesni_avx2_glue.c index 1148fd4cd57f..fec0ab7a63dd 100644 --- a/arch/x86/crypto/sm4_aesni_avx2_glue.c +++ b/arch/x86/crypto/sm4_aesni_avx2_glue.c @@ -8,11 +8,10 @@ * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> */ +#include <asm/fpu/api.h> #include <linux/module.h> #include <linux/crypto.h> #include <linux/kernel.h> -#include <asm/simd.h> -#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/sm4.h> #include "sm4-avx.h" @@ -48,10 +47,9 @@ static int ctr_crypt(struct skcipher_request *req) static struct skcipher_alg sm4_aesni_avx2_skciphers[] = { { .base = { - .cra_name = "__ecb(sm4)", - .cra_driver_name = "__ecb-sm4-aesni-avx2", + .cra_name = "ecb(sm4)", + .cra_driver_name = "ecb-sm4-aesni-avx2", .cra_priority = 500, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = SM4_BLOCK_SIZE, .cra_ctxsize = sizeof(struct sm4_ctx), .cra_module = THIS_MODULE, @@ -64,10 +62,9 @@ static struct skcipher_alg sm4_aesni_avx2_skciphers[] = { .decrypt = sm4_avx_ecb_decrypt, }, { .base = { - .cra_name = "__cbc(sm4)", - .cra_driver_name = "__cbc-sm4-aesni-avx2", + .cra_name = "cbc(sm4)", + .cra_driver_name = "cbc-sm4-aesni-avx2", .cra_priority = 500, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = SM4_BLOCK_SIZE, .cra_ctxsize = sizeof(struct sm4_ctx), .cra_module = THIS_MODULE, @@ -81,10 +78,9 @@ static struct skcipher_alg sm4_aesni_avx2_skciphers[] = { .decrypt = cbc_decrypt, }, { .base = { - .cra_name = "__ctr(sm4)", - .cra_driver_name = "__ctr-sm4-aesni-avx2", + .cra_name = "ctr(sm4)", + .cra_driver_name = "ctr-sm4-aesni-avx2", .cra_priority = 500, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct sm4_ctx), .cra_module = THIS_MODULE, @@ -100,9 +96,6 @@ static struct skcipher_alg sm4_aesni_avx2_skciphers[] = { } }; -static struct simd_skcipher_alg * -simd_sm4_aesni_avx2_skciphers[ARRAY_SIZE(sm4_aesni_avx2_skciphers)]; - static int __init sm4_init(void) { const char *feature_name; @@ -121,16 +114,14 @@ static int __init sm4_init(void) return -ENODEV; } - return simd_register_skciphers_compat(sm4_aesni_avx2_skciphers, - ARRAY_SIZE(sm4_aesni_avx2_skciphers), - simd_sm4_aesni_avx2_skciphers); + return crypto_register_skciphers(sm4_aesni_avx2_skciphers, + ARRAY_SIZE(sm4_aesni_avx2_skciphers)); } static void __exit sm4_exit(void) { - simd_unregister_skciphers(sm4_aesni_avx2_skciphers, - ARRAY_SIZE(sm4_aesni_avx2_skciphers), - simd_sm4_aesni_avx2_skciphers); + crypto_unregister_skciphers(sm4_aesni_avx2_skciphers, + ARRAY_SIZE(sm4_aesni_avx2_skciphers)); } module_init(sm4_init); diff --git a/arch/x86/crypto/sm4_aesni_avx_glue.c b/arch/x86/crypto/sm4_aesni_avx_glue.c index 85b4ca78b47b..72867fc49ce8 100644 --- a/arch/x86/crypto/sm4_aesni_avx_glue.c +++ b/arch/x86/crypto/sm4_aesni_avx_glue.c @@ -8,11 +8,10 @@ * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> */ +#include <asm/fpu/api.h> #include <linux/module.h> #include <linux/crypto.h> #include <linux/kernel.h> -#include <asm/simd.h> -#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/sm4.h> #include "sm4-avx.h" @@ -263,10 +262,9 @@ static int ctr_crypt(struct skcipher_request *req) static struct skcipher_alg sm4_aesni_avx_skciphers[] = { { .base = { - .cra_name = "__ecb(sm4)", - .cra_driver_name = "__ecb-sm4-aesni-avx", + .cra_name = "ecb(sm4)", + .cra_driver_name = "ecb-sm4-aesni-avx", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = SM4_BLOCK_SIZE, .cra_ctxsize = sizeof(struct sm4_ctx), .cra_module = THIS_MODULE, @@ -279,10 +277,9 @@ static struct skcipher_alg sm4_aesni_avx_skciphers[] = { .decrypt = sm4_avx_ecb_decrypt, }, { .base = { - .cra_name = "__cbc(sm4)", - .cra_driver_name = "__cbc-sm4-aesni-avx", + .cra_name = "cbc(sm4)", + .cra_driver_name = "cbc-sm4-aesni-avx", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = SM4_BLOCK_SIZE, .cra_ctxsize = sizeof(struct sm4_ctx), .cra_module = THIS_MODULE, @@ -296,10 +293,9 @@ static struct skcipher_alg sm4_aesni_avx_skciphers[] = { .decrypt = cbc_decrypt, }, { .base = { - .cra_name = "__ctr(sm4)", - .cra_driver_name = "__ctr-sm4-aesni-avx", + .cra_name = "ctr(sm4)", + .cra_driver_name = "ctr-sm4-aesni-avx", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct sm4_ctx), .cra_module = THIS_MODULE, @@ -315,9 +311,6 @@ static struct skcipher_alg sm4_aesni_avx_skciphers[] = { } }; -static struct simd_skcipher_alg * -simd_sm4_aesni_avx_skciphers[ARRAY_SIZE(sm4_aesni_avx_skciphers)]; - static int __init sm4_init(void) { const char *feature_name; @@ -335,16 +328,14 @@ static int __init sm4_init(void) return -ENODEV; } - return simd_register_skciphers_compat(sm4_aesni_avx_skciphers, - ARRAY_SIZE(sm4_aesni_avx_skciphers), - simd_sm4_aesni_avx_skciphers); + return crypto_register_skciphers(sm4_aesni_avx_skciphers, + ARRAY_SIZE(sm4_aesni_avx_skciphers)); } static void __exit sm4_exit(void) { - simd_unregister_skciphers(sm4_aesni_avx_skciphers, - ARRAY_SIZE(sm4_aesni_avx_skciphers), - simd_sm4_aesni_avx_skciphers); + crypto_unregister_skciphers(sm4_aesni_avx_skciphers, + ARRAY_SIZE(sm4_aesni_avx_skciphers)); } module_init(sm4_init); diff --git a/arch/x86/crypto/twofish_avx_glue.c b/arch/x86/crypto/twofish_avx_glue.c index 3eb3440b477a..9e20db013750 100644 --- a/arch/x86/crypto/twofish_avx_glue.c +++ b/arch/x86/crypto/twofish_avx_glue.c @@ -13,7 +13,6 @@ #include <linux/crypto.h> #include <linux/err.h> #include <crypto/algapi.h> -#include <crypto/internal/simd.h> #include <crypto/twofish.h> #include "twofish.h" @@ -74,10 +73,9 @@ static int cbc_decrypt(struct skcipher_request *req) static struct skcipher_alg twofish_algs[] = { { - .base.cra_name = "__ecb(twofish)", - .base.cra_driver_name = "__ecb-twofish-avx", + .base.cra_name = "ecb(twofish)", + .base.cra_driver_name = "ecb-twofish-avx", .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = TF_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct twofish_ctx), .base.cra_module = THIS_MODULE, @@ -87,10 +85,9 @@ static struct skcipher_alg twofish_algs[] = { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(twofish)", - .base.cra_driver_name = "__cbc-twofish-avx", + .base.cra_name = "cbc(twofish)", + .base.cra_driver_name = "cbc-twofish-avx", .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = TF_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct twofish_ctx), .base.cra_module = THIS_MODULE, @@ -103,8 +100,6 @@ static struct skcipher_alg twofish_algs[] = { }, }; -static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)]; - static int __init twofish_init(void) { const char *feature_name; @@ -114,15 +109,13 @@ static int __init twofish_init(void) return -ENODEV; } - return simd_register_skciphers_compat(twofish_algs, - ARRAY_SIZE(twofish_algs), - twofish_simd_algs); + return crypto_register_skciphers(twofish_algs, + ARRAY_SIZE(twofish_algs)); } static void __exit twofish_exit(void) { - simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs), - twofish_simd_algs); + crypto_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs)); } module_init(twofish_init); diff --git a/arch/x86/include/asm/simd.h b/arch/x86/include/asm/simd.h index a341c878e977..b8027b63cd7a 100644 --- a/arch/x86/include/asm/simd.h +++ b/arch/x86/include/asm/simd.h @@ -1,6 +1,10 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SIMD_H +#define _ASM_SIMD_H #include <asm/fpu/api.h> +#include <linux/compiler_attributes.h> +#include <linux/types.h> /* * may_use_simd - whether it is allowable at this time to issue SIMD @@ -10,3 +14,5 @@ static __must_check inline bool may_use_simd(void) { return irq_fpu_usable(); } + +#endif /* _ASM_SIMD_H */ diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 0bc708ee2788..7be4e1647903 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2933,6 +2933,7 @@ void __init sev_set_cpu_caps(void) void __init sev_hardware_setup(void) { unsigned int eax, ebx, ecx, edx, sev_asid_count, sev_es_asid_count; + struct sev_platform_init_args init_args = {0}; bool sev_snp_supported = false; bool sev_es_supported = false; bool sev_supported = false; @@ -3059,6 +3060,15 @@ out: sev_supported_vmsa_features = 0; if (sev_es_debug_swap_enabled) sev_supported_vmsa_features |= SVM_SEV_FEAT_DEBUG_SWAP; + + if (!sev_enabled) + return; + + /* + * Do both SNP and SEV initialization at KVM module load. + */ + init_args.probe = true; + sev_platform_init(&init_args); } void sev_hardware_unsetup(void) @@ -3074,6 +3084,8 @@ void sev_hardware_unsetup(void) misc_cg_set_capacity(MISC_CG_RES_SEV, 0); misc_cg_set_capacity(MISC_CG_RES_SEV_ES, 0); + + sev_platform_shutdown(); } int sev_cpu_init(struct svm_cpu_data *sd) diff --git a/crypto/842.c b/crypto/842.c index 5fb37a925989..881945d44328 100644 --- a/crypto/842.c +++ b/crypto/842.c @@ -23,10 +23,6 @@ #include <linux/module.h> #include <linux/sw842.h> -struct crypto842_ctx { - void *wmem; /* working memory for compress */ -}; - static void *crypto842_alloc_ctx(void) { void *ctx; diff --git a/crypto/Kconfig b/crypto/Kconfig index dbf97c4e7c59..9322e42e562d 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1012,13 +1012,10 @@ config CRYPTO_SHA3 help SHA-3 secure hash algorithms (FIPS 202, ISO/IEC 10118-3) -config CRYPTO_SM3 - tristate - config CRYPTO_SM3_GENERIC tristate "SM3 (ShangMi 3)" select CRYPTO_HASH - select CRYPTO_SM3 + select CRYPTO_LIB_SM3 help SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012, ISO/IEC 10118-3) diff --git a/crypto/Makefile b/crypto/Makefile index 0e6ab5ffd3f7..5d2f2a28d8a0 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -79,7 +79,6 @@ obj-$(CONFIG_CRYPTO_SHA1) += sha1_generic.o obj-$(CONFIG_CRYPTO_SHA256) += sha256_generic.o obj-$(CONFIG_CRYPTO_SHA512) += sha512_generic.o obj-$(CONFIG_CRYPTO_SHA3) += sha3_generic.o -obj-$(CONFIG_CRYPTO_SM3) += sm3.o obj-$(CONFIG_CRYPTO_SM3_GENERIC) += sm3_generic.o obj-$(CONFIG_CRYPTO_STREEBOG) += streebog_generic.o obj-$(CONFIG_CRYPTO_WP512) += wp512.o @@ -148,8 +147,10 @@ obj-$(CONFIG_CRYPTO_KHAZAD) += khazad.o obj-$(CONFIG_CRYPTO_ANUBIS) += anubis.o obj-$(CONFIG_CRYPTO_SEED) += seed.o obj-$(CONFIG_CRYPTO_ARIA) += aria_generic.o -obj-$(CONFIG_CRYPTO_CHACHA20) += chacha_generic.o -obj-$(CONFIG_CRYPTO_POLY1305) += poly1305_generic.o +obj-$(CONFIG_CRYPTO_CHACHA20) += chacha.o +CFLAGS_chacha.o += -DARCH=$(ARCH) +obj-$(CONFIG_CRYPTO_POLY1305) += poly1305.o +CFLAGS_poly1305.o += -DARCH=$(ARCH) obj-$(CONFIG_CRYPTO_DEFLATE) += deflate.o obj-$(CONFIG_CRYPTO_MICHAEL_MIC) += michael_mic.o obj-$(CONFIG_CRYPTO_CRC32C) += crc32c_generic.o diff --git a/crypto/acompress.c b/crypto/acompress.c index f7a3fbe5447e..4c665c6fb5d6 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -8,20 +8,34 @@ */ #include <crypto/internal/acompress.h> +#include <crypto/scatterwalk.h> #include <linux/cryptouser.h> +#include <linux/cpumask.h> #include <linux/errno.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/page-flags.h> +#include <linux/percpu.h> +#include <linux/scatterlist.h> +#include <linux/sched.h> #include <linux/seq_file.h> #include <linux/slab.h> +#include <linux/smp.h> +#include <linux/spinlock.h> #include <linux/string.h> +#include <linux/workqueue.h> #include <net/netlink.h> #include "compress.h" struct crypto_scomp; +enum { + ACOMP_WALK_SLEEP = 1 << 0, + ACOMP_WALK_SRC_LINEAR = 1 << 1, + ACOMP_WALK_DST_LINEAR = 1 << 2, +}; + static const struct crypto_type crypto_acomp_type; static void acomp_reqchain_done(void *data, int err); @@ -95,7 +109,7 @@ static int crypto_acomp_init_tfm(struct crypto_tfm *tfm) acomp->compress = alg->compress; acomp->decompress = alg->decompress; - acomp->reqsize = alg->reqsize; + acomp->reqsize = alg->base.cra_reqsize; acomp->base.exit = crypto_acomp_exit_tfm; @@ -136,6 +150,7 @@ static const struct crypto_type crypto_acomp_type = { .maskset = CRYPTO_ALG_TYPE_ACOMPRESS_MASK, .type = CRYPTO_ALG_TYPE_ACOMPRESS, .tfmsize = offsetof(struct crypto_acomp, base), + .algsize = offsetof(struct acomp_alg, base), }; struct crypto_acomp *crypto_alloc_acomp(const char *alg_name, u32 type, @@ -161,7 +176,6 @@ static void acomp_save_req(struct acomp_req *req, crypto_completion_t cplt) state->data = req->base.data; req->base.complete = cplt; req->base.data = state; - state->req0 = req; } static void acomp_restore_req(struct acomp_req *req) @@ -172,23 +186,16 @@ static void acomp_restore_req(struct acomp_req *req) req->base.data = state->data; } -static void acomp_reqchain_virt(struct acomp_req_chain *state, int err) +static void acomp_reqchain_virt(struct acomp_req *req) { - struct acomp_req *req = state->cur; + struct acomp_req_chain *state = &req->chain; unsigned int slen = req->slen; unsigned int dlen = req->dlen; - req->base.err = err; - state = &req->chain; - if (state->flags & CRYPTO_ACOMP_REQ_SRC_VIRT) acomp_request_set_src_dma(req, state->src, slen); - else if (state->flags & CRYPTO_ACOMP_REQ_SRC_FOLIO) - acomp_request_set_src_folio(req, state->sfolio, state->soff, slen); if (state->flags & CRYPTO_ACOMP_REQ_DST_VIRT) acomp_request_set_dst_dma(req, state->dst, dlen); - else if (state->flags & CRYPTO_ACOMP_REQ_DST_FOLIO) - acomp_request_set_dst_folio(req, state->dfolio, state->doff, dlen); } static void acomp_virt_to_sg(struct acomp_req *req) @@ -196,9 +203,7 @@ static void acomp_virt_to_sg(struct acomp_req *req) struct acomp_req_chain *state = &req->chain; state->flags = req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | - CRYPTO_ACOMP_REQ_DST_VIRT | - CRYPTO_ACOMP_REQ_SRC_FOLIO | - CRYPTO_ACOMP_REQ_DST_FOLIO); + CRYPTO_ACOMP_REQ_DST_VIRT); if (acomp_request_src_isvirt(req)) { unsigned int slen = req->slen; @@ -207,17 +212,6 @@ static void acomp_virt_to_sg(struct acomp_req *req) state->src = svirt; sg_init_one(&state->ssg, svirt, slen); acomp_request_set_src_sg(req, &state->ssg, slen); - } else if (acomp_request_src_isfolio(req)) { - struct folio *folio = req->sfolio; - unsigned int slen = req->slen; - size_t off = req->soff; - - state->sfolio = folio; - state->soff = off; - sg_init_table(&state->ssg, 1); - sg_set_page(&state->ssg, folio_page(folio, off / PAGE_SIZE), - slen, off % PAGE_SIZE); - acomp_request_set_src_sg(req, &state->ssg, slen); } if (acomp_request_dst_isvirt(req)) { @@ -227,39 +221,15 @@ static void acomp_virt_to_sg(struct acomp_req *req) state->dst = dvirt; sg_init_one(&state->dsg, dvirt, dlen); acomp_request_set_dst_sg(req, &state->dsg, dlen); - } else if (acomp_request_dst_isfolio(req)) { - struct folio *folio = req->dfolio; - unsigned int dlen = req->dlen; - size_t off = req->doff; - - state->dfolio = folio; - state->doff = off; - sg_init_table(&state->dsg, 1); - sg_set_page(&state->dsg, folio_page(folio, off / PAGE_SIZE), - dlen, off % PAGE_SIZE); - acomp_request_set_src_sg(req, &state->dsg, dlen); } } -static int acomp_do_nondma(struct acomp_req_chain *state, - struct acomp_req *req) +static int acomp_do_nondma(struct acomp_req *req, bool comp) { - u32 keep = CRYPTO_ACOMP_REQ_SRC_VIRT | - CRYPTO_ACOMP_REQ_SRC_NONDMA | - CRYPTO_ACOMP_REQ_DST_VIRT | - CRYPTO_ACOMP_REQ_DST_NONDMA; - ACOMP_REQUEST_ON_STACK(fbreq, crypto_acomp_reqtfm(req)); + ACOMP_FBREQ_ON_STACK(fbreq, req); int err; - acomp_request_set_callback(fbreq, req->base.flags, NULL, NULL); - fbreq->base.flags &= ~keep; - fbreq->base.flags |= req->base.flags & keep; - fbreq->src = req->src; - fbreq->dst = req->dst; - fbreq->slen = req->slen; - fbreq->dlen = req->dlen; - - if (state->op == crypto_acomp_reqtfm(req)->compress) + if (comp) err = crypto_acomp_compress(fbreq); else err = crypto_acomp_decompress(fbreq); @@ -268,114 +238,74 @@ static int acomp_do_nondma(struct acomp_req_chain *state, return err; } -static int acomp_do_one_req(struct acomp_req_chain *state, - struct acomp_req *req) +static int acomp_do_one_req(struct acomp_req *req, bool comp) { - state->cur = req; - if (acomp_request_isnondma(req)) - return acomp_do_nondma(state, req); + return acomp_do_nondma(req, comp); acomp_virt_to_sg(req); - return state->op(req); + return comp ? crypto_acomp_reqtfm(req)->compress(req) : + crypto_acomp_reqtfm(req)->decompress(req); } -static int acomp_reqchain_finish(struct acomp_req *req0, int err, u32 mask) +static int acomp_reqchain_finish(struct acomp_req *req, int err) { - struct acomp_req_chain *state = req0->base.data; - struct acomp_req *req = state->cur; - struct acomp_req *n; - - acomp_reqchain_virt(state, err); - - if (req != req0) - list_add_tail(&req->base.list, &req0->base.list); - - list_for_each_entry_safe(req, n, &state->head, base.list) { - list_del_init(&req->base.list); - - req->base.flags &= mask; - req->base.complete = acomp_reqchain_done; - req->base.data = state; - - err = acomp_do_one_req(state, req); - - if (err == -EINPROGRESS) { - if (!list_empty(&state->head)) - err = -EBUSY; - goto out; - } - - if (err == -EBUSY) - goto out; - - acomp_reqchain_virt(state, err); - list_add_tail(&req->base.list, &req0->base.list); - } - - acomp_restore_req(req0); - -out: + acomp_reqchain_virt(req); + acomp_restore_req(req); return err; } static void acomp_reqchain_done(void *data, int err) { - struct acomp_req_chain *state = data; - crypto_completion_t compl = state->compl; + struct acomp_req *req = data; + crypto_completion_t compl; - data = state->data; + compl = req->chain.compl; + data = req->chain.data; - if (err == -EINPROGRESS) { - if (!list_empty(&state->head)) - return; + if (err == -EINPROGRESS) goto notify; - } - err = acomp_reqchain_finish(state->req0, err, - CRYPTO_TFM_REQ_MAY_BACKLOG); - if (err == -EBUSY) - return; + err = acomp_reqchain_finish(req, err); notify: compl(data, err); } -static int acomp_do_req_chain(struct acomp_req *req, - int (*op)(struct acomp_req *req)) +static int acomp_do_req_chain(struct acomp_req *req, bool comp) { - struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); - struct acomp_req_chain *state; int err; - if (crypto_acomp_req_chain(tfm) || - (!acomp_request_chained(req) && acomp_request_issg(req))) - return op(req); - acomp_save_req(req, acomp_reqchain_done); - state = req->base.data; - state->op = op; - state->src = NULL; - INIT_LIST_HEAD(&state->head); - list_splice_init(&req->base.list, &state->head); - - err = acomp_do_one_req(state, req); + err = acomp_do_one_req(req, comp); if (err == -EBUSY || err == -EINPROGRESS) - return -EBUSY; + return err; - return acomp_reqchain_finish(req, err, ~0); + return acomp_reqchain_finish(req, err); } int crypto_acomp_compress(struct acomp_req *req) { - return acomp_do_req_chain(req, crypto_acomp_reqtfm(req)->compress); + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + + if (acomp_req_on_stack(req) && acomp_is_async(tfm)) + return -EAGAIN; + if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) + return crypto_acomp_reqtfm(req)->compress(req); + return acomp_do_req_chain(req, true); } EXPORT_SYMBOL_GPL(crypto_acomp_compress); int crypto_acomp_decompress(struct acomp_req *req) { - return acomp_do_req_chain(req, crypto_acomp_reqtfm(req)->decompress); + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + + if (acomp_req_on_stack(req) && acomp_is_async(tfm)) + return -EAGAIN; + if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) + return crypto_acomp_reqtfm(req)->decompress(req); + return acomp_do_req_chain(req, false); } EXPORT_SYMBOL_GPL(crypto_acomp_decompress); @@ -434,5 +364,230 @@ void crypto_unregister_acomps(struct acomp_alg *algs, int count) } EXPORT_SYMBOL_GPL(crypto_unregister_acomps); +static void acomp_stream_workfn(struct work_struct *work) +{ + struct crypto_acomp_streams *s = + container_of(work, struct crypto_acomp_streams, stream_work); + struct crypto_acomp_stream __percpu *streams = s->streams; + int cpu; + + for_each_cpu(cpu, &s->stream_want) { + struct crypto_acomp_stream *ps; + void *ctx; + + ps = per_cpu_ptr(streams, cpu); + if (ps->ctx) + continue; + + ctx = s->alloc_ctx(); + if (IS_ERR(ctx)) + break; + + spin_lock_bh(&ps->lock); + ps->ctx = ctx; + spin_unlock_bh(&ps->lock); + + cpumask_clear_cpu(cpu, &s->stream_want); + } +} + +void crypto_acomp_free_streams(struct crypto_acomp_streams *s) +{ + struct crypto_acomp_stream __percpu *streams = s->streams; + void (*free_ctx)(void *); + int i; + + s->streams = NULL; + if (!streams) + return; + + cancel_work_sync(&s->stream_work); + free_ctx = s->free_ctx; + + for_each_possible_cpu(i) { + struct crypto_acomp_stream *ps = per_cpu_ptr(streams, i); + + if (!ps->ctx) + continue; + + free_ctx(ps->ctx); + } + + free_percpu(streams); +} +EXPORT_SYMBOL_GPL(crypto_acomp_free_streams); + +int crypto_acomp_alloc_streams(struct crypto_acomp_streams *s) +{ + struct crypto_acomp_stream __percpu *streams; + struct crypto_acomp_stream *ps; + unsigned int i; + void *ctx; + + if (s->streams) + return 0; + + streams = alloc_percpu(struct crypto_acomp_stream); + if (!streams) + return -ENOMEM; + + ctx = s->alloc_ctx(); + if (IS_ERR(ctx)) { + free_percpu(streams); + return PTR_ERR(ctx); + } + + i = cpumask_first(cpu_possible_mask); + ps = per_cpu_ptr(streams, i); + ps->ctx = ctx; + + for_each_possible_cpu(i) { + ps = per_cpu_ptr(streams, i); + spin_lock_init(&ps->lock); + } + + s->streams = streams; + + INIT_WORK(&s->stream_work, acomp_stream_workfn); + return 0; +} +EXPORT_SYMBOL_GPL(crypto_acomp_alloc_streams); + +struct crypto_acomp_stream *crypto_acomp_lock_stream_bh( + struct crypto_acomp_streams *s) __acquires(stream) +{ + struct crypto_acomp_stream __percpu *streams = s->streams; + int cpu = raw_smp_processor_id(); + struct crypto_acomp_stream *ps; + + ps = per_cpu_ptr(streams, cpu); + spin_lock_bh(&ps->lock); + if (likely(ps->ctx)) + return ps; + spin_unlock(&ps->lock); + + cpumask_set_cpu(cpu, &s->stream_want); + schedule_work(&s->stream_work); + + ps = per_cpu_ptr(streams, cpumask_first(cpu_possible_mask)); + spin_lock(&ps->lock); + return ps; +} +EXPORT_SYMBOL_GPL(crypto_acomp_lock_stream_bh); + +void acomp_walk_done_src(struct acomp_walk *walk, int used) +{ + walk->slen -= used; + if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) + scatterwalk_advance(&walk->in, used); + else + scatterwalk_done_src(&walk->in, used); + + if ((walk->flags & ACOMP_WALK_SLEEP)) + cond_resched(); +} +EXPORT_SYMBOL_GPL(acomp_walk_done_src); + +void acomp_walk_done_dst(struct acomp_walk *walk, int used) +{ + walk->dlen -= used; + if ((walk->flags & ACOMP_WALK_DST_LINEAR)) + scatterwalk_advance(&walk->out, used); + else + scatterwalk_done_dst(&walk->out, used); + + if ((walk->flags & ACOMP_WALK_SLEEP)) + cond_resched(); +} +EXPORT_SYMBOL_GPL(acomp_walk_done_dst); + +int acomp_walk_next_src(struct acomp_walk *walk) +{ + unsigned int slen = walk->slen; + unsigned int max = UINT_MAX; + + if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP)) + max = PAGE_SIZE; + if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) { + walk->in.__addr = (void *)(((u8 *)walk->in.sg) + + walk->in.offset); + return min(slen, max); + } + + return slen ? scatterwalk_next(&walk->in, slen) : 0; +} +EXPORT_SYMBOL_GPL(acomp_walk_next_src); + +int acomp_walk_next_dst(struct acomp_walk *walk) +{ + unsigned int dlen = walk->dlen; + unsigned int max = UINT_MAX; + + if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP)) + max = PAGE_SIZE; + if ((walk->flags & ACOMP_WALK_DST_LINEAR)) { + walk->out.__addr = (void *)(((u8 *)walk->out.sg) + + walk->out.offset); + return min(dlen, max); + } + + return dlen ? scatterwalk_next(&walk->out, dlen) : 0; +} +EXPORT_SYMBOL_GPL(acomp_walk_next_dst); + +int acomp_walk_virt(struct acomp_walk *__restrict walk, + struct acomp_req *__restrict req, bool atomic) +{ + struct scatterlist *src = req->src; + struct scatterlist *dst = req->dst; + + walk->slen = req->slen; + walk->dlen = req->dlen; + + if (!walk->slen || !walk->dlen) + return -EINVAL; + + walk->flags = 0; + if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) && !atomic) + walk->flags |= ACOMP_WALK_SLEEP; + if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT)) + walk->flags |= ACOMP_WALK_SRC_LINEAR; + if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_VIRT)) + walk->flags |= ACOMP_WALK_DST_LINEAR; + + if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) { + walk->in.sg = (void *)req->svirt; + walk->in.offset = 0; + } else + scatterwalk_start(&walk->in, src); + if ((walk->flags & ACOMP_WALK_DST_LINEAR)) { + walk->out.sg = (void *)req->dvirt; + walk->out.offset = 0; + } else + scatterwalk_start(&walk->out, dst); + + return 0; +} +EXPORT_SYMBOL_GPL(acomp_walk_virt); + +struct acomp_req *acomp_request_clone(struct acomp_req *req, + size_t total, gfp_t gfp) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + struct acomp_req *nreq; + + nreq = kmalloc(total, gfp); + if (!nreq) { + acomp_request_set_tfm(req, tfm->fb); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + return req; + } + + memcpy(nreq, req, total); + acomp_request_set_tfm(req, tfm); + return req; +} +EXPORT_SYMBOL_GPL(acomp_request_clone); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous compression type"); diff --git a/crypto/aead.c b/crypto/aead.c index 12f5b42171af..5d14b775036e 100644 --- a/crypto/aead.c +++ b/crypto/aead.c @@ -186,6 +186,7 @@ static const struct crypto_type crypto_aead_type = { .maskset = CRYPTO_ALG_TYPE_MASK, .type = CRYPTO_ALG_TYPE_AEAD, .tfmsize = offsetof(struct crypto_aead, base), + .algsize = offsetof(struct aead_alg, base), }; int crypto_grab_aead(struct crypto_aead_spawn *spawn, diff --git a/crypto/ahash.c b/crypto/ahash.c index 2d9eec2b2b1c..7a74092323b9 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -43,10 +43,7 @@ struct crypto_hash_walk { }; struct ahash_save_req_state { - struct list_head head; struct ahash_request *req0; - struct ahash_request *cur; - int (*op)(struct ahash_request *req); crypto_completion_t compl; void *data; struct scatterlist sg; @@ -54,9 +51,9 @@ struct ahash_save_req_state { u8 *page; unsigned int offset; unsigned int nbytes; + bool update; }; -static void ahash_reqchain_done(void *data, int err); static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt); static void ahash_restore_req(struct ahash_request *req); static void ahash_def_finup_done1(void *data, int err); @@ -303,6 +300,8 @@ int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, int err; err = alg->setkey(tfm, key, keylen); + if (!err && ahash_is_async(tfm)) + err = crypto_ahash_setkey(tfm->fb, key, keylen); if (unlikely(err)) { ahash_set_needkey(tfm, alg); return err; @@ -313,21 +312,17 @@ int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, } EXPORT_SYMBOL_GPL(crypto_ahash_setkey); -static bool ahash_request_hasvirt(struct ahash_request *req) -{ - return ahash_request_isvirt(req); -} - static int ahash_reqchain_virt(struct ahash_save_req_state *state, int err, u32 mask) { - struct ahash_request *req = state->cur; + struct ahash_request *req = state->req0; + struct crypto_ahash *tfm; + + tfm = crypto_ahash_reqtfm(req); for (;;) { unsigned len = state->nbytes; - req->base.err = err; - if (!state->offset) break; @@ -346,10 +341,9 @@ static int ahash_reqchain_virt(struct ahash_save_req_state *state, state->offset += len; req->nbytes = len; - err = state->op(req); + err = crypto_ahash_alg(tfm)->update(req); if (err == -EINPROGRESS) { - if (!list_empty(&state->head) || - state->offset < state->nbytes) + if (state->offset < state->nbytes) err = -EBUSY; break; } @@ -365,64 +359,12 @@ static int ahash_reqchain_finish(struct ahash_request *req0, struct ahash_save_req_state *state, int err, u32 mask) { - struct ahash_request *req = state->cur; - struct crypto_ahash *tfm; - struct ahash_request *n; - bool update; u8 *page; err = ahash_reqchain_virt(state, err, mask); if (err == -EINPROGRESS || err == -EBUSY) goto out; - if (req != req0) - list_add_tail(&req->base.list, &req0->base.list); - - tfm = crypto_ahash_reqtfm(req); - update = state->op == crypto_ahash_alg(tfm)->update; - - list_for_each_entry_safe(req, n, &state->head, base.list) { - list_del_init(&req->base.list); - - req->base.flags &= mask; - req->base.complete = ahash_reqchain_done; - req->base.data = state; - state->cur = req; - - if (update && ahash_request_isvirt(req) && req->nbytes) { - unsigned len = req->nbytes; - u8 *result = req->result; - - state->src = req->svirt; - state->nbytes = len; - - len = min(PAGE_SIZE, len); - - memcpy(state->page, req->svirt, len); - state->offset = len; - - ahash_request_set_crypt(req, &state->sg, result, len); - } - - err = state->op(req); - - if (err == -EINPROGRESS) { - if (!list_empty(&state->head) || - state->offset < state->nbytes) - err = -EBUSY; - goto out; - } - - if (err == -EBUSY) - goto out; - - err = ahash_reqchain_virt(state, err, mask); - if (err == -EINPROGRESS || err == -EBUSY) - goto out; - - list_add_tail(&req->base.list, &req0->base.list); - } - page = state->page; if (page) { memset(page, 0, PAGE_SIZE); @@ -442,7 +384,7 @@ static void ahash_reqchain_done(void *data, int err) data = state->data; if (err == -EINPROGRESS) { - if (!list_empty(&state->head) || state->offset < state->nbytes) + if (state->offset < state->nbytes) return; goto notify; } @@ -467,21 +409,14 @@ static int ahash_do_req_chain(struct ahash_request *req, int err; if (crypto_ahash_req_chain(tfm) || - (!ahash_request_chained(req) && - (!update || !ahash_request_isvirt(req)))) + !update || !ahash_request_isvirt(req)) return op(req); - if (update && ahash_request_hasvirt(req)) { - gfp_t gfp; - u32 flags; - - flags = ahash_request_flags(req); - gfp = (flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? - GFP_KERNEL : GFP_ATOMIC; - page = (void *)__get_free_page(gfp); + if (update && ahash_request_isvirt(req)) { + page = (void *)__get_free_page(GFP_ATOMIC); err = -ENOMEM; if (!page) - goto out_set_chain; + goto out; } state = &state0; @@ -493,12 +428,10 @@ static int ahash_do_req_chain(struct ahash_request *req, state = req->base.data; } - state->op = op; - state->cur = req; + state->update = update; state->page = page; state->offset = 0; state->nbytes = 0; - INIT_LIST_HEAD(&state->head); if (page) sg_init_one(&state->sg, page, PAGE_SIZE); @@ -519,16 +452,18 @@ static int ahash_do_req_chain(struct ahash_request *req, } err = op(req); - if (err == -EBUSY || err == -EINPROGRESS) - return -EBUSY; + if (err == -EINPROGRESS || err == -EBUSY) { + if (state->offset < state->nbytes) + err = -EBUSY; + return err; + } return ahash_reqchain_finish(req, state, err, ~0); out_free_page: free_page((unsigned long)page); -out_set_chain: - req->base.err = err; +out: return err; } @@ -536,17 +471,12 @@ int crypto_ahash_init(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) { - int err; - - err = crypto_shash_init(prepare_shash_desc(req, tfm)); - req->base.err = err; - return err; - } - + if (likely(tfm->using_shash)) + return crypto_shash_init(prepare_shash_desc(req, tfm)); if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; - + if (ahash_req_on_stack(req) && ahash_is_async(tfm)) + return -EAGAIN; return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->init); } EXPORT_SYMBOL_GPL(crypto_ahash_init); @@ -555,15 +485,11 @@ static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct ahash_save_req_state *state; - gfp_t gfp; - u32 flags; if (!ahash_is_async(tfm)) return 0; - flags = ahash_request_flags(req); - gfp = (flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; - state = kmalloc(sizeof(*state), gfp); + state = kmalloc(sizeof(*state), GFP_ATOMIC); if (!state) return -ENOMEM; @@ -596,14 +522,10 @@ int crypto_ahash_update(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) { - int err; - - err = shash_ahash_update(req, ahash_request_ctx(req)); - req->base.err = err; - return err; - } - + if (likely(tfm->using_shash)) + return shash_ahash_update(req, ahash_request_ctx(req)); + if (ahash_req_on_stack(req) && ahash_is_async(tfm)) + return -EAGAIN; return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->update); } EXPORT_SYMBOL_GPL(crypto_ahash_update); @@ -612,14 +534,10 @@ int crypto_ahash_final(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) { - int err; - - err = crypto_shash_final(ahash_request_ctx(req), req->result); - req->base.err = err; - return err; - } - + if (likely(tfm->using_shash)) + return crypto_shash_final(ahash_request_ctx(req), req->result); + if (ahash_req_on_stack(req) && ahash_is_async(tfm)) + return -EAGAIN; return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->final); } EXPORT_SYMBOL_GPL(crypto_ahash_final); @@ -628,18 +546,13 @@ int crypto_ahash_finup(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) { - int err; - - err = shash_ahash_finup(req, ahash_request_ctx(req)); - req->base.err = err; - return err; - } - + if (likely(tfm->using_shash)) + return shash_ahash_finup(req, ahash_request_ctx(req)); + if (ahash_req_on_stack(req) && ahash_is_async(tfm)) + return -EAGAIN; if (!crypto_ahash_alg(tfm)->finup || - (!crypto_ahash_req_chain(tfm) && ahash_request_hasvirt(req))) + (!crypto_ahash_req_chain(tfm) && ahash_request_isvirt(req))) return ahash_def_finup(req); - return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->finup); } EXPORT_SYMBOL_GPL(crypto_ahash_finup); @@ -706,20 +619,14 @@ int crypto_ahash_digest(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) { - int err; - - err = shash_ahash_digest(req, prepare_shash_desc(req, tfm)); - req->base.err = err; - return err; - } - - if (!crypto_ahash_req_chain(tfm) && ahash_request_hasvirt(req)) + if (likely(tfm->using_shash)) + return shash_ahash_digest(req, prepare_shash_desc(req, tfm)); + if (ahash_req_on_stack(req) && ahash_is_async(tfm)) + return -EAGAIN; + if (!crypto_ahash_req_chain(tfm) && ahash_request_isvirt(req)) return ahash_def_digest(req); - if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; - return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->digest); } EXPORT_SYMBOL_GPL(crypto_ahash_digest); @@ -819,26 +726,63 @@ static void crypto_ahash_exit_tfm(struct crypto_tfm *tfm) struct crypto_ahash *hash = __crypto_ahash_cast(tfm); struct ahash_alg *alg = crypto_ahash_alg(hash); - alg->exit_tfm(hash); + if (alg->exit_tfm) + alg->exit_tfm(hash); + else if (tfm->__crt_alg->cra_exit) + tfm->__crt_alg->cra_exit(tfm); + + if (ahash_is_async(hash)) + crypto_free_ahash(hash->fb); } static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) { struct crypto_ahash *hash = __crypto_ahash_cast(tfm); struct ahash_alg *alg = crypto_ahash_alg(hash); + struct crypto_ahash *fb = NULL; + int err; crypto_ahash_set_statesize(hash, alg->halg.statesize); - crypto_ahash_set_reqsize(hash, alg->reqsize); + crypto_ahash_set_reqsize(hash, crypto_tfm_alg_reqsize(tfm)); + + hash->fb = hash; if (tfm->__crt_alg->cra_type == &crypto_shash_type) return crypto_init_ahash_using_shash(tfm); + if (ahash_is_async(hash)) { + fb = crypto_alloc_ahash(crypto_ahash_alg_name(hash), + 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(fb)) + return PTR_ERR(fb); + + hash->fb = fb; + } + ahash_set_needkey(hash, alg); - if (alg->exit_tfm) - tfm->exit = crypto_ahash_exit_tfm; + tfm->exit = crypto_ahash_exit_tfm; + + if (!alg->init_tfm) { + if (!tfm->__crt_alg->cra_init) + return 0; + + err = tfm->__crt_alg->cra_init(tfm); + if (err) + goto out_free_sync_hash; - return alg->init_tfm ? alg->init_tfm(hash) : 0; + return 0; + } + + err = alg->init_tfm(hash); + if (err) + goto out_free_sync_hash; + + return 0; + +out_free_sync_hash: + crypto_free_ahash(fb); + return err; } static unsigned int crypto_ahash_extsize(struct crypto_alg *alg) @@ -897,6 +841,7 @@ static const struct crypto_type crypto_ahash_type = { .maskset = CRYPTO_ALG_TYPE_AHASH_MASK, .type = CRYPTO_ALG_TYPE_AHASH, .tfmsize = offsetof(struct crypto_ahash, base), + .algsize = offsetof(struct ahash_alg, halg.base), }; int crypto_grab_ahash(struct crypto_ahash_spawn *spawn, @@ -994,7 +939,7 @@ static int ahash_prepare_alg(struct ahash_alg *alg) if (alg->halg.statesize == 0) return -EINVAL; - if (alg->reqsize && alg->reqsize < alg->halg.statesize) + if (base->cra_reqsize && base->cra_reqsize < alg->halg.statesize) return -EINVAL; err = hash_prepare_alg(&alg->halg); @@ -1074,5 +1019,54 @@ int ahash_register_instance(struct crypto_template *tmpl, } EXPORT_SYMBOL_GPL(ahash_register_instance); +void ahash_request_free(struct ahash_request *req) +{ + if (unlikely(!req)) + return; + + if (!ahash_req_on_stack(req)) { + kfree(req); + return; + } + + ahash_request_zero(req); +} +EXPORT_SYMBOL_GPL(ahash_request_free); + +int crypto_hash_digest(struct crypto_ahash *tfm, const u8 *data, + unsigned int len, u8 *out) +{ + HASH_REQUEST_ON_STACK(req, tfm->fb); + int err; + + ahash_request_set_callback(req, 0, NULL, NULL); + ahash_request_set_virt(req, data, out, len); + err = crypto_ahash_digest(req); + + ahash_request_zero(req); + + return err; +} +EXPORT_SYMBOL_GPL(crypto_hash_digest); + +struct ahash_request *ahash_request_clone(struct ahash_request *req, + size_t total, gfp_t gfp) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ahash_request *nreq; + + nreq = kmalloc(total, gfp); + if (!nreq) { + ahash_request_set_tfm(req, tfm->fb); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + return req; + } + + memcpy(nreq, req, total); + ahash_request_set_tfm(req, tfm); + return req; +} +EXPORT_SYMBOL_GPL(ahash_request_clone); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous cryptographic hash type"); diff --git a/crypto/akcipher.c b/crypto/akcipher.c index 72c82d9aa077..a36f50c83827 100644 --- a/crypto/akcipher.c +++ b/crypto/akcipher.c @@ -97,6 +97,7 @@ static const struct crypto_type crypto_akcipher_type = { .maskset = CRYPTO_ALG_TYPE_AHASH_MASK, .type = CRYPTO_ALG_TYPE_AKCIPHER, .tfmsize = offsetof(struct crypto_akcipher, base), + .algsize = offsetof(struct akcipher_alg, base), }; int crypto_grab_akcipher(struct crypto_akcipher_spawn *spawn, diff --git a/crypto/algapi.c b/crypto/algapi.c index ea9ed9580aa8..532d3efc3c7d 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -71,12 +71,23 @@ static void crypto_free_instance(struct crypto_instance *inst) static void crypto_destroy_instance_workfn(struct work_struct *w) { - struct crypto_instance *inst = container_of(w, struct crypto_instance, + struct crypto_template *tmpl = container_of(w, struct crypto_template, free_work); - struct crypto_template *tmpl = inst->tmpl; + struct crypto_instance *inst; + struct hlist_node *n; + HLIST_HEAD(list); + + down_write(&crypto_alg_sem); + hlist_for_each_entry_safe(inst, n, &tmpl->dead, list) { + if (refcount_read(&inst->alg.cra_refcnt) != -1) + continue; + hlist_del(&inst->list); + hlist_add_head(&inst->list, &list); + } + up_write(&crypto_alg_sem); - crypto_free_instance(inst); - crypto_tmpl_put(tmpl); + hlist_for_each_entry_safe(inst, n, &list, list) + crypto_free_instance(inst); } static void crypto_destroy_instance(struct crypto_alg *alg) @@ -84,9 +95,10 @@ static void crypto_destroy_instance(struct crypto_alg *alg) struct crypto_instance *inst = container_of(alg, struct crypto_instance, alg); + struct crypto_template *tmpl = inst->tmpl; - INIT_WORK(&inst->free_work, crypto_destroy_instance_workfn); - schedule_work(&inst->free_work); + refcount_set(&alg->cra_refcnt, -1); + schedule_work(&tmpl->free_work); } /* @@ -132,14 +144,16 @@ static void crypto_remove_instance(struct crypto_instance *inst, inst->alg.cra_flags |= CRYPTO_ALG_DEAD; - if (!tmpl || !crypto_tmpl_get(tmpl)) + if (!tmpl) return; - list_move(&inst->alg.cra_list, list); + list_del_init(&inst->alg.cra_list); hlist_del(&inst->list); - inst->alg.cra_destroy = crypto_destroy_instance; + hlist_add_head(&inst->list, &tmpl->dead); BUG_ON(!list_empty(&inst->alg.cra_users)); + + crypto_alg_put(&inst->alg); } /* @@ -404,6 +418,15 @@ void crypto_remove_final(struct list_head *list) } EXPORT_SYMBOL_GPL(crypto_remove_final); +static void crypto_free_alg(struct crypto_alg *alg) +{ + unsigned int algsize = alg->cra_type->algsize; + u8 *p = (u8 *)alg - algsize; + + crypto_destroy_alg(alg); + kfree(p); +} + int crypto_register_alg(struct crypto_alg *alg) { struct crypto_larval *larval; @@ -416,6 +439,19 @@ int crypto_register_alg(struct crypto_alg *alg) if (err) return err; + if (alg->cra_flags & CRYPTO_ALG_DUP_FIRST && + !WARN_ON_ONCE(alg->cra_destroy)) { + unsigned int algsize = alg->cra_type->algsize; + u8 *p = (u8 *)alg - algsize; + + p = kmemdup(p, algsize + sizeof(*alg), GFP_KERNEL); + if (!p) + return -ENOMEM; + + alg = (void *)(p + algsize); + alg->cra_destroy = crypto_free_alg; + } + down_write(&crypto_alg_sem); larval = __crypto_register_alg(alg, &algs_to_put); if (!IS_ERR_OR_NULL(larval)) { @@ -424,8 +460,10 @@ int crypto_register_alg(struct crypto_alg *alg) } up_write(&crypto_alg_sem); - if (IS_ERR(larval)) + if (IS_ERR(larval)) { + crypto_alg_put(alg); return PTR_ERR(larval); + } if (test_started) crypto_schedule_test(larval); @@ -461,11 +499,9 @@ void crypto_unregister_alg(struct crypto_alg *alg) if (WARN(ret, "Algorithm %s is not registered", alg->cra_driver_name)) return; - if (WARN_ON(refcount_read(&alg->cra_refcnt) != 1)) - return; - - crypto_alg_put(alg); + WARN_ON(!alg->cra_destroy && refcount_read(&alg->cra_refcnt) != 1); + list_add(&alg->cra_list, &list); crypto_remove_final(&list); } EXPORT_SYMBOL_GPL(crypto_unregister_alg); @@ -504,6 +540,8 @@ int crypto_register_template(struct crypto_template *tmpl) struct crypto_template *q; int err = -EEXIST; + INIT_WORK(&tmpl->free_work, crypto_destroy_instance_workfn); + down_write(&crypto_alg_sem); crypto_check_module_sig(tmpl->module); @@ -565,6 +603,8 @@ void crypto_unregister_template(struct crypto_template *tmpl) crypto_free_instance(inst); } crypto_remove_final(&users); + + flush_work(&tmpl->free_work); } EXPORT_SYMBOL_GPL(crypto_unregister_template); @@ -618,6 +658,7 @@ int crypto_register_instance(struct crypto_template *tmpl, inst->alg.cra_module = tmpl->module; inst->alg.cra_flags |= CRYPTO_ALG_INSTANCE; + inst->alg.cra_destroy = crypto_destroy_instance; down_write(&crypto_alg_sem); diff --git a/crypto/api.c b/crypto/api.c index 3416e98128a0..e427cc5662b5 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -707,7 +707,6 @@ void crypto_destroy_alg(struct crypto_alg *alg) { if (alg->cra_type && alg->cra_type->destroy) alg->cra_type->destroy(alg); - if (alg->cra_destroy) alg->cra_destroy(alg); } diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c index bf165d321440..e5b177c8e842 100644 --- a/crypto/asymmetric_keys/public_key.c +++ b/crypto/asymmetric_keys/public_key.c @@ -163,10 +163,8 @@ static u8 *pkey_pack_u32(u8 *dst, u32 val) static int software_key_query(const struct kernel_pkey_params *params, struct kernel_pkey_query *info) { - struct crypto_akcipher *tfm; struct public_key *pkey = params->key->payload.data[asym_crypto]; char alg_name[CRYPTO_MAX_ALG_NAME]; - struct crypto_sig *sig; u8 *key, *ptr; int ret, len; bool issig; @@ -188,7 +186,11 @@ static int software_key_query(const struct kernel_pkey_params *params, ptr = pkey_pack_u32(ptr, pkey->paramlen); memcpy(ptr, pkey->params, pkey->paramlen); + memset(info, 0, sizeof(*info)); + if (issig) { + struct crypto_sig *sig; + sig = crypto_alloc_sig(alg_name, 0, 0); if (IS_ERR(sig)) { ret = PTR_ERR(sig); @@ -200,9 +202,10 @@ static int software_key_query(const struct kernel_pkey_params *params, else ret = crypto_sig_set_pubkey(sig, key, pkey->keylen); if (ret < 0) - goto error_free_tfm; + goto error_free_sig; len = crypto_sig_keysize(sig); + info->key_size = len; info->max_sig_size = crypto_sig_maxsize(sig); info->max_data_size = crypto_sig_digestsize(sig); @@ -211,11 +214,19 @@ static int software_key_query(const struct kernel_pkey_params *params, info->supported_ops |= KEYCTL_SUPPORTS_SIGN; if (strcmp(params->encoding, "pkcs1") == 0) { + info->max_enc_size = len / BITS_PER_BYTE; + info->max_dec_size = len / BITS_PER_BYTE; + info->supported_ops |= KEYCTL_SUPPORTS_ENCRYPT; if (pkey->key_is_private) info->supported_ops |= KEYCTL_SUPPORTS_DECRYPT; } + +error_free_sig: + crypto_free_sig(sig); } else { + struct crypto_akcipher *tfm; + tfm = crypto_alloc_akcipher(alg_name, 0, 0); if (IS_ERR(tfm)) { ret = PTR_ERR(tfm); @@ -227,28 +238,23 @@ static int software_key_query(const struct kernel_pkey_params *params, else ret = crypto_akcipher_set_pub_key(tfm, key, pkey->keylen); if (ret < 0) - goto error_free_tfm; + goto error_free_akcipher; len = crypto_akcipher_maxsize(tfm); + info->key_size = len * BITS_PER_BYTE; info->max_sig_size = len; info->max_data_size = len; + info->max_enc_size = len; + info->max_dec_size = len; info->supported_ops = KEYCTL_SUPPORTS_ENCRYPT; if (pkey->key_is_private) info->supported_ops |= KEYCTL_SUPPORTS_DECRYPT; - } - - info->key_size = len * 8; - info->max_enc_size = len; - info->max_dec_size = len; - - ret = 0; -error_free_tfm: - if (issig) - crypto_free_sig(sig); - else +error_free_akcipher: crypto_free_akcipher(tfm); + } + error_free_key: kfree_sensitive(key); pr_devel("<==%s() = %d\n", __func__, ret); diff --git a/crypto/asymmetric_keys/x509_cert_parser.c b/crypto/asymmetric_keys/x509_cert_parser.c index ee2fdab42334..2ffe4ae90bea 100644 --- a/crypto/asymmetric_keys/x509_cert_parser.c +++ b/crypto/asymmetric_keys/x509_cert_parser.c @@ -372,10 +372,9 @@ static int x509_fabricate_name(struct x509_parse_context *ctx, size_t hdrlen, /* Empty name string if no material */ if (!ctx->cn_size && !ctx->o_size && !ctx->email_size) { - buffer = kmalloc(1, GFP_KERNEL); + buffer = kzalloc(1, GFP_KERNEL); if (!buffer) return -ENOMEM; - buffer[0] = 0; goto done; } diff --git a/crypto/blake2b_generic.c b/crypto/blake2b_generic.c index 04a712ddfb43..6fa38965a493 100644 --- a/crypto/blake2b_generic.c +++ b/crypto/blake2b_generic.c @@ -15,12 +15,12 @@ * More information about BLAKE2 can be found at https://blake2.net. */ -#include <linux/unaligned.h> -#include <linux/module.h> -#include <linux/kernel.h> -#include <linux/bitops.h> #include <crypto/internal/blake2b.h> #include <crypto/internal/hash.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> +#include <linux/unaligned.h> static const u8 blake2b_sigma[12][16] = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, @@ -111,8 +111,8 @@ static void blake2b_compress_one_generic(struct blake2b_state *S, #undef G #undef ROUND -void blake2b_compress_generic(struct blake2b_state *state, - const u8 *block, size_t nblocks, u32 inc) +static void blake2b_compress_generic(struct blake2b_state *state, + const u8 *block, size_t nblocks, u32 inc) { do { blake2b_increment_counter(state, inc); @@ -120,17 +120,19 @@ void blake2b_compress_generic(struct blake2b_state *state, block += BLAKE2B_BLOCK_SIZE; } while (--nblocks); } -EXPORT_SYMBOL(blake2b_compress_generic); static int crypto_blake2b_update_generic(struct shash_desc *desc, const u8 *in, unsigned int inlen) { - return crypto_blake2b_update(desc, in, inlen, blake2b_compress_generic); + return crypto_blake2b_update_bo(desc, in, inlen, + blake2b_compress_generic); } -static int crypto_blake2b_final_generic(struct shash_desc *desc, u8 *out) +static int crypto_blake2b_finup_generic(struct shash_desc *desc, const u8 *in, + unsigned int inlen, u8 *out) { - return crypto_blake2b_final(desc, out, blake2b_compress_generic); + return crypto_blake2b_finup(desc, in, inlen, out, + blake2b_compress_generic); } #define BLAKE2B_ALG(name, driver_name, digest_size) \ @@ -138,7 +140,9 @@ static int crypto_blake2b_final_generic(struct shash_desc *desc, u8 *out) .base.cra_name = name, \ .base.cra_driver_name = driver_name, \ .base.cra_priority = 100, \ - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY | \ + CRYPTO_AHASH_ALG_BLOCK_ONLY | \ + CRYPTO_AHASH_ALG_FINAL_NONZERO, \ .base.cra_blocksize = BLAKE2B_BLOCK_SIZE, \ .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), \ .base.cra_module = THIS_MODULE, \ @@ -146,8 +150,9 @@ static int crypto_blake2b_final_generic(struct shash_desc *desc, u8 *out) .setkey = crypto_blake2b_setkey, \ .init = crypto_blake2b_init, \ .update = crypto_blake2b_update_generic, \ - .final = crypto_blake2b_final_generic, \ - .descsize = sizeof(struct blake2b_state), \ + .finup = crypto_blake2b_finup_generic, \ + .descsize = BLAKE2B_DESC_SIZE, \ + .statesize = BLAKE2B_STATE_SIZE, \ } static struct shash_alg blake2b_algs[] = { diff --git a/crypto/ccm.c b/crypto/ccm.c index 06476b53b491..f3f455e4908b 100644 --- a/crypto/ccm.c +++ b/crypto/ccm.c @@ -10,11 +10,12 @@ #include <crypto/internal/hash.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> +#include <crypto/utils.h> #include <linux/err.h> -#include <linux/init.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/slab.h> +#include <linux/string.h> struct ccm_instance_ctx { struct crypto_skcipher_spawn ctr; @@ -54,11 +55,6 @@ struct cbcmac_tfm_ctx { struct crypto_cipher *child; }; -struct cbcmac_desc_ctx { - unsigned int len; - u8 dg[]; -}; - static inline struct crypto_ccm_req_priv_ctx *crypto_ccm_reqctx( struct aead_request *req) { @@ -783,12 +779,10 @@ static int crypto_cbcmac_digest_setkey(struct crypto_shash *parent, static int crypto_cbcmac_digest_init(struct shash_desc *pdesc) { - struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc); int bs = crypto_shash_digestsize(pdesc->tfm); + u8 *dg = shash_desc_ctx(pdesc); - ctx->len = 0; - memset(ctx->dg, 0, bs); - + memset(dg, 0, bs); return 0; } @@ -797,39 +791,34 @@ static int crypto_cbcmac_digest_update(struct shash_desc *pdesc, const u8 *p, { struct crypto_shash *parent = pdesc->tfm; struct cbcmac_tfm_ctx *tctx = crypto_shash_ctx(parent); - struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc); struct crypto_cipher *tfm = tctx->child; int bs = crypto_shash_digestsize(parent); - - while (len > 0) { - unsigned int l = min(len, bs - ctx->len); - - crypto_xor(&ctx->dg[ctx->len], p, l); - ctx->len +=l; - len -= l; - p += l; - - if (ctx->len == bs) { - crypto_cipher_encrypt_one(tfm, ctx->dg, ctx->dg); - ctx->len = 0; - } - } - - return 0; + u8 *dg = shash_desc_ctx(pdesc); + + do { + crypto_xor(dg, p, bs); + crypto_cipher_encrypt_one(tfm, dg, dg); + p += bs; + len -= bs; + } while (len >= bs); + return len; } -static int crypto_cbcmac_digest_final(struct shash_desc *pdesc, u8 *out) +static int crypto_cbcmac_digest_finup(struct shash_desc *pdesc, const u8 *src, + unsigned int len, u8 *out) { struct crypto_shash *parent = pdesc->tfm; struct cbcmac_tfm_ctx *tctx = crypto_shash_ctx(parent); - struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc); struct crypto_cipher *tfm = tctx->child; int bs = crypto_shash_digestsize(parent); + u8 *dg = shash_desc_ctx(pdesc); - if (ctx->len) - crypto_cipher_encrypt_one(tfm, ctx->dg, ctx->dg); - - memcpy(out, ctx->dg, bs); + if (len) { + crypto_xor(dg, src, len); + crypto_cipher_encrypt_one(tfm, out, dg); + return 0; + } + memcpy(out, dg, bs); return 0; } @@ -883,19 +872,19 @@ static int cbcmac_create(struct crypto_template *tmpl, struct rtattr **tb) goto err_free_inst; inst->alg.base.cra_priority = alg->cra_priority; - inst->alg.base.cra_blocksize = 1; + inst->alg.base.cra_blocksize = alg->cra_blocksize; inst->alg.digestsize = alg->cra_blocksize; - inst->alg.descsize = sizeof(struct cbcmac_desc_ctx) + - alg->cra_blocksize; + inst->alg.descsize = alg->cra_blocksize; + inst->alg.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY; inst->alg.base.cra_ctxsize = sizeof(struct cbcmac_tfm_ctx); inst->alg.base.cra_init = cbcmac_init_tfm; inst->alg.base.cra_exit = cbcmac_exit_tfm; inst->alg.init = crypto_cbcmac_digest_init; inst->alg.update = crypto_cbcmac_digest_update; - inst->alg.final = crypto_cbcmac_digest_final; + inst->alg.finup = crypto_cbcmac_digest_finup; inst->alg.setkey = crypto_cbcmac_digest_setkey; inst->free = shash_free_singlespawn_instance; diff --git a/crypto/chacha.c b/crypto/chacha.c new file mode 100644 index 000000000000..5103bc0b2881 --- /dev/null +++ b/crypto/chacha.c @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Crypto API wrappers for the ChaCha20, XChaCha20, and XChaCha12 stream ciphers + * + * Copyright (C) 2015 Martin Willi + * Copyright (C) 2018 Google LLC + */ + +#include <linux/unaligned.h> +#include <crypto/algapi.h> +#include <crypto/chacha.h> +#include <crypto/internal/skcipher.h> +#include <linux/module.h> + +struct chacha_ctx { + u32 key[8]; + int nrounds; +}; + +static int chacha_setkey(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keysize, int nrounds) +{ + struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); + int i; + + if (keysize != CHACHA_KEY_SIZE) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(ctx->key); i++) + ctx->key[i] = get_unaligned_le32(key + i * sizeof(u32)); + + ctx->nrounds = nrounds; + return 0; +} + +static int chacha20_setkey(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keysize) +{ + return chacha_setkey(tfm, key, keysize, 20); +} + +static int chacha12_setkey(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keysize) +{ + return chacha_setkey(tfm, key, keysize, 12); +} + +static int chacha_stream_xor(struct skcipher_request *req, + const struct chacha_ctx *ctx, const u8 *iv, + bool arch) +{ + struct skcipher_walk walk; + u32 state[16]; + int err; + + err = skcipher_walk_virt(&walk, req, false); + + chacha_init(state, ctx->key, iv); + + while (walk.nbytes > 0) { + unsigned int nbytes = walk.nbytes; + + if (nbytes < walk.total) + nbytes = round_down(nbytes, CHACHA_BLOCK_SIZE); + + if (arch) + chacha_crypt(state, walk.dst.virt.addr, + walk.src.virt.addr, nbytes, ctx->nrounds); + else + chacha_crypt_generic(state, walk.dst.virt.addr, + walk.src.virt.addr, nbytes, + ctx->nrounds); + err = skcipher_walk_done(&walk, walk.nbytes - nbytes); + } + + return err; +} + +static int crypto_chacha_crypt_generic(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); + + return chacha_stream_xor(req, ctx, req->iv, false); +} + +static int crypto_chacha_crypt_arch(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); + + return chacha_stream_xor(req, ctx, req->iv, true); +} + +static int crypto_xchacha_crypt(struct skcipher_request *req, bool arch) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); + struct chacha_ctx subctx; + u32 state[16]; + u8 real_iv[16]; + + /* Compute the subkey given the original key and first 128 nonce bits */ + chacha_init(state, ctx->key, req->iv); + if (arch) + hchacha_block(state, subctx.key, ctx->nrounds); + else + hchacha_block_generic(state, subctx.key, ctx->nrounds); + subctx.nrounds = ctx->nrounds; + + /* Build the real IV */ + memcpy(&real_iv[0], req->iv + 24, 8); /* stream position */ + memcpy(&real_iv[8], req->iv + 16, 8); /* remaining 64 nonce bits */ + + /* Generate the stream and XOR it with the data */ + return chacha_stream_xor(req, &subctx, real_iv, arch); +} + +static int crypto_xchacha_crypt_generic(struct skcipher_request *req) +{ + return crypto_xchacha_crypt(req, false); +} + +static int crypto_xchacha_crypt_arch(struct skcipher_request *req) +{ + return crypto_xchacha_crypt(req, true); +} + +static struct skcipher_alg algs[] = { + { + .base.cra_name = "chacha20", + .base.cra_driver_name = "chacha20-generic", + .base.cra_priority = 100, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct chacha_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = CHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, + .setkey = chacha20_setkey, + .encrypt = crypto_chacha_crypt_generic, + .decrypt = crypto_chacha_crypt_generic, + }, + { + .base.cra_name = "xchacha20", + .base.cra_driver_name = "xchacha20-generic", + .base.cra_priority = 100, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct chacha_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = XCHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, + .setkey = chacha20_setkey, + .encrypt = crypto_xchacha_crypt_generic, + .decrypt = crypto_xchacha_crypt_generic, + }, + { + .base.cra_name = "xchacha12", + .base.cra_driver_name = "xchacha12-generic", + .base.cra_priority = 100, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct chacha_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = XCHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, + .setkey = chacha12_setkey, + .encrypt = crypto_xchacha_crypt_generic, + .decrypt = crypto_xchacha_crypt_generic, + }, + { + .base.cra_name = "chacha20", + .base.cra_driver_name = "chacha20-" __stringify(ARCH), + .base.cra_priority = 300, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct chacha_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = CHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, + .setkey = chacha20_setkey, + .encrypt = crypto_chacha_crypt_arch, + .decrypt = crypto_chacha_crypt_arch, + }, + { + .base.cra_name = "xchacha20", + .base.cra_driver_name = "xchacha20-" __stringify(ARCH), + .base.cra_priority = 300, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct chacha_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = XCHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, + .setkey = chacha20_setkey, + .encrypt = crypto_xchacha_crypt_arch, + .decrypt = crypto_xchacha_crypt_arch, + }, + { + .base.cra_name = "xchacha12", + .base.cra_driver_name = "xchacha12-" __stringify(ARCH), + .base.cra_priority = 300, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct chacha_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = XCHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, + .setkey = chacha12_setkey, + .encrypt = crypto_xchacha_crypt_arch, + .decrypt = crypto_xchacha_crypt_arch, + } +}; + +static unsigned int num_algs; + +static int __init crypto_chacha_mod_init(void) +{ + /* register the arch flavours only if they differ from generic */ + num_algs = ARRAY_SIZE(algs); + BUILD_BUG_ON(ARRAY_SIZE(algs) % 2 != 0); + if (!chacha_is_arch_optimized()) + num_algs /= 2; + + return crypto_register_skciphers(algs, num_algs); +} + +static void __exit crypto_chacha_mod_fini(void) +{ + crypto_unregister_skciphers(algs, num_algs); +} + +subsys_initcall(crypto_chacha_mod_init); +module_exit(crypto_chacha_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Martin Willi <martin@strongswan.org>"); +MODULE_DESCRIPTION("Crypto API wrappers for the ChaCha20, XChaCha20, and XChaCha12 stream ciphers"); +MODULE_ALIAS_CRYPTO("chacha20"); +MODULE_ALIAS_CRYPTO("chacha20-generic"); +MODULE_ALIAS_CRYPTO("chacha20-" __stringify(ARCH)); +MODULE_ALIAS_CRYPTO("xchacha20"); +MODULE_ALIAS_CRYPTO("xchacha20-generic"); +MODULE_ALIAS_CRYPTO("xchacha20-" __stringify(ARCH)); +MODULE_ALIAS_CRYPTO("xchacha12"); +MODULE_ALIAS_CRYPTO("xchacha12-generic"); +MODULE_ALIAS_CRYPTO("xchacha12-" __stringify(ARCH)); diff --git a/crypto/chacha_generic.c b/crypto/chacha_generic.c deleted file mode 100644 index 1fb9fbd302c6..000000000000 --- a/crypto/chacha_generic.c +++ /dev/null @@ -1,139 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * ChaCha and XChaCha stream ciphers, including ChaCha20 (RFC7539) - * - * Copyright (C) 2015 Martin Willi - * Copyright (C) 2018 Google LLC - */ - -#include <linux/unaligned.h> -#include <crypto/algapi.h> -#include <crypto/internal/chacha.h> -#include <crypto/internal/skcipher.h> -#include <linux/module.h> - -static int chacha_stream_xor(struct skcipher_request *req, - const struct chacha_ctx *ctx, const u8 *iv) -{ - struct skcipher_walk walk; - u32 state[16]; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - chacha_init(state, ctx->key, iv); - - while (walk.nbytes > 0) { - unsigned int nbytes = walk.nbytes; - - if (nbytes < walk.total) - nbytes = round_down(nbytes, CHACHA_BLOCK_SIZE); - - chacha_crypt_generic(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, ctx->nrounds); - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } - - return err; -} - -static int crypto_chacha_crypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - - return chacha_stream_xor(req, ctx, req->iv); -} - -static int crypto_xchacha_crypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - struct chacha_ctx subctx; - u32 state[16]; - u8 real_iv[16]; - - /* Compute the subkey given the original key and first 128 nonce bits */ - chacha_init(state, ctx->key, req->iv); - hchacha_block_generic(state, subctx.key, ctx->nrounds); - subctx.nrounds = ctx->nrounds; - - /* Build the real IV */ - memcpy(&real_iv[0], req->iv + 24, 8); /* stream position */ - memcpy(&real_iv[8], req->iv + 16, 8); /* remaining 64 nonce bits */ - - /* Generate the stream and XOR it with the data */ - return chacha_stream_xor(req, &subctx, real_iv); -} - -static struct skcipher_alg algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-generic", - .base.cra_priority = 100, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = crypto_chacha_crypt, - .decrypt = crypto_chacha_crypt, - }, { - .base.cra_name = "xchacha20", - .base.cra_driver_name = "xchacha20-generic", - .base.cra_priority = 100, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = crypto_xchacha_crypt, - .decrypt = crypto_xchacha_crypt, - }, { - .base.cra_name = "xchacha12", - .base.cra_driver_name = "xchacha12-generic", - .base.cra_priority = 100, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha12_setkey, - .encrypt = crypto_xchacha_crypt, - .decrypt = crypto_xchacha_crypt, - } -}; - -static int __init chacha_generic_mod_init(void) -{ - return crypto_register_skciphers(algs, ARRAY_SIZE(algs)); -} - -static void __exit chacha_generic_mod_fini(void) -{ - crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); -} - -subsys_initcall(chacha_generic_mod_init); -module_exit(chacha_generic_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Martin Willi <martin@strongswan.org>"); -MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (generic)"); -MODULE_ALIAS_CRYPTO("chacha20"); -MODULE_ALIAS_CRYPTO("chacha20-generic"); -MODULE_ALIAS_CRYPTO("xchacha20"); -MODULE_ALIAS_CRYPTO("xchacha20-generic"); -MODULE_ALIAS_CRYPTO("xchacha12"); -MODULE_ALIAS_CRYPTO("xchacha12-generic"); diff --git a/crypto/cmac.c b/crypto/cmac.c index c66a0f4d8808..f297042a324b 100644 --- a/crypto/cmac.c +++ b/crypto/cmac.c @@ -13,9 +13,12 @@ #include <crypto/internal/cipher.h> #include <crypto/internal/hash.h> +#include <crypto/utils.h> #include <linux/err.h> #include <linux/kernel.h> #include <linux/module.h> +#include <linux/slab.h> +#include <linux/string.h> /* * +------------------------ @@ -31,22 +34,6 @@ struct cmac_tfm_ctx { __be64 consts[]; }; -/* - * +------------------------ - * | <shash desc> - * +------------------------ - * | cmac_desc_ctx - * +------------------------ - * | odds (block size) - * +------------------------ - * | prev (block size) - * +------------------------ - */ -struct cmac_desc_ctx { - unsigned int len; - u8 odds[]; -}; - static int crypto_cmac_digest_setkey(struct crypto_shash *parent, const u8 *inkey, unsigned int keylen) { @@ -102,13 +89,10 @@ static int crypto_cmac_digest_setkey(struct crypto_shash *parent, static int crypto_cmac_digest_init(struct shash_desc *pdesc) { - struct cmac_desc_ctx *ctx = shash_desc_ctx(pdesc); int bs = crypto_shash_blocksize(pdesc->tfm); - u8 *prev = &ctx->odds[bs]; + u8 *prev = shash_desc_ctx(pdesc); - ctx->len = 0; memset(prev, 0, bs); - return 0; } @@ -117,77 +101,36 @@ static int crypto_cmac_digest_update(struct shash_desc *pdesc, const u8 *p, { struct crypto_shash *parent = pdesc->tfm; struct cmac_tfm_ctx *tctx = crypto_shash_ctx(parent); - struct cmac_desc_ctx *ctx = shash_desc_ctx(pdesc); struct crypto_cipher *tfm = tctx->child; int bs = crypto_shash_blocksize(parent); - u8 *odds = ctx->odds; - u8 *prev = odds + bs; - - /* checking the data can fill the block */ - if ((ctx->len + len) <= bs) { - memcpy(odds + ctx->len, p, len); - ctx->len += len; - return 0; - } - - /* filling odds with new data and encrypting it */ - memcpy(odds + ctx->len, p, bs - ctx->len); - len -= bs - ctx->len; - p += bs - ctx->len; - - crypto_xor(prev, odds, bs); - crypto_cipher_encrypt_one(tfm, prev, prev); + u8 *prev = shash_desc_ctx(pdesc); - /* clearing the length */ - ctx->len = 0; - - /* encrypting the rest of data */ - while (len > bs) { + do { crypto_xor(prev, p, bs); crypto_cipher_encrypt_one(tfm, prev, prev); p += bs; len -= bs; - } - - /* keeping the surplus of blocksize */ - if (len) { - memcpy(odds, p, len); - ctx->len = len; - } - - return 0; + } while (len >= bs); + return len; } -static int crypto_cmac_digest_final(struct shash_desc *pdesc, u8 *out) +static int crypto_cmac_digest_finup(struct shash_desc *pdesc, const u8 *src, + unsigned int len, u8 *out) { struct crypto_shash *parent = pdesc->tfm; struct cmac_tfm_ctx *tctx = crypto_shash_ctx(parent); - struct cmac_desc_ctx *ctx = shash_desc_ctx(pdesc); struct crypto_cipher *tfm = tctx->child; int bs = crypto_shash_blocksize(parent); - u8 *odds = ctx->odds; - u8 *prev = odds + bs; + u8 *prev = shash_desc_ctx(pdesc); unsigned int offset = 0; - if (ctx->len != bs) { - unsigned int rlen; - u8 *p = odds + ctx->len; - - *p = 0x80; - p++; - - rlen = bs - ctx->len - 1; - if (rlen) - memset(p, 0, rlen); - + crypto_xor(prev, src, len); + if (len != bs) { + prev[len] ^= 0x80; offset += bs; } - - crypto_xor(prev, odds, bs); crypto_xor(prev, (const u8 *)tctx->consts + offset, bs); - crypto_cipher_encrypt_one(tfm, out, prev); - return 0; } @@ -269,13 +212,14 @@ static int cmac_create(struct crypto_template *tmpl, struct rtattr **tb) inst->alg.base.cra_blocksize = alg->cra_blocksize; inst->alg.base.cra_ctxsize = sizeof(struct cmac_tfm_ctx) + alg->cra_blocksize * 2; + inst->alg.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINAL_NONZERO; inst->alg.digestsize = alg->cra_blocksize; - inst->alg.descsize = sizeof(struct cmac_desc_ctx) + - alg->cra_blocksize * 2; + inst->alg.descsize = alg->cra_blocksize; inst->alg.init = crypto_cmac_digest_init; inst->alg.update = crypto_cmac_digest_update; - inst->alg.final = crypto_cmac_digest_final; + inst->alg.finup = crypto_cmac_digest_finup; inst->alg.setkey = crypto_cmac_digest_setkey; inst->alg.init_tfm = cmac_init_tfm; inst->alg.clone_tfm = cmac_clone_tfm; diff --git a/crypto/deflate.c b/crypto/deflate.c index 5c346c544093..0d2b64d96d6e 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -6,246 +6,241 @@ * by IPCOMP (RFC 3173 & RFC 2394). * * Copyright (c) 2003 James Morris <jmorris@intercode.com.au> - * - * FIXME: deflate transforms will require up to a total of about 436k of kernel - * memory on i386 (390k for compression, the rest for decompression), as the - * current zlib kernel code uses a worst case pre-allocation system by default. - * This needs to be fixed so that the amount of memory required is properly - * related to the winbits and memlevel parameters. - * - * The default winbits of 11 should suit most packets, and it may be something - * to configure on a per-tfm basis in the future. - * - * Currently, compression history is not maintained between tfm calls, as - * it is not needed for IPCOMP and keeps the code simpler. It can be - * implemented if someone wants it. + * Copyright (c) 2023 Google, LLC. <ardb@kernel.org> + * Copyright (c) 2025 Herbert Xu <herbert@gondor.apana.org.au> */ +#include <crypto/internal/acompress.h> +#include <crypto/scatterwalk.h> #include <linux/init.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <linux/crypto.h> +#include <linux/mutex.h> +#include <linux/percpu.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> +#include <linux/spinlock.h> #include <linux/zlib.h> -#include <linux/vmalloc.h> -#include <linux/interrupt.h> -#include <linux/mm.h> -#include <linux/net.h> -#include <crypto/internal/scompress.h> #define DEFLATE_DEF_LEVEL Z_DEFAULT_COMPRESSION #define DEFLATE_DEF_WINBITS 11 #define DEFLATE_DEF_MEMLEVEL MAX_MEM_LEVEL -struct deflate_ctx { - struct z_stream_s comp_stream; - struct z_stream_s decomp_stream; +struct deflate_stream { + struct z_stream_s stream; + u8 workspace[]; }; -static int deflate_comp_init(struct deflate_ctx *ctx) -{ - int ret = 0; - struct z_stream_s *stream = &ctx->comp_stream; - - stream->workspace = vzalloc(zlib_deflate_workspacesize( - -DEFLATE_DEF_WINBITS, MAX_MEM_LEVEL)); - if (!stream->workspace) { - ret = -ENOMEM; - goto out; - } - ret = zlib_deflateInit2(stream, DEFLATE_DEF_LEVEL, Z_DEFLATED, - -DEFLATE_DEF_WINBITS, DEFLATE_DEF_MEMLEVEL, - Z_DEFAULT_STRATEGY); - if (ret != Z_OK) { - ret = -EINVAL; - goto out_free; - } -out: - return ret; -out_free: - vfree(stream->workspace); - goto out; -} +static DEFINE_MUTEX(deflate_stream_lock); -static int deflate_decomp_init(struct deflate_ctx *ctx) +static void *deflate_alloc_stream(void) { - int ret = 0; - struct z_stream_s *stream = &ctx->decomp_stream; + size_t size = max(zlib_inflate_workspacesize(), + zlib_deflate_workspacesize(-DEFLATE_DEF_WINBITS, + DEFLATE_DEF_MEMLEVEL)); + struct deflate_stream *ctx; - stream->workspace = vzalloc(zlib_inflate_workspacesize()); - if (!stream->workspace) { - ret = -ENOMEM; - goto out; - } - ret = zlib_inflateInit2(stream, -DEFLATE_DEF_WINBITS); - if (ret != Z_OK) { - ret = -EINVAL; - goto out_free; - } -out: - return ret; -out_free: - vfree(stream->workspace); - goto out; -} + ctx = kvmalloc(sizeof(*ctx) + size, GFP_KERNEL); + if (!ctx) + return ERR_PTR(-ENOMEM); -static void deflate_comp_exit(struct deflate_ctx *ctx) -{ - zlib_deflateEnd(&ctx->comp_stream); - vfree(ctx->comp_stream.workspace); -} + ctx->stream.workspace = ctx->workspace; -static void deflate_decomp_exit(struct deflate_ctx *ctx) -{ - zlib_inflateEnd(&ctx->decomp_stream); - vfree(ctx->decomp_stream.workspace); + return ctx; } -static int __deflate_init(void *ctx) +static struct crypto_acomp_streams deflate_streams = { + .alloc_ctx = deflate_alloc_stream, + .cfree_ctx = kvfree, +}; + +static int deflate_compress_one(struct acomp_req *req, + struct deflate_stream *ds) { + struct z_stream_s *stream = &ds->stream; + struct acomp_walk walk; int ret; - ret = deflate_comp_init(ctx); + ret = acomp_walk_virt(&walk, req, true); if (ret) - goto out; - ret = deflate_decomp_init(ctx); - if (ret) - deflate_comp_exit(ctx); -out: - return ret; -} + return ret; -static void *deflate_alloc_ctx(void) -{ - struct deflate_ctx *ctx; - int ret; + do { + unsigned int dcur; - ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); - if (!ctx) - return ERR_PTR(-ENOMEM); + dcur = acomp_walk_next_dst(&walk); + if (!dcur) + return -ENOSPC; - ret = __deflate_init(ctx); - if (ret) { - kfree(ctx); - return ERR_PTR(ret); - } + stream->avail_out = dcur; + stream->next_out = walk.dst.virt.addr; - return ctx; -} + do { + int flush = Z_FINISH; + unsigned int scur; -static void __deflate_exit(void *ctx) -{ - deflate_comp_exit(ctx); - deflate_decomp_exit(ctx); -} + stream->avail_in = 0; + stream->next_in = NULL; -static void deflate_free_ctx(void *ctx) -{ - __deflate_exit(ctx); - kfree_sensitive(ctx); + scur = acomp_walk_next_src(&walk); + if (scur) { + if (acomp_walk_more_src(&walk, scur)) + flush = Z_NO_FLUSH; + stream->avail_in = scur; + stream->next_in = walk.src.virt.addr; + } + + ret = zlib_deflate(stream, flush); + + if (scur) { + scur -= stream->avail_in; + acomp_walk_done_src(&walk, scur); + } + } while (ret == Z_OK && stream->avail_out); + + acomp_walk_done_dst(&walk, dcur); + } while (ret == Z_OK); + + if (ret != Z_STREAM_END) + return -EINVAL; + + req->dlen = stream->total_out; + return 0; } -static int __deflate_compress(const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx) +static int deflate_compress(struct acomp_req *req) { - int ret = 0; - struct deflate_ctx *dctx = ctx; - struct z_stream_s *stream = &dctx->comp_stream; + struct crypto_acomp_stream *s; + struct deflate_stream *ds; + int err; + + s = crypto_acomp_lock_stream_bh(&deflate_streams); + ds = s->ctx; - ret = zlib_deflateReset(stream); - if (ret != Z_OK) { - ret = -EINVAL; + err = zlib_deflateInit2(&ds->stream, DEFLATE_DEF_LEVEL, Z_DEFLATED, + -DEFLATE_DEF_WINBITS, DEFLATE_DEF_MEMLEVEL, + Z_DEFAULT_STRATEGY); + if (err != Z_OK) { + err = -EINVAL; goto out; } - stream->next_in = (u8 *)src; - stream->avail_in = slen; - stream->next_out = (u8 *)dst; - stream->avail_out = *dlen; + err = deflate_compress_one(req, ds); - ret = zlib_deflate(stream, Z_FINISH); - if (ret != Z_STREAM_END) { - ret = -EINVAL; - goto out; - } - ret = 0; - *dlen = stream->total_out; out: - return ret; + crypto_acomp_unlock_stream_bh(s); + + return err; } -static int deflate_scompress(struct crypto_scomp *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int *dlen, - void *ctx) +static int deflate_decompress_one(struct acomp_req *req, + struct deflate_stream *ds) { - return __deflate_compress(src, slen, dst, dlen, ctx); + struct z_stream_s *stream = &ds->stream; + bool out_of_space = false; + struct acomp_walk walk; + int ret; + + ret = acomp_walk_virt(&walk, req, true); + if (ret) + return ret; + + do { + unsigned int scur; + + stream->avail_in = 0; + stream->next_in = NULL; + + scur = acomp_walk_next_src(&walk); + if (scur) { + stream->avail_in = scur; + stream->next_in = walk.src.virt.addr; + } + + do { + unsigned int dcur; + + dcur = acomp_walk_next_dst(&walk); + if (!dcur) { + out_of_space = true; + break; + } + + stream->avail_out = dcur; + stream->next_out = walk.dst.virt.addr; + + ret = zlib_inflate(stream, Z_NO_FLUSH); + + dcur -= stream->avail_out; + acomp_walk_done_dst(&walk, dcur); + } while (ret == Z_OK && stream->avail_in); + + if (scur) + acomp_walk_done_src(&walk, scur); + + if (out_of_space) + return -ENOSPC; + } while (ret == Z_OK); + + if (ret != Z_STREAM_END) + return -EINVAL; + + req->dlen = stream->total_out; + return 0; } -static int __deflate_decompress(const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx) +static int deflate_decompress(struct acomp_req *req) { + struct crypto_acomp_stream *s; + struct deflate_stream *ds; + int err; - int ret = 0; - struct deflate_ctx *dctx = ctx; - struct z_stream_s *stream = &dctx->decomp_stream; + s = crypto_acomp_lock_stream_bh(&deflate_streams); + ds = s->ctx; - ret = zlib_inflateReset(stream); - if (ret != Z_OK) { - ret = -EINVAL; + err = zlib_inflateInit2(&ds->stream, -DEFLATE_DEF_WINBITS); + if (err != Z_OK) { + err = -EINVAL; goto out; } - stream->next_in = (u8 *)src; - stream->avail_in = slen; - stream->next_out = (u8 *)dst; - stream->avail_out = *dlen; - - ret = zlib_inflate(stream, Z_SYNC_FLUSH); - /* - * Work around a bug in zlib, which sometimes wants to taste an extra - * byte when being used in the (undocumented) raw deflate mode. - * (From USAGI). - */ - if (ret == Z_OK && !stream->avail_in && stream->avail_out) { - u8 zerostuff = 0; - stream->next_in = &zerostuff; - stream->avail_in = 1; - ret = zlib_inflate(stream, Z_FINISH); - } - if (ret != Z_STREAM_END) { - ret = -EINVAL; - goto out; - } - ret = 0; - *dlen = stream->total_out; + err = deflate_decompress_one(req, ds); + out: - return ret; + crypto_acomp_unlock_stream_bh(s); + + return err; } -static int deflate_sdecompress(struct crypto_scomp *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int *dlen, - void *ctx) +static int deflate_init(struct crypto_acomp *tfm) { - return __deflate_decompress(src, slen, dst, dlen, ctx); + int ret; + + mutex_lock(&deflate_stream_lock); + ret = crypto_acomp_alloc_streams(&deflate_streams); + mutex_unlock(&deflate_stream_lock); + + return ret; } -static struct scomp_alg scomp = { - .alloc_ctx = deflate_alloc_ctx, - .free_ctx = deflate_free_ctx, - .compress = deflate_scompress, - .decompress = deflate_sdecompress, - .base = { - .cra_name = "deflate", - .cra_driver_name = "deflate-scomp", - .cra_module = THIS_MODULE, - } +static struct acomp_alg acomp = { + .compress = deflate_compress, + .decompress = deflate_decompress, + .init = deflate_init, + .base.cra_name = "deflate", + .base.cra_driver_name = "deflate-generic", + .base.cra_flags = CRYPTO_ALG_REQ_CHAIN, + .base.cra_module = THIS_MODULE, }; static int __init deflate_mod_init(void) { - return crypto_register_scomp(&scomp); + return crypto_register_acomp(&acomp); } static void __exit deflate_mod_fini(void) { - crypto_unregister_scomp(&scomp); + crypto_unregister_acomp(&acomp); + crypto_acomp_free_streams(&deflate_streams); } subsys_initcall(deflate_mod_init); @@ -254,5 +249,7 @@ module_exit(deflate_mod_fini); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Deflate Compression Algorithm for IPCOMP"); MODULE_AUTHOR("James Morris <jmorris@intercode.com.au>"); +MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>"); +MODULE_AUTHOR("Herbert Xu <herbert@gondor.apana.org.au>"); MODULE_ALIAS_CRYPTO("deflate"); MODULE_ALIAS_CRYPTO("deflate-generic"); diff --git a/crypto/ecdsa-p1363.c b/crypto/ecdsa-p1363.c index 4454f1f8f33f..e0c55c64711c 100644 --- a/crypto/ecdsa-p1363.c +++ b/crypto/ecdsa-p1363.c @@ -21,7 +21,8 @@ static int ecdsa_p1363_verify(struct crypto_sig *tfm, const void *digest, unsigned int dlen) { struct ecdsa_p1363_ctx *ctx = crypto_sig_ctx(tfm); - unsigned int keylen = crypto_sig_keysize(ctx->child); + unsigned int keylen = DIV_ROUND_UP_POW2(crypto_sig_keysize(ctx->child), + BITS_PER_BYTE); unsigned int ndigits = DIV_ROUND_UP_POW2(keylen, sizeof(u64)); struct ecdsa_raw_sig sig; @@ -45,7 +46,8 @@ static unsigned int ecdsa_p1363_max_size(struct crypto_sig *tfm) { struct ecdsa_p1363_ctx *ctx = crypto_sig_ctx(tfm); - return 2 * crypto_sig_keysize(ctx->child); + return 2 * DIV_ROUND_UP_POW2(crypto_sig_keysize(ctx->child), + BITS_PER_BYTE); } static unsigned int ecdsa_p1363_digest_size(struct crypto_sig *tfm) diff --git a/crypto/ecdsa-x962.c b/crypto/ecdsa-x962.c index 90a04f4b9a2f..ee71594d10a0 100644 --- a/crypto/ecdsa-x962.c +++ b/crypto/ecdsa-x962.c @@ -82,7 +82,7 @@ static int ecdsa_x962_verify(struct crypto_sig *tfm, int err; sig_ctx.ndigits = DIV_ROUND_UP_POW2(crypto_sig_keysize(ctx->child), - sizeof(u64)); + sizeof(u64) * BITS_PER_BYTE); err = asn1_ber_decoder(&ecdsasignature_decoder, &sig_ctx, src, slen); if (err < 0) @@ -103,7 +103,8 @@ static unsigned int ecdsa_x962_max_size(struct crypto_sig *tfm) { struct ecdsa_x962_ctx *ctx = crypto_sig_ctx(tfm); struct sig_alg *alg = crypto_sig_alg(ctx->child); - int slen = crypto_sig_keysize(ctx->child); + int slen = DIV_ROUND_UP_POW2(crypto_sig_keysize(ctx->child), + BITS_PER_BYTE); /* * Verify takes ECDSA-Sig-Value (described in RFC 5480) as input, diff --git a/crypto/ecdsa.c b/crypto/ecdsa.c index 117526d15dde..a70b60a90a3c 100644 --- a/crypto/ecdsa.c +++ b/crypto/ecdsa.c @@ -167,7 +167,7 @@ static unsigned int ecdsa_key_size(struct crypto_sig *tfm) { struct ecc_ctx *ctx = crypto_sig_ctx(tfm); - return DIV_ROUND_UP(ctx->curve->nbits, 8); + return ctx->curve->nbits; } static unsigned int ecdsa_digest_size(struct crypto_sig *tfm) diff --git a/crypto/ecrdsa.c b/crypto/ecrdsa.c index b3dd8a3ddeb7..2c0602f0cd40 100644 --- a/crypto/ecrdsa.c +++ b/crypto/ecrdsa.c @@ -249,7 +249,7 @@ static unsigned int ecrdsa_key_size(struct crypto_sig *tfm) * Verify doesn't need any output, so it's just informational * for keyctl to determine the key bit size. */ - return ctx->pub_key.ndigits * sizeof(u64); + return ctx->pub_key.ndigits * sizeof(u64) * BITS_PER_BYTE; } static unsigned int ecrdsa_max_size(struct crypto_sig *tfm) diff --git a/crypto/essiv.c b/crypto/essiv.c index ec0ec8992c2d..cfce8ef7ab1f 100644 --- a/crypto/essiv.c +++ b/crypto/essiv.c @@ -548,8 +548,7 @@ static int essiv_create(struct crypto_template *tmpl, struct rtattr **tb) } /* record the driver name so we can instantiate this exact algo later */ - strscpy(ictx->shash_driver_name, hash_alg->base.cra_driver_name, - CRYPTO_MAX_ALG_NAME); + strscpy(ictx->shash_driver_name, hash_alg->base.cra_driver_name); /* Instance fields */ diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c index c70d163c1ac9..b5fc20a0dafc 100644 --- a/crypto/ghash-generic.c +++ b/crypto/ghash-generic.c @@ -34,14 +34,14 @@ * (https://csrc.nist.gov/publications/detail/sp/800-38d/final) */ -#include <crypto/algapi.h> #include <crypto/gf128mul.h> #include <crypto/ghash.h> #include <crypto/internal/hash.h> -#include <linux/crypto.h> -#include <linux/init.h> +#include <crypto/utils.h> +#include <linux/err.h> #include <linux/kernel.h> #include <linux/module.h> +#include <linux/string.h> static int ghash_init(struct shash_desc *desc) { @@ -82,59 +82,36 @@ static int ghash_update(struct shash_desc *desc, struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); u8 *dst = dctx->buffer; - if (dctx->bytes) { - int n = min(srclen, dctx->bytes); - u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes); - - dctx->bytes -= n; - srclen -= n; - - while (n--) - *pos++ ^= *src++; - - if (!dctx->bytes) - gf128mul_4k_lle((be128 *)dst, ctx->gf128); - } - - while (srclen >= GHASH_BLOCK_SIZE) { + do { crypto_xor(dst, src, GHASH_BLOCK_SIZE); gf128mul_4k_lle((be128 *)dst, ctx->gf128); src += GHASH_BLOCK_SIZE; srclen -= GHASH_BLOCK_SIZE; - } - - if (srclen) { - dctx->bytes = GHASH_BLOCK_SIZE - srclen; - while (srclen--) - *dst++ ^= *src++; - } + } while (srclen >= GHASH_BLOCK_SIZE); - return 0; + return srclen; } -static void ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx) +static void ghash_flush(struct shash_desc *desc, const u8 *src, + unsigned int len) { + struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); + struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); u8 *dst = dctx->buffer; - if (dctx->bytes) { - u8 *tmp = dst + (GHASH_BLOCK_SIZE - dctx->bytes); - - while (dctx->bytes--) - *tmp++ ^= 0; - + if (len) { + crypto_xor(dst, src, len); gf128mul_4k_lle((be128 *)dst, ctx->gf128); } - - dctx->bytes = 0; } -static int ghash_final(struct shash_desc *desc, u8 *dst) +static int ghash_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *dst) { struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); u8 *buf = dctx->buffer; - ghash_flush(ctx, dctx); + ghash_flush(desc, src, len); memcpy(dst, buf, GHASH_BLOCK_SIZE); return 0; @@ -151,13 +128,14 @@ static struct shash_alg ghash_alg = { .digestsize = GHASH_DIGEST_SIZE, .init = ghash_init, .update = ghash_update, - .final = ghash_final, + .finup = ghash_finup, .setkey = ghash_setkey, .descsize = sizeof(struct ghash_desc_ctx), .base = { .cra_name = "ghash", .cra_driver_name = "ghash-generic", .cra_priority = 100, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = GHASH_BLOCK_SIZE, .cra_ctxsize = sizeof(struct ghash_ctx), .cra_module = THIS_MODULE, diff --git a/crypto/hmac.c b/crypto/hmac.c index 7cec25ff9889..dfb153511865 100644 --- a/crypto/hmac.c +++ b/crypto/hmac.c @@ -146,9 +146,6 @@ static int hmac_init_tfm(struct crypto_shash *parent) if (IS_ERR(hash)) return PTR_ERR(hash); - parent->descsize = sizeof(struct shash_desc) + - crypto_shash_descsize(hash); - tctx->hash = hash; return 0; } @@ -222,6 +219,7 @@ static int hmac_create(struct crypto_template *tmpl, struct rtattr **tb) inst->alg.digestsize = ds; inst->alg.statesize = ss; + inst->alg.descsize = sizeof(struct shash_desc) + salg->descsize; inst->alg.init = hmac_init; inst->alg.update = hmac_update; inst->alg.final = hmac_final; diff --git a/crypto/internal.h b/crypto/internal.h index 11567ea24fc3..2ed79bf208ca 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -46,6 +46,7 @@ struct crypto_type { unsigned int maskclear; unsigned int maskset; unsigned int tfmsize; + unsigned int algsize; }; enum { @@ -128,7 +129,6 @@ void *crypto_create_tfm_node(struct crypto_alg *alg, const struct crypto_type *frontend, int node); void *crypto_clone_tfm(const struct crypto_type *frontend, struct crypto_tfm *otfm); -void crypto_destroy_alg(struct crypto_alg *alg); static inline void *crypto_create_tfm(struct crypto_alg *alg, const struct crypto_type *frontend) @@ -163,6 +163,8 @@ static inline struct crypto_alg *crypto_alg_get(struct crypto_alg *alg) return alg; } +void crypto_destroy_alg(struct crypto_alg *alg); + static inline void crypto_alg_put(struct crypto_alg *alg) { if (refcount_dec_and_test(&alg->cra_refcnt)) diff --git a/crypto/kpp.c b/crypto/kpp.c index ecc63a1a948d..2e0cefe7a25f 100644 --- a/crypto/kpp.c +++ b/crypto/kpp.c @@ -80,6 +80,7 @@ static const struct crypto_type crypto_kpp_type = { .maskset = CRYPTO_ALG_TYPE_MASK, .type = CRYPTO_ALG_TYPE_KPP, .tfmsize = offsetof(struct crypto_kpp, base), + .algsize = offsetof(struct kpp_alg, base), }; struct crypto_kpp *crypto_alloc_kpp(const char *alg_name, u32 type, u32 mask) diff --git a/crypto/lskcipher.c b/crypto/lskcipher.c index cdb4897c63e6..c2e2c38b5aa8 100644 --- a/crypto/lskcipher.c +++ b/crypto/lskcipher.c @@ -294,6 +294,7 @@ static const struct crypto_type crypto_lskcipher_type = { .maskset = CRYPTO_ALG_TYPE_MASK, .type = CRYPTO_ALG_TYPE_LSKCIPHER, .tfmsize = offsetof(struct crypto_lskcipher, base), + .algsize = offsetof(struct lskcipher_alg, co.base), }; static void crypto_lskcipher_exit_tfm_sg(struct crypto_tfm *tfm) diff --git a/crypto/lz4.c b/crypto/lz4.c index 82588607fb2e..9661ed01692f 100644 --- a/crypto/lz4.c +++ b/crypto/lz4.c @@ -12,10 +12,6 @@ #include <linux/lz4.h> #include <crypto/internal/scompress.h> -struct lz4_ctx { - void *lz4_comp_mem; -}; - static void *lz4_alloc_ctx(void) { void *ctx; diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c index 997e76c0183a..a637fddc1ccd 100644 --- a/crypto/lz4hc.c +++ b/crypto/lz4hc.c @@ -10,10 +10,6 @@ #include <linux/vmalloc.h> #include <linux/lz4.h> -struct lz4hc_ctx { - void *lz4hc_comp_mem; -}; - static void *lz4hc_alloc_ctx(void) { void *ctx; diff --git a/crypto/lzo-rle.c b/crypto/lzo-rle.c index b1350ae278b8..e7efcf107179 100644 --- a/crypto/lzo-rle.c +++ b/crypto/lzo-rle.c @@ -9,10 +9,6 @@ #include <linux/module.h> #include <linux/slab.h> -struct lzorle_ctx { - void *lzorle_comp_mem; -}; - static void *lzorle_alloc_ctx(void) { void *ctx; diff --git a/crypto/lzo.c b/crypto/lzo.c index dfe5a07ca35f..f1b36a1ca6f6 100644 --- a/crypto/lzo.c +++ b/crypto/lzo.c @@ -9,10 +9,6 @@ #include <linux/module.h> #include <linux/slab.h> -struct lzo_ctx { - void *lzo_comp_mem; -}; - static void *lzo_alloc_ctx(void) { void *ctx; diff --git a/crypto/md5.c b/crypto/md5.c index 72c0c46fb5ee..994005cd977d 100644 --- a/crypto/md5.c +++ b/crypto/md5.c @@ -17,11 +17,9 @@ */ #include <crypto/internal/hash.h> #include <crypto/md5.h> -#include <linux/init.h> +#include <linux/kernel.h> #include <linux/module.h> #include <linux/string.h> -#include <linux/types.h> -#include <asm/byteorder.h> const u8 md5_zero_message_hash[MD5_DIGEST_SIZE] = { 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, @@ -120,10 +118,11 @@ static void md5_transform(__u32 *hash, __u32 const *in) hash[3] += d; } -static inline void md5_transform_helper(struct md5_state *ctx) +static inline void md5_transform_helper(struct md5_state *ctx, + u32 block[MD5_BLOCK_WORDS]) { - le32_to_cpu_array(ctx->block, sizeof(ctx->block) / sizeof(u32)); - md5_transform(ctx->hash, ctx->block); + le32_to_cpu_array(block, MD5_BLOCK_WORDS); + md5_transform(ctx->hash, block); } static int md5_init(struct shash_desc *desc) @@ -142,76 +141,53 @@ static int md5_init(struct shash_desc *desc) static int md5_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct md5_state *mctx = shash_desc_ctx(desc); - const u32 avail = sizeof(mctx->block) - (mctx->byte_count & 0x3f); + u32 block[MD5_BLOCK_WORDS]; mctx->byte_count += len; - - if (avail > len) { - memcpy((char *)mctx->block + (sizeof(mctx->block) - avail), - data, len); - return 0; - } - - memcpy((char *)mctx->block + (sizeof(mctx->block) - avail), - data, avail); - - md5_transform_helper(mctx); - data += avail; - len -= avail; - - while (len >= sizeof(mctx->block)) { - memcpy(mctx->block, data, sizeof(mctx->block)); - md5_transform_helper(mctx); - data += sizeof(mctx->block); - len -= sizeof(mctx->block); - } - - memcpy(mctx->block, data, len); - - return 0; + do { + memcpy(block, data, sizeof(block)); + md5_transform_helper(mctx, block); + data += sizeof(block); + len -= sizeof(block); + } while (len >= sizeof(block)); + memzero_explicit(block, sizeof(block)); + mctx->byte_count -= len; + return len; } -static int md5_final(struct shash_desc *desc, u8 *out) +static int md5_finup(struct shash_desc *desc, const u8 *data, unsigned int len, + u8 *out) { struct md5_state *mctx = shash_desc_ctx(desc); - const unsigned int offset = mctx->byte_count & 0x3f; - char *p = (char *)mctx->block + offset; - int padding = 56 - (offset + 1); + u32 block[MD5_BLOCK_WORDS]; + unsigned int offset; + int padding; + char *p; + + memcpy(block, data, len); + + offset = len; + p = (char *)block + offset; + padding = 56 - (offset + 1); *p++ = 0x80; if (padding < 0) { memset(p, 0x00, padding + sizeof (u64)); - md5_transform_helper(mctx); - p = (char *)mctx->block; + md5_transform_helper(mctx, block); + p = (char *)block; padding = 56; } memset(p, 0, padding); - mctx->block[14] = mctx->byte_count << 3; - mctx->block[15] = mctx->byte_count >> 29; - le32_to_cpu_array(mctx->block, (sizeof(mctx->block) - - sizeof(u64)) / sizeof(u32)); - md5_transform(mctx->hash, mctx->block); + mctx->byte_count += len; + block[14] = mctx->byte_count << 3; + block[15] = mctx->byte_count >> 29; + le32_to_cpu_array(block, (sizeof(block) - sizeof(u64)) / sizeof(u32)); + md5_transform(mctx->hash, block); + memzero_explicit(block, sizeof(block)); cpu_to_le32_array(mctx->hash, sizeof(mctx->hash) / sizeof(u32)); memcpy(out, mctx->hash, sizeof(mctx->hash)); - memset(mctx, 0, sizeof(*mctx)); - - return 0; -} - -static int md5_export(struct shash_desc *desc, void *out) -{ - struct md5_state *ctx = shash_desc_ctx(desc); - - memcpy(out, ctx, sizeof(*ctx)); - return 0; -} - -static int md5_import(struct shash_desc *desc, const void *in) -{ - struct md5_state *ctx = shash_desc_ctx(desc); - memcpy(ctx, in, sizeof(*ctx)); return 0; } @@ -219,14 +195,12 @@ static struct shash_alg alg = { .digestsize = MD5_DIGEST_SIZE, .init = md5_init, .update = md5_update, - .final = md5_final, - .export = md5_export, - .import = md5_import, - .descsize = sizeof(struct md5_state), - .statesize = sizeof(struct md5_state), + .finup = md5_finup, + .descsize = MD5_STATE_SIZE, .base = { .cra_name = "md5", .cra_driver_name = "md5-generic", + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = MD5_HMAC_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/crypto/poly1305.c b/crypto/poly1305.c new file mode 100644 index 000000000000..e0436bdc462b --- /dev/null +++ b/crypto/poly1305.c @@ -0,0 +1,152 @@ +/* + * Crypto API wrapper for the Poly1305 library functions + * + * Copyright (C) 2015 Martin Willi + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include <crypto/algapi.h> +#include <crypto/internal/hash.h> +#include <crypto/internal/poly1305.h> +#include <linux/crypto.h> +#include <linux/kernel.h> +#include <linux/module.h> + +struct crypto_poly1305_desc_ctx { + struct poly1305_desc_ctx base; + u8 key[POLY1305_KEY_SIZE]; + unsigned int keysize; +}; + +static int crypto_poly1305_init(struct shash_desc *desc) +{ + struct crypto_poly1305_desc_ctx *dctx = shash_desc_ctx(desc); + + dctx->keysize = 0; + return 0; +} + +static int crypto_poly1305_update(struct shash_desc *desc, + const u8 *src, unsigned int srclen, bool arch) +{ + struct crypto_poly1305_desc_ctx *dctx = shash_desc_ctx(desc); + unsigned int bytes; + + /* + * The key is passed as the first 32 "data" bytes. The actual + * poly1305_init() can be called only once the full key is available. + */ + if (dctx->keysize < POLY1305_KEY_SIZE) { + bytes = min(srclen, POLY1305_KEY_SIZE - dctx->keysize); + memcpy(&dctx->key[dctx->keysize], src, bytes); + dctx->keysize += bytes; + if (dctx->keysize < POLY1305_KEY_SIZE) + return 0; + if (arch) + poly1305_init(&dctx->base, dctx->key); + else + poly1305_init_generic(&dctx->base, dctx->key); + src += bytes; + srclen -= bytes; + } + + if (arch) + poly1305_update(&dctx->base, src, srclen); + else + poly1305_update_generic(&dctx->base, src, srclen); + + return 0; +} + +static int crypto_poly1305_update_generic(struct shash_desc *desc, + const u8 *src, unsigned int srclen) +{ + return crypto_poly1305_update(desc, src, srclen, false); +} + +static int crypto_poly1305_update_arch(struct shash_desc *desc, + const u8 *src, unsigned int srclen) +{ + return crypto_poly1305_update(desc, src, srclen, true); +} + +static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst, bool arch) +{ + struct crypto_poly1305_desc_ctx *dctx = shash_desc_ctx(desc); + + if (unlikely(dctx->keysize != POLY1305_KEY_SIZE)) + return -ENOKEY; + + if (arch) + poly1305_final(&dctx->base, dst); + else + poly1305_final_generic(&dctx->base, dst); + memzero_explicit(&dctx->key, sizeof(dctx->key)); + return 0; +} + +static int crypto_poly1305_final_generic(struct shash_desc *desc, u8 *dst) +{ + return crypto_poly1305_final(desc, dst, false); +} + +static int crypto_poly1305_final_arch(struct shash_desc *desc, u8 *dst) +{ + return crypto_poly1305_final(desc, dst, true); +} + +static struct shash_alg poly1305_algs[] = { + { + .base.cra_name = "poly1305", + .base.cra_driver_name = "poly1305-generic", + .base.cra_priority = 100, + .base.cra_blocksize = POLY1305_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = POLY1305_DIGEST_SIZE, + .init = crypto_poly1305_init, + .update = crypto_poly1305_update_generic, + .final = crypto_poly1305_final_generic, + .descsize = sizeof(struct crypto_poly1305_desc_ctx), + }, + { + .base.cra_name = "poly1305", + .base.cra_driver_name = "poly1305-" __stringify(ARCH), + .base.cra_priority = 300, + .base.cra_blocksize = POLY1305_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = POLY1305_DIGEST_SIZE, + .init = crypto_poly1305_init, + .update = crypto_poly1305_update_arch, + .final = crypto_poly1305_final_arch, + .descsize = sizeof(struct crypto_poly1305_desc_ctx), + }, +}; + +static int num_algs; + +static int __init poly1305_mod_init(void) +{ + /* register the arch flavours only if they differ from generic */ + num_algs = poly1305_is_arch_optimized() ? 2 : 1; + + return crypto_register_shashes(poly1305_algs, num_algs); +} + +static void __exit poly1305_mod_exit(void) +{ + crypto_unregister_shashes(poly1305_algs, num_algs); +} + +subsys_initcall(poly1305_mod_init); +module_exit(poly1305_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Martin Willi <martin@strongswan.org>"); +MODULE_DESCRIPTION("Crypto API wrapper for the Poly1305 library functions"); +MODULE_ALIAS_CRYPTO("poly1305"); +MODULE_ALIAS_CRYPTO("poly1305-generic"); +MODULE_ALIAS_CRYPTO("poly1305-" __stringify(ARCH)); diff --git a/crypto/poly1305_generic.c b/crypto/poly1305_generic.c deleted file mode 100644 index e6f29a98725a..000000000000 --- a/crypto/poly1305_generic.c +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Poly1305 authenticator algorithm, RFC7539 - * - * Copyright (C) 2015 Martin Willi - * - * Based on public domain code by Andrew Moon and Daniel J. Bernstein. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - */ - -#include <crypto/algapi.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/poly1305.h> -#include <linux/crypto.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/unaligned.h> - -static int crypto_poly1305_init(struct shash_desc *desc) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - poly1305_core_init(&dctx->h); - dctx->buflen = 0; - dctx->rset = 0; - dctx->sset = false; - - return 0; -} - -static unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx, - const u8 *src, unsigned int srclen) -{ - if (!dctx->sset) { - if (!dctx->rset && srclen >= POLY1305_BLOCK_SIZE) { - poly1305_core_setkey(&dctx->core_r, src); - src += POLY1305_BLOCK_SIZE; - srclen -= POLY1305_BLOCK_SIZE; - dctx->rset = 2; - } - if (srclen >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(src + 0); - dctx->s[1] = get_unaligned_le32(src + 4); - dctx->s[2] = get_unaligned_le32(src + 8); - dctx->s[3] = get_unaligned_le32(src + 12); - src += POLY1305_BLOCK_SIZE; - srclen -= POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - } - return srclen; -} - -static void poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, - unsigned int srclen) -{ - unsigned int datalen; - - if (unlikely(!dctx->sset)) { - datalen = crypto_poly1305_setdesckey(dctx, src, srclen); - src += srclen - datalen; - srclen = datalen; - } - - poly1305_core_blocks(&dctx->h, &dctx->core_r, src, - srclen / POLY1305_BLOCK_SIZE, 1); -} - -static int crypto_poly1305_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - unsigned int bytes; - - if (unlikely(dctx->buflen)) { - bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - srclen -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - poly1305_blocks(dctx, dctx->buf, - POLY1305_BLOCK_SIZE); - dctx->buflen = 0; - } - } - - if (likely(srclen >= POLY1305_BLOCK_SIZE)) { - poly1305_blocks(dctx, src, srclen); - src += srclen - (srclen % POLY1305_BLOCK_SIZE); - srclen %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(srclen)) { - dctx->buflen = srclen; - memcpy(dctx->buf, src, srclen); - } - - return 0; -} - -static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (unlikely(!dctx->sset)) - return -ENOKEY; - - poly1305_final_generic(dctx, dst); - return 0; -} - -static struct shash_alg poly1305_alg = { - .digestsize = POLY1305_DIGEST_SIZE, - .init = crypto_poly1305_init, - .update = crypto_poly1305_update, - .final = crypto_poly1305_final, - .descsize = sizeof(struct poly1305_desc_ctx), - .base = { - .cra_name = "poly1305", - .cra_driver_name = "poly1305-generic", - .cra_priority = 100, - .cra_blocksize = POLY1305_BLOCK_SIZE, - .cra_module = THIS_MODULE, - }, -}; - -static int __init poly1305_mod_init(void) -{ - return crypto_register_shash(&poly1305_alg); -} - -static void __exit poly1305_mod_exit(void) -{ - crypto_unregister_shash(&poly1305_alg); -} - -subsys_initcall(poly1305_mod_init); -module_exit(poly1305_mod_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Martin Willi <martin@strongswan.org>"); -MODULE_DESCRIPTION("Poly1305 authenticator"); -MODULE_ALIAS_CRYPTO("poly1305"); -MODULE_ALIAS_CRYPTO("poly1305-generic"); diff --git a/crypto/rng.c b/crypto/rng.c index 9d8804e46422..b8ae6ebc091d 100644 --- a/crypto/rng.c +++ b/crypto/rng.c @@ -98,6 +98,7 @@ static const struct crypto_type crypto_rng_type = { .maskset = CRYPTO_ALG_TYPE_MASK, .type = CRYPTO_ALG_TYPE_RNG, .tfmsize = offsetof(struct crypto_rng, base), + .algsize = offsetof(struct rng_alg, base), }; struct crypto_rng *crypto_alloc_rng(const char *alg_name, u32 type, u32 mask) diff --git a/crypto/rsassa-pkcs1.c b/crypto/rsassa-pkcs1.c index d01ac75635e0..94fa5e9600e7 100644 --- a/crypto/rsassa-pkcs1.c +++ b/crypto/rsassa-pkcs1.c @@ -301,7 +301,7 @@ static unsigned int rsassa_pkcs1_key_size(struct crypto_sig *tfm) { struct rsassa_pkcs1_ctx *ctx = crypto_sig_ctx(tfm); - return ctx->key_size; + return ctx->key_size * BITS_PER_BYTE; } static int rsassa_pkcs1_set_pub_key(struct crypto_sig *tfm, diff --git a/crypto/scompress.c b/crypto/scompress.c index ffeedcf20b0f..15148c58d648 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -7,9 +7,9 @@ * Author: Giovanni Cabiddu <giovanni.cabiddu@intel.com> */ -#include <crypto/internal/acompress.h> #include <crypto/internal/scompress.h> #include <crypto/scatterwalk.h> +#include <linux/cpumask.h> #include <linux/cryptouser.h> #include <linux/err.h> #include <linux/highmem.h> @@ -20,20 +20,17 @@ #include <linux/seq_file.h> #include <linux/slab.h> #include <linux/string.h> -#include <linux/vmalloc.h> +#include <linux/workqueue.h> #include <net/netlink.h> #include "compress.h" -#define SCOMP_SCRATCH_SIZE 65400 - struct scomp_scratch { spinlock_t lock; union { void *src; unsigned long saddr; }; - void *dst; }; static DEFINE_PER_CPU(struct scomp_scratch, scomp_scratch) = { @@ -44,6 +41,10 @@ static const struct crypto_type crypto_scomp_type; static int scomp_scratch_users; static DEFINE_MUTEX(scomp_lock); +static cpumask_t scomp_scratch_want; +static void scomp_scratch_workfn(struct work_struct *work); +static DECLARE_WORK(scomp_scratch_work, scomp_scratch_workfn); + static int __maybe_unused crypto_scomp_report( struct sk_buff *skb, struct crypto_alg *alg) { @@ -74,82 +75,48 @@ static void crypto_scomp_free_scratches(void) scratch = per_cpu_ptr(&scomp_scratch, i); free_page(scratch->saddr); - vfree(scratch->dst); scratch->src = NULL; - scratch->dst = NULL; } } -static int crypto_scomp_alloc_scratches(void) +static int scomp_alloc_scratch(struct scomp_scratch *scratch, int cpu) { - struct scomp_scratch *scratch; - int i; - - for_each_possible_cpu(i) { - struct page *page; - void *mem; - - scratch = per_cpu_ptr(&scomp_scratch, i); + int node = cpu_to_node(cpu); + struct page *page; - page = alloc_pages_node(cpu_to_node(i), GFP_KERNEL, 0); - if (!page) - goto error; - scratch->src = page_address(page); - mem = vmalloc_node(SCOMP_SCRATCH_SIZE, cpu_to_node(i)); - if (!mem) - goto error; - scratch->dst = mem; - } + page = alloc_pages_node(node, GFP_KERNEL, 0); + if (!page) + return -ENOMEM; + spin_lock_bh(&scratch->lock); + scratch->src = page_address(page); + spin_unlock_bh(&scratch->lock); return 0; -error: - crypto_scomp_free_scratches(); - return -ENOMEM; } -static void scomp_free_streams(struct scomp_alg *alg) +static void scomp_scratch_workfn(struct work_struct *work) { - struct crypto_acomp_stream __percpu *stream = alg->stream; - int i; + int cpu; - alg->stream = NULL; - if (!stream) - return; + for_each_cpu(cpu, &scomp_scratch_want) { + struct scomp_scratch *scratch; - for_each_possible_cpu(i) { - struct crypto_acomp_stream *ps = per_cpu_ptr(stream, i); - - if (IS_ERR_OR_NULL(ps->ctx)) + scratch = per_cpu_ptr(&scomp_scratch, cpu); + if (scratch->src) + continue; + if (scomp_alloc_scratch(scratch, cpu)) break; - alg->free_ctx(ps->ctx); + cpumask_clear_cpu(cpu, &scomp_scratch_want); } - - free_percpu(stream); } -static int scomp_alloc_streams(struct scomp_alg *alg) +static int crypto_scomp_alloc_scratches(void) { - struct crypto_acomp_stream __percpu *stream; - int i; - - stream = alloc_percpu(struct crypto_acomp_stream); - if (!stream) - return -ENOMEM; - - alg->stream = stream; - - for_each_possible_cpu(i) { - struct crypto_acomp_stream *ps = per_cpu_ptr(stream, i); - - ps->ctx = alg->alloc_ctx(); - if (IS_ERR(ps->ctx)) { - scomp_free_streams(alg); - return PTR_ERR(ps->ctx); - } + unsigned int i = cpumask_first(cpu_possible_mask); + struct scomp_scratch *scratch; - spin_lock_init(&ps->lock); - } - return 0; + scratch = per_cpu_ptr(&scomp_scratch, i); + return scomp_alloc_scratch(scratch, i); } static int crypto_scomp_init_tfm(struct crypto_tfm *tfm) @@ -158,11 +125,9 @@ static int crypto_scomp_init_tfm(struct crypto_tfm *tfm) int ret = 0; mutex_lock(&scomp_lock); - if (!alg->stream) { - ret = scomp_alloc_streams(alg); - if (ret) - goto unlock; - } + ret = crypto_acomp_alloc_streams(&alg->streams); + if (ret) + goto unlock; if (!scomp_scratch_users++) { ret = crypto_scomp_alloc_scratches(); if (ret) @@ -174,13 +139,40 @@ unlock: return ret; } +static struct scomp_scratch *scomp_lock_scratch(void) __acquires(scratch) +{ + int cpu = raw_smp_processor_id(); + struct scomp_scratch *scratch; + + scratch = per_cpu_ptr(&scomp_scratch, cpu); + spin_lock(&scratch->lock); + if (likely(scratch->src)) + return scratch; + spin_unlock(&scratch->lock); + + cpumask_set_cpu(cpu, &scomp_scratch_want); + schedule_work(&scomp_scratch_work); + + scratch = per_cpu_ptr(&scomp_scratch, cpumask_first(cpu_possible_mask)); + spin_lock(&scratch->lock); + return scratch; +} + +static inline void scomp_unlock_scratch(struct scomp_scratch *scratch) + __releases(scratch) +{ + spin_unlock(&scratch->lock); +} + static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) { - struct scomp_scratch *scratch = raw_cpu_ptr(&scomp_scratch); struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); struct crypto_scomp **tfm_ctx = acomp_tfm_ctx(tfm); + bool src_isvirt = acomp_request_src_isvirt(req); + bool dst_isvirt = acomp_request_dst_isvirt(req); struct crypto_scomp *scomp = *tfm_ctx; struct crypto_acomp_stream *stream; + struct scomp_scratch *scratch; unsigned int slen = req->slen; unsigned int dlen = req->dlen; struct page *spage, *dpage; @@ -197,15 +189,32 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) if (!req->dst || !dlen) return -EINVAL; - if (acomp_request_src_isvirt(req)) + if (dst_isvirt) + dst = req->dvirt; + else { + if (dlen <= req->dst->length) { + dpage = sg_page(req->dst); + doff = req->dst->offset; + } else + return -ENOSYS; + + dpage = nth_page(dpage, doff / PAGE_SIZE); + doff = offset_in_page(doff); + + n = (dlen - 1) / PAGE_SIZE; + n += (offset_in_page(dlen - 1) + doff) / PAGE_SIZE; + if (PageHighMem(dpage + n) && + size_add(doff, dlen) > PAGE_SIZE) + return -ENOSYS; + dst = kmap_local_page(dpage) + doff; + } + + if (src_isvirt) src = req->svirt; else { - src = scratch->src; + src = NULL; do { - if (acomp_request_src_isfolio(req)) { - spage = folio_page(req->sfolio, 0); - soff = req->soff; - } else if (slen <= req->src->length) { + if (slen <= req->src->length) { spage = sg_page(req->src); soff = req->src->offset; } else @@ -223,59 +232,37 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) } while (0); } - if (acomp_request_dst_isvirt(req)) - dst = req->dvirt; - else { - unsigned int max = SCOMP_SCRATCH_SIZE; - - dst = scratch->dst; - do { - if (acomp_request_dst_isfolio(req)) { - dpage = folio_page(req->dfolio, 0); - doff = req->doff; - } else if (dlen <= req->dst->length) { - dpage = sg_page(req->dst); - doff = req->dst->offset; - } else - break; - - dpage = nth_page(dpage, doff / PAGE_SIZE); - doff = offset_in_page(doff); + stream = crypto_acomp_lock_stream_bh(&crypto_scomp_alg(scomp)->streams); - n = (dlen - 1) / PAGE_SIZE; - n += (offset_in_page(dlen - 1) + doff) / PAGE_SIZE; - if (PageHighMem(nth_page(dpage, n)) && - size_add(doff, dlen) > PAGE_SIZE) - break; - dst = kmap_local_page(dpage) + doff; - max = dlen; - } while (0); - dlen = min(dlen, max); - } - - spin_lock_bh(&scratch->lock); + if (!src_isvirt && !src) { + const u8 *src; - if (src == scratch->src) + scratch = scomp_lock_scratch(); + src = scratch->src; memcpy_from_sglist(scratch->src, req->src, 0, slen); - stream = raw_cpu_ptr(crypto_scomp_alg(scomp)->stream); - spin_lock(&stream->lock); - if (dir) + if (dir) + ret = crypto_scomp_compress(scomp, src, slen, + dst, &dlen, stream->ctx); + else + ret = crypto_scomp_decompress(scomp, src, slen, + dst, &dlen, stream->ctx); + + scomp_unlock_scratch(scratch); + } else if (dir) ret = crypto_scomp_compress(scomp, src, slen, dst, &dlen, stream->ctx); else ret = crypto_scomp_decompress(scomp, src, slen, dst, &dlen, stream->ctx); - if (dst == scratch->dst) - memcpy_to_sglist(req->dst, 0, dst, dlen); - - spin_unlock(&stream->lock); - spin_unlock_bh(&scratch->lock); + crypto_acomp_unlock_stream_bh(stream); req->dlen = dlen; - if (!acomp_request_dst_isvirt(req) && dst != scratch->dst) { + if (!src_isvirt && src) + kunmap_local(src); + if (!dst_isvirt) { kunmap_local(dst); dlen += doff; for (;;) { @@ -286,34 +273,18 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) dpage = nth_page(dpage, 1); } } - if (!acomp_request_src_isvirt(req) && src != scratch->src) - kunmap_local(src); return ret; } -static int scomp_acomp_chain(struct acomp_req *req, int dir) -{ - struct acomp_req *r2; - int err; - - err = scomp_acomp_comp_decomp(req, dir); - req->base.err = err; - - list_for_each_entry(r2, &req->base.list, base.list) - r2->base.err = scomp_acomp_comp_decomp(r2, dir); - - return err; -} - static int scomp_acomp_compress(struct acomp_req *req) { - return scomp_acomp_chain(req, 1); + return scomp_acomp_comp_decomp(req, 1); } static int scomp_acomp_decompress(struct acomp_req *req) { - return scomp_acomp_chain(req, 0); + return scomp_acomp_comp_decomp(req, 0); } static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm) @@ -322,6 +293,7 @@ static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm) crypto_free_scomp(*ctx); + flush_work(&scomp_scratch_work); mutex_lock(&scomp_lock); if (!--scomp_scratch_users) crypto_scomp_free_scratches(); @@ -355,7 +327,9 @@ int crypto_init_scomp_ops_async(struct crypto_tfm *tfm) static void crypto_scomp_destroy(struct crypto_alg *alg) { - scomp_free_streams(__crypto_scomp_alg(alg)); + struct scomp_alg *scomp = __crypto_scomp_alg(alg); + + crypto_acomp_free_streams(&scomp->streams); } static const struct crypto_type crypto_scomp_type = { @@ -372,6 +346,7 @@ static const struct crypto_type crypto_scomp_type = { .maskset = CRYPTO_ALG_TYPE_MASK, .type = CRYPTO_ALG_TYPE_SCOMPRESS, .tfmsize = offsetof(struct crypto_scomp, base), + .algsize = offsetof(struct scomp_alg, base), }; static void scomp_prepare_alg(struct scomp_alg *alg) diff --git a/crypto/sha1_generic.c b/crypto/sha1_generic.c index 325b57fe28dc..7a3c837923b5 100644 --- a/crypto/sha1_generic.c +++ b/crypto/sha1_generic.c @@ -12,13 +12,11 @@ * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> */ #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> #include <crypto/sha1.h> #include <crypto/sha1_base.h> -#include <asm/byteorder.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE] = { 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, @@ -39,38 +37,31 @@ static void sha1_generic_block_fn(struct sha1_state *sst, u8 const *src, memzero_explicit(temp, sizeof(temp)); } -int crypto_sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int crypto_sha1_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - return sha1_base_do_update(desc, data, len, sha1_generic_block_fn); + return sha1_base_do_update_blocks(desc, data, len, + sha1_generic_block_fn); } -EXPORT_SYMBOL(crypto_sha1_update); -static int sha1_final(struct shash_desc *desc, u8 *out) +static int crypto_sha1_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - sha1_base_do_finalize(desc, sha1_generic_block_fn); + sha1_base_do_finup(desc, data, len, sha1_generic_block_fn); return sha1_base_finish(desc, out); } -int crypto_sha1_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha1_base_do_update(desc, data, len, sha1_generic_block_fn); - return sha1_final(desc, out); -} -EXPORT_SYMBOL(crypto_sha1_finup); - static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = crypto_sha1_update, - .final = sha1_final, .finup = crypto_sha1_finup, - .descsize = sizeof(struct sha1_state), + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name= "sha1-generic", .cra_priority = 100, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/crypto/sha256_generic.c b/crypto/sha256_generic.c index b00521f1a6d4..05084e5bbaec 100644 --- a/crypto/sha256_generic.c +++ b/crypto/sha256_generic.c @@ -8,14 +8,10 @@ * SHA224 Support Copyright 2007 Intel Corporation <jonathan.lynch@intel.com> */ #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> #include <crypto/sha2.h> #include <crypto/sha256_base.h> -#include <asm/byteorder.h> -#include <linux/unaligned.h> +#include <linux/kernel.h> +#include <linux/module.h> const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE] = { 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47, @@ -33,42 +29,37 @@ const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE] = { }; EXPORT_SYMBOL_GPL(sha256_zero_message_hash); -int crypto_sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static void sha256_block(struct crypto_sha256_state *sctx, const u8 *input, + int blocks) { - sha256_update(shash_desc_ctx(desc), data, len); - return 0; + sha256_transform_blocks(sctx, input, blocks); } -EXPORT_SYMBOL(crypto_sha256_update); -static int crypto_sha256_final(struct shash_desc *desc, u8 *out) +static int crypto_sha256_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - if (crypto_shash_digestsize(desc->tfm) == SHA224_DIGEST_SIZE) - sha224_final(shash_desc_ctx(desc), out); - else - sha256_final(shash_desc_ctx(desc), out); - return 0; + return sha256_base_do_update_blocks(desc, data, len, sha256_block); } -int crypto_sha256_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash) +static int crypto_sha256_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *hash) { - sha256_update(shash_desc_ctx(desc), data, len); - return crypto_sha256_final(desc, hash); + sha256_base_do_finup(desc, data, len, sha256_block); + return sha256_base_finish(desc, hash); } -EXPORT_SYMBOL(crypto_sha256_finup); static struct shash_alg sha256_algs[2] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = crypto_sha256_update, - .final = crypto_sha256_final, .finup = crypto_sha256_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name= "sha256-generic", .cra_priority = 100, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -76,13 +67,14 @@ static struct shash_alg sha256_algs[2] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = crypto_sha256_update, - .final = crypto_sha256_final, .finup = crypto_sha256_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name= "sha224-generic", .cra_priority = 100, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/crypto/sha3_generic.c b/crypto/sha3_generic.c index b103642b56ea..41d1e506e6de 100644 --- a/crypto/sha3_generic.c +++ b/crypto/sha3_generic.c @@ -9,10 +9,10 @@ * Ard Biesheuvel <ard.biesheuvel@linaro.org> */ #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/types.h> #include <crypto/sha3.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> #include <linux/unaligned.h> /* @@ -161,68 +161,51 @@ static void keccakf(u64 st[25]) int crypto_sha3_init(struct shash_desc *desc) { struct sha3_state *sctx = shash_desc_ctx(desc); - unsigned int digest_size = crypto_shash_digestsize(desc->tfm); - - sctx->rsiz = 200 - 2 * digest_size; - sctx->rsizw = sctx->rsiz / 8; - sctx->partial = 0; memset(sctx->st, 0, sizeof(sctx->st)); return 0; } EXPORT_SYMBOL(crypto_sha3_init); -int crypto_sha3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int crypto_sha3_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { + unsigned int rsiz = crypto_shash_blocksize(desc->tfm); struct sha3_state *sctx = shash_desc_ctx(desc); - unsigned int done; - const u8 *src; - - done = 0; - src = data; - - if ((sctx->partial + len) > (sctx->rsiz - 1)) { - if (sctx->partial) { - done = -sctx->partial; - memcpy(sctx->buf + sctx->partial, data, - done + sctx->rsiz); - src = sctx->buf; - } + unsigned int rsizw = rsiz / 8; - do { - unsigned int i; + do { + int i; - for (i = 0; i < sctx->rsizw; i++) - sctx->st[i] ^= get_unaligned_le64(src + 8 * i); - keccakf(sctx->st); + for (i = 0; i < rsizw; i++) + sctx->st[i] ^= get_unaligned_le64(data + 8 * i); + keccakf(sctx->st); - done += sctx->rsiz; - src = data + done; - } while (done + (sctx->rsiz - 1) < len); - - sctx->partial = 0; - } - memcpy(sctx->buf + sctx->partial, src, len - done); - sctx->partial += (len - done); - - return 0; + data += rsiz; + len -= rsiz; + } while (len >= rsiz); + return len; } -EXPORT_SYMBOL(crypto_sha3_update); -int crypto_sha3_final(struct shash_desc *desc, u8 *out) +static int crypto_sha3_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - struct sha3_state *sctx = shash_desc_ctx(desc); - unsigned int i, inlen = sctx->partial; unsigned int digest_size = crypto_shash_digestsize(desc->tfm); + unsigned int rsiz = crypto_shash_blocksize(desc->tfm); + struct sha3_state *sctx = shash_desc_ctx(desc); + __le64 block[SHA3_224_BLOCK_SIZE / 8] = {}; __le64 *digest = (__le64 *)out; + unsigned int rsizw = rsiz / 8; + u8 *p; + int i; - sctx->buf[inlen++] = 0x06; - memset(sctx->buf + inlen, 0, sctx->rsiz - inlen); - sctx->buf[sctx->rsiz - 1] |= 0x80; + p = memcpy(block, src, len); + p[len++] = 0x06; + p[rsiz - 1] |= 0x80; - for (i = 0; i < sctx->rsizw; i++) - sctx->st[i] ^= get_unaligned_le64(sctx->buf + 8 * i); + for (i = 0; i < rsizw; i++) + sctx->st[i] ^= le64_to_cpu(block[i]); + memzero_explicit(block, sizeof(block)); keccakf(sctx->st); @@ -232,49 +215,51 @@ int crypto_sha3_final(struct shash_desc *desc, u8 *out) if (digest_size & 4) put_unaligned_le32(sctx->st[i], (__le32 *)digest); - memset(sctx, 0, sizeof(*sctx)); return 0; } -EXPORT_SYMBOL(crypto_sha3_final); static struct shash_alg algs[] = { { .digestsize = SHA3_224_DIGEST_SIZE, .init = crypto_sha3_init, .update = crypto_sha3_update, - .final = crypto_sha3_final, - .descsize = sizeof(struct sha3_state), + .finup = crypto_sha3_finup, + .descsize = SHA3_STATE_SIZE, .base.cra_name = "sha3-224", .base.cra_driver_name = "sha3-224-generic", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = SHA3_224_BLOCK_SIZE, .base.cra_module = THIS_MODULE, }, { .digestsize = SHA3_256_DIGEST_SIZE, .init = crypto_sha3_init, .update = crypto_sha3_update, - .final = crypto_sha3_final, - .descsize = sizeof(struct sha3_state), + .finup = crypto_sha3_finup, + .descsize = SHA3_STATE_SIZE, .base.cra_name = "sha3-256", .base.cra_driver_name = "sha3-256-generic", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = SHA3_256_BLOCK_SIZE, .base.cra_module = THIS_MODULE, }, { .digestsize = SHA3_384_DIGEST_SIZE, .init = crypto_sha3_init, .update = crypto_sha3_update, - .final = crypto_sha3_final, - .descsize = sizeof(struct sha3_state), + .finup = crypto_sha3_finup, + .descsize = SHA3_STATE_SIZE, .base.cra_name = "sha3-384", .base.cra_driver_name = "sha3-384-generic", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = SHA3_384_BLOCK_SIZE, .base.cra_module = THIS_MODULE, }, { .digestsize = SHA3_512_DIGEST_SIZE, .init = crypto_sha3_init, .update = crypto_sha3_update, - .final = crypto_sha3_final, - .descsize = sizeof(struct sha3_state), + .finup = crypto_sha3_finup, + .descsize = SHA3_STATE_SIZE, .base.cra_name = "sha3-512", .base.cra_driver_name = "sha3-512-generic", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = SHA3_512_BLOCK_SIZE, .base.cra_module = THIS_MODULE, } }; @@ -289,7 +274,7 @@ static void __exit sha3_generic_mod_fini(void) crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); } -subsys_initcall(sha3_generic_mod_init); +module_init(sha3_generic_mod_init); module_exit(sha3_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c index ed81813bd420..bfea65f4181c 100644 --- a/crypto/sha512_generic.c +++ b/crypto/sha512_generic.c @@ -6,16 +6,10 @@ * Copyright (c) 2003 Kyle McMartin <kyle@debian.org> */ #include <crypto/internal/hash.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/init.h> -#include <linux/crypto.h> -#include <linux/types.h> #include <crypto/sha2.h> #include <crypto/sha512_base.h> -#include <linux/percpu.h> -#include <asm/byteorder.h> +#include <linux/kernel.h> +#include <linux/module.h> #include <linux/unaligned.h> const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE] = { @@ -145,47 +139,42 @@ sha512_transform(u64 *state, const u8 *input) state[4] += e; state[5] += f; state[6] += g; state[7] += h; } -static void sha512_generic_block_fn(struct sha512_state *sst, u8 const *src, - int blocks) +void sha512_generic_block_fn(struct sha512_state *sst, u8 const *src, + int blocks) { - while (blocks--) { + do { sha512_transform(sst->state, src); src += SHA512_BLOCK_SIZE; - } + } while (--blocks); } +EXPORT_SYMBOL_GPL(sha512_generic_block_fn); -int crypto_sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int crypto_sha512_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - return sha512_base_do_update(desc, data, len, sha512_generic_block_fn); + return sha512_base_do_update_blocks(desc, data, len, + sha512_generic_block_fn); } -EXPORT_SYMBOL(crypto_sha512_update); -static int sha512_final(struct shash_desc *desc, u8 *hash) +static int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *hash) { - sha512_base_do_finalize(desc, sha512_generic_block_fn); + sha512_base_do_finup(desc, data, len, sha512_generic_block_fn); return sha512_base_finish(desc, hash); } -int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash) -{ - sha512_base_do_update(desc, data, len, sha512_generic_block_fn); - return sha512_final(desc, hash); -} -EXPORT_SYMBOL(crypto_sha512_finup); - static struct shash_alg sha512_algs[2] = { { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, .update = crypto_sha512_update, - .final = sha512_final, .finup = crypto_sha512_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha512", .cra_driver_name = "sha512-generic", .cra_priority = 100, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -193,13 +182,14 @@ static struct shash_alg sha512_algs[2] = { { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, .update = crypto_sha512_update, - .final = sha512_final, .finup = crypto_sha512_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha384", .cra_driver_name = "sha384-generic", .cra_priority = 100, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/crypto/shash.c b/crypto/shash.c index 301ab42bf849..b6c79a4a044a 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -16,6 +16,24 @@ #include "hash.h" +static inline bool crypto_shash_block_only(struct crypto_shash *tfm) +{ + return crypto_shash_alg(tfm)->base.cra_flags & + CRYPTO_AHASH_ALG_BLOCK_ONLY; +} + +static inline bool crypto_shash_final_nonzero(struct crypto_shash *tfm) +{ + return crypto_shash_alg(tfm)->base.cra_flags & + CRYPTO_AHASH_ALG_FINAL_NONZERO; +} + +static inline bool crypto_shash_finup_max(struct crypto_shash *tfm) +{ + return crypto_shash_alg(tfm)->base.cra_flags & + CRYPTO_AHASH_ALG_FINUP_MAX; +} + int shash_no_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) { @@ -46,18 +64,27 @@ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key, } EXPORT_SYMBOL_GPL(crypto_shash_setkey); -int crypto_shash_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int __crypto_shash_init(struct shash_desc *desc) { - return crypto_shash_alg(desc->tfm)->update(desc, data, len); + struct crypto_shash *tfm = desc->tfm; + + if (crypto_shash_block_only(tfm)) { + u8 *buf = shash_desc_ctx(desc); + + buf += crypto_shash_descsize(tfm) - 1; + *buf = 0; + } + + return crypto_shash_alg(tfm)->init(desc); } -EXPORT_SYMBOL_GPL(crypto_shash_update); -int crypto_shash_final(struct shash_desc *desc, u8 *out) +int crypto_shash_init(struct shash_desc *desc) { - return crypto_shash_alg(desc->tfm)->final(desc, out); + if (crypto_shash_get_flags(desc->tfm) & CRYPTO_TFM_NEED_KEY) + return -ENOKEY; + return __crypto_shash_init(desc); } -EXPORT_SYMBOL_GPL(crypto_shash_final); +EXPORT_SYMBOL_GPL(crypto_shash_init); static int shash_default_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) @@ -68,20 +95,89 @@ static int shash_default_finup(struct shash_desc *desc, const u8 *data, shash->final(desc, out); } -int crypto_shash_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) +static int crypto_shash_op_and_zero( + int (*op)(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out), + struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - return crypto_shash_alg(desc->tfm)->finup(desc, data, len, out); + int err; + + err = op(desc, data, len, out); + memset(shash_desc_ctx(desc), 0, crypto_shash_descsize(desc->tfm)); + return err; +} + +int crypto_shash_finup(struct shash_desc *restrict desc, const u8 *data, + unsigned int len, u8 *restrict out) +{ + struct crypto_shash *tfm = desc->tfm; + u8 *blenp = shash_desc_ctx(desc); + bool finup_max, nonzero; + unsigned int bs; + int err; + u8 *buf; + + if (!crypto_shash_block_only(tfm)) { + if (out) + goto finup; + return crypto_shash_alg(tfm)->update(desc, data, len); + } + + finup_max = out && crypto_shash_finup_max(tfm); + + /* Retain extra block for final nonzero algorithms. */ + nonzero = crypto_shash_final_nonzero(tfm); + + /* + * The partial block buffer follows the algorithm desc context. + * The byte following that contains the length. + */ + blenp += crypto_shash_descsize(tfm) - 1; + bs = crypto_shash_blocksize(tfm); + buf = blenp - bs; + + if (likely(!*blenp && finup_max)) + goto finup; + + while ((*blenp + len) >= bs + nonzero) { + unsigned int nbytes = len - nonzero; + const u8 *src = data; + + if (*blenp) { + memcpy(buf + *blenp, data, bs - *blenp); + nbytes = bs; + src = buf; + } + + err = crypto_shash_alg(tfm)->update(desc, src, nbytes); + if (err < 0) + return err; + + data += nbytes - err - *blenp; + len -= nbytes - err - *blenp; + *blenp = 0; + } + + if (*blenp || !out) { + memcpy(buf + *blenp, data, len); + *blenp += len; + if (!out) + return 0; + data = buf; + len = *blenp; + } + +finup: + return crypto_shash_op_and_zero(crypto_shash_alg(tfm)->finup, desc, + data, len, out); } EXPORT_SYMBOL_GPL(crypto_shash_finup); static int shash_default_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - struct shash_alg *shash = crypto_shash_alg(desc->tfm); - - return shash->init(desc) ?: - shash->finup(desc, data, len, out); + return __crypto_shash_init(desc) ?: + crypto_shash_finup(desc, data, len, out); } int crypto_shash_digest(struct shash_desc *desc, const u8 *data, @@ -92,7 +188,8 @@ int crypto_shash_digest(struct shash_desc *desc, const u8 *data, if (crypto_shash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; - return crypto_shash_alg(tfm)->digest(desc, data, len, out); + return crypto_shash_op_and_zero(crypto_shash_alg(tfm)->digest, desc, + data, len, out); } EXPORT_SYMBOL_GPL(crypto_shash_digest); @@ -100,44 +197,92 @@ int crypto_shash_tfm_digest(struct crypto_shash *tfm, const u8 *data, unsigned int len, u8 *out) { SHASH_DESC_ON_STACK(desc, tfm); - int err; desc->tfm = tfm; + return crypto_shash_digest(desc, data, len, out); +} +EXPORT_SYMBOL_GPL(crypto_shash_tfm_digest); - err = crypto_shash_digest(desc, data, len, out); - - shash_desc_zero(desc); +int crypto_shash_export_core(struct shash_desc *desc, void *out) +{ + int (*export)(struct shash_desc *desc, void *out); + struct crypto_shash *tfm = desc->tfm; + u8 *buf = shash_desc_ctx(desc); + unsigned int plen, ss; + + plen = crypto_shash_blocksize(tfm) + 1; + ss = crypto_shash_statesize(tfm); + if (crypto_shash_block_only(tfm)) + ss -= plen; + export = crypto_shash_alg(tfm)->export; + if (!export) { + memcpy(out, buf, ss); + return 0; + } - return err; + return export(desc, out); } -EXPORT_SYMBOL_GPL(crypto_shash_tfm_digest); +EXPORT_SYMBOL_GPL(crypto_shash_export_core); int crypto_shash_export(struct shash_desc *desc, void *out) { struct crypto_shash *tfm = desc->tfm; - struct shash_alg *shash = crypto_shash_alg(tfm); - if (shash->export) - return shash->export(desc, out); + if (crypto_shash_block_only(tfm)) { + unsigned int plen = crypto_shash_blocksize(tfm) + 1; + unsigned int descsize = crypto_shash_descsize(tfm); + unsigned int ss = crypto_shash_statesize(tfm); + u8 *buf = shash_desc_ctx(desc); - memcpy(out, shash_desc_ctx(desc), crypto_shash_descsize(tfm)); - return 0; + memcpy(out + ss - plen, buf + descsize - plen, plen); + } + return crypto_shash_export_core(desc, out); } EXPORT_SYMBOL_GPL(crypto_shash_export); -int crypto_shash_import(struct shash_desc *desc, const void *in) +int crypto_shash_import_core(struct shash_desc *desc, const void *in) { + int (*import)(struct shash_desc *desc, const void *in); struct crypto_shash *tfm = desc->tfm; - struct shash_alg *shash = crypto_shash_alg(tfm); + unsigned int descsize, plen, ss; + u8 *buf = shash_desc_ctx(desc); if (crypto_shash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; - if (shash->import) - return shash->import(desc, in); + plen = crypto_shash_blocksize(tfm) + 1; + descsize = crypto_shash_descsize(tfm); + ss = crypto_shash_statesize(tfm); + buf[descsize - 1] = 0; + if (crypto_shash_block_only(tfm)) + ss -= plen; + import = crypto_shash_alg(tfm)->import; + if (!import) { + memcpy(buf, in, ss); + return 0; + } - memcpy(shash_desc_ctx(desc), in, crypto_shash_descsize(tfm)); - return 0; + return import(desc, in); +} +EXPORT_SYMBOL_GPL(crypto_shash_import_core); + +int crypto_shash_import(struct shash_desc *desc, const void *in) +{ + struct crypto_shash *tfm = desc->tfm; + int err; + + err = crypto_shash_import_core(desc, in); + if (crypto_shash_block_only(tfm)) { + unsigned int plen = crypto_shash_blocksize(tfm) + 1; + unsigned int descsize = crypto_shash_descsize(tfm); + unsigned int ss = crypto_shash_statesize(tfm); + u8 *buf = shash_desc_ctx(desc); + + memcpy(buf + descsize - plen, in + ss - plen, plen); + if (buf[descsize - 1] >= plen) + err = -EOVERFLOW; + } + return err; } EXPORT_SYMBOL_GPL(crypto_shash_import); @@ -153,9 +298,6 @@ static int crypto_shash_init_tfm(struct crypto_tfm *tfm) { struct crypto_shash *hash = __crypto_shash_cast(tfm); struct shash_alg *alg = crypto_shash_alg(hash); - int err; - - hash->descsize = alg->descsize; shash_set_needkey(hash, alg); @@ -165,18 +307,7 @@ static int crypto_shash_init_tfm(struct crypto_tfm *tfm) if (!alg->init_tfm) return 0; - err = alg->init_tfm(hash); - if (err) - return err; - - /* ->init_tfm() may have increased the descsize. */ - if (WARN_ON_ONCE(hash->descsize > HASH_MAX_DESCSIZE)) { - if (alg->exit_tfm) - alg->exit_tfm(hash); - return -EINVAL; - } - - return 0; + return alg->init_tfm(hash); } static void crypto_shash_free_instance(struct crypto_instance *inst) @@ -227,6 +358,7 @@ const struct crypto_type crypto_shash_type = { .maskset = CRYPTO_ALG_TYPE_MASK, .type = CRYPTO_ALG_TYPE_SHASH, .tfmsize = offsetof(struct crypto_shash, base), + .algsize = offsetof(struct shash_alg, base), }; int crypto_grab_shash(struct crypto_shash_spawn *spawn, @@ -273,8 +405,6 @@ struct crypto_shash *crypto_clone_shash(struct crypto_shash *hash) if (IS_ERR(nhash)) return nhash; - nhash->descsize = hash->descsize; - if (alg->clone_tfm) { err = alg->clone_tfm(nhash, hash); if (err) { @@ -308,9 +438,6 @@ static int shash_prepare_alg(struct shash_alg *alg) struct crypto_alg *base = &alg->halg.base; int err; - if (alg->descsize > HASH_MAX_DESCSIZE) - return -EINVAL; - if ((alg->export && !alg->import) || (alg->import && !alg->export)) return -EINVAL; @@ -336,11 +463,20 @@ static int shash_prepare_alg(struct shash_alg *alg) alg->finup = shash_default_finup; if (!alg->digest) alg->digest = shash_default_digest; - if (!alg->export) + if (!alg->export && !alg->halg.statesize) alg->halg.statesize = alg->descsize; if (!alg->setkey) alg->setkey = shash_no_setkey; + if (base->cra_flags & CRYPTO_AHASH_ALG_BLOCK_ONLY) { + BUILD_BUG_ON(MAX_ALGAPI_BLOCKSIZE >= 256); + alg->descsize += base->cra_blocksize + 1; + alg->statesize += base->cra_blocksize + 1; + } + + if (alg->descsize > HASH_MAX_DESCSIZE) + return -EINVAL; + return 0; } diff --git a/crypto/sig.c b/crypto/sig.c index dfc7cae90802..beba745b6405 100644 --- a/crypto/sig.c +++ b/crypto/sig.c @@ -74,6 +74,7 @@ static const struct crypto_type crypto_sig_type = { .maskset = CRYPTO_ALG_TYPE_MASK, .type = CRYPTO_ALG_TYPE_SIG, .tfmsize = offsetof(struct crypto_sig, base), + .algsize = offsetof(struct sig_alg, base), }; struct crypto_sig *crypto_alloc_sig(const char *alg_name, u32 type, u32 mask) @@ -102,6 +103,11 @@ static int sig_default_set_key(struct crypto_sig *tfm, return -ENOSYS; } +static unsigned int sig_default_size(struct crypto_sig *tfm) +{ + return DIV_ROUND_UP_POW2(crypto_sig_keysize(tfm), BITS_PER_BYTE); +} + static int sig_prepare_alg(struct sig_alg *alg) { struct crypto_alg *base = &alg->base; @@ -117,9 +123,9 @@ static int sig_prepare_alg(struct sig_alg *alg) if (!alg->key_size) return -EINVAL; if (!alg->max_size) - alg->max_size = alg->key_size; + alg->max_size = sig_default_size; if (!alg->digest_size) - alg->digest_size = alg->key_size; + alg->digest_size = sig_default_size; base->cra_type = &crypto_sig_type; base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; diff --git a/crypto/skcipher.c b/crypto/skcipher.c index 132075a905d9..319215cfded5 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -620,6 +620,7 @@ static const struct crypto_type crypto_skcipher_type = { .maskset = CRYPTO_ALG_TYPE_SKCIPHER_MASK, .type = CRYPTO_ALG_TYPE_SKCIPHER, .tfmsize = offsetof(struct crypto_skcipher, base), + .algsize = offsetof(struct skcipher_alg, base), }; int crypto_grab_skcipher(struct crypto_skcipher_spawn *spawn, diff --git a/crypto/sm3_generic.c b/crypto/sm3_generic.c index a2d23a46924e..4fb6957c2f0c 100644 --- a/crypto/sm3_generic.c +++ b/crypto/sm3_generic.c @@ -9,15 +9,10 @@ */ #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> #include <crypto/sm3.h> #include <crypto/sm3_base.h> -#include <linux/bitops.h> -#include <asm/byteorder.h> -#include <linux/unaligned.h> +#include <linux/kernel.h> +#include <linux/module.h> const u8 sm3_zero_message_hash[SM3_DIGEST_SIZE] = { 0x1A, 0xB2, 0x1D, 0x83, 0x55, 0xCF, 0xA1, 0x7F, @@ -30,38 +25,28 @@ EXPORT_SYMBOL_GPL(sm3_zero_message_hash); static int crypto_sm3_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - sm3_update(shash_desc_ctx(desc), data, len); - return 0; -} - -static int crypto_sm3_final(struct shash_desc *desc, u8 *out) -{ - sm3_final(shash_desc_ctx(desc), out); - return 0; + return sm3_base_do_update_blocks(desc, data, len, sm3_block_generic); } static int crypto_sm3_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *hash) { - struct sm3_state *sctx = shash_desc_ctx(desc); - - if (len) - sm3_update(sctx, data, len); - sm3_final(sctx, hash); - return 0; + sm3_base_do_finup(desc, data, len, sm3_block_generic); + return sm3_base_finish(desc, hash); } static struct shash_alg sm3_alg = { .digestsize = SM3_DIGEST_SIZE, .init = sm3_base_init, .update = crypto_sm3_update, - .final = crypto_sm3_final, .finup = crypto_sm3_finup, - .descsize = sizeof(struct sm3_state), + .descsize = SM3_STATE_SIZE, .base = { .cra_name = "sm3", .cra_driver_name = "sm3-generic", .cra_priority = 100, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SM3_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 96f4a66be14c..879fc21dcc16 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -716,207 +716,6 @@ static inline int do_one_ahash_op(struct ahash_request *req, int ret) return crypto_wait_req(ret, wait); } -struct test_mb_ahash_data { - struct scatterlist sg[XBUFSIZE]; - char result[64]; - struct ahash_request *req; - struct crypto_wait wait; - char *xbuf[XBUFSIZE]; -}; - -static inline int do_mult_ahash_op(struct test_mb_ahash_data *data, u32 num_mb, - int *rc) -{ - int i, err; - - /* Fire up a bunch of concurrent requests */ - err = crypto_ahash_digest(data[0].req); - - /* Wait for all requests to finish */ - err = crypto_wait_req(err, &data[0].wait); - if (num_mb < 2) - return err; - - for (i = 0; i < num_mb; i++) { - rc[i] = ahash_request_err(data[i].req); - if (rc[i]) { - pr_info("concurrent request %d error %d\n", i, rc[i]); - err = rc[i]; - } - } - - return err; -} - -static int test_mb_ahash_jiffies(struct test_mb_ahash_data *data, int blen, - int secs, u32 num_mb) -{ - unsigned long start, end; - int bcount; - int ret = 0; - int *rc; - - rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL); - if (!rc) - return -ENOMEM; - - for (start = jiffies, end = start + secs * HZ, bcount = 0; - time_before(jiffies, end); bcount++) { - ret = do_mult_ahash_op(data, num_mb, rc); - if (ret) - goto out; - } - - pr_cont("%d operations in %d seconds (%llu bytes)\n", - bcount * num_mb, secs, (u64)bcount * blen * num_mb); - -out: - kfree(rc); - return ret; -} - -static int test_mb_ahash_cycles(struct test_mb_ahash_data *data, int blen, - u32 num_mb) -{ - unsigned long cycles = 0; - int ret = 0; - int i; - int *rc; - - rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL); - if (!rc) - return -ENOMEM; - - /* Warm-up run. */ - for (i = 0; i < 4; i++) { - ret = do_mult_ahash_op(data, num_mb, rc); - if (ret) - goto out; - } - - /* The real thing. */ - for (i = 0; i < 8; i++) { - cycles_t start, end; - - start = get_cycles(); - ret = do_mult_ahash_op(data, num_mb, rc); - end = get_cycles(); - - if (ret) - goto out; - - cycles += end - start; - } - - pr_cont("1 operation in %lu cycles (%d bytes)\n", - (cycles + 4) / (8 * num_mb), blen); - -out: - kfree(rc); - return ret; -} - -static void test_mb_ahash_speed(const char *algo, unsigned int secs, - struct hash_speed *speed, u32 num_mb) -{ - struct test_mb_ahash_data *data; - struct crypto_ahash *tfm; - unsigned int i, j, k; - int ret; - - data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL); - if (!data) - return; - - tfm = crypto_alloc_ahash(algo, 0, 0); - if (IS_ERR(tfm)) { - pr_err("failed to load transform for %s: %ld\n", - algo, PTR_ERR(tfm)); - goto free_data; - } - - for (i = 0; i < num_mb; ++i) { - if (testmgr_alloc_buf(data[i].xbuf)) - goto out; - - crypto_init_wait(&data[i].wait); - - data[i].req = ahash_request_alloc(tfm, GFP_KERNEL); - if (!data[i].req) { - pr_err("alg: hash: Failed to allocate request for %s\n", - algo); - goto out; - } - - - if (i) { - ahash_request_set_callback(data[i].req, 0, NULL, NULL); - ahash_request_chain(data[i].req, data[0].req); - } else - ahash_request_set_callback(data[0].req, 0, - crypto_req_done, - &data[0].wait); - - sg_init_table(data[i].sg, XBUFSIZE); - for (j = 0; j < XBUFSIZE; j++) { - sg_set_buf(data[i].sg + j, data[i].xbuf[j], PAGE_SIZE); - memset(data[i].xbuf[j], 0xff, PAGE_SIZE); - } - } - - pr_info("\ntesting speed of multibuffer %s (%s)\n", algo, - get_driver_name(crypto_ahash, tfm)); - - for (i = 0; speed[i].blen != 0; i++) { - /* For some reason this only tests digests. */ - if (speed[i].blen != speed[i].plen) - continue; - - if (speed[i].blen > XBUFSIZE * PAGE_SIZE) { - pr_err("template (%u) too big for tvmem (%lu)\n", - speed[i].blen, XBUFSIZE * PAGE_SIZE); - goto out; - } - - if (klen) - crypto_ahash_setkey(tfm, tvmem[0], klen); - - for (k = 0; k < num_mb; k++) - ahash_request_set_crypt(data[k].req, data[k].sg, - data[k].result, speed[i].blen); - - pr_info("test%3u " - "(%5u byte blocks,%5u bytes per update,%4u updates): ", - i, speed[i].blen, speed[i].plen, - speed[i].blen / speed[i].plen); - - if (secs) { - ret = test_mb_ahash_jiffies(data, speed[i].blen, secs, - num_mb); - cond_resched(); - } else { - ret = test_mb_ahash_cycles(data, speed[i].blen, num_mb); - } - - - if (ret) { - pr_err("At least one hashing failed ret=%d\n", ret); - break; - } - } - -out: - ahash_request_free(data[0].req); - - for (k = 0; k < num_mb; ++k) - testmgr_free_buf(data[k].xbuf); - - crypto_free_ahash(tfm); - -free_data: - kfree(data); -} - static int test_ahash_jiffies_digest(struct ahash_request *req, int blen, char *out, int secs) { @@ -2584,36 +2383,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_ahash_speed("sm3", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; fallthrough; - case 450: - test_mb_ahash_speed("sha1", sec, generic_hash_speed_template, - num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; - case 451: - test_mb_ahash_speed("sha256", sec, generic_hash_speed_template, - num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; - case 452: - test_mb_ahash_speed("sha512", sec, generic_hash_speed_template, - num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; - case 453: - test_mb_ahash_speed("sm3", sec, generic_hash_speed_template, - num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; - case 454: - test_mb_ahash_speed("streebog256", sec, - generic_hash_speed_template, num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; - case 455: - test_mb_ahash_speed("streebog512", sec, - generic_hash_speed_template, num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; case 499: break; diff --git a/crypto/xcbc.c b/crypto/xcbc.c index fc785667b134..970ff581dc58 100644 --- a/crypto/xcbc.c +++ b/crypto/xcbc.c @@ -8,9 +8,12 @@ #include <crypto/internal/cipher.h> #include <crypto/internal/hash.h> +#include <crypto/utils.h> #include <linux/err.h> #include <linux/kernel.h> #include <linux/module.h> +#include <linux/slab.h> +#include <linux/string.h> static u_int32_t ks[12] = {0x01010101, 0x01010101, 0x01010101, 0x01010101, 0x02020202, 0x02020202, 0x02020202, 0x02020202, @@ -30,22 +33,6 @@ struct xcbc_tfm_ctx { u8 consts[]; }; -/* - * +------------------------ - * | <shash desc> - * +------------------------ - * | xcbc_desc_ctx - * +------------------------ - * | odds (block size) - * +------------------------ - * | prev (block size) - * +------------------------ - */ -struct xcbc_desc_ctx { - unsigned int len; - u8 odds[]; -}; - #define XCBC_BLOCKSIZE 16 static int crypto_xcbc_digest_setkey(struct crypto_shash *parent, @@ -70,13 +57,10 @@ static int crypto_xcbc_digest_setkey(struct crypto_shash *parent, static int crypto_xcbc_digest_init(struct shash_desc *pdesc) { - struct xcbc_desc_ctx *ctx = shash_desc_ctx(pdesc); int bs = crypto_shash_blocksize(pdesc->tfm); - u8 *prev = &ctx->odds[bs]; + u8 *prev = shash_desc_ctx(pdesc); - ctx->len = 0; memset(prev, 0, bs); - return 0; } @@ -85,77 +69,36 @@ static int crypto_xcbc_digest_update(struct shash_desc *pdesc, const u8 *p, { struct crypto_shash *parent = pdesc->tfm; struct xcbc_tfm_ctx *tctx = crypto_shash_ctx(parent); - struct xcbc_desc_ctx *ctx = shash_desc_ctx(pdesc); struct crypto_cipher *tfm = tctx->child; int bs = crypto_shash_blocksize(parent); - u8 *odds = ctx->odds; - u8 *prev = odds + bs; - - /* checking the data can fill the block */ - if ((ctx->len + len) <= bs) { - memcpy(odds + ctx->len, p, len); - ctx->len += len; - return 0; - } - - /* filling odds with new data and encrypting it */ - memcpy(odds + ctx->len, p, bs - ctx->len); - len -= bs - ctx->len; - p += bs - ctx->len; - - crypto_xor(prev, odds, bs); - crypto_cipher_encrypt_one(tfm, prev, prev); + u8 *prev = shash_desc_ctx(pdesc); - /* clearing the length */ - ctx->len = 0; - - /* encrypting the rest of data */ - while (len > bs) { + do { crypto_xor(prev, p, bs); crypto_cipher_encrypt_one(tfm, prev, prev); p += bs; len -= bs; - } - - /* keeping the surplus of blocksize */ - if (len) { - memcpy(odds, p, len); - ctx->len = len; - } - - return 0; + } while (len >= bs); + return len; } -static int crypto_xcbc_digest_final(struct shash_desc *pdesc, u8 *out) +static int crypto_xcbc_digest_finup(struct shash_desc *pdesc, const u8 *src, + unsigned int len, u8 *out) { struct crypto_shash *parent = pdesc->tfm; struct xcbc_tfm_ctx *tctx = crypto_shash_ctx(parent); - struct xcbc_desc_ctx *ctx = shash_desc_ctx(pdesc); struct crypto_cipher *tfm = tctx->child; int bs = crypto_shash_blocksize(parent); - u8 *odds = ctx->odds; - u8 *prev = odds + bs; + u8 *prev = shash_desc_ctx(pdesc); unsigned int offset = 0; - if (ctx->len != bs) { - unsigned int rlen; - u8 *p = odds + ctx->len; - - *p = 0x80; - p++; - - rlen = bs - ctx->len -1; - if (rlen) - memset(p, 0, rlen); - + crypto_xor(prev, src, len); + if (len != bs) { + prev[len] ^= 0x80; offset += bs; } - - crypto_xor(prev, odds, bs); crypto_xor(prev, &tctx->consts[offset], bs); - crypto_cipher_encrypt_one(tfm, out, prev); - return 0; } @@ -216,17 +159,18 @@ static int xcbc_create(struct crypto_template *tmpl, struct rtattr **tb) inst->alg.base.cra_blocksize = alg->cra_blocksize; inst->alg.base.cra_ctxsize = sizeof(struct xcbc_tfm_ctx) + alg->cra_blocksize * 2; + inst->alg.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINAL_NONZERO; inst->alg.digestsize = alg->cra_blocksize; - inst->alg.descsize = sizeof(struct xcbc_desc_ctx) + - alg->cra_blocksize * 2; + inst->alg.descsize = alg->cra_blocksize; inst->alg.base.cra_init = xcbc_init_tfm; inst->alg.base.cra_exit = xcbc_exit_tfm; inst->alg.init = crypto_xcbc_digest_init; inst->alg.update = crypto_xcbc_digest_update; - inst->alg.final = crypto_xcbc_digest_final; + inst->alg.finup = crypto_xcbc_digest_finup; inst->alg.setkey = crypto_xcbc_digest_setkey; inst->free = shash_free_singlespawn_instance; diff --git a/drivers/char/hw_random/atmel-rng.c b/drivers/char/hw_random/atmel-rng.c index 143406bc6939..d2b00458761e 100644 --- a/drivers/char/hw_random/atmel-rng.c +++ b/drivers/char/hw_random/atmel-rng.c @@ -37,6 +37,7 @@ struct atmel_trng { struct clk *clk; void __iomem *base; struct hwrng rng; + struct device *dev; bool has_half_rate; }; @@ -59,9 +60,9 @@ static int atmel_trng_read(struct hwrng *rng, void *buf, size_t max, u32 *data = buf; int ret; - ret = pm_runtime_get_sync((struct device *)trng->rng.priv); + ret = pm_runtime_get_sync(trng->dev); if (ret < 0) { - pm_runtime_put_sync((struct device *)trng->rng.priv); + pm_runtime_put_sync(trng->dev); return ret; } @@ -79,8 +80,8 @@ static int atmel_trng_read(struct hwrng *rng, void *buf, size_t max, ret = 4; out: - pm_runtime_mark_last_busy((struct device *)trng->rng.priv); - pm_runtime_put_sync_autosuspend((struct device *)trng->rng.priv); + pm_runtime_mark_last_busy(trng->dev); + pm_runtime_put_sync_autosuspend(trng->dev); return ret; } @@ -134,9 +135,9 @@ static int atmel_trng_probe(struct platform_device *pdev) return -ENODEV; trng->has_half_rate = data->has_half_rate; + trng->dev = &pdev->dev; trng->rng.name = pdev->name; trng->rng.read = atmel_trng_read; - trng->rng.priv = (unsigned long)&pdev->dev; platform_set_drvdata(pdev, trng); #ifndef CONFIG_PM diff --git a/drivers/char/hw_random/mtk-rng.c b/drivers/char/hw_random/mtk-rng.c index 1e3048f2bb38..b7fa1bc1122b 100644 --- a/drivers/char/hw_random/mtk-rng.c +++ b/drivers/char/hw_random/mtk-rng.c @@ -36,6 +36,7 @@ struct mtk_rng { void __iomem *base; struct clk *clk; struct hwrng rng; + struct device *dev; }; static int mtk_rng_init(struct hwrng *rng) @@ -85,7 +86,7 @@ static int mtk_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) struct mtk_rng *priv = to_mtk_rng(rng); int retval = 0; - pm_runtime_get_sync((struct device *)priv->rng.priv); + pm_runtime_get_sync(priv->dev); while (max >= sizeof(u32)) { if (!mtk_rng_wait_ready(rng, wait)) @@ -97,8 +98,8 @@ static int mtk_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) max -= sizeof(u32); } - pm_runtime_mark_last_busy((struct device *)priv->rng.priv); - pm_runtime_put_sync_autosuspend((struct device *)priv->rng.priv); + pm_runtime_mark_last_busy(priv->dev); + pm_runtime_put_sync_autosuspend(priv->dev); return retval || !wait ? retval : -EIO; } @@ -112,13 +113,13 @@ static int mtk_rng_probe(struct platform_device *pdev) if (!priv) return -ENOMEM; + priv->dev = &pdev->dev; priv->rng.name = pdev->name; #ifndef CONFIG_PM priv->rng.init = mtk_rng_init; priv->rng.cleanup = mtk_rng_cleanup; #endif priv->rng.read = mtk_rng_read; - priv->rng.priv = (unsigned long)&pdev->dev; priv->rng.quality = 900; priv->clk = devm_clk_get(&pdev->dev, "rng"); diff --git a/drivers/char/hw_random/npcm-rng.c b/drivers/char/hw_random/npcm-rng.c index 9ff00f096f38..3e308c890bd2 100644 --- a/drivers/char/hw_random/npcm-rng.c +++ b/drivers/char/hw_random/npcm-rng.c @@ -32,6 +32,7 @@ struct npcm_rng { void __iomem *base; struct hwrng rng; + struct device *dev; u32 clkp; }; @@ -57,7 +58,7 @@ static int npcm_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) int retval = 0; int ready; - pm_runtime_get_sync((struct device *)priv->rng.priv); + pm_runtime_get_sync(priv->dev); while (max) { if (wait) { @@ -79,8 +80,8 @@ static int npcm_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) max--; } - pm_runtime_mark_last_busy((struct device *)priv->rng.priv); - pm_runtime_put_sync_autosuspend((struct device *)priv->rng.priv); + pm_runtime_mark_last_busy(priv->dev); + pm_runtime_put_sync_autosuspend(priv->dev); return retval || !wait ? retval : -EIO; } @@ -109,7 +110,7 @@ static int npcm_rng_probe(struct platform_device *pdev) #endif priv->rng.name = pdev->name; priv->rng.read = npcm_rng_read; - priv->rng.priv = (unsigned long)&pdev->dev; + priv->dev = &pdev->dev; priv->clkp = (u32)(uintptr_t)of_device_get_match_data(&pdev->dev); writel(NPCM_RNG_M1ROSEL, priv->base + NPCM_RNGMODE_REG); diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 47082782008a..5686369779be 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -530,13 +530,6 @@ source "drivers/crypto/cavium/nitrox/Kconfig" source "drivers/crypto/marvell/Kconfig" source "drivers/crypto/intel/Kconfig" -config CRYPTO_DEV_CAVIUM_ZIP - tristate "Cavium ZIP driver" - depends on PCI && 64BIT && (ARM64 || COMPILE_TEST) - help - Select this option if you want to enable compression/decompression - acceleration on Cavium's ARM based SoCs - config CRYPTO_DEV_QCE tristate "Qualcomm crypto engine accelerator" depends on ARCH_QCOM || COMPILE_TEST diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index c97f0ebc55ec..22eadcc8f4a2 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -8,12 +8,9 @@ obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_I2C) += atmel-i2c.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_ECC) += atmel-ecc.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA204A) += atmel-sha204a.o -obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/ obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ obj-$(CONFIG_CRYPTO_DEV_CCREE) += ccree/ obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chelsio/ -obj-$(CONFIG_CRYPTO_DEV_CPT) += cavium/cpt/ -obj-$(CONFIG_CRYPTO_DEV_NITROX) += cavium/nitrox/ obj-$(CONFIG_CRYPTO_DEV_EXYNOS_RNG) += exynos-rng.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += caam/ obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o @@ -50,3 +47,4 @@ obj-y += hisilicon/ obj-$(CONFIG_CRYPTO_DEV_AMLOGIC_GXL) += amlogic/ obj-y += intel/ obj-y += starfive/ +obj-y += cavium/ diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index 6072dd9f390b..ba13fb75c05d 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -23,6 +23,18 @@ #include <linux/string.h> #include "sun8i-ce.h" +static void sun8i_ce_hash_stat_fb_inc(struct crypto_ahash *tfm) +{ + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { + struct sun8i_ce_alg_template *algt __maybe_unused; + struct ahash_alg *alg = crypto_ahash_alg(tfm); + + algt = container_of(alg, struct sun8i_ce_alg_template, + alg.hash.base); + algt->stat_fb++; + } +} + int sun8i_ce_hash_init_tfm(struct crypto_ahash *tfm) { struct sun8i_ce_hash_tfm_ctx *op = crypto_ahash_ctx(tfm); @@ -48,8 +60,10 @@ int sun8i_ce_hash_init_tfm(struct crypto_ahash *tfm) sizeof(struct sun8i_ce_hash_reqctx) + crypto_ahash_reqsize(op->fallback_tfm)); - memcpy(algt->fbname, crypto_ahash_driver_name(op->fallback_tfm), - CRYPTO_MAX_ALG_NAME); + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + memcpy(algt->fbname, + crypto_ahash_driver_name(op->fallback_tfm), + CRYPTO_MAX_ALG_NAME); err = pm_runtime_get_sync(op->ce->dev); if (err < 0) @@ -78,7 +92,9 @@ int sun8i_ce_hash_init(struct ahash_request *areq) memset(rctx, 0, sizeof(struct sun8i_ce_hash_reqctx)); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_init(&rctx->fallback_req); } @@ -90,7 +106,9 @@ int sun8i_ce_hash_export(struct ahash_request *areq, void *out) struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_export(&rctx->fallback_req, out); } @@ -102,7 +120,9 @@ int sun8i_ce_hash_import(struct ahash_request *areq, const void *in) struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_import(&rctx->fallback_req, in); } @@ -113,21 +133,13 @@ int sun8i_ce_hash_final(struct ahash_request *areq) struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); - ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = areq->result; - - if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { - struct sun8i_ce_alg_template *algt __maybe_unused; - struct ahash_alg *alg = crypto_ahash_alg(tfm); + sun8i_ce_hash_stat_fb_inc(tfm); - algt = container_of(alg, struct sun8i_ce_alg_template, - alg.hash.base); -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG - algt->stat_fb++; -#endif - } + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, areq->result, 0); return crypto_ahash_final(&rctx->fallback_req); } @@ -139,10 +151,10 @@ int sun8i_ce_hash_update(struct ahash_request *areq) struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, NULL, areq->nbytes); return crypto_ahash_update(&rctx->fallback_req); } @@ -153,24 +165,14 @@ int sun8i_ce_hash_finup(struct ahash_request *areq) struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); - ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; - rctx->fallback_req.result = areq->result; + sun8i_ce_hash_stat_fb_inc(tfm); - if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { - struct sun8i_ce_alg_template *algt __maybe_unused; - struct ahash_alg *alg = crypto_ahash_alg(tfm); - - algt = container_of(alg, struct sun8i_ce_alg_template, - alg.hash.base); -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG - algt->stat_fb++; -#endif - } + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); return crypto_ahash_finup(&rctx->fallback_req); } @@ -181,24 +183,14 @@ static int sun8i_ce_hash_digest_fb(struct ahash_request *areq) struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); - ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; - rctx->fallback_req.result = areq->result; - - if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { - struct sun8i_ce_alg_template *algt __maybe_unused; - struct ahash_alg *alg = crypto_ahash_alg(tfm); + sun8i_ce_hash_stat_fb_inc(tfm); - algt = container_of(alg, struct sun8i_ce_alg_template, - alg.hash.base); -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG - algt->stat_fb++; -#endif - } + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); return crypto_ahash_digest(&rctx->fallback_req); } @@ -213,22 +205,30 @@ static bool sun8i_ce_hash_need_fallback(struct ahash_request *areq) algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base); if (areq->nbytes == 0) { - algt->stat_fb_len0++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_len0++; + return true; } /* we need to reserve one SG for padding one */ if (sg_nents_for_len(areq->src, areq->nbytes) > MAX_SG - 1) { - algt->stat_fb_maxsg++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_maxsg++; + return true; } sg = areq->src; while (sg) { if (sg->length % 4) { - algt->stat_fb_srclen++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_srclen++; + return true; } if (!IS_ALIGNED(sg->offset, sizeof(u32))) { - algt->stat_fb_srcali++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_srcali++; + return true; } sg = sg_next(sg); @@ -244,21 +244,11 @@ int sun8i_ce_hash_digest(struct ahash_request *areq) struct sun8i_ce_alg_template *algt; struct sun8i_ce_dev *ce; struct crypto_engine *engine; - struct scatterlist *sg; - int nr_sgs, e, i; + int e; if (sun8i_ce_hash_need_fallback(areq)) return sun8i_ce_hash_digest_fb(areq); - nr_sgs = sg_nents_for_len(areq->src, areq->nbytes); - if (nr_sgs > MAX_SG - 1) - return sun8i_ce_hash_digest_fb(areq); - - for_each_sg(areq->src, sg, nr_sgs, i) { - if (sg->length % 4 || !IS_ALIGNED(sg->offset, sizeof(u32))) - return sun8i_ce_hash_digest_fb(areq); - } - algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base); ce = algt->ce; @@ -343,9 +333,8 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) u32 common; u64 byte_count; __le32 *bf; - void *buf = NULL; + void *buf, *result; int j, i, todo; - void *result = NULL; u64 bs; int digestsize; dma_addr_t addr_res, addr_pad; @@ -365,22 +354,22 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) buf = kcalloc(2, bs, GFP_KERNEL | GFP_DMA); if (!buf) { err = -ENOMEM; - goto theend; + goto err_out; } bf = (__le32 *)buf; result = kzalloc(digestsize, GFP_KERNEL | GFP_DMA); if (!result) { err = -ENOMEM; - goto theend; + goto err_free_buf; } flow = rctx->flow; chan = &ce->chanlist[flow]; -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG - algt->stat_req++; -#endif + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_req++; + dev_dbg(ce->dev, "%s %s len=%d\n", __func__, crypto_tfm_alg_name(areq->base.tfm), areq->nbytes); cet = chan->tl; @@ -398,7 +387,7 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) if (nr_sgs <= 0 || nr_sgs > MAX_SG) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; - goto theend; + goto err_free_result; } len = areq->nbytes; @@ -411,7 +400,7 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) if (len > 0) { dev_err(ce->dev, "remaining len %d\n", len); err = -EINVAL; - goto theend; + goto err_unmap_src; } addr_res = dma_map_single(ce->dev, result, digestsize, DMA_FROM_DEVICE); cet->t_dst[0].addr = desc_addr_val_le32(ce, addr_res); @@ -419,7 +408,7 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) if (dma_mapping_error(ce->dev, addr_res)) { dev_err(ce->dev, "DMA map dest\n"); err = -EINVAL; - goto theend; + goto err_unmap_src; } byte_count = areq->nbytes; @@ -441,7 +430,7 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) } if (!j) { err = -EINVAL; - goto theend; + goto err_unmap_result; } addr_pad = dma_map_single(ce->dev, buf, j * 4, DMA_TO_DEVICE); @@ -450,7 +439,7 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) if (dma_mapping_error(ce->dev, addr_pad)) { dev_err(ce->dev, "DMA error on padding SG\n"); err = -EINVAL; - goto theend; + goto err_unmap_result; } if (ce->variant->hash_t_dlen_in_bits) @@ -463,16 +452,25 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) err = sun8i_ce_run_task(ce, flow, crypto_ahash_alg_name(tfm)); dma_unmap_single(ce->dev, addr_pad, j * 4, DMA_TO_DEVICE); - dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); + +err_unmap_result: dma_unmap_single(ce->dev, addr_res, digestsize, DMA_FROM_DEVICE); + if (!err) + memcpy(areq->result, result, algt->alg.hash.base.halg.digestsize); +err_unmap_src: + dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); - memcpy(areq->result, result, algt->alg.hash.base.halg.digestsize); -theend: - kfree(buf); +err_free_result: kfree(result); + +err_free_buf: + kfree(buf); + +err_out: local_bh_disable(); crypto_finalize_hash_request(engine, breq, err); local_bh_enable(); + return 0; } diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c index 753f67a36dc5..8bc08089f044 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -150,7 +150,9 @@ int sun8i_ss_hash_init(struct ahash_request *areq) memset(rctx, 0, sizeof(struct sun8i_ss_hash_reqctx)); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_init(&rctx->fallback_req); } @@ -162,7 +164,9 @@ int sun8i_ss_hash_export(struct ahash_request *areq, void *out) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_export(&rctx->fallback_req, out); } @@ -174,7 +178,9 @@ int sun8i_ss_hash_import(struct ahash_request *areq, const void *in) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_import(&rctx->fallback_req, in); } @@ -186,9 +192,10 @@ int sun8i_ss_hash_final(struct ahash_request *areq) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = areq->result; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, areq->result, 0); if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) { struct ahash_alg *alg = crypto_ahash_alg(tfm); @@ -212,10 +219,10 @@ int sun8i_ss_hash_update(struct ahash_request *areq) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, NULL, areq->nbytes); return crypto_ahash_update(&rctx->fallback_req); } @@ -227,12 +234,11 @@ int sun8i_ss_hash_finup(struct ahash_request *areq) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; - rctx->fallback_req.result = areq->result; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) { struct ahash_alg *alg = crypto_ahash_alg(tfm); @@ -256,12 +262,11 @@ static int sun8i_ss_hash_digest_fb(struct ahash_request *areq) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; - rctx->fallback_req.result = areq->result; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) { struct ahash_alg *alg = crypto_ahash_alg(tfm); diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index e0af611a95d8..289750f34ccf 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -72,7 +72,7 @@ static inline int crypto4xx_crypt(struct skcipher_request *req, { struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher); - __le32 iv[AES_IV_SIZE]; + __le32 iv[AES_IV_SIZE / 4]; if (check_blocksize && !IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE)) return -EINVAL; @@ -429,7 +429,7 @@ static int crypto4xx_crypt_aes_ccm(struct aead_request *req, bool decrypt) struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); struct crypto4xx_aead_reqctx *rctx = aead_request_ctx(req); struct crypto_aead *aead = crypto_aead_reqtfm(req); - __le32 iv[16]; + __le32 iv[4]; u32 tmp_sa[SA_AES128_CCM_LEN + 4]; struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *)tmp_sa; unsigned int len = req->cryptlen; diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index ec3ccfa60445..c77d06ddb1ec 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -676,7 +676,7 @@ int crypto4xx_build_pd(struct crypto_async_request *req, struct scatterlist *src, struct scatterlist *dst, const unsigned int datalen, - const __le32 *iv, const u32 iv_len, + const void *iv, const u32 iv_len, const struct dynamic_sa_ctl *req_sa, const unsigned int sa_len, const unsigned int assoclen, diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 3adcc5e65694..9c56c7ac6e4c 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -147,6 +147,12 @@ struct crypto4xx_alg { struct crypto4xx_device *dev; }; +#if IS_ENABLED(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION >= 120000 +#define BUILD_PD_ACCESS __attribute__((access(read_only, 6, 7))) +#else +#define BUILD_PD_ACCESS +#endif + int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size); void crypto4xx_free_sa(struct crypto4xx_ctx *ctx); int crypto4xx_build_pd(struct crypto_async_request *req, @@ -154,11 +160,11 @@ int crypto4xx_build_pd(struct crypto_async_request *req, struct scatterlist *src, struct scatterlist *dst, const unsigned int datalen, - const __le32 *iv, const u32 iv_len, + const void *iv, const u32 iv_len, const struct dynamic_sa_ctl *sa, const unsigned int sa_len, const unsigned int assoclen, - struct scatterlist *dst_tmp); + struct scatterlist *dst_tmp) BUILD_PD_ACCESS; int crypto4xx_setkey_aes_cbc(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen); int crypto4xx_setkey_aes_ctr(struct crypto_skcipher *cipher, diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index 14bf86957d31..27c5d000b4b2 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -1743,7 +1743,8 @@ static struct skcipher_alg aes_xts_alg = { .base.cra_driver_name = "atmel-xts-aes", .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct atmel_aes_xts_ctx), - .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, @@ -2220,7 +2221,7 @@ static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd) static void atmel_aes_crypto_alg_init(struct crypto_alg *alg) { - alg->cra_flags |= CRYPTO_ALG_ASYNC; + alg->cra_flags |= CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; alg->cra_alignmask = 0xf; alg->cra_priority = ATMEL_AES_PRIORITY; alg->cra_module = THIS_MODULE; diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index 67a170608566..2cc36da163e8 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -1254,7 +1254,8 @@ static int atmel_sha_cra_init(struct crypto_tfm *tfm) static void atmel_sha_alg_init(struct ahash_alg *alg) { alg->halg.base.cra_priority = ATMEL_SHA_PRIORITY; - alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC; + alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY; alg->halg.base.cra_ctxsize = sizeof(struct atmel_sha_ctx); alg->halg.base.cra_module = THIS_MODULE; alg->halg.base.cra_init = atmel_sha_cra_init; @@ -2041,7 +2042,8 @@ static void atmel_sha_hmac_cra_exit(struct crypto_tfm *tfm) static void atmel_sha_hmac_alg_init(struct ahash_alg *alg) { alg->halg.base.cra_priority = ATMEL_SHA_PRIORITY; - alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC; + alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY; alg->halg.base.cra_ctxsize = sizeof(struct atmel_sha_hmac_ctx); alg->halg.base.cra_module = THIS_MODULE; alg->halg.base.cra_init = atmel_sha_hmac_cra_init; diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index de9717e221e4..098f5532f389 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -785,7 +785,7 @@ static int atmel_tdes_init_tfm(struct crypto_skcipher *tfm) static void atmel_tdes_skcipher_alg_init(struct skcipher_alg *alg) { alg->base.cra_priority = ATMEL_TDES_PRIORITY; - alg->base.cra_flags = CRYPTO_ALG_ASYNC; + alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; alg->base.cra_ctxsize = sizeof(struct atmel_tdes_ctx); alg->base.cra_module = THIS_MODULE; diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index d4b39184dbdb..38ff931059b4 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -573,6 +573,7 @@ static const struct soc_device_attribute caam_imx_soc_table[] = { { .soc_id = "i.MX7*", .data = &caam_imx7_data }, { .soc_id = "i.MX8M*", .data = &caam_imx7_data }, { .soc_id = "i.MX8ULP", .data = &caam_imx8ulp_data }, + { .soc_id = "i.MX8QM", .data = &caam_imx8ulp_data }, { .soc_id = "VF*", .data = &caam_vf610_data }, { .family = "Freescale i.MX" }, { /* sentinel */ } diff --git a/drivers/crypto/cavium/Makefile b/drivers/crypto/cavium/Makefile index 4679c06b611f..75227c587ed0 100644 --- a/drivers/crypto/cavium/Makefile +++ b/drivers/crypto/cavium/Makefile @@ -2,4 +2,5 @@ # # Makefile for Cavium crypto device drivers # -obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += zip/ +obj-$(CONFIG_CRYPTO_DEV_CPT) += cpt/ +obj-$(CONFIG_CRYPTO_DEV_NITROX) += nitrox/ diff --git a/drivers/crypto/cavium/zip/Makefile b/drivers/crypto/cavium/zip/Makefile deleted file mode 100644 index 020d189d793d..000000000000 --- a/drivers/crypto/cavium/zip/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for Cavium's ZIP Driver. -# - -obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += thunderx_zip.o -thunderx_zip-y := zip_main.o \ - zip_device.o \ - zip_crypto.o \ - zip_mem.o \ - zip_deflate.o \ - zip_inflate.o diff --git a/drivers/crypto/cavium/zip/common.h b/drivers/crypto/cavium/zip/common.h deleted file mode 100644 index 54f6fb054119..000000000000 --- a/drivers/crypto/cavium/zip/common.h +++ /dev/null @@ -1,222 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __COMMON_H__ -#define __COMMON_H__ - -#include <linux/delay.h> -#include <linux/init.h> -#include <linux/interrupt.h> -#include <linux/io.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/pci.h> -#include <linux/seq_file.h> -#include <linux/string.h> -#include <linux/types.h> - -/* Device specific zlib function definitions */ -#include "zip_device.h" - -/* ZIP device definitions */ -#include "zip_main.h" - -/* ZIP memory allocation/deallocation related definitions */ -#include "zip_mem.h" - -/* Device specific structure definitions */ -#include "zip_regs.h" - -#define ZIP_ERROR -1 - -#define ZIP_FLUSH_FINISH 4 - -#define RAW_FORMAT 0 /* for rawpipe */ -#define ZLIB_FORMAT 1 /* for zpipe */ -#define GZIP_FORMAT 2 /* for gzpipe */ -#define LZS_FORMAT 3 /* for lzspipe */ - -/* Max number of ZIP devices supported */ -#define MAX_ZIP_DEVICES 2 - -/* Configures the number of zip queues to be used */ -#define ZIP_NUM_QUEUES 2 - -#define DYNAMIC_STOP_EXCESS 1024 - -/* Maximum buffer sizes in direct mode */ -#define MAX_INPUT_BUFFER_SIZE (64 * 1024) -#define MAX_OUTPUT_BUFFER_SIZE (64 * 1024) - -/** - * struct zip_operation - common data structure for comp and decomp operations - * @input: Next input byte is read from here - * @output: Next output byte written here - * @ctx_addr: Inflate context buffer address - * @history: Pointer to the history buffer - * @input_len: Number of bytes available at next_in - * @input_total_len: Total number of input bytes read - * @output_len: Remaining free space at next_out - * @output_total_len: Total number of bytes output so far - * @csum: Checksum value of the uncompressed data - * @flush: Flush flag - * @format: Format (depends on stream's wrap) - * @speed: Speed depends on stream's level - * @ccode: Compression code ( stream's strategy) - * @lzs_flag: Flag for LZS support - * @begin_file: Beginning of file indication for inflate - * @history_len: Size of the history data - * @end_file: Ending of the file indication for inflate - * @compcode: Completion status of the ZIP invocation - * @bytes_read: Input bytes read in current instruction - * @bits_processed: Total bits processed for entire file - * @sizeofptr: To distinguish between ILP32 and LP64 - * @sizeofzops: Optional just for padding - * - * This structure is used to maintain the required meta data for the - * comp and decomp operations. - */ -struct zip_operation { - u8 *input; - u8 *output; - u64 ctx_addr; - u64 history; - - u32 input_len; - u32 input_total_len; - - u32 output_len; - u32 output_total_len; - - u32 csum; - u32 flush; - - u32 format; - u32 speed; - u32 ccode; - u32 lzs_flag; - - u32 begin_file; - u32 history_len; - - u32 end_file; - u32 compcode; - u32 bytes_read; - u32 bits_processed; - - u32 sizeofptr; - u32 sizeofzops; -}; - -static inline int zip_poll_result(union zip_zres_s *result) -{ - int retries = 1000; - - while (!result->s.compcode) { - if (!--retries) { - pr_err("ZIP ERR: request timed out"); - return -ETIMEDOUT; - } - udelay(10); - /* - * Force re-reading of compcode which is updated - * by the ZIP coprocessor. - */ - rmb(); - } - return 0; -} - -/* error messages */ -#define zip_err(fmt, args...) pr_err("ZIP ERR:%s():%d: " \ - fmt "\n", __func__, __LINE__, ## args) - -#ifdef MSG_ENABLE -/* Enable all messages */ -#define zip_msg(fmt, args...) pr_info("ZIP_MSG:" fmt "\n", ## args) -#else -#define zip_msg(fmt, args...) -#endif - -#if defined(ZIP_DEBUG_ENABLE) && defined(MSG_ENABLE) - -#ifdef DEBUG_LEVEL - -#define FILE_NAME (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : \ - strrchr(__FILE__, '\\') ? strrchr(__FILE__, '\\') + 1 : __FILE__) - -#if DEBUG_LEVEL >= 4 - -#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s: %s() : %d: " \ - fmt "\n", FILE_NAME, __func__, __LINE__, ## args) - -#elif DEBUG_LEVEL >= 3 - -#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s: %s() : %d: " \ - fmt "\n", FILE_NAME, __func__, __LINE__, ## args) - -#elif DEBUG_LEVEL >= 2 - -#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s() : %d: " \ - fmt "\n", __func__, __LINE__, ## args) - -#else - -#define zip_dbg(fmt, args...) pr_info("ZIP DBG:" fmt "\n", ## args) - -#endif /* DEBUG LEVEL >=4 */ - -#else - -#define zip_dbg(fmt, args...) pr_info("ZIP DBG:" fmt "\n", ## args) - -#endif /* DEBUG_LEVEL */ -#else - -#define zip_dbg(fmt, args...) - -#endif /* ZIP_DEBUG_ENABLE && MSG_ENABLE*/ - -#endif diff --git a/drivers/crypto/cavium/zip/zip_crypto.c b/drivers/crypto/cavium/zip/zip_crypto.c deleted file mode 100644 index 02e87f2d50db..000000000000 --- a/drivers/crypto/cavium/zip/zip_crypto.c +++ /dev/null @@ -1,261 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include "zip_crypto.h" - -static void zip_static_init_zip_ops(struct zip_operation *zip_ops, - int lzs_flag) -{ - zip_ops->flush = ZIP_FLUSH_FINISH; - - /* equivalent to level 6 of opensource zlib */ - zip_ops->speed = 1; - - if (!lzs_flag) { - zip_ops->ccode = 0; /* Auto Huffman */ - zip_ops->lzs_flag = 0; - zip_ops->format = ZLIB_FORMAT; - } else { - zip_ops->ccode = 3; /* LZS Encoding */ - zip_ops->lzs_flag = 1; - zip_ops->format = LZS_FORMAT; - } - zip_ops->begin_file = 1; - zip_ops->history_len = 0; - zip_ops->end_file = 1; - zip_ops->compcode = 0; - zip_ops->csum = 1; /* Adler checksum desired */ -} - -static int zip_ctx_init(struct zip_kernel_ctx *zip_ctx, int lzs_flag) -{ - struct zip_operation *comp_ctx = &zip_ctx->zip_comp; - struct zip_operation *decomp_ctx = &zip_ctx->zip_decomp; - - zip_static_init_zip_ops(comp_ctx, lzs_flag); - zip_static_init_zip_ops(decomp_ctx, lzs_flag); - - comp_ctx->input = zip_data_buf_alloc(MAX_INPUT_BUFFER_SIZE); - if (!comp_ctx->input) - return -ENOMEM; - - comp_ctx->output = zip_data_buf_alloc(MAX_OUTPUT_BUFFER_SIZE); - if (!comp_ctx->output) - goto err_comp_input; - - decomp_ctx->input = zip_data_buf_alloc(MAX_INPUT_BUFFER_SIZE); - if (!decomp_ctx->input) - goto err_comp_output; - - decomp_ctx->output = zip_data_buf_alloc(MAX_OUTPUT_BUFFER_SIZE); - if (!decomp_ctx->output) - goto err_decomp_input; - - return 0; - -err_decomp_input: - zip_data_buf_free(decomp_ctx->input, MAX_INPUT_BUFFER_SIZE); - -err_comp_output: - zip_data_buf_free(comp_ctx->output, MAX_OUTPUT_BUFFER_SIZE); - -err_comp_input: - zip_data_buf_free(comp_ctx->input, MAX_INPUT_BUFFER_SIZE); - - return -ENOMEM; -} - -static void zip_ctx_exit(struct zip_kernel_ctx *zip_ctx) -{ - struct zip_operation *comp_ctx = &zip_ctx->zip_comp; - struct zip_operation *dec_ctx = &zip_ctx->zip_decomp; - - zip_data_buf_free(comp_ctx->input, MAX_INPUT_BUFFER_SIZE); - zip_data_buf_free(comp_ctx->output, MAX_OUTPUT_BUFFER_SIZE); - - zip_data_buf_free(dec_ctx->input, MAX_INPUT_BUFFER_SIZE); - zip_data_buf_free(dec_ctx->output, MAX_OUTPUT_BUFFER_SIZE); -} - -static int zip_compress(const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, - struct zip_kernel_ctx *zip_ctx) -{ - struct zip_operation *zip_ops = NULL; - struct zip_state *zip_state; - struct zip_device *zip = NULL; - int ret; - - if (!zip_ctx || !src || !dst || !dlen) - return -ENOMEM; - - zip = zip_get_device(zip_get_node_id()); - if (!zip) - return -ENODEV; - - zip_state = kzalloc(sizeof(*zip_state), GFP_ATOMIC); - if (!zip_state) - return -ENOMEM; - - zip_ops = &zip_ctx->zip_comp; - - zip_ops->input_len = slen; - zip_ops->output_len = *dlen; - memcpy(zip_ops->input, src, slen); - - ret = zip_deflate(zip_ops, zip_state, zip); - - if (!ret) { - *dlen = zip_ops->output_len; - memcpy(dst, zip_ops->output, *dlen); - } - kfree(zip_state); - return ret; -} - -static int zip_decompress(const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, - struct zip_kernel_ctx *zip_ctx) -{ - struct zip_operation *zip_ops = NULL; - struct zip_state *zip_state; - struct zip_device *zip = NULL; - int ret; - - if (!zip_ctx || !src || !dst || !dlen) - return -ENOMEM; - - zip = zip_get_device(zip_get_node_id()); - if (!zip) - return -ENODEV; - - zip_state = kzalloc(sizeof(*zip_state), GFP_ATOMIC); - if (!zip_state) - return -ENOMEM; - - zip_ops = &zip_ctx->zip_decomp; - memcpy(zip_ops->input, src, slen); - - /* Work around for a bug in zlib which needs an extra bytes sometimes */ - if (zip_ops->ccode != 3) /* Not LZS Encoding */ - zip_ops->input[slen++] = 0; - - zip_ops->input_len = slen; - zip_ops->output_len = *dlen; - - ret = zip_inflate(zip_ops, zip_state, zip); - - if (!ret) { - *dlen = zip_ops->output_len; - memcpy(dst, zip_ops->output, *dlen); - } - kfree(zip_state); - return ret; -} - -/* SCOMP framework start */ -void *zip_alloc_scomp_ctx_deflate(void) -{ - int ret; - struct zip_kernel_ctx *zip_ctx; - - zip_ctx = kzalloc(sizeof(*zip_ctx), GFP_KERNEL); - if (!zip_ctx) - return ERR_PTR(-ENOMEM); - - ret = zip_ctx_init(zip_ctx, 0); - - if (ret) { - kfree_sensitive(zip_ctx); - return ERR_PTR(ret); - } - - return zip_ctx; -} - -void *zip_alloc_scomp_ctx_lzs(void) -{ - int ret; - struct zip_kernel_ctx *zip_ctx; - - zip_ctx = kzalloc(sizeof(*zip_ctx), GFP_KERNEL); - if (!zip_ctx) - return ERR_PTR(-ENOMEM); - - ret = zip_ctx_init(zip_ctx, 1); - - if (ret) { - kfree_sensitive(zip_ctx); - return ERR_PTR(ret); - } - - return zip_ctx; -} - -void zip_free_scomp_ctx(void *ctx) -{ - struct zip_kernel_ctx *zip_ctx = ctx; - - zip_ctx_exit(zip_ctx); - kfree_sensitive(zip_ctx); -} - -int zip_scomp_compress(struct crypto_scomp *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx) -{ - struct zip_kernel_ctx *zip_ctx = ctx; - - return zip_compress(src, slen, dst, dlen, zip_ctx); -} - -int zip_scomp_decompress(struct crypto_scomp *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx) -{ - struct zip_kernel_ctx *zip_ctx = ctx; - - return zip_decompress(src, slen, dst, dlen, zip_ctx); -} /* SCOMP framework end */ diff --git a/drivers/crypto/cavium/zip/zip_crypto.h b/drivers/crypto/cavium/zip/zip_crypto.h deleted file mode 100644 index 10899ece2d1f..000000000000 --- a/drivers/crypto/cavium/zip/zip_crypto.h +++ /dev/null @@ -1,68 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_CRYPTO_H__ -#define __ZIP_CRYPTO_H__ - -#include <crypto/internal/scompress.h> -#include "common.h" -#include "zip_deflate.h" -#include "zip_inflate.h" - -struct zip_kernel_ctx { - struct zip_operation zip_comp; - struct zip_operation zip_decomp; -}; - -void *zip_alloc_scomp_ctx_deflate(void); -void *zip_alloc_scomp_ctx_lzs(void); -void zip_free_scomp_ctx(void *zip_ctx); -int zip_scomp_compress(struct crypto_scomp *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx); -int zip_scomp_decompress(struct crypto_scomp *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx); -#endif diff --git a/drivers/crypto/cavium/zip/zip_deflate.c b/drivers/crypto/cavium/zip/zip_deflate.c deleted file mode 100644 index d7133f857d67..000000000000 --- a/drivers/crypto/cavium/zip/zip_deflate.c +++ /dev/null @@ -1,200 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include <linux/delay.h> -#include <linux/sched.h> - -#include "common.h" -#include "zip_deflate.h" - -/* Prepares the deflate zip command */ -static int prepare_zip_command(struct zip_operation *zip_ops, - struct zip_state *s, union zip_inst_s *zip_cmd) -{ - union zip_zres_s *result_ptr = &s->result; - - memset(zip_cmd, 0, sizeof(s->zip_cmd)); - memset(result_ptr, 0, sizeof(s->result)); - - /* IWORD #0 */ - /* History gather */ - zip_cmd->s.hg = 0; - /* compression enable = 1 for deflate */ - zip_cmd->s.ce = 1; - /* sf (sync flush) */ - zip_cmd->s.sf = 1; - /* ef (end of file) */ - if (zip_ops->flush == ZIP_FLUSH_FINISH) { - zip_cmd->s.ef = 1; - zip_cmd->s.sf = 0; - } - - zip_cmd->s.cc = zip_ops->ccode; - /* ss (compression speed/storage) */ - zip_cmd->s.ss = zip_ops->speed; - - /* IWORD #1 */ - /* adler checksum */ - zip_cmd->s.adlercrc32 = zip_ops->csum; - zip_cmd->s.historylength = zip_ops->history_len; - zip_cmd->s.dg = 0; - - /* IWORD # 6 and 7 - compression input/history pointer */ - zip_cmd->s.inp_ptr_addr.s.addr = __pa(zip_ops->input); - zip_cmd->s.inp_ptr_ctl.s.length = (zip_ops->input_len + - zip_ops->history_len); - zip_cmd->s.ds = 0; - - /* IWORD # 8 and 9 - Output pointer */ - zip_cmd->s.out_ptr_addr.s.addr = __pa(zip_ops->output); - zip_cmd->s.out_ptr_ctl.s.length = zip_ops->output_len; - /* maximum number of output-stream bytes that can be written */ - zip_cmd->s.totaloutputlength = zip_ops->output_len; - - /* IWORD # 10 and 11 - Result pointer */ - zip_cmd->s.res_ptr_addr.s.addr = __pa(result_ptr); - /* Clearing completion code */ - result_ptr->s.compcode = 0; - - return 0; -} - -/** - * zip_deflate - API to offload deflate operation to hardware - * @zip_ops: Pointer to zip operation structure - * @s: Pointer to the structure representing zip state - * @zip_dev: Pointer to zip device structure - * - * This function prepares the zip deflate command and submits it to the zip - * engine for processing. - * - * Return: 0 if successful or error code - */ -int zip_deflate(struct zip_operation *zip_ops, struct zip_state *s, - struct zip_device *zip_dev) -{ - union zip_inst_s *zip_cmd = &s->zip_cmd; - union zip_zres_s *result_ptr = &s->result; - u32 queue; - - /* Prepares zip command based on the input parameters */ - prepare_zip_command(zip_ops, s, zip_cmd); - - atomic64_add(zip_ops->input_len, &zip_dev->stats.comp_in_bytes); - /* Loads zip command into command queues and rings door bell */ - queue = zip_load_instr(zip_cmd, zip_dev); - - /* Stats update for compression requests submitted */ - atomic64_inc(&zip_dev->stats.comp_req_submit); - - /* Wait for completion or error */ - zip_poll_result(result_ptr); - - /* Stats update for compression requests completed */ - atomic64_inc(&zip_dev->stats.comp_req_complete); - - zip_ops->compcode = result_ptr->s.compcode; - switch (zip_ops->compcode) { - case ZIP_CMD_NOTDONE: - zip_dbg("Zip instruction not yet completed"); - return ZIP_ERROR; - - case ZIP_CMD_SUCCESS: - zip_dbg("Zip instruction completed successfully"); - zip_update_cmd_bufs(zip_dev, queue); - break; - - case ZIP_CMD_DTRUNC: - zip_dbg("Output Truncate error"); - /* Returning ZIP_ERROR to avoid copy to user */ - return ZIP_ERROR; - - default: - zip_err("Zip instruction failed. Code:%d", zip_ops->compcode); - return ZIP_ERROR; - } - - /* Update the CRC depending on the format */ - switch (zip_ops->format) { - case RAW_FORMAT: - zip_dbg("RAW Format: %d ", zip_ops->format); - /* Get checksum from engine, need to feed it again */ - zip_ops->csum = result_ptr->s.adler32; - break; - - case ZLIB_FORMAT: - zip_dbg("ZLIB Format: %d ", zip_ops->format); - zip_ops->csum = result_ptr->s.adler32; - break; - - case GZIP_FORMAT: - zip_dbg("GZIP Format: %d ", zip_ops->format); - zip_ops->csum = result_ptr->s.crc32; - break; - - case LZS_FORMAT: - zip_dbg("LZS Format: %d ", zip_ops->format); - break; - - default: - zip_err("Unknown Format:%d\n", zip_ops->format); - } - - atomic64_add(result_ptr->s.totalbyteswritten, - &zip_dev->stats.comp_out_bytes); - - /* Update output_len */ - if (zip_ops->output_len < result_ptr->s.totalbyteswritten) { - /* Dynamic stop && strm->output_len < zipconstants[onfsize] */ - zip_err("output_len (%d) < total bytes written(%d)\n", - zip_ops->output_len, result_ptr->s.totalbyteswritten); - zip_ops->output_len = 0; - - } else { - zip_ops->output_len = result_ptr->s.totalbyteswritten; - } - - return 0; -} diff --git a/drivers/crypto/cavium/zip/zip_deflate.h b/drivers/crypto/cavium/zip/zip_deflate.h deleted file mode 100644 index 1d32e76edc4d..000000000000 --- a/drivers/crypto/cavium/zip/zip_deflate.h +++ /dev/null @@ -1,62 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_DEFLATE_H__ -#define __ZIP_DEFLATE_H__ - -/** - * zip_deflate - API to offload deflate operation to hardware - * @zip_ops: Pointer to zip operation structure - * @s: Pointer to the structure representing zip state - * @zip_dev: Pointer to the structure representing zip device - * - * This function prepares the zip deflate command and submits it to the zip - * engine by ringing the doorbell. - * - * Return: 0 if successful or error code - */ -int zip_deflate(struct zip_operation *zip_ops, struct zip_state *s, - struct zip_device *zip_dev); -#endif diff --git a/drivers/crypto/cavium/zip/zip_device.c b/drivers/crypto/cavium/zip/zip_device.c deleted file mode 100644 index f174ec29ed69..000000000000 --- a/drivers/crypto/cavium/zip/zip_device.c +++ /dev/null @@ -1,202 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include "common.h" -#include "zip_deflate.h" - -/** - * zip_cmd_queue_consumed - Calculates the space consumed in the command queue. - * - * @zip_dev: Pointer to zip device structure - * @queue: Queue number - * - * Return: Bytes consumed in the command queue buffer. - */ -static inline u32 zip_cmd_queue_consumed(struct zip_device *zip_dev, int queue) -{ - return ((zip_dev->iq[queue].sw_head - zip_dev->iq[queue].sw_tail) * - sizeof(u64 *)); -} - -/** - * zip_load_instr - Submits the instruction into the ZIP command queue - * @instr: Pointer to the instruction to be submitted - * @zip_dev: Pointer to ZIP device structure to which the instruction is to - * be submitted - * - * This function copies the ZIP instruction to the command queue and rings the - * doorbell to notify the engine of the instruction submission. The command - * queue is maintained in a circular fashion. When there is space for exactly - * one instruction in the queue, next chunk pointer of the queue is made to - * point to the head of the queue, thus maintaining a circular queue. - * - * Return: Queue number to which the instruction was submitted - */ -u32 zip_load_instr(union zip_inst_s *instr, - struct zip_device *zip_dev) -{ - union zip_quex_doorbell dbell; - u32 queue = 0; - u32 consumed = 0; - u64 *ncb_ptr = NULL; - union zip_nptr_s ncp; - - /* - * Distribute the instructions between the enabled queues based on - * the CPU id. - */ - if (raw_smp_processor_id() % 2 == 0) - queue = 0; - else - queue = 1; - - zip_dbg("CPU Core: %d Queue number:%d", raw_smp_processor_id(), queue); - - /* Take cmd buffer lock */ - spin_lock(&zip_dev->iq[queue].lock); - - /* - * Command Queue implementation - * 1. If there is place for new instructions, push the cmd at sw_head. - * 2. If there is place for exactly one instruction, push the new cmd - * at the sw_head. Make sw_head point to the sw_tail to make it - * circular. Write sw_head's physical address to the "Next-Chunk - * Buffer Ptr" to make it cmd_hw_tail. - * 3. Ring the door bell. - */ - zip_dbg("sw_head : %lx", zip_dev->iq[queue].sw_head); - zip_dbg("sw_tail : %lx", zip_dev->iq[queue].sw_tail); - - consumed = zip_cmd_queue_consumed(zip_dev, queue); - /* Check if there is space to push just one cmd */ - if ((consumed + 128) == (ZIP_CMD_QBUF_SIZE - 8)) { - zip_dbg("Cmd queue space available for single command"); - /* Space for one cmd, pust it and make it circular queue */ - memcpy((u8 *)zip_dev->iq[queue].sw_head, (u8 *)instr, - sizeof(union zip_inst_s)); - zip_dev->iq[queue].sw_head += 16; /* 16 64_bit words = 128B */ - - /* Now, point the "Next-Chunk Buffer Ptr" to sw_head */ - ncb_ptr = zip_dev->iq[queue].sw_head; - - zip_dbg("ncb addr :0x%lx sw_head addr :0x%lx", - ncb_ptr, zip_dev->iq[queue].sw_head - 16); - - /* Using Circular command queue */ - zip_dev->iq[queue].sw_head = zip_dev->iq[queue].sw_tail; - /* Mark this buffer for free */ - zip_dev->iq[queue].free_flag = 1; - - /* Write new chunk buffer address at "Next-Chunk Buffer Ptr" */ - ncp.u_reg64 = 0ull; - ncp.s.addr = __pa(zip_dev->iq[queue].sw_head); - *ncb_ptr = ncp.u_reg64; - zip_dbg("*ncb_ptr :0x%lx sw_head[phys] :0x%lx", - *ncb_ptr, __pa(zip_dev->iq[queue].sw_head)); - - zip_dev->iq[queue].pend_cnt++; - - } else { - zip_dbg("Enough space is available for commands"); - /* Push this cmd to cmd queue buffer */ - memcpy((u8 *)zip_dev->iq[queue].sw_head, (u8 *)instr, - sizeof(union zip_inst_s)); - zip_dev->iq[queue].sw_head += 16; /* 16 64_bit words = 128B */ - - zip_dev->iq[queue].pend_cnt++; - } - zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", - zip_dev->iq[queue].sw_head, zip_dev->iq[queue].sw_tail, - zip_dev->iq[queue].hw_tail); - - zip_dbg(" Pushed the new cmd : pend_cnt : %d", - zip_dev->iq[queue].pend_cnt); - - /* Ring the doorbell */ - dbell.u_reg64 = 0ull; - dbell.s.dbell_cnt = 1; - zip_reg_write(dbell.u_reg64, - (zip_dev->reg_base + ZIP_QUEX_DOORBELL(queue))); - - /* Unlock cmd buffer lock */ - spin_unlock(&zip_dev->iq[queue].lock); - - return queue; -} - -/** - * zip_update_cmd_bufs - Updates the queue statistics after posting the - * instruction - * @zip_dev: Pointer to zip device structure - * @queue: Queue number - */ -void zip_update_cmd_bufs(struct zip_device *zip_dev, u32 queue) -{ - /* Take cmd buffer lock */ - spin_lock(&zip_dev->iq[queue].lock); - - /* Check if the previous buffer can be freed */ - if (zip_dev->iq[queue].free_flag == 1) { - zip_dbg("Free flag. Free cmd buffer, adjust sw head and tail"); - /* Reset the free flag */ - zip_dev->iq[queue].free_flag = 0; - - /* Point the hw_tail to start of the new chunk buffer */ - zip_dev->iq[queue].hw_tail = zip_dev->iq[queue].sw_head; - } else { - zip_dbg("Free flag not set. increment hw tail"); - zip_dev->iq[queue].hw_tail += 16; /* 16 64_bit words = 128B */ - } - - zip_dev->iq[queue].done_cnt++; - zip_dev->iq[queue].pend_cnt--; - - zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", - zip_dev->iq[queue].sw_head, zip_dev->iq[queue].sw_tail, - zip_dev->iq[queue].hw_tail); - zip_dbg(" Got CC : pend_cnt : %d\n", zip_dev->iq[queue].pend_cnt); - - spin_unlock(&zip_dev->iq[queue].lock); -} diff --git a/drivers/crypto/cavium/zip/zip_device.h b/drivers/crypto/cavium/zip/zip_device.h deleted file mode 100644 index 9e18b3b93d38..000000000000 --- a/drivers/crypto/cavium/zip/zip_device.h +++ /dev/null @@ -1,108 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_DEVICE_H__ -#define __ZIP_DEVICE_H__ - -#include <linux/types.h> -#include "zip_main.h" - -struct sg_info { - /* - * Pointer to the input data when scatter_gather == 0 and - * pointer to the input gather list buffer when scatter_gather == 1 - */ - union zip_zptr_s *gather; - - /* - * Pointer to the output data when scatter_gather == 0 and - * pointer to the output scatter list buffer when scatter_gather == 1 - */ - union zip_zptr_s *scatter; - - /* - * Holds size of the output buffer pointed by scatter list - * when scatter_gather == 1 - */ - u64 scatter_buf_size; - - /* for gather data */ - u64 gather_enable; - - /* for scatter data */ - u64 scatter_enable; - - /* Number of gather list pointers for gather data */ - u32 gbuf_cnt; - - /* Number of scatter list pointers for scatter data */ - u32 sbuf_cnt; - - /* Buffers allocation state */ - u8 alloc_state; -}; - -/** - * struct zip_state - Structure representing the required information related - * to a command - * @zip_cmd: Pointer to zip instruction structure - * @result: Pointer to zip result structure - * @ctx: Context pointer for inflate - * @history: Decompression history pointer - * @sginfo: Scatter-gather info structure - */ -struct zip_state { - union zip_inst_s zip_cmd; - union zip_zres_s result; - union zip_zptr_s *ctx; - union zip_zptr_s *history; - struct sg_info sginfo; -}; - -#define ZIP_CONTEXT_SIZE 2048 -#define ZIP_INFLATE_HISTORY_SIZE 32768 -#define ZIP_DEFLATE_HISTORY_SIZE 32768 - -#endif diff --git a/drivers/crypto/cavium/zip/zip_inflate.c b/drivers/crypto/cavium/zip/zip_inflate.c deleted file mode 100644 index 7e0d73e2f89e..000000000000 --- a/drivers/crypto/cavium/zip/zip_inflate.c +++ /dev/null @@ -1,223 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include <linux/delay.h> -#include <linux/sched.h> - -#include "common.h" -#include "zip_inflate.h" - -static int prepare_inflate_zcmd(struct zip_operation *zip_ops, - struct zip_state *s, union zip_inst_s *zip_cmd) -{ - union zip_zres_s *result_ptr = &s->result; - - memset(zip_cmd, 0, sizeof(s->zip_cmd)); - memset(result_ptr, 0, sizeof(s->result)); - - /* IWORD#0 */ - - /* Decompression History Gather list - no gather list */ - zip_cmd->s.hg = 0; - /* For decompression, CE must be 0x0. */ - zip_cmd->s.ce = 0; - /* For decompression, SS must be 0x0. */ - zip_cmd->s.ss = 0; - /* For decompression, SF should always be set. */ - zip_cmd->s.sf = 1; - - /* Begin File */ - if (zip_ops->begin_file == 0) - zip_cmd->s.bf = 0; - else - zip_cmd->s.bf = 1; - - zip_cmd->s.ef = 1; - /* 0: for Deflate decompression, 3: for LZS decompression */ - zip_cmd->s.cc = zip_ops->ccode; - - /* IWORD #1*/ - - /* adler checksum */ - zip_cmd->s.adlercrc32 = zip_ops->csum; - - /* - * HISTORYLENGTH must be 0x0 for any ZIP decompress operation. - * History data is added to a decompression operation via IWORD3. - */ - zip_cmd->s.historylength = 0; - zip_cmd->s.ds = 0; - - /* IWORD # 8 and 9 - Output pointer */ - zip_cmd->s.out_ptr_addr.s.addr = __pa(zip_ops->output); - zip_cmd->s.out_ptr_ctl.s.length = zip_ops->output_len; - - /* Maximum number of output-stream bytes that can be written */ - zip_cmd->s.totaloutputlength = zip_ops->output_len; - - zip_dbg("Data Direct Input case "); - - /* IWORD # 6 and 7 - input pointer */ - zip_cmd->s.dg = 0; - zip_cmd->s.inp_ptr_addr.s.addr = __pa((u8 *)zip_ops->input); - zip_cmd->s.inp_ptr_ctl.s.length = zip_ops->input_len; - - /* IWORD # 10 and 11 - Result pointer */ - zip_cmd->s.res_ptr_addr.s.addr = __pa(result_ptr); - - /* Clearing completion code */ - result_ptr->s.compcode = 0; - - /* Returning 0 for time being.*/ - return 0; -} - -/** - * zip_inflate - API to offload inflate operation to hardware - * @zip_ops: Pointer to zip operation structure - * @s: Pointer to the structure representing zip state - * @zip_dev: Pointer to zip device structure - * - * This function prepares the zip inflate command and submits it to the zip - * engine for processing. - * - * Return: 0 if successful or error code - */ -int zip_inflate(struct zip_operation *zip_ops, struct zip_state *s, - struct zip_device *zip_dev) -{ - union zip_inst_s *zip_cmd = &s->zip_cmd; - union zip_zres_s *result_ptr = &s->result; - u32 queue; - - /* Prepare inflate zip command */ - prepare_inflate_zcmd(zip_ops, s, zip_cmd); - - atomic64_add(zip_ops->input_len, &zip_dev->stats.decomp_in_bytes); - - /* Load inflate command to zip queue and ring the doorbell */ - queue = zip_load_instr(zip_cmd, zip_dev); - - /* Decompression requests submitted stats update */ - atomic64_inc(&zip_dev->stats.decomp_req_submit); - - /* Wait for completion or error */ - zip_poll_result(result_ptr); - - /* Decompression requests completed stats update */ - atomic64_inc(&zip_dev->stats.decomp_req_complete); - - zip_ops->compcode = result_ptr->s.compcode; - switch (zip_ops->compcode) { - case ZIP_CMD_NOTDONE: - zip_dbg("Zip Instruction not yet completed\n"); - return ZIP_ERROR; - - case ZIP_CMD_SUCCESS: - zip_dbg("Zip Instruction completed successfully\n"); - break; - - case ZIP_CMD_DYNAMIC_STOP: - zip_dbg(" Dynamic stop Initiated\n"); - break; - - default: - zip_dbg("Instruction failed. Code = %d\n", zip_ops->compcode); - atomic64_inc(&zip_dev->stats.decomp_bad_reqs); - zip_update_cmd_bufs(zip_dev, queue); - return ZIP_ERROR; - } - - zip_update_cmd_bufs(zip_dev, queue); - - if ((zip_ops->ccode == 3) && (zip_ops->flush == 4) && - (zip_ops->compcode != ZIP_CMD_DYNAMIC_STOP)) - result_ptr->s.ef = 1; - - zip_ops->csum = result_ptr->s.adler32; - - atomic64_add(result_ptr->s.totalbyteswritten, - &zip_dev->stats.decomp_out_bytes); - - if (zip_ops->output_len < result_ptr->s.totalbyteswritten) { - zip_err("output_len (%d) < total bytes written (%d)\n", - zip_ops->output_len, result_ptr->s.totalbyteswritten); - zip_ops->output_len = 0; - } else { - zip_ops->output_len = result_ptr->s.totalbyteswritten; - } - - zip_ops->bytes_read = result_ptr->s.totalbytesread; - zip_ops->bits_processed = result_ptr->s.totalbitsprocessed; - zip_ops->end_file = result_ptr->s.ef; - if (zip_ops->end_file) { - switch (zip_ops->format) { - case RAW_FORMAT: - zip_dbg("RAW Format: %d ", zip_ops->format); - /* Get checksum from engine */ - zip_ops->csum = result_ptr->s.adler32; - break; - - case ZLIB_FORMAT: - zip_dbg("ZLIB Format: %d ", zip_ops->format); - zip_ops->csum = result_ptr->s.adler32; - break; - - case GZIP_FORMAT: - zip_dbg("GZIP Format: %d ", zip_ops->format); - zip_ops->csum = result_ptr->s.crc32; - break; - - case LZS_FORMAT: - zip_dbg("LZS Format: %d ", zip_ops->format); - break; - - default: - zip_err("Format error:%d\n", zip_ops->format); - } - } - - return 0; -} diff --git a/drivers/crypto/cavium/zip/zip_inflate.h b/drivers/crypto/cavium/zip/zip_inflate.h deleted file mode 100644 index 6b20f179978e..000000000000 --- a/drivers/crypto/cavium/zip/zip_inflate.h +++ /dev/null @@ -1,62 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_INFLATE_H__ -#define __ZIP_INFLATE_H__ - -/** - * zip_inflate - API to offload inflate operation to hardware - * @zip_ops: Pointer to zip operation structure - * @s: Pointer to the structure representing zip state - * @zip_dev: Pointer to the structure representing zip device - * - * This function prepares the zip inflate command and submits it to the zip - * engine for processing. - * - * Return: 0 if successful or error code - */ -int zip_inflate(struct zip_operation *zip_ops, struct zip_state *s, - struct zip_device *zip_dev); -#endif diff --git a/drivers/crypto/cavium/zip/zip_main.c b/drivers/crypto/cavium/zip/zip_main.c deleted file mode 100644 index abd58de4343d..000000000000 --- a/drivers/crypto/cavium/zip/zip_main.c +++ /dev/null @@ -1,603 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include "common.h" -#include "zip_crypto.h" - -#define DRV_NAME "ThunderX-ZIP" - -static struct zip_device *zip_dev[MAX_ZIP_DEVICES]; - -static const struct pci_device_id zip_id_table[] = { - { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDERX_ZIP) }, - { 0, } -}; - -static void zip_debugfs_init(void); -static void zip_debugfs_exit(void); -static int zip_register_compression_device(void); -static void zip_unregister_compression_device(void); - -void zip_reg_write(u64 val, u64 __iomem *addr) -{ - writeq(val, addr); -} - -u64 zip_reg_read(u64 __iomem *addr) -{ - return readq(addr); -} - -/* - * Allocates new ZIP device structure - * Returns zip_device pointer or NULL if cannot allocate memory for zip_device - */ -static struct zip_device *zip_alloc_device(struct pci_dev *pdev) -{ - struct zip_device *zip = NULL; - int idx; - - for (idx = 0; idx < MAX_ZIP_DEVICES; idx++) { - if (!zip_dev[idx]) - break; - } - - /* To ensure that the index is within the limit */ - if (idx < MAX_ZIP_DEVICES) - zip = devm_kzalloc(&pdev->dev, sizeof(*zip), GFP_KERNEL); - - if (!zip) - return NULL; - - zip_dev[idx] = zip; - zip->index = idx; - return zip; -} - -/** - * zip_get_device - Get ZIP device based on node id of cpu - * - * @node: Node id of the current cpu - * Return: Pointer to Zip device structure - */ -struct zip_device *zip_get_device(int node) -{ - if ((node < MAX_ZIP_DEVICES) && (node >= 0)) - return zip_dev[node]; - - zip_err("ZIP device not found for node id %d\n", node); - return NULL; -} - -/** - * zip_get_node_id - Get the node id of the current cpu - * - * Return: Node id of the current cpu - */ -int zip_get_node_id(void) -{ - return cpu_to_node(raw_smp_processor_id()); -} - -/* Initializes the ZIP h/w sub-system */ -static int zip_init_hw(struct zip_device *zip) -{ - union zip_cmd_ctl cmd_ctl; - union zip_constants constants; - union zip_que_ena que_ena; - union zip_quex_map que_map; - union zip_que_pri que_pri; - - union zip_quex_sbuf_addr que_sbuf_addr; - union zip_quex_sbuf_ctl que_sbuf_ctl; - - int q = 0; - - /* Enable the ZIP Engine(Core) Clock */ - cmd_ctl.u_reg64 = zip_reg_read(zip->reg_base + ZIP_CMD_CTL); - cmd_ctl.s.forceclk = 1; - zip_reg_write(cmd_ctl.u_reg64 & 0xFF, (zip->reg_base + ZIP_CMD_CTL)); - - zip_msg("ZIP_CMD_CTL : 0x%016llx", - zip_reg_read(zip->reg_base + ZIP_CMD_CTL)); - - constants.u_reg64 = zip_reg_read(zip->reg_base + ZIP_CONSTANTS); - zip->depth = constants.s.depth; - zip->onfsize = constants.s.onfsize; - zip->ctxsize = constants.s.ctxsize; - - zip_msg("depth: 0x%016llx , onfsize : 0x%016llx , ctxsize : 0x%016llx", - zip->depth, zip->onfsize, zip->ctxsize); - - /* - * Program ZIP_QUE(0..7)_SBUF_ADDR and ZIP_QUE(0..7)_SBUF_CTL to - * have the correct buffer pointer and size configured for each - * instruction queue. - */ - for (q = 0; q < ZIP_NUM_QUEUES; q++) { - que_sbuf_ctl.u_reg64 = 0ull; - que_sbuf_ctl.s.size = (ZIP_CMD_QBUF_SIZE / sizeof(u64)); - que_sbuf_ctl.s.inst_be = 0; - que_sbuf_ctl.s.stream_id = 0; - zip_reg_write(que_sbuf_ctl.u_reg64, - (zip->reg_base + ZIP_QUEX_SBUF_CTL(q))); - - zip_msg("QUEX_SBUF_CTL[%d]: 0x%016llx", q, - zip_reg_read(zip->reg_base + ZIP_QUEX_SBUF_CTL(q))); - } - - for (q = 0; q < ZIP_NUM_QUEUES; q++) { - memset(&zip->iq[q], 0x0, sizeof(struct zip_iq)); - - spin_lock_init(&zip->iq[q].lock); - - if (zip_cmd_qbuf_alloc(zip, q)) { - while (q != 0) { - q--; - zip_cmd_qbuf_free(zip, q); - } - return -ENOMEM; - } - - /* Initialize tail ptr to head */ - zip->iq[q].sw_tail = zip->iq[q].sw_head; - zip->iq[q].hw_tail = zip->iq[q].sw_head; - - /* Write the physical addr to register */ - que_sbuf_addr.u_reg64 = 0ull; - que_sbuf_addr.s.ptr = (__pa(zip->iq[q].sw_head) >> - ZIP_128B_ALIGN); - - zip_msg("QUE[%d]_PTR(PHYS): 0x%016llx", q, - (u64)que_sbuf_addr.s.ptr); - - zip_reg_write(que_sbuf_addr.u_reg64, - (zip->reg_base + ZIP_QUEX_SBUF_ADDR(q))); - - zip_msg("QUEX_SBUF_ADDR[%d]: 0x%016llx", q, - zip_reg_read(zip->reg_base + ZIP_QUEX_SBUF_ADDR(q))); - - zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", - zip->iq[q].sw_head, zip->iq[q].sw_tail, - zip->iq[q].hw_tail); - zip_dbg("sw_head phy addr : 0x%lx", que_sbuf_addr.s.ptr); - } - - /* - * Queue-to-ZIP core mapping - * If a queue is not mapped to a particular core, it is equivalent to - * the ZIP core being disabled. - */ - que_ena.u_reg64 = 0x0ull; - /* Enabling queues based on ZIP_NUM_QUEUES */ - for (q = 0; q < ZIP_NUM_QUEUES; q++) - que_ena.s.ena |= (0x1 << q); - zip_reg_write(que_ena.u_reg64, (zip->reg_base + ZIP_QUE_ENA)); - - zip_msg("QUE_ENA : 0x%016llx", - zip_reg_read(zip->reg_base + ZIP_QUE_ENA)); - - for (q = 0; q < ZIP_NUM_QUEUES; q++) { - que_map.u_reg64 = 0ull; - /* Mapping each queue to two ZIP cores */ - que_map.s.zce = 0x3; - zip_reg_write(que_map.u_reg64, - (zip->reg_base + ZIP_QUEX_MAP(q))); - - zip_msg("QUE_MAP(%d) : 0x%016llx", q, - zip_reg_read(zip->reg_base + ZIP_QUEX_MAP(q))); - } - - que_pri.u_reg64 = 0ull; - for (q = 0; q < ZIP_NUM_QUEUES; q++) - que_pri.s.pri |= (0x1 << q); /* Higher Priority RR */ - zip_reg_write(que_pri.u_reg64, (zip->reg_base + ZIP_QUE_PRI)); - - zip_msg("QUE_PRI %016llx", zip_reg_read(zip->reg_base + ZIP_QUE_PRI)); - - return 0; -} - -static void zip_reset(struct zip_device *zip) -{ - union zip_cmd_ctl cmd_ctl; - - cmd_ctl.u_reg64 = 0x0ull; - cmd_ctl.s.reset = 1; /* Forces ZIP cores to do reset */ - zip_reg_write(cmd_ctl.u_reg64, (zip->reg_base + ZIP_CMD_CTL)); -} - -static int zip_probe(struct pci_dev *pdev, const struct pci_device_id *ent) -{ - struct device *dev = &pdev->dev; - struct zip_device *zip = NULL; - int err; - - zip = zip_alloc_device(pdev); - if (!zip) - return -ENOMEM; - - dev_info(dev, "Found ZIP device %d %x:%x on Node %d\n", zip->index, - pdev->vendor, pdev->device, dev_to_node(dev)); - - pci_set_drvdata(pdev, zip); - zip->pdev = pdev; - - err = pci_enable_device(pdev); - if (err) { - dev_err(dev, "Failed to enable PCI device"); - goto err_free_device; - } - - err = pci_request_regions(pdev, DRV_NAME); - if (err) { - dev_err(dev, "PCI request regions failed 0x%x", err); - goto err_disable_device; - } - - err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); - goto err_release_regions; - } - - /* MAP configuration registers */ - zip->reg_base = pci_ioremap_bar(pdev, PCI_CFG_ZIP_PF_BAR0); - if (!zip->reg_base) { - dev_err(dev, "ZIP: Cannot map BAR0 CSR memory space, aborting"); - err = -ENOMEM; - goto err_release_regions; - } - - /* Initialize ZIP Hardware */ - err = zip_init_hw(zip); - if (err) - goto err_release_regions; - - /* Register with the Kernel Crypto Interface */ - err = zip_register_compression_device(); - if (err < 0) { - zip_err("ZIP: Kernel Crypto Registration failed\n"); - goto err_register; - } - - /* comp-decomp statistics are handled with debugfs interface */ - zip_debugfs_init(); - - return 0; - -err_register: - zip_reset(zip); - -err_release_regions: - if (zip->reg_base) - iounmap(zip->reg_base); - pci_release_regions(pdev); - -err_disable_device: - pci_disable_device(pdev); - -err_free_device: - pci_set_drvdata(pdev, NULL); - - /* Remove zip_dev from zip_device list, free the zip_device memory */ - zip_dev[zip->index] = NULL; - devm_kfree(dev, zip); - - return err; -} - -static void zip_remove(struct pci_dev *pdev) -{ - struct zip_device *zip = pci_get_drvdata(pdev); - int q = 0; - - if (!zip) - return; - - zip_debugfs_exit(); - - zip_unregister_compression_device(); - - if (zip->reg_base) { - zip_reset(zip); - iounmap(zip->reg_base); - } - - pci_release_regions(pdev); - pci_disable_device(pdev); - - /* - * Free Command Queue buffers. This free should be called for all - * the enabled Queues. - */ - for (q = 0; q < ZIP_NUM_QUEUES; q++) - zip_cmd_qbuf_free(zip, q); - - pci_set_drvdata(pdev, NULL); - /* remove zip device from zip device list */ - zip_dev[zip->index] = NULL; -} - -/* PCI Sub-System Interface */ -static struct pci_driver zip_driver = { - .name = DRV_NAME, - .id_table = zip_id_table, - .probe = zip_probe, - .remove = zip_remove, -}; - -/* Kernel Crypto Subsystem Interface */ - -static struct scomp_alg zip_scomp_deflate = { - .alloc_ctx = zip_alloc_scomp_ctx_deflate, - .free_ctx = zip_free_scomp_ctx, - .compress = zip_scomp_compress, - .decompress = zip_scomp_decompress, - .base = { - .cra_name = "deflate", - .cra_driver_name = "deflate-scomp-cavium", - .cra_module = THIS_MODULE, - .cra_priority = 300, - } -}; - -static struct scomp_alg zip_scomp_lzs = { - .alloc_ctx = zip_alloc_scomp_ctx_lzs, - .free_ctx = zip_free_scomp_ctx, - .compress = zip_scomp_compress, - .decompress = zip_scomp_decompress, - .base = { - .cra_name = "lzs", - .cra_driver_name = "lzs-scomp-cavium", - .cra_module = THIS_MODULE, - .cra_priority = 300, - } -}; - -static int zip_register_compression_device(void) -{ - int ret; - - ret = crypto_register_scomp(&zip_scomp_deflate); - if (ret < 0) { - zip_err("Deflate scomp algorithm registration failed\n"); - return ret; - } - - ret = crypto_register_scomp(&zip_scomp_lzs); - if (ret < 0) { - zip_err("LZS scomp algorithm registration failed\n"); - goto err_unregister_scomp_deflate; - } - - return ret; - -err_unregister_scomp_deflate: - crypto_unregister_scomp(&zip_scomp_deflate); - - return ret; -} - -static void zip_unregister_compression_device(void) -{ - crypto_unregister_scomp(&zip_scomp_deflate); - crypto_unregister_scomp(&zip_scomp_lzs); -} - -/* - * debugfs functions - */ -#ifdef CONFIG_DEBUG_FS -#include <linux/debugfs.h> - -/* Displays ZIP device statistics */ -static int zip_stats_show(struct seq_file *s, void *unused) -{ - u64 val = 0ull; - u64 avg_chunk = 0ull, avg_cr = 0ull; - u32 q = 0; - - int index = 0; - struct zip_device *zip; - struct zip_stats *st; - - for (index = 0; index < MAX_ZIP_DEVICES; index++) { - u64 pending = 0; - - if (zip_dev[index]) { - zip = zip_dev[index]; - st = &zip->stats; - - /* Get all the pending requests */ - for (q = 0; q < ZIP_NUM_QUEUES; q++) { - val = zip_reg_read((zip->reg_base + - ZIP_DBG_QUEX_STA(q))); - pending += val >> 32 & 0xffffff; - } - - val = atomic64_read(&st->comp_req_complete); - avg_chunk = (val) ? atomic64_read(&st->comp_in_bytes) / val : 0; - - val = atomic64_read(&st->comp_out_bytes); - avg_cr = (val) ? atomic64_read(&st->comp_in_bytes) / val : 0; - seq_printf(s, " ZIP Device %d Stats\n" - "-----------------------------------\n" - "Comp Req Submitted : \t%lld\n" - "Comp Req Completed : \t%lld\n" - "Compress In Bytes : \t%lld\n" - "Compressed Out Bytes : \t%lld\n" - "Average Chunk size : \t%llu\n" - "Average Compression ratio : \t%llu\n" - "Decomp Req Submitted : \t%lld\n" - "Decomp Req Completed : \t%lld\n" - "Decompress In Bytes : \t%lld\n" - "Decompressed Out Bytes : \t%lld\n" - "Decompress Bad requests : \t%lld\n" - "Pending Req : \t%lld\n" - "---------------------------------\n", - index, - (u64)atomic64_read(&st->comp_req_submit), - (u64)atomic64_read(&st->comp_req_complete), - (u64)atomic64_read(&st->comp_in_bytes), - (u64)atomic64_read(&st->comp_out_bytes), - avg_chunk, - avg_cr, - (u64)atomic64_read(&st->decomp_req_submit), - (u64)atomic64_read(&st->decomp_req_complete), - (u64)atomic64_read(&st->decomp_in_bytes), - (u64)atomic64_read(&st->decomp_out_bytes), - (u64)atomic64_read(&st->decomp_bad_reqs), - pending); - } - } - return 0; -} - -/* Clears stats data */ -static int zip_clear_show(struct seq_file *s, void *unused) -{ - int index = 0; - - for (index = 0; index < MAX_ZIP_DEVICES; index++) { - if (zip_dev[index]) { - memset(&zip_dev[index]->stats, 0, - sizeof(struct zip_stats)); - seq_printf(s, "Cleared stats for zip %d\n", index); - } - } - - return 0; -} - -static struct zip_registers zipregs[64] = { - {"ZIP_CMD_CTL ", 0x0000ull}, - {"ZIP_THROTTLE ", 0x0010ull}, - {"ZIP_CONSTANTS ", 0x00A0ull}, - {"ZIP_QUE0_MAP ", 0x1400ull}, - {"ZIP_QUE1_MAP ", 0x1408ull}, - {"ZIP_QUE_ENA ", 0x0500ull}, - {"ZIP_QUE_PRI ", 0x0508ull}, - {"ZIP_QUE0_DONE ", 0x2000ull}, - {"ZIP_QUE1_DONE ", 0x2008ull}, - {"ZIP_QUE0_DOORBELL ", 0x4000ull}, - {"ZIP_QUE1_DOORBELL ", 0x4008ull}, - {"ZIP_QUE0_SBUF_ADDR ", 0x1000ull}, - {"ZIP_QUE1_SBUF_ADDR ", 0x1008ull}, - {"ZIP_QUE0_SBUF_CTL ", 0x1200ull}, - {"ZIP_QUE1_SBUF_CTL ", 0x1208ull}, - { NULL, 0} -}; - -/* Prints registers' contents */ -static int zip_regs_show(struct seq_file *s, void *unused) -{ - u64 val = 0; - int i = 0, index = 0; - - for (index = 0; index < MAX_ZIP_DEVICES; index++) { - if (zip_dev[index]) { - seq_printf(s, "--------------------------------\n" - " ZIP Device %d Registers\n" - "--------------------------------\n", - index); - - i = 0; - - while (zipregs[i].reg_name) { - val = zip_reg_read((zip_dev[index]->reg_base + - zipregs[i].reg_offset)); - seq_printf(s, "%s: 0x%016llx\n", - zipregs[i].reg_name, val); - i++; - } - } - } - return 0; -} - -DEFINE_SHOW_ATTRIBUTE(zip_stats); -DEFINE_SHOW_ATTRIBUTE(zip_clear); -DEFINE_SHOW_ATTRIBUTE(zip_regs); - -/* Root directory for thunderx_zip debugfs entry */ -static struct dentry *zip_debugfs_root; - -static void zip_debugfs_init(void) -{ - if (!debugfs_initialized()) - return; - - zip_debugfs_root = debugfs_create_dir("thunderx_zip", NULL); - - /* Creating files for entries inside thunderx_zip directory */ - debugfs_create_file("zip_stats", 0444, zip_debugfs_root, NULL, - &zip_stats_fops); - - debugfs_create_file("zip_clear", 0444, zip_debugfs_root, NULL, - &zip_clear_fops); - - debugfs_create_file("zip_regs", 0444, zip_debugfs_root, NULL, - &zip_regs_fops); - -} - -static void zip_debugfs_exit(void) -{ - debugfs_remove_recursive(zip_debugfs_root); -} - -#else -static void __init zip_debugfs_init(void) { } -static void __exit zip_debugfs_exit(void) { } -#endif -/* debugfs - end */ - -module_pci_driver(zip_driver); - -MODULE_AUTHOR("Cavium Inc"); -MODULE_DESCRIPTION("Cavium Inc ThunderX ZIP Driver"); -MODULE_LICENSE("GPL v2"); -MODULE_DEVICE_TABLE(pci, zip_id_table); diff --git a/drivers/crypto/cavium/zip/zip_main.h b/drivers/crypto/cavium/zip/zip_main.h deleted file mode 100644 index e1e4fa92ce80..000000000000 --- a/drivers/crypto/cavium/zip/zip_main.h +++ /dev/null @@ -1,120 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_MAIN_H__ -#define __ZIP_MAIN_H__ - -#include "zip_device.h" -#include "zip_regs.h" - -/* PCI device IDs */ -#define PCI_DEVICE_ID_THUNDERX_ZIP 0xA01A - -/* ZIP device BARs */ -#define PCI_CFG_ZIP_PF_BAR0 0 /* Base addr for normal regs */ - -/* Maximum available zip queues */ -#define ZIP_MAX_NUM_QUEUES 8 - -#define ZIP_128B_ALIGN 7 - -/* Command queue buffer size */ -#define ZIP_CMD_QBUF_SIZE (8064 + 8) - -struct zip_registers { - char *reg_name; - u64 reg_offset; -}; - -/* ZIP Compression - Decompression stats */ -struct zip_stats { - atomic64_t comp_req_submit; - atomic64_t comp_req_complete; - atomic64_t decomp_req_submit; - atomic64_t decomp_req_complete; - atomic64_t comp_in_bytes; - atomic64_t comp_out_bytes; - atomic64_t decomp_in_bytes; - atomic64_t decomp_out_bytes; - atomic64_t decomp_bad_reqs; -}; - -/* ZIP Instruction Queue */ -struct zip_iq { - u64 *sw_head; - u64 *sw_tail; - u64 *hw_tail; - u64 done_cnt; - u64 pend_cnt; - u64 free_flag; - - /* ZIP IQ lock */ - spinlock_t lock; -}; - -/* ZIP Device */ -struct zip_device { - u32 index; - void __iomem *reg_base; - struct pci_dev *pdev; - - /* Different ZIP Constants */ - u64 depth; - u64 onfsize; - u64 ctxsize; - - struct zip_iq iq[ZIP_MAX_NUM_QUEUES]; - struct zip_stats stats; -}; - -/* Prototypes */ -struct zip_device *zip_get_device(int node_id); -int zip_get_node_id(void); -void zip_reg_write(u64 val, u64 __iomem *addr); -u64 zip_reg_read(u64 __iomem *addr); -void zip_update_cmd_bufs(struct zip_device *zip_dev, u32 queue); -u32 zip_load_instr(union zip_inst_s *instr, struct zip_device *zip_dev); - -#endif /* ZIP_MAIN_H */ diff --git a/drivers/crypto/cavium/zip/zip_mem.c b/drivers/crypto/cavium/zip/zip_mem.c deleted file mode 100644 index b3e0843a9169..000000000000 --- a/drivers/crypto/cavium/zip/zip_mem.c +++ /dev/null @@ -1,114 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include <linux/types.h> -#include <linux/vmalloc.h> - -#include "common.h" - -/** - * zip_cmd_qbuf_alloc - Allocates a cmd buffer for ZIP Instruction Queue - * @zip: Pointer to zip device structure - * @q: Queue number to allocate bufffer to - * Return: 0 if successful, -ENOMEM otherwise - */ -int zip_cmd_qbuf_alloc(struct zip_device *zip, int q) -{ - zip->iq[q].sw_head = (u64 *)__get_free_pages((GFP_KERNEL | GFP_DMA), - get_order(ZIP_CMD_QBUF_SIZE)); - - if (!zip->iq[q].sw_head) - return -ENOMEM; - - memset(zip->iq[q].sw_head, 0, ZIP_CMD_QBUF_SIZE); - - zip_dbg("cmd_qbuf_alloc[%d] Success : %p\n", q, zip->iq[q].sw_head); - return 0; -} - -/** - * zip_cmd_qbuf_free - Frees the cmd Queue buffer - * @zip: Pointer to zip device structure - * @q: Queue number to free buffer of - */ -void zip_cmd_qbuf_free(struct zip_device *zip, int q) -{ - zip_dbg("Freeing cmd_qbuf 0x%lx\n", zip->iq[q].sw_tail); - - free_pages((u64)zip->iq[q].sw_tail, get_order(ZIP_CMD_QBUF_SIZE)); -} - -/** - * zip_data_buf_alloc - Allocates memory for a data bufffer - * @size: Size of the buffer to allocate - * Returns: Pointer to the buffer allocated - */ -u8 *zip_data_buf_alloc(u64 size) -{ - u8 *ptr; - - ptr = (u8 *)__get_free_pages((GFP_KERNEL | GFP_DMA), - get_order(size)); - - if (!ptr) - return NULL; - - memset(ptr, 0, size); - - zip_dbg("Data buffer allocation success\n"); - return ptr; -} - -/** - * zip_data_buf_free - Frees the memory of a data buffer - * @ptr: Pointer to the buffer - * @size: Buffer size - */ -void zip_data_buf_free(u8 *ptr, u64 size) -{ - zip_dbg("Freeing data buffer 0x%lx\n", ptr); - - free_pages((u64)ptr, get_order(size)); -} diff --git a/drivers/crypto/cavium/zip/zip_mem.h b/drivers/crypto/cavium/zip/zip_mem.h deleted file mode 100644 index f8f2f08c4a5c..000000000000 --- a/drivers/crypto/cavium/zip/zip_mem.h +++ /dev/null @@ -1,78 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_MEM_H__ -#define __ZIP_MEM_H__ - -/** - * zip_cmd_qbuf_free - Frees the cmd Queue buffer - * @zip: Pointer to zip device structure - * @q: Queue nmber to free buffer of - */ -void zip_cmd_qbuf_free(struct zip_device *zip, int q); - -/** - * zip_cmd_qbuf_alloc - Allocates a Chunk/cmd buffer for ZIP Inst(cmd) Queue - * @zip: Pointer to zip device structure - * @q: Queue number to allocate bufffer to - * Return: 0 if successful, 1 otherwise - */ -int zip_cmd_qbuf_alloc(struct zip_device *zip, int q); - -/** - * zip_data_buf_alloc - Allocates memory for a data bufffer - * @size: Size of the buffer to allocate - * Returns: Pointer to the buffer allocated - */ -u8 *zip_data_buf_alloc(u64 size); - -/** - * zip_data_buf_free - Frees the memory of a data buffer - * @ptr: Pointer to the buffer - * @size: Buffer size - */ -void zip_data_buf_free(u8 *ptr, u64 size); - -#endif diff --git a/drivers/crypto/cavium/zip/zip_regs.h b/drivers/crypto/cavium/zip/zip_regs.h deleted file mode 100644 index 874e0236c87e..000000000000 --- a/drivers/crypto/cavium/zip/zip_regs.h +++ /dev/null @@ -1,1347 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_REGS_H__ -#define __ZIP_REGS_H__ - -/* - * Configuration and status register (CSR) address and type definitions for - * Cavium ZIP. - */ - -#include <linux/kern_levels.h> - -/* ZIP invocation result completion status codes */ -#define ZIP_CMD_NOTDONE 0x0 - -/* Successful completion. */ -#define ZIP_CMD_SUCCESS 0x1 - -/* Output truncated */ -#define ZIP_CMD_DTRUNC 0x2 - -/* Dynamic Stop */ -#define ZIP_CMD_DYNAMIC_STOP 0x3 - -/* Uncompress ran out of input data when IWORD0[EF] was set */ -#define ZIP_CMD_ITRUNC 0x4 - -/* Uncompress found the reserved block type 3 */ -#define ZIP_CMD_RBLOCK 0x5 - -/* - * Uncompress found LEN != ZIP_CMD_NLEN in an uncompressed block in the input. - */ -#define ZIP_CMD_NLEN 0x6 - -/* Uncompress found a bad code in the main Huffman codes. */ -#define ZIP_CMD_BADCODE 0x7 - -/* Uncompress found a bad code in the 19 Huffman codes encoding lengths. */ -#define ZIP_CMD_BADCODE2 0x8 - -/* Compress found a zero-length input. */ -#define ZIP_CMD_ZERO_LEN 0x9 - -/* The compress or decompress encountered an internal parity error. */ -#define ZIP_CMD_PARITY 0xA - -/* - * Uncompress found a string identifier that precedes the uncompressed data and - * decompression history. - */ -#define ZIP_CMD_FATAL 0xB - -/** - * enum zip_int_vec_e - ZIP MSI-X Vector Enumeration, enumerates the MSI-X - * interrupt vectors. - */ -enum zip_int_vec_e { - ZIP_INT_VEC_E_ECCE = 0x10, - ZIP_INT_VEC_E_FIFE = 0x11, - ZIP_INT_VEC_E_QUE0_DONE = 0x0, - ZIP_INT_VEC_E_QUE0_ERR = 0x8, - ZIP_INT_VEC_E_QUE1_DONE = 0x1, - ZIP_INT_VEC_E_QUE1_ERR = 0x9, - ZIP_INT_VEC_E_QUE2_DONE = 0x2, - ZIP_INT_VEC_E_QUE2_ERR = 0xa, - ZIP_INT_VEC_E_QUE3_DONE = 0x3, - ZIP_INT_VEC_E_QUE3_ERR = 0xb, - ZIP_INT_VEC_E_QUE4_DONE = 0x4, - ZIP_INT_VEC_E_QUE4_ERR = 0xc, - ZIP_INT_VEC_E_QUE5_DONE = 0x5, - ZIP_INT_VEC_E_QUE5_ERR = 0xd, - ZIP_INT_VEC_E_QUE6_DONE = 0x6, - ZIP_INT_VEC_E_QUE6_ERR = 0xe, - ZIP_INT_VEC_E_QUE7_DONE = 0x7, - ZIP_INT_VEC_E_QUE7_ERR = 0xf, - ZIP_INT_VEC_E_ENUM_LAST = 0x12, -}; - -/** - * union zip_zptr_addr_s - ZIP Generic Pointer Structure for ADDR. - * - * It is the generic format of pointers in ZIP_INST_S. - */ -union zip_zptr_addr_s { - u64 u_reg64; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_49_63 : 15; - u64 addr : 49; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 addr : 49; - u64 reserved_49_63 : 15; -#endif - } s; - -}; - -/** - * union zip_zptr_ctl_s - ZIP Generic Pointer Structure for CTL. - * - * It is the generic format of pointers in ZIP_INST_S. - */ -union zip_zptr_ctl_s { - u64 u_reg64; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_112_127 : 16; - u64 length : 16; - u64 reserved_67_95 : 29; - u64 fw : 1; - u64 nc : 1; - u64 data_be : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 data_be : 1; - u64 nc : 1; - u64 fw : 1; - u64 reserved_67_95 : 29; - u64 length : 16; - u64 reserved_112_127 : 16; -#endif - } s; -}; - -/** - * union zip_inst_s - ZIP Instruction Structure. - * Each ZIP instruction has 16 words (they are called IWORD0 to IWORD15 within - * the structure). - */ -union zip_inst_s { - u64 u_reg64[16]; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 doneint : 1; - u64 reserved_56_62 : 7; - u64 totaloutputlength : 24; - u64 reserved_27_31 : 5; - u64 exn : 3; - u64 reserved_23_23 : 1; - u64 exbits : 7; - u64 reserved_12_15 : 4; - u64 sf : 1; - u64 ss : 2; - u64 cc : 2; - u64 ef : 1; - u64 bf : 1; - u64 ce : 1; - u64 reserved_3_3 : 1; - u64 ds : 1; - u64 dg : 1; - u64 hg : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 hg : 1; - u64 dg : 1; - u64 ds : 1; - u64 reserved_3_3 : 1; - u64 ce : 1; - u64 bf : 1; - u64 ef : 1; - u64 cc : 2; - u64 ss : 2; - u64 sf : 1; - u64 reserved_12_15 : 4; - u64 exbits : 7; - u64 reserved_23_23 : 1; - u64 exn : 3; - u64 reserved_27_31 : 5; - u64 totaloutputlength : 24; - u64 reserved_56_62 : 7; - u64 doneint : 1; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 historylength : 16; - u64 reserved_96_111 : 16; - u64 adlercrc32 : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 adlercrc32 : 32; - u64 reserved_96_111 : 16; - u64 historylength : 16; -#endif - union zip_zptr_addr_s ctx_ptr_addr; - union zip_zptr_ctl_s ctx_ptr_ctl; - union zip_zptr_addr_s his_ptr_addr; - union zip_zptr_ctl_s his_ptr_ctl; - union zip_zptr_addr_s inp_ptr_addr; - union zip_zptr_ctl_s inp_ptr_ctl; - union zip_zptr_addr_s out_ptr_addr; - union zip_zptr_ctl_s out_ptr_ctl; - union zip_zptr_addr_s res_ptr_addr; - union zip_zptr_ctl_s res_ptr_ctl; -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_817_831 : 15; - u64 wq_ptr : 49; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 wq_ptr : 49; - u64 reserved_817_831 : 15; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_882_895 : 14; - u64 tt : 2; - u64 reserved_874_879 : 6; - u64 grp : 10; - u64 tag : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 tag : 32; - u64 grp : 10; - u64 reserved_874_879 : 6; - u64 tt : 2; - u64 reserved_882_895 : 14; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_896_959 : 64; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 reserved_896_959 : 64; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_960_1023 : 64; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 reserved_960_1023 : 64; -#endif - } s; -}; - -/** - * union zip_nptr_s - ZIP Instruction Next-Chunk-Buffer Pointer (NPTR) - * Structure - * - * ZIP_NPTR structure is used to chain all the zip instruction buffers - * together. ZIP instruction buffers are managed (allocated and released) by - * the software. - */ -union zip_nptr_s { - u64 u_reg64; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_49_63 : 15; - u64 addr : 49; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 addr : 49; - u64 reserved_49_63 : 15; -#endif - } s; -}; - -/** - * union zip_zptr_s - ZIP Generic Pointer Structure. - * - * It is the generic format of pointers in ZIP_INST_S. - */ -union zip_zptr_s { - u64 u_reg64[2]; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_49_63 : 15; - u64 addr : 49; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 addr : 49; - u64 reserved_49_63 : 15; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_112_127 : 16; - u64 length : 16; - u64 reserved_67_95 : 29; - u64 fw : 1; - u64 nc : 1; - u64 data_be : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 data_be : 1; - u64 nc : 1; - u64 fw : 1; - u64 reserved_67_95 : 29; - u64 length : 16; - u64 reserved_112_127 : 16; -#endif - } s; -}; - -/** - * union zip_zres_s - ZIP Result Structure - * - * The ZIP coprocessor writes the result structure after it completes the - * invocation. The result structure is exactly 24 bytes, and each invocation of - * the ZIP coprocessor produces exactly one result structure. - */ -union zip_zres_s { - u64 u_reg64[3]; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 crc32 : 32; - u64 adler32 : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 adler32 : 32; - u64 crc32 : 32; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 totalbyteswritten : 32; - u64 totalbytesread : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 totalbytesread : 32; - u64 totalbyteswritten : 32; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 totalbitsprocessed : 32; - u64 doneint : 1; - u64 reserved_155_158 : 4; - u64 exn : 3; - u64 reserved_151_151 : 1; - u64 exbits : 7; - u64 reserved_137_143 : 7; - u64 ef : 1; - - volatile u64 compcode : 8; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - - volatile u64 compcode : 8; - u64 ef : 1; - u64 reserved_137_143 : 7; - u64 exbits : 7; - u64 reserved_151_151 : 1; - u64 exn : 3; - u64 reserved_155_158 : 4; - u64 doneint : 1; - u64 totalbitsprocessed : 32; -#endif - } s; -}; - -/** - * union zip_cmd_ctl - Structure representing the register that controls - * clock and reset. - */ -union zip_cmd_ctl { - u64 u_reg64; - struct zip_cmd_ctl_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_2_63 : 62; - u64 forceclk : 1; - u64 reset : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 reset : 1; - u64 forceclk : 1; - u64 reserved_2_63 : 62; -#endif - } s; -}; - -#define ZIP_CMD_CTL 0x0ull - -/** - * union zip_constants - Data structure representing the register that contains - * all of the current implementation-related parameters of the zip core in this - * chip. - */ -union zip_constants { - u64 u_reg64; - struct zip_constants_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 nexec : 8; - u64 reserved_49_55 : 7; - u64 syncflush_capable : 1; - u64 depth : 16; - u64 onfsize : 12; - u64 ctxsize : 12; - u64 reserved_1_7 : 7; - u64 disabled : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 disabled : 1; - u64 reserved_1_7 : 7; - u64 ctxsize : 12; - u64 onfsize : 12; - u64 depth : 16; - u64 syncflush_capable : 1; - u64 reserved_49_55 : 7; - u64 nexec : 8; -#endif - } s; -}; - -#define ZIP_CONSTANTS 0x00A0ull - -/** - * union zip_corex_bist_status - Represents registers which have the BIST - * status of memories in zip cores. - * - * Each bit is the BIST result of an individual memory - * (per bit, 0 = pass and 1 = fail). - */ -union zip_corex_bist_status { - u64 u_reg64; - struct zip_corex_bist_status_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_53_63 : 11; - u64 bstatus : 53; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 bstatus : 53; - u64 reserved_53_63 : 11; -#endif - } s; -}; - -static inline u64 ZIP_COREX_BIST_STATUS(u64 param1) -{ - if (param1 <= 1) - return 0x0520ull + (param1 & 1) * 0x8ull; - pr_err("ZIP_COREX_BIST_STATUS: %llu\n", param1); - return 0; -} - -/** - * union zip_ctl_bist_status - Represents register that has the BIST status of - * memories in ZIP_CTL (instruction buffer, G/S pointer FIFO, input data - * buffer, output data buffers). - * - * Each bit is the BIST result of an individual memory - * (per bit, 0 = pass and 1 = fail). - */ -union zip_ctl_bist_status { - u64 u_reg64; - struct zip_ctl_bist_status_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_9_63 : 55; - u64 bstatus : 9; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 bstatus : 9; - u64 reserved_9_63 : 55; -#endif - } s; -}; - -#define ZIP_CTL_BIST_STATUS 0x0510ull - -/** - * union zip_ctl_cfg - Represents the register that controls the behavior of - * the ZIP DMA engines. - * - * It is recommended to keep default values for normal operation. Changing the - * values of the fields may be useful for diagnostics. - */ -union zip_ctl_cfg { - u64 u_reg64; - struct zip_ctl_cfg_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_52_63 : 12; - u64 ildf : 4; - u64 reserved_36_47 : 12; - u64 drtf : 4; - u64 reserved_27_31 : 5; - u64 stcf : 3; - u64 reserved_19_23 : 5; - u64 ldf : 3; - u64 reserved_2_15 : 14; - u64 busy : 1; - u64 reserved_0_0 : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 reserved_0_0 : 1; - u64 busy : 1; - u64 reserved_2_15 : 14; - u64 ldf : 3; - u64 reserved_19_23 : 5; - u64 stcf : 3; - u64 reserved_27_31 : 5; - u64 drtf : 4; - u64 reserved_36_47 : 12; - u64 ildf : 4; - u64 reserved_52_63 : 12; -#endif - } s; -}; - -#define ZIP_CTL_CFG 0x0560ull - -/** - * union zip_dbg_corex_inst - Represents the registers that reflect the status - * of the current instruction that the ZIP core is executing or has executed. - * - * These registers are only for debug use. - */ -union zip_dbg_corex_inst { - u64 u_reg64; - struct zip_dbg_corex_inst_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 busy : 1; - u64 reserved_35_62 : 28; - u64 qid : 3; - u64 iid : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 iid : 32; - u64 qid : 3; - u64 reserved_35_62 : 28; - u64 busy : 1; -#endif - } s; -}; - -static inline u64 ZIP_DBG_COREX_INST(u64 param1) -{ - if (param1 <= 1) - return 0x0640ull + (param1 & 1) * 0x8ull; - pr_err("ZIP_DBG_COREX_INST: %llu\n", param1); - return 0; -} - -/** - * union zip_dbg_corex_sta - Represents registers that reflect the status of - * the zip cores. - * - * They are for debug use only. - */ -union zip_dbg_corex_sta { - u64 u_reg64; - struct zip_dbg_corex_sta_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 busy : 1; - u64 reserved_37_62 : 26; - u64 ist : 5; - u64 nie : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 nie : 32; - u64 ist : 5; - u64 reserved_37_62 : 26; - u64 busy : 1; -#endif - } s; -}; - -static inline u64 ZIP_DBG_COREX_STA(u64 param1) -{ - if (param1 <= 1) - return 0x0680ull + (param1 & 1) * 0x8ull; - pr_err("ZIP_DBG_COREX_STA: %llu\n", param1); - return 0; -} - -/** - * union zip_dbg_quex_sta - Represets registers that reflect status of the zip - * instruction queues. - * - * They are for debug use only. - */ -union zip_dbg_quex_sta { - u64 u_reg64; - struct zip_dbg_quex_sta_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 busy : 1; - u64 reserved_56_62 : 7; - u64 rqwc : 24; - u64 nii : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 nii : 32; - u64 rqwc : 24; - u64 reserved_56_62 : 7; - u64 busy : 1; -#endif - } s; -}; - -static inline u64 ZIP_DBG_QUEX_STA(u64 param1) -{ - if (param1 <= 7) - return 0x1800ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_DBG_QUEX_STA: %llu\n", param1); - return 0; -} - -/** - * union zip_ecc_ctl - Represents the register that enables ECC for each - * individual internal memory that requires ECC. - * - * For debug purpose, it can also flip one or two bits in the ECC data. - */ -union zip_ecc_ctl { - u64 u_reg64; - struct zip_ecc_ctl_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_19_63 : 45; - u64 vmem_cdis : 1; - u64 vmem_fs : 2; - u64 reserved_15_15 : 1; - u64 idf1_cdis : 1; - u64 idf1_fs : 2; - u64 reserved_11_11 : 1; - u64 idf0_cdis : 1; - u64 idf0_fs : 2; - u64 reserved_7_7 : 1; - u64 gspf_cdis : 1; - u64 gspf_fs : 2; - u64 reserved_3_3 : 1; - u64 iqf_cdis : 1; - u64 iqf_fs : 2; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 iqf_fs : 2; - u64 iqf_cdis : 1; - u64 reserved_3_3 : 1; - u64 gspf_fs : 2; - u64 gspf_cdis : 1; - u64 reserved_7_7 : 1; - u64 idf0_fs : 2; - u64 idf0_cdis : 1; - u64 reserved_11_11 : 1; - u64 idf1_fs : 2; - u64 idf1_cdis : 1; - u64 reserved_15_15 : 1; - u64 vmem_fs : 2; - u64 vmem_cdis : 1; - u64 reserved_19_63 : 45; -#endif - } s; -}; - -#define ZIP_ECC_CTL 0x0568ull - -/* NCB - zip_ecce_ena_w1c */ -union zip_ecce_ena_w1c { - u64 u_reg64; - struct zip_ecce_ena_w1c_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_37_63 : 27; - u64 dbe : 5; - u64 reserved_5_31 : 27; - u64 sbe : 5; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 sbe : 5; - u64 reserved_5_31 : 27; - u64 dbe : 5; - u64 reserved_37_63 : 27; -#endif - } s; -}; - -#define ZIP_ECCE_ENA_W1C 0x0598ull - -/* NCB - zip_ecce_ena_w1s */ -union zip_ecce_ena_w1s { - u64 u_reg64; - struct zip_ecce_ena_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_37_63 : 27; - u64 dbe : 5; - u64 reserved_5_31 : 27; - u64 sbe : 5; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 sbe : 5; - u64 reserved_5_31 : 27; - u64 dbe : 5; - u64 reserved_37_63 : 27; -#endif - } s; -}; - -#define ZIP_ECCE_ENA_W1S 0x0590ull - -/** - * union zip_ecce_int - Represents the register that contains the status of the - * ECC interrupt sources. - */ -union zip_ecce_int { - u64 u_reg64; - struct zip_ecce_int_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_37_63 : 27; - u64 dbe : 5; - u64 reserved_5_31 : 27; - u64 sbe : 5; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 sbe : 5; - u64 reserved_5_31 : 27; - u64 dbe : 5; - u64 reserved_37_63 : 27; -#endif - } s; -}; - -#define ZIP_ECCE_INT 0x0580ull - -/* NCB - zip_ecce_int_w1s */ -union zip_ecce_int_w1s { - u64 u_reg64; - struct zip_ecce_int_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_37_63 : 27; - u64 dbe : 5; - u64 reserved_5_31 : 27; - u64 sbe : 5; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 sbe : 5; - u64 reserved_5_31 : 27; - u64 dbe : 5; - u64 reserved_37_63 : 27; -#endif - } s; -}; - -#define ZIP_ECCE_INT_W1S 0x0588ull - -/* NCB - zip_fife_ena_w1c */ -union zip_fife_ena_w1c { - u64 u_reg64; - struct zip_fife_ena_w1c_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_42_63 : 22; - u64 asserts : 42; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 asserts : 42; - u64 reserved_42_63 : 22; -#endif - } s; -}; - -#define ZIP_FIFE_ENA_W1C 0x0090ull - -/* NCB - zip_fife_ena_w1s */ -union zip_fife_ena_w1s { - u64 u_reg64; - struct zip_fife_ena_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_42_63 : 22; - u64 asserts : 42; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 asserts : 42; - u64 reserved_42_63 : 22; -#endif - } s; -}; - -#define ZIP_FIFE_ENA_W1S 0x0088ull - -/* NCB - zip_fife_int */ -union zip_fife_int { - u64 u_reg64; - struct zip_fife_int_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_42_63 : 22; - u64 asserts : 42; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 asserts : 42; - u64 reserved_42_63 : 22; -#endif - } s; -}; - -#define ZIP_FIFE_INT 0x0078ull - -/* NCB - zip_fife_int_w1s */ -union zip_fife_int_w1s { - u64 u_reg64; - struct zip_fife_int_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_42_63 : 22; - u64 asserts : 42; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 asserts : 42; - u64 reserved_42_63 : 22; -#endif - } s; -}; - -#define ZIP_FIFE_INT_W1S 0x0080ull - -/** - * union zip_msix_pbax - Represents the register that is the MSI-X PBA table - * - * The bit number is indexed by the ZIP_INT_VEC_E enumeration. - */ -union zip_msix_pbax { - u64 u_reg64; - struct zip_msix_pbax_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 pend : 64; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 pend : 64; -#endif - } s; -}; - -static inline u64 ZIP_MSIX_PBAX(u64 param1) -{ - if (param1 == 0) - return 0x0000838000FF0000ull; - pr_err("ZIP_MSIX_PBAX: %llu\n", param1); - return 0; -} - -/** - * union zip_msix_vecx_addr - Represents the register that is the MSI-X vector - * table, indexed by the ZIP_INT_VEC_E enumeration. - */ -union zip_msix_vecx_addr { - u64 u_reg64; - struct zip_msix_vecx_addr_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_49_63 : 15; - u64 addr : 47; - u64 reserved_1_1 : 1; - u64 secvec : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 secvec : 1; - u64 reserved_1_1 : 1; - u64 addr : 47; - u64 reserved_49_63 : 15; -#endif - } s; -}; - -static inline u64 ZIP_MSIX_VECX_ADDR(u64 param1) -{ - if (param1 <= 17) - return 0x0000838000F00000ull + (param1 & 31) * 0x10ull; - pr_err("ZIP_MSIX_VECX_ADDR: %llu\n", param1); - return 0; -} - -/** - * union zip_msix_vecx_ctl - Represents the register that is the MSI-X vector - * table, indexed by the ZIP_INT_VEC_E enumeration. - */ -union zip_msix_vecx_ctl { - u64 u_reg64; - struct zip_msix_vecx_ctl_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_33_63 : 31; - u64 mask : 1; - u64 reserved_20_31 : 12; - u64 data : 20; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 data : 20; - u64 reserved_20_31 : 12; - u64 mask : 1; - u64 reserved_33_63 : 31; -#endif - } s; -}; - -static inline u64 ZIP_MSIX_VECX_CTL(u64 param1) -{ - if (param1 <= 17) - return 0x0000838000F00008ull + (param1 & 31) * 0x10ull; - pr_err("ZIP_MSIX_VECX_CTL: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_done - Represents the registers that contain the per-queue - * instruction done count. - */ -union zip_quex_done { - u64 u_reg64; - struct zip_quex_done_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_20_63 : 44; - u64 done : 20; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 done : 20; - u64 reserved_20_63 : 44; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DONE(u64 param1) -{ - if (param1 <= 7) - return 0x2000ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DONE: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_done_ack - Represents the registers on write to which will - * decrement the per-queue instructiona done count. - */ -union zip_quex_done_ack { - u64 u_reg64; - struct zip_quex_done_ack_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_20_63 : 44; - u64 done_ack : 20; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 done_ack : 20; - u64 reserved_20_63 : 44; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DONE_ACK(u64 param1) -{ - if (param1 <= 7) - return 0x2200ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DONE_ACK: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_done_ena_w1c - Represents the register which when written - * 1 to will disable the DONEINT interrupt for the queue. - */ -union zip_quex_done_ena_w1c { - u64 u_reg64; - struct zip_quex_done_ena_w1c_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_1_63 : 63; - u64 done_ena : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 done_ena : 1; - u64 reserved_1_63 : 63; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DONE_ENA_W1C(u64 param1) -{ - if (param1 <= 7) - return 0x2600ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DONE_ENA_W1C: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_done_ena_w1s - Represents the register that when written 1 to - * will enable the DONEINT interrupt for the queue. - */ -union zip_quex_done_ena_w1s { - u64 u_reg64; - struct zip_quex_done_ena_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_1_63 : 63; - u64 done_ena : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 done_ena : 1; - u64 reserved_1_63 : 63; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DONE_ENA_W1S(u64 param1) -{ - if (param1 <= 7) - return 0x2400ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DONE_ENA_W1S: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_done_wait - Represents the register that specifies the per - * queue interrupt coalescing settings. - */ -union zip_quex_done_wait { - u64 u_reg64; - struct zip_quex_done_wait_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_48_63 : 16; - u64 time_wait : 16; - u64 reserved_20_31 : 12; - u64 num_wait : 20; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 num_wait : 20; - u64 reserved_20_31 : 12; - u64 time_wait : 16; - u64 reserved_48_63 : 16; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DONE_WAIT(u64 param1) -{ - if (param1 <= 7) - return 0x2800ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DONE_WAIT: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_doorbell - Represents doorbell registers for the ZIP - * instruction queues. - */ -union zip_quex_doorbell { - u64 u_reg64; - struct zip_quex_doorbell_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_20_63 : 44; - u64 dbell_cnt : 20; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 dbell_cnt : 20; - u64 reserved_20_63 : 44; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DOORBELL(u64 param1) -{ - if (param1 <= 7) - return 0x4000ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DOORBELL: %llu\n", param1); - return 0; -} - -union zip_quex_err_ena_w1c { - u64 u_reg64; - struct zip_quex_err_ena_w1c_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_5_63 : 59; - u64 mdbe : 1; - u64 nwrp : 1; - u64 nrrp : 1; - u64 irde : 1; - u64 dovf : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 dovf : 1; - u64 irde : 1; - u64 nrrp : 1; - u64 nwrp : 1; - u64 mdbe : 1; - u64 reserved_5_63 : 59; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_ERR_ENA_W1C(u64 param1) -{ - if (param1 <= 7) - return 0x3600ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_ERR_ENA_W1C: %llu\n", param1); - return 0; -} - -union zip_quex_err_ena_w1s { - u64 u_reg64; - struct zip_quex_err_ena_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_5_63 : 59; - u64 mdbe : 1; - u64 nwrp : 1; - u64 nrrp : 1; - u64 irde : 1; - u64 dovf : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 dovf : 1; - u64 irde : 1; - u64 nrrp : 1; - u64 nwrp : 1; - u64 mdbe : 1; - u64 reserved_5_63 : 59; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_ERR_ENA_W1S(u64 param1) -{ - if (param1 <= 7) - return 0x3400ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_ERR_ENA_W1S: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_err_int - Represents registers that contain the per-queue - * error interrupts. - */ -union zip_quex_err_int { - u64 u_reg64; - struct zip_quex_err_int_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_5_63 : 59; - u64 mdbe : 1; - u64 nwrp : 1; - u64 nrrp : 1; - u64 irde : 1; - u64 dovf : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 dovf : 1; - u64 irde : 1; - u64 nrrp : 1; - u64 nwrp : 1; - u64 mdbe : 1; - u64 reserved_5_63 : 59; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_ERR_INT(u64 param1) -{ - if (param1 <= 7) - return 0x3000ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_ERR_INT: %llu\n", param1); - return 0; -} - -/* NCB - zip_que#_err_int_w1s */ -union zip_quex_err_int_w1s { - u64 u_reg64; - struct zip_quex_err_int_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_5_63 : 59; - u64 mdbe : 1; - u64 nwrp : 1; - u64 nrrp : 1; - u64 irde : 1; - u64 dovf : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 dovf : 1; - u64 irde : 1; - u64 nrrp : 1; - u64 nwrp : 1; - u64 mdbe : 1; - u64 reserved_5_63 : 59; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_ERR_INT_W1S(u64 param1) -{ - if (param1 <= 7) - return 0x3200ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_ERR_INT_W1S: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_gcfg - Represents the registers that reflect status of the - * zip instruction queues,debug use only. - */ -union zip_quex_gcfg { - u64 u_reg64; - struct zip_quex_gcfg_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_4_63 : 60; - u64 iqb_ldwb : 1; - u64 cbw_sty : 1; - u64 l2ld_cmd : 2; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 l2ld_cmd : 2; - u64 cbw_sty : 1; - u64 iqb_ldwb : 1; - u64 reserved_4_63 : 60; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_GCFG(u64 param1) -{ - if (param1 <= 7) - return 0x1A00ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_GCFG: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_map - Represents the registers that control how each - * instruction queue maps to zip cores. - */ -union zip_quex_map { - u64 u_reg64; - struct zip_quex_map_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_2_63 : 62; - u64 zce : 2; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 zce : 2; - u64 reserved_2_63 : 62; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_MAP(u64 param1) -{ - if (param1 <= 7) - return 0x1400ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_MAP: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_sbuf_addr - Represents the registers that set the buffer - * parameters for the instruction queues. - * - * When quiescent (i.e. outstanding doorbell count is 0), it is safe to rewrite - * this register to effectively reset the command buffer state machine. - * These registers must be programmed after SW programs the corresponding - * ZIP_QUE(0..7)_SBUF_CTL. - */ -union zip_quex_sbuf_addr { - u64 u_reg64; - struct zip_quex_sbuf_addr_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_49_63 : 15; - u64 ptr : 42; - u64 off : 7; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 off : 7; - u64 ptr : 42; - u64 reserved_49_63 : 15; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_SBUF_ADDR(u64 param1) -{ - if (param1 <= 7) - return 0x1000ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_SBUF_ADDR: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_sbuf_ctl - Represents the registers that set the buffer - * parameters for the instruction queues. - * - * When quiescent (i.e. outstanding doorbell count is 0), it is safe to rewrite - * this register to effectively reset the command buffer state machine. - * These registers must be programmed before SW programs the corresponding - * ZIP_QUE(0..7)_SBUF_ADDR. - */ -union zip_quex_sbuf_ctl { - u64 u_reg64; - struct zip_quex_sbuf_ctl_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_45_63 : 19; - u64 size : 13; - u64 inst_be : 1; - u64 reserved_24_30 : 7; - u64 stream_id : 8; - u64 reserved_12_15 : 4; - u64 aura : 12; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 aura : 12; - u64 reserved_12_15 : 4; - u64 stream_id : 8; - u64 reserved_24_30 : 7; - u64 inst_be : 1; - u64 size : 13; - u64 reserved_45_63 : 19; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_SBUF_CTL(u64 param1) -{ - if (param1 <= 7) - return 0x1200ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_SBUF_CTL: %llu\n", param1); - return 0; -} - -/** - * union zip_que_ena - Represents queue enable register - * - * If a queue is disabled, ZIP_CTL stops fetching instructions from the queue. - */ -union zip_que_ena { - u64 u_reg64; - struct zip_que_ena_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_8_63 : 56; - u64 ena : 8; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 ena : 8; - u64 reserved_8_63 : 56; -#endif - } s; -}; - -#define ZIP_QUE_ENA 0x0500ull - -/** - * union zip_que_pri - Represents the register that defines the priority - * between instruction queues. - */ -union zip_que_pri { - u64 u_reg64; - struct zip_que_pri_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_8_63 : 56; - u64 pri : 8; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 pri : 8; - u64 reserved_8_63 : 56; -#endif - } s; -}; - -#define ZIP_QUE_PRI 0x0508ull - -/** - * union zip_throttle - Represents the register that controls the maximum - * number of in-flight X2I data fetch transactions. - * - * Writing 0 to this register causes the ZIP module to temporarily suspend NCB - * accesses; it is not recommended for normal operation, but may be useful for - * diagnostics. - */ -union zip_throttle { - u64 u_reg64; - struct zip_throttle_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_6_63 : 58; - u64 ld_infl : 6; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 ld_infl : 6; - u64 reserved_6_63 : 58; -#endif - } s; -}; - -#define ZIP_THROTTLE 0x0010ull - -#endif /* _CSRS_ZIP__ */ diff --git a/drivers/crypto/ccp/ccp-crypto-aes.c b/drivers/crypto/ccp/ccp-crypto-aes.c index d11daaf47f06..685d42ec7ade 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes.c +++ b/drivers/crypto/ccp/ccp-crypto-aes.c @@ -7,15 +7,16 @@ * Author: Tom Lendacky <thomas.lendacky@amd.com> */ -#include <linux/module.h> -#include <linux/sched.h> -#include <linux/delay.h> -#include <linux/scatterlist.h> -#include <linux/crypto.h> -#include <crypto/algapi.h> #include <crypto/aes.h> #include <crypto/ctr.h> -#include <crypto/scatterwalk.h> +#include <crypto/internal/skcipher.h> +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> +#include <linux/string.h> #include "ccp-crypto.h" diff --git a/drivers/crypto/ccp/ccp-crypto-des3.c b/drivers/crypto/ccp/ccp-crypto-des3.c index afae30adb703..91b1189c47de 100644 --- a/drivers/crypto/ccp/ccp-crypto-des3.c +++ b/drivers/crypto/ccp/ccp-crypto-des3.c @@ -7,14 +7,15 @@ * Author: Gary R Hook <ghook@amd.com> */ +#include <crypto/internal/des.h> +#include <crypto/internal/skcipher.h> +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/list.h> #include <linux/module.h> -#include <linux/sched.h> -#include <linux/delay.h> #include <linux/scatterlist.h> -#include <linux/crypto.h> -#include <crypto/algapi.h> -#include <crypto/scatterwalk.h> -#include <crypto/internal/des.h> +#include <linux/slab.h> +#include <linux/string.h> #include "ccp-crypto.h" diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c index ecd58b38c46e..bc90aba5162a 100644 --- a/drivers/crypto/ccp/ccp-crypto-main.c +++ b/drivers/crypto/ccp/ccp-crypto-main.c @@ -7,14 +7,17 @@ * Author: Tom Lendacky <thomas.lendacky@amd.com> */ -#include <linux/module.h> -#include <linux/moduleparam.h> +#include <crypto/internal/akcipher.h> +#include <crypto/internal/hash.h> +#include <crypto/internal/skcipher.h> +#include <linux/ccp.h> +#include <linux/err.h> #include <linux/kernel.h> #include <linux/list.h> -#include <linux/ccp.h> +#include <linux/module.h> #include <linux/scatterlist.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/akcipher.h> +#include <linux/slab.h> +#include <linux/spinlock.h> #include "ccp-crypto.h" diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 2e87ca0e292a..1ccff5e3f4bc 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -109,6 +109,15 @@ static void *sev_init_ex_buffer; */ static struct sev_data_range_list *snp_range_list; +static void __sev_firmware_shutdown(struct sev_device *sev, bool panic); + +static int snp_shutdown_on_panic(struct notifier_block *nb, + unsigned long reason, void *arg); + +static struct notifier_block snp_panic_notifier = { + .notifier_call = snp_shutdown_on_panic, +}; + static inline bool sev_version_greater_or_equal(u8 maj, u8 min) { struct sev_device *sev = psp_master->sev_data; @@ -1112,7 +1121,7 @@ static int __sev_snp_init_locked(int *error) if (!sev_version_greater_or_equal(SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR)) { dev_dbg(sev->dev, "SEV-SNP support requires firmware version >= %d:%d\n", SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR); - return 0; + return -EOPNOTSUPP; } /* SNP_INIT requires MSR_VM_HSAVE_PA to be cleared on all CPUs. */ @@ -1176,21 +1185,34 @@ static int __sev_snp_init_locked(int *error) wbinvd_on_all_cpus(); rc = __sev_do_cmd_locked(cmd, arg, error); - if (rc) + if (rc) { + dev_err(sev->dev, "SEV-SNP: %s failed rc %d, error %#x\n", + cmd == SEV_CMD_SNP_INIT_EX ? "SNP_INIT_EX" : "SNP_INIT", + rc, *error); return rc; + } /* Prepare for first SNP guest launch after INIT. */ wbinvd_on_all_cpus(); rc = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, error); - if (rc) + if (rc) { + dev_err(sev->dev, "SEV-SNP: SNP_DF_FLUSH failed rc %d, error %#x\n", + rc, *error); return rc; + } sev->snp_initialized = true; dev_dbg(sev->dev, "SEV-SNP firmware initialized\n"); + dev_info(sev->dev, "SEV-SNP API:%d.%d build:%d\n", sev->api_major, + sev->api_minor, sev->build); + + atomic_notifier_chain_register(&panic_notifier_list, + &snp_panic_notifier); + sev_es_tmr_size = SNP_TMR_SIZE; - return rc; + return 0; } static void __sev_platform_init_handle_tmr(struct sev_device *sev) @@ -1287,16 +1309,22 @@ static int __sev_platform_init_locked(int *error) if (error) *error = psp_ret; - if (rc) + if (rc) { + dev_err(sev->dev, "SEV: %s failed %#x, rc %d\n", + sev_init_ex_buffer ? "INIT_EX" : "INIT", psp_ret, rc); return rc; + } sev->state = SEV_STATE_INIT; /* Prepare for first SEV guest launch after INIT */ wbinvd_on_all_cpus(); rc = __sev_do_cmd_locked(SEV_CMD_DF_FLUSH, NULL, error); - if (rc) + if (rc) { + dev_err(sev->dev, "SEV: DF_FLUSH failed %#x, rc %d\n", + *error, rc); return rc; + } dev_dbg(sev->dev, "SEV firmware initialized\n"); @@ -1319,19 +1347,9 @@ static int _sev_platform_init_locked(struct sev_platform_init_args *args) if (sev->state == SEV_STATE_INIT) return 0; - /* - * Legacy guests cannot be running while SNP_INIT(_EX) is executing, - * so perform SEV-SNP initialization at probe time. - */ rc = __sev_snp_init_locked(&args->error); - if (rc && rc != -ENODEV) { - /* - * Don't abort the probe if SNP INIT failed, - * continue to initialize the legacy SEV firmware. - */ - dev_err(sev->dev, "SEV-SNP: failed to INIT rc %d, error %#x\n", - rc, args->error); - } + if (rc && rc != -ENODEV) + return rc; /* Defer legacy SEV/SEV-ES support if allowed by caller/module. */ if (args->probe && !psp_init_on_probe) @@ -1367,8 +1385,11 @@ static int __sev_platform_shutdown_locked(int *error) return 0; ret = __sev_do_cmd_locked(SEV_CMD_SHUTDOWN, NULL, error); - if (ret) + if (ret) { + dev_err(sev->dev, "SEV: failed to SHUTDOWN error %#x, rc %d\n", + *error, ret); return ret; + } sev->state = SEV_STATE_UNINIT; dev_dbg(sev->dev, "SEV firmware shutdown\n"); @@ -1389,6 +1410,37 @@ static int sev_get_platform_state(int *state, int *error) return rc; } +static int sev_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required) +{ + struct sev_platform_init_args init_args = {0}; + int rc; + + rc = _sev_platform_init_locked(&init_args); + if (rc) { + argp->error = SEV_RET_INVALID_PLATFORM_STATE; + return rc; + } + + *shutdown_required = true; + + return 0; +} + +static int snp_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required) +{ + int error, rc; + + rc = __sev_snp_init_locked(&error); + if (rc) { + argp->error = SEV_RET_INVALID_PLATFORM_STATE; + return rc; + } + + *shutdown_required = true; + + return 0; +} + static int sev_ioctl_do_reset(struct sev_issue_cmd *argp, bool writable) { int state, rc; @@ -1441,24 +1493,31 @@ static int sev_ioctl_do_platform_status(struct sev_issue_cmd *argp) static int sev_ioctl_do_pek_pdh_gen(int cmd, struct sev_issue_cmd *argp, bool writable) { struct sev_device *sev = psp_master->sev_data; + bool shutdown_required = false; int rc; if (!writable) return -EPERM; if (sev->state == SEV_STATE_UNINIT) { - rc = __sev_platform_init_locked(&argp->error); + rc = sev_move_to_init_state(argp, &shutdown_required); if (rc) return rc; } - return __sev_do_cmd_locked(cmd, NULL, &argp->error); + rc = __sev_do_cmd_locked(cmd, NULL, &argp->error); + + if (shutdown_required) + __sev_firmware_shutdown(sev, false); + + return rc; } static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable) { struct sev_device *sev = psp_master->sev_data; struct sev_user_data_pek_csr input; + bool shutdown_required = false; struct sev_data_pek_csr data; void __user *input_address; void *blob = NULL; @@ -1490,7 +1549,7 @@ static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable) cmd: if (sev->state == SEV_STATE_UNINIT) { - ret = __sev_platform_init_locked(&argp->error); + ret = sev_move_to_init_state(argp, &shutdown_required); if (ret) goto e_free_blob; } @@ -1511,6 +1570,9 @@ cmd: } e_free_blob: + if (shutdown_required) + __sev_firmware_shutdown(sev, false); + kfree(blob); return ret; } @@ -1682,9 +1744,12 @@ static int __sev_snp_shutdown_locked(int *error, bool panic) ret = __sev_do_cmd_locked(SEV_CMD_SNP_SHUTDOWN_EX, &data, error); /* SHUTDOWN may require DF_FLUSH */ if (*error == SEV_RET_DFFLUSH_REQUIRED) { - ret = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, NULL); + int dfflush_error = SEV_RET_NO_FW_CALL; + + ret = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, &dfflush_error); if (ret) { - dev_err(sev->dev, "SEV-SNP DF_FLUSH failed\n"); + dev_err(sev->dev, "SEV-SNP DF_FLUSH failed, ret = %d, error = %#x\n", + ret, dfflush_error); return ret; } /* reissue the shutdown command */ @@ -1692,7 +1757,8 @@ static int __sev_snp_shutdown_locked(int *error, bool panic) error); } if (ret) { - dev_err(sev->dev, "SEV-SNP firmware shutdown failed\n"); + dev_err(sev->dev, "SEV-SNP firmware shutdown failed, rc %d, error %#x\n", + ret, *error); return ret; } @@ -1718,6 +1784,12 @@ static int __sev_snp_shutdown_locked(int *error, bool panic) sev->snp_initialized = false; dev_dbg(sev->dev, "SEV-SNP firmware shutdown\n"); + atomic_notifier_chain_unregister(&panic_notifier_list, + &snp_panic_notifier); + + /* Reset TMR size back to default */ + sev_es_tmr_size = SEV_TMR_SIZE; + return ret; } @@ -1726,6 +1798,7 @@ static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp, bool writable) struct sev_device *sev = psp_master->sev_data; struct sev_user_data_pek_cert_import input; struct sev_data_pek_cert_import data; + bool shutdown_required = false; void *pek_blob, *oca_blob; int ret; @@ -1756,7 +1829,7 @@ static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp, bool writable) /* If platform is not in INIT state then transition it to INIT */ if (sev->state != SEV_STATE_INIT) { - ret = __sev_platform_init_locked(&argp->error); + ret = sev_move_to_init_state(argp, &shutdown_required); if (ret) goto e_free_oca; } @@ -1764,6 +1837,9 @@ static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp, bool writable) ret = __sev_do_cmd_locked(SEV_CMD_PEK_CERT_IMPORT, &data, &argp->error); e_free_oca: + if (shutdown_required) + __sev_firmware_shutdown(sev, false); + kfree(oca_blob); e_free_pek: kfree(pek_blob); @@ -1880,32 +1956,23 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) struct sev_data_pdh_cert_export data; void __user *input_cert_chain_address; void __user *input_pdh_cert_address; + bool shutdown_required = false; int ret; - /* If platform is not in INIT state then transition it to INIT. */ - if (sev->state != SEV_STATE_INIT) { - if (!writable) - return -EPERM; - - ret = __sev_platform_init_locked(&argp->error); - if (ret) - return ret; - } - if (copy_from_user(&input, (void __user *)argp->data, sizeof(input))) return -EFAULT; memset(&data, 0, sizeof(data)); + input_pdh_cert_address = (void __user *)input.pdh_cert_address; + input_cert_chain_address = (void __user *)input.cert_chain_address; + /* Userspace wants to query the certificate length. */ if (!input.pdh_cert_address || !input.pdh_cert_len || !input.cert_chain_address) goto cmd; - input_pdh_cert_address = (void __user *)input.pdh_cert_address; - input_cert_chain_address = (void __user *)input.cert_chain_address; - /* Allocate a physically contiguous buffer to store the PDH blob. */ if (input.pdh_cert_len > SEV_FW_BLOB_MAX_SIZE) return -EFAULT; @@ -1931,6 +1998,17 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) data.cert_chain_len = input.cert_chain_len; cmd: + /* If platform is not in INIT state then transition it to INIT. */ + if (sev->state != SEV_STATE_INIT) { + if (!writable) { + ret = -EPERM; + goto e_free_cert; + } + ret = sev_move_to_init_state(argp, &shutdown_required); + if (ret) + goto e_free_cert; + } + ret = __sev_do_cmd_locked(SEV_CMD_PDH_CERT_EXPORT, &data, &argp->error); /* If we query the length, FW responded with expected data. */ @@ -1957,6 +2035,9 @@ cmd: } e_free_cert: + if (shutdown_required) + __sev_firmware_shutdown(sev, false); + kfree(cert_blob); e_free_pdh: kfree(pdh_blob); @@ -1966,12 +2047,13 @@ e_free_pdh: static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) { struct sev_device *sev = psp_master->sev_data; + bool shutdown_required = false; struct sev_data_snp_addr buf; struct page *status_page; + int ret, error; void *data; - int ret; - if (!sev->snp_initialized || !argp->data) + if (!argp->data) return -EINVAL; status_page = alloc_page(GFP_KERNEL_ACCOUNT); @@ -1980,6 +2062,12 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) data = page_address(status_page); + if (!sev->snp_initialized) { + ret = snp_move_to_init_state(argp, &shutdown_required); + if (ret) + goto cleanup; + } + /* * Firmware expects status page to be in firmware-owned state, otherwise * it will report firmware error code INVALID_PAGE_STATE (0x1A). @@ -2008,6 +2096,9 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) ret = -EFAULT; cleanup: + if (shutdown_required) + __sev_snp_shutdown_locked(&error, false); + __free_pages(status_page, 0); return ret; } @@ -2016,21 +2107,33 @@ static int sev_ioctl_do_snp_commit(struct sev_issue_cmd *argp) { struct sev_device *sev = psp_master->sev_data; struct sev_data_snp_commit buf; + bool shutdown_required = false; + int ret, error; - if (!sev->snp_initialized) - return -EINVAL; + if (!sev->snp_initialized) { + ret = snp_move_to_init_state(argp, &shutdown_required); + if (ret) + return ret; + } buf.len = sizeof(buf); - return __sev_do_cmd_locked(SEV_CMD_SNP_COMMIT, &buf, &argp->error); + ret = __sev_do_cmd_locked(SEV_CMD_SNP_COMMIT, &buf, &argp->error); + + if (shutdown_required) + __sev_snp_shutdown_locked(&error, false); + + return ret; } static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable) { struct sev_device *sev = psp_master->sev_data; struct sev_user_data_snp_config config; + bool shutdown_required = false; + int ret, error; - if (!sev->snp_initialized || !argp->data) + if (!argp->data) return -EINVAL; if (!writable) @@ -2039,17 +2142,29 @@ static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable if (copy_from_user(&config, (void __user *)argp->data, sizeof(config))) return -EFAULT; - return __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error); + if (!sev->snp_initialized) { + ret = snp_move_to_init_state(argp, &shutdown_required); + if (ret) + return ret; + } + + ret = __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error); + + if (shutdown_required) + __sev_snp_shutdown_locked(&error, false); + + return ret; } static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable) { struct sev_device *sev = psp_master->sev_data; struct sev_user_data_snp_vlek_load input; + bool shutdown_required = false; + int ret, error; void *blob; - int ret; - if (!sev->snp_initialized || !argp->data) + if (!argp->data) return -EINVAL; if (!writable) @@ -2068,8 +2183,18 @@ static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable) input.vlek_wrapped_address = __psp_pa(blob); + if (!sev->snp_initialized) { + ret = snp_move_to_init_state(argp, &shutdown_required); + if (ret) + goto cleanup; + } + ret = __sev_do_cmd_locked(SEV_CMD_SNP_VLEK_LOAD, &input, &argp->error); + if (shutdown_required) + __sev_snp_shutdown_locked(&error, false); + +cleanup: kfree(blob); return ret; @@ -2339,6 +2464,15 @@ static void sev_firmware_shutdown(struct sev_device *sev) mutex_unlock(&sev_cmd_mutex); } +void sev_platform_shutdown(void) +{ + if (!psp_master || !psp_master->sev_data) + return; + + sev_firmware_shutdown(psp_master->sev_data); +} +EXPORT_SYMBOL_GPL(sev_platform_shutdown); + void sev_dev_destroy(struct psp_device *psp) { struct sev_device *sev = psp->sev_data; @@ -2373,10 +2507,6 @@ static int snp_shutdown_on_panic(struct notifier_block *nb, return NOTIFY_DONE; } -static struct notifier_block snp_panic_notifier = { - .notifier_call = snp_shutdown_on_panic, -}; - int sev_issue_cmd_external_user(struct file *filep, unsigned int cmd, void *data, int *error) { @@ -2390,9 +2520,7 @@ EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user); void sev_pci_init(void) { struct sev_device *sev = psp_master->sev_data; - struct sev_platform_init_args args = {0}; u8 api_major, api_minor, build; - int rc; if (!sev) return; @@ -2415,18 +2543,6 @@ void sev_pci_init(void) api_major, api_minor, build, sev->api_major, sev->api_minor, sev->build); - /* Initialize the platform */ - args.probe = true; - rc = sev_platform_init(&args); - if (rc) - dev_err(sev->dev, "SEV: failed to INIT error %#x, rc %d\n", - args.error, rc); - - dev_info(sev->dev, "SEV%s API:%d.%d build:%d\n", sev->snp_initialized ? - "-SNP" : "", sev->api_major, sev->api_minor, sev->build); - - atomic_notifier_chain_register(&panic_notifier_list, - &snp_panic_notifier); return; err: @@ -2443,7 +2559,4 @@ void sev_pci_exit(void) return; sev_firmware_shutdown(sev); - - atomic_notifier_chain_unregister(&panic_notifier_list, - &snp_panic_notifier); } diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c index 1dc2378aa88b..e050f5ff5efb 100644 --- a/drivers/crypto/img-hash.c +++ b/drivers/crypto/img-hash.c @@ -491,8 +491,9 @@ static int img_hash_init(struct ahash_request *req) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_init(&rctx->fallback_req); } @@ -555,10 +556,10 @@ static int img_hash_update(struct ahash_request *req) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); return crypto_ahash_update(&rctx->fallback_req); } @@ -570,9 +571,10 @@ static int img_hash_final(struct ahash_request *req) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); return crypto_ahash_final(&rctx->fallback_req); } @@ -584,11 +586,12 @@ static int img_hash_finup(struct ahash_request *req) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, + req->nbytes); + return crypto_ahash_finup(&rctx->fallback_req); } @@ -600,8 +603,9 @@ static int img_hash_import(struct ahash_request *req, const void *in) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_import(&rctx->fallback_req, in); } @@ -613,8 +617,9 @@ static int img_hash_export(struct ahash_request *req, void *out) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_export(&rctx->fallback_req, out); } diff --git a/drivers/crypto/inside-secure/eip93/eip93-hash.c b/drivers/crypto/inside-secure/eip93/eip93-hash.c index df1b05ac5a57..ac13d90a2b7c 100644 --- a/drivers/crypto/inside-secure/eip93/eip93-hash.c +++ b/drivers/crypto/inside-secure/eip93/eip93-hash.c @@ -97,12 +97,20 @@ void eip93_hash_handle_result(struct crypto_async_request *async, int err) static void eip93_hash_init_sa_state_digest(u32 hash, u8 *digest) { - u32 sha256_init[] = { SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, - SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7 }; - u32 sha224_init[] = { SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, - SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7 }; - u32 sha1_init[] = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }; - u32 md5_init[] = { MD5_H0, MD5_H1, MD5_H2, MD5_H3 }; + static const u32 sha256_init[] = { + SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, + SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7 + }; + static const u32 sha224_init[] = { + SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, + SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7 + }; + static const u32 sha1_init[] = { + SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 + }; + static const u32 md5_init[] = { + MD5_H0, MD5_H1, MD5_H2, MD5_H3 + }; /* Init HASH constant */ switch (hash) { diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index f44c08f5f5ec..d2b632193beb 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -2043,7 +2043,7 @@ struct safexcel_alg_template safexcel_alg_cbcmac = { .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = 1, + .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), .cra_init = safexcel_ahash_cra_init, .cra_exit = safexcel_ahash_cra_exit, diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index 09d9589f2d68..be3b899c6977 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -999,12 +999,9 @@ out: static int deflate_generic_decompress(struct acomp_req *req) { - ACOMP_REQUEST_ON_STACK(fbreq, crypto_acomp_reqtfm(req)); + ACOMP_FBREQ_ON_STACK(fbreq, req); int ret; - acomp_request_set_callback(fbreq, 0, NULL, NULL); - acomp_request_set_params(fbreq, req->src, req->dst, req->slen, - req->dlen); ret = crypto_acomp_decompress(fbreq); req->dlen = fbreq->dlen; @@ -1020,8 +1017,7 @@ static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - u32 compression_crc); + dma_addr_t dst_addr, unsigned int *dlen); static void iaa_desc_complete(struct idxd_desc *idxd_desc, enum idxd_complete_type comp_type, @@ -1087,10 +1083,10 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, } if (ctx->compress && compression_ctx->verify_compress) { + u32 *compression_crc = acomp_request_ctx(ctx->req); dma_addr_t src_addr, dst_addr; - u32 compression_crc; - compression_crc = idxd_desc->iax_completion->crc; + *compression_crc = idxd_desc->iax_completion->crc; ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr); if (ret) { @@ -1100,8 +1096,7 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, } ret = iaa_compress_verify(ctx->tfm, ctx->req, iaa_wq->wq, src_addr, - ctx->req->slen, dst_addr, &ctx->req->dlen, - compression_crc); + ctx->req->slen, dst_addr, &ctx->req->dlen); if (ret) { dev_dbg(dev, "%s: compress verify failed ret=%d\n", __func__, ret); err = -EIO; @@ -1130,11 +1125,11 @@ out: static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - u32 *compression_crc) + dma_addr_t dst_addr, unsigned int *dlen) { struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + u32 *compression_crc = acomp_request_ctx(req); struct iaa_device *iaa_device; struct idxd_desc *idxd_desc; struct iax_hw_desc *desc; @@ -1187,8 +1182,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, " src_addr %llx, dst_addr %llx\n", __func__, active_compression_mode->name, src_addr, dst_addr); - } else if (ctx->async_mode) - req->base.data = idxd_desc; + } dev_dbg(dev, "%s: compression mode %s," " desc->src1_addr %llx, desc->src1_size %d," @@ -1282,11 +1276,11 @@ out: static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - u32 compression_crc) + dma_addr_t dst_addr, unsigned int *dlen) { struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + u32 *compression_crc = acomp_request_ctx(req); struct iaa_device *iaa_device; struct idxd_desc *idxd_desc; struct iax_hw_desc *desc; @@ -1346,10 +1340,10 @@ static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, goto err; } - if (compression_crc != idxd_desc->iax_completion->crc) { + if (*compression_crc != idxd_desc->iax_completion->crc) { ret = -EINVAL; dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:" - " comp=0x%x, decomp=0x%x\n", compression_crc, + " comp=0x%x, decomp=0x%x\n", *compression_crc, idxd_desc->iax_completion->crc); print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, 8, 1, idxd_desc->iax_completion, 64, 0); @@ -1369,8 +1363,7 @@ err: static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - bool disable_async) + dma_addr_t dst_addr, unsigned int *dlen) { struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); @@ -1412,7 +1405,7 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, desc->src1_size = slen; desc->completion_addr = idxd_desc->compl_dma; - if (ctx->use_irq && !disable_async) { + if (ctx->use_irq) { desc->flags |= IDXD_OP_FLAG_RCI; idxd_desc->crypto.req = req; @@ -1425,8 +1418,7 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, " src_addr %llx, dst_addr %llx\n", __func__, active_compression_mode->name, src_addr, dst_addr); - } else if (ctx->async_mode && !disable_async) - req->base.data = idxd_desc; + } dev_dbg(dev, "%s: decompression mode %s," " desc->src1_addr %llx, desc->src1_size %d," @@ -1446,7 +1438,7 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, update_total_decomp_calls(); update_wq_decomp_calls(wq); - if (ctx->async_mode && !disable_async) { + if (ctx->async_mode) { ret = -EINPROGRESS; dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__); goto out; @@ -1474,7 +1466,7 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, *dlen = req->dlen; - if (!ctx->async_mode || disable_async) + if (!ctx->async_mode) idxd_free_desc(wq, idxd_desc); /* Update stats */ @@ -1496,7 +1488,6 @@ static int iaa_comp_acompress(struct acomp_req *req) dma_addr_t src_addr, dst_addr; int nr_sgs, cpu, ret = 0; struct iaa_wq *iaa_wq; - u32 compression_crc; struct idxd_wq *wq; struct device *dev; @@ -1557,7 +1548,7 @@ static int iaa_comp_acompress(struct acomp_req *req) req->dst, req->dlen, sg_dma_len(req->dst)); ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr, - &req->dlen, &compression_crc); + &req->dlen); if (ret == -EINPROGRESS) return ret; @@ -1569,7 +1560,7 @@ static int iaa_comp_acompress(struct acomp_req *req) } ret = iaa_compress_verify(tfm, req, wq, src_addr, req->slen, - dst_addr, &req->dlen, compression_crc); + dst_addr, &req->dlen); if (ret) dev_dbg(dev, "asynchronous compress verification failed ret=%d\n", ret); @@ -1655,7 +1646,7 @@ static int iaa_comp_adecompress(struct acomp_req *req) req->dst, req->dlen, sg_dma_len(req->dst)); ret = iaa_decompress(tfm, req, wq, src_addr, req->slen, - dst_addr, &req->dlen, false); + dst_addr, &req->dlen); if (ret == -EINPROGRESS) return ret; @@ -1699,6 +1690,7 @@ static struct acomp_alg iaa_acomp_fixed_deflate = { .cra_driver_name = "deflate-iaa", .cra_flags = CRYPTO_ALG_ASYNC, .cra_ctxsize = sizeof(struct iaa_compression_ctx), + .cra_reqsize = sizeof(u32), .cra_module = THIS_MODULE, .cra_priority = IAA_ALG_PRIORITY, } diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c index 4feeef83f7a3..795f4598400b 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c @@ -93,7 +93,6 @@ static const struct adf_fw_config adf_fw_dcc_config[] = { static struct adf_hw_device_class adf_420xx_class = { .name = ADF_420XX_DEVICE_NAME, .type = DEV_420XX, - .instances = 0, }; static u32 get_ae_mask(struct adf_hw_device_data *self) diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c index 8084aa0f7f41..cfa00daeb4fb 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c @@ -14,7 +14,7 @@ #include "adf_420xx_hw_data.h" static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, ADF_420XX_PCI_DEVICE_ID), }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_420XX) }, { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); @@ -186,11 +186,19 @@ static void adf_remove(struct pci_dev *pdev) adf_cleanup_accel(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + static struct pci_driver adf_driver = { .id_table = adf_pci_tbl, .name = ADF_420XX_DEVICE_NAME, .probe = adf_probe, .remove = adf_remove, + .shutdown = adf_shutdown, .sriov_configure = adf_sriov_configure, .err_handler = &adf_err_handler, }; diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c index 4eb6ef99efdd..7d4c366aa8b2 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -96,7 +96,6 @@ static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_dcc_config)); static struct adf_hw_device_class adf_4xxx_class = { .name = ADF_4XXX_DEVICE_NAME, .type = DEV_4XXX, - .instances = 0, }; static u32 get_ae_mask(struct adf_hw_device_data *self) @@ -422,13 +421,13 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id) hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK; hw_data->num_rps = ADF_GEN4_MAX_RPS; switch (dev_id) { - case ADF_402XX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_402XX: hw_data->fw_name = ADF_402XX_FW; hw_data->fw_mmp_name = ADF_402XX_MMP; hw_data->uof_get_name = uof_get_name_402xx; hw_data->get_ena_thd_mask = get_ena_thd_mask; break; - case ADF_401XX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_401XX: hw_data->fw_name = ADF_4XXX_FW; hw_data->fw_mmp_name = ADF_4XXX_MMP; hw_data->uof_get_name = uof_get_name_4xxx; diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c index 5537a9991e4e..c9be5dcddb27 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c @@ -14,9 +14,9 @@ #include "adf_4xxx_hw_data.h" static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, ADF_4XXX_PCI_DEVICE_ID), }, - { PCI_VDEVICE(INTEL, ADF_401XX_PCI_DEVICE_ID), }, - { PCI_VDEVICE(INTEL, ADF_402XX_PCI_DEVICE_ID), }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_4XXX) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_401XX) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_402XX) }, { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); @@ -188,11 +188,19 @@ static void adf_remove(struct pci_dev *pdev) adf_cleanup_accel(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + static struct pci_driver adf_driver = { .id_table = adf_pci_tbl, .name = ADF_4XXX_DEVICE_NAME, .probe = adf_probe, .remove = adf_remove, + .shutdown = adf_shutdown, .sriov_configure = adf_sriov_configure, .err_handler = &adf_err_handler, }; diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c index e78f7bfd30b8..9425af26d34c 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -22,7 +22,6 @@ static const u32 thrd_to_arb_map[ADF_C3XXX_MAX_ACCELENGINES] = { static struct adf_hw_device_class c3xxx_class = { .name = ADF_C3XXX_DEVICE_NAME, .type = DEV_C3XXX, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c index b825b35ab4bf..bceb5dd8b148 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c @@ -19,24 +19,6 @@ #include <adf_dbgfs.h> #include "adf_c3xxx_hw_data.h" -static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX), }, - { } -}; -MODULE_DEVICE_TABLE(pci, adf_pci_tbl); - -static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); -static void adf_remove(struct pci_dev *dev); - -static struct pci_driver adf_driver = { - .id_table = adf_pci_tbl, - .name = ADF_C3XXX_DEVICE_NAME, - .probe = adf_probe, - .remove = adf_remove, - .sriov_configure = adf_sriov_configure, - .err_handler = &adf_err_handler, -}; - static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) { pci_release_regions(accel_dev->accel_pci_dev.pci_dev); @@ -227,6 +209,29 @@ static void adf_remove(struct pci_dev *pdev) kfree(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX) }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_C3XXX_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .shutdown = adf_shutdown, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; + static int __init adfdrv_init(void) { request_module("intel_qat"); diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index a512ca4efd3f..f73d9a4a9ab7 100644 --- a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -13,7 +13,6 @@ static struct adf_hw_device_class c3xxxiov_class = { .name = ADF_C3XXXVF_DEVICE_NAME, .type = DEV_C3XXXVF, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c index 32ebe09477a8..1a2f36b603fb 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c @@ -22,7 +22,6 @@ static const u32 thrd_to_arb_map[ADF_C62X_MAX_ACCELENGINES] = { static struct adf_hw_device_class c62x_class = { .name = ADF_C62X_DEVICE_NAME, .type = DEV_C62X, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c index 8a7bdec358d6..23ccb72b6ea2 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c @@ -19,24 +19,6 @@ #include <adf_dbgfs.h> #include "adf_c62x_hw_data.h" -static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X), }, - { } -}; -MODULE_DEVICE_TABLE(pci, adf_pci_tbl); - -static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); -static void adf_remove(struct pci_dev *dev); - -static struct pci_driver adf_driver = { - .id_table = adf_pci_tbl, - .name = ADF_C62X_DEVICE_NAME, - .probe = adf_probe, - .remove = adf_remove, - .sriov_configure = adf_sriov_configure, - .err_handler = &adf_err_handler, -}; - static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) { pci_release_regions(accel_dev->accel_pci_dev.pci_dev); @@ -227,6 +209,29 @@ static void adf_remove(struct pci_dev *pdev) kfree(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X) }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_C62X_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .shutdown = adf_shutdown, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; + static int __init adfdrv_init(void) { request_module("intel_qat"); diff --git a/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c index 4aaaaf921734..29e53b41a895 100644 --- a/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -13,7 +13,6 @@ static struct adf_hw_device_class c62xiov_class = { .name = ADF_C62XVF_DEVICE_NAME, .type = DEV_C62XVF, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index dc21551153cb..1e301a20c244 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -25,14 +25,14 @@ #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" #define ADF_4XXX_DEVICE_NAME "4xxx" #define ADF_420XX_DEVICE_NAME "420xx" -#define ADF_4XXX_PCI_DEVICE_ID 0x4940 -#define ADF_4XXXIOV_PCI_DEVICE_ID 0x4941 -#define ADF_401XX_PCI_DEVICE_ID 0x4942 -#define ADF_401XXIOV_PCI_DEVICE_ID 0x4943 -#define ADF_402XX_PCI_DEVICE_ID 0x4944 -#define ADF_402XXIOV_PCI_DEVICE_ID 0x4945 -#define ADF_420XX_PCI_DEVICE_ID 0x4946 -#define ADF_420XXIOV_PCI_DEVICE_ID 0x4947 +#define PCI_DEVICE_ID_INTEL_QAT_4XXX 0x4940 +#define PCI_DEVICE_ID_INTEL_QAT_4XXXIOV 0x4941 +#define PCI_DEVICE_ID_INTEL_QAT_401XX 0x4942 +#define PCI_DEVICE_ID_INTEL_QAT_401XXIOV 0x4943 +#define PCI_DEVICE_ID_INTEL_QAT_402XX 0x4944 +#define PCI_DEVICE_ID_INTEL_QAT_402XXIOV 0x4945 +#define PCI_DEVICE_ID_INTEL_QAT_420XX 0x4946 +#define PCI_DEVICE_ID_INTEL_QAT_420XXIOV 0x4947 #define ADF_DEVICE_FUSECTL_OFFSET 0x40 #define ADF_DEVICE_LEGFUSE_OFFSET 0x4C #define ADF_DEVICE_FUSECTL_MASK 0x80000000 diff --git a/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c b/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c index a6e02405d402..a0a29b97a749 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c +++ b/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c @@ -241,13 +241,13 @@ static struct acomp_alg qat_acomp[] = { { .cra_priority = 4001, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, .cra_ctxsize = sizeof(struct qat_compression_ctx), + .cra_reqsize = sizeof(struct qat_compression_req), .cra_module = THIS_MODULE, }, .init = qat_comp_alg_init_tfm, .exit = qat_comp_alg_exit_tfm, .compress = qat_comp_alg_compress, .decompress = qat_comp_alg_decompress, - .reqsize = sizeof(struct qat_compression_req), }}; int qat_comp_algs_register(void) diff --git a/drivers/crypto/intel/qat/qat_common/qat_hal.c b/drivers/crypto/intel/qat/qat_common/qat_hal.c index ef8a9cf74f0c..841c1d7d3ffe 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_hal.c +++ b/drivers/crypto/intel/qat/qat_common/qat_hal.c @@ -694,16 +694,16 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->pci_dev = pci_info->pci_dev; switch (handle->pci_dev->device) { - case ADF_4XXX_PCI_DEVICE_ID: - case ADF_401XX_PCI_DEVICE_ID: - case ADF_402XX_PCI_DEVICE_ID: - case ADF_420XX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_4XXX: + case PCI_DEVICE_ID_INTEL_QAT_401XX: + case PCI_DEVICE_ID_INTEL_QAT_402XX: + case PCI_DEVICE_ID_INTEL_QAT_420XX: handle->chip_info->mmp_sram_size = 0; handle->chip_info->nn = false; handle->chip_info->lm2lm3 = true; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X; handle->chip_info->icp_rst_csr = ICP_RESET_CPP0; - if (handle->pci_dev->device == ADF_420XX_PCI_DEVICE_ID) + if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_420XX) handle->chip_info->icp_rst_mask = 0x100155; else handle->chip_info->icp_rst_mask = 0x100015; diff --git a/drivers/crypto/intel/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c index 7678a93c6853..620300e70238 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2014 - 2020 Intel Corporation */ #include <linux/align.h> +#include <linux/bitops.h> #include <linux/slab.h> #include <linux/ctype.h> #include <linux/kernel.h> @@ -731,10 +732,10 @@ qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) return ICP_QAT_AC_C62X_DEV_TYPE; case PCI_DEVICE_ID_INTEL_QAT_C3XXX: return ICP_QAT_AC_C3XXX_DEV_TYPE; - case ADF_4XXX_PCI_DEVICE_ID: - case ADF_401XX_PCI_DEVICE_ID: - case ADF_402XX_PCI_DEVICE_ID: - case ADF_420XX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_4XXX: + case PCI_DEVICE_ID_INTEL_QAT_401XX: + case PCI_DEVICE_ID_INTEL_QAT_402XX: + case PCI_DEVICE_ID_INTEL_QAT_420XX: return ICP_QAT_AC_4XXX_A_DEV_TYPE; default: pr_err("QAT: unsupported device 0x%x\n", @@ -1205,7 +1206,6 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, } #define ADD_ADDR(high, low) ((((u64)high) << 32) + low) -#define BITS_IN_DWORD 32 static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc) @@ -1223,7 +1223,7 @@ static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, fcu_dram_hi_csr = handle->chip_info->fcu_dram_addr_hi; fcu_dram_lo_csr = handle->chip_info->fcu_dram_addr_lo; - SET_CAP_CSR(handle, fcu_dram_hi_csr, (bus_addr >> BITS_IN_DWORD)); + SET_CAP_CSR(handle, fcu_dram_hi_csr, bus_addr >> BITS_PER_TYPE(u32)); SET_CAP_CSR(handle, fcu_dram_lo_csr, bus_addr); SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH); @@ -1438,7 +1438,7 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, virt_base = (uintptr_t)img_desc.dram_base_addr_v + simg_offset; bus_addr = img_desc.dram_bus_addr + simg_offset; auth_desc = img_desc.dram_base_addr_v; - auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); + auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); auth_desc->css_hdr_low = (unsigned int)bus_addr; virt_addr = virt_base; @@ -1448,7 +1448,7 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, sizeof(*css_hdr); virt_addr = virt_addr + sizeof(*css_hdr); - auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); + auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); auth_desc->fwsk_pub_low = (unsigned int)bus_addr; memcpy((void *)(uintptr_t)virt_addr, @@ -1470,7 +1470,7 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, auth_desc->fwsk_pub_low) + ICP_QAT_CSS_FWSK_PUB_LEN(handle); virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(handle); - auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); + auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); auth_desc->signature_low = (unsigned int)bus_addr; memcpy((void *)(uintptr_t)virt_addr, @@ -1484,7 +1484,7 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, ICP_QAT_CSS_SIGNATURE_LEN(handle); virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); - auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); + auth_desc->img_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); auth_desc->img_low = (unsigned int)bus_addr; auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(handle); if (bus_addr + auth_desc->img_len > img_desc.dram_bus_addr + @@ -1507,12 +1507,12 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, auth_desc->img_ae_mode_data_low) + sizeof(struct icp_qat_simg_ae_mode); - auth_desc->img_ae_init_data_high = (unsigned int) - (bus_addr >> BITS_IN_DWORD); + auth_desc->img_ae_init_data_high = + (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); auth_desc->img_ae_init_data_low = (unsigned int)bus_addr; bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN; - auth_desc->img_ae_insts_high = (unsigned int) - (bus_addr >> BITS_IN_DWORD); + auth_desc->img_ae_insts_high = + (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); auth_desc->img_ae_insts_low = (unsigned int)bus_addr; virt_addr += sizeof(struct icp_qat_css_hdr); virt_addr += ICP_QAT_CSS_FWSK_PUB_LEN(handle); diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index e48bcf1818cd..bf9e8f34f451 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -24,7 +24,6 @@ static const u32 thrd_to_arb_map[ADF_DH895XCC_MAX_ACCELENGINES] = { static struct adf_hw_device_class dh895xcc_class = { .name = ADF_DH895XCC_DEVICE_NAME, .type = DEV_DH895XCC, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c index 07e9d7e52861..b59e0cc49e52 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c @@ -19,24 +19,6 @@ #include <adf_dbgfs.h> #include "adf_dh895xcc_hw_data.h" -static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC), }, - { } -}; -MODULE_DEVICE_TABLE(pci, adf_pci_tbl); - -static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); -static void adf_remove(struct pci_dev *dev); - -static struct pci_driver adf_driver = { - .id_table = adf_pci_tbl, - .name = ADF_DH895XCC_DEVICE_NAME, - .probe = adf_probe, - .remove = adf_remove, - .sriov_configure = adf_sriov_configure, - .err_handler = &adf_err_handler, -}; - static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) { pci_release_regions(accel_dev->accel_pci_dev.pci_dev); @@ -227,6 +209,29 @@ static void adf_remove(struct pci_dev *pdev) kfree(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC) }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_DH895XCC_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .shutdown = adf_shutdown, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; + static int __init adfdrv_init(void) { request_module("intel_qat"); diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index f4ee4c2e00da..bc59c1473eef 100644 --- a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -13,7 +13,6 @@ static struct adf_hw_device_class dh895xcciov_class = { .name = ADF_DH895XCCVF_DEVICE_NAME, .type = DEV_DH895XCCVF, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) diff --git a/drivers/crypto/nx/nx-aes-cbc.c b/drivers/crypto/nx/nx-aes-cbc.c index 0e440f704a8f..35fa5bad1d9f 100644 --- a/drivers/crypto/nx/nx-aes-cbc.c +++ b/drivers/crypto/nx/nx-aes-cbc.c @@ -8,10 +8,12 @@ */ #include <crypto/aes.h> -#include <crypto/algapi.h> +#include <crypto/internal/skcipher.h> +#include <linux/err.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <linux/types.h> -#include <linux/crypto.h> +#include <linux/spinlock.h> +#include <linux/string.h> #include <asm/vio.h> #include "nx_csbcpb.h" diff --git a/drivers/crypto/nx/nx-aes-ctr.c b/drivers/crypto/nx/nx-aes-ctr.c index dfa3ad1a12f2..709b3ee74657 100644 --- a/drivers/crypto/nx/nx-aes-ctr.c +++ b/drivers/crypto/nx/nx-aes-ctr.c @@ -9,10 +9,12 @@ #include <crypto/aes.h> #include <crypto/ctr.h> -#include <crypto/algapi.h> +#include <crypto/internal/skcipher.h> +#include <linux/err.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <linux/types.h> -#include <linux/crypto.h> +#include <linux/spinlock.h> +#include <linux/string.h> #include <asm/vio.h> #include "nx_csbcpb.h" diff --git a/drivers/crypto/nx/nx-aes-ecb.c b/drivers/crypto/nx/nx-aes-ecb.c index 502a565074e9..4039cf3b22d4 100644 --- a/drivers/crypto/nx/nx-aes-ecb.c +++ b/drivers/crypto/nx/nx-aes-ecb.c @@ -8,10 +8,12 @@ */ #include <crypto/aes.h> -#include <crypto/algapi.h> +#include <crypto/internal/skcipher.h> +#include <linux/err.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <linux/types.h> -#include <linux/crypto.h> +#include <linux/spinlock.h> +#include <linux/string.h> #include <asm/vio.h> #include "nx_csbcpb.h" diff --git a/drivers/crypto/nx/nx-aes-xcbc.c b/drivers/crypto/nx/nx-aes-xcbc.c index eb5c8f689360..bf465d824e2c 100644 --- a/drivers/crypto/nx/nx-aes-xcbc.c +++ b/drivers/crypto/nx/nx-aes-xcbc.c @@ -7,13 +7,14 @@ * Author: Kent Yoder <yoder1@us.ibm.com> */ -#include <crypto/internal/hash.h> #include <crypto/aes.h> -#include <crypto/algapi.h> +#include <crypto/internal/hash.h> +#include <linux/atomic.h> +#include <linux/errno.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <linux/types.h> -#include <linux/crypto.h> -#include <asm/vio.h> +#include <linux/spinlock.h> +#include <linux/string.h> #include "nx_csbcpb.h" #include "nx.h" @@ -21,8 +22,6 @@ struct xcbc_state { u8 state[AES_BLOCK_SIZE]; - unsigned int count; - u8 buffer[AES_BLOCK_SIZE]; }; static int nx_xcbc_set_key(struct crypto_shash *desc, @@ -58,7 +57,7 @@ static int nx_xcbc_set_key(struct crypto_shash *desc, */ static int nx_xcbc_empty(struct shash_desc *desc, u8 *out) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; struct nx_sg *in_sg, *out_sg; u8 keys[2][AES_BLOCK_SIZE]; @@ -135,9 +134,9 @@ out: return rc; } -static int nx_crypto_ctx_aes_xcbc_init2(struct crypto_tfm *tfm) +static int nx_crypto_ctx_aes_xcbc_init2(struct crypto_shash *tfm) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm); + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; int err; @@ -166,31 +165,24 @@ static int nx_xcbc_update(struct shash_desc *desc, const u8 *data, unsigned int len) { + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct xcbc_state *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; struct nx_sg *in_sg; struct nx_sg *out_sg; - u32 to_process = 0, leftover, total; unsigned int max_sg_len; unsigned long irq_flags; + u32 to_process, total; int rc = 0; int data_len; spin_lock_irqsave(&nx_ctx->lock, irq_flags); + memcpy(csbcpb->cpb.aes_xcbc.out_cv_mac, sctx->state, AES_BLOCK_SIZE); + NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; - total = sctx->count + len; - - /* 2 cases for total data len: - * 1: <= AES_BLOCK_SIZE: copy into state, return 0 - * 2: > AES_BLOCK_SIZE: process X blocks, copy in leftover - */ - if (total <= AES_BLOCK_SIZE) { - memcpy(sctx->buffer + sctx->count, data, len); - sctx->count += len; - goto out; - } + total = len; in_sg = nx_ctx->in_sg; max_sg_len = min_t(u64, nx_driver.of.max_sg_len/sizeof(struct nx_sg), @@ -200,7 +192,7 @@ static int nx_xcbc_update(struct shash_desc *desc, data_len = AES_BLOCK_SIZE; out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *)sctx->state, - &len, nx_ctx->ap->sglen); + &data_len, nx_ctx->ap->sglen); if (data_len != AES_BLOCK_SIZE) { rc = -EINVAL; @@ -210,56 +202,21 @@ static int nx_xcbc_update(struct shash_desc *desc, nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg); do { - to_process = total - to_process; - to_process = to_process & ~(AES_BLOCK_SIZE - 1); - - leftover = total - to_process; - - /* the hardware will not accept a 0 byte operation for this - * algorithm and the operation MUST be finalized to be correct. - * So if we happen to get an update that falls on a block sized - * boundary, we must save off the last block to finalize with - * later. */ - if (!leftover) { - to_process -= AES_BLOCK_SIZE; - leftover = AES_BLOCK_SIZE; - } - - if (sctx->count) { - data_len = sctx->count; - in_sg = nx_build_sg_list(nx_ctx->in_sg, - (u8 *) sctx->buffer, - &data_len, - max_sg_len); - if (data_len != sctx->count) { - rc = -EINVAL; - goto out; - } - } + to_process = total & ~(AES_BLOCK_SIZE - 1); - data_len = to_process - sctx->count; in_sg = nx_build_sg_list(in_sg, (u8 *) data, - &data_len, + &to_process, max_sg_len); - if (data_len != to_process - sctx->count) { - rc = -EINVAL; - goto out; - } - nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); /* we've hit the nx chip previously and we're updating again, * so copy over the partial digest */ - if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) { - memcpy(csbcpb->cpb.aes_xcbc.cv, - csbcpb->cpb.aes_xcbc.out_cv_mac, - AES_BLOCK_SIZE); - } + memcpy(csbcpb->cpb.aes_xcbc.cv, + csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE); - NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) { rc = -EINVAL; goto out; @@ -271,28 +228,24 @@ static int nx_xcbc_update(struct shash_desc *desc, atomic_inc(&(nx_ctx->stats->aes_ops)); - /* everything after the first update is continuation */ - NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; - total -= to_process; - data += to_process - sctx->count; - sctx->count = 0; + data += to_process; in_sg = nx_ctx->in_sg; - } while (leftover > AES_BLOCK_SIZE); + } while (total >= AES_BLOCK_SIZE); - /* copy the leftover back into the state struct */ - memcpy(sctx->buffer, data, leftover); - sctx->count = leftover; + rc = total; + memcpy(sctx->state, csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE); out: spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); return rc; } -static int nx_xcbc_final(struct shash_desc *desc, u8 *out) +static int nx_xcbc_finup(struct shash_desc *desc, const u8 *src, + unsigned int nbytes, u8 *out) { + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct xcbc_state *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; struct nx_sg *in_sg, *out_sg; unsigned long irq_flags; @@ -301,12 +254,10 @@ static int nx_xcbc_final(struct shash_desc *desc, u8 *out) spin_lock_irqsave(&nx_ctx->lock, irq_flags); - if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) { - /* we've hit the nx chip previously, now we're finalizing, - * so copy over the partial digest */ - memcpy(csbcpb->cpb.aes_xcbc.cv, - csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE); - } else if (sctx->count == 0) { + if (nbytes) { + /* non-zero final, so copy over the partial digest */ + memcpy(csbcpb->cpb.aes_xcbc.cv, sctx->state, AES_BLOCK_SIZE); + } else { /* * we've never seen an update, so this is a 0 byte op. The * hardware cannot handle a 0 byte op, so just ECB to @@ -320,11 +271,11 @@ static int nx_xcbc_final(struct shash_desc *desc, u8 *out) * this is not an intermediate operation */ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; - len = sctx->count; - in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)sctx->buffer, - &len, nx_ctx->ap->sglen); + len = nbytes; + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len, + nx_ctx->ap->sglen); - if (len != sctx->count) { + if (len != nbytes) { rc = -EINVAL; goto out; } @@ -362,18 +313,19 @@ struct shash_alg nx_shash_aes_xcbc_alg = { .digestsize = AES_BLOCK_SIZE, .init = nx_xcbc_init, .update = nx_xcbc_update, - .final = nx_xcbc_final, + .finup = nx_xcbc_finup, .setkey = nx_xcbc_set_key, .descsize = sizeof(struct xcbc_state), - .statesize = sizeof(struct xcbc_state), + .init_tfm = nx_crypto_ctx_aes_xcbc_init2, + .exit_tfm = nx_crypto_ctx_shash_exit, .base = { .cra_name = "xcbc(aes)", .cra_driver_name = "xcbc-aes-nx", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINAL_NONZERO, .cra_blocksize = AES_BLOCK_SIZE, .cra_module = THIS_MODULE, .cra_ctxsize = sizeof(struct nx_crypto_ctx), - .cra_init = nx_crypto_ctx_aes_xcbc_init2, - .cra_exit = nx_crypto_ctx_exit, } }; diff --git a/drivers/crypto/nx/nx-sha256.c b/drivers/crypto/nx/nx-sha256.c index c3bebf0feabe..5b29dd026df2 100644 --- a/drivers/crypto/nx/nx-sha256.c +++ b/drivers/crypto/nx/nx-sha256.c @@ -9,9 +9,12 @@ #include <crypto/internal/hash.h> #include <crypto/sha2.h> +#include <linux/errno.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <asm/vio.h> -#include <asm/byteorder.h> +#include <linux/spinlock.h> +#include <linux/string.h> +#include <linux/unaligned.h> #include "nx_csbcpb.h" #include "nx.h" @@ -19,12 +22,11 @@ struct sha256_state_be { __be32 state[SHA256_DIGEST_SIZE / 4]; u64 count; - u8 buf[SHA256_BLOCK_SIZE]; }; -static int nx_crypto_ctx_sha256_init(struct crypto_tfm *tfm) +static int nx_crypto_ctx_sha256_init(struct crypto_shash *tfm) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm); + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm); int err; err = nx_crypto_ctx_sha_init(tfm); @@ -40,11 +42,10 @@ static int nx_crypto_ctx_sha256_init(struct crypto_tfm *tfm) return 0; } -static int nx_sha256_init(struct shash_desc *desc) { +static int nx_sha256_init(struct shash_desc *desc) +{ struct sha256_state_be *sctx = shash_desc_ctx(desc); - memset(sctx, 0, sizeof *sctx); - sctx->state[0] = __cpu_to_be32(SHA256_H0); sctx->state[1] = __cpu_to_be32(SHA256_H1); sctx->state[2] = __cpu_to_be32(SHA256_H2); @@ -61,30 +62,18 @@ static int nx_sha256_init(struct shash_desc *desc) { static int nx_sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len) { + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct sha256_state_be *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; + u64 to_process, leftover, total = len; struct nx_sg *out_sg; - u64 to_process = 0, leftover, total; unsigned long irq_flags; int rc = 0; int data_len; u32 max_sg_len; - u64 buf_len = (sctx->count % SHA256_BLOCK_SIZE); spin_lock_irqsave(&nx_ctx->lock, irq_flags); - /* 2 cases for total data len: - * 1: < SHA256_BLOCK_SIZE: copy into state, return 0 - * 2: >= SHA256_BLOCK_SIZE: process X blocks, copy in leftover - */ - total = (sctx->count % SHA256_BLOCK_SIZE) + len; - if (total < SHA256_BLOCK_SIZE) { - memcpy(sctx->buf + buf_len, data, len); - sctx->count += len; - goto out; - } - memcpy(csbcpb->cpb.sha256.message_digest, sctx->state, SHA256_DIGEST_SIZE); NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; @@ -105,41 +94,17 @@ static int nx_sha256_update(struct shash_desc *desc, const u8 *data, } do { - int used_sgs = 0; struct nx_sg *in_sg = nx_ctx->in_sg; - if (buf_len) { - data_len = buf_len; - in_sg = nx_build_sg_list(in_sg, - (u8 *) sctx->buf, - &data_len, - max_sg_len); - - if (data_len != buf_len) { - rc = -EINVAL; - goto out; - } - used_sgs = in_sg - nx_ctx->in_sg; - } + to_process = total & ~(SHA256_BLOCK_SIZE - 1); - /* to_process: SHA256_BLOCK_SIZE aligned chunk to be - * processed in this iteration. This value is restricted - * by sg list limits and number of sgs we already used - * for leftover data. (see above) - * In ideal case, we could allow NX_PAGE_SIZE * max_sg_len, - * but because data may not be aligned, we need to account - * for that too. */ - to_process = min_t(u64, total, - (max_sg_len - 1 - used_sgs) * NX_PAGE_SIZE); - to_process = to_process & ~(SHA256_BLOCK_SIZE - 1); - - data_len = to_process - buf_len; + data_len = to_process; in_sg = nx_build_sg_list(in_sg, (u8 *) data, &data_len, max_sg_len); nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); - to_process = data_len + buf_len; + to_process = data_len; leftover = total - to_process; /* @@ -162,26 +127,22 @@ static int nx_sha256_update(struct shash_desc *desc, const u8 *data, atomic_inc(&(nx_ctx->stats->sha256_ops)); total -= to_process; - data += to_process - buf_len; - buf_len = 0; - + data += to_process; + sctx->count += to_process; } while (leftover >= SHA256_BLOCK_SIZE); - /* copy the leftover back into the state struct */ - if (leftover) - memcpy(sctx->buf, data, leftover); - - sctx->count += len; + rc = leftover; memcpy(sctx->state, csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE); out: spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); return rc; } -static int nx_sha256_final(struct shash_desc *desc, u8 *out) +static int nx_sha256_finup(struct shash_desc *desc, const u8 *src, + unsigned int nbytes, u8 *out) { + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct sha256_state_be *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; struct nx_sg *in_sg, *out_sg; unsigned long irq_flags; @@ -197,25 +158,19 @@ static int nx_sha256_final(struct shash_desc *desc, u8 *out) nx_ctx->ap->databytelen/NX_PAGE_SIZE); /* final is represented by continuing the operation and indicating that - * this is not an intermediate operation */ - if (sctx->count >= SHA256_BLOCK_SIZE) { - /* we've hit the nx chip previously, now we're finalizing, - * so copy over the partial digest */ - memcpy(csbcpb->cpb.sha256.input_partial_digest, sctx->state, SHA256_DIGEST_SIZE); - NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; - NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; - } else { - NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; - NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION; - } + * this is not an intermediate operation + * copy over the partial digest */ + memcpy(csbcpb->cpb.sha256.input_partial_digest, sctx->state, SHA256_DIGEST_SIZE); + NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; + sctx->count += nbytes; csbcpb->cpb.sha256.message_bit_length = (u64) (sctx->count * 8); - len = sctx->count & (SHA256_BLOCK_SIZE - 1); - in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) sctx->buf, - &len, max_sg_len); + len = nbytes; + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len, max_sg_len); - if (len != (sctx->count & (SHA256_BLOCK_SIZE - 1))) { + if (len != nbytes) { rc = -EINVAL; goto out; } @@ -251,18 +206,34 @@ out: static int nx_sha256_export(struct shash_desc *desc, void *out) { struct sha256_state_be *sctx = shash_desc_ctx(desc); + union { + u8 *u8; + u32 *u32; + u64 *u64; + } p = { .u8 = out }; + int i; - memcpy(out, sctx, sizeof(*sctx)); + for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(*p.u32); i++) + put_unaligned(be32_to_cpu(sctx->state[i]), p.u32++); + put_unaligned(sctx->count, p.u64++); return 0; } static int nx_sha256_import(struct shash_desc *desc, const void *in) { struct sha256_state_be *sctx = shash_desc_ctx(desc); + union { + const u8 *u8; + const u32 *u32; + const u64 *u64; + } p = { .u8 = in }; + int i; - memcpy(sctx, in, sizeof(*sctx)); + for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(*p.u32); i++) + sctx->state[i] = cpu_to_be32(get_unaligned(p.u32++)); + sctx->count = get_unaligned(p.u64++); return 0; } @@ -270,19 +241,20 @@ struct shash_alg nx_shash_sha256_alg = { .digestsize = SHA256_DIGEST_SIZE, .init = nx_sha256_init, .update = nx_sha256_update, - .final = nx_sha256_final, + .finup = nx_sha256_finup, .export = nx_sha256_export, .import = nx_sha256_import, + .init_tfm = nx_crypto_ctx_sha256_init, + .exit_tfm = nx_crypto_ctx_shash_exit, .descsize = sizeof(struct sha256_state_be), .statesize = sizeof(struct sha256_state_be), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-nx", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, .cra_ctxsize = sizeof(struct nx_crypto_ctx), - .cra_init = nx_crypto_ctx_sha256_init, - .cra_exit = nx_crypto_ctx_exit, } }; diff --git a/drivers/crypto/nx/nx-sha512.c b/drivers/crypto/nx/nx-sha512.c index 1ffb40d2c324..f74776b7d7d7 100644 --- a/drivers/crypto/nx/nx-sha512.c +++ b/drivers/crypto/nx/nx-sha512.c @@ -9,8 +9,12 @@ #include <crypto/internal/hash.h> #include <crypto/sha2.h> +#include <linux/errno.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <asm/vio.h> +#include <linux/spinlock.h> +#include <linux/string.h> +#include <linux/unaligned.h> #include "nx_csbcpb.h" #include "nx.h" @@ -18,12 +22,11 @@ struct sha512_state_be { __be64 state[SHA512_DIGEST_SIZE / 8]; u64 count[2]; - u8 buf[SHA512_BLOCK_SIZE]; }; -static int nx_crypto_ctx_sha512_init(struct crypto_tfm *tfm) +static int nx_crypto_ctx_sha512_init(struct crypto_shash *tfm) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm); + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm); int err; err = nx_crypto_ctx_sha_init(tfm); @@ -43,8 +46,6 @@ static int nx_sha512_init(struct shash_desc *desc) { struct sha512_state_be *sctx = shash_desc_ctx(desc); - memset(sctx, 0, sizeof *sctx); - sctx->state[0] = __cpu_to_be64(SHA512_H0); sctx->state[1] = __cpu_to_be64(SHA512_H1); sctx->state[2] = __cpu_to_be64(SHA512_H2); @@ -54,6 +55,7 @@ static int nx_sha512_init(struct shash_desc *desc) sctx->state[6] = __cpu_to_be64(SHA512_H6); sctx->state[7] = __cpu_to_be64(SHA512_H7); sctx->count[0] = 0; + sctx->count[1] = 0; return 0; } @@ -61,30 +63,18 @@ static int nx_sha512_init(struct shash_desc *desc) static int nx_sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) { + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct sha512_state_be *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; + u64 to_process, leftover, total = len; struct nx_sg *out_sg; - u64 to_process, leftover = 0, total; unsigned long irq_flags; int rc = 0; int data_len; u32 max_sg_len; - u64 buf_len = (sctx->count[0] % SHA512_BLOCK_SIZE); spin_lock_irqsave(&nx_ctx->lock, irq_flags); - /* 2 cases for total data len: - * 1: < SHA512_BLOCK_SIZE: copy into state, return 0 - * 2: >= SHA512_BLOCK_SIZE: process X blocks, copy in leftover - */ - total = (sctx->count[0] % SHA512_BLOCK_SIZE) + len; - if (total < SHA512_BLOCK_SIZE) { - memcpy(sctx->buf + buf_len, data, len); - sctx->count[0] += len; - goto out; - } - memcpy(csbcpb->cpb.sha512.message_digest, sctx->state, SHA512_DIGEST_SIZE); NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; @@ -105,45 +95,17 @@ static int nx_sha512_update(struct shash_desc *desc, const u8 *data, } do { - int used_sgs = 0; struct nx_sg *in_sg = nx_ctx->in_sg; - if (buf_len) { - data_len = buf_len; - in_sg = nx_build_sg_list(in_sg, - (u8 *) sctx->buf, - &data_len, max_sg_len); - - if (data_len != buf_len) { - rc = -EINVAL; - goto out; - } - used_sgs = in_sg - nx_ctx->in_sg; - } + to_process = total & ~(SHA512_BLOCK_SIZE - 1); - /* to_process: SHA512_BLOCK_SIZE aligned chunk to be - * processed in this iteration. This value is restricted - * by sg list limits and number of sgs we already used - * for leftover data. (see above) - * In ideal case, we could allow NX_PAGE_SIZE * max_sg_len, - * but because data may not be aligned, we need to account - * for that too. */ - to_process = min_t(u64, total, - (max_sg_len - 1 - used_sgs) * NX_PAGE_SIZE); - to_process = to_process & ~(SHA512_BLOCK_SIZE - 1); - - data_len = to_process - buf_len; + data_len = to_process; in_sg = nx_build_sg_list(in_sg, (u8 *) data, &data_len, max_sg_len); nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); - if (data_len != (to_process - buf_len)) { - rc = -EINVAL; - goto out; - } - - to_process = data_len + buf_len; + to_process = data_len; leftover = total - to_process; /* @@ -166,30 +128,29 @@ static int nx_sha512_update(struct shash_desc *desc, const u8 *data, atomic_inc(&(nx_ctx->stats->sha512_ops)); total -= to_process; - data += to_process - buf_len; - buf_len = 0; - + data += to_process; + sctx->count[0] += to_process; + if (sctx->count[0] < to_process) + sctx->count[1]++; } while (leftover >= SHA512_BLOCK_SIZE); - /* copy the leftover back into the state struct */ - if (leftover) - memcpy(sctx->buf, data, leftover); - sctx->count[0] += len; + rc = leftover; memcpy(sctx->state, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE); out: spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); return rc; } -static int nx_sha512_final(struct shash_desc *desc, u8 *out) +static int nx_sha512_finup(struct shash_desc *desc, const u8 *src, + unsigned int nbytes, u8 *out) { struct sha512_state_be *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; struct nx_sg *in_sg, *out_sg; u32 max_sg_len; - u64 count0; unsigned long irq_flags; + u64 count0, count1; int rc = 0; int len; @@ -201,30 +162,23 @@ static int nx_sha512_final(struct shash_desc *desc, u8 *out) nx_ctx->ap->databytelen/NX_PAGE_SIZE); /* final is represented by continuing the operation and indicating that - * this is not an intermediate operation */ - if (sctx->count[0] >= SHA512_BLOCK_SIZE) { - /* we've hit the nx chip previously, now we're finalizing, - * so copy over the partial digest */ - memcpy(csbcpb->cpb.sha512.input_partial_digest, sctx->state, - SHA512_DIGEST_SIZE); - NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; - NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; - } else { - NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; - NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION; - } - + * this is not an intermediate operation + * copy over the partial digest */ + memcpy(csbcpb->cpb.sha512.input_partial_digest, sctx->state, SHA512_DIGEST_SIZE); NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; - count0 = sctx->count[0] * 8; + count0 = sctx->count[0] + nbytes; + count1 = sctx->count[1]; - csbcpb->cpb.sha512.message_bit_length_lo = count0; + csbcpb->cpb.sha512.message_bit_length_lo = count0 << 3; + csbcpb->cpb.sha512.message_bit_length_hi = (count1 << 3) | + (count0 >> 61); - len = sctx->count[0] & (SHA512_BLOCK_SIZE - 1); - in_sg = nx_build_sg_list(nx_ctx->in_sg, sctx->buf, &len, - max_sg_len); + len = nbytes; + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len, max_sg_len); - if (len != (sctx->count[0] & (SHA512_BLOCK_SIZE - 1))) { + if (len != nbytes) { rc = -EINVAL; goto out; } @@ -246,7 +200,7 @@ static int nx_sha512_final(struct shash_desc *desc, u8 *out) goto out; atomic_inc(&(nx_ctx->stats->sha512_ops)); - atomic64_add(sctx->count[0], &(nx_ctx->stats->sha512_bytes)); + atomic64_add(count0, &(nx_ctx->stats->sha512_bytes)); memcpy(out, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE); out: @@ -257,18 +211,34 @@ out: static int nx_sha512_export(struct shash_desc *desc, void *out) { struct sha512_state_be *sctx = shash_desc_ctx(desc); + union { + u8 *u8; + u64 *u64; + } p = { .u8 = out }; + int i; - memcpy(out, sctx, sizeof(*sctx)); + for (i = 0; i < SHA512_DIGEST_SIZE / sizeof(*p.u64); i++) + put_unaligned(be64_to_cpu(sctx->state[i]), p.u64++); + put_unaligned(sctx->count[0], p.u64++); + put_unaligned(sctx->count[1], p.u64++); return 0; } static int nx_sha512_import(struct shash_desc *desc, const void *in) { struct sha512_state_be *sctx = shash_desc_ctx(desc); + union { + const u8 *u8; + const u64 *u64; + } p = { .u8 = in }; + int i; - memcpy(sctx, in, sizeof(*sctx)); + for (i = 0; i < SHA512_DIGEST_SIZE / sizeof(*p.u64); i++) + sctx->state[i] = cpu_to_be64(get_unaligned(p.u64++)); + sctx->count[0] = get_unaligned(p.u64++); + sctx->count[1] = get_unaligned(p.u64++); return 0; } @@ -276,19 +246,20 @@ struct shash_alg nx_shash_sha512_alg = { .digestsize = SHA512_DIGEST_SIZE, .init = nx_sha512_init, .update = nx_sha512_update, - .final = nx_sha512_final, + .finup = nx_sha512_finup, .export = nx_sha512_export, .import = nx_sha512_import, + .init_tfm = nx_crypto_ctx_sha512_init, + .exit_tfm = nx_crypto_ctx_shash_exit, .descsize = sizeof(struct sha512_state_be), .statesize = sizeof(struct sha512_state_be), .base = { .cra_name = "sha512", .cra_driver_name = "sha512-nx", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, .cra_ctxsize = sizeof(struct nx_crypto_ctx), - .cra_init = nx_crypto_ctx_sha512_init, - .cra_exit = nx_crypto_ctx_exit, } }; diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index a3b979193d9b..78135fb13f5c 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -7,11 +7,11 @@ * Author: Kent Yoder <yoder1@us.ibm.com> */ +#include <crypto/aes.h> #include <crypto/internal/aead.h> #include <crypto/internal/hash.h> -#include <crypto/aes.h> +#include <crypto/internal/skcipher.h> #include <crypto/sha2.h> -#include <crypto/algapi.h> #include <crypto/scatterwalk.h> #include <linux/module.h> #include <linux/moduleparam.h> @@ -124,8 +124,6 @@ struct nx_sg *nx_build_sg_list(struct nx_sg *sg_head, } if ((sg - sg_head) == sgmax) { - pr_err("nx: scatter/gather list overflow, pid: %d\n", - current->pid); sg++; break; } @@ -702,14 +700,14 @@ int nx_crypto_ctx_aes_ecb_init(struct crypto_skcipher *tfm) NX_MODE_AES_ECB); } -int nx_crypto_ctx_sha_init(struct crypto_tfm *tfm) +int nx_crypto_ctx_sha_init(struct crypto_shash *tfm) { - return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_SHA, NX_MODE_SHA); + return nx_crypto_ctx_init(crypto_shash_ctx(tfm), NX_FC_SHA, NX_MODE_SHA); } -int nx_crypto_ctx_aes_xcbc_init(struct crypto_tfm *tfm) +int nx_crypto_ctx_aes_xcbc_init(struct crypto_shash *tfm) { - return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES, + return nx_crypto_ctx_init(crypto_shash_ctx(tfm), NX_FC_AES, NX_MODE_AES_XCBC_MAC); } @@ -744,6 +742,11 @@ void nx_crypto_ctx_aead_exit(struct crypto_aead *tfm) kfree_sensitive(nx_ctx->kmem); } +void nx_crypto_ctx_shash_exit(struct crypto_shash *tfm) +{ + nx_crypto_ctx_exit(crypto_shash_ctx(tfm)); +} + static int nx_probe(struct vio_dev *viodev, const struct vio_device_id *id) { dev_dbg(&viodev->dev, "driver probed: %s resource id: 0x%x\n", diff --git a/drivers/crypto/nx/nx.h b/drivers/crypto/nx/nx.h index e1b4b6927bec..36974f08490a 100644 --- a/drivers/crypto/nx/nx.h +++ b/drivers/crypto/nx/nx.h @@ -3,7 +3,11 @@ #ifndef __NX_H__ #define __NX_H__ +#include <asm/vio.h> #include <crypto/ctr.h> +#include <crypto/internal/aead.h> +#include <crypto/internal/hash.h> +#include <crypto/internal/skcipher.h> #define NX_NAME "nx-crypto" #define NX_STRING "IBM Power7+ Nest Accelerator Crypto Driver" @@ -139,19 +143,20 @@ struct nx_crypto_ctx { } priv; }; -struct crypto_aead; +struct scatterlist; /* prototypes */ int nx_crypto_ctx_aes_ccm_init(struct crypto_aead *tfm); int nx_crypto_ctx_aes_gcm_init(struct crypto_aead *tfm); -int nx_crypto_ctx_aes_xcbc_init(struct crypto_tfm *tfm); +int nx_crypto_ctx_aes_xcbc_init(struct crypto_shash *tfm); int nx_crypto_ctx_aes_ctr_init(struct crypto_skcipher *tfm); int nx_crypto_ctx_aes_cbc_init(struct crypto_skcipher *tfm); int nx_crypto_ctx_aes_ecb_init(struct crypto_skcipher *tfm); -int nx_crypto_ctx_sha_init(struct crypto_tfm *tfm); +int nx_crypto_ctx_sha_init(struct crypto_shash *tfm); void nx_crypto_ctx_exit(struct crypto_tfm *tfm); void nx_crypto_ctx_skcipher_exit(struct crypto_skcipher *tfm); void nx_crypto_ctx_aead_exit(struct crypto_aead *tfm); +void nx_crypto_ctx_shash_exit(struct crypto_shash *tfm); void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function); int nx_hcall_sync(struct nx_crypto_ctx *ctx, struct vio_pfo_op *op, u32 may_sleep); diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 551dd32a8db0..1ecf5f6ac04e 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -1086,10 +1086,7 @@ static struct attribute *omap_aes_attrs[] = { &dev_attr_fallback.attr, NULL, }; - -static const struct attribute_group omap_aes_attr_group = { - .attrs = omap_aes_attrs, -}; +ATTRIBUTE_GROUPS(omap_aes); static int omap_aes_probe(struct platform_device *pdev) { @@ -1215,12 +1212,6 @@ static int omap_aes_probe(struct platform_device *pdev) } } - err = sysfs_create_group(&dev->kobj, &omap_aes_attr_group); - if (err) { - dev_err(dev, "could not create sysfs device attrs\n"); - goto err_aead_algs; - } - return 0; err_aead_algs: for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) { @@ -1277,8 +1268,6 @@ static void omap_aes_remove(struct platform_device *pdev) tasklet_kill(&dd->done_task); omap_aes_dma_cleanup(dd); pm_runtime_disable(dd->dev); - - sysfs_remove_group(&dd->dev->kobj, &omap_aes_attr_group); } #ifdef CONFIG_PM_SLEEP @@ -1304,6 +1293,7 @@ static struct platform_driver omap_aes_driver = { .name = "omap-aes", .pm = &omap_aes_pm_ops, .of_match_table = omap_aes_of_match, + .dev_groups = omap_aes_groups, }, }; diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 7021481bf027..56f192cb976d 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -2039,10 +2039,7 @@ static struct attribute *omap_sham_attrs[] = { &dev_attr_fallback.attr, NULL, }; - -static const struct attribute_group omap_sham_attr_group = { - .attrs = omap_sham_attrs, -}; +ATTRIBUTE_GROUPS(omap_sham); static int omap_sham_probe(struct platform_device *pdev) { @@ -2158,12 +2155,6 @@ static int omap_sham_probe(struct platform_device *pdev) } } - err = sysfs_create_group(&dev->kobj, &omap_sham_attr_group); - if (err) { - dev_err(dev, "could not create sysfs device attrs\n"); - goto err_algs; - } - return 0; err_algs: @@ -2210,8 +2201,6 @@ static void omap_sham_remove(struct platform_device *pdev) if (!dd->polling_mode) dma_release_channel(dd->dma_lch); - - sysfs_remove_group(&dd->dev->kobj, &omap_sham_attr_group); } static struct platform_driver omap_sham_driver = { @@ -2220,6 +2209,7 @@ static struct platform_driver omap_sham_driver = { .driver = { .name = "omap-sham", .of_match_table = omap_sham_of_match, + .dev_groups = omap_sham_groups, }, }; diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c index db9e84c0c9fb..c89b9c6b5f4c 100644 --- a/drivers/crypto/padlock-sha.c +++ b/drivers/crypto/padlock-sha.c @@ -7,59 +7,83 @@ * Copyright (c) 2006 Michal Ludvig <michal@logix.cz> */ +#include <asm/cpu_device_id.h> #include <crypto/internal/hash.h> #include <crypto/padlock.h> #include <crypto/sha1.h> #include <crypto/sha2.h> +#include <linux/cpufeature.h> #include <linux/err.h> -#include <linux/module.h> -#include <linux/init.h> -#include <linux/errno.h> -#include <linux/interrupt.h> #include <linux/kernel.h> -#include <linux/scatterlist.h> -#include <asm/cpu_device_id.h> -#include <asm/fpu/api.h> +#include <linux/module.h> -struct padlock_sha_desc { - struct shash_desc fallback; -}; +#define PADLOCK_SHA_DESCSIZE (128 + ((PADLOCK_ALIGNMENT - 1) & \ + ~(CRYPTO_MINALIGN - 1))) struct padlock_sha_ctx { - struct crypto_shash *fallback; + struct crypto_ahash *fallback; }; -static int padlock_sha_init(struct shash_desc *desc) +static inline void *padlock_shash_desc_ctx(struct shash_desc *desc) { - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); + return PTR_ALIGN(shash_desc_ctx(desc), PADLOCK_ALIGNMENT); +} + +static int padlock_sha1_init(struct shash_desc *desc) +{ + struct sha1_state *sctx = padlock_shash_desc_ctx(desc); + + *sctx = (struct sha1_state){ + .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, + }; + + return 0; +} + +static int padlock_sha256_init(struct shash_desc *desc) +{ + struct sha256_state *sctx = padlock_shash_desc_ctx(desc); - dctx->fallback.tfm = ctx->fallback; - return crypto_shash_init(&dctx->fallback); + sha256_init(sctx); + return 0; } static int padlock_sha_update(struct shash_desc *desc, const u8 *data, unsigned int length) { - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - - return crypto_shash_update(&dctx->fallback, data, length); + struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); + u8 *state = padlock_shash_desc_ctx(desc); + HASH_REQUEST_ON_STACK(req, ctx->fallback); + int remain; + + ahash_request_set_callback(req, 0, NULL, NULL); + ahash_request_set_virt(req, data, NULL, length); + remain = crypto_ahash_import(req, state) ?: + crypto_ahash_update(req); + if (remain < 0) + return remain; + return crypto_ahash_export(req, state) ?: remain; } static int padlock_sha_export(struct shash_desc *desc, void *out) { - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - - return crypto_shash_export(&dctx->fallback, out); + memcpy(out, padlock_shash_desc_ctx(desc), + crypto_shash_coresize(desc->tfm)); + return 0; } static int padlock_sha_import(struct shash_desc *desc, const void *in) { - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); + unsigned int bs = crypto_shash_blocksize(desc->tfm); + unsigned int ss = crypto_shash_coresize(desc->tfm); + u64 *state = padlock_shash_desc_ctx(desc); + + memcpy(state, in, ss); - dctx->fallback.tfm = ctx->fallback; - return crypto_shash_import(&dctx->fallback, in); + /* Stop evil imports from generating a fault. */ + state[ss / 8 - 1] &= ~(bs - 1); + + return 0; } static inline void padlock_output_block(uint32_t *src, @@ -69,65 +93,38 @@ static inline void padlock_output_block(uint32_t *src, *dst++ = swab32(*src++); } +static int padlock_sha_finup(struct shash_desc *desc, const u8 *in, + unsigned int count, u8 *out) +{ + struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); + HASH_REQUEST_ON_STACK(req, ctx->fallback); + + ahash_request_set_callback(req, 0, NULL, NULL); + ahash_request_set_virt(req, in, out, count); + return crypto_ahash_import(req, padlock_shash_desc_ctx(desc)) ?: + crypto_ahash_finup(req); +} + static int padlock_sha1_finup(struct shash_desc *desc, const u8 *in, unsigned int count, u8 *out) { /* We can't store directly to *out as it may be unaligned. */ /* BTW Don't reduce the buffer size below 128 Bytes! * PadLock microcode needs it that big. */ - char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ - ((aligned(STACK_ALIGN))); - char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - struct sha1_state state; - unsigned int space; - unsigned int leftover; - int err; - - err = crypto_shash_export(&dctx->fallback, &state); - if (err) - goto out; - - if (state.count + count > ULONG_MAX) - return crypto_shash_finup(&dctx->fallback, in, count, out); - - leftover = ((state.count - 1) & (SHA1_BLOCK_SIZE - 1)) + 1; - space = SHA1_BLOCK_SIZE - leftover; - if (space) { - if (count > space) { - err = crypto_shash_update(&dctx->fallback, in, space) ?: - crypto_shash_export(&dctx->fallback, &state); - if (err) - goto out; - count -= space; - in += space; - } else { - memcpy(state.buffer + leftover, in, count); - in = state.buffer; - count += leftover; - state.count &= ~(SHA1_BLOCK_SIZE - 1); - } - } + struct sha1_state *state = padlock_shash_desc_ctx(desc); + u64 start = state->count; - memcpy(result, &state.state, SHA1_DIGEST_SIZE); + if (start + count > ULONG_MAX) + return padlock_sha_finup(desc, in, count, out); asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" /* rep xsha1 */ : \ - : "c"((unsigned long)state.count + count), \ - "a"((unsigned long)state.count), \ - "S"(in), "D"(result)); + : "c"((unsigned long)start + count), \ + "a"((unsigned long)start), \ + "S"(in), "D"(state)); - padlock_output_block((uint32_t *)result, (uint32_t *)out, 5); - -out: - return err; -} - -static int padlock_sha1_final(struct shash_desc *desc, u8 *out) -{ - const u8 *buf = (void *)desc; - - return padlock_sha1_finup(desc, buf, 0, out); + padlock_output_block(state->state, (uint32_t *)out, 5); + return 0; } static int padlock_sha256_finup(struct shash_desc *desc, const u8 *in, @@ -136,78 +133,46 @@ static int padlock_sha256_finup(struct shash_desc *desc, const u8 *in, /* We can't store directly to *out as it may be unaligned. */ /* BTW Don't reduce the buffer size below 128 Bytes! * PadLock microcode needs it that big. */ - char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ - ((aligned(STACK_ALIGN))); - char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - struct sha256_state state; - unsigned int space; - unsigned int leftover; - int err; - - err = crypto_shash_export(&dctx->fallback, &state); - if (err) - goto out; - - if (state.count + count > ULONG_MAX) - return crypto_shash_finup(&dctx->fallback, in, count, out); - - leftover = ((state.count - 1) & (SHA256_BLOCK_SIZE - 1)) + 1; - space = SHA256_BLOCK_SIZE - leftover; - if (space) { - if (count > space) { - err = crypto_shash_update(&dctx->fallback, in, space) ?: - crypto_shash_export(&dctx->fallback, &state); - if (err) - goto out; - count -= space; - in += space; - } else { - memcpy(state.buf + leftover, in, count); - in = state.buf; - count += leftover; - state.count &= ~(SHA1_BLOCK_SIZE - 1); - } - } + struct sha256_state *state = padlock_shash_desc_ctx(desc); + u64 start = state->count; - memcpy(result, &state.state, SHA256_DIGEST_SIZE); + if (start + count > ULONG_MAX) + return padlock_sha_finup(desc, in, count, out); asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" /* rep xsha256 */ : \ - : "c"((unsigned long)state.count + count), \ - "a"((unsigned long)state.count), \ - "S"(in), "D"(result)); - - padlock_output_block((uint32_t *)result, (uint32_t *)out, 8); - -out: - return err; -} + : "c"((unsigned long)start + count), \ + "a"((unsigned long)start), \ + "S"(in), "D"(state)); -static int padlock_sha256_final(struct shash_desc *desc, u8 *out) -{ - const u8 *buf = (void *)desc; - - return padlock_sha256_finup(desc, buf, 0, out); + padlock_output_block(state->state, (uint32_t *)out, 8); + return 0; } static int padlock_init_tfm(struct crypto_shash *hash) { const char *fallback_driver_name = crypto_shash_alg_name(hash); struct padlock_sha_ctx *ctx = crypto_shash_ctx(hash); - struct crypto_shash *fallback_tfm; + struct crypto_ahash *fallback_tfm; /* Allocate a fallback and abort if it failed. */ - fallback_tfm = crypto_alloc_shash(fallback_driver_name, 0, - CRYPTO_ALG_NEED_FALLBACK); + fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); if (IS_ERR(fallback_tfm)) { printk(KERN_WARNING PFX "Fallback driver '%s' could not be loaded!\n", fallback_driver_name); return PTR_ERR(fallback_tfm); } + if (crypto_shash_statesize(hash) < + crypto_ahash_statesize(fallback_tfm)) { + crypto_free_ahash(fallback_tfm); + return -EINVAL; + } + ctx->fallback = fallback_tfm; - hash->descsize += crypto_shash_descsize(fallback_tfm); + return 0; } @@ -215,26 +180,27 @@ static void padlock_exit_tfm(struct crypto_shash *hash) { struct padlock_sha_ctx *ctx = crypto_shash_ctx(hash); - crypto_free_shash(ctx->fallback); + crypto_free_ahash(ctx->fallback); } static struct shash_alg sha1_alg = { .digestsize = SHA1_DIGEST_SIZE, - .init = padlock_sha_init, + .init = padlock_sha1_init, .update = padlock_sha_update, .finup = padlock_sha1_finup, - .final = padlock_sha1_final, .export = padlock_sha_export, .import = padlock_sha_import, .init_tfm = padlock_init_tfm, .exit_tfm = padlock_exit_tfm, - .descsize = sizeof(struct padlock_sha_desc), - .statesize = sizeof(struct sha1_state), + .descsize = PADLOCK_SHA_DESCSIZE, + .statesize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-padlock", .cra_priority = PADLOCK_CRA_PRIORITY, - .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_ctxsize = sizeof(struct padlock_sha_ctx), .cra_module = THIS_MODULE, @@ -243,21 +209,22 @@ static struct shash_alg sha1_alg = { static struct shash_alg sha256_alg = { .digestsize = SHA256_DIGEST_SIZE, - .init = padlock_sha_init, + .init = padlock_sha256_init, .update = padlock_sha_update, .finup = padlock_sha256_finup, - .final = padlock_sha256_final, + .init_tfm = padlock_init_tfm, .export = padlock_sha_export, .import = padlock_sha_import, - .init_tfm = padlock_init_tfm, .exit_tfm = padlock_exit_tfm, - .descsize = sizeof(struct padlock_sha_desc), - .statesize = sizeof(struct sha256_state), + .descsize = PADLOCK_SHA_DESCSIZE, + .statesize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-padlock", .cra_priority = PADLOCK_CRA_PRIORITY, - .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct padlock_sha_ctx), .cra_module = THIS_MODULE, @@ -266,207 +233,58 @@ static struct shash_alg sha256_alg = { /* Add two shash_alg instance for hardware-implemented * * multiple-parts hash supported by VIA Nano Processor.*/ -static int padlock_sha1_init_nano(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha1_state){ - .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, - }; - - return 0; -} static int padlock_sha1_update_nano(struct shash_desc *desc, - const u8 *data, unsigned int len) + const u8 *src, unsigned int len) { - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int partial, done; - const u8 *src; /*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/ - u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ - ((aligned(STACK_ALIGN))); - u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); - - partial = sctx->count & 0x3f; - sctx->count += len; - done = 0; - src = data; - memcpy(dst, (u8 *)(sctx->state), SHA1_DIGEST_SIZE); - - if ((partial + len) >= SHA1_BLOCK_SIZE) { - - /* Append the bytes in state's buffer to a block to handle */ - if (partial) { - done = -partial; - memcpy(sctx->buffer + partial, data, - done + SHA1_BLOCK_SIZE); - src = sctx->buffer; - asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" - : "+S"(src), "+D"(dst) \ - : "a"((long)-1), "c"((unsigned long)1)); - done += SHA1_BLOCK_SIZE; - src = data + done; - } - - /* Process the left bytes from the input data */ - if (len - done >= SHA1_BLOCK_SIZE) { - asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" - : "+S"(src), "+D"(dst) - : "a"((long)-1), - "c"((unsigned long)((len - done) / SHA1_BLOCK_SIZE))); - done += ((len - done) - (len - done) % SHA1_BLOCK_SIZE); - src = data + done; - } - partial = 0; - } - memcpy((u8 *)(sctx->state), dst, SHA1_DIGEST_SIZE); - memcpy(sctx->buffer + partial, src, len - done); - - return 0; -} - -static int padlock_sha1_final_nano(struct shash_desc *desc, u8 *out) -{ - struct sha1_state *state = (struct sha1_state *)shash_desc_ctx(desc); - unsigned int partial, padlen; - __be64 bits; - static const u8 padding[64] = { 0x80, }; - - bits = cpu_to_be64(state->count << 3); - - /* Pad out to 56 mod 64 */ - partial = state->count & 0x3f; - padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial); - padlock_sha1_update_nano(desc, padding, padlen); - - /* Append length field bytes */ - padlock_sha1_update_nano(desc, (const u8 *)&bits, sizeof(bits)); - - /* Swap to output */ - padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 5); - - return 0; -} - -static int padlock_sha256_init_nano(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha256_state){ - .state = { SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, \ - SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7}, - }; - - return 0; + struct sha1_state *state = padlock_shash_desc_ctx(desc); + int blocks = len / SHA1_BLOCK_SIZE; + + len -= blocks * SHA1_BLOCK_SIZE; + state->count += blocks * SHA1_BLOCK_SIZE; + + /* Process the left bytes from the input data */ + asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" + : "+S"(src), "+D"(state) + : "a"((long)-1), + "c"((unsigned long)blocks)); + return len; } -static int padlock_sha256_update_nano(struct shash_desc *desc, const u8 *data, +static int padlock_sha256_update_nano(struct shash_desc *desc, const u8 *src, unsigned int len) { - struct sha256_state *sctx = shash_desc_ctx(desc); - unsigned int partial, done; - const u8 *src; /*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/ - u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ - ((aligned(STACK_ALIGN))); - u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); - - partial = sctx->count & 0x3f; - sctx->count += len; - done = 0; - src = data; - memcpy(dst, (u8 *)(sctx->state), SHA256_DIGEST_SIZE); - - if ((partial + len) >= SHA256_BLOCK_SIZE) { - - /* Append the bytes in state's buffer to a block to handle */ - if (partial) { - done = -partial; - memcpy(sctx->buf + partial, data, - done + SHA256_BLOCK_SIZE); - src = sctx->buf; - asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" - : "+S"(src), "+D"(dst) - : "a"((long)-1), "c"((unsigned long)1)); - done += SHA256_BLOCK_SIZE; - src = data + done; - } - - /* Process the left bytes from input data*/ - if (len - done >= SHA256_BLOCK_SIZE) { - asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" - : "+S"(src), "+D"(dst) - : "a"((long)-1), - "c"((unsigned long)((len - done) / 64))); - done += ((len - done) - (len - done) % 64); - src = data + done; - } - partial = 0; - } - memcpy((u8 *)(sctx->state), dst, SHA256_DIGEST_SIZE); - memcpy(sctx->buf + partial, src, len - done); - - return 0; -} - -static int padlock_sha256_final_nano(struct shash_desc *desc, u8 *out) -{ - struct sha256_state *state = - (struct sha256_state *)shash_desc_ctx(desc); - unsigned int partial, padlen; - __be64 bits; - static const u8 padding[64] = { 0x80, }; - - bits = cpu_to_be64(state->count << 3); - - /* Pad out to 56 mod 64 */ - partial = state->count & 0x3f; - padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial); - padlock_sha256_update_nano(desc, padding, padlen); - - /* Append length field bytes */ - padlock_sha256_update_nano(desc, (const u8 *)&bits, sizeof(bits)); - - /* Swap to output */ - padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 8); - - return 0; -} - -static int padlock_sha_export_nano(struct shash_desc *desc, - void *out) -{ - int statesize = crypto_shash_statesize(desc->tfm); - void *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, statesize); - return 0; -} - -static int padlock_sha_import_nano(struct shash_desc *desc, - const void *in) -{ - int statesize = crypto_shash_statesize(desc->tfm); - void *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, statesize); - return 0; + struct crypto_sha256_state *state = padlock_shash_desc_ctx(desc); + int blocks = len / SHA256_BLOCK_SIZE; + + len -= blocks * SHA256_BLOCK_SIZE; + state->count += blocks * SHA256_BLOCK_SIZE; + + /* Process the left bytes from input data*/ + asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" + : "+S"(src), "+D"(state) + : "a"((long)-1), + "c"((unsigned long)blocks)); + return len; } static struct shash_alg sha1_alg_nano = { .digestsize = SHA1_DIGEST_SIZE, - .init = padlock_sha1_init_nano, + .init = padlock_sha1_init, .update = padlock_sha1_update_nano, - .final = padlock_sha1_final_nano, - .export = padlock_sha_export_nano, - .import = padlock_sha_import_nano, - .descsize = sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), + .finup = padlock_sha1_finup, + .export = padlock_sha_export, + .import = padlock_sha_import, + .descsize = PADLOCK_SHA_DESCSIZE, + .statesize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-padlock-nano", .cra_priority = PADLOCK_CRA_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -474,17 +292,19 @@ static struct shash_alg sha1_alg_nano = { static struct shash_alg sha256_alg_nano = { .digestsize = SHA256_DIGEST_SIZE, - .init = padlock_sha256_init_nano, + .init = padlock_sha256_init, .update = padlock_sha256_update_nano, - .final = padlock_sha256_final_nano, - .export = padlock_sha_export_nano, - .import = padlock_sha_import_nano, - .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), + .finup = padlock_sha256_finup, + .export = padlock_sha_export, + .import = padlock_sha_import, + .descsize = PADLOCK_SHA_DESCSIZE, + .statesize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-padlock-nano", .cra_priority = PADLOCK_CRA_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/drivers/crypto/rockchip/rk3288_crypto_ahash.c b/drivers/crypto/rockchip/rk3288_crypto_ahash.c index 69d6019d8abc..d6928ebe9526 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_ahash.c +++ b/drivers/crypto/rockchip/rk3288_crypto_ahash.c @@ -52,12 +52,11 @@ static int rk_ahash_digest_fb(struct ahash_request *areq) algt->stat_fb++; ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; - rctx->fallback_req.result = areq->result; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); return crypto_ahash_digest(&rctx->fallback_req); } @@ -124,8 +123,9 @@ static int rk_ahash_init(struct ahash_request *req) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_init(&rctx->fallback_req); } @@ -137,10 +137,10 @@ static int rk_ahash_update(struct ahash_request *req) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); return crypto_ahash_update(&rctx->fallback_req); } @@ -152,9 +152,10 @@ static int rk_ahash_final(struct ahash_request *req) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); return crypto_ahash_final(&rctx->fallback_req); } @@ -166,12 +167,11 @@ static int rk_ahash_finup(struct ahash_request *req) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, + req->nbytes); return crypto_ahash_finup(&rctx->fallback_req); } @@ -183,8 +183,9 @@ static int rk_ahash_import(struct ahash_request *req, const void *in) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_import(&rctx->fallback_req, in); } @@ -196,8 +197,9 @@ static int rk_ahash_export(struct ahash_request *req, void *out) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_export(&rctx->fallback_req, out); } diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index b4c3c14dafd5..b829c84f60f2 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -9,11 +9,17 @@ // // Hash part based on omap-sham.c driver. +#include <crypto/aes.h> +#include <crypto/ctr.h> +#include <crypto/internal/hash.h> +#include <crypto/internal/skcipher.h> +#include <crypto/md5.h> +#include <crypto/scatterwalk.h> +#include <crypto/sha1.h> +#include <crypto/sha2.h> #include <linux/clk.h> -#include <linux/crypto.h> #include <linux/dma-mapping.h> #include <linux/err.h> -#include <linux/errno.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/io.h> @@ -22,17 +28,9 @@ #include <linux/of.h> #include <linux/platform_device.h> #include <linux/scatterlist.h> - -#include <crypto/ctr.h> -#include <crypto/aes.h> -#include <crypto/algapi.h> -#include <crypto/scatterwalk.h> - -#include <crypto/hash.h> -#include <crypto/md5.h> -#include <crypto/sha1.h> -#include <crypto/sha2.h> -#include <crypto/internal/hash.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/string.h> #define _SBF(s, v) ((v) << (s)) diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index 091612b066f1..fdc0b2486069 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -1415,22 +1415,13 @@ static int sa_sha_run(struct ahash_request *req) (auth_len >= SA_UNSAFE_DATA_SZ_MIN && auth_len <= SA_UNSAFE_DATA_SZ_MAX)) { struct ahash_request *subreq = &rctx->fallback_req; - int ret = 0; + int ret; ahash_request_set_tfm(subreq, ctx->fallback.ahash); - subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - - crypto_ahash_init(subreq); - - subreq->nbytes = auth_len; - subreq->src = req->src; - subreq->result = req->result; - - ret |= crypto_ahash_update(subreq); - - subreq->nbytes = 0; + ahash_request_set_callback(subreq, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(subreq, req->src, req->result, auth_len); - ret |= crypto_ahash_final(subreq); + ret = crypto_ahash_digest(subreq); return ret; } @@ -1502,8 +1493,7 @@ static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) return ret; if (alg_base) { - ctx->shash = crypto_alloc_shash(alg_base, 0, - CRYPTO_ALG_NEED_FALLBACK); + ctx->shash = crypto_alloc_shash(alg_base, 0, 0); if (IS_ERR(ctx->shash)) { dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", alg_base); @@ -1511,8 +1501,7 @@ static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) } /* for fallback */ ctx->fallback.ahash = - crypto_alloc_ahash(alg_base, 0, - CRYPTO_ALG_NEED_FALLBACK); + crypto_alloc_ahash(alg_base, 0, CRYPTO_ALG_ASYNC); if (IS_ERR(ctx->fallback.ahash)) { dev_err(ctx->dev_data->dev, "Could not load fallback driver\n"); @@ -1546,54 +1535,38 @@ static int sa_sha_init(struct ahash_request *req) crypto_ahash_digestsize(tfm), rctx); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); - rctx->fallback_req.base.flags = - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(&rctx->fallback_req, NULL, NULL, 0); return crypto_ahash_init(&rctx->fallback_req); } static int sa_sha_update(struct ahash_request *req) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); - struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); - rctx->fallback_req.base.flags = - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); return crypto_ahash_update(&rctx->fallback_req); } static int sa_sha_final(struct ahash_request *req) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); - struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); - rctx->fallback_req.base.flags = - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); return crypto_ahash_final(&rctx->fallback_req); } static int sa_sha_finup(struct ahash_request *req) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); - struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); - rctx->fallback_req.base.flags = - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, req->nbytes); return crypto_ahash_finup(&rctx->fallback_req); } @@ -1605,8 +1578,7 @@ static int sa_sha_import(struct ahash_request *req, const void *in) struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); return crypto_ahash_import(&rctx->fallback_req, in); } @@ -1614,12 +1586,9 @@ static int sa_sha_import(struct ahash_request *req, const void *in) static int sa_sha_export(struct ahash_request *req, void *out) { struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); struct ahash_request *subreq = &rctx->fallback_req; - ahash_request_set_tfm(subreq, ctx->fallback.ahash); - subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(subreq, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); return crypto_ahash_export(subreq, out); } diff --git a/drivers/crypto/tegra/tegra-se-hash.c b/drivers/crypto/tegra/tegra-se-hash.c index 42d007b7af45..d09b4aaeecef 100644 --- a/drivers/crypto/tegra/tegra-se-hash.c +++ b/drivers/crypto/tegra/tegra-se-hash.c @@ -117,8 +117,9 @@ static int tegra_sha_fallback_init(struct ahash_request *req) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_init(&rctx->fallback_req); } @@ -130,10 +131,10 @@ static int tegra_sha_fallback_update(struct ahash_request *req) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); return crypto_ahash_update(&rctx->fallback_req); } @@ -145,9 +146,10 @@ static int tegra_sha_fallback_final(struct ahash_request *req) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); return crypto_ahash_final(&rctx->fallback_req); } @@ -159,12 +161,11 @@ static int tegra_sha_fallback_finup(struct ahash_request *req) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, + req->nbytes); return crypto_ahash_finup(&rctx->fallback_req); } @@ -176,12 +177,11 @@ static int tegra_sha_fallback_digest(struct ahash_request *req) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, + req->nbytes); return crypto_ahash_digest(&rctx->fallback_req); } @@ -193,8 +193,9 @@ static int tegra_sha_fallback_import(struct ahash_request *req, const void *in) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_import(&rctx->fallback_req, in); } @@ -206,8 +207,9 @@ static int tegra_sha_fallback_export(struct ahash_request *req, void *out) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_export(&rctx->fallback_req, out); } diff --git a/drivers/crypto/xilinx/zynqmp-sha.c b/drivers/crypto/xilinx/zynqmp-sha.c index 580649f9bff8..67cf8d990a1d 100644 --- a/drivers/crypto/xilinx/zynqmp-sha.c +++ b/drivers/crypto/xilinx/zynqmp-sha.c @@ -3,18 +3,18 @@ * Xilinx ZynqMP SHA Driver. * Copyright (c) 2022 Xilinx Inc. */ -#include <linux/cacheflush.h> -#include <crypto/hash.h> #include <crypto/internal/hash.h> #include <crypto/sha3.h> -#include <linux/crypto.h> +#include <linux/cacheflush.h> +#include <linux/cleanup.h> #include <linux/device.h> #include <linux/dma-mapping.h> +#include <linux/err.h> #include <linux/firmware/xlnx-zynqmp.h> -#include <linux/init.h> #include <linux/io.h> #include <linux/kernel.h> #include <linux/module.h> +#include <linux/spinlock.h> #include <linux/platform_device.h> #define ZYNQMP_DMA_BIT_MASK 32U @@ -36,13 +36,11 @@ struct zynqmp_sha_tfm_ctx { struct crypto_shash *fbk_tfm; }; -struct zynqmp_sha_desc_ctx { - struct shash_desc fbk_req; -}; - static dma_addr_t update_dma_addr, final_dma_addr; static char *ubuf, *fbuf; +static DEFINE_SPINLOCK(zynqmp_sha_lock); + static int zynqmp_sha_init_tfm(struct crypto_shash *hash) { const char *fallback_driver_name = crypto_shash_alg_name(hash); @@ -60,8 +58,13 @@ static int zynqmp_sha_init_tfm(struct crypto_shash *hash) if (IS_ERR(fallback_tfm)) return PTR_ERR(fallback_tfm); + if (crypto_shash_descsize(hash) < + crypto_shash_descsize(tfm_ctx->fbk_tfm)) { + crypto_free_shash(fallback_tfm); + return -EINVAL; + } + tfm_ctx->fbk_tfm = fallback_tfm; - hash->descsize += crypto_shash_descsize(tfm_ctx->fbk_tfm); return 0; } @@ -70,61 +73,45 @@ static void zynqmp_sha_exit_tfm(struct crypto_shash *hash) { struct zynqmp_sha_tfm_ctx *tfm_ctx = crypto_shash_ctx(hash); - if (tfm_ctx->fbk_tfm) { - crypto_free_shash(tfm_ctx->fbk_tfm); - tfm_ctx->fbk_tfm = NULL; - } - - memzero_explicit(tfm_ctx, sizeof(struct zynqmp_sha_tfm_ctx)); + crypto_free_shash(tfm_ctx->fbk_tfm); } static int zynqmp_sha_init(struct shash_desc *desc) { - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); struct zynqmp_sha_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + struct crypto_shash *fbtfm = tctx->fbk_tfm; + SHASH_DESC_ON_STACK(fbdesc, fbtfm); - dctx->fbk_req.tfm = tctx->fbk_tfm; - return crypto_shash_init(&dctx->fbk_req); + fbdesc->tfm = fbtfm; + return crypto_shash_init(fbdesc) ?: + crypto_shash_export_core(fbdesc, shash_desc_ctx(desc)); } static int zynqmp_sha_update(struct shash_desc *desc, const u8 *data, unsigned int length) { - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); - - return crypto_shash_update(&dctx->fbk_req, data, length); -} - -static int zynqmp_sha_final(struct shash_desc *desc, u8 *out) -{ - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); + struct zynqmp_sha_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + struct crypto_shash *fbtfm = tctx->fbk_tfm; + SHASH_DESC_ON_STACK(fbdesc, fbtfm); - return crypto_shash_final(&dctx->fbk_req, out); + fbdesc->tfm = fbtfm; + return crypto_shash_import_core(fbdesc, shash_desc_ctx(desc)) ?: + crypto_shash_update(fbdesc, data, length) ?: + crypto_shash_export_core(fbdesc, shash_desc_ctx(desc)); } static int zynqmp_sha_finup(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out) { - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); - - return crypto_shash_finup(&dctx->fbk_req, data, length, out); -} - -static int zynqmp_sha_import(struct shash_desc *desc, const void *in) -{ - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); struct zynqmp_sha_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + struct crypto_shash *fbtfm = tctx->fbk_tfm; + SHASH_DESC_ON_STACK(fbdesc, fbtfm); - dctx->fbk_req.tfm = tctx->fbk_tfm; - return crypto_shash_import(&dctx->fbk_req, in); + fbdesc->tfm = fbtfm; + return crypto_shash_import_core(fbdesc, shash_desc_ctx(desc)) ?: + crypto_shash_finup(fbdesc, data, length, out); } -static int zynqmp_sha_export(struct shash_desc *desc, void *out) -{ - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); - - return crypto_shash_export(&dctx->fbk_req, out); -} - -static int zynqmp_sha_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) +static int __zynqmp_sha_digest(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { unsigned int remaining_len = len; int update_size; @@ -159,27 +146,30 @@ static int zynqmp_sha_digest(struct shash_desc *desc, const u8 *data, unsigned i return ret; } +static int zynqmp_sha_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) +{ + scoped_guard(spinlock_bh, &zynqmp_sha_lock) + return __zynqmp_sha_digest(desc, data, len, out); +} + static struct zynqmp_sha_drv_ctx sha3_drv_ctx = { .sha3_384 = { .init = zynqmp_sha_init, .update = zynqmp_sha_update, - .final = zynqmp_sha_final, .finup = zynqmp_sha_finup, .digest = zynqmp_sha_digest, - .export = zynqmp_sha_export, - .import = zynqmp_sha_import, .init_tfm = zynqmp_sha_init_tfm, .exit_tfm = zynqmp_sha_exit_tfm, - .descsize = sizeof(struct zynqmp_sha_desc_ctx), - .statesize = sizeof(struct sha3_state), + .descsize = sizeof(struct sha3_state), .digestsize = SHA3_384_DIGEST_SIZE, .base = { .cra_name = "sha3-384", .cra_driver_name = "zynqmp-sha3-384", .cra_priority = 300, .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA3_384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct zynqmp_sha_tfm_ctx), .cra_module = THIS_MODULE, diff --git a/fs/ubifs/compress.c b/fs/ubifs/compress.c index ea6f06adcd43..059a02691edd 100644 --- a/fs/ubifs/compress.c +++ b/fs/ubifs/compress.c @@ -19,6 +19,11 @@ #include <linux/highmem.h> #include "ubifs.h" +union ubifs_in_ptr { + const void *buf; + struct folio *folio; +}; + /* Fake description object for the "none" compressor */ static struct ubifs_compressor none_compr = { .compr_type = UBIFS_COMPR_NONE, @@ -68,28 +73,61 @@ static struct ubifs_compressor zstd_compr = { /* All UBIFS compressors */ struct ubifs_compressor *ubifs_compressors[UBIFS_COMPR_TYPES_CNT]; -static int ubifs_compress_req(const struct ubifs_info *c, - struct acomp_req *req, - void *out_buf, int *out_len, - const char *compr_name) +static void ubifs_compress_common(int *compr_type, union ubifs_in_ptr in_ptr, + size_t in_offset, int in_len, bool in_folio, + void *out_buf, int *out_len) { - struct crypto_wait wait; - int in_len = req->slen; + struct ubifs_compressor *compr = ubifs_compressors[*compr_type]; int dlen = *out_len; int err; + if (*compr_type == UBIFS_COMPR_NONE) + goto no_compr; + + /* If the input data is small, do not even try to compress it */ + if (in_len < UBIFS_MIN_COMPR_LEN) + goto no_compr; + dlen = min(dlen, in_len - UBIFS_MIN_COMPRESS_DIFF); - crypto_init_wait(&wait); - acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_req_done, &wait); - acomp_request_set_dst_dma(req, out_buf, dlen); - err = crypto_acomp_compress(req); - err = crypto_wait_req(err, &wait); - *out_len = req->dlen; - acomp_request_free(req); + do { + ACOMP_REQUEST_ON_STACK(req, compr->cc); + DECLARE_CRYPTO_WAIT(wait); + + acomp_request_set_callback(req, 0, NULL, NULL); + if (in_folio) + acomp_request_set_src_folio(req, in_ptr.folio, + in_offset, in_len); + else + acomp_request_set_src_dma(req, in_ptr.buf, in_len); + acomp_request_set_dst_dma(req, out_buf, dlen); + err = crypto_acomp_compress(req); + dlen = req->dlen; + if (err != -EAGAIN) + break; + + req = ACOMP_REQUEST_CLONE(req, GFP_NOFS | __GFP_NOWARN); + acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + err = crypto_acomp_compress(req); + err = crypto_wait_req(err, &wait); + dlen = req->dlen; + acomp_request_free(req); + } while (0); + + *out_len = dlen; + if (err) + goto no_compr; - return err; + return; + +no_compr: + if (in_folio) + memcpy_from_folio(out_buf, in_ptr.folio, in_offset, in_len); + else + memcpy(out_buf, in_ptr.buf, in_len); + *out_len = in_len; + *compr_type = UBIFS_COMPR_NONE; } /** @@ -114,32 +152,10 @@ static int ubifs_compress_req(const struct ubifs_info *c, void ubifs_compress(const struct ubifs_info *c, const void *in_buf, int in_len, void *out_buf, int *out_len, int *compr_type) { - int err; - struct ubifs_compressor *compr = ubifs_compressors[*compr_type]; - - if (*compr_type == UBIFS_COMPR_NONE) - goto no_compr; + union ubifs_in_ptr in_ptr = { .buf = in_buf }; - /* If the input data is small, do not even try to compress it */ - if (in_len < UBIFS_MIN_COMPR_LEN) - goto no_compr; - - { - ACOMP_REQUEST_ALLOC(req, compr->cc, GFP_NOFS | __GFP_NOWARN); - - acomp_request_set_src_dma(req, in_buf, in_len); - err = ubifs_compress_req(c, req, out_buf, out_len, compr->name); - } - - if (err) - goto no_compr; - - return; - -no_compr: - memcpy(out_buf, in_buf, in_len); - *out_len = in_len; - *compr_type = UBIFS_COMPR_NONE; + ubifs_compress_common(compr_type, in_ptr, 0, in_len, false, + out_buf, out_len); } /** @@ -166,55 +182,71 @@ void ubifs_compress_folio(const struct ubifs_info *c, struct folio *in_folio, size_t in_offset, int in_len, void *out_buf, int *out_len, int *compr_type) { - int err; - struct ubifs_compressor *compr = ubifs_compressors[*compr_type]; + union ubifs_in_ptr in_ptr = { .folio = in_folio }; - if (*compr_type == UBIFS_COMPR_NONE) - goto no_compr; - - /* If the input data is small, do not even try to compress it */ - if (in_len < UBIFS_MIN_COMPR_LEN) - goto no_compr; + ubifs_compress_common(compr_type, in_ptr, in_offset, in_len, true, + out_buf, out_len); +} - { - ACOMP_REQUEST_ALLOC(req, compr->cc, GFP_NOFS | __GFP_NOWARN); +static int ubifs_decompress_common(const struct ubifs_info *c, + const void *in_buf, int in_len, + void *out_ptr, size_t out_offset, + int *out_len, bool out_folio, + int compr_type) +{ + struct ubifs_compressor *compr; + int dlen = *out_len; + int err; - acomp_request_set_src_folio(req, in_folio, in_offset, in_len); - err = ubifs_compress_req(c, req, out_buf, out_len, compr->name); + if (unlikely(compr_type < 0 || compr_type >= UBIFS_COMPR_TYPES_CNT)) { + ubifs_err(c, "invalid compression type %d", compr_type); + return -EINVAL; } - if (err) - goto no_compr; - - return; + compr = ubifs_compressors[compr_type]; -no_compr: - memcpy_from_folio(out_buf, in_folio, in_offset, in_len); - *out_len = in_len; - *compr_type = UBIFS_COMPR_NONE; -} + if (unlikely(!compr->capi_name)) { + ubifs_err(c, "%s compression is not compiled in", compr->name); + return -EINVAL; + } -static int ubifs_decompress_req(const struct ubifs_info *c, - struct acomp_req *req, - const void *in_buf, int in_len, int *out_len, - const char *compr_name) -{ - struct crypto_wait wait; - int err; + if (compr_type == UBIFS_COMPR_NONE) { + if (out_folio) + memcpy_to_folio(out_ptr, out_offset, in_buf, in_len); + else + memcpy(out_ptr, in_buf, in_len); + *out_len = in_len; + return 0; + } - crypto_init_wait(&wait); - acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_req_done, &wait); - acomp_request_set_src_dma(req, in_buf, in_len); - err = crypto_acomp_decompress(req); - err = crypto_wait_req(err, &wait); - *out_len = req->dlen; + do { + ACOMP_REQUEST_ON_STACK(req, compr->cc); + DECLARE_CRYPTO_WAIT(wait); + acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + acomp_request_set_src_dma(req, in_buf, in_len); + if (out_folio) + acomp_request_set_dst_folio(req, out_ptr, out_offset, + dlen); + else + acomp_request_set_dst_dma(req, out_ptr, dlen); + err = crypto_acomp_decompress(req); + dlen = req->dlen; + if (err != -EAGAIN) + break; + + req = ACOMP_REQUEST_CLONE(req, GFP_NOFS | __GFP_NOWARN); + err = crypto_acomp_decompress(req); + err = crypto_wait_req(err, &wait); + dlen = req->dlen; + acomp_request_free(req); + } while (0); + + *out_len = dlen; if (err) ubifs_err(c, "cannot decompress %d bytes, compressor %s, error %d", - in_len, compr_name, err); - - acomp_request_free(req); + in_len, compr->name, err); return err; } @@ -235,33 +267,8 @@ static int ubifs_decompress_req(const struct ubifs_info *c, int ubifs_decompress(const struct ubifs_info *c, const void *in_buf, int in_len, void *out_buf, int *out_len, int compr_type) { - struct ubifs_compressor *compr; - - if (unlikely(compr_type < 0 || compr_type >= UBIFS_COMPR_TYPES_CNT)) { - ubifs_err(c, "invalid compression type %d", compr_type); - return -EINVAL; - } - - compr = ubifs_compressors[compr_type]; - - if (unlikely(!compr->capi_name)) { - ubifs_err(c, "%s compression is not compiled in", compr->name); - return -EINVAL; - } - - if (compr_type == UBIFS_COMPR_NONE) { - memcpy(out_buf, in_buf, in_len); - *out_len = in_len; - return 0; - } - - { - ACOMP_REQUEST_ALLOC(req, compr->cc, GFP_NOFS | __GFP_NOWARN); - - acomp_request_set_dst_dma(req, out_buf, *out_len); - return ubifs_decompress_req(c, req, in_buf, in_len, out_len, - compr->name); - } + return ubifs_decompress_common(c, in_buf, in_len, out_buf, 0, out_len, + false, compr_type); } /** @@ -283,34 +290,8 @@ int ubifs_decompress_folio(const struct ubifs_info *c, const void *in_buf, int in_len, struct folio *out_folio, size_t out_offset, int *out_len, int compr_type) { - struct ubifs_compressor *compr; - - if (unlikely(compr_type < 0 || compr_type >= UBIFS_COMPR_TYPES_CNT)) { - ubifs_err(c, "invalid compression type %d", compr_type); - return -EINVAL; - } - - compr = ubifs_compressors[compr_type]; - - if (unlikely(!compr->capi_name)) { - ubifs_err(c, "%s compression is not compiled in", compr->name); - return -EINVAL; - } - - if (compr_type == UBIFS_COMPR_NONE) { - memcpy_to_folio(out_folio, out_offset, in_buf, in_len); - *out_len = in_len; - return 0; - } - - { - ACOMP_REQUEST_ALLOC(req, compr->cc, GFP_NOFS | __GFP_NOWARN); - - acomp_request_set_dst_folio(req, out_folio, out_offset, - *out_len); - return ubifs_decompress_req(c, req, in_buf, in_len, out_len, - compr->name); - } + return ubifs_decompress_common(c, in_buf, in_len, out_folio, + out_offset, out_len, true, compr_type); } /** diff --git a/include/asm-generic/simd.h b/include/asm-generic/simd.h index d0343d58a74a..ac29a22eb7cf 100644 --- a/include/asm-generic/simd.h +++ b/include/asm-generic/simd.h @@ -1,6 +1,10 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_GENERIC_SIMD_H +#define _ASM_GENERIC_SIMD_H -#include <linux/hardirq.h> +#include <linux/compiler_attributes.h> +#include <linux/preempt.h> +#include <linux/types.h> /* * may_use_simd - whether it is allowable at this time to issue SIMD @@ -13,3 +17,5 @@ static __must_check inline bool may_use_simd(void) { return !in_interrupt(); } + +#endif /* _ASM_GENERIC_SIMD_H */ diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index c497c73baf13..1b30290d6380 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -32,30 +32,28 @@ /* Set this bit for if virtual address destination cannot be used for DMA. */ #define CRYPTO_ACOMP_REQ_DST_NONDMA 0x00000010 -/* Set this bit if source is a folio. */ -#define CRYPTO_ACOMP_REQ_SRC_FOLIO 0x00000020 - -/* Set this bit if destination is a folio. */ -#define CRYPTO_ACOMP_REQ_DST_FOLIO 0x00000040 +/* Private flags that should not be touched by the user. */ +#define CRYPTO_ACOMP_REQ_PRIVATE \ + (CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | \ + CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA) #define CRYPTO_ACOMP_DST_MAX 131072 #define MAX_SYNC_COMP_REQSIZE 0 -#define ACOMP_REQUEST_ALLOC(name, tfm, gfp) \ +#define ACOMP_REQUEST_ON_STACK(name, tfm) \ char __##name##_req[sizeof(struct acomp_req) + \ MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ struct acomp_req *name = acomp_request_on_stack_init( \ - __##name##_req, (tfm), (gfp), false) + __##name##_req, (tfm)) + +#define ACOMP_REQUEST_CLONE(name, gfp) \ + acomp_request_clone(name, sizeof(__##name##_req), gfp) struct acomp_req; struct folio; struct acomp_req_chain { - struct list_head head; - struct acomp_req *req0; - struct acomp_req *cur; - int (*op)(struct acomp_req *req); crypto_completion_t compl; void *data; struct scatterlist ssg; @@ -68,8 +66,6 @@ struct acomp_req_chain { u8 *dst; struct folio *dfolio; }; - size_t soff; - size_t doff; u32 flags; }; @@ -81,10 +77,6 @@ struct acomp_req_chain { * @dst: Destination scatterlist * @svirt: Source virtual address * @dvirt: Destination virtual address - * @sfolio: Source folio - * @soff: Source folio offset - * @dfolio: Destination folio - * @doff: Destination folio offset * @slen: Size of the input buffer * @dlen: Size of the output buffer and number of bytes produced * @chain: Private API code data, do not use @@ -95,15 +87,11 @@ struct acomp_req { union { struct scatterlist *src; const u8 *svirt; - struct folio *sfolio; }; union { struct scatterlist *dst; u8 *dvirt; - struct folio *dfolio; }; - size_t soff; - size_t doff; unsigned int slen; unsigned int dlen; @@ -130,14 +118,8 @@ struct crypto_acomp { struct crypto_tfm base; }; -struct crypto_acomp_stream { - spinlock_t lock; - void *ctx; -}; - #define COMP_ALG_COMMON { \ struct crypto_alg base; \ - struct crypto_acomp_stream __percpu *stream; \ } struct comp_alg_common COMP_ALG_COMMON; @@ -213,7 +195,7 @@ static inline unsigned int crypto_acomp_reqsize(struct crypto_acomp *tfm) static inline void acomp_request_set_tfm(struct acomp_req *req, struct crypto_acomp *tfm) { - req->base.tfm = crypto_acomp_tfm(tfm); + crypto_request_set_tfm(&req->base, crypto_acomp_tfm(tfm)); } static inline bool acomp_is_async(struct crypto_acomp *tfm) @@ -310,6 +292,11 @@ static inline void *acomp_request_extra(struct acomp_req *req) return (void *)((char *)req + len); } +static inline bool acomp_req_on_stack(struct acomp_req *req) +{ + return crypto_req_on_stack(&req->base); +} + /** * acomp_request_free() -- zeroize and free asynchronous (de)compression * request as well as the output buffer if allocated @@ -319,7 +306,7 @@ static inline void *acomp_request_extra(struct acomp_req *req) */ static inline void acomp_request_free(struct acomp_req *req) { - if (!req || (req->base.flags & CRYPTO_TFM_REQ_ON_STACK)) + if (!req || acomp_req_on_stack(req)) return; kfree_sensitive(req); } @@ -340,17 +327,9 @@ static inline void acomp_request_set_callback(struct acomp_req *req, crypto_completion_t cmpl, void *data) { - u32 keep = CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | - CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA | - CRYPTO_ACOMP_REQ_SRC_FOLIO | CRYPTO_ACOMP_REQ_DST_FOLIO | - CRYPTO_TFM_REQ_ON_STACK; - - req->base.complete = cmpl; - req->base.data = data; - req->base.flags &= keep; - req->base.flags |= flgs & ~keep; - - crypto_reqchain_init(&req->base); + flgs &= ~CRYPTO_ACOMP_REQ_PRIVATE; + flgs |= req->base.flags & CRYPTO_ACOMP_REQ_PRIVATE; + crypto_request_set_callback(&req->base, flgs, cmpl, data); } /** @@ -379,8 +358,6 @@ static inline void acomp_request_set_params(struct acomp_req *req, req->base.flags &= ~(CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | - CRYPTO_ACOMP_REQ_SRC_FOLIO | - CRYPTO_ACOMP_REQ_DST_FOLIO | CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA); } @@ -403,7 +380,6 @@ static inline void acomp_request_set_src_sg(struct acomp_req *req, req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_VIRT; - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; } /** @@ -423,7 +399,6 @@ static inline void acomp_request_set_src_dma(struct acomp_req *req, req->slen = slen; req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; } @@ -444,7 +419,6 @@ static inline void acomp_request_set_src_nondma(struct acomp_req *req, req->svirt = src; req->slen = slen; - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_SRC_NONDMA; req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; } @@ -463,13 +437,9 @@ static inline void acomp_request_set_src_folio(struct acomp_req *req, struct folio *folio, size_t off, unsigned int len) { - req->sfolio = folio; - req->soff = off; - req->slen = len; - - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_VIRT; - req->base.flags |= CRYPTO_ACOMP_REQ_SRC_FOLIO; + sg_init_table(&req->chain.ssg, 1); + sg_set_folio(&req->chain.ssg, folio, len, off); + acomp_request_set_src_sg(req, &req->chain.ssg, len); } /** @@ -490,7 +460,6 @@ static inline void acomp_request_set_dst_sg(struct acomp_req *req, req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_VIRT; - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; } /** @@ -510,7 +479,6 @@ static inline void acomp_request_set_dst_dma(struct acomp_req *req, req->dlen = dlen; req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; } @@ -530,7 +498,6 @@ static inline void acomp_request_set_dst_nondma(struct acomp_req *req, req->dvirt = dst; req->dlen = dlen; - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_DST_NONDMA; req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; } @@ -549,19 +516,9 @@ static inline void acomp_request_set_dst_folio(struct acomp_req *req, struct folio *folio, size_t off, unsigned int len) { - req->dfolio = folio; - req->doff = off; - req->dlen = len; - - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_VIRT; - req->base.flags |= CRYPTO_ACOMP_REQ_DST_FOLIO; -} - -static inline void acomp_request_chain(struct acomp_req *req, - struct acomp_req *head) -{ - crypto_request_chain(&req->base, &head->base); + sg_init_table(&req->chain.dsg, 1); + sg_set_folio(&req->chain.dsg, folio, len, off); + acomp_request_set_dst_sg(req, &req->chain.dsg, len); } /** @@ -587,18 +544,16 @@ int crypto_acomp_compress(struct acomp_req *req); int crypto_acomp_decompress(struct acomp_req *req); static inline struct acomp_req *acomp_request_on_stack_init( - char *buf, struct crypto_acomp *tfm, gfp_t gfp, bool stackonly) + char *buf, struct crypto_acomp *tfm) { - struct acomp_req *req; + struct acomp_req *req = (void *)buf; - if (!stackonly && (req = acomp_request_alloc(tfm, gfp))) - return req; - - req = (void *)buf; - acomp_request_set_tfm(req, tfm->fb); + acomp_request_set_tfm(req, tfm); req->base.flags = CRYPTO_TFM_REQ_ON_STACK; - return req; } +struct acomp_req *acomp_request_clone(struct acomp_req *req, + size_t total, gfp_t gfp); + #endif diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 6e07bbc04089..6999e10ea09e 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -68,16 +68,17 @@ struct crypto_instance { struct crypto_spawn *spawns; }; - struct work_struct free_work; - void *__ctx[] CRYPTO_MINALIGN_ATTR; }; struct crypto_template { struct list_head list; struct hlist_head instances; + struct hlist_head dead; struct module *module; + struct work_struct free_work; + int (*create)(struct crypto_template *tmpl, struct rtattr **tb); char name[CRYPTO_MAX_ALG_NAME]; @@ -266,14 +267,14 @@ static inline u32 crypto_tfm_alg_type(struct crypto_tfm *tfm) return tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK; } -static inline bool crypto_request_chained(struct crypto_async_request *req) +static inline bool crypto_tfm_req_chain(struct crypto_tfm *tfm) { - return !list_empty(&req->list); + return tfm->__crt_alg->cra_flags & CRYPTO_ALG_REQ_CHAIN; } -static inline bool crypto_tfm_req_chain(struct crypto_tfm *tfm) +static inline u32 crypto_request_flags(struct crypto_async_request *req) { - return tfm->__crt_alg->cra_flags & CRYPTO_ALG_REQ_CHAIN; + return req->flags & ~CRYPTO_TFM_REQ_ON_STACK; } #endif /* _CRYPTO_ALGAPI_H */ diff --git a/include/crypto/blake2b.h b/include/crypto/blake2b.h index 0c0176285349..dd7694477e50 100644 --- a/include/crypto/blake2b.h +++ b/include/crypto/blake2b.h @@ -7,10 +7,20 @@ #include <linux/types.h> #include <linux/string.h> +struct blake2b_state { + /* 'h', 't', and 'f' are used in assembly code, so keep them as-is. */ + u64 h[8]; + u64 t[2]; + /* The true state ends here. The rest is temporary storage. */ + u64 f[2]; +}; + enum blake2b_lengths { BLAKE2B_BLOCK_SIZE = 128, BLAKE2B_HASH_SIZE = 64, BLAKE2B_KEY_SIZE = 64, + BLAKE2B_STATE_SIZE = offsetof(struct blake2b_state, f), + BLAKE2B_DESC_SIZE = sizeof(struct blake2b_state), BLAKE2B_160_HASH_SIZE = 20, BLAKE2B_256_HASH_SIZE = 32, @@ -18,16 +28,6 @@ enum blake2b_lengths { BLAKE2B_512_HASH_SIZE = 64, }; -struct blake2b_state { - /* 'h', 't', and 'f' are used in assembly code, so keep them as-is. */ - u64 h[8]; - u64 t[2]; - u64 f[2]; - u8 buf[BLAKE2B_BLOCK_SIZE]; - unsigned int buflen; - unsigned int outlen; -}; - enum blake2b_iv { BLAKE2B_IV0 = 0x6A09E667F3BCC908ULL, BLAKE2B_IV1 = 0xBB67AE8584CAA73BULL, @@ -40,7 +40,7 @@ enum blake2b_iv { }; static inline void __blake2b_init(struct blake2b_state *state, size_t outlen, - const void *key, size_t keylen) + size_t keylen) { state->h[0] = BLAKE2B_IV0 ^ (0x01010000 | keylen << 8 | outlen); state->h[1] = BLAKE2B_IV1; @@ -52,15 +52,6 @@ static inline void __blake2b_init(struct blake2b_state *state, size_t outlen, state->h[7] = BLAKE2B_IV7; state->t[0] = 0; state->t[1] = 0; - state->f[0] = 0; - state->f[1] = 0; - state->buflen = 0; - state->outlen = outlen; - if (keylen) { - memcpy(state->buf, key, keylen); - memset(&state->buf[keylen], 0, BLAKE2B_BLOCK_SIZE - keylen); - state->buflen = BLAKE2B_BLOCK_SIZE; - } } #endif /* _CRYPTO_BLAKE2B_H */ diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h index f8cc073bba41..58129e18cc31 100644 --- a/include/crypto/chacha.h +++ b/include/crypto/chacha.h @@ -99,4 +99,13 @@ static inline void chacha20_crypt(u32 *state, u8 *dst, const u8 *src, chacha_crypt(state, dst, src, bytes, 20); } +#if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CHACHA) +bool chacha_is_arch_optimized(void); +#else +static inline bool chacha_is_arch_optimized(void) +{ + return false; +} +#endif + #endif /* _CRYPTO_CHACHA_H */ diff --git a/include/crypto/ctr.h b/include/crypto/ctr.h index da1ee73e9ce9..06984a26c8cf 100644 --- a/include/crypto/ctr.h +++ b/include/crypto/ctr.h @@ -8,58 +8,8 @@ #ifndef _CRYPTO_CTR_H #define _CRYPTO_CTR_H -#include <crypto/algapi.h> -#include <crypto/internal/skcipher.h> -#include <linux/string.h> -#include <linux/types.h> - #define CTR_RFC3686_NONCE_SIZE 4 #define CTR_RFC3686_IV_SIZE 8 #define CTR_RFC3686_BLOCK_SIZE 16 -static inline int crypto_ctr_encrypt_walk(struct skcipher_request *req, - void (*fn)(struct crypto_skcipher *, - const u8 *, u8 *)) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - int blocksize = crypto_skcipher_chunksize(tfm); - u8 buf[MAX_CIPHER_BLOCKSIZE]; - struct skcipher_walk walk; - int err; - - /* avoid integer division due to variable blocksize parameter */ - if (WARN_ON_ONCE(!is_power_of_2(blocksize))) - return -EINVAL; - - err = skcipher_walk_virt(&walk, req, false); - - while (walk.nbytes > 0) { - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - int nbytes = walk.nbytes; - int tail = 0; - - if (nbytes < walk.total) { - tail = walk.nbytes & (blocksize - 1); - nbytes -= tail; - } - - do { - int bsize = min(nbytes, blocksize); - - fn(tfm, walk.iv, buf); - - crypto_xor_cpy(dst, src, buf, bsize); - crypto_inc(walk.iv, blocksize); - - dst += bsize; - src += bsize; - nbytes -= bsize; - } while (nbytes > 0); - - err = skcipher_walk_done(&walk, tail); - } - return err; -} - #endif /* _CRYPTO_CTR_H */ diff --git a/include/crypto/ghash.h b/include/crypto/ghash.h index f832c9f2aca3..043d938e9a2c 100644 --- a/include/crypto/ghash.h +++ b/include/crypto/ghash.h @@ -7,18 +7,18 @@ #define __CRYPTO_GHASH_H__ #include <linux/types.h> -#include <crypto/gf128mul.h> #define GHASH_BLOCK_SIZE 16 #define GHASH_DIGEST_SIZE 16 +struct gf128mul_4k; + struct ghash_ctx { struct gf128mul_4k *gf128; }; struct ghash_desc_ctx { u8 buffer[GHASH_BLOCK_SIZE]; - u32 bytes; }; #endif diff --git a/include/crypto/hash.h b/include/crypto/hash.h index a67988316d06..5f87d1040a7c 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -16,6 +16,9 @@ /* Set this bit for virtual address instead of SG list. */ #define CRYPTO_AHASH_REQ_VIRT 0x00000001 +#define CRYPTO_AHASH_REQ_PRIVATE \ + CRYPTO_AHASH_REQ_VIRT + struct crypto_ahash; /** @@ -82,6 +85,8 @@ struct ahash_request { * transformation object. Data processing can happen synchronously * [SHASH] or asynchronously [AHASH] at this point. Driver must not use * req->result. + * For block-only algorithms, @update must return the number + * of bytes to store in the API partial block buffer. * @final: **[mandatory]** Retrieve result from the driver. This function finalizes the * transformation and retrieves the resulting hash from the driver and * pushes it back to upper layers. No data processing happens at this @@ -136,7 +141,6 @@ struct ahash_request { * This is a counterpart to @init_tfm, used to remove * various changes set in @init_tfm. * @clone_tfm: Copy transform into new object, may allocate memory. - * @reqsize: Size of the request context. * @halg: see struct hash_alg_common */ struct ahash_alg { @@ -153,8 +157,6 @@ struct ahash_alg { void (*exit_tfm)(struct crypto_ahash *tfm); int (*clone_tfm)(struct crypto_ahash *dst, struct crypto_ahash *src); - unsigned int reqsize; - struct hash_alg_common halg; }; @@ -166,16 +168,26 @@ struct shash_desc { #define HASH_MAX_DIGESTSIZE 64 /* - * Worst case is hmac(sha3-224-generic). Its context is a nested 'shash_desc' - * containing a 'struct sha3_state'. + * Worst case is hmac(sha-224-s390). Its context is a nested 'shash_desc' + * containing a 'struct s390_sha_ctx'. */ #define HASH_MAX_DESCSIZE (sizeof(struct shash_desc) + 360) +#define MAX_SYNC_HASH_REQSIZE HASH_MAX_DESCSIZE #define SHASH_DESC_ON_STACK(shash, ctx) \ char __##shash##_desc[sizeof(struct shash_desc) + HASH_MAX_DESCSIZE] \ __aligned(__alignof__(struct shash_desc)); \ struct shash_desc *shash = (struct shash_desc *)__##shash##_desc +#define HASH_REQUEST_ON_STACK(name, _tfm) \ + char __##name##_req[sizeof(struct ahash_request) + \ + MAX_SYNC_HASH_REQSIZE] CRYPTO_MINALIGN_ATTR; \ + struct ahash_request *name = \ + ahash_request_on_stack_init(__##name##_req, (_tfm)) + +#define HASH_REQUEST_CLONE(name, gfp) \ + hash_request_clone(name, sizeof(__##name##_req), gfp) + /** * struct shash_alg - synchronous message digest definition * @init: see struct ahash_alg @@ -234,11 +246,11 @@ struct crypto_ahash { bool using_shash; /* Underlying algorithm is shash, not ahash */ unsigned int statesize; unsigned int reqsize; + struct crypto_ahash *fb; struct crypto_tfm base; }; struct crypto_shash { - unsigned int descsize; struct crypto_tfm base; }; @@ -252,6 +264,11 @@ struct crypto_shash { * CRYPTO_ALG_TYPE_SKCIPHER API applies here as well. */ +static inline bool ahash_req_on_stack(struct ahash_request *req) +{ + return crypto_req_on_stack(&req->base); +} + static inline struct crypto_ahash *__crypto_ahash_cast(struct crypto_tfm *tfm) { return container_of(tfm, struct crypto_ahash, base); @@ -548,7 +565,7 @@ int crypto_ahash_update(struct ahash_request *req); static inline void ahash_request_set_tfm(struct ahash_request *req, struct crypto_ahash *tfm) { - req->base.tfm = crypto_ahash_tfm(tfm); + crypto_request_set_tfm(&req->base, crypto_ahash_tfm(tfm)); } /** @@ -582,9 +599,12 @@ static inline struct ahash_request *ahash_request_alloc_noprof( * ahash_request_free() - zeroize and free the request data structure * @req: request data structure cipher handle to be freed */ -static inline void ahash_request_free(struct ahash_request *req) +void ahash_request_free(struct ahash_request *req); + +static inline void ahash_request_zero(struct ahash_request *req) { - kfree_sensitive(req); + memzero_explicit(req, sizeof(*req) + + crypto_ahash_reqsize(crypto_ahash_reqtfm(req))); } static inline struct ahash_request *ahash_request_cast( @@ -623,14 +643,9 @@ static inline void ahash_request_set_callback(struct ahash_request *req, crypto_completion_t compl, void *data) { - u32 keep = CRYPTO_AHASH_REQ_VIRT; - - req->base.complete = compl; - req->base.data = data; - flags &= ~keep; - req->base.flags &= keep; - req->base.flags |= flags; - crypto_reqchain_init(&req->base); + flags &= ~CRYPTO_AHASH_REQ_PRIVATE; + flags |= req->base.flags & CRYPTO_AHASH_REQ_PRIVATE; + crypto_request_set_callback(&req->base, flags, compl, data); } /** @@ -679,12 +694,6 @@ static inline void ahash_request_set_virt(struct ahash_request *req, req->base.flags |= CRYPTO_AHASH_REQ_VIRT; } -static inline void ahash_request_chain(struct ahash_request *req, - struct ahash_request *head) -{ - crypto_request_chain(&req->base, &head->base); -} - /** * DOC: Synchronous Message Digest API * @@ -820,7 +829,7 @@ static inline void crypto_shash_clear_flags(struct crypto_shash *tfm, u32 flags) */ static inline unsigned int crypto_shash_descsize(struct crypto_shash *tfm) { - return tfm->descsize; + return crypto_shash_alg(tfm)->descsize; } static inline void *shash_desc_ctx(struct shash_desc *desc) @@ -838,7 +847,7 @@ static inline void *shash_desc_ctx(struct shash_desc *desc) * cipher handle must point to a keyed message digest cipher in order for this * function to succeed. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the setting of the key was successful; < 0 if an error occurred */ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key, @@ -855,7 +864,7 @@ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key, * crypto_shash_update and crypto_shash_final. The parameters have the same * meaning as discussed for those separate three functions. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the message digest creation was successful; < 0 if an error * occurred */ @@ -875,12 +884,15 @@ int crypto_shash_digest(struct shash_desc *desc, const u8 *data, * directly, and it allocates a hash descriptor on the stack internally. * Note that this stack allocation may be fairly large. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 on success; < 0 if an error occurred. */ int crypto_shash_tfm_digest(struct crypto_shash *tfm, const u8 *data, unsigned int len, u8 *out); +int crypto_hash_digest(struct crypto_ahash *tfm, const u8 *data, + unsigned int len, u8 *out); + /** * crypto_shash_export() - extract operational state for message digest * @desc: reference to the operational state handle whose state is exported @@ -890,12 +902,24 @@ int crypto_shash_tfm_digest(struct crypto_shash *tfm, const u8 *data, * caller-allocated output buffer out which must have sufficient size (e.g. by * calling crypto_shash_descsize). * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the export creation was successful; < 0 if an error occurred */ int crypto_shash_export(struct shash_desc *desc, void *out); /** + * crypto_shash_export_core() - extract core state for message digest + * @desc: reference to the operational state handle whose state is exported + * @out: output buffer of sufficient size that can hold the hash state + * + * Export the hash state without the partial block buffer. + * + * Context: Softirq or process context. + * Return: 0 if the export creation was successful; < 0 if an error occurred + */ +int crypto_shash_export_core(struct shash_desc *desc, void *out); + +/** * crypto_shash_import() - import operational state * @desc: reference to the operational state handle the state imported into * @in: buffer holding the state @@ -904,12 +928,24 @@ int crypto_shash_export(struct shash_desc *desc, void *out); * the input buffer. That buffer should have been generated with the * crypto_ahash_export function. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the import was successful; < 0 if an error occurred */ int crypto_shash_import(struct shash_desc *desc, const void *in); /** + * crypto_shash_import_core() - import core state + * @desc: reference to the operational state handle the state imported into + * @in: buffer holding the state + * + * Import the hash state without the partial block buffer. + * + * Context: Softirq or process context. + * Return: 0 if the import was successful; < 0 if an error occurred + */ +int crypto_shash_import_core(struct shash_desc *desc, const void *in); + +/** * crypto_shash_init() - (re)initialize message digest * @desc: operational state handle that is already filled * @@ -917,19 +953,29 @@ int crypto_shash_import(struct shash_desc *desc, const void *in); * operational state handle. Any potentially existing state created by * previous operations is discarded. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the message digest initialization was successful; < 0 if an * error occurred */ -static inline int crypto_shash_init(struct shash_desc *desc) -{ - struct crypto_shash *tfm = desc->tfm; +int crypto_shash_init(struct shash_desc *desc); - if (crypto_shash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) - return -ENOKEY; - - return crypto_shash_alg(tfm)->init(desc); -} +/** + * crypto_shash_finup() - calculate message digest of buffer + * @desc: see crypto_shash_final() + * @data: see crypto_shash_update() + * @len: see crypto_shash_update() + * @out: see crypto_shash_final() + * + * This function is a "short-hand" for the function calls of + * crypto_shash_update and crypto_shash_final. The parameters have the same + * meaning as discussed for those separate functions. + * + * Context: Softirq or process context. + * Return: 0 if the message digest creation was successful; < 0 if an error + * occurred + */ +int crypto_shash_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out); /** * crypto_shash_update() - add data to message digest for processing @@ -939,12 +985,15 @@ static inline int crypto_shash_init(struct shash_desc *desc) * * Updates the message digest state of the operational state handle. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the message digest update was successful; < 0 if an error * occurred */ -int crypto_shash_update(struct shash_desc *desc, const u8 *data, - unsigned int len); +static inline int crypto_shash_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + return crypto_shash_finup(desc, data, len, NULL); +} /** * crypto_shash_final() - calculate message digest @@ -956,29 +1005,14 @@ int crypto_shash_update(struct shash_desc *desc, const u8 *data, * into the output buffer. The caller must ensure that the output buffer is * large enough by using crypto_shash_digestsize. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the message digest creation was successful; < 0 if an error * occurred */ -int crypto_shash_final(struct shash_desc *desc, u8 *out); - -/** - * crypto_shash_finup() - calculate message digest of buffer - * @desc: see crypto_shash_final() - * @data: see crypto_shash_update() - * @len: see crypto_shash_update() - * @out: see crypto_shash_final() - * - * This function is a "short-hand" for the function calls of - * crypto_shash_update and crypto_shash_final. The parameters have the same - * meaning as discussed for those separate functions. - * - * Context: Any context. - * Return: 0 if the message digest creation was successful; < 0 if an error - * occurred - */ -int crypto_shash_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out); +static inline int crypto_shash_final(struct shash_desc *desc, u8 *out) +{ + return crypto_shash_finup(desc, NULL, 0, out); +} static inline void shash_desc_zero(struct shash_desc *desc) { @@ -986,14 +1020,22 @@ static inline void shash_desc_zero(struct shash_desc *desc) sizeof(*desc) + crypto_shash_descsize(desc->tfm)); } -static inline int ahash_request_err(struct ahash_request *req) +static inline bool ahash_is_async(struct crypto_ahash *tfm) { - return req->base.err; + return crypto_tfm_is_async(&tfm->base); } -static inline bool ahash_is_async(struct crypto_ahash *tfm) +static inline struct ahash_request *ahash_request_on_stack_init( + char *buf, struct crypto_ahash *tfm) { - return crypto_tfm_is_async(&tfm->base); + struct ahash_request *req = (void *)buf; + + ahash_request_set_tfm(req, tfm); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + return req; } +struct ahash_request *ahash_request_clone(struct ahash_request *req, + size_t total, gfp_t gfp); + #endif /* _CRYPTO_HASH_H */ diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index aaf59f3236fa..7eda32619024 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -11,12 +11,17 @@ #include <crypto/acompress.h> #include <crypto/algapi.h> +#include <crypto/scatterwalk.h> +#include <linux/compiler_types.h> +#include <linux/cpumask_types.h> +#include <linux/spinlock.h> +#include <linux/workqueue_types.h> -#define ACOMP_REQUEST_ON_STACK(name, tfm) \ +#define ACOMP_FBREQ_ON_STACK(name, req) \ char __##name##_req[sizeof(struct acomp_req) + \ MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ - struct acomp_req *name = acomp_request_on_stack_init( \ - __##name##_req, (tfm), 0, true) + struct acomp_req *name = acomp_fbreq_on_stack_init( \ + __##name##_req, (req)) /** * struct acomp_alg - asynchronous compression algorithm @@ -35,9 +40,7 @@ * counterpart to @init, used to remove various changes set in * @init. * - * @reqsize: Context size for (de)compression requests * @base: Common crypto API algorithm data structure - * @stream: Per-cpu memory for algorithm * @calg: Cmonn algorithm data structure shared with scomp */ struct acomp_alg { @@ -46,14 +49,61 @@ struct acomp_alg { int (*init)(struct crypto_acomp *tfm); void (*exit)(struct crypto_acomp *tfm); - unsigned int reqsize; - union { struct COMP_ALG_COMMON; struct comp_alg_common calg; }; }; +struct crypto_acomp_stream { + spinlock_t lock; + void *ctx; +}; + +struct crypto_acomp_streams { + /* These must come first because of struct scomp_alg. */ + void *(*alloc_ctx)(void); + union { + void (*free_ctx)(void *); + void (*cfree_ctx)(const void *); + }; + + struct crypto_acomp_stream __percpu *streams; + struct work_struct stream_work; + cpumask_t stream_want; +}; + +struct acomp_walk { + union { + /* Virtual address of the source. */ + struct { + struct { + const void *const addr; + } virt; + } src; + + /* Private field for the API, do not use. */ + struct scatter_walk in; + }; + + union { + /* Virtual address of the destination. */ + struct { + struct { + void *const addr; + } virt; + } dst; + + /* Private field for the API, do not use. */ + struct scatter_walk out; + }; + + unsigned int slen; + unsigned int dlen; + + int flags; +}; + /* * Transform internal helpers. */ @@ -98,17 +148,10 @@ void crypto_unregister_acomp(struct acomp_alg *alg); int crypto_register_acomps(struct acomp_alg *algs, int count); void crypto_unregister_acomps(struct acomp_alg *algs, int count); -static inline bool acomp_request_chained(struct acomp_req *req) -{ - return crypto_request_chained(&req->base); -} - static inline bool acomp_request_issg(struct acomp_req *req) { return !(req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | - CRYPTO_ACOMP_REQ_DST_VIRT | - CRYPTO_ACOMP_REQ_SRC_FOLIO | - CRYPTO_ACOMP_REQ_DST_FOLIO)); + CRYPTO_ACOMP_REQ_DST_VIRT)); } static inline bool acomp_request_src_isvirt(struct acomp_req *req) @@ -143,19 +186,57 @@ static inline bool acomp_request_isnondma(struct acomp_req *req) CRYPTO_ACOMP_REQ_DST_NONDMA); } -static inline bool acomp_request_src_isfolio(struct acomp_req *req) +static inline bool crypto_acomp_req_chain(struct crypto_acomp *tfm) { - return req->base.flags & CRYPTO_ACOMP_REQ_SRC_FOLIO; + return crypto_tfm_req_chain(&tfm->base); } -static inline bool acomp_request_dst_isfolio(struct acomp_req *req) +void crypto_acomp_free_streams(struct crypto_acomp_streams *s); +int crypto_acomp_alloc_streams(struct crypto_acomp_streams *s); + +struct crypto_acomp_stream *crypto_acomp_lock_stream_bh( + struct crypto_acomp_streams *s) __acquires(stream); + +static inline void crypto_acomp_unlock_stream_bh( + struct crypto_acomp_stream *stream) __releases(stream) { - return req->base.flags & CRYPTO_ACOMP_REQ_DST_FOLIO; + spin_unlock_bh(&stream->lock); } -static inline bool crypto_acomp_req_chain(struct crypto_acomp *tfm) +void acomp_walk_done_src(struct acomp_walk *walk, int used); +void acomp_walk_done_dst(struct acomp_walk *walk, int used); +int acomp_walk_next_src(struct acomp_walk *walk); +int acomp_walk_next_dst(struct acomp_walk *walk); +int acomp_walk_virt(struct acomp_walk *__restrict walk, + struct acomp_req *__restrict req, bool atomic); + +static inline bool acomp_walk_more_src(const struct acomp_walk *walk, int cur) { - return crypto_tfm_req_chain(&tfm->base); + return walk->slen != cur; +} + +static inline u32 acomp_request_flags(struct acomp_req *req) +{ + return crypto_request_flags(&req->base) & ~CRYPTO_ACOMP_REQ_PRIVATE; +} + +static inline struct acomp_req *acomp_fbreq_on_stack_init( + char *buf, struct acomp_req *old) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(old); + struct acomp_req *req = (void *)buf; + + acomp_request_set_tfm(req, tfm->fb); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + acomp_request_set_callback(req, acomp_request_flags(old), NULL, NULL); + req->base.flags &= ~CRYPTO_ACOMP_REQ_PRIVATE; + req->base.flags |= old->base.flags & CRYPTO_ACOMP_REQ_PRIVATE; + req->src = old->src; + req->dst = old->dst; + req->slen = old->slen; + req->dlen = old->dlen; + + return req; } #endif diff --git a/include/crypto/internal/blake2b.h b/include/crypto/internal/blake2b.h index 982fe5e8471c..3e09e2485306 100644 --- a/include/crypto/internal/blake2b.h +++ b/include/crypto/internal/blake2b.h @@ -7,65 +7,36 @@ #ifndef _CRYPTO_INTERNAL_BLAKE2B_H #define _CRYPTO_INTERNAL_BLAKE2B_H +#include <asm/byteorder.h> #include <crypto/blake2b.h> #include <crypto/internal/hash.h> +#include <linux/array_size.h> +#include <linux/compiler.h> +#include <linux/build_bug.h> +#include <linux/errno.h> +#include <linux/math.h> #include <linux/string.h> - -void blake2b_compress_generic(struct blake2b_state *state, - const u8 *block, size_t nblocks, u32 inc); +#include <linux/types.h> static inline void blake2b_set_lastblock(struct blake2b_state *state) { state->f[0] = -1; + state->f[1] = 0; } -typedef void (*blake2b_compress_t)(struct blake2b_state *state, - const u8 *block, size_t nblocks, u32 inc); - -static inline void __blake2b_update(struct blake2b_state *state, - const u8 *in, size_t inlen, - blake2b_compress_t compress) +static inline void blake2b_set_nonlast(struct blake2b_state *state) { - const size_t fill = BLAKE2B_BLOCK_SIZE - state->buflen; - - if (unlikely(!inlen)) - return; - if (inlen > fill) { - memcpy(state->buf + state->buflen, in, fill); - (*compress)(state, state->buf, 1, BLAKE2B_BLOCK_SIZE); - state->buflen = 0; - in += fill; - inlen -= fill; - } - if (inlen > BLAKE2B_BLOCK_SIZE) { - const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2B_BLOCK_SIZE); - /* Hash one less (full) block than strictly possible */ - (*compress)(state, in, nblocks - 1, BLAKE2B_BLOCK_SIZE); - in += BLAKE2B_BLOCK_SIZE * (nblocks - 1); - inlen -= BLAKE2B_BLOCK_SIZE * (nblocks - 1); - } - memcpy(state->buf + state->buflen, in, inlen); - state->buflen += inlen; + state->f[0] = 0; + state->f[1] = 0; } -static inline void __blake2b_final(struct blake2b_state *state, u8 *out, - blake2b_compress_t compress) -{ - int i; - - blake2b_set_lastblock(state); - memset(state->buf + state->buflen, 0, - BLAKE2B_BLOCK_SIZE - state->buflen); /* Padding */ - (*compress)(state, state->buf, 1, state->buflen); - for (i = 0; i < ARRAY_SIZE(state->h); i++) - __cpu_to_le64s(&state->h[i]); - memcpy(out, state->h, state->outlen); -} +typedef void (*blake2b_compress_t)(struct blake2b_state *state, + const u8 *block, size_t nblocks, u32 inc); /* Helper functions for shash implementations of BLAKE2b */ struct blake2b_tfm_ctx { - u8 key[BLAKE2B_KEY_SIZE]; + u8 key[BLAKE2B_BLOCK_SIZE]; unsigned int keylen; }; @@ -74,10 +45,13 @@ static inline int crypto_blake2b_setkey(struct crypto_shash *tfm, { struct blake2b_tfm_ctx *tctx = crypto_shash_ctx(tfm); - if (keylen == 0 || keylen > BLAKE2B_KEY_SIZE) + if (keylen > BLAKE2B_KEY_SIZE) return -EINVAL; + BUILD_BUG_ON(BLAKE2B_KEY_SIZE > BLAKE2B_BLOCK_SIZE); + memcpy(tctx->key, key, keylen); + memset(tctx->key + keylen, 0, BLAKE2B_BLOCK_SIZE - keylen); tctx->keylen = keylen; return 0; @@ -89,26 +63,38 @@ static inline int crypto_blake2b_init(struct shash_desc *desc) struct blake2b_state *state = shash_desc_ctx(desc); unsigned int outlen = crypto_shash_digestsize(desc->tfm); - __blake2b_init(state, outlen, tctx->key, tctx->keylen); - return 0; + __blake2b_init(state, outlen, tctx->keylen); + return tctx->keylen ? + crypto_shash_update(desc, tctx->key, BLAKE2B_BLOCK_SIZE) : 0; } -static inline int crypto_blake2b_update(struct shash_desc *desc, - const u8 *in, unsigned int inlen, - blake2b_compress_t compress) +static inline int crypto_blake2b_update_bo(struct shash_desc *desc, + const u8 *in, unsigned int inlen, + blake2b_compress_t compress) { struct blake2b_state *state = shash_desc_ctx(desc); - __blake2b_update(state, in, inlen, compress); - return 0; + blake2b_set_nonlast(state); + compress(state, in, inlen / BLAKE2B_BLOCK_SIZE, BLAKE2B_BLOCK_SIZE); + return inlen - round_down(inlen, BLAKE2B_BLOCK_SIZE); } -static inline int crypto_blake2b_final(struct shash_desc *desc, u8 *out, +static inline int crypto_blake2b_finup(struct shash_desc *desc, const u8 *in, + unsigned int inlen, u8 *out, blake2b_compress_t compress) { struct blake2b_state *state = shash_desc_ctx(desc); + u8 buf[BLAKE2B_BLOCK_SIZE]; + int i; - __blake2b_final(state, out, compress); + memcpy(buf, in, inlen); + memset(buf + inlen, 0, BLAKE2B_BLOCK_SIZE - inlen); + blake2b_set_lastblock(state); + compress(state, buf, 1, inlen); + for (i = 0; i < ARRAY_SIZE(state->h); i++) + __cpu_to_le64s(&state->h[i]); + memcpy(out, state->h, crypto_shash_digestsize(desc->tfm)); + memzero_explicit(buf, sizeof(buf)); return 0; } diff --git a/include/crypto/internal/chacha.h b/include/crypto/internal/chacha.h deleted file mode 100644 index b085dc1ac151..000000000000 --- a/include/crypto/internal/chacha.h +++ /dev/null @@ -1,43 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#ifndef _CRYPTO_INTERNAL_CHACHA_H -#define _CRYPTO_INTERNAL_CHACHA_H - -#include <crypto/chacha.h> -#include <crypto/internal/skcipher.h> -#include <linux/crypto.h> - -struct chacha_ctx { - u32 key[8]; - int nrounds; -}; - -static inline int chacha_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keysize, int nrounds) -{ - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - int i; - - if (keysize != CHACHA_KEY_SIZE) - return -EINVAL; - - for (i = 0; i < ARRAY_SIZE(ctx->key); i++) - ctx->key[i] = get_unaligned_le32(key + i * sizeof(u32)); - - ctx->nrounds = nrounds; - return 0; -} - -static inline int chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keysize) -{ - return chacha_setkey(tfm, key, keysize, 20); -} - -static inline int chacha12_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keysize) -{ - return chacha_setkey(tfm, key, keysize, 12); -} - -#endif /* _CRYPTO_CHACHA_H */ diff --git a/include/crypto/internal/engine.h b/include/crypto/internal/engine.h index fbf4be56cf12..b6a4ea2240fc 100644 --- a/include/crypto/internal/engine.h +++ b/include/crypto/internal/engine.h @@ -27,10 +27,10 @@ struct device; * @retry_support: indication that the hardware allows re-execution * of a failed backlog request * crypto-engine, in head position to keep order + * @rt: whether this queue is set to run as a realtime task * @list: link with the global crypto engine list * @queue_lock: spinlock to synchronise access to request queue * @queue: the crypto queue of the engine - * @rt: whether this queue is set to run as a realtime task * @prepare_crypt_hardware: a request will soon arrive from the queue * so the subsystem requests the driver to prepare the hardware * by issuing this call @@ -51,14 +51,13 @@ struct crypto_engine { bool running; bool retry_support; + bool rt; struct list_head list; spinlock_t queue_lock; struct crypto_queue queue; struct device *dev; - bool rt; - int (*prepare_crypt_hardware)(struct crypto_engine *engine); int (*unprepare_crypt_hardware)(struct crypto_engine *engine); int (*do_batch_requests)(struct crypto_engine *engine); diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 052ac7924af3..1e80dd084a23 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -11,6 +11,21 @@ #include <crypto/algapi.h> #include <crypto/hash.h> +/* Set this bit to handle partial blocks in the API. */ +#define CRYPTO_AHASH_ALG_BLOCK_ONLY 0x01000000 + +/* Set this bit if final requires at least one byte. */ +#define CRYPTO_AHASH_ALG_FINAL_NONZERO 0x02000000 + +/* Set this bit if finup can deal with multiple blocks. */ +#define CRYPTO_AHASH_ALG_FINUP_MAX 0x04000000 + +#define HASH_FBREQ_ON_STACK(name, req) \ + char __##name##_req[sizeof(struct ahash_request) + \ + MAX_SYNC_HASH_REQSIZE] CRYPTO_MINALIGN_ATTR; \ + struct ahash_request *name = ahash_fbreq_on_stack_init( \ + __##name##_req, (req)) + struct ahash_request; struct ahash_instance { @@ -187,7 +202,7 @@ static inline void ahash_request_complete(struct ahash_request *req, int err) static inline u32 ahash_request_flags(struct ahash_request *req) { - return req->base.flags; + return crypto_request_flags(&req->base) & ~CRYPTO_AHASH_REQ_PRIVATE; } static inline struct crypto_ahash *crypto_spawn_ahash( @@ -247,11 +262,6 @@ static inline struct crypto_shash *__crypto_shash_cast(struct crypto_tfm *tfm) return container_of(tfm, struct crypto_shash, base); } -static inline bool ahash_request_chained(struct ahash_request *req) -{ - return false; -} - static inline bool ahash_request_isvirt(struct ahash_request *req) { return req->base.flags & CRYPTO_AHASH_REQ_VIRT; @@ -262,5 +272,29 @@ static inline bool crypto_ahash_req_chain(struct crypto_ahash *tfm) return crypto_tfm_req_chain(&tfm->base); } +static inline struct ahash_request *ahash_fbreq_on_stack_init( + char *buf, struct ahash_request *old) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(old); + struct ahash_request *req = (void *)buf; + + ahash_request_set_tfm(req, tfm->fb); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + ahash_request_set_callback(req, ahash_request_flags(old), NULL, NULL); + req->base.flags &= ~CRYPTO_AHASH_REQ_PRIVATE; + req->base.flags |= old->base.flags & CRYPTO_AHASH_REQ_PRIVATE; + req->src = old->src; + req->result = old->result; + req->nbytes = old->nbytes; + + return req; +} + +/* Return the state size without partial block for block-only algorithms. */ +static inline unsigned int crypto_shash_coresize(struct crypto_shash *tfm) +{ + return crypto_shash_statesize(tfm) - crypto_shash_blocksize(tfm) - 1; +} + #endif /* _CRYPTO_INTERNAL_HASH_H */ diff --git a/include/crypto/internal/scompress.h b/include/crypto/internal/scompress.h index f25aa2ea3b48..533d6c16a491 100644 --- a/include/crypto/internal/scompress.h +++ b/include/crypto/internal/scompress.h @@ -9,10 +9,7 @@ #ifndef _CRYPTO_SCOMP_INT_H #define _CRYPTO_SCOMP_INT_H -#include <crypto/acompress.h> -#include <crypto/algapi.h> - -struct acomp_req; +#include <crypto/internal/acompress.h> struct crypto_scomp { struct crypto_tfm base; @@ -26,12 +23,10 @@ struct crypto_scomp { * @compress: Function performs a compress operation * @decompress: Function performs a de-compress operation * @base: Common crypto API algorithm data structure - * @stream: Per-cpu memory for algorithm + * @streams: Per-cpu memory for algorithm * @calg: Cmonn algorithm data structure shared with acomp */ struct scomp_alg { - void *(*alloc_ctx)(void); - void (*free_ctx)(void *ctx); int (*compress)(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx); @@ -40,6 +35,14 @@ struct scomp_alg { void *ctx); union { + struct { + void *(*alloc_ctx)(void); + void (*free_ctx)(void *ctx); + }; + struct crypto_acomp_streams streams; + }; + + union { struct COMP_ALG_COMMON; struct comp_alg_common calg; }; diff --git a/include/crypto/internal/simd.h b/include/crypto/internal/simd.h index be97b97a75dd..f56049bd1660 100644 --- a/include/crypto/internal/simd.h +++ b/include/crypto/internal/simd.h @@ -6,6 +6,7 @@ #ifndef _CRYPTO_INTERNAL_SIMD_H #define _CRYPTO_INTERNAL_SIMD_H +#include <asm/simd.h> #include <linux/percpu.h> #include <linux/types.h> @@ -46,9 +47,6 @@ void simd_unregister_aeads(struct aead_alg *algs, int count, * self-tests, in order to test the no-SIMD fallback code. This override is * currently limited to configurations where the extra self-tests are enabled, * because it might be a bit too invasive to be part of the regular self-tests. - * - * This is a macro so that <asm/simd.h>, which some architectures don't have, - * doesn't have to be included directly here. */ #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS DECLARE_PER_CPU(bool, crypto_simd_disabled_for_test); diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h index a958ab0636ad..0cad8e7364c8 100644 --- a/include/crypto/internal/skcipher.h +++ b/include/crypto/internal/skcipher.h @@ -67,8 +67,6 @@ struct skcipher_walk { struct scatter_walk in; }; - unsigned int nbytes; - union { /* Virtual address of the destination. */ struct { @@ -81,6 +79,7 @@ struct skcipher_walk { struct scatter_walk out; }; + unsigned int nbytes; unsigned int total; u8 *page; diff --git a/include/crypto/md5.h b/include/crypto/md5.h index cf9e9dec3d21..198b5d69b92f 100644 --- a/include/crypto/md5.h +++ b/include/crypto/md5.h @@ -8,6 +8,7 @@ #define MD5_HMAC_BLOCK_SIZE 64 #define MD5_BLOCK_WORDS 16 #define MD5_HASH_WORDS 4 +#define MD5_STATE_SIZE 24 #define MD5_H0 0x67452301UL #define MD5_H1 0xefcdab89UL @@ -18,8 +19,8 @@ extern const u8 md5_zero_message_hash[MD5_DIGEST_SIZE]; struct md5_state { u32 hash[MD5_HASH_WORDS]; - u32 block[MD5_BLOCK_WORDS]; u64 byte_count; + u32 block[MD5_BLOCK_WORDS]; }; #endif diff --git a/include/crypto/poly1305.h b/include/crypto/poly1305.h index 090692ec3bc7..6e21ec2d1dc2 100644 --- a/include/crypto/poly1305.h +++ b/include/crypto/poly1305.h @@ -43,10 +43,6 @@ struct poly1305_desc_ctx { u8 buf[POLY1305_BLOCK_SIZE]; /* bytes used in partial buffer */ unsigned int buflen; - /* how many keys have been set in r[] */ - unsigned short rset; - /* whether s[] has been set */ - bool sset; /* finalize key */ u32 s[4]; /* accumulator */ @@ -96,4 +92,13 @@ static inline void poly1305_final(struct poly1305_desc_ctx *desc, u8 *digest) poly1305_final_generic(desc, digest); } +#if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305) +bool poly1305_is_arch_optimized(void); +#else +static inline bool poly1305_is_arch_optimized(void) +{ + return false; +} +#endif + #endif diff --git a/include/crypto/sha1.h b/include/crypto/sha1.h index 044ecea60ac8..f48230b1413c 100644 --- a/include/crypto/sha1.h +++ b/include/crypto/sha1.h @@ -10,6 +10,7 @@ #define SHA1_DIGEST_SIZE 20 #define SHA1_BLOCK_SIZE 64 +#define SHA1_STATE_SIZE offsetof(struct sha1_state, buffer) #define SHA1_H0 0x67452301UL #define SHA1_H1 0xefcdab89UL @@ -25,14 +26,6 @@ struct sha1_state { u8 buffer[SHA1_BLOCK_SIZE]; }; -struct shash_desc; - -extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -extern int crypto_sha1_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); - /* * An implementation of SHA-1's compression function. Don't use in new code! * You shouldn't be using SHA-1, and even if you *have* to use SHA-1, this isn't diff --git a/include/crypto/sha1_base.h b/include/crypto/sha1_base.h index 0c342ed0d038..62701d136c79 100644 --- a/include/crypto/sha1_base.h +++ b/include/crypto/sha1_base.h @@ -10,10 +10,9 @@ #include <crypto/internal/hash.h> #include <crypto/sha1.h> -#include <linux/crypto.h> -#include <linux/module.h> +#include <linux/math.h> #include <linux/string.h> - +#include <linux/types.h> #include <linux/unaligned.h> typedef void (sha1_block_fn)(struct sha1_state *sst, u8 const *src, int blocks); @@ -32,63 +31,38 @@ static inline int sha1_base_init(struct shash_desc *desc) return 0; } -static inline int sha1_base_do_update(struct shash_desc *desc, - const u8 *data, - unsigned int len, - sha1_block_fn *block_fn) +static inline int sha1_base_do_update_blocks(struct shash_desc *desc, + const u8 *data, + unsigned int len, + sha1_block_fn *block_fn) { + unsigned int remain = len - round_down(len, SHA1_BLOCK_SIZE); struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; - - sctx->count += len; - - if (unlikely((partial + len) >= SHA1_BLOCK_SIZE)) { - int blocks; - - if (partial) { - int p = SHA1_BLOCK_SIZE - partial; - - memcpy(sctx->buffer + partial, data, p); - data += p; - len -= p; - - block_fn(sctx, sctx->buffer, 1); - } - blocks = len / SHA1_BLOCK_SIZE; - len %= SHA1_BLOCK_SIZE; - - if (blocks) { - block_fn(sctx, data, blocks); - data += blocks * SHA1_BLOCK_SIZE; - } - partial = 0; - } - if (len) - memcpy(sctx->buffer + partial, data, len); - - return 0; + sctx->count += len - remain; + block_fn(sctx, data, len / SHA1_BLOCK_SIZE); + return remain; } -static inline int sha1_base_do_finalize(struct shash_desc *desc, - sha1_block_fn *block_fn) +static inline int sha1_base_do_finup(struct shash_desc *desc, + const u8 *src, unsigned int len, + sha1_block_fn *block_fn) { - const int bit_offset = SHA1_BLOCK_SIZE - sizeof(__be64); + unsigned int bit_offset = SHA1_BLOCK_SIZE / 8 - 1; struct sha1_state *sctx = shash_desc_ctx(desc); - __be64 *bits = (__be64 *)(sctx->buffer + bit_offset); - unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; - - sctx->buffer[partial++] = 0x80; - if (partial > bit_offset) { - memset(sctx->buffer + partial, 0x0, SHA1_BLOCK_SIZE - partial); - partial = 0; - - block_fn(sctx, sctx->buffer, 1); - } - - memset(sctx->buffer + partial, 0x0, bit_offset - partial); - *bits = cpu_to_be64(sctx->count << 3); - block_fn(sctx, sctx->buffer, 1); + union { + __be64 b64[SHA1_BLOCK_SIZE / 4]; + u8 u8[SHA1_BLOCK_SIZE * 2]; + } block = {}; + + if (len >= bit_offset * 8) + bit_offset += SHA1_BLOCK_SIZE / 8; + memcpy(&block, src, len); + block.u8[len] = 0x80; + sctx->count += len; + block.b64[bit_offset] = cpu_to_be64(sctx->count << 3); + block_fn(sctx, block.u8, (bit_offset + 1) * 8 / SHA1_BLOCK_SIZE); + memzero_explicit(&block, sizeof(block)); return 0; } @@ -102,7 +76,6 @@ static inline int sha1_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(__be32); i++) put_unaligned_be32(sctx->state[i], digest++); - memzero_explicit(sctx, sizeof(*sctx)); return 0; } diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index b9e9281d76c9..abbd882f7849 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -19,6 +19,7 @@ #define SHA512_DIGEST_SIZE 64 #define SHA512_BLOCK_SIZE 128 +#define SHA512_STATE_SIZE 80 #define SHA224_H0 0xc1059ed8UL #define SHA224_H1 0x367cd507UL @@ -64,6 +65,11 @@ extern const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE]; extern const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE]; +struct crypto_sha256_state { + u32 state[SHA256_DIGEST_SIZE / 4]; + u64 count; +}; + struct sha256_state { u32 state[SHA256_DIGEST_SIZE / 4]; u64 count; @@ -76,20 +82,6 @@ struct sha512_state { u8 buf[SHA512_BLOCK_SIZE]; }; -struct shash_desc; - -extern int crypto_sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -extern int crypto_sha256_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); - -extern int crypto_sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -extern int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); - /* * Stand-alone implementation of the SHA256 algorithm. It is designed to * have as little dependencies as possible so it can be used in the diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h index e0418818d63c..08cd5e41d4fd 100644 --- a/include/crypto/sha256_base.h +++ b/include/crypto/sha256_base.h @@ -8,14 +8,14 @@ #ifndef _CRYPTO_SHA256_BASE_H #define _CRYPTO_SHA256_BASE_H -#include <asm/byteorder.h> -#include <linux/unaligned.h> #include <crypto/internal/hash.h> #include <crypto/sha2.h> +#include <linux/math.h> #include <linux/string.h> #include <linux/types.h> +#include <linux/unaligned.h> -typedef void (sha256_block_fn)(struct sha256_state *sst, u8 const *src, +typedef void (sha256_block_fn)(struct crypto_sha256_state *sst, u8 const *src, int blocks); static inline int sha224_base_init(struct shash_desc *desc) @@ -40,6 +40,7 @@ static inline int lib_sha256_base_do_update(struct sha256_state *sctx, sha256_block_fn *block_fn) { unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; + struct crypto_sha256_state *state = (void *)sctx; sctx->count += len; @@ -53,14 +54,14 @@ static inline int lib_sha256_base_do_update(struct sha256_state *sctx, data += p; len -= p; - block_fn(sctx, sctx->buf, 1); + block_fn(state, sctx->buf, 1); } blocks = len / SHA256_BLOCK_SIZE; len %= SHA256_BLOCK_SIZE; if (blocks) { - block_fn(sctx, data, blocks); + block_fn(state, data, blocks); data += blocks * SHA256_BLOCK_SIZE; } partial = 0; @@ -71,36 +72,72 @@ static inline int lib_sha256_base_do_update(struct sha256_state *sctx, return 0; } -static inline int sha256_base_do_update(struct shash_desc *desc, - const u8 *data, - unsigned int len, - sha256_block_fn *block_fn) +static inline int lib_sha256_base_do_update_blocks( + struct crypto_sha256_state *sctx, const u8 *data, unsigned int len, + sha256_block_fn *block_fn) { - struct sha256_state *sctx = shash_desc_ctx(desc); + unsigned int remain = len - round_down(len, SHA256_BLOCK_SIZE); - return lib_sha256_base_do_update(sctx, data, len, block_fn); + sctx->count += len - remain; + block_fn(sctx, data, len / SHA256_BLOCK_SIZE); + return remain; } -static inline int lib_sha256_base_do_finalize(struct sha256_state *sctx, - sha256_block_fn *block_fn) +static inline int sha256_base_do_update_blocks( + struct shash_desc *desc, const u8 *data, unsigned int len, + sha256_block_fn *block_fn) { - const int bit_offset = SHA256_BLOCK_SIZE - sizeof(__be64); - __be64 *bits = (__be64 *)(sctx->buf + bit_offset); - unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; + return lib_sha256_base_do_update_blocks(shash_desc_ctx(desc), data, + len, block_fn); +} - sctx->buf[partial++] = 0x80; - if (partial > bit_offset) { - memset(sctx->buf + partial, 0x0, SHA256_BLOCK_SIZE - partial); - partial = 0; +static inline int lib_sha256_base_do_finup(struct crypto_sha256_state *sctx, + const u8 *src, unsigned int len, + sha256_block_fn *block_fn) +{ + unsigned int bit_offset = SHA256_BLOCK_SIZE / 8 - 1; + union { + __be64 b64[SHA256_BLOCK_SIZE / 4]; + u8 u8[SHA256_BLOCK_SIZE * 2]; + } block = {}; + + if (len >= bit_offset * 8) + bit_offset += SHA256_BLOCK_SIZE / 8; + memcpy(&block, src, len); + block.u8[len] = 0x80; + sctx->count += len; + block.b64[bit_offset] = cpu_to_be64(sctx->count << 3); + block_fn(sctx, block.u8, (bit_offset + 1) * 8 / SHA256_BLOCK_SIZE); + memzero_explicit(&block, sizeof(block)); + + return 0; +} + +static inline int sha256_base_do_finup(struct shash_desc *desc, + const u8 *src, unsigned int len, + sha256_block_fn *block_fn) +{ + struct crypto_sha256_state *sctx = shash_desc_ctx(desc); + + if (len >= SHA256_BLOCK_SIZE) { + int remain; - block_fn(sctx, sctx->buf, 1); + remain = lib_sha256_base_do_update_blocks(sctx, src, len, + block_fn); + src += len - remain; + len = remain; } + return lib_sha256_base_do_finup(sctx, src, len, block_fn); +} - memset(sctx->buf + partial, 0x0, bit_offset - partial); - *bits = cpu_to_be64(sctx->count << 3); - block_fn(sctx, sctx->buf, 1); +static inline int lib_sha256_base_do_finalize(struct sha256_state *sctx, + sha256_block_fn *block_fn) +{ + unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; + struct crypto_sha256_state *state = (void *)sctx; - return 0; + sctx->count -= partial; + return lib_sha256_base_do_finup(state, sctx->buf, partial, block_fn); } static inline int sha256_base_do_finalize(struct shash_desc *desc, @@ -111,25 +148,33 @@ static inline int sha256_base_do_finalize(struct shash_desc *desc, return lib_sha256_base_do_finalize(sctx, block_fn); } -static inline int lib_sha256_base_finish(struct sha256_state *sctx, u8 *out, - unsigned int digest_size) +static inline int __sha256_base_finish(u32 state[SHA256_DIGEST_SIZE / 4], + u8 *out, unsigned int digest_size) { __be32 *digest = (__be32 *)out; int i; for (i = 0; digest_size > 0; i++, digest_size -= sizeof(__be32)) - put_unaligned_be32(sctx->state[i], digest++); + put_unaligned_be32(state[i], digest++); + return 0; +} +static inline void lib_sha256_base_finish(struct sha256_state *sctx, u8 *out, + unsigned int digest_size) +{ + __sha256_base_finish(sctx->state, out, digest_size); memzero_explicit(sctx, sizeof(*sctx)); - return 0; } static inline int sha256_base_finish(struct shash_desc *desc, u8 *out) { unsigned int digest_size = crypto_shash_digestsize(desc->tfm); - struct sha256_state *sctx = shash_desc_ctx(desc); + struct crypto_sha256_state *sctx = shash_desc_ctx(desc); - return lib_sha256_base_finish(sctx, out, digest_size); + return __sha256_base_finish(sctx->state, out, digest_size); } +void sha256_transform_blocks(struct crypto_sha256_state *sst, + const u8 *input, int blocks); + #endif /* _CRYPTO_SHA256_BASE_H */ diff --git a/include/crypto/sha3.h b/include/crypto/sha3.h index 080f60c2e6b1..3c2559f51ada 100644 --- a/include/crypto/sha3.h +++ b/include/crypto/sha3.h @@ -5,6 +5,8 @@ #ifndef __CRYPTO_SHA3_H__ #define __CRYPTO_SHA3_H__ +#include <linux/types.h> + #define SHA3_224_DIGEST_SIZE (224 / 8) #define SHA3_224_BLOCK_SIZE (200 - 2 * SHA3_224_DIGEST_SIZE) @@ -17,18 +19,14 @@ #define SHA3_512_DIGEST_SIZE (512 / 8) #define SHA3_512_BLOCK_SIZE (200 - 2 * SHA3_512_DIGEST_SIZE) -struct sha3_state { - u64 st[25]; - unsigned int rsiz; - unsigned int rsizw; +#define SHA3_STATE_SIZE 200 - unsigned int partial; - u8 buf[SHA3_224_BLOCK_SIZE]; +struct shash_desc; + +struct sha3_state { + u64 st[SHA3_STATE_SIZE / 8]; }; int crypto_sha3_init(struct shash_desc *desc); -int crypto_sha3_update(struct shash_desc *desc, const u8 *data, - unsigned int len); -int crypto_sha3_final(struct shash_desc *desc, u8 *out); #endif diff --git a/include/crypto/sha512_base.h b/include/crypto/sha512_base.h index 679916a84cb2..aa814bab442d 100644 --- a/include/crypto/sha512_base.h +++ b/include/crypto/sha512_base.h @@ -10,10 +10,10 @@ #include <crypto/internal/hash.h> #include <crypto/sha2.h> -#include <linux/crypto.h> -#include <linux/module.h> +#include <linux/compiler.h> +#include <linux/math.h> #include <linux/string.h> - +#include <linux/types.h> #include <linux/unaligned.h> typedef void (sha512_block_fn)(struct sha512_state *sst, u8 const *src, @@ -53,66 +53,51 @@ static inline int sha512_base_init(struct shash_desc *desc) return 0; } -static inline int sha512_base_do_update(struct shash_desc *desc, - const u8 *data, - unsigned int len, - sha512_block_fn *block_fn) +static inline int sha512_base_do_update_blocks(struct shash_desc *desc, + const u8 *data, + unsigned int len, + sha512_block_fn *block_fn) { + unsigned int remain = len - round_down(len, SHA512_BLOCK_SIZE); struct sha512_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE; + len -= remain; sctx->count[0] += len; if (sctx->count[0] < len) sctx->count[1]++; - - if (unlikely((partial + len) >= SHA512_BLOCK_SIZE)) { - int blocks; - - if (partial) { - int p = SHA512_BLOCK_SIZE - partial; - - memcpy(sctx->buf + partial, data, p); - data += p; - len -= p; - - block_fn(sctx, sctx->buf, 1); - } - - blocks = len / SHA512_BLOCK_SIZE; - len %= SHA512_BLOCK_SIZE; - - if (blocks) { - block_fn(sctx, data, blocks); - data += blocks * SHA512_BLOCK_SIZE; - } - partial = 0; - } - if (len) - memcpy(sctx->buf + partial, data, len); - - return 0; + block_fn(sctx, data, len / SHA512_BLOCK_SIZE); + return remain; } -static inline int sha512_base_do_finalize(struct shash_desc *desc, - sha512_block_fn *block_fn) +static inline int sha512_base_do_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, + sha512_block_fn *block_fn) { - const int bit_offset = SHA512_BLOCK_SIZE - sizeof(__be64[2]); + unsigned int bit_offset = SHA512_BLOCK_SIZE / 8 - 2; struct sha512_state *sctx = shash_desc_ctx(desc); - __be64 *bits = (__be64 *)(sctx->buf + bit_offset); - unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE; + union { + __be64 b64[SHA512_BLOCK_SIZE / 4]; + u8 u8[SHA512_BLOCK_SIZE * 2]; + } block = {}; - sctx->buf[partial++] = 0x80; - if (partial > bit_offset) { - memset(sctx->buf + partial, 0x0, SHA512_BLOCK_SIZE - partial); - partial = 0; + if (len >= SHA512_BLOCK_SIZE) { + int remain; - block_fn(sctx, sctx->buf, 1); + remain = sha512_base_do_update_blocks(desc, src, len, block_fn); + src += len - remain; + len = remain; } - memset(sctx->buf + partial, 0x0, bit_offset - partial); - bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61); - bits[1] = cpu_to_be64(sctx->count[0] << 3); - block_fn(sctx, sctx->buf, 1); + if (len >= bit_offset * 8) + bit_offset += SHA512_BLOCK_SIZE / 8; + memcpy(&block, src, len); + block.u8[len] = 0x80; + sctx->count[0] += len; + block.b64[bit_offset] = cpu_to_be64(sctx->count[1] << 3 | + sctx->count[0] >> 61); + block.b64[bit_offset + 1] = cpu_to_be64(sctx->count[0] << 3); + block_fn(sctx, block.u8, (bit_offset + 2) * 8 / SHA512_BLOCK_SIZE); + memzero_explicit(&block, sizeof(block)); return 0; } @@ -126,9 +111,10 @@ static inline int sha512_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; digest_size > 0; i++, digest_size -= sizeof(__be64)) put_unaligned_be64(sctx->state[i], digest++); - - memzero_explicit(sctx, sizeof(*sctx)); return 0; } +void sha512_generic_block_fn(struct sha512_state *sst, u8 const *src, + int blocks); + #endif /* _CRYPTO_SHA512_BASE_H */ diff --git a/include/crypto/sig.h b/include/crypto/sig.h index 11024708c069..fa6dafafab3f 100644 --- a/include/crypto/sig.h +++ b/include/crypto/sig.h @@ -128,7 +128,7 @@ static inline void crypto_free_sig(struct crypto_sig *tfm) /** * crypto_sig_keysize() - Get key size * - * Function returns the key size in bytes. + * Function returns the key size in bits. * Function assumes that the key is already set in the transformation. If this * function is called without a setkey or with a failed setkey, you may end up * in a NULL dereference. diff --git a/include/crypto/sm3.h b/include/crypto/sm3.h index 1f021ad0533f..c8d02c86c298 100644 --- a/include/crypto/sm3.h +++ b/include/crypto/sm3.h @@ -14,6 +14,7 @@ #define SM3_DIGEST_SIZE 32 #define SM3_BLOCK_SIZE 64 +#define SM3_STATE_SIZE 40 #define SM3_T1 0x79CC4519 #define SM3_T2 0x7A879D8A @@ -58,7 +59,6 @@ static inline void sm3_init(struct sm3_state *sctx) sctx->count = 0; } -void sm3_update(struct sm3_state *sctx, const u8 *data, unsigned int len); -void sm3_final(struct sm3_state *sctx, u8 *out); +void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks); #endif diff --git a/include/crypto/sm3_base.h b/include/crypto/sm3_base.h index b33ed39c2bce..7c53570bc05e 100644 --- a/include/crypto/sm3_base.h +++ b/include/crypto/sm3_base.h @@ -11,87 +11,59 @@ #include <crypto/internal/hash.h> #include <crypto/sm3.h> -#include <linux/crypto.h> +#include <linux/math.h> #include <linux/module.h> #include <linux/string.h> +#include <linux/types.h> #include <linux/unaligned.h> typedef void (sm3_block_fn)(struct sm3_state *sst, u8 const *src, int blocks); static inline int sm3_base_init(struct shash_desc *desc) { - struct sm3_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SM3_IVA; - sctx->state[1] = SM3_IVB; - sctx->state[2] = SM3_IVC; - sctx->state[3] = SM3_IVD; - sctx->state[4] = SM3_IVE; - sctx->state[5] = SM3_IVF; - sctx->state[6] = SM3_IVG; - sctx->state[7] = SM3_IVH; - sctx->count = 0; - + sm3_init(shash_desc_ctx(desc)); return 0; } -static inline int sm3_base_do_update(struct shash_desc *desc, - const u8 *data, - unsigned int len, - sm3_block_fn *block_fn) +static inline int sm3_base_do_update_blocks(struct shash_desc *desc, + const u8 *data, unsigned int len, + sm3_block_fn *block_fn) { + unsigned int remain = len - round_down(len, SM3_BLOCK_SIZE); struct sm3_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SM3_BLOCK_SIZE; - - sctx->count += len; - - if (unlikely((partial + len) >= SM3_BLOCK_SIZE)) { - int blocks; - - if (partial) { - int p = SM3_BLOCK_SIZE - partial; - - memcpy(sctx->buffer + partial, data, p); - data += p; - len -= p; - block_fn(sctx, sctx->buffer, 1); - } - - blocks = len / SM3_BLOCK_SIZE; - len %= SM3_BLOCK_SIZE; - - if (blocks) { - block_fn(sctx, data, blocks); - data += blocks * SM3_BLOCK_SIZE; - } - partial = 0; - } - if (len) - memcpy(sctx->buffer + partial, data, len); - - return 0; + sctx->count += len - remain; + block_fn(sctx, data, len / SM3_BLOCK_SIZE); + return remain; } -static inline int sm3_base_do_finalize(struct shash_desc *desc, - sm3_block_fn *block_fn) +static inline int sm3_base_do_finup(struct shash_desc *desc, + const u8 *src, unsigned int len, + sm3_block_fn *block_fn) { - const int bit_offset = SM3_BLOCK_SIZE - sizeof(__be64); + unsigned int bit_offset = SM3_BLOCK_SIZE / 8 - 1; struct sm3_state *sctx = shash_desc_ctx(desc); - __be64 *bits = (__be64 *)(sctx->buffer + bit_offset); - unsigned int partial = sctx->count % SM3_BLOCK_SIZE; + union { + __be64 b64[SM3_BLOCK_SIZE / 4]; + u8 u8[SM3_BLOCK_SIZE * 2]; + } block = {}; - sctx->buffer[partial++] = 0x80; - if (partial > bit_offset) { - memset(sctx->buffer + partial, 0x0, SM3_BLOCK_SIZE - partial); - partial = 0; + if (len >= SM3_BLOCK_SIZE) { + int remain; - block_fn(sctx, sctx->buffer, 1); + remain = sm3_base_do_update_blocks(desc, src, len, block_fn); + src += len - remain; + len = remain; } - memset(sctx->buffer + partial, 0x0, bit_offset - partial); - *bits = cpu_to_be64(sctx->count << 3); - block_fn(sctx, sctx->buffer, 1); + if (len >= bit_offset * 8) + bit_offset += SM3_BLOCK_SIZE / 8; + memcpy(&block, src, len); + block.u8[len] = 0x80; + sctx->count += len; + block.b64[bit_offset] = cpu_to_be64(sctx->count << 3); + block_fn(sctx, block.u8, (bit_offset + 1) * 8 / SM3_BLOCK_SIZE); + memzero_explicit(&block, sizeof(block)); return 0; } @@ -104,8 +76,6 @@ static inline int sm3_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; i < SM3_DIGEST_SIZE / sizeof(__be32); i++) put_unaligned_be32(sctx->state[i], digest++); - - memzero_explicit(sctx, sizeof(*sctx)); return 0; } diff --git a/include/linux/crypto.h b/include/linux/crypto.h index 1e3809d28abd..f691ce01745e 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -14,7 +14,6 @@ #include <linux/completion.h> #include <linux/errno.h> -#include <linux/list.h> #include <linux/refcount.h> #include <linux/slab.h> #include <linux/types.h> @@ -51,6 +50,15 @@ #define CRYPTO_ALG_NEED_FALLBACK 0x00000100 /* + * Set if the algorithm data structure should be duplicated into + * kmalloc memory before registration. This is useful for hardware + * that can be disconnected at will. Do not use this if the data + * structure is embedded into a bigger one. Duplicate the overall + * data structure in the driver in that case. + */ +#define CRYPTO_ALG_DUP_FIRST 0x00000200 + +/* * Set if the algorithm has passed automated run-time testing. Note that * if there is no run-time testing for a given algorithm it is considered * to have passed. @@ -128,6 +136,8 @@ /* Set if the algorithm supports request chains and virtual addresses. */ #define CRYPTO_ALG_REQ_CHAIN 0x00040000 +/* The high bits 0xff000000 are reserved for type-specific flags. */ + /* * Transform masks and values (for crt_flags). */ @@ -179,7 +189,6 @@ struct crypto_async_request { struct crypto_tfm *tfm; u32 flags; - int err; }; /** @@ -278,6 +287,7 @@ struct cipher_alg { * to the alignmask of the algorithm being used, in order to * avoid the API having to realign them. Note: the alignmask is * not supported for hash algorithms and is always 0 for them. + * @cra_reqsize: Size of the request context for this algorithm. * @cra_priority: Priority of this transformation implementation. In case * multiple transformations with same @cra_name are available to * the Crypto API, the kernel will use the one with highest @@ -302,17 +312,8 @@ struct cipher_alg { * by @cra_type and @cra_flags above, the associated structure must be * filled with callbacks. This field might be empty. This is the case * for ahash, shash. - * @cra_init: Initialize the cryptographic transformation object. This function - * is used to initialize the cryptographic transformation object. - * This function is called only once at the instantiation time, right - * after the transformation context was allocated. In case the - * cryptographic hardware has some special requirements which need to - * be handled by software, this function shall check for the precise - * requirement of the transformation and put any software fallbacks - * in place. - * @cra_exit: Deinitialize the cryptographic transformation object. This is a - * counterpart to @cra_init, used to remove various changes set in - * @cra_init. + * @cra_init: Deprecated, do not use. + * @cra_exit: Deprecated, do not use. * @cra_u.cipher: Union member which contains a single-block symmetric cipher * definition. See @struct @cipher_alg. * @cra_module: Owner of this transformation implementation. Set to THIS_MODULE @@ -333,6 +334,7 @@ struct crypto_alg { unsigned int cra_blocksize; unsigned int cra_ctxsize; unsigned int cra_alignmask; + unsigned int cra_reqsize; int cra_priority; refcount_t cra_refcnt; @@ -452,6 +454,11 @@ static inline unsigned int crypto_tfm_alg_alignmask(struct crypto_tfm *tfm) return tfm->__crt_alg->cra_alignmask; } +static inline unsigned int crypto_tfm_alg_reqsize(struct crypto_tfm *tfm) +{ + return tfm->__crt_alg->cra_reqsize; +} + static inline u32 crypto_tfm_get_flags(struct crypto_tfm *tfm) { return tfm->crt_flags; @@ -473,22 +480,33 @@ static inline unsigned int crypto_tfm_ctx_alignment(void) return __alignof__(tfm->__crt_ctx); } -static inline void crypto_reqchain_init(struct crypto_async_request *req) +static inline bool crypto_tfm_is_async(struct crypto_tfm *tfm) { - req->err = -EINPROGRESS; - INIT_LIST_HEAD(&req->list); + return tfm->__crt_alg->cra_flags & CRYPTO_ALG_ASYNC; } -static inline void crypto_request_chain(struct crypto_async_request *req, - struct crypto_async_request *head) +static inline bool crypto_req_on_stack(struct crypto_async_request *req) { - req->err = -EINPROGRESS; - list_add_tail(&req->list, &head->list); + return req->flags & CRYPTO_TFM_REQ_ON_STACK; } -static inline bool crypto_tfm_is_async(struct crypto_tfm *tfm) +static inline void crypto_request_set_callback( + struct crypto_async_request *req, u32 flags, + crypto_completion_t compl, void *data) { - return tfm->__crt_alg->cra_flags & CRYPTO_ALG_ASYNC; + u32 keep = CRYPTO_TFM_REQ_ON_STACK; + + req->complete = compl; + req->data = data; + req->flags &= keep; + req->flags |= flags & ~keep; +} + +static inline void crypto_request_set_tfm(struct crypto_async_request *req, + struct crypto_tfm *tfm) +{ + req->tfm = tfm; + req->flags &= ~CRYPTO_TFM_REQ_ON_STACK; } #endif /* _LINUX_CRYPTO_H */ diff --git a/include/linux/psp-sev.h b/include/linux/psp-sev.h index f3cad182d4ef..0b3a36bdaa90 100644 --- a/include/linux/psp-sev.h +++ b/include/linux/psp-sev.h @@ -954,6 +954,7 @@ int sev_do_cmd(int cmd, void *data, int *psp_ret); void *psp_copy_user_blob(u64 uaddr, u32 len); void *snp_alloc_firmware_page(gfp_t mask); void snp_free_firmware_page(void *addr); +void sev_platform_shutdown(void); #else /* !CONFIG_CRYPTO_DEV_SP_PSP */ @@ -988,6 +989,8 @@ static inline void *snp_alloc_firmware_page(gfp_t mask) static inline void snp_free_firmware_page(void *addr) { } +static inline void sev_platform_shutdown(void) { } + #endif /* CONFIG_CRYPTO_DEV_SP_PSP */ #endif /* __PSP_SEV_H__ */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 798972b29b68..2c6ab80e0cdc 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -152,4 +152,7 @@ config CRYPTO_LIB_SHA1 config CRYPTO_LIB_SHA256 tristate +config CRYPTO_LIB_SM3 + tristate + endmenu diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 01fac1cd05a1..4dd62bc5bee3 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -60,3 +60,6 @@ endif obj-$(CONFIG_MPILIB) += mpi/ obj-$(CONFIG_CRYPTO_MANAGER_EXTRA_TESTS) += simd.o + +obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o +libsm3-y := sm3.o diff --git a/lib/crypto/poly1305.c b/lib/crypto/poly1305.c index 6e80214ebad8..b633b043f0f6 100644 --- a/lib/crypto/poly1305.c +++ b/lib/crypto/poly1305.c @@ -22,8 +22,6 @@ void poly1305_init_generic(struct poly1305_desc_ctx *desc, desc->s[3] = get_unaligned_le32(key + 28); poly1305_core_init(&desc->h); desc->buflen = 0; - desc->sset = true; - desc->rset = 2; } EXPORT_SYMBOL_GPL(poly1305_init_generic); diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 04c1f2557e6c..a89bab377de1 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -118,18 +118,19 @@ static void sha256_transform(u32 *state, const u8 *input, u32 *W) state[4] += e; state[5] += f; state[6] += g; state[7] += h; } -static void sha256_transform_blocks(struct sha256_state *sctx, - const u8 *input, int blocks) +void sha256_transform_blocks(struct crypto_sha256_state *sst, + const u8 *input, int blocks) { u32 W[64]; do { - sha256_transform(sctx->state, input, W); + sha256_transform(sst->state, input, W); input += SHA256_BLOCK_SIZE; } while (--blocks); memzero_explicit(W, sizeof(W)); } +EXPORT_SYMBOL_GPL(sha256_transform_blocks); void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len) { diff --git a/crypto/sm3.c b/lib/crypto/sm3.c index 18c2fb73ba16..efff0e267d84 100644 --- a/crypto/sm3.c +++ b/lib/crypto/sm3.c @@ -8,9 +8,11 @@ * Copyright (C) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> */ +#include <crypto/sm3.h> +#include <linux/kernel.h> #include <linux/module.h> +#include <linux/string.h> #include <linux/unaligned.h> -#include <crypto/sm3.h> static const u32 ____cacheline_aligned K[64] = { 0x79cc4519, 0xf3988a32, 0xe7311465, 0xce6228cb, @@ -166,81 +168,18 @@ static void sm3_transform(struct sm3_state *sctx, u8 const *data, u32 W[16]) #undef W1 #undef W2 -static inline void sm3_block(struct sm3_state *sctx, - u8 const *data, int blocks, u32 W[16]) -{ - while (blocks--) { - sm3_transform(sctx, data, W); - data += SM3_BLOCK_SIZE; - } -} - -void sm3_update(struct sm3_state *sctx, const u8 *data, unsigned int len) +void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks) { - unsigned int partial = sctx->count % SM3_BLOCK_SIZE; u32 W[16]; - sctx->count += len; - - if ((partial + len) >= SM3_BLOCK_SIZE) { - int blocks; - - if (partial) { - int p = SM3_BLOCK_SIZE - partial; - - memcpy(sctx->buffer + partial, data, p); - data += p; - len -= p; - - sm3_block(sctx, sctx->buffer, 1, W); - } - - blocks = len / SM3_BLOCK_SIZE; - len %= SM3_BLOCK_SIZE; - - if (blocks) { - sm3_block(sctx, data, blocks, W); - data += blocks * SM3_BLOCK_SIZE; - } - - memzero_explicit(W, sizeof(W)); - - partial = 0; - } - if (len) - memcpy(sctx->buffer + partial, data, len); -} -EXPORT_SYMBOL_GPL(sm3_update); - -void sm3_final(struct sm3_state *sctx, u8 *out) -{ - const int bit_offset = SM3_BLOCK_SIZE - sizeof(u64); - __be64 *bits = (__be64 *)(sctx->buffer + bit_offset); - __be32 *digest = (__be32 *)out; - unsigned int partial = sctx->count % SM3_BLOCK_SIZE; - u32 W[16]; - int i; - - sctx->buffer[partial++] = 0x80; - if (partial > bit_offset) { - memset(sctx->buffer + partial, 0, SM3_BLOCK_SIZE - partial); - partial = 0; - - sm3_block(sctx, sctx->buffer, 1, W); - } - - memset(sctx->buffer + partial, 0, bit_offset - partial); - *bits = cpu_to_be64(sctx->count << 3); - sm3_block(sctx, sctx->buffer, 1, W); - - for (i = 0; i < 8; i++) - put_unaligned_be32(sctx->state[i], digest++); + do { + sm3_transform(sctx, data, W); + data += SM3_BLOCK_SIZE; + } while (--blocks); - /* Zeroize sensitive information. */ memzero_explicit(W, sizeof(W)); - memzero_explicit(sctx, sizeof(*sctx)); } -EXPORT_SYMBOL_GPL(sm3_final); +EXPORT_SYMBOL_GPL(sm3_block_generic); MODULE_DESCRIPTION("Generic SM3 library"); MODULE_LICENSE("GPL v2"); |