microsoft
/
WSL2-Linux-Kernel
зеркало из https://github.com/microsoft/WSL2-Linux-Kernel.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

crypto: arm64/gcm-aes-ce - remove non-SIMD fallback path 

Now that kernel mode SIMD is guaranteed to be available when executing
in task or softirq context, we no longer need scalar fallbacks to use
when the NEON is unavailable. So get rid of them.

Reviewed-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
This commit is contained in:
Ard Biesheuvel 2021-08-27 09:03:36 +02:00 коммит произвёл Herbert Xu
Родитель 4a7e1e5fc2
Коммит b9e699f912
1 изменённых файлов: 55 добавлений и 162 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

113
arch/arm64/crypto/ghash-ce-glue.c
Просмотреть файл

@ -362,7 +362,6 @@ static int gcm_encrypt(struct aead_request *req)
err = skcipher_walk_aead_encrypt(&walk, req, false);
if (likely(crypto_simd_usable())) {
do {
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
@ -393,53 +392,6 @@ static int gcm_encrypt(struct aead_request *req)
err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
} while (walk.nbytes);
} else {
while (walk.nbytes >= AES_BLOCK_SIZE) {
int blocks = walk.nbytes / AES_BLOCK_SIZE;
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
int remaining = blocks;
do {
aes_encrypt(&ctx->aes_key, buf, iv);
crypto_xor_cpy(dst, src, buf, AES_BLOCK_SIZE);
crypto_inc(iv, AES_BLOCK_SIZE);
dst += AES_BLOCK_SIZE;
src += AES_BLOCK_SIZE;
} while (--remaining > 0);
ghash_do_update(blocks, dg, walk.dst.virt.addr,
&ctx->ghash_key, NULL);
err = skcipher_walk_done(&walk,
walk.nbytes % AES_BLOCK_SIZE);
}
/* handle the tail */
if (walk.nbytes) {
aes_encrypt(&ctx->aes_key, buf, iv);
crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr,
buf, walk.nbytes);
memcpy(buf, walk.dst.virt.addr, walk.nbytes);
memset(buf + walk.nbytes, 0, sizeof(buf) - walk.nbytes);
}
tag = (u8 *)&lengths;
ghash_do_update(1, dg, tag, &ctx->ghash_key,
walk.nbytes ? buf : NULL);
if (walk.nbytes)
err = skcipher_walk_done(&walk, 0);
put_unaligned_be64(dg[1], tag);
put_unaligned_be64(dg[0], tag + 8);
put_unaligned_be32(1, iv + GCM_IV_SIZE);
aes_encrypt(&ctx->aes_key, iv, iv);
crypto_xor(tag, iv, AES_BLOCK_SIZE);
}
if (err)
return err;
@ -464,6 +416,7 @@ static int gcm_decrypt(struct aead_request *req)
u64 dg[2] = {};
be128 lengths;
u8 *tag;
int ret;
int err;
lengths.a = cpu_to_be64(req->assoclen * 8);
@ -481,9 +434,6 @@ static int gcm_decrypt(struct aead_request *req)
err = skcipher_walk_aead_decrypt(&walk, req, false);
if (likely(crypto_simd_usable())) {
int ret;
do {
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
@ -500,8 +450,7 @@ static int gcm_decrypt(struct aead_request *req)
}
kernel_neon_begin();
ret = pmull_gcm_decrypt(nbytes, dst, src,
ctx->ghash_key.h,
ret = pmull_gcm_decrypt(nbytes, dst, src, ctx->ghash_key.h,
dg, iv, ctx->aes_key.key_enc,
nrounds, tag, otag, authsize);
kernel_neon_end();
@ -518,64 +467,8 @@ static int gcm_decrypt(struct aead_request *req)
if (err)
return err;
if (ret)
return -EBADMSG;
} else {
while (walk.nbytes >= AES_BLOCK_SIZE) {
int blocks = walk.nbytes / AES_BLOCK_SIZE;
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
ghash_do_update(blocks, dg, walk.src.virt.addr,
&ctx->ghash_key, NULL);
do {
aes_encrypt(&ctx->aes_key, buf, iv);
crypto_xor_cpy(dst, src, buf, AES_BLOCK_SIZE);
crypto_inc(iv, AES_BLOCK_SIZE);
dst += AES_BLOCK_SIZE;
src += AES_BLOCK_SIZE;
} while (--blocks > 0);
err = skcipher_walk_done(&walk,
walk.nbytes % AES_BLOCK_SIZE);
}
/* handle the tail */
if (walk.nbytes) {
memcpy(buf, walk.src.virt.addr, walk.nbytes);
memset(buf + walk.nbytes, 0, sizeof(buf) - walk.nbytes);
}
tag = (u8 *)&lengths;
ghash_do_update(1, dg, tag, &ctx->ghash_key,
walk.nbytes ? buf : NULL);
if (walk.nbytes) {
aes_encrypt(&ctx->aes_key, buf, iv);
crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr,
buf, walk.nbytes);
err = skcipher_walk_done(&walk, 0);
}
if (err)
return err;
put_unaligned_be64(dg[1], tag);
put_unaligned_be64(dg[0], tag + 8);
put_unaligned_be32(1, iv + GCM_IV_SIZE);
aes_encrypt(&ctx->aes_key, iv, iv);
crypto_xor(tag, iv, AES_BLOCK_SIZE);
if (crypto_memneq(tag, otag, authsize)) {
memzero_explicit(tag, AES_BLOCK_SIZE);
return -EBADMSG;
}
}
return 0;
return ret ? -EBADMSG : 0;
}
static struct aead_alg gcm_aes_alg = {