crypto: aesni - Merge encode and decode to GCM_ENC_DEC macro
Make a macro for the main encode/decode routine. Only a small handful of lines differ for enc and dec. This will also become the main scatter/gather update routine. Signed-off-by: Dave Watson <davejwatson@fb.com> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
This commit is contained in:
Dave Watson
Herbert Xu
Родитель
adcadab369
Коммит
ba45833e3e
|
|
@ -223,6 +223,118 @@ ALL_F: .octa 0xffffffffffffffffffffffffffffffff
|
||||||
mov %r13, %r12
|
mov %r13, %r12
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
|
# GCM_ENC_DEC Encodes/Decodes given data. Assumes that the passed gcm_context
|
||||||
|
# struct has been initialized by GCM_INIT.
|
||||||
|
# Requires the input data be at least 1 byte long because of READ_PARTIAL_BLOCK
|
||||||
|
# Clobbers rax, r10-r13, and xmm0-xmm15
|
||||||
|
.macro GCM_ENC_DEC operation
|
||||||
|
# Encrypt/Decrypt first few blocks
|
||||||
|
|
||||||
|
and $(3<<4), %r12
|
||||||
|
jz _initial_num_blocks_is_0_\@
|
||||||
|
cmp $(2<<4), %r12
|
||||||
|
jb _initial_num_blocks_is_1_\@
|
||||||
|
je _initial_num_blocks_is_2_\@
|
||||||
|
_initial_num_blocks_is_3_\@:
|
||||||
|
INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
|
||||||
|
%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 5, 678, \operation
|
||||||
|
sub $48, %r13
|
||||||
|
jmp _initial_blocks_\@
|
||||||
|
_initial_num_blocks_is_2_\@:
|
||||||
|
INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
|
||||||
|
%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 6, 78, \operation
|
||||||
|
sub $32, %r13
|
||||||
|
jmp _initial_blocks_\@
|
||||||
|
_initial_num_blocks_is_1_\@:
|
||||||
|
INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
|
||||||
|
%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 7, 8, \operation
|
||||||
|
sub $16, %r13
|
||||||
|
jmp _initial_blocks_\@
|
||||||
|
_initial_num_blocks_is_0_\@:
|
||||||
|
INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
|
||||||
|
%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 8, 0, \operation
|
||||||
|
_initial_blocks_\@:
|
||||||
|
|
||||||
|
# Main loop - Encrypt/Decrypt remaining blocks
|
||||||
|
|
||||||
|
cmp $0, %r13
|
||||||
|
je _zero_cipher_left_\@
|
||||||
|
sub $64, %r13
|
||||||
|
je _four_cipher_left_\@
|
||||||
|
_crypt_by_4_\@:
|
||||||
|
GHASH_4_ENCRYPT_4_PARALLEL_\operation %xmm9, %xmm10, %xmm11, %xmm12, \
|
||||||
|
%xmm13, %xmm14, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, \
|
||||||
|
%xmm7, %xmm8, enc
|
||||||
|
add $64, %r11
|
||||||
|
sub $64, %r13
|
||||||
|
jne _crypt_by_4_\@
|
||||||
|
_four_cipher_left_\@:
|
||||||
|
GHASH_LAST_4 %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, \
|
||||||
|
%xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm8
|
||||||
|
_zero_cipher_left_\@:
|
||||||
|
mov %arg4, %r13
|
||||||
|
and $15, %r13 # %r13 = arg4 (mod 16)
|
||||||
|
je _multiple_of_16_bytes_\@
|
||||||
|
|
||||||
|
# Handle the last <16 Byte block separately
|
||||||
|
paddd ONE(%rip), %xmm0 # INCR CNT to get Yn
|
||||||
|
movdqa SHUF_MASK(%rip), %xmm10
|
||||||
|
PSHUFB_XMM %xmm10, %xmm0
|
||||||
|
|
||||||
|
ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # Encrypt(K, Yn)
|
||||||
|
|
||||||
|
lea (%arg3,%r11,1), %r10
|
||||||
|
mov %r13, %r12
|
||||||
|
READ_PARTIAL_BLOCK %r10 %r12 %xmm2 %xmm1
|
||||||
|
|
||||||
|
lea ALL_F+16(%rip), %r12
|
||||||
|
sub %r13, %r12
|
||||||
|
.ifc \operation, dec
|
||||||
|
movdqa %xmm1, %xmm2
|
||||||
|
.endif
|
||||||
|
pxor %xmm1, %xmm0 # XOR Encrypt(K, Yn)
|
||||||
|
movdqu (%r12), %xmm1
|
||||||
|
# get the appropriate mask to mask out top 16-r13 bytes of xmm0
|
||||||
|
pand %xmm1, %xmm0 # mask out top 16-r13 bytes of xmm0
|
||||||
|
.ifc \operation, dec
|
||||||
|
pand %xmm1, %xmm2
|
||||||
|
movdqa SHUF_MASK(%rip), %xmm10
|
||||||
|
PSHUFB_XMM %xmm10 ,%xmm2
|
||||||
|
|
||||||
|
pxor %xmm2, %xmm8
|
||||||
|
.else
|
||||||
|
movdqa SHUF_MASK(%rip), %xmm10
|
||||||
|
PSHUFB_XMM %xmm10,%xmm0
|
||||||
|
|
||||||
|
pxor %xmm0, %xmm8
|
||||||
|
.endif
|
||||||
|
|
||||||
|
GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
|
||||||
|
.ifc \operation, enc
|
||||||
|
# GHASH computation for the last <16 byte block
|
||||||
|
movdqa SHUF_MASK(%rip), %xmm10
|
||||||
|
# shuffle xmm0 back to output as ciphertext
|
||||||
|
PSHUFB_XMM %xmm10, %xmm0
|
||||||
|
.endif
|
||||||
|
|
||||||
|
# Output %r13 bytes
|
||||||
|
MOVQ_R64_XMM %xmm0, %rax
|
||||||
|
cmp $8, %r13
|
||||||
|
jle _less_than_8_bytes_left_\@
|
||||||
|
mov %rax, (%arg2 , %r11, 1)
|
||||||
|
add $8, %r11
|
||||||
|
psrldq $8, %xmm0
|
||||||
|
MOVQ_R64_XMM %xmm0, %rax
|
||||||
|
sub $8, %r13
|
||||||
|
_less_than_8_bytes_left_\@:
|
||||||
|
mov %al, (%arg2, %r11, 1)
|
||||||
|
add $1, %r11
|
||||||
|
shr $8, %rax
|
||||||
|
sub $1, %r13
|
||||||
|
jne _less_than_8_bytes_left_\@
|
||||||
|
_multiple_of_16_bytes_\@:
|
||||||
|
.endm
|
||||||
|
|
||||||
# GCM_COMPLETE Finishes update of tag of last partial block
|
# GCM_COMPLETE Finishes update of tag of last partial block
|
||||||
# Output: Authorization Tag (AUTH_TAG)
|
# Output: Authorization Tag (AUTH_TAG)
|
||||||
# Clobbers rax, r10-r12, and xmm0, xmm1, xmm5-xmm15
|
# Clobbers rax, r10-r12, and xmm0, xmm1, xmm5-xmm15
|
||||||
|
|
@ -1246,93 +1358,7 @@ ENTRY(aesni_gcm_dec)
|
||||||
FUNC_SAVE
|
FUNC_SAVE
|
||||||
|
|
||||||
GCM_INIT
|
GCM_INIT
|
||||||
|
GCM_ENC_DEC dec
|
||||||
# Decrypt first few blocks
|
|
||||||
|
|
||||||
and $(3<<4), %r12
|
|
||||||
jz _initial_num_blocks_is_0_decrypt
|
|
||||||
cmp $(2<<4), %r12
|
|
||||||
jb _initial_num_blocks_is_1_decrypt
|
|
||||||
je _initial_num_blocks_is_2_decrypt
|
|
||||||
_initial_num_blocks_is_3_decrypt:
|
|
||||||
INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
|
|
||||||
%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 5, 678, dec
|
|
||||||
sub $48, %r13
|
|
||||||
jmp _initial_blocks_decrypted
|
|
||||||
_initial_num_blocks_is_2_decrypt:
|
|
||||||
INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
|
|
||||||
%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 6, 78, dec
|
|
||||||
sub $32, %r13
|
|
||||||
jmp _initial_blocks_decrypted
|
|
||||||
_initial_num_blocks_is_1_decrypt:
|
|
||||||
INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
|
|
||||||
%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 7, 8, dec
|
|
||||||
sub $16, %r13
|
|
||||||
jmp _initial_blocks_decrypted
|
|
||||||
_initial_num_blocks_is_0_decrypt:
|
|
||||||
INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
|
|
||||||
%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 8, 0, dec
|
|
||||||
_initial_blocks_decrypted:
|
|
||||||
cmp $0, %r13
|
|
||||||
je _zero_cipher_left_decrypt
|
|
||||||
sub $64, %r13
|
|
||||||
je _four_cipher_left_decrypt
|
|
||||||
_decrypt_by_4:
|
|
||||||
GHASH_4_ENCRYPT_4_PARALLEL_DEC %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, \
|
|
||||||
%xmm14, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, dec
|
|
||||||
add $64, %r11
|
|
||||||
sub $64, %r13
|
|
||||||
jne _decrypt_by_4
|
|
||||||
_four_cipher_left_decrypt:
|
|
||||||
GHASH_LAST_4 %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, \
|
|
||||||
%xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm8
|
|
||||||
_zero_cipher_left_decrypt:
|
|
||||||
mov %arg4, %r13
|
|
||||||
and $15, %r13 # %r13 = arg4 (mod 16)
|
|
||||||
je _multiple_of_16_bytes_decrypt
|
|
||||||
|
|
||||||
# Handle the last <16 byte block separately
|
|
||||||
|
|
||||||
paddd ONE(%rip), %xmm0 # increment CNT to get Yn
|
|
||||||
movdqa SHUF_MASK(%rip), %xmm10
|
|
||||||
PSHUFB_XMM %xmm10, %xmm0
|
|
||||||
|
|
||||||
ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # E(K, Yn)
|
|
||||||
|
|
||||||
lea (%arg3,%r11,1), %r10
|
|
||||||
mov %r13, %r12
|
|
||||||
READ_PARTIAL_BLOCK %r10 %r12 %xmm2 %xmm1
|
|
||||||
|
|
||||||
lea ALL_F+16(%rip), %r12
|
|
||||||
sub %r13, %r12
|
|
||||||
movdqa %xmm1, %xmm2
|
|
||||||
pxor %xmm1, %xmm0 # Ciphertext XOR E(K, Yn)
|
|
||||||
movdqu (%r12), %xmm1
|
|
||||||
# get the appropriate mask to mask out top 16-%r13 bytes of %xmm0
|
|
||||||
pand %xmm1, %xmm0 # mask out top 16-%r13 bytes of %xmm0
|
|
||||||
pand %xmm1, %xmm2
|
|
||||||
movdqa SHUF_MASK(%rip), %xmm10
|
|
||||||
PSHUFB_XMM %xmm10 ,%xmm2
|
|
||||||
|
|
||||||
pxor %xmm2, %xmm8
|
|
||||||
GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
|
|
||||||
|
|
||||||
# output %r13 bytes
|
|
||||||
MOVQ_R64_XMM %xmm0, %rax
|
|
||||||
cmp $8, %r13
|
|
||||||
jle _less_than_8_bytes_left_decrypt
|
|
||||||
mov %rax, (%arg2 , %r11, 1)
|
|
||||||
add $8, %r11
|
|
||||||
psrldq $8, %xmm0
|
|
||||||
MOVQ_R64_XMM %xmm0, %rax
|
|
||||||
sub $8, %r13
|
|
||||||
_less_than_8_bytes_left_decrypt:
|
|
||||||
mov %al, (%arg2, %r11, 1)
|
|
||||||
add $1, %r11
|
|
||||||
shr $8, %rax
|
|
||||||
sub $1, %r13
|
|
||||||
jne _less_than_8_bytes_left_decrypt
|
|
||||||
_multiple_of_16_bytes_decrypt:
|
|
||||||
GCM_COMPLETE
|
GCM_COMPLETE
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
ret
|
||||||
|
|
@ -1418,98 +1444,7 @@ ENTRY(aesni_gcm_enc)
|
||||||
FUNC_SAVE
|
FUNC_SAVE
|
||||||
|
|
||||||
GCM_INIT
|
GCM_INIT
|
||||||
# Encrypt first few blocks
|
GCM_ENC_DEC enc
|
||||||
|
|
||||||
and $(3<<4), %r12
|
|
||||||
jz _initial_num_blocks_is_0_encrypt
|
|
||||||
cmp $(2<<4), %r12
|
|
||||||
jb _initial_num_blocks_is_1_encrypt
|
|
||||||
je _initial_num_blocks_is_2_encrypt
|
|
||||||
_initial_num_blocks_is_3_encrypt:
|
|
||||||
INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
|
|
||||||
%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 5, 678, enc
|
|
||||||
sub $48, %r13
|
|
||||||
jmp _initial_blocks_encrypted
|
|
||||||
_initial_num_blocks_is_2_encrypt:
|
|
||||||
INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
|
|
||||||
%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 6, 78, enc
|
|
||||||
sub $32, %r13
|
|
||||||
jmp _initial_blocks_encrypted
|
|
||||||
_initial_num_blocks_is_1_encrypt:
|
|
||||||
INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
|
|
||||||
%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 7, 8, enc
|
|
||||||
sub $16, %r13
|
|
||||||
jmp _initial_blocks_encrypted
|
|
||||||
_initial_num_blocks_is_0_encrypt:
|
|
||||||
INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
|
|
||||||
%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 8, 0, enc
|
|
||||||
_initial_blocks_encrypted:
|
|
||||||
|
|
||||||
# Main loop - Encrypt remaining blocks
|
|
||||||
|
|
||||||
cmp $0, %r13
|
|
||||||
je _zero_cipher_left_encrypt
|
|
||||||
sub $64, %r13
|
|
||||||
je _four_cipher_left_encrypt
|
|
||||||
_encrypt_by_4_encrypt:
|
|
||||||
GHASH_4_ENCRYPT_4_PARALLEL_ENC %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, \
|
|
||||||
%xmm14, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, enc
|
|
||||||
add $64, %r11
|
|
||||||
sub $64, %r13
|
|
||||||
jne _encrypt_by_4_encrypt
|
|
||||||
_four_cipher_left_encrypt:
|
|
||||||
GHASH_LAST_4 %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, \
|
|
||||||
%xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm8
|
|
||||||
_zero_cipher_left_encrypt:
|
|
||||||
mov %arg4, %r13
|
|
||||||
and $15, %r13 # %r13 = arg4 (mod 16)
|
|
||||||
je _multiple_of_16_bytes_encrypt
|
|
||||||
|
|
||||||
# Handle the last <16 Byte block separately
|
|
||||||
paddd ONE(%rip), %xmm0 # INCR CNT to get Yn
|
|
||||||
movdqa SHUF_MASK(%rip), %xmm10
|
|
||||||
PSHUFB_XMM %xmm10, %xmm0
|
|
||||||
|
|
||||||
ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # Encrypt(K, Yn)
|
|
||||||
|
|
||||||
lea (%arg3,%r11,1), %r10
|
|
||||||
mov %r13, %r12
|
|
||||||
READ_PARTIAL_BLOCK %r10 %r12 %xmm2 %xmm1
|
|
||||||
|
|
||||||
lea ALL_F+16(%rip), %r12
|
|
||||||
sub %r13, %r12
|
|
||||||
pxor %xmm1, %xmm0 # Plaintext XOR Encrypt(K, Yn)
|
|
||||||
movdqu (%r12), %xmm1
|
|
||||||
# get the appropriate mask to mask out top 16-r13 bytes of xmm0
|
|
||||||
pand %xmm1, %xmm0 # mask out top 16-r13 bytes of xmm0
|
|
||||||
movdqa SHUF_MASK(%rip), %xmm10
|
|
||||||
PSHUFB_XMM %xmm10,%xmm0
|
|
||||||
|
|
||||||
pxor %xmm0, %xmm8
|
|
||||||
GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
|
|
||||||
# GHASH computation for the last <16 byte block
|
|
||||||
movdqa SHUF_MASK(%rip), %xmm10
|
|
||||||
PSHUFB_XMM %xmm10, %xmm0
|
|
||||||
|
|
||||||
# shuffle xmm0 back to output as ciphertext
|
|
||||||
|
|
||||||
# Output %r13 bytes
|
|
||||||
MOVQ_R64_XMM %xmm0, %rax
|
|
||||||
cmp $8, %r13
|
|
||||||
jle _less_than_8_bytes_left_encrypt
|
|
||||||
mov %rax, (%arg2 , %r11, 1)
|
|
||||||
add $8, %r11
|
|
||||||
psrldq $8, %xmm0
|
|
||||||
MOVQ_R64_XMM %xmm0, %rax
|
|
||||||
sub $8, %r13
|
|
||||||
_less_than_8_bytes_left_encrypt:
|
|
||||||
mov %al, (%arg2, %r11, 1)
|
|
||||||
add $1, %r11
|
|
||||||
shr $8, %rax
|
|
||||||
sub $1, %r13
|
|
||||||
jne _less_than_8_bytes_left_encrypt
|
|
||||||
_multiple_of_16_bytes_encrypt:
|
|
||||||
_return_T_encrypt:
|
|
||||||
GCM_COMPLETE
|
GCM_COMPLETE
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
ret
|
||||||
|
|
|
||||||
Загрузка…
Ссылка в новой задаче