microsoft
/
WSL2-Linux-Kernel
зеркало из https://github.com/microsoft/WSL2-Linux-Kernel.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

MIPS: lib: csum_partial: Add macro to build csum_partial symbols 

In preparation for EVA support, we use a macro to build the
__csum_partial_copy_user main code so it can be shared across
multiple implementations. EVA uses the same code but it replaces
the load/store/prefetch instructions with the EVA specific ones
therefore using a macro avoids unnecessary code duplications.

Signed-off-by: Markos Chandras <markos.chandras@imgtec.com>
This commit is contained in:
Markos Chandras 2014-01-17 10:48:46 +00:00 коммит произвёл Ralf Baechle
Родитель 2ab82e6648
Коммит e89fb56c8b
1 изменённых файлов: 108 добавлений и 92 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

200
arch/mips/lib/csum_partial.S
Просмотреть файл

@ -331,6 +331,10 @@ LEAF(csum_partial)
/* Instruction type */ /* Instruction type */
#define LD_INSN 1 #define LD_INSN 1
#define ST_INSN 2 #define ST_INSN 2
#define LEGACY_MODE 1
#define EVA_MODE 2
#define USEROP 1
#define KERNELOP 2
/* /*
* Wrapper to add an entry in the exception table * Wrapper to add an entry in the exception table
@ -343,10 +347,12 @@ LEAF(csum_partial)
* handler : Exception handler * handler : Exception handler
*/ */
#define EXC(insn, type, reg, addr, handler) \ #define EXC(insn, type, reg, addr, handler) \
9: insn reg, addr; \ .if \mode == LEGACY_MODE; \
.section __ex_table,"a"; \ 9: insn reg, addr; \
PTR 9b, handler; \ .section __ex_table,"a"; \
.previous PTR 9b, handler; \
.previous; \
.endif
#undef LOAD #undef LOAD
@ -419,16 +425,20 @@ LEAF(csum_partial)
.set at=v1 .set at=v1
#endif #endif
LEAF(__csum_partial_copy_kernel) .macro __BUILD_CSUM_PARTIAL_COPY_USER mode, from, to, __nocheck
FEXPORT(__csum_partial_copy_to_user)
FEXPORT(__csum_partial_copy_from_user)
PTR_ADDU AT, src, len /* See (1) above. */ PTR_ADDU AT, src, len /* See (1) above. */
/* initialize __nocheck if this the first time we execute this
* macro
*/
#ifdef CONFIG_64BIT #ifdef CONFIG_64BIT
move errptr, a4 move errptr, a4
#else #else
lw errptr, 16(sp) lw errptr, 16(sp)
#endif #endif
FEXPORT(csum_partial_copy_nocheck) .if \__nocheck == 1
FEXPORT(csum_partial_copy_nocheck)
.endif
move sum, zero move sum, zero
move odd, zero move odd, zero
/* /*
@ -444,48 +454,48 @@ FEXPORT(csum_partial_copy_nocheck)
*/ */
sltu t2, len, NBYTES sltu t2, len, NBYTES
and t1, dst, ADDRMASK and t1, dst, ADDRMASK
bnez t2, .Lcopy_bytes_checklen bnez t2, .Lcopy_bytes_checklen\@
and t0, src, ADDRMASK and t0, src, ADDRMASK
andi odd, dst, 0x1 /* odd buffer? */ andi odd, dst, 0x1 /* odd buffer? */
bnez t1, .Ldst_unaligned bnez t1, .Ldst_unaligned\@
nop nop
bnez t0, .Lsrc_unaligned_dst_aligned bnez t0, .Lsrc_unaligned_dst_aligned\@
/* /*
* use delay slot for fall-through * use delay slot for fall-through
* src and dst are aligned; need to compute rem * src and dst are aligned; need to compute rem
*/ */
.Lboth_aligned: .Lboth_aligned\@:
SRL t0, len, LOG_NBYTES+3 # +3 for 8 units/iter SRL t0, len, LOG_NBYTES+3 # +3 for 8 units/iter
beqz t0, .Lcleanup_both_aligned # len < 8*NBYTES beqz t0, .Lcleanup_both_aligned\@ # len < 8*NBYTES
nop nop
SUB len, 8*NBYTES # subtract here for bgez loop SUB len, 8*NBYTES # subtract here for bgez loop
.align 4 .align 4
1: 1:
LOAD(t0, UNIT(0)(src), .Ll_exc) LOAD(t0, UNIT(0)(src), .Ll_exc\@)
LOAD(t1, UNIT(1)(src), .Ll_exc_copy) LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@)
LOAD(t2, UNIT(2)(src), .Ll_exc_copy) LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@)
LOAD(t3, UNIT(3)(src), .Ll_exc_copy) LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@)
LOAD(t4, UNIT(4)(src), .Ll_exc_copy) LOAD(t4, UNIT(4)(src), .Ll_exc_copy\@)
LOAD(t5, UNIT(5)(src), .Ll_exc_copy) LOAD(t5, UNIT(5)(src), .Ll_exc_copy\@)
LOAD(t6, UNIT(6)(src), .Ll_exc_copy) LOAD(t6, UNIT(6)(src), .Ll_exc_copy\@)
LOAD(t7, UNIT(7)(src), .Ll_exc_copy) LOAD(t7, UNIT(7)(src), .Ll_exc_copy\@)
SUB len, len, 8*NBYTES SUB len, len, 8*NBYTES
ADD src, src, 8*NBYTES ADD src, src, 8*NBYTES
STORE(t0, UNIT(0)(dst), .Ls_exc) STORE(t0, UNIT(0)(dst), .Ls_exc\@)
ADDC(sum, t0) ADDC(sum, t0)
STORE(t1, UNIT(1)(dst), .Ls_exc) STORE(t1, UNIT(1)(dst), .Ls_exc\@)
ADDC(sum, t1) ADDC(sum, t1)
STORE(t2, UNIT(2)(dst), .Ls_exc) STORE(t2, UNIT(2)(dst), .Ls_exc\@)
ADDC(sum, t2) ADDC(sum, t2)
STORE(t3, UNIT(3)(dst), .Ls_exc) STORE(t3, UNIT(3)(dst), .Ls_exc\@)
ADDC(sum, t3) ADDC(sum, t3)
STORE(t4, UNIT(4)(dst), .Ls_exc) STORE(t4, UNIT(4)(dst), .Ls_exc\@)
ADDC(sum, t4) ADDC(sum, t4)
STORE(t5, UNIT(5)(dst), .Ls_exc) STORE(t5, UNIT(5)(dst), .Ls_exc\@)
ADDC(sum, t5) ADDC(sum, t5)
STORE(t6, UNIT(6)(dst), .Ls_exc) STORE(t6, UNIT(6)(dst), .Ls_exc\@)
ADDC(sum, t6) ADDC(sum, t6)
STORE(t7, UNIT(7)(dst), .Ls_exc) STORE(t7, UNIT(7)(dst), .Ls_exc\@)
ADDC(sum, t7) ADDC(sum, t7)
.set reorder /* DADDI_WAR */ .set reorder /* DADDI_WAR */
ADD dst, dst, 8*NBYTES ADD dst, dst, 8*NBYTES
@ -496,44 +506,44 @@ FEXPORT(csum_partial_copy_nocheck)
/* /*
* len == the number of bytes left to copy < 8*NBYTES * len == the number of bytes left to copy < 8*NBYTES
*/ */
.Lcleanup_both_aligned: .Lcleanup_both_aligned\@:
#define rem t7 #define rem t7
beqz len, .Ldone beqz len, .Ldone\@
sltu t0, len, 4*NBYTES sltu t0, len, 4*NBYTES
bnez t0, .Lless_than_4units bnez t0, .Lless_than_4units\@
and rem, len, (NBYTES-1) # rem = len % NBYTES and rem, len, (NBYTES-1) # rem = len % NBYTES
/* /*
* len >= 4*NBYTES * len >= 4*NBYTES
*/ */
LOAD(t0, UNIT(0)(src), .Ll_exc) LOAD(t0, UNIT(0)(src), .Ll_exc\@)
LOAD(t1, UNIT(1)(src), .Ll_exc_copy) LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@)
LOAD(t2, UNIT(2)(src), .Ll_exc_copy) LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@)
LOAD(t3, UNIT(3)(src), .Ll_exc_copy) LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@)
SUB len, len, 4*NBYTES SUB len, len, 4*NBYTES
ADD src, src, 4*NBYTES ADD src, src, 4*NBYTES
STORE(t0, UNIT(0)(dst), .Ls_exc) STORE(t0, UNIT(0)(dst), .Ls_exc\@)
ADDC(sum, t0) ADDC(sum, t0)
STORE(t1, UNIT(1)(dst), .Ls_exc) STORE(t1, UNIT(1)(dst), .Ls_exc\@)
ADDC(sum, t1) ADDC(sum, t1)
STORE(t2, UNIT(2)(dst), .Ls_exc) STORE(t2, UNIT(2)(dst), .Ls_exc\@)
ADDC(sum, t2) ADDC(sum, t2)
STORE(t3, UNIT(3)(dst), .Ls_exc) STORE(t3, UNIT(3)(dst), .Ls_exc\@)
ADDC(sum, t3) ADDC(sum, t3)
.set reorder /* DADDI_WAR */ .set reorder /* DADDI_WAR */
ADD dst, dst, 4*NBYTES ADD dst, dst, 4*NBYTES
beqz len, .Ldone beqz len, .Ldone\@
.set noreorder .set noreorder
.Lless_than_4units: .Lless_than_4units\@:
/* /*
* rem = len % NBYTES * rem = len % NBYTES
*/ */
beq rem, len, .Lcopy_bytes beq rem, len, .Lcopy_bytes\@
nop nop
1: 1:
LOAD(t0, 0(src), .Ll_exc) LOAD(t0, 0(src), .Ll_exc\@)
ADD src, src, NBYTES ADD src, src, NBYTES
SUB len, len, NBYTES SUB len, len, NBYTES
STORE(t0, 0(dst), .Ls_exc) STORE(t0, 0(dst), .Ls_exc\@)
ADDC(sum, t0) ADDC(sum, t0)
.set reorder /* DADDI_WAR */ .set reorder /* DADDI_WAR */
ADD dst, dst, NBYTES ADD dst, dst, NBYTES
@ -552,20 +562,20 @@ FEXPORT(csum_partial_copy_nocheck)
* more instruction-level parallelism. * more instruction-level parallelism.
*/ */
#define bits t2 #define bits t2
beqz len, .Ldone beqz len, .Ldone\@
ADD t1, dst, len # t1 is just past last byte of dst ADD t1, dst, len # t1 is just past last byte of dst
li bits, 8*NBYTES li bits, 8*NBYTES
SLL rem, len, 3 # rem = number of bits to keep SLL rem, len, 3 # rem = number of bits to keep
LOAD(t0, 0(src), .Ll_exc) LOAD(t0, 0(src), .Ll_exc\@)
SUB bits, bits, rem # bits = number of bits to discard SUB bits, bits, rem # bits = number of bits to discard
SHIFT_DISCARD t0, t0, bits SHIFT_DISCARD t0, t0, bits
STREST(t0, -1(t1), .Ls_exc) STREST(t0, -1(t1), .Ls_exc\@)
SHIFT_DISCARD_REVERT t0, t0, bits SHIFT_DISCARD_REVERT t0, t0, bits
.set reorder .set reorder
ADDC(sum, t0) ADDC(sum, t0)
b .Ldone b .Ldone\@
.set noreorder .set noreorder
.Ldst_unaligned: .Ldst_unaligned\@:
/* /*
* dst is unaligned * dst is unaligned
* t0 = src & ADDRMASK * t0 = src & ADDRMASK
@ -576,25 +586,25 @@ FEXPORT(csum_partial_copy_nocheck)
* Set match = (src and dst have same alignment) * Set match = (src and dst have same alignment)
*/ */
#define match rem #define match rem
LDFIRST(t3, FIRST(0)(src), .Ll_exc) LDFIRST(t3, FIRST(0)(src), .Ll_exc\@)
ADD t2, zero, NBYTES ADD t2, zero, NBYTES
LDREST(t3, REST(0)(src), .Ll_exc_copy) LDREST(t3, REST(0)(src), .Ll_exc_copy\@)
SUB t2, t2, t1 # t2 = number of bytes copied SUB t2, t2, t1 # t2 = number of bytes copied
xor match, t0, t1 xor match, t0, t1
STFIRST(t3, FIRST(0)(dst), .Ls_exc) STFIRST(t3, FIRST(0)(dst), .Ls_exc\@)
SLL t4, t1, 3 # t4 = number of bits to discard SLL t4, t1, 3 # t4 = number of bits to discard
SHIFT_DISCARD t3, t3, t4 SHIFT_DISCARD t3, t3, t4
/* no SHIFT_DISCARD_REVERT to handle odd buffer properly */ /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
ADDC(sum, t3) ADDC(sum, t3)
beq len, t2, .Ldone beq len, t2, .Ldone\@
SUB len, len, t2 SUB len, len, t2
ADD dst, dst, t2 ADD dst, dst, t2
beqz match, .Lboth_aligned beqz match, .Lboth_aligned\@
ADD src, src, t2 ADD src, src, t2
.Lsrc_unaligned_dst_aligned: .Lsrc_unaligned_dst_aligned\@:
SRL t0, len, LOG_NBYTES+2 # +2 for 4 units/iter SRL t0, len, LOG_NBYTES+2 # +2 for 4 units/iter
beqz t0, .Lcleanup_src_unaligned beqz t0, .Lcleanup_src_unaligned\@
and rem, len, (4*NBYTES-1) # rem = len % 4*NBYTES and rem, len, (4*NBYTES-1) # rem = len % 4*NBYTES
1: 1:
/* /*
@ -603,53 +613,53 @@ FEXPORT(csum_partial_copy_nocheck)
* It's OK to load FIRST(N+1) before REST(N) because the two addresses * It's OK to load FIRST(N+1) before REST(N) because the two addresses
* are to the same unit (unless src is aligned, but it's not). * are to the same unit (unless src is aligned, but it's not).
*/ */
LDFIRST(t0, FIRST(0)(src), .Ll_exc) LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy) LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy\@)
SUB len, len, 4*NBYTES SUB len, len, 4*NBYTES
LDREST(t0, REST(0)(src), .Ll_exc_copy) LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
LDREST(t1, REST(1)(src), .Ll_exc_copy) LDREST(t1, REST(1)(src), .Ll_exc_copy\@)
LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy) LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy\@)
LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy) LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy\@)
LDREST(t2, REST(2)(src), .Ll_exc_copy) LDREST(t2, REST(2)(src), .Ll_exc_copy\@)
LDREST(t3, REST(3)(src), .Ll_exc_copy) LDREST(t3, REST(3)(src), .Ll_exc_copy\@)
ADD src, src, 4*NBYTES ADD src, src, 4*NBYTES
#ifdef CONFIG_CPU_SB1 #ifdef CONFIG_CPU_SB1
nop # improves slotting nop # improves slotting
#endif #endif
STORE(t0, UNIT(0)(dst), .Ls_exc) STORE(t0, UNIT(0)(dst), .Ls_exc\@)
ADDC(sum, t0) ADDC(sum, t0)
STORE(t1, UNIT(1)(dst), .Ls_exc) STORE(t1, UNIT(1)(dst), .Ls_exc\@)
ADDC(sum, t1) ADDC(sum, t1)
STORE(t2, UNIT(2)(dst), .Ls_exc) STORE(t2, UNIT(2)(dst), .Ls_exc\@)
ADDC(sum, t2) ADDC(sum, t2)
STORE(t3, UNIT(3)(dst), .Ls_exc) STORE(t3, UNIT(3)(dst), .Ls_exc\@)
ADDC(sum, t3) ADDC(sum, t3)
.set reorder /* DADDI_WAR */ .set reorder /* DADDI_WAR */
ADD dst, dst, 4*NBYTES ADD dst, dst, 4*NBYTES
bne len, rem, 1b bne len, rem, 1b
.set noreorder .set noreorder
.Lcleanup_src_unaligned: .Lcleanup_src_unaligned\@:
beqz len, .Ldone beqz len, .Ldone\@
and rem, len, NBYTES-1 # rem = len % NBYTES and rem, len, NBYTES-1 # rem = len % NBYTES
beq rem, len, .Lcopy_bytes beq rem, len, .Lcopy_bytes\@
nop nop
1: 1:
LDFIRST(t0, FIRST(0)(src), .Ll_exc) LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
LDREST(t0, REST(0)(src), .Ll_exc_copy) LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
ADD src, src, NBYTES ADD src, src, NBYTES
SUB len, len, NBYTES SUB len, len, NBYTES
STORE(t0, 0(dst), .Ls_exc) STORE(t0, 0(dst), .Ls_exc\@)
ADDC(sum, t0) ADDC(sum, t0)
.set reorder /* DADDI_WAR */ .set reorder /* DADDI_WAR */
ADD dst, dst, NBYTES ADD dst, dst, NBYTES
bne len, rem, 1b bne len, rem, 1b
.set noreorder .set noreorder
.Lcopy_bytes_checklen: .Lcopy_bytes_checklen\@:
beqz len, .Ldone beqz len, .Ldone\@
nop nop
.Lcopy_bytes: .Lcopy_bytes\@:
/* 0 < len < NBYTES */ /* 0 < len < NBYTES */
#ifdef CONFIG_CPU_LITTLE_ENDIAN #ifdef CONFIG_CPU_LITTLE_ENDIAN
#define SHIFT_START 0 #define SHIFT_START 0
@ -662,12 +672,12 @@ FEXPORT(csum_partial_copy_nocheck)
li t3, SHIFT_START # shift li t3, SHIFT_START # shift
/* use .Ll_exc_copy here to return correct sum on fault */ /* use .Ll_exc_copy here to return correct sum on fault */
#define COPY_BYTE(N) \ #define COPY_BYTE(N) \
LOADBU(t0, N(src), .Ll_exc_copy); \ LOADBU(t0, N(src), .Ll_exc_copy\@); \
SUB len, len, 1; \ SUB len, len, 1; \
STOREB(t0, N(dst), .Ls_exc); \ STOREB(t0, N(dst), .Ls_exc\@); \
SLLV t0, t0, t3; \ SLLV t0, t0, t3; \
addu t3, SHIFT_INC; \ addu t3, SHIFT_INC; \
beqz len, .Lcopy_bytes_done; \ beqz len, .Lcopy_bytes_done\@; \
or t2, t0 or t2, t0
COPY_BYTE(0) COPY_BYTE(0)
@ -678,14 +688,14 @@ FEXPORT(csum_partial_copy_nocheck)
COPY_BYTE(4) COPY_BYTE(4)
COPY_BYTE(5) COPY_BYTE(5)
#endif #endif
LOADBU(t0, NBYTES-2(src), .Ll_exc_copy) LOADBU(t0, NBYTES-2(src), .Ll_exc_copy\@)
SUB len, len, 1 SUB len, len, 1
STOREB(t0, NBYTES-2(dst), .Ls_exc) STOREB(t0, NBYTES-2(dst), .Ls_exc\@)
SLLV t0, t0, t3 SLLV t0, t0, t3
or t2, t0 or t2, t0
.Lcopy_bytes_done: .Lcopy_bytes_done\@:
ADDC(sum, t2) ADDC(sum, t2)
.Ldone: .Ldone\@:
/* fold checksum */ /* fold checksum */
#ifdef USE_DOUBLE #ifdef USE_DOUBLE
dsll32 v1, sum, 0 dsll32 v1, sum, 0
@ -714,7 +724,7 @@ FEXPORT(csum_partial_copy_nocheck)
jr ra jr ra
.set noreorder .set noreorder
.Ll_exc_copy: .Ll_exc_copy\@:
/* /*
* Copy bytes from src until faulting load address (or until a * Copy bytes from src until faulting load address (or until a
* lb faults) * lb faults)
@ -729,7 +739,7 @@ FEXPORT(csum_partial_copy_nocheck)
li t2, SHIFT_START li t2, SHIFT_START
LOADK t0, THREAD_BUADDR(t0) LOADK t0, THREAD_BUADDR(t0)
1: 1:
LOADBU(t1, 0(src), .Ll_exc) LOADBU(t1, 0(src), .Ll_exc\@)
ADD src, src, 1 ADD src, src, 1
sb t1, 0(dst) # can't fault -- we're copy_from_user sb t1, 0(dst) # can't fault -- we're copy_from_user
SLLV t1, t1, t2 SLLV t1, t1, t2
@ -739,7 +749,7 @@ FEXPORT(csum_partial_copy_nocheck)
ADD dst, dst, 1 ADD dst, dst, 1
bne src, t0, 1b bne src, t0, 1b
.set noreorder .set noreorder
.Ll_exc: .Ll_exc\@:
LOADK t0, TI_TASK($28) LOADK t0, TI_TASK($28)
nop nop
LOADK t0, THREAD_BUADDR(t0) # t0 is just past last good address LOADK t0, THREAD_BUADDR(t0) # t0 is just past last good address
@ -758,7 +768,7 @@ FEXPORT(csum_partial_copy_nocheck)
*/ */
.set reorder /* DADDI_WAR */ .set reorder /* DADDI_WAR */
SUB src, len, 1 SUB src, len, 1
beqz len, .Ldone beqz len, .Ldone\@
.set noreorder .set noreorder
1: sb zero, 0(dst) 1: sb zero, 0(dst)
ADD dst, dst, 1 ADD dst, dst, 1
@ -773,13 +783,19 @@ FEXPORT(csum_partial_copy_nocheck)
SUB src, src, v1 SUB src, src, v1
#endif #endif
li v1, -EFAULT li v1, -EFAULT
b .Ldone b .Ldone\@
sw v1, (errptr) sw v1, (errptr)
.Ls_exc: .Ls_exc\@:
li v0, -1 /* invalid checksum */ li v0, -1 /* invalid checksum */
li v1, -EFAULT li v1, -EFAULT
jr ra jr ra
sw v1, (errptr) sw v1, (errptr)
.set pop .set pop
END(__csum_partial_copy_kernel) .endm
LEAF(__csum_partial_copy_kernel)
FEXPORT(__csum_partial_copy_to_user)
FEXPORT(__csum_partial_copy_from_user)
__BUILD_CSUM_PARTIAL_COPY_USER LEGACY_MODE USEROP USEROP 1
END(__csum_partial_copy_kernel)