x86/asm/entry: Do mass removal of 'ARGOFFSET'
ARGOFFSET is zero now, removing it changes no code. A few macros lost "offset" parameter, since it is always zero now too. No code changes - verified with objdump. Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com> Signed-off-by: Andy Lutomirski <luto@amacapital.net> Cc: Alexei Starovoitov <ast@plumgrid.com> Cc: Borislav Petkov <bp@alien8.de> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Kees Cook <keescook@chromium.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Will Drewry <wad@chromium.org> Link: http://lkml.kernel.org/r/8689f937622d9d2db0ab8be82331fa15e4ed4713.1424989793.git.luto@amacapital.net Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
Родитель
0d55083698
Коммит
f2db9382c1
|
@ -41,13 +41,13 @@
|
||||||
movl %edx,%edx /* zero extension */
|
movl %edx,%edx /* zero extension */
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
/* clobbers %eax */
|
/* clobbers %rax */
|
||||||
.macro CLEAR_RREGS offset=0, _r9=rax
|
.macro CLEAR_RREGS _r9=rax
|
||||||
xorl %eax,%eax
|
xorl %eax,%eax
|
||||||
movq %rax,\offset+R11(%rsp)
|
movq %rax,R11(%rsp)
|
||||||
movq %rax,\offset+R10(%rsp)
|
movq %rax,R10(%rsp)
|
||||||
movq %\_r9,\offset+R9(%rsp)
|
movq %\_r9,R9(%rsp)
|
||||||
movq %rax,\offset+R8(%rsp)
|
movq %rax,R8(%rsp)
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -60,14 +60,14 @@
|
||||||
* If it's -1 to make us punt the syscall, then (u32)-1 is still
|
* If it's -1 to make us punt the syscall, then (u32)-1 is still
|
||||||
* an appropriately invalid value.
|
* an appropriately invalid value.
|
||||||
*/
|
*/
|
||||||
.macro LOAD_ARGS32 offset, _r9=0
|
.macro LOAD_ARGS32 _r9=0
|
||||||
.if \_r9
|
.if \_r9
|
||||||
movl \offset+R9(%rsp),%r9d
|
movl R9(%rsp),%r9d
|
||||||
.endif
|
.endif
|
||||||
movl \offset+RCX(%rsp),%ecx
|
movl RCX(%rsp),%ecx
|
||||||
movl \offset+RDX(%rsp),%edx
|
movl RDX(%rsp),%edx
|
||||||
movl \offset+RSI(%rsp),%esi
|
movl RSI(%rsp),%esi
|
||||||
movl \offset+RDI(%rsp),%edi
|
movl RDI(%rsp),%edi
|
||||||
movl %eax,%eax /* zero extension */
|
movl %eax,%eax /* zero extension */
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
|
@ -158,12 +158,12 @@ ENTRY(ia32_sysenter_target)
|
||||||
* ourselves. To save a few cycles, we can check whether
|
* ourselves. To save a few cycles, we can check whether
|
||||||
* NT was set instead of doing an unconditional popfq.
|
* NT was set instead of doing an unconditional popfq.
|
||||||
*/
|
*/
|
||||||
testl $X86_EFLAGS_NT,EFLAGS-ARGOFFSET(%rsp)
|
testl $X86_EFLAGS_NT,EFLAGS(%rsp)
|
||||||
jnz sysenter_fix_flags
|
jnz sysenter_fix_flags
|
||||||
sysenter_flags_fixed:
|
sysenter_flags_fixed:
|
||||||
|
|
||||||
orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
|
||||||
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
|
||||||
CFI_REMEMBER_STATE
|
CFI_REMEMBER_STATE
|
||||||
jnz sysenter_tracesys
|
jnz sysenter_tracesys
|
||||||
cmpq $(IA32_NR_syscalls-1),%rax
|
cmpq $(IA32_NR_syscalls-1),%rax
|
||||||
|
@ -172,16 +172,16 @@ sysenter_do_call:
|
||||||
IA32_ARG_FIXUP
|
IA32_ARG_FIXUP
|
||||||
sysenter_dispatch:
|
sysenter_dispatch:
|
||||||
call *ia32_sys_call_table(,%rax,8)
|
call *ia32_sys_call_table(,%rax,8)
|
||||||
movq %rax,RAX-ARGOFFSET(%rsp)
|
movq %rax,RAX(%rsp)
|
||||||
DISABLE_INTERRUPTS(CLBR_NONE)
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
||||||
TRACE_IRQS_OFF
|
TRACE_IRQS_OFF
|
||||||
testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP)
|
||||||
jnz sysexit_audit
|
jnz sysexit_audit
|
||||||
sysexit_from_sys_call:
|
sysexit_from_sys_call:
|
||||||
andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
|
||||||
/* clear IF, that popfq doesn't enable interrupts early */
|
/* clear IF, that popfq doesn't enable interrupts early */
|
||||||
andl $~0x200,EFLAGS-ARGOFFSET(%rsp)
|
andl $~0x200,EFLAGS(%rsp)
|
||||||
movl RIP-ARGOFFSET(%rsp),%edx /* User %eip */
|
movl RIP(%rsp),%edx /* User %eip */
|
||||||
CFI_REGISTER rip,rdx
|
CFI_REGISTER rip,rdx
|
||||||
RESTORE_RSI_RDI
|
RESTORE_RSI_RDI
|
||||||
REMOVE_PT_GPREGS_FROM_STACK 3*8
|
REMOVE_PT_GPREGS_FROM_STACK 3*8
|
||||||
|
@ -207,18 +207,18 @@ sysexit_from_sys_call:
|
||||||
movl %ebx,%esi /* 2nd arg: 1st syscall arg */
|
movl %ebx,%esi /* 2nd arg: 1st syscall arg */
|
||||||
movl %eax,%edi /* 1st arg: syscall number */
|
movl %eax,%edi /* 1st arg: syscall number */
|
||||||
call __audit_syscall_entry
|
call __audit_syscall_entry
|
||||||
movl RAX-ARGOFFSET(%rsp),%eax /* reload syscall number */
|
movl RAX(%rsp),%eax /* reload syscall number */
|
||||||
cmpq $(IA32_NR_syscalls-1),%rax
|
cmpq $(IA32_NR_syscalls-1),%rax
|
||||||
ja ia32_badsys
|
ja ia32_badsys
|
||||||
movl %ebx,%edi /* reload 1st syscall arg */
|
movl %ebx,%edi /* reload 1st syscall arg */
|
||||||
movl RCX-ARGOFFSET(%rsp),%esi /* reload 2nd syscall arg */
|
movl RCX(%rsp),%esi /* reload 2nd syscall arg */
|
||||||
movl RDX-ARGOFFSET(%rsp),%edx /* reload 3rd syscall arg */
|
movl RDX(%rsp),%edx /* reload 3rd syscall arg */
|
||||||
movl RSI-ARGOFFSET(%rsp),%ecx /* reload 4th syscall arg */
|
movl RSI(%rsp),%ecx /* reload 4th syscall arg */
|
||||||
movl RDI-ARGOFFSET(%rsp),%r8d /* reload 5th syscall arg */
|
movl RDI(%rsp),%r8d /* reload 5th syscall arg */
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
.macro auditsys_exit exit
|
.macro auditsys_exit exit
|
||||||
testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP)
|
||||||
jnz ia32_ret_from_sys_call
|
jnz ia32_ret_from_sys_call
|
||||||
TRACE_IRQS_ON
|
TRACE_IRQS_ON
|
||||||
ENABLE_INTERRUPTS(CLBR_NONE)
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
||||||
|
@ -229,13 +229,13 @@ sysexit_from_sys_call:
|
||||||
1: setbe %al /* 1 if error, 0 if not */
|
1: setbe %al /* 1 if error, 0 if not */
|
||||||
movzbl %al,%edi /* zero-extend that into %edi */
|
movzbl %al,%edi /* zero-extend that into %edi */
|
||||||
call __audit_syscall_exit
|
call __audit_syscall_exit
|
||||||
movq RAX-ARGOFFSET(%rsp),%rax /* reload syscall return value */
|
movq RAX(%rsp),%rax /* reload syscall return value */
|
||||||
movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
|
movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
|
||||||
DISABLE_INTERRUPTS(CLBR_NONE)
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
||||||
TRACE_IRQS_OFF
|
TRACE_IRQS_OFF
|
||||||
testl %edi,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
testl %edi,TI_flags+THREAD_INFO(%rsp,RIP)
|
||||||
jz \exit
|
jz \exit
|
||||||
CLEAR_RREGS -ARGOFFSET
|
CLEAR_RREGS
|
||||||
jmp int_with_check
|
jmp int_with_check
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
|
@ -255,7 +255,7 @@ sysenter_fix_flags:
|
||||||
|
|
||||||
sysenter_tracesys:
|
sysenter_tracesys:
|
||||||
#ifdef CONFIG_AUDITSYSCALL
|
#ifdef CONFIG_AUDITSYSCALL
|
||||||
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP)
|
||||||
jz sysenter_auditsys
|
jz sysenter_auditsys
|
||||||
#endif
|
#endif
|
||||||
SAVE_EXTRA_REGS
|
SAVE_EXTRA_REGS
|
||||||
|
@ -263,7 +263,7 @@ sysenter_tracesys:
|
||||||
movq $-ENOSYS,RAX(%rsp)/* ptrace can change this for a bad syscall */
|
movq $-ENOSYS,RAX(%rsp)/* ptrace can change this for a bad syscall */
|
||||||
movq %rsp,%rdi /* &pt_regs -> arg1 */
|
movq %rsp,%rdi /* &pt_regs -> arg1 */
|
||||||
call syscall_trace_enter
|
call syscall_trace_enter
|
||||||
LOAD_ARGS32 ARGOFFSET /* reload args from stack in case ptrace changed it */
|
LOAD_ARGS32 /* reload args from stack in case ptrace changed it */
|
||||||
RESTORE_EXTRA_REGS
|
RESTORE_EXTRA_REGS
|
||||||
cmpq $(IA32_NR_syscalls-1),%rax
|
cmpq $(IA32_NR_syscalls-1),%rax
|
||||||
ja int_ret_from_sys_call /* sysenter_tracesys has set RAX(%rsp) */
|
ja int_ret_from_sys_call /* sysenter_tracesys has set RAX(%rsp) */
|
||||||
|
@ -309,17 +309,17 @@ ENTRY(ia32_cstar_target)
|
||||||
ALLOC_PT_GPREGS_ON_STACK 8
|
ALLOC_PT_GPREGS_ON_STACK 8
|
||||||
SAVE_C_REGS_EXCEPT_RCX_R891011
|
SAVE_C_REGS_EXCEPT_RCX_R891011
|
||||||
movl %eax,%eax /* zero extension */
|
movl %eax,%eax /* zero extension */
|
||||||
movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
|
movq %rax,ORIG_RAX(%rsp)
|
||||||
movq %rcx,RIP-ARGOFFSET(%rsp)
|
movq %rcx,RIP(%rsp)
|
||||||
CFI_REL_OFFSET rip,RIP-ARGOFFSET
|
CFI_REL_OFFSET rip,RIP
|
||||||
movq %rbp,RCX-ARGOFFSET(%rsp) /* this lies slightly to ptrace */
|
movq %rbp,RCX(%rsp) /* this lies slightly to ptrace */
|
||||||
movl %ebp,%ecx
|
movl %ebp,%ecx
|
||||||
movq $__USER32_CS,CS-ARGOFFSET(%rsp)
|
movq $__USER32_CS,CS(%rsp)
|
||||||
movq $__USER32_DS,SS-ARGOFFSET(%rsp)
|
movq $__USER32_DS,SS(%rsp)
|
||||||
movq %r11,EFLAGS-ARGOFFSET(%rsp)
|
movq %r11,EFLAGS(%rsp)
|
||||||
/*CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
|
/*CFI_REL_OFFSET rflags,EFLAGS*/
|
||||||
movq %r8,RSP-ARGOFFSET(%rsp)
|
movq %r8,RSP(%rsp)
|
||||||
CFI_REL_OFFSET rsp,RSP-ARGOFFSET
|
CFI_REL_OFFSET rsp,RSP
|
||||||
/* no need to do an access_ok check here because r8 has been
|
/* no need to do an access_ok check here because r8 has been
|
||||||
32bit zero extended */
|
32bit zero extended */
|
||||||
/* hardware stack frame is complete now */
|
/* hardware stack frame is complete now */
|
||||||
|
@ -327,8 +327,8 @@ ENTRY(ia32_cstar_target)
|
||||||
1: movl (%r8),%r9d
|
1: movl (%r8),%r9d
|
||||||
_ASM_EXTABLE(1b,ia32_badarg)
|
_ASM_EXTABLE(1b,ia32_badarg)
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
|
||||||
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
|
||||||
CFI_REMEMBER_STATE
|
CFI_REMEMBER_STATE
|
||||||
jnz cstar_tracesys
|
jnz cstar_tracesys
|
||||||
cmpq $IA32_NR_syscalls-1,%rax
|
cmpq $IA32_NR_syscalls-1,%rax
|
||||||
|
@ -337,32 +337,32 @@ cstar_do_call:
|
||||||
IA32_ARG_FIXUP 1
|
IA32_ARG_FIXUP 1
|
||||||
cstar_dispatch:
|
cstar_dispatch:
|
||||||
call *ia32_sys_call_table(,%rax,8)
|
call *ia32_sys_call_table(,%rax,8)
|
||||||
movq %rax,RAX-ARGOFFSET(%rsp)
|
movq %rax,RAX(%rsp)
|
||||||
DISABLE_INTERRUPTS(CLBR_NONE)
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
||||||
TRACE_IRQS_OFF
|
TRACE_IRQS_OFF
|
||||||
testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP)
|
||||||
jnz sysretl_audit
|
jnz sysretl_audit
|
||||||
sysretl_from_sys_call:
|
sysretl_from_sys_call:
|
||||||
andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
|
||||||
RESTORE_RSI_RDI_RDX
|
RESTORE_RSI_RDI_RDX
|
||||||
movl RIP-ARGOFFSET(%rsp),%ecx
|
movl RIP(%rsp),%ecx
|
||||||
CFI_REGISTER rip,rcx
|
CFI_REGISTER rip,rcx
|
||||||
movl EFLAGS-ARGOFFSET(%rsp),%r11d
|
movl EFLAGS(%rsp),%r11d
|
||||||
/*CFI_REGISTER rflags,r11*/
|
/*CFI_REGISTER rflags,r11*/
|
||||||
xorq %r10,%r10
|
xorq %r10,%r10
|
||||||
xorq %r9,%r9
|
xorq %r9,%r9
|
||||||
xorq %r8,%r8
|
xorq %r8,%r8
|
||||||
TRACE_IRQS_ON
|
TRACE_IRQS_ON
|
||||||
movl RSP-ARGOFFSET(%rsp),%esp
|
movl RSP(%rsp),%esp
|
||||||
CFI_RESTORE rsp
|
CFI_RESTORE rsp
|
||||||
USERGS_SYSRET32
|
USERGS_SYSRET32
|
||||||
|
|
||||||
#ifdef CONFIG_AUDITSYSCALL
|
#ifdef CONFIG_AUDITSYSCALL
|
||||||
cstar_auditsys:
|
cstar_auditsys:
|
||||||
CFI_RESTORE_STATE
|
CFI_RESTORE_STATE
|
||||||
movl %r9d,R9-ARGOFFSET(%rsp) /* register to be clobbered by call */
|
movl %r9d,R9(%rsp) /* register to be clobbered by call */
|
||||||
auditsys_entry_common
|
auditsys_entry_common
|
||||||
movl R9-ARGOFFSET(%rsp),%r9d /* reload 6th syscall arg */
|
movl R9(%rsp),%r9d /* reload 6th syscall arg */
|
||||||
jmp cstar_dispatch
|
jmp cstar_dispatch
|
||||||
|
|
||||||
sysretl_audit:
|
sysretl_audit:
|
||||||
|
@ -371,16 +371,16 @@ sysretl_audit:
|
||||||
|
|
||||||
cstar_tracesys:
|
cstar_tracesys:
|
||||||
#ifdef CONFIG_AUDITSYSCALL
|
#ifdef CONFIG_AUDITSYSCALL
|
||||||
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP)
|
||||||
jz cstar_auditsys
|
jz cstar_auditsys
|
||||||
#endif
|
#endif
|
||||||
xchgl %r9d,%ebp
|
xchgl %r9d,%ebp
|
||||||
SAVE_EXTRA_REGS
|
SAVE_EXTRA_REGS
|
||||||
CLEAR_RREGS 0, r9
|
CLEAR_RREGS r9
|
||||||
movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
|
movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
|
||||||
movq %rsp,%rdi /* &pt_regs -> arg1 */
|
movq %rsp,%rdi /* &pt_regs -> arg1 */
|
||||||
call syscall_trace_enter
|
call syscall_trace_enter
|
||||||
LOAD_ARGS32 ARGOFFSET, 1 /* reload args from stack in case ptrace changed it */
|
LOAD_ARGS32 1 /* reload args from stack in case ptrace changed it */
|
||||||
RESTORE_EXTRA_REGS
|
RESTORE_EXTRA_REGS
|
||||||
xchgl %ebp,%r9d
|
xchgl %ebp,%r9d
|
||||||
cmpq $(IA32_NR_syscalls-1),%rax
|
cmpq $(IA32_NR_syscalls-1),%rax
|
||||||
|
@ -438,8 +438,8 @@ ENTRY(ia32_syscall)
|
||||||
this could be a problem. */
|
this could be a problem. */
|
||||||
ALLOC_PT_GPREGS_ON_STACK
|
ALLOC_PT_GPREGS_ON_STACK
|
||||||
SAVE_C_REGS_EXCEPT_R891011
|
SAVE_C_REGS_EXCEPT_R891011
|
||||||
orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
|
||||||
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
|
||||||
jnz ia32_tracesys
|
jnz ia32_tracesys
|
||||||
cmpq $(IA32_NR_syscalls-1),%rax
|
cmpq $(IA32_NR_syscalls-1),%rax
|
||||||
ja ia32_badsys
|
ja ia32_badsys
|
||||||
|
@ -447,9 +447,9 @@ ia32_do_call:
|
||||||
IA32_ARG_FIXUP
|
IA32_ARG_FIXUP
|
||||||
call *ia32_sys_call_table(,%rax,8) # xxx: rip relative
|
call *ia32_sys_call_table(,%rax,8) # xxx: rip relative
|
||||||
ia32_sysret:
|
ia32_sysret:
|
||||||
movq %rax,RAX-ARGOFFSET(%rsp)
|
movq %rax,RAX(%rsp)
|
||||||
ia32_ret_from_sys_call:
|
ia32_ret_from_sys_call:
|
||||||
CLEAR_RREGS -ARGOFFSET
|
CLEAR_RREGS
|
||||||
jmp int_ret_from_sys_call
|
jmp int_ret_from_sys_call
|
||||||
|
|
||||||
ia32_tracesys:
|
ia32_tracesys:
|
||||||
|
@ -458,7 +458,7 @@ ia32_tracesys:
|
||||||
movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
|
movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
|
||||||
movq %rsp,%rdi /* &pt_regs -> arg1 */
|
movq %rsp,%rdi /* &pt_regs -> arg1 */
|
||||||
call syscall_trace_enter
|
call syscall_trace_enter
|
||||||
LOAD_ARGS32 ARGOFFSET /* reload args from stack in case ptrace changed it */
|
LOAD_ARGS32 /* reload args from stack in case ptrace changed it */
|
||||||
RESTORE_EXTRA_REGS
|
RESTORE_EXTRA_REGS
|
||||||
cmpq $(IA32_NR_syscalls-1),%rax
|
cmpq $(IA32_NR_syscalls-1),%rax
|
||||||
ja int_ret_from_sys_call /* ia32_tracesys has set RAX(%rsp) */
|
ja int_ret_from_sys_call /* ia32_tracesys has set RAX(%rsp) */
|
||||||
|
@ -466,7 +466,7 @@ ia32_tracesys:
|
||||||
END(ia32_syscall)
|
END(ia32_syscall)
|
||||||
|
|
||||||
ia32_badsys:
|
ia32_badsys:
|
||||||
movq $0,ORIG_RAX-ARGOFFSET(%rsp)
|
movq $0,ORIG_RAX(%rsp)
|
||||||
movq $-ENOSYS,%rax
|
movq $-ENOSYS,%rax
|
||||||
jmp ia32_sysret
|
jmp ia32_sysret
|
||||||
|
|
||||||
|
@ -499,17 +499,17 @@ ia32_ptregs_common:
|
||||||
CFI_ENDPROC
|
CFI_ENDPROC
|
||||||
CFI_STARTPROC32 simple
|
CFI_STARTPROC32 simple
|
||||||
CFI_SIGNAL_FRAME
|
CFI_SIGNAL_FRAME
|
||||||
CFI_DEF_CFA rsp,SS+8-ARGOFFSET
|
CFI_DEF_CFA rsp,SS+8
|
||||||
CFI_REL_OFFSET rax,RAX-ARGOFFSET
|
CFI_REL_OFFSET rax,RAX
|
||||||
CFI_REL_OFFSET rcx,RCX-ARGOFFSET
|
CFI_REL_OFFSET rcx,RCX
|
||||||
CFI_REL_OFFSET rdx,RDX-ARGOFFSET
|
CFI_REL_OFFSET rdx,RDX
|
||||||
CFI_REL_OFFSET rsi,RSI-ARGOFFSET
|
CFI_REL_OFFSET rsi,RSI
|
||||||
CFI_REL_OFFSET rdi,RDI-ARGOFFSET
|
CFI_REL_OFFSET rdi,RDI
|
||||||
CFI_REL_OFFSET rip,RIP-ARGOFFSET
|
CFI_REL_OFFSET rip,RIP
|
||||||
/* CFI_REL_OFFSET cs,CS-ARGOFFSET*/
|
/* CFI_REL_OFFSET cs,CS*/
|
||||||
/* CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
|
/* CFI_REL_OFFSET rflags,EFLAGS*/
|
||||||
CFI_REL_OFFSET rsp,RSP-ARGOFFSET
|
CFI_REL_OFFSET rsp,RSP
|
||||||
/* CFI_REL_OFFSET ss,SS-ARGOFFSET*/
|
/* CFI_REL_OFFSET ss,SS*/
|
||||||
SAVE_EXTRA_REGS 8
|
SAVE_EXTRA_REGS 8
|
||||||
call *%rax
|
call *%rax
|
||||||
RESTORE_EXTRA_REGS 8
|
RESTORE_EXTRA_REGS 8
|
||||||
|
|
|
@ -88,8 +88,6 @@ For 32-bit we have the following conventions - kernel is built with
|
||||||
#define RSP 19*8
|
#define RSP 19*8
|
||||||
#define SS 20*8
|
#define SS 20*8
|
||||||
|
|
||||||
#define ARGOFFSET 0
|
|
||||||
|
|
||||||
.macro ALLOC_PT_GPREGS_ON_STACK addskip=0
|
.macro ALLOC_PT_GPREGS_ON_STACK addskip=0
|
||||||
subq $15*8+\addskip, %rsp
|
subq $15*8+\addskip, %rsp
|
||||||
CFI_ADJUST_CFA_OFFSET 15*8+\addskip
|
CFI_ADJUST_CFA_OFFSET 15*8+\addskip
|
||||||
|
|
|
@ -73,9 +73,9 @@ ENDPROC(native_usergs_sysret64)
|
||||||
#endif /* CONFIG_PARAVIRT */
|
#endif /* CONFIG_PARAVIRT */
|
||||||
|
|
||||||
|
|
||||||
.macro TRACE_IRQS_IRETQ offset=ARGOFFSET
|
.macro TRACE_IRQS_IRETQ
|
||||||
#ifdef CONFIG_TRACE_IRQFLAGS
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
||||||
bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
|
bt $9,EFLAGS(%rsp) /* interrupts off? */
|
||||||
jnc 1f
|
jnc 1f
|
||||||
TRACE_IRQS_ON
|
TRACE_IRQS_ON
|
||||||
1:
|
1:
|
||||||
|
@ -107,8 +107,8 @@ ENDPROC(native_usergs_sysret64)
|
||||||
call debug_stack_reset
|
call debug_stack_reset
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
.macro TRACE_IRQS_IRETQ_DEBUG offset=ARGOFFSET
|
.macro TRACE_IRQS_IRETQ_DEBUG
|
||||||
bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
|
bt $9,EFLAGS(%rsp) /* interrupts off? */
|
||||||
jnc 1f
|
jnc 1f
|
||||||
TRACE_IRQS_ON_DEBUG
|
TRACE_IRQS_ON_DEBUG
|
||||||
1:
|
1:
|
||||||
|
@ -184,16 +184,16 @@ ENDPROC(native_usergs_sysret64)
|
||||||
* frame that enables passing a complete pt_regs to a C function.
|
* frame that enables passing a complete pt_regs to a C function.
|
||||||
*/
|
*/
|
||||||
.macro DEFAULT_FRAME start=1 offset=0
|
.macro DEFAULT_FRAME start=1 offset=0
|
||||||
XCPT_FRAME \start, ORIG_RAX+\offset-ARGOFFSET
|
XCPT_FRAME \start, ORIG_RAX+\offset
|
||||||
CFI_REL_OFFSET rdi, RDI+\offset-ARGOFFSET
|
CFI_REL_OFFSET rdi, RDI+\offset
|
||||||
CFI_REL_OFFSET rsi, RSI+\offset-ARGOFFSET
|
CFI_REL_OFFSET rsi, RSI+\offset
|
||||||
CFI_REL_OFFSET rdx, RDX+\offset-ARGOFFSET
|
CFI_REL_OFFSET rdx, RDX+\offset
|
||||||
CFI_REL_OFFSET rcx, RCX+\offset-ARGOFFSET
|
CFI_REL_OFFSET rcx, RCX+\offset
|
||||||
CFI_REL_OFFSET rax, RAX+\offset-ARGOFFSET
|
CFI_REL_OFFSET rax, RAX+\offset
|
||||||
CFI_REL_OFFSET r8, R8+\offset-ARGOFFSET
|
CFI_REL_OFFSET r8, R8+\offset
|
||||||
CFI_REL_OFFSET r9, R9+\offset-ARGOFFSET
|
CFI_REL_OFFSET r9, R9+\offset
|
||||||
CFI_REL_OFFSET r10, R10+\offset-ARGOFFSET
|
CFI_REL_OFFSET r10, R10+\offset
|
||||||
CFI_REL_OFFSET r11, R11+\offset-ARGOFFSET
|
CFI_REL_OFFSET r11, R11+\offset
|
||||||
CFI_REL_OFFSET rbx, RBX+\offset
|
CFI_REL_OFFSET rbx, RBX+\offset
|
||||||
CFI_REL_OFFSET rbp, RBP+\offset
|
CFI_REL_OFFSET rbp, RBP+\offset
|
||||||
CFI_REL_OFFSET r12, R12+\offset
|
CFI_REL_OFFSET r12, R12+\offset
|
||||||
|
@ -237,13 +237,13 @@ ENTRY(ret_from_fork)
|
||||||
|
|
||||||
RESTORE_EXTRA_REGS
|
RESTORE_EXTRA_REGS
|
||||||
|
|
||||||
testl $3, CS-ARGOFFSET(%rsp) # from kernel_thread?
|
testl $3,CS(%rsp) # from kernel_thread?
|
||||||
jz 1f
|
jz 1f
|
||||||
|
|
||||||
testl $_TIF_IA32, TI_flags(%rcx) # 32-bit compat task needs IRET
|
testl $_TIF_IA32, TI_flags(%rcx) # 32-bit compat task needs IRET
|
||||||
jnz int_ret_from_sys_call
|
jnz int_ret_from_sys_call
|
||||||
|
|
||||||
RESTORE_TOP_OF_STACK %rdi, -ARGOFFSET
|
RESTORE_TOP_OF_STACK %rdi
|
||||||
jmp ret_from_sys_call # go to the SYSRET fastpath
|
jmp ret_from_sys_call # go to the SYSRET fastpath
|
||||||
|
|
||||||
1:
|
1:
|
||||||
|
@ -310,11 +310,11 @@ GLOBAL(system_call_after_swapgs)
|
||||||
ENABLE_INTERRUPTS(CLBR_NONE)
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
||||||
ALLOC_PT_GPREGS_ON_STACK 8
|
ALLOC_PT_GPREGS_ON_STACK 8
|
||||||
SAVE_C_REGS_EXCEPT_RAX_RCX
|
SAVE_C_REGS_EXCEPT_RAX_RCX
|
||||||
movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
|
movq $-ENOSYS,RAX(%rsp)
|
||||||
movq_cfi rax,(ORIG_RAX-ARGOFFSET)
|
movq_cfi rax,ORIG_RAX
|
||||||
movq %rcx,RIP-ARGOFFSET(%rsp)
|
movq %rcx,RIP(%rsp)
|
||||||
CFI_REL_OFFSET rip,RIP-ARGOFFSET
|
CFI_REL_OFFSET rip,RIP
|
||||||
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
|
||||||
jnz tracesys
|
jnz tracesys
|
||||||
system_call_fastpath:
|
system_call_fastpath:
|
||||||
#if __SYSCALL_MASK == ~0
|
#if __SYSCALL_MASK == ~0
|
||||||
|
@ -326,13 +326,13 @@ system_call_fastpath:
|
||||||
ja ret_from_sys_call /* and return regs->ax */
|
ja ret_from_sys_call /* and return regs->ax */
|
||||||
movq %r10,%rcx
|
movq %r10,%rcx
|
||||||
call *sys_call_table(,%rax,8) # XXX: rip relative
|
call *sys_call_table(,%rax,8) # XXX: rip relative
|
||||||
movq %rax,RAX-ARGOFFSET(%rsp)
|
movq %rax,RAX(%rsp)
|
||||||
/*
|
/*
|
||||||
* Syscall return path ending with SYSRET (fast path)
|
* Syscall return path ending with SYSRET (fast path)
|
||||||
* Has incomplete stack frame and undefined top of stack.
|
* Has incomplete stack frame and undefined top of stack.
|
||||||
*/
|
*/
|
||||||
ret_from_sys_call:
|
ret_from_sys_call:
|
||||||
testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP)
|
||||||
jnz int_ret_from_sys_call_fixup /* Go the the slow path */
|
jnz int_ret_from_sys_call_fixup /* Go the the slow path */
|
||||||
|
|
||||||
LOCKDEP_SYS_EXIT
|
LOCKDEP_SYS_EXIT
|
||||||
|
@ -344,7 +344,7 @@ ret_from_sys_call:
|
||||||
*/
|
*/
|
||||||
TRACE_IRQS_ON
|
TRACE_IRQS_ON
|
||||||
RESTORE_C_REGS_EXCEPT_RCX
|
RESTORE_C_REGS_EXCEPT_RCX
|
||||||
movq RIP-ARGOFFSET(%rsp),%rcx
|
movq RIP(%rsp),%rcx
|
||||||
CFI_REGISTER rip,rcx
|
CFI_REGISTER rip,rcx
|
||||||
/*CFI_REGISTER rflags,r11*/
|
/*CFI_REGISTER rflags,r11*/
|
||||||
movq PER_CPU_VAR(old_rsp), %rsp
|
movq PER_CPU_VAR(old_rsp), %rsp
|
||||||
|
@ -353,7 +353,7 @@ ret_from_sys_call:
|
||||||
CFI_RESTORE_STATE
|
CFI_RESTORE_STATE
|
||||||
|
|
||||||
int_ret_from_sys_call_fixup:
|
int_ret_from_sys_call_fixup:
|
||||||
FIXUP_TOP_OF_STACK %r11, -ARGOFFSET
|
FIXUP_TOP_OF_STACK %r11
|
||||||
jmp int_ret_from_sys_call
|
jmp int_ret_from_sys_call
|
||||||
|
|
||||||
/* Do syscall tracing */
|
/* Do syscall tracing */
|
||||||
|
@ -364,7 +364,7 @@ tracesys:
|
||||||
test %rax, %rax
|
test %rax, %rax
|
||||||
jnz tracesys_phase2 /* if needed, run the slow path */
|
jnz tracesys_phase2 /* if needed, run the slow path */
|
||||||
RESTORE_C_REGS_EXCEPT_RAX /* else restore clobbered regs */
|
RESTORE_C_REGS_EXCEPT_RAX /* else restore clobbered regs */
|
||||||
movq ORIG_RAX-ARGOFFSET(%rsp), %rax
|
movq ORIG_RAX(%rsp), %rax
|
||||||
jmp system_call_fastpath /* and return to the fast path */
|
jmp system_call_fastpath /* and return to the fast path */
|
||||||
|
|
||||||
tracesys_phase2:
|
tracesys_phase2:
|
||||||
|
@ -391,7 +391,7 @@ tracesys_phase2:
|
||||||
ja int_ret_from_sys_call /* RAX(%rsp) is already set */
|
ja int_ret_from_sys_call /* RAX(%rsp) is already set */
|
||||||
movq %r10,%rcx /* fixup for C */
|
movq %r10,%rcx /* fixup for C */
|
||||||
call *sys_call_table(,%rax,8)
|
call *sys_call_table(,%rax,8)
|
||||||
movq %rax,RAX-ARGOFFSET(%rsp)
|
movq %rax,RAX(%rsp)
|
||||||
/* Use IRET because user could have changed frame */
|
/* Use IRET because user could have changed frame */
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -475,9 +475,9 @@ END(stub_\func)
|
||||||
ENTRY(\label)
|
ENTRY(\label)
|
||||||
CFI_STARTPROC
|
CFI_STARTPROC
|
||||||
DEFAULT_FRAME 0, 8 /* offset 8: return address */
|
DEFAULT_FRAME 0, 8 /* offset 8: return address */
|
||||||
FIXUP_TOP_OF_STACK %r11, 8-ARGOFFSET
|
FIXUP_TOP_OF_STACK %r11, 8
|
||||||
call \func
|
call \func
|
||||||
RESTORE_TOP_OF_STACK %r11, 8-ARGOFFSET
|
RESTORE_TOP_OF_STACK %r11, 8
|
||||||
ret
|
ret
|
||||||
CFI_ENDPROC
|
CFI_ENDPROC
|
||||||
END(\label)
|
END(\label)
|
||||||
|
@ -677,7 +677,7 @@ common_interrupt:
|
||||||
ASM_CLAC
|
ASM_CLAC
|
||||||
addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */
|
addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */
|
||||||
interrupt do_IRQ
|
interrupt do_IRQ
|
||||||
/* 0(%rsp): old_rsp-ARGOFFSET */
|
/* 0(%rsp): old_rsp */
|
||||||
ret_from_intr:
|
ret_from_intr:
|
||||||
DISABLE_INTERRUPTS(CLBR_NONE)
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
||||||
TRACE_IRQS_OFF
|
TRACE_IRQS_OFF
|
||||||
|
@ -687,13 +687,13 @@ ret_from_intr:
|
||||||
popq %rsi
|
popq %rsi
|
||||||
CFI_DEF_CFA rsi,SS+8-RBP /* reg/off reset after def_cfa_expr */
|
CFI_DEF_CFA rsi,SS+8-RBP /* reg/off reset after def_cfa_expr */
|
||||||
/* return code expects complete pt_regs - adjust rsp accordingly: */
|
/* return code expects complete pt_regs - adjust rsp accordingly: */
|
||||||
leaq ARGOFFSET-RBP(%rsi), %rsp
|
leaq -RBP(%rsi),%rsp
|
||||||
CFI_DEF_CFA_REGISTER rsp
|
CFI_DEF_CFA_REGISTER rsp
|
||||||
CFI_ADJUST_CFA_OFFSET RBP-ARGOFFSET
|
CFI_ADJUST_CFA_OFFSET RBP
|
||||||
|
|
||||||
exit_intr:
|
exit_intr:
|
||||||
GET_THREAD_INFO(%rcx)
|
GET_THREAD_INFO(%rcx)
|
||||||
testl $3,CS-ARGOFFSET(%rsp)
|
testl $3,CS(%rsp)
|
||||||
je retint_kernel
|
je retint_kernel
|
||||||
|
|
||||||
/* Interrupt came from user space */
|
/* Interrupt came from user space */
|
||||||
|
@ -721,8 +721,8 @@ retint_swapgs: /* return to user-space */
|
||||||
* Try to use SYSRET instead of IRET if we're returning to
|
* Try to use SYSRET instead of IRET if we're returning to
|
||||||
* a completely clean 64-bit userspace context.
|
* a completely clean 64-bit userspace context.
|
||||||
*/
|
*/
|
||||||
movq (RCX-ARGOFFSET)(%rsp), %rcx
|
movq RCX(%rsp),%rcx
|
||||||
cmpq %rcx,(RIP-ARGOFFSET)(%rsp) /* RCX == RIP */
|
cmpq %rcx,RIP(%rsp) /* RCX == RIP */
|
||||||
jne opportunistic_sysret_failed
|
jne opportunistic_sysret_failed
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -743,11 +743,11 @@ retint_swapgs: /* return to user-space */
|
||||||
shr $__VIRTUAL_MASK_SHIFT, %rcx
|
shr $__VIRTUAL_MASK_SHIFT, %rcx
|
||||||
jnz opportunistic_sysret_failed
|
jnz opportunistic_sysret_failed
|
||||||
|
|
||||||
cmpq $__USER_CS,(CS-ARGOFFSET)(%rsp) /* CS must match SYSRET */
|
cmpq $__USER_CS,CS(%rsp) /* CS must match SYSRET */
|
||||||
jne opportunistic_sysret_failed
|
jne opportunistic_sysret_failed
|
||||||
|
|
||||||
movq (R11-ARGOFFSET)(%rsp), %r11
|
movq R11(%rsp),%r11
|
||||||
cmpq %r11,(EFLAGS-ARGOFFSET)(%rsp) /* R11 == RFLAGS */
|
cmpq %r11,EFLAGS(%rsp) /* R11 == RFLAGS */
|
||||||
jne opportunistic_sysret_failed
|
jne opportunistic_sysret_failed
|
||||||
|
|
||||||
testq $X86_EFLAGS_RF,%r11 /* sysret can't restore RF */
|
testq $X86_EFLAGS_RF,%r11 /* sysret can't restore RF */
|
||||||
|
@ -755,7 +755,7 @@ retint_swapgs: /* return to user-space */
|
||||||
|
|
||||||
/* nothing to check for RSP */
|
/* nothing to check for RSP */
|
||||||
|
|
||||||
cmpq $__USER_DS,(SS-ARGOFFSET)(%rsp) /* SS must match SYSRET */
|
cmpq $__USER_DS,SS(%rsp) /* SS must match SYSRET */
|
||||||
jne opportunistic_sysret_failed
|
jne opportunistic_sysret_failed
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -870,7 +870,7 @@ retint_signal:
|
||||||
ENTRY(retint_kernel)
|
ENTRY(retint_kernel)
|
||||||
cmpl $0,PER_CPU_VAR(__preempt_count)
|
cmpl $0,PER_CPU_VAR(__preempt_count)
|
||||||
jnz retint_restore_args
|
jnz retint_restore_args
|
||||||
bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
|
bt $9,EFLAGS(%rsp) /* interrupts off? */
|
||||||
jnc retint_restore_args
|
jnc retint_restore_args
|
||||||
call preempt_schedule_irq
|
call preempt_schedule_irq
|
||||||
jmp exit_intr
|
jmp exit_intr
|
||||||
|
@ -1286,11 +1286,11 @@ ENTRY(paranoid_exit)
|
||||||
TRACE_IRQS_OFF_DEBUG
|
TRACE_IRQS_OFF_DEBUG
|
||||||
testl %ebx,%ebx /* swapgs needed? */
|
testl %ebx,%ebx /* swapgs needed? */
|
||||||
jnz paranoid_exit_no_swapgs
|
jnz paranoid_exit_no_swapgs
|
||||||
TRACE_IRQS_IRETQ 0
|
TRACE_IRQS_IRETQ
|
||||||
SWAPGS_UNSAFE_STACK
|
SWAPGS_UNSAFE_STACK
|
||||||
jmp paranoid_exit_restore
|
jmp paranoid_exit_restore
|
||||||
paranoid_exit_no_swapgs:
|
paranoid_exit_no_swapgs:
|
||||||
TRACE_IRQS_IRETQ_DEBUG 0
|
TRACE_IRQS_IRETQ_DEBUG
|
||||||
paranoid_exit_restore:
|
paranoid_exit_restore:
|
||||||
RESTORE_EXTRA_REGS
|
RESTORE_EXTRA_REGS
|
||||||
RESTORE_C_REGS
|
RESTORE_C_REGS
|
||||||
|
|
Загрузка…
Ссылка в новой задаче