x86: Make __put_user() generate an out-of-line call
Instead of inlining the stac/mov/clac sequence (which also requires individual exception table entries and several asm instruction alternatives entries), just generate "call __put_user_nocheck_X" for the __put_user() cases, the same way we changed __get_user earlier. Unlike the get_user() case, we didn't have the same nice infrastructure to just generate the call with a single case, so this actually has to change some of the infrastructure in order to do this. But that only cleans up the code further. So now, instead of using a case statement for the sizes, we just do the same thing we've done on the get_user() side for a long time: use the size as an immediate constant to the asm, and generate the asm that way directly. In order to handle the special case of 64-bit data on a 32-bit kernel, I needed to change the calling convention slightly: the data is passed in %eax[:%edx], the pointer in %ecx, and the return value is also returned in %ecx. It used to be returned in %eax, but because of how %eax can now be a double register input, we don't want mix that with a single-register output. The actual low-level asm is easier to handle: we'll just share the code between the checking and non-checking case, with the non-checking case jumping into the middle of the function. That may sound a bit too special, but this code is all very very special anyway, so... Cc: Al Viro <viro@zeniv.linux.org.uk> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Ingo Molnar <mingo@kernel.org> Cc: Borislav Petkov <bp@alien8.de> Cc: Peter Zijlstra <peterz@infradead.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Linus Torvalds
Родитель
ea6f043fc9
Коммит
d55564cfc2
|
|
@ -201,11 +201,6 @@ extern int __get_user_bad(void);
|
|||
*/
|
||||
#define __get_user(x,ptr) do_get_user_call(get_user_nocheck,x,ptr)
|
||||
|
||||
#define __put_user_x(size, x, ptr, __ret_pu) \
|
||||
asm volatile("call __put_user_" #size : "=a" (__ret_pu) \
|
||||
: "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
|
||||
|
||||
|
||||
|
||||
#ifdef CONFIG_X86_32
|
||||
#define __put_user_goto_u64(x, addr, label) \
|
||||
|
|
@ -217,25 +212,41 @@ extern int __get_user_bad(void);
|
|||
: : "A" (x), "r" (addr) \
|
||||
: : label)
|
||||
|
||||
#define __put_user_x8(x, ptr, __ret_pu) \
|
||||
asm volatile("call __put_user_8" : "=a" (__ret_pu) \
|
||||
: "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
|
||||
#else
|
||||
#define __put_user_goto_u64(x, ptr, label) \
|
||||
__put_user_goto(x, ptr, "q", "er", label)
|
||||
#define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
|
||||
#endif
|
||||
|
||||
extern void __put_user_bad(void);
|
||||
|
||||
/*
|
||||
* Strange magic calling convention: pointer in %ecx,
|
||||
* value in %eax(:%edx), return value in %eax. clobbers %rbx
|
||||
* value in %eax(:%edx), return value in %ecx. clobbers %rbx
|
||||
*/
|
||||
extern void __put_user_1(void);
|
||||
extern void __put_user_2(void);
|
||||
extern void __put_user_4(void);
|
||||
extern void __put_user_8(void);
|
||||
extern void __put_user_nocheck_1(void);
|
||||
extern void __put_user_nocheck_2(void);
|
||||
extern void __put_user_nocheck_4(void);
|
||||
extern void __put_user_nocheck_8(void);
|
||||
|
||||
#define do_put_user_call(fn,x,ptr) \
|
||||
({ \
|
||||
int __ret_pu; \
|
||||
register __typeof__(*(ptr)) __val_pu asm("%"_ASM_AX); \
|
||||
__chk_user_ptr(ptr); \
|
||||
__val_pu = (x); \
|
||||
asm volatile("call __" #fn "_%P[size]" \
|
||||
: "=c" (__ret_pu), \
|
||||
ASM_CALL_CONSTRAINT \
|
||||
: "0" (ptr), \
|
||||
"r" (__val_pu), \
|
||||
[size] "i" (sizeof(*(ptr))) \
|
||||
:"ebx"); \
|
||||
__builtin_expect(__ret_pu, 0); \
|
||||
})
|
||||
|
||||
/**
|
||||
* put_user - Write a simple value into user space.
|
||||
|
|
@ -254,32 +265,29 @@ extern void __put_user_8(void);
|
|||
*
|
||||
* Return: zero on success, or -EFAULT on error.
|
||||
*/
|
||||
#define put_user(x, ptr) \
|
||||
({ \
|
||||
int __ret_pu; \
|
||||
__typeof__(*(ptr)) __pu_val; \
|
||||
__chk_user_ptr(ptr); \
|
||||
might_fault(); \
|
||||
__pu_val = x; \
|
||||
switch (sizeof(*(ptr))) { \
|
||||
case 1: \
|
||||
__put_user_x(1, __pu_val, ptr, __ret_pu); \
|
||||
break; \
|
||||
case 2: \
|
||||
__put_user_x(2, __pu_val, ptr, __ret_pu); \
|
||||
break; \
|
||||
case 4: \
|
||||
__put_user_x(4, __pu_val, ptr, __ret_pu); \
|
||||
break; \
|
||||
case 8: \
|
||||
__put_user_x8(__pu_val, ptr, __ret_pu); \
|
||||
break; \
|
||||
default: \
|
||||
__put_user_x(X, __pu_val, ptr, __ret_pu); \
|
||||
break; \
|
||||
} \
|
||||
__builtin_expect(__ret_pu, 0); \
|
||||
})
|
||||
#define put_user(x, ptr) ({ might_fault(); do_put_user_call(put_user,x,ptr); })
|
||||
|
||||
/**
|
||||
* __put_user - Write a simple value into user space, with less checking.
|
||||
* @x: Value to copy to user space.
|
||||
* @ptr: Destination address, in user space.
|
||||
*
|
||||
* Context: User context only. This function may sleep if pagefaults are
|
||||
* enabled.
|
||||
*
|
||||
* This macro copies a single simple value from kernel space to user
|
||||
* space. It supports simple types like char and int, but not larger
|
||||
* data types like structures or arrays.
|
||||
*
|
||||
* @ptr must have pointer-to-simple-variable type, and @x must be assignable
|
||||
* to the result of dereferencing @ptr.
|
||||
*
|
||||
* Caller must check the pointer with access_ok() before calling this
|
||||
* function.
|
||||
*
|
||||
* Return: zero on success, or -EFAULT on error.
|
||||
*/
|
||||
#define __put_user(x, ptr) do_put_user_call(put_user_nocheck,x,ptr)
|
||||
|
||||
#define __put_user_size(x, ptr, size, label) \
|
||||
do { \
|
||||
|
|
@ -370,21 +378,6 @@ do { \
|
|||
: [umem] "m" (__m(addr)), \
|
||||
[efault] "i" (-EFAULT), "0" (err))
|
||||
|
||||
#define __put_user_nocheck(x, ptr, size) \
|
||||
({ \
|
||||
__label__ __pu_label; \
|
||||
int __pu_err = -EFAULT; \
|
||||
__typeof__(*(ptr)) __pu_val = (x); \
|
||||
__typeof__(ptr) __pu_ptr = (ptr); \
|
||||
__typeof__(size) __pu_size = (size); \
|
||||
__uaccess_begin(); \
|
||||
__put_user_size(__pu_val, __pu_ptr, __pu_size, __pu_label); \
|
||||
__pu_err = 0; \
|
||||
__pu_label: \
|
||||
__uaccess_end(); \
|
||||
__builtin_expect(__pu_err, 0); \
|
||||
})
|
||||
|
||||
/* FIXME: this hack is definitely wrong -AK */
|
||||
struct __large_struct { unsigned long buf[100]; };
|
||||
#define __m(x) (*(struct __large_struct __user *)(x))
|
||||
|
|
@ -401,30 +394,6 @@ struct __large_struct { unsigned long buf[100]; };
|
|||
: : ltype(x), "m" (__m(addr)) \
|
||||
: : label)
|
||||
|
||||
/**
|
||||
* __put_user - Write a simple value into user space, with less checking.
|
||||
* @x: Value to copy to user space.
|
||||
* @ptr: Destination address, in user space.
|
||||
*
|
||||
* Context: User context only. This function may sleep if pagefaults are
|
||||
* enabled.
|
||||
*
|
||||
* This macro copies a single simple value from kernel space to user
|
||||
* space. It supports simple types like char and int, but not larger
|
||||
* data types like structures or arrays.
|
||||
*
|
||||
* @ptr must have pointer-to-simple-variable type, and @x must be assignable
|
||||
* to the result of dereferencing @ptr.
|
||||
*
|
||||
* Caller must check the pointer with access_ok() before calling this
|
||||
* function.
|
||||
*
|
||||
* Return: zero on success, or -EFAULT on error.
|
||||
*/
|
||||
|
||||
#define __put_user(x, ptr) \
|
||||
__put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
|
||||
|
||||
extern unsigned long
|
||||
copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
|
||||
extern __must_check long
|
||||
|
|
|
|||
|
|
@ -25,7 +25,9 @@
|
|||
* Inputs: %eax[:%edx] contains the data
|
||||
* %ecx contains the address
|
||||
*
|
||||
* Outputs: %eax is error code (0 or -EFAULT)
|
||||
* Outputs: %ecx is error code (0 or -EFAULT)
|
||||
*
|
||||
* Clobbers: %ebx needed for task pointer
|
||||
*
|
||||
* These functions should not modify any other registers,
|
||||
* as they get called from within inline assembly.
|
||||
|
|
@ -38,13 +40,15 @@ SYM_FUNC_START(__put_user_1)
|
|||
ENTER
|
||||
cmp TASK_addr_limit(%_ASM_BX),%_ASM_CX
|
||||
jae .Lbad_put_user
|
||||
SYM_INNER_LABEL(__put_user_nocheck_1, SYM_L_GLOBAL)
|
||||
ASM_STAC
|
||||
1: movb %al,(%_ASM_CX)
|
||||
xor %eax,%eax
|
||||
xor %ecx,%ecx
|
||||
ASM_CLAC
|
||||
ret
|
||||
SYM_FUNC_END(__put_user_1)
|
||||
EXPORT_SYMBOL(__put_user_1)
|
||||
EXPORT_SYMBOL(__put_user_nocheck_1)
|
||||
|
||||
SYM_FUNC_START(__put_user_2)
|
||||
ENTER
|
||||
|
|
@ -52,13 +56,15 @@ SYM_FUNC_START(__put_user_2)
|
|||
sub $1,%_ASM_BX
|
||||
cmp %_ASM_BX,%_ASM_CX
|
||||
jae .Lbad_put_user
|
||||
SYM_INNER_LABEL(__put_user_nocheck_2, SYM_L_GLOBAL)
|
||||
ASM_STAC
|
||||
2: movw %ax,(%_ASM_CX)
|
||||
xor %eax,%eax
|
||||
xor %ecx,%ecx
|
||||
ASM_CLAC
|
||||
ret
|
||||
SYM_FUNC_END(__put_user_2)
|
||||
EXPORT_SYMBOL(__put_user_2)
|
||||
EXPORT_SYMBOL(__put_user_nocheck_2)
|
||||
|
||||
SYM_FUNC_START(__put_user_4)
|
||||
ENTER
|
||||
|
|
@ -66,13 +72,15 @@ SYM_FUNC_START(__put_user_4)
|
|||
sub $3,%_ASM_BX
|
||||
cmp %_ASM_BX,%_ASM_CX
|
||||
jae .Lbad_put_user
|
||||
SYM_INNER_LABEL(__put_user_nocheck_4, SYM_L_GLOBAL)
|
||||
ASM_STAC
|
||||
3: movl %eax,(%_ASM_CX)
|
||||
xor %eax,%eax
|
||||
xor %ecx,%ecx
|
||||
ASM_CLAC
|
||||
ret
|
||||
SYM_FUNC_END(__put_user_4)
|
||||
EXPORT_SYMBOL(__put_user_4)
|
||||
EXPORT_SYMBOL(__put_user_nocheck_4)
|
||||
|
||||
SYM_FUNC_START(__put_user_8)
|
||||
ENTER
|
||||
|
|
@ -80,21 +88,23 @@ SYM_FUNC_START(__put_user_8)
|
|||
sub $7,%_ASM_BX
|
||||
cmp %_ASM_BX,%_ASM_CX
|
||||
jae .Lbad_put_user
|
||||
SYM_INNER_LABEL(__put_user_nocheck_8, SYM_L_GLOBAL)
|
||||
ASM_STAC
|
||||
4: mov %_ASM_AX,(%_ASM_CX)
|
||||
#ifdef CONFIG_X86_32
|
||||
5: movl %edx,4(%_ASM_CX)
|
||||
#endif
|
||||
xor %eax,%eax
|
||||
xor %ecx,%ecx
|
||||
ASM_CLAC
|
||||
RET
|
||||
SYM_FUNC_END(__put_user_8)
|
||||
EXPORT_SYMBOL(__put_user_8)
|
||||
EXPORT_SYMBOL(__put_user_nocheck_8)
|
||||
|
||||
SYM_CODE_START_LOCAL(.Lbad_put_user_clac)
|
||||
ASM_CLAC
|
||||
.Lbad_put_user:
|
||||
movl $-EFAULT,%eax
|
||||
movl $-EFAULT,%ecx
|
||||
RET
|
||||
SYM_CODE_END(.Lbad_put_user_clac)
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче