[ARM] clearpage: provide our own clear_user_highpage()
For similar reasons as copy_user_page(), we want to avoid the additional kmap_atomic if it's unnecessary. Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
This commit is contained in:
Russell King
Russell King
Родитель
063b0a4207
Коммит
303c644365
|
|
@ -111,7 +111,7 @@
|
|||
struct page;
|
||||
|
||||
struct cpu_user_fns {
|
||||
void (*cpu_clear_user_page)(void *p, unsigned long user);
|
||||
void (*cpu_clear_user_highpage)(struct page *page, unsigned long vaddr);
|
||||
void (*cpu_copy_user_highpage)(struct page *to, struct page *from,
|
||||
unsigned long vaddr);
|
||||
};
|
||||
|
|
@ -119,20 +119,21 @@ struct cpu_user_fns {
|
|||
#ifdef MULTI_USER
|
||||
extern struct cpu_user_fns cpu_user;
|
||||
|
||||
#define __cpu_clear_user_page cpu_user.cpu_clear_user_page
|
||||
#define __cpu_clear_user_highpage cpu_user.cpu_clear_user_highpage
|
||||
#define __cpu_copy_user_highpage cpu_user.cpu_copy_user_highpage
|
||||
|
||||
#else
|
||||
|
||||
#define __cpu_clear_user_page __glue(_USER,_clear_user_page)
|
||||
#define __cpu_clear_user_highpage __glue(_USER,_clear_user_highpage)
|
||||
#define __cpu_copy_user_highpage __glue(_USER,_copy_user_highpage)
|
||||
|
||||
extern void __cpu_clear_user_page(void *p, unsigned long user);
|
||||
extern void __cpu_clear_user_highpage(struct page *page, unsigned long vaddr);
|
||||
extern void __cpu_copy_user_highpage(struct page *to, struct page *from,
|
||||
unsigned long vaddr);
|
||||
#endif
|
||||
|
||||
#define clear_user_page(addr,vaddr,pg) __cpu_clear_user_page(addr, vaddr)
|
||||
#define clear_user_highpage(page,vaddr) \
|
||||
__cpu_clear_user_highpage(page, vaddr)
|
||||
|
||||
#define __HAVE_ARCH_COPY_USER_HIGHPAGE
|
||||
#define copy_user_highpage(to,from,vaddr,vma) \
|
||||
|
|
|
|||
|
|
@ -79,12 +79,11 @@ void feroceon_copy_user_highpage(struct page *to, struct page *from,
|
|||
kunmap_atomic(kto, KM_USER0);
|
||||
}
|
||||
|
||||
void __attribute__((naked))
|
||||
feroceon_clear_user_page(void *kaddr, unsigned long vaddr)
|
||||
void feroceon_clear_user_highpage(struct page *page, unsigned long vaddr)
|
||||
{
|
||||
void *kaddr = kmap_atomic(page, KM_USER0);
|
||||
asm("\
|
||||
stmfd sp!, {r4-r7, lr} \n\
|
||||
mov r1, %0 \n\
|
||||
mov r1, %1 \n\
|
||||
mov r2, #0 \n\
|
||||
mov r3, #0 \n\
|
||||
mov r4, #0 \n\
|
||||
|
|
@ -93,19 +92,20 @@ feroceon_clear_user_page(void *kaddr, unsigned long vaddr)
|
|||
mov r7, #0 \n\
|
||||
mov ip, #0 \n\
|
||||
mov lr, #0 \n\
|
||||
1: stmia r0, {r2-r7, ip, lr} \n\
|
||||
1: stmia %0, {r2-r7, ip, lr} \n\
|
||||
subs r1, r1, #1 \n\
|
||||
mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D line\n\
|
||||
mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
|
||||
add r0, r0, #32 \n\
|
||||
bne 1b \n\
|
||||
mcr p15, 0, r1, c7, c10, 4 @ drain WB\n\
|
||||
ldmfd sp!, {r4-r7, pc}"
|
||||
mcr p15, 0, r1, c7, c10, 4 @ drain WB"
|
||||
:
|
||||
: "I" (PAGE_SIZE / 32));
|
||||
: "r" (kaddr), "I" (PAGE_SIZE / 32)
|
||||
: "r1", "r2", "r3", "r4", "r5", "r6", "r7", "ip", "lr");
|
||||
kunmap_atomic(kaddr, KM_USER0);
|
||||
}
|
||||
|
||||
struct cpu_user_fns feroceon_user_fns __initdata = {
|
||||
.cpu_clear_user_page = feroceon_clear_user_page,
|
||||
.cpu_clear_user_highpage = feroceon_clear_user_highpage,
|
||||
.cpu_copy_user_highpage = feroceon_copy_user_highpage,
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -54,10 +54,10 @@ void v3_copy_user_highpage(struct page *to, struct page *from,
|
|||
*
|
||||
* FIXME: do we need to handle cache stuff...
|
||||
*/
|
||||
void __attribute__((naked)) v3_clear_user_page(void *kaddr, unsigned long vaddr)
|
||||
void v3_clear_user_highpage(struct page *page, unsigned long vaddr)
|
||||
{
|
||||
void *kaddr = kmap_atomic(page, KM_USER0);
|
||||
asm("\n\
|
||||
str lr, [sp, #-4]!\n\
|
||||
mov r1, %1 @ 1\n\
|
||||
mov r2, #0 @ 1\n\
|
||||
mov r3, #0 @ 1\n\
|
||||
|
|
@ -68,13 +68,14 @@ void __attribute__((naked)) v3_clear_user_page(void *kaddr, unsigned long vaddr)
|
|||
stmia %0!, {r2, r3, ip, lr} @ 4\n\
|
||||
stmia %0!, {r2, r3, ip, lr} @ 4\n\
|
||||
subs r1, r1, #1 @ 1\n\
|
||||
bne 1b @ 1\n\
|
||||
ldr pc, [sp], #4"
|
||||
bne 1b @ 1"
|
||||
:
|
||||
: "r" (kaddr), "I" (PAGE_SIZE / 64));
|
||||
: "r" (kaddr), "I" (PAGE_SIZE / 64)
|
||||
: "r1", "r2", "r3", "ip", "lr");
|
||||
kunmap_atomic(kaddr, KM_USER0);
|
||||
}
|
||||
|
||||
struct cpu_user_fns v3_user_fns __initdata = {
|
||||
.cpu_clear_user_page = v3_clear_user_page,
|
||||
.cpu_clear_user_highpage = v3_clear_user_highpage,
|
||||
.cpu_copy_user_highpage = v3_copy_user_highpage,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -91,30 +91,30 @@ void v4_mc_copy_user_highpage(struct page *from, struct page *to,
|
|||
/*
|
||||
* ARMv4 optimised clear_user_page
|
||||
*/
|
||||
void __attribute__((naked))
|
||||
v4_mc_clear_user_page(void *kaddr, unsigned long vaddr)
|
||||
void v4_mc_clear_user_highpage(struct page *page, unsigned long vaddr)
|
||||
{
|
||||
asm volatile(
|
||||
"str lr, [sp, #-4]!\n\
|
||||
void *kaddr = kmap_atomic(page, KM_USER0);
|
||||
asm volatile("\
|
||||
mov r1, %0 @ 1\n\
|
||||
mov r2, #0 @ 1\n\
|
||||
mov r3, #0 @ 1\n\
|
||||
mov ip, #0 @ 1\n\
|
||||
mov lr, #0 @ 1\n\
|
||||
1: mcr p15, 0, r0, c7, c6, 1 @ 1 invalidate D line\n\
|
||||
stmia r0!, {r2, r3, ip, lr} @ 4\n\
|
||||
stmia r0!, {r2, r3, ip, lr} @ 4\n\
|
||||
mcr p15, 0, r0, c7, c6, 1 @ 1 invalidate D line\n\
|
||||
stmia r0!, {r2, r3, ip, lr} @ 4\n\
|
||||
stmia r0!, {r2, r3, ip, lr} @ 4\n\
|
||||
1: mcr p15, 0, %0, c7, c6, 1 @ 1 invalidate D line\n\
|
||||
stmia %0!, {r2, r3, ip, lr} @ 4\n\
|
||||
stmia %0!, {r2, r3, ip, lr} @ 4\n\
|
||||
mcr p15, 0, %0, c7, c6, 1 @ 1 invalidate D line\n\
|
||||
stmia %0!, {r2, r3, ip, lr} @ 4\n\
|
||||
stmia %0!, {r2, r3, ip, lr} @ 4\n\
|
||||
subs r1, r1, #1 @ 1\n\
|
||||
bne 1b @ 1\n\
|
||||
ldr pc, [sp], #4"
|
||||
bne 1b @ 1"
|
||||
:
|
||||
: "I" (PAGE_SIZE / 64));
|
||||
: "r" (kaddr), "I" (PAGE_SIZE / 64)
|
||||
: "r1", "r2", "r3", "ip", "lr");
|
||||
kunmap_atomic(kaddr, KM_USER0);
|
||||
}
|
||||
|
||||
struct cpu_user_fns v4_mc_user_fns __initdata = {
|
||||
.cpu_clear_user_page = v4_mc_clear_user_page,
|
||||
.cpu_clear_user_highpage = v4_mc_clear_user_highpage,
|
||||
.cpu_copy_user_highpage = v4_mc_copy_user_highpage,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -64,31 +64,31 @@ void v4wb_copy_user_highpage(struct page *to, struct page *from,
|
|||
*
|
||||
* Same story as above.
|
||||
*/
|
||||
void __attribute__((naked))
|
||||
v4wb_clear_user_page(void *kaddr, unsigned long vaddr)
|
||||
void v4wb_clear_user_highpage(struct page *page, unsigned long vaddr)
|
||||
{
|
||||
void *kaddr = kmap_atomic(page, KM_USER0);
|
||||
asm("\
|
||||
str lr, [sp, #-4]!\n\
|
||||
mov r1, %0 @ 1\n\
|
||||
mov r1, %1 @ 1\n\
|
||||
mov r2, #0 @ 1\n\
|
||||
mov r3, #0 @ 1\n\
|
||||
mov ip, #0 @ 1\n\
|
||||
mov lr, #0 @ 1\n\
|
||||
1: mcr p15, 0, r0, c7, c6, 1 @ 1 invalidate D line\n\
|
||||
stmia r0!, {r2, r3, ip, lr} @ 4\n\
|
||||
stmia r0!, {r2, r3, ip, lr} @ 4\n\
|
||||
mcr p15, 0, r0, c7, c6, 1 @ 1 invalidate D line\n\
|
||||
stmia r0!, {r2, r3, ip, lr} @ 4\n\
|
||||
stmia r0!, {r2, r3, ip, lr} @ 4\n\
|
||||
1: mcr p15, 0, %0, c7, c6, 1 @ 1 invalidate D line\n\
|
||||
stmia %0!, {r2, r3, ip, lr} @ 4\n\
|
||||
stmia %0!, {r2, r3, ip, lr} @ 4\n\
|
||||
mcr p15, 0, %0, c7, c6, 1 @ 1 invalidate D line\n\
|
||||
stmia %0!, {r2, r3, ip, lr} @ 4\n\
|
||||
stmia %0!, {r2, r3, ip, lr} @ 4\n\
|
||||
subs r1, r1, #1 @ 1\n\
|
||||
bne 1b @ 1\n\
|
||||
mcr p15, 0, r1, c7, c10, 4 @ 1 drain WB\n\
|
||||
ldr pc, [sp], #4"
|
||||
mcr p15, 0, r1, c7, c10, 4 @ 1 drain WB"
|
||||
:
|
||||
: "I" (PAGE_SIZE / 64));
|
||||
: "r" (kaddr), "I" (PAGE_SIZE / 64)
|
||||
: "r1", "r2", "r3", "ip", "lr");
|
||||
kunmap_atomic(kaddr, KM_USER0);
|
||||
}
|
||||
|
||||
struct cpu_user_fns v4wb_user_fns __initdata = {
|
||||
.cpu_clear_user_page = v4wb_clear_user_page,
|
||||
.cpu_clear_user_highpage = v4wb_clear_user_highpage,
|
||||
.cpu_copy_user_highpage = v4wb_copy_user_highpage,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -60,29 +60,29 @@ void v4wt_copy_user_highpage(struct page *to, struct page *from,
|
|||
*
|
||||
* Same story as above.
|
||||
*/
|
||||
void __attribute__((naked))
|
||||
v4wt_clear_user_page(void *kaddr, unsigned long vaddr)
|
||||
void v4wt_clear_user_highpage(struct page *page, unsigned long vaddr)
|
||||
{
|
||||
void *kaddr = kmap_atomic(page, KM_USER0);
|
||||
asm("\
|
||||
str lr, [sp, #-4]!\n\
|
||||
mov r1, %0 @ 1\n\
|
||||
mov r1, %1 @ 1\n\
|
||||
mov r2, #0 @ 1\n\
|
||||
mov r3, #0 @ 1\n\
|
||||
mov ip, #0 @ 1\n\
|
||||
mov lr, #0 @ 1\n\
|
||||
1: stmia r0!, {r2, r3, ip, lr} @ 4\n\
|
||||
stmia r0!, {r2, r3, ip, lr} @ 4\n\
|
||||
stmia r0!, {r2, r3, ip, lr} @ 4\n\
|
||||
stmia r0!, {r2, r3, ip, lr} @ 4\n\
|
||||
1: stmia %0!, {r2, r3, ip, lr} @ 4\n\
|
||||
stmia %0!, {r2, r3, ip, lr} @ 4\n\
|
||||
stmia %0!, {r2, r3, ip, lr} @ 4\n\
|
||||
stmia %0!, {r2, r3, ip, lr} @ 4\n\
|
||||
subs r1, r1, #1 @ 1\n\
|
||||
bne 1b @ 1\n\
|
||||
mcr p15, 0, r2, c7, c7, 0 @ flush ID cache\n\
|
||||
ldr pc, [sp], #4"
|
||||
mcr p15, 0, r2, c7, c7, 0 @ flush ID cache"
|
||||
:
|
||||
: "I" (PAGE_SIZE / 64));
|
||||
: "r" (kaddr), "I" (PAGE_SIZE / 64)
|
||||
: "r1", "r2", "r3", "ip", "lr");
|
||||
kunmap_atomic(kaddr, KM_USER0);
|
||||
}
|
||||
|
||||
struct cpu_user_fns v4wt_user_fns __initdata = {
|
||||
.cpu_clear_user_page = v4wt_clear_user_page,
|
||||
.cpu_clear_user_highpage = v4wt_clear_user_highpage,
|
||||
.cpu_copy_user_highpage = v4wt_copy_user_highpage,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -49,9 +49,11 @@ static void v6_copy_user_highpage_nonaliasing(struct page *to,
|
|||
* Clear the user page. No aliasing to deal with so we can just
|
||||
* attack the kernel's existing mapping of this page.
|
||||
*/
|
||||
static void v6_clear_user_page_nonaliasing(void *kaddr, unsigned long vaddr)
|
||||
static void v6_clear_user_highpage_nonaliasing(struct page *page, unsigned long vaddr)
|
||||
{
|
||||
void *kaddr = kmap_atomic(page, KM_USER0);
|
||||
clear_page(kaddr);
|
||||
kunmap_atomic(kaddr, KM_USER0);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -107,20 +109,13 @@ static void v6_copy_user_highpage_aliasing(struct page *to,
|
|||
* so remap the kernel page into the same cache colour as the user
|
||||
* page.
|
||||
*/
|
||||
static void v6_clear_user_page_aliasing(void *kaddr, unsigned long vaddr)
|
||||
static void v6_clear_user_highpage_aliasing(struct page *page, unsigned long vaddr)
|
||||
{
|
||||
unsigned int offset = CACHE_COLOUR(vaddr);
|
||||
unsigned long to = to_address + (offset << PAGE_SHIFT);
|
||||
|
||||
/*
|
||||
* Discard data in the kernel mapping for the new page
|
||||
* FIXME: needs this MCRR to be supported.
|
||||
*/
|
||||
__asm__("mcrr p15, 0, %1, %0, c6 @ 0xec401f06"
|
||||
:
|
||||
: "r" (kaddr),
|
||||
"r" ((unsigned long)kaddr + PAGE_SIZE - L1_CACHE_BYTES)
|
||||
: "cc");
|
||||
/* FIXME: not highmem safe */
|
||||
discard_old_kernel_data(page_address(page));
|
||||
|
||||
/*
|
||||
* Now clear the page using the same cache colour as
|
||||
|
|
@ -128,7 +123,7 @@ static void v6_clear_user_page_aliasing(void *kaddr, unsigned long vaddr)
|
|||
*/
|
||||
spin_lock(&v6_lock);
|
||||
|
||||
set_pte_ext(TOP_PTE(to_address) + offset, pfn_pte(__pa(kaddr) >> PAGE_SHIFT, PAGE_KERNEL), 0);
|
||||
set_pte_ext(TOP_PTE(to_address) + offset, pfn_pte(page_to_pfn(page), PAGE_KERNEL), 0);
|
||||
flush_tlb_kernel_page(to);
|
||||
clear_page((void *)to);
|
||||
|
||||
|
|
@ -136,14 +131,14 @@ static void v6_clear_user_page_aliasing(void *kaddr, unsigned long vaddr)
|
|||
}
|
||||
|
||||
struct cpu_user_fns v6_user_fns __initdata = {
|
||||
.cpu_clear_user_page = v6_clear_user_page_nonaliasing,
|
||||
.cpu_clear_user_highpage = v6_clear_user_highpage_nonaliasing,
|
||||
.cpu_copy_user_highpage = v6_copy_user_highpage_nonaliasing,
|
||||
};
|
||||
|
||||
static int __init v6_userpage_init(void)
|
||||
{
|
||||
if (cache_is_vipt_aliasing()) {
|
||||
cpu_user.cpu_clear_user_page = v6_clear_user_page_aliasing;
|
||||
cpu_user.cpu_clear_user_highpage = v6_clear_user_highpage_aliasing;
|
||||
cpu_user.cpu_copy_user_highpage = v6_copy_user_highpage_aliasing;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -87,26 +87,27 @@ void xsc3_mc_copy_user_highpage(struct page *to, struct page *from,
|
|||
* r0 = destination
|
||||
* r1 = virtual user address of ultimate destination page
|
||||
*/
|
||||
void __attribute__((naked))
|
||||
xsc3_mc_clear_user_page(void *kaddr, unsigned long vaddr)
|
||||
void xsc3_mc_clear_user_highpage(struct page *page, unsigned long vaddr)
|
||||
{
|
||||
void *kaddr = kmap_atomic(page, KM_USER0);
|
||||
asm("\
|
||||
mov r1, %0 \n\
|
||||
mov r1, %1 \n\
|
||||
mov r2, #0 \n\
|
||||
mov r3, #0 \n\
|
||||
1: mcr p15, 0, r0, c7, c6, 1 @ invalidate line\n\
|
||||
strd r2, [r0], #8 \n\
|
||||
strd r2, [r0], #8 \n\
|
||||
strd r2, [r0], #8 \n\
|
||||
strd r2, [r0], #8 \n\
|
||||
1: mcr p15, 0, %0, c7, c6, 1 @ invalidate line\n\
|
||||
strd r2, [%0], #8 \n\
|
||||
strd r2, [%0], #8 \n\
|
||||
strd r2, [%0], #8 \n\
|
||||
strd r2, [%0], #8 \n\
|
||||
subs r1, r1, #1 \n\
|
||||
bne 1b \n\
|
||||
mov pc, lr"
|
||||
bne 1b"
|
||||
:
|
||||
: "I" (PAGE_SIZE / 32));
|
||||
: "r" (kaddr), "I" (PAGE_SIZE / 32)
|
||||
: "r1", "r2", "r3");
|
||||
kunmap_atomic(kaddr, KM_USER0);
|
||||
}
|
||||
|
||||
struct cpu_user_fns xsc3_mc_user_fns __initdata = {
|
||||
.cpu_clear_user_page = xsc3_mc_clear_user_page,
|
||||
.cpu_clear_user_highpage = xsc3_mc_clear_user_highpage,
|
||||
.cpu_copy_user_highpage = xsc3_mc_copy_user_highpage,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -113,28 +113,30 @@ void xscale_mc_copy_user_highpage(struct page *to, struct page *from,
|
|||
/*
|
||||
* XScale optimised clear_user_page
|
||||
*/
|
||||
void __attribute__((naked))
|
||||
xscale_mc_clear_user_page(void *kaddr, unsigned long vaddr)
|
||||
void
|
||||
xscale_mc_clear_user_highpage(struct page *page, unsigned long vaddr)
|
||||
{
|
||||
void *kaddr = kmap_atomic(page, KM_USER0);
|
||||
asm volatile(
|
||||
"mov r1, %0 \n\
|
||||
"mov r1, %1 \n\
|
||||
mov r2, #0 \n\
|
||||
mov r3, #0 \n\
|
||||
1: mov ip, r0 \n\
|
||||
strd r2, [r0], #8 \n\
|
||||
strd r2, [r0], #8 \n\
|
||||
strd r2, [r0], #8 \n\
|
||||
strd r2, [r0], #8 \n\
|
||||
1: mov ip, %0 \n\
|
||||
strd r2, [%0], #8 \n\
|
||||
strd r2, [%0], #8 \n\
|
||||
strd r2, [%0], #8 \n\
|
||||
strd r2, [%0], #8 \n\
|
||||
mcr p15, 0, ip, c7, c10, 1 @ clean D line\n\
|
||||
subs r1, r1, #1 \n\
|
||||
mcr p15, 0, ip, c7, c6, 1 @ invalidate D line\n\
|
||||
bne 1b \n\
|
||||
mov pc, lr"
|
||||
bne 1b"
|
||||
:
|
||||
: "I" (PAGE_SIZE / 32));
|
||||
: "r" (kaddr), "I" (PAGE_SIZE / 32)
|
||||
: "r1", "r2", "r3", "ip");
|
||||
kunmap_atomic(kaddr, KM_USER0);
|
||||
}
|
||||
|
||||
struct cpu_user_fns xscale_mc_user_fns __initdata = {
|
||||
.cpu_clear_user_page = xscale_mc_clear_user_page,
|
||||
.cpu_clear_user_highpage = xscale_mc_clear_user_highpage,
|
||||
.cpu_copy_user_highpage = xscale_mc_copy_user_highpage,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ EXPORT_SYMBOL(cpu_cache);
|
|||
|
||||
#ifdef CONFIG_MMU
|
||||
#ifndef MULTI_USER
|
||||
EXPORT_SYMBOL(__cpu_clear_user_page);
|
||||
EXPORT_SYMBOL(__cpu_clear_user_highpage);
|
||||
EXPORT_SYMBOL(__cpu_copy_user_highpage);
|
||||
#else
|
||||
EXPORT_SYMBOL(cpu_user);
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче