riscv: mm: Support kasan for sv57
This patchset add kasan_populate and kasan_shallow_populate for sv57, and is tested on both qemu and unmatched with CONFIG_KASAN and CONFIG_KASAN_VMALLOC on. Signed-off-by: Qinglin Pan <panqinglin2020@iscas.ac.cn> Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
This commit is contained in:
Qinglin Pan
Palmer Dabbelt
Родитель
011f09d120
Коммит
8fbdccd2b1
|
|
@ -111,6 +111,8 @@ static void __init kasan_populate_pud(pgd_t *pgd,
|
|||
* pt_ops facility.
|
||||
*/
|
||||
base_pud = pt_ops.get_pud_virt(pfn_to_phys(_pgd_pfn(*pgd)));
|
||||
} else if (pgd_none(*pgd)) {
|
||||
base_pud = memblock_alloc(PTRS_PER_PUD * sizeof(pud_t), PAGE_SIZE);
|
||||
} else {
|
||||
base_pud = (pud_t *)pgd_page_vaddr(*pgd);
|
||||
if (base_pud == lm_alias(kasan_early_shadow_pud))
|
||||
|
|
@ -149,13 +151,72 @@ static void __init kasan_populate_pud(pgd_t *pgd,
|
|||
set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_pud)), PAGE_TABLE));
|
||||
}
|
||||
|
||||
#define kasan_early_shadow_pgd_next (pgtable_l4_enabled ? \
|
||||
static void __init kasan_populate_p4d(pgd_t *pgd,
|
||||
unsigned long vaddr, unsigned long end,
|
||||
bool early)
|
||||
{
|
||||
phys_addr_t phys_addr;
|
||||
p4d_t *p4dp, *base_p4d;
|
||||
unsigned long next;
|
||||
|
||||
if (early) {
|
||||
/*
|
||||
* We can't use pgd_page_vaddr here as it would return a linear
|
||||
* mapping address but it is not mapped yet, but when populating
|
||||
* early_pg_dir, we need the physical address and when populating
|
||||
* swapper_pg_dir, we need the kernel virtual address so use
|
||||
* pt_ops facility.
|
||||
*/
|
||||
base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(*pgd)));
|
||||
} else {
|
||||
base_p4d = (p4d_t *)pgd_page_vaddr(*pgd);
|
||||
if (base_p4d == lm_alias(kasan_early_shadow_p4d))
|
||||
base_p4d = memblock_alloc(PTRS_PER_PUD * sizeof(p4d_t), PAGE_SIZE);
|
||||
}
|
||||
|
||||
p4dp = base_p4d + p4d_index(vaddr);
|
||||
|
||||
do {
|
||||
next = p4d_addr_end(vaddr, end);
|
||||
|
||||
if (p4d_none(*p4dp) && IS_ALIGNED(vaddr, P4D_SIZE) && (next - vaddr) >= P4D_SIZE) {
|
||||
if (early) {
|
||||
phys_addr = __pa(((uintptr_t)kasan_early_shadow_pud));
|
||||
set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_TABLE));
|
||||
continue;
|
||||
} else {
|
||||
phys_addr = memblock_phys_alloc(P4D_SIZE, P4D_SIZE);
|
||||
if (phys_addr) {
|
||||
set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_KERNEL));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
kasan_populate_pud((pgd_t *)p4dp, vaddr, next, early);
|
||||
} while (p4dp++, vaddr = next, vaddr != end);
|
||||
|
||||
/*
|
||||
* Wait for the whole P4D to be populated before setting the P4D in
|
||||
* the page table, otherwise, if we did set the P4D before populating
|
||||
* it entirely, memblock could allocate a page at a physical address
|
||||
* where KASAN is not populated yet and then we'd get a page fault.
|
||||
*/
|
||||
if (!early)
|
||||
set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_p4d)), PAGE_TABLE));
|
||||
}
|
||||
|
||||
#define kasan_early_shadow_pgd_next (pgtable_l5_enabled ? \
|
||||
(uintptr_t)kasan_early_shadow_p4d : \
|
||||
(pgtable_l4_enabled ? \
|
||||
(uintptr_t)kasan_early_shadow_pud : \
|
||||
(uintptr_t)kasan_early_shadow_pmd)
|
||||
(uintptr_t)kasan_early_shadow_pmd))
|
||||
#define kasan_populate_pgd_next(pgdp, vaddr, next, early) \
|
||||
(pgtable_l5_enabled ? \
|
||||
kasan_populate_p4d(pgdp, vaddr, next, early) : \
|
||||
(pgtable_l4_enabled ? \
|
||||
kasan_populate_pud(pgdp, vaddr, next, early) : \
|
||||
kasan_populate_pmd((pud_t *)pgdp, vaddr, next))
|
||||
kasan_populate_pmd((pud_t *)pgdp, vaddr, next)))
|
||||
|
||||
static void __init kasan_populate_pgd(pgd_t *pgdp,
|
||||
unsigned long vaddr, unsigned long end,
|
||||
|
|
@ -219,6 +280,14 @@ asmlinkage void __init kasan_early_init(void)
|
|||
PAGE_TABLE));
|
||||
}
|
||||
|
||||
if (pgtable_l5_enabled) {
|
||||
for (i = 0; i < PTRS_PER_P4D; ++i)
|
||||
set_p4d(kasan_early_shadow_p4d + i,
|
||||
pfn_p4d(PFN_DOWN
|
||||
(__pa(((uintptr_t)kasan_early_shadow_pud))),
|
||||
PAGE_TABLE));
|
||||
}
|
||||
|
||||
kasan_populate_pgd(early_pg_dir + pgd_index(KASAN_SHADOW_START),
|
||||
KASAN_SHADOW_START, KASAN_SHADOW_END, true);
|
||||
|
||||
|
|
@ -244,9 +313,27 @@ static void __init kasan_populate(void *start, void *end)
|
|||
memset(start, KASAN_SHADOW_INIT, end - start);
|
||||
}
|
||||
|
||||
static void __init kasan_shallow_populate_pmd(pgd_t *pgdp,
|
||||
unsigned long vaddr, unsigned long end)
|
||||
{
|
||||
unsigned long next;
|
||||
pmd_t *pmdp, *base_pmd;
|
||||
bool is_kasan_pte;
|
||||
|
||||
base_pmd = (pmd_t *)pgd_page_vaddr(*pgdp);
|
||||
pmdp = base_pmd + pmd_index(vaddr);
|
||||
|
||||
do {
|
||||
next = pmd_addr_end(vaddr, end);
|
||||
is_kasan_pte = (pmd_pgtable(*pmdp) == lm_alias(kasan_early_shadow_pte));
|
||||
|
||||
if (is_kasan_pte)
|
||||
pmd_clear(pmdp);
|
||||
} while (pmdp++, vaddr = next, vaddr != end);
|
||||
}
|
||||
|
||||
static void __init kasan_shallow_populate_pud(pgd_t *pgdp,
|
||||
unsigned long vaddr, unsigned long end,
|
||||
bool kasan_populate)
|
||||
unsigned long vaddr, unsigned long end)
|
||||
{
|
||||
unsigned long next;
|
||||
pud_t *pudp, *base_pud;
|
||||
|
|
@ -256,21 +343,60 @@ static void __init kasan_shallow_populate_pud(pgd_t *pgdp,
|
|||
base_pud = (pud_t *)pgd_page_vaddr(*pgdp);
|
||||
pudp = base_pud + pud_index(vaddr);
|
||||
|
||||
if (kasan_populate)
|
||||
memcpy(base_pud, (void *)kasan_early_shadow_pgd_next,
|
||||
sizeof(pud_t) * PTRS_PER_PUD);
|
||||
|
||||
do {
|
||||
next = pud_addr_end(vaddr, end);
|
||||
is_kasan_pmd = (pud_pgtable(*pudp) == lm_alias(kasan_early_shadow_pmd));
|
||||
|
||||
if (is_kasan_pmd) {
|
||||
if (!is_kasan_pmd)
|
||||
continue;
|
||||
|
||||
base_pmd = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
|
||||
set_pud(pudp, pfn_pud(PFN_DOWN(__pa(base_pmd)), PAGE_TABLE));
|
||||
}
|
||||
|
||||
if (IS_ALIGNED(vaddr, PUD_SIZE) && (next - vaddr) >= PUD_SIZE)
|
||||
continue;
|
||||
|
||||
memcpy(base_pmd, (void *)kasan_early_shadow_pmd, PAGE_SIZE);
|
||||
kasan_shallow_populate_pmd((pgd_t *)pudp, vaddr, next);
|
||||
} while (pudp++, vaddr = next, vaddr != end);
|
||||
}
|
||||
|
||||
static void __init kasan_shallow_populate_p4d(pgd_t *pgdp,
|
||||
unsigned long vaddr, unsigned long end)
|
||||
{
|
||||
unsigned long next;
|
||||
p4d_t *p4dp, *base_p4d;
|
||||
pud_t *base_pud;
|
||||
bool is_kasan_pud;
|
||||
|
||||
base_p4d = (p4d_t *)pgd_page_vaddr(*pgdp);
|
||||
p4dp = base_p4d + p4d_index(vaddr);
|
||||
|
||||
do {
|
||||
next = p4d_addr_end(vaddr, end);
|
||||
is_kasan_pud = (p4d_pgtable(*p4dp) == lm_alias(kasan_early_shadow_pud));
|
||||
|
||||
if (!is_kasan_pud)
|
||||
continue;
|
||||
|
||||
base_pud = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
|
||||
set_p4d(p4dp, pfn_p4d(PFN_DOWN(__pa(base_pud)), PAGE_TABLE));
|
||||
|
||||
if (IS_ALIGNED(vaddr, P4D_SIZE) && (next - vaddr) >= P4D_SIZE)
|
||||
continue;
|
||||
|
||||
memcpy(base_pud, (void *)kasan_early_shadow_pud, PAGE_SIZE);
|
||||
kasan_shallow_populate_pud((pgd_t *)p4dp, vaddr, next);
|
||||
} while (p4dp++, vaddr = next, vaddr != end);
|
||||
}
|
||||
|
||||
#define kasan_shallow_populate_pgd_next(pgdp, vaddr, next) \
|
||||
(pgtable_l5_enabled ? \
|
||||
kasan_shallow_populate_p4d(pgdp, vaddr, next) : \
|
||||
(pgtable_l4_enabled ? \
|
||||
kasan_shallow_populate_pud(pgdp, vaddr, next) : \
|
||||
kasan_shallow_populate_pmd(pgdp, vaddr, next)))
|
||||
|
||||
static void __init kasan_shallow_populate_pgd(unsigned long vaddr, unsigned long end)
|
||||
{
|
||||
unsigned long next;
|
||||
|
|
@ -291,7 +417,8 @@ static void __init kasan_shallow_populate_pgd(unsigned long vaddr, unsigned long
|
|||
if (IS_ALIGNED(vaddr, PGDIR_SIZE) && (next - vaddr) >= PGDIR_SIZE)
|
||||
continue;
|
||||
|
||||
kasan_shallow_populate_pud(pgd_k, vaddr, next, is_kasan_pgd_next);
|
||||
memcpy(p, (void *)kasan_early_shadow_pgd_next, PAGE_SIZE);
|
||||
kasan_shallow_populate_pgd_next(pgd_k, vaddr, next);
|
||||
} while (pgd_k++, vaddr = next, vaddr != end);
|
||||
}
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче