riscv: mm: Prepare pt_ops helper functions for sv57
This patch prepare some pt_ops helper functions which will be used in creating sv57 mappings during boot time. Signed-off-by: Qinglin Pan <panqinglin2020@iscas.ac.cn> Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
This commit is contained in:
Qinglin Pan
Palmer Dabbelt
Родитель
d10efa21a9
Коммит
677b9eb881
|
|
@ -25,6 +25,7 @@ enum fixed_addresses {
|
|||
FIX_PTE,
|
||||
FIX_PMD,
|
||||
FIX_PUD,
|
||||
FIX_P4D,
|
||||
FIX_TEXT_POKE1,
|
||||
FIX_TEXT_POKE0,
|
||||
FIX_EARLYCON_MEM_BASE,
|
||||
|
|
|
|||
|
|
@ -133,6 +133,8 @@ struct pt_alloc_ops {
|
|||
phys_addr_t (*alloc_pmd)(uintptr_t va);
|
||||
pud_t *(*get_pud_virt)(phys_addr_t pa);
|
||||
phys_addr_t (*alloc_pud)(uintptr_t va);
|
||||
p4d_t *(*get_p4d_virt)(phys_addr_t pa);
|
||||
phys_addr_t (*alloc_p4d)(uintptr_t va);
|
||||
#endif
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -229,6 +229,7 @@ pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss;
|
|||
static pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss;
|
||||
|
||||
pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);
|
||||
static p4d_t __maybe_unused early_dtb_p4d[PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE);
|
||||
static pud_t __maybe_unused early_dtb_pud[PTRS_PER_PUD] __initdata __aligned(PAGE_SIZE);
|
||||
static pmd_t __maybe_unused early_dtb_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE);
|
||||
|
||||
|
|
@ -319,6 +320,16 @@ static pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE);
|
|||
#define early_pmd ((pmd_t *)XIP_FIXUP(early_pmd))
|
||||
#endif /* CONFIG_XIP_KERNEL */
|
||||
|
||||
static p4d_t trampoline_p4d[PTRS_PER_P4D] __page_aligned_bss;
|
||||
static p4d_t fixmap_p4d[PTRS_PER_P4D] __page_aligned_bss;
|
||||
static p4d_t early_p4d[PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE);
|
||||
|
||||
#ifdef CONFIG_XIP_KERNEL
|
||||
#define trampoline_p4d ((p4d_t *)XIP_FIXUP(trampoline_p4d))
|
||||
#define fixmap_p4d ((p4d_t *)XIP_FIXUP(fixmap_p4d))
|
||||
#define early_p4d ((p4d_t *)XIP_FIXUP(early_p4d))
|
||||
#endif /* CONFIG_XIP_KERNEL */
|
||||
|
||||
static pud_t trampoline_pud[PTRS_PER_PUD] __page_aligned_bss;
|
||||
static pud_t fixmap_pud[PTRS_PER_PUD] __page_aligned_bss;
|
||||
static pud_t early_pud[PTRS_PER_PUD] __initdata __aligned(PAGE_SIZE);
|
||||
|
|
@ -433,6 +444,44 @@ static phys_addr_t alloc_pud_late(uintptr_t va)
|
|||
return __pa(vaddr);
|
||||
}
|
||||
|
||||
static p4d_t *__init get_p4d_virt_early(phys_addr_t pa)
|
||||
{
|
||||
return (p4d_t *)((uintptr_t)pa);
|
||||
}
|
||||
|
||||
static p4d_t *__init get_p4d_virt_fixmap(phys_addr_t pa)
|
||||
{
|
||||
clear_fixmap(FIX_P4D);
|
||||
return (p4d_t *)set_fixmap_offset(FIX_P4D, pa);
|
||||
}
|
||||
|
||||
static p4d_t *__init get_p4d_virt_late(phys_addr_t pa)
|
||||
{
|
||||
return (p4d_t *)__va(pa);
|
||||
}
|
||||
|
||||
static phys_addr_t __init alloc_p4d_early(uintptr_t va)
|
||||
{
|
||||
/* Only one P4D is available for early mapping */
|
||||
BUG_ON((va - kernel_map.virt_addr) >> PGDIR_SHIFT);
|
||||
|
||||
return (uintptr_t)early_p4d;
|
||||
}
|
||||
|
||||
static phys_addr_t __init alloc_p4d_fixmap(uintptr_t va)
|
||||
{
|
||||
return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE);
|
||||
}
|
||||
|
||||
static phys_addr_t alloc_p4d_late(uintptr_t va)
|
||||
{
|
||||
unsigned long vaddr;
|
||||
|
||||
vaddr = __get_free_page(GFP_KERNEL);
|
||||
BUG_ON(!vaddr);
|
||||
return __pa(vaddr);
|
||||
}
|
||||
|
||||
static void __init create_pud_mapping(pud_t *pudp,
|
||||
uintptr_t va, phys_addr_t pa,
|
||||
phys_addr_t sz, pgprot_t prot)
|
||||
|
|
@ -460,21 +509,55 @@ static void __init create_pud_mapping(pud_t *pudp,
|
|||
create_pmd_mapping(nextp, va, pa, sz, prot);
|
||||
}
|
||||
|
||||
#define pgd_next_t pud_t
|
||||
#define alloc_pgd_next(__va) (pgtable_l4_enabled ? \
|
||||
pt_ops.alloc_pud(__va) : pt_ops.alloc_pmd(__va))
|
||||
#define get_pgd_next_virt(__pa) (pgtable_l4_enabled ? \
|
||||
pt_ops.get_pud_virt(__pa) : (pgd_next_t *)pt_ops.get_pmd_virt(__pa))
|
||||
static void __init create_p4d_mapping(p4d_t *p4dp,
|
||||
uintptr_t va, phys_addr_t pa,
|
||||
phys_addr_t sz, pgprot_t prot)
|
||||
{
|
||||
pud_t *nextp;
|
||||
phys_addr_t next_phys;
|
||||
uintptr_t p4d_index = p4d_index(va);
|
||||
|
||||
if (sz == P4D_SIZE) {
|
||||
if (p4d_val(p4dp[p4d_index]) == 0)
|
||||
p4dp[p4d_index] = pfn_p4d(PFN_DOWN(pa), prot);
|
||||
return;
|
||||
}
|
||||
|
||||
if (p4d_val(p4dp[p4d_index]) == 0) {
|
||||
next_phys = pt_ops.alloc_pud(va);
|
||||
p4dp[p4d_index] = pfn_p4d(PFN_DOWN(next_phys), PAGE_TABLE);
|
||||
nextp = pt_ops.get_pud_virt(next_phys);
|
||||
memset(nextp, 0, PAGE_SIZE);
|
||||
} else {
|
||||
next_phys = PFN_PHYS(_p4d_pfn(p4dp[p4d_index]));
|
||||
nextp = pt_ops.get_pud_virt(next_phys);
|
||||
}
|
||||
|
||||
create_pud_mapping(nextp, va, pa, sz, prot);
|
||||
}
|
||||
|
||||
#define pgd_next_t p4d_t
|
||||
#define alloc_pgd_next(__va) (pgtable_l5_enabled ? \
|
||||
pt_ops.alloc_p4d(__va) : (pgtable_l4_enabled ? \
|
||||
pt_ops.alloc_pud(__va) : pt_ops.alloc_pmd(__va)))
|
||||
#define get_pgd_next_virt(__pa) (pgtable_l5_enabled ? \
|
||||
pt_ops.get_p4d_virt(__pa) : (pgd_next_t *)(pgtable_l4_enabled ? \
|
||||
pt_ops.get_pud_virt(__pa) : (pud_t *)pt_ops.get_pmd_virt(__pa)))
|
||||
#define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \
|
||||
(pgtable_l5_enabled ? \
|
||||
create_p4d_mapping(__nextp, __va, __pa, __sz, __prot) : \
|
||||
(pgtable_l4_enabled ? \
|
||||
create_pud_mapping(__nextp, __va, __pa, __sz, __prot) : \
|
||||
create_pmd_mapping((pmd_t *)__nextp, __va, __pa, __sz, __prot))
|
||||
#define fixmap_pgd_next (pgtable_l4_enabled ? \
|
||||
(uintptr_t)fixmap_pud : (uintptr_t)fixmap_pmd)
|
||||
#define trampoline_pgd_next (pgtable_l4_enabled ? \
|
||||
(uintptr_t)trampoline_pud : (uintptr_t)trampoline_pmd)
|
||||
#define early_dtb_pgd_next (pgtable_l4_enabled ? \
|
||||
(uintptr_t)early_dtb_pud : (uintptr_t)early_dtb_pmd)
|
||||
create_pud_mapping((pud_t *)__nextp, __va, __pa, __sz, __prot) : \
|
||||
create_pmd_mapping((pmd_t *)__nextp, __va, __pa, __sz, __prot)))
|
||||
#define fixmap_pgd_next (pgtable_l5_enabled ? \
|
||||
(uintptr_t)fixmap_p4d : (pgtable_l4_enabled ? \
|
||||
(uintptr_t)fixmap_pud : (uintptr_t)fixmap_pmd))
|
||||
#define trampoline_pgd_next (pgtable_l5_enabled ? \
|
||||
(uintptr_t)trampoline_p4d : (pgtable_l4_enabled ? \
|
||||
(uintptr_t)trampoline_pud : (uintptr_t)trampoline_pmd))
|
||||
#define early_dtb_pgd_next (pgtable_l5_enabled ? \
|
||||
(uintptr_t)early_dtb_p4d : (pgtable_l4_enabled ? \
|
||||
(uintptr_t)early_dtb_pud : (uintptr_t)early_dtb_pmd))
|
||||
#else
|
||||
#define pgd_next_t pte_t
|
||||
#define alloc_pgd_next(__va) pt_ops.alloc_pte(__va)
|
||||
|
|
@ -483,6 +566,7 @@ static void __init create_pud_mapping(pud_t *pudp,
|
|||
create_pte_mapping(__nextp, __va, __pa, __sz, __prot)
|
||||
#define fixmap_pgd_next ((uintptr_t)fixmap_pte)
|
||||
#define early_dtb_pgd_next ((uintptr_t)early_dtb_pmd)
|
||||
#define create_p4d_mapping(__pmdp, __va, __pa, __sz, __prot)
|
||||
#define create_pud_mapping(__pmdp, __va, __pa, __sz, __prot)
|
||||
#define create_pmd_mapping(__pmdp, __va, __pa, __sz, __prot)
|
||||
#endif /* __PAGETABLE_PMD_FOLDED */
|
||||
|
|
@ -693,10 +777,13 @@ static void __init create_fdt_early_page_table(pgd_t *pgdir, uintptr_t dtb_pa)
|
|||
PGDIR_SIZE,
|
||||
IS_ENABLED(CONFIG_64BIT) ? PAGE_TABLE : PAGE_KERNEL);
|
||||
|
||||
if (pgtable_l4_enabled) {
|
||||
if (pgtable_l5_enabled)
|
||||
create_p4d_mapping(early_dtb_p4d, DTB_EARLY_BASE_VA,
|
||||
(uintptr_t)early_dtb_pud, P4D_SIZE, PAGE_TABLE);
|
||||
|
||||
if (pgtable_l4_enabled)
|
||||
create_pud_mapping(early_dtb_pud, DTB_EARLY_BASE_VA,
|
||||
(uintptr_t)early_dtb_pmd, PUD_SIZE, PAGE_TABLE);
|
||||
}
|
||||
|
||||
if (IS_ENABLED(CONFIG_64BIT)) {
|
||||
create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA,
|
||||
|
|
@ -732,6 +819,8 @@ void __init pt_ops_set_early(void)
|
|||
pt_ops.get_pmd_virt = get_pmd_virt_early;
|
||||
pt_ops.alloc_pud = alloc_pud_early;
|
||||
pt_ops.get_pud_virt = get_pud_virt_early;
|
||||
pt_ops.alloc_p4d = alloc_p4d_early;
|
||||
pt_ops.get_p4d_virt = get_p4d_virt_early;
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
@ -752,6 +841,8 @@ void __init pt_ops_set_fixmap(void)
|
|||
pt_ops.get_pmd_virt = kernel_mapping_pa_to_va((uintptr_t)get_pmd_virt_fixmap);
|
||||
pt_ops.alloc_pud = kernel_mapping_pa_to_va((uintptr_t)alloc_pud_fixmap);
|
||||
pt_ops.get_pud_virt = kernel_mapping_pa_to_va((uintptr_t)get_pud_virt_fixmap);
|
||||
pt_ops.alloc_p4d = kernel_mapping_pa_to_va((uintptr_t)alloc_p4d_fixmap);
|
||||
pt_ops.get_p4d_virt = kernel_mapping_pa_to_va((uintptr_t)get_p4d_virt_fixmap);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
@ -768,6 +859,8 @@ void __init pt_ops_set_late(void)
|
|||
pt_ops.get_pmd_virt = get_pmd_virt_late;
|
||||
pt_ops.alloc_pud = alloc_pud_late;
|
||||
pt_ops.get_pud_virt = get_pud_virt_late;
|
||||
pt_ops.alloc_p4d = alloc_p4d_late;
|
||||
pt_ops.get_p4d_virt = get_p4d_virt_late;
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
@ -828,6 +921,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)
|
|||
fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE);
|
||||
|
||||
#ifndef __PAGETABLE_PMD_FOLDED
|
||||
/* Setup fixmap P4D and PUD */
|
||||
if (pgtable_l5_enabled)
|
||||
create_p4d_mapping(fixmap_p4d, FIXADDR_START,
|
||||
(uintptr_t)fixmap_pud, P4D_SIZE, PAGE_TABLE);
|
||||
/* Setup fixmap PUD and PMD */
|
||||
if (pgtable_l4_enabled)
|
||||
create_pud_mapping(fixmap_pud, FIXADDR_START,
|
||||
|
|
@ -837,6 +934,9 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)
|
|||
/* Setup trampoline PGD and PMD */
|
||||
create_pgd_mapping(trampoline_pg_dir, kernel_map.virt_addr,
|
||||
trampoline_pgd_next, PGDIR_SIZE, PAGE_TABLE);
|
||||
if (pgtable_l5_enabled)
|
||||
create_p4d_mapping(trampoline_p4d, kernel_map.virt_addr,
|
||||
(uintptr_t)trampoline_pud, P4D_SIZE, PAGE_TABLE);
|
||||
if (pgtable_l4_enabled)
|
||||
create_pud_mapping(trampoline_pud, kernel_map.virt_addr,
|
||||
(uintptr_t)trampoline_pmd, PUD_SIZE, PAGE_TABLE);
|
||||
|
|
@ -938,6 +1038,7 @@ static void __init setup_vm_final(void)
|
|||
clear_fixmap(FIX_PTE);
|
||||
clear_fixmap(FIX_PMD);
|
||||
clear_fixmap(FIX_PUD);
|
||||
clear_fixmap(FIX_P4D);
|
||||
|
||||
/* Move to swapper page table */
|
||||
csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | satp_mode);
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче