2019-05-29 17:17:58 +03:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2014-08-19 23:41:43 +04:00
|
|
|
/*
|
|
|
|
* Copyright (c) 2014, The Linux Foundation. All rights reserved.
|
|
|
|
*/
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/sched.h>
|
2016-01-27 12:50:19 +03:00
|
|
|
#include <linux/vmalloc.h>
|
2014-08-19 23:41:43 +04:00
|
|
|
|
2020-09-16 05:20:47 +03:00
|
|
|
#include <asm/cacheflush.h>
|
2017-05-09 01:58:05 +03:00
|
|
|
#include <asm/set_memory.h>
|
2014-08-19 23:41:43 +04:00
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
|
|
|
|
struct page_change_data {
|
|
|
|
pgprot_t set_mask;
|
|
|
|
pgprot_t clear_mask;
|
|
|
|
};
|
|
|
|
|
arm64: mm: apply r/o permissions of VM areas to its linear alias as well
On arm64, we use block mappings and contiguous hints to map the linear
region, to minimize the TLB footprint. However, this means that the
entire region is mapped using read/write permissions, which we cannot
modify at page granularity without having to take intrusive measures to
prevent TLB conflicts.
This means the linear aliases of pages belonging to read-only mappings
(executable or otherwise) in the vmalloc region are also mapped read/write,
and could potentially be abused to modify things like module code, bpf JIT
code or other read-only data.
So let's fix this, by extending the set_memory_ro/rw routines to take
the linear alias into account. The consequence of enabling this is
that we can no longer use block mappings or contiguous hints, so in
cases where the TLB footprint of the linear region is a bottleneck,
performance may be affected.
Therefore, allow this feature to be runtime en/disabled, by setting
rodata=full (or 'on' to disable just this enhancement, or 'off' to
disable read-only mappings for code and r/o data entirely) on the
kernel command line. Also, allow the default value to be set via a
Kconfig option.
Tested-by: Laura Abbott <labbott@redhat.com>
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2018-11-07 13:36:20 +03:00
|
|
|
bool rodata_full __ro_after_init = IS_ENABLED(CONFIG_RODATA_FULL_DEFAULT_ENABLED);
|
|
|
|
|
2019-07-12 06:58:43 +03:00
|
|
|
static int change_page_range(pte_t *ptep, unsigned long addr, void *data)
|
2014-08-19 23:41:43 +04:00
|
|
|
{
|
|
|
|
struct page_change_data *cdata = data;
|
2018-02-15 14:14:56 +03:00
|
|
|
pte_t pte = READ_ONCE(*ptep);
|
2014-08-19 23:41:43 +04:00
|
|
|
|
|
|
|
pte = clear_pte_bit(pte, cdata->clear_mask);
|
|
|
|
pte = set_pte_bit(pte, cdata->set_mask);
|
|
|
|
|
|
|
|
set_pte(ptep, pte);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-02-06 03:24:47 +03:00
|
|
|
/*
|
|
|
|
* This function assumes that the range is mapped with PAGE_SIZE pages.
|
|
|
|
*/
|
|
|
|
static int __change_memory_common(unsigned long start, unsigned long size,
|
|
|
|
pgprot_t set_mask, pgprot_t clear_mask)
|
|
|
|
{
|
|
|
|
struct page_change_data data;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
data.set_mask = set_mask;
|
|
|
|
data.clear_mask = clear_mask;
|
|
|
|
|
|
|
|
ret = apply_to_page_range(&init_mm, start, size, change_page_range,
|
|
|
|
&data);
|
|
|
|
|
|
|
|
flush_tlb_kernel_range(start, start + size);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2014-08-19 23:41:43 +04:00
|
|
|
static int change_memory_common(unsigned long addr, int numpages,
|
|
|
|
pgprot_t set_mask, pgprot_t clear_mask)
|
|
|
|
{
|
|
|
|
unsigned long start = addr;
|
2019-12-17 16:34:04 +03:00
|
|
|
unsigned long size = PAGE_SIZE * numpages;
|
2014-08-19 23:41:43 +04:00
|
|
|
unsigned long end = start + size;
|
2016-01-27 12:50:19 +03:00
|
|
|
struct vm_struct *area;
|
arm64: mm: apply r/o permissions of VM areas to its linear alias as well
On arm64, we use block mappings and contiguous hints to map the linear
region, to minimize the TLB footprint. However, this means that the
entire region is mapped using read/write permissions, which we cannot
modify at page granularity without having to take intrusive measures to
prevent TLB conflicts.
This means the linear aliases of pages belonging to read-only mappings
(executable or otherwise) in the vmalloc region are also mapped read/write,
and could potentially be abused to modify things like module code, bpf JIT
code or other read-only data.
So let's fix this, by extending the set_memory_ro/rw routines to take
the linear alias into account. The consequence of enabling this is
that we can no longer use block mappings or contiguous hints, so in
cases where the TLB footprint of the linear region is a bottleneck,
performance may be affected.
Therefore, allow this feature to be runtime en/disabled, by setting
rodata=full (or 'on' to disable just this enhancement, or 'off' to
disable read-only mappings for code and r/o data entirely) on the
kernel command line. Also, allow the default value to be set via a
Kconfig option.
Tested-by: Laura Abbott <labbott@redhat.com>
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2018-11-07 13:36:20 +03:00
|
|
|
int i;
|
2014-08-19 23:41:43 +04:00
|
|
|
|
2015-10-26 14:26:57 +03:00
|
|
|
if (!PAGE_ALIGNED(addr)) {
|
2014-09-12 02:10:32 +04:00
|
|
|
start &= PAGE_MASK;
|
|
|
|
end = start + size;
|
2014-08-19 23:41:43 +04:00
|
|
|
WARN_ON_ONCE(1);
|
|
|
|
}
|
|
|
|
|
2016-01-27 12:50:19 +03:00
|
|
|
/*
|
|
|
|
* Kernel VA mappings are always live, and splitting live section
|
|
|
|
* mappings into page mappings may cause TLB conflicts. This means
|
|
|
|
* we have to ensure that changing the permission bits of the range
|
|
|
|
* we are operating on does not result in such splitting.
|
|
|
|
*
|
|
|
|
* Let's restrict ourselves to mappings created by vmalloc (or vmap).
|
|
|
|
* Those are guaranteed to consist entirely of page mappings, and
|
|
|
|
* splitting is never needed.
|
|
|
|
*
|
|
|
|
* So check whether the [addr, addr + size) interval is entirely
|
|
|
|
* covered by precisely one VM area that has the VM_ALLOC flag set.
|
|
|
|
*/
|
|
|
|
area = find_vm_area((void *)addr);
|
|
|
|
if (!area ||
|
|
|
|
end > (unsigned long)area->addr + area->size ||
|
|
|
|
!(area->flags & VM_ALLOC))
|
2014-08-19 23:41:43 +04:00
|
|
|
return -EINVAL;
|
|
|
|
|
2016-01-26 18:47:25 +03:00
|
|
|
if (!numpages)
|
|
|
|
return 0;
|
|
|
|
|
arm64: mm: apply r/o permissions of VM areas to its linear alias as well
On arm64, we use block mappings and contiguous hints to map the linear
region, to minimize the TLB footprint. However, this means that the
entire region is mapped using read/write permissions, which we cannot
modify at page granularity without having to take intrusive measures to
prevent TLB conflicts.
This means the linear aliases of pages belonging to read-only mappings
(executable or otherwise) in the vmalloc region are also mapped read/write,
and could potentially be abused to modify things like module code, bpf JIT
code or other read-only data.
So let's fix this, by extending the set_memory_ro/rw routines to take
the linear alias into account. The consequence of enabling this is
that we can no longer use block mappings or contiguous hints, so in
cases where the TLB footprint of the linear region is a bottleneck,
performance may be affected.
Therefore, allow this feature to be runtime en/disabled, by setting
rodata=full (or 'on' to disable just this enhancement, or 'off' to
disable read-only mappings for code and r/o data entirely) on the
kernel command line. Also, allow the default value to be set via a
Kconfig option.
Tested-by: Laura Abbott <labbott@redhat.com>
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2018-11-07 13:36:20 +03:00
|
|
|
/*
|
|
|
|
* If we are manipulating read-only permissions, apply the same
|
|
|
|
* change to the linear mapping of the pages that back this VM area.
|
|
|
|
*/
|
|
|
|
if (rodata_full && (pgprot_val(set_mask) == PTE_RDONLY ||
|
|
|
|
pgprot_val(clear_mask) == PTE_RDONLY)) {
|
|
|
|
for (i = 0; i < area->nr_pages; i++) {
|
|
|
|
__change_memory_common((u64)page_address(area->pages[i]),
|
|
|
|
PAGE_SIZE, set_mask, clear_mask);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-11-07 13:36:19 +03:00
|
|
|
/*
|
|
|
|
* Get rid of potentially aliasing lazily unmapped vm areas that may
|
|
|
|
* have permissions set that deviate from the ones we are setting here.
|
|
|
|
*/
|
|
|
|
vm_unmap_aliases();
|
|
|
|
|
2016-02-06 03:24:47 +03:00
|
|
|
return __change_memory_common(start, size, set_mask, clear_mask);
|
2014-08-19 23:41:43 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
int set_memory_ro(unsigned long addr, int numpages)
|
|
|
|
{
|
|
|
|
return change_memory_common(addr, numpages,
|
|
|
|
__pgprot(PTE_RDONLY),
|
|
|
|
__pgprot(PTE_WRITE));
|
|
|
|
}
|
|
|
|
|
|
|
|
int set_memory_rw(unsigned long addr, int numpages)
|
|
|
|
{
|
|
|
|
return change_memory_common(addr, numpages,
|
|
|
|
__pgprot(PTE_WRITE),
|
|
|
|
__pgprot(PTE_RDONLY));
|
|
|
|
}
|
|
|
|
|
|
|
|
int set_memory_nx(unsigned long addr, int numpages)
|
|
|
|
{
|
|
|
|
return change_memory_common(addr, numpages,
|
|
|
|
__pgprot(PTE_PXN),
|
2020-05-06 22:51:33 +03:00
|
|
|
__pgprot(PTE_MAYBE_GP));
|
2014-08-19 23:41:43 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
int set_memory_x(unsigned long addr, int numpages)
|
|
|
|
{
|
|
|
|
return change_memory_common(addr, numpages,
|
2020-05-06 22:51:33 +03:00
|
|
|
__pgprot(PTE_MAYBE_GP),
|
2014-08-19 23:41:43 +04:00
|
|
|
__pgprot(PTE_PXN));
|
|
|
|
}
|
2016-02-06 03:24:47 +03:00
|
|
|
|
2017-04-03 05:24:33 +03:00
|
|
|
int set_memory_valid(unsigned long addr, int numpages, int enable)
|
2016-02-06 03:24:47 +03:00
|
|
|
{
|
|
|
|
if (enable)
|
2017-04-03 05:24:33 +03:00
|
|
|
return __change_memory_common(addr, PAGE_SIZE * numpages,
|
2016-02-06 03:24:47 +03:00
|
|
|
__pgprot(PTE_VALID),
|
|
|
|
__pgprot(0));
|
|
|
|
else
|
2017-04-03 05:24:33 +03:00
|
|
|
return __change_memory_common(addr, PAGE_SIZE * numpages,
|
2016-02-06 03:24:47 +03:00
|
|
|
__pgprot(0),
|
|
|
|
__pgprot(PTE_VALID));
|
|
|
|
}
|
2017-04-03 05:24:33 +03:00
|
|
|
|
2019-05-23 13:22:54 +03:00
|
|
|
int set_direct_map_invalid_noflush(struct page *page)
|
|
|
|
{
|
|
|
|
struct page_change_data data = {
|
|
|
|
.set_mask = __pgprot(0),
|
|
|
|
.clear_mask = __pgprot(PTE_VALID),
|
|
|
|
};
|
|
|
|
|
2020-12-15 06:10:30 +03:00
|
|
|
if (!debug_pagealloc_enabled() && !rodata_full)
|
2019-05-23 13:22:54 +03:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
return apply_to_page_range(&init_mm,
|
|
|
|
(unsigned long)page_address(page),
|
|
|
|
PAGE_SIZE, change_page_range, &data);
|
|
|
|
}
|
|
|
|
|
|
|
|
int set_direct_map_default_noflush(struct page *page)
|
|
|
|
{
|
|
|
|
struct page_change_data data = {
|
|
|
|
.set_mask = __pgprot(PTE_VALID | PTE_WRITE),
|
|
|
|
.clear_mask = __pgprot(PTE_RDONLY),
|
|
|
|
};
|
|
|
|
|
2020-12-15 06:10:30 +03:00
|
|
|
if (!debug_pagealloc_enabled() && !rodata_full)
|
2019-05-23 13:22:54 +03:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
return apply_to_page_range(&init_mm,
|
|
|
|
(unsigned long)page_address(page),
|
|
|
|
PAGE_SIZE, change_page_range, &data);
|
|
|
|
}
|
|
|
|
|
2020-12-15 06:10:30 +03:00
|
|
|
#ifdef CONFIG_DEBUG_PAGEALLOC
|
2017-04-03 05:24:33 +03:00
|
|
|
void __kernel_map_pages(struct page *page, int numpages, int enable)
|
|
|
|
{
|
2019-05-23 13:22:54 +03:00
|
|
|
if (!debug_pagealloc_enabled() && !rodata_full)
|
|
|
|
return;
|
|
|
|
|
2017-04-03 05:24:33 +03:00
|
|
|
set_memory_valid((unsigned long)page_address(page), numpages, enable);
|
|
|
|
}
|
2020-12-15 06:10:35 +03:00
|
|
|
#endif /* CONFIG_DEBUG_PAGEALLOC */
|
2019-05-23 13:22:54 +03:00
|
|
|
|
2016-08-24 20:27:30 +03:00
|
|
|
/*
|
2019-05-23 13:22:54 +03:00
|
|
|
* This function is used to determine if a linear map page has been marked as
|
|
|
|
* not-valid. Walk the page table and check the PTE_VALID bit. This is based
|
|
|
|
* on kern_addr_valid(), which almost does what we need.
|
2016-08-24 20:27:30 +03:00
|
|
|
*
|
|
|
|
* Because this is only called on the kernel linear map, p?d_sect() implies
|
|
|
|
* p?d_present(). When debug_pagealloc is enabled, sections mappings are
|
|
|
|
* disabled.
|
|
|
|
*/
|
|
|
|
bool kernel_page_present(struct page *page)
|
|
|
|
{
|
2018-02-15 14:14:56 +03:00
|
|
|
pgd_t *pgdp;
|
2020-06-05 02:46:23 +03:00
|
|
|
p4d_t *p4dp;
|
2018-02-15 14:14:56 +03:00
|
|
|
pud_t *pudp, pud;
|
|
|
|
pmd_t *pmdp, pmd;
|
|
|
|
pte_t *ptep;
|
2016-08-24 20:27:30 +03:00
|
|
|
unsigned long addr = (unsigned long)page_address(page);
|
|
|
|
|
2019-05-23 13:22:54 +03:00
|
|
|
if (!debug_pagealloc_enabled() && !rodata_full)
|
|
|
|
return true;
|
|
|
|
|
2018-02-15 14:14:56 +03:00
|
|
|
pgdp = pgd_offset_k(addr);
|
|
|
|
if (pgd_none(READ_ONCE(*pgdp)))
|
2016-08-24 20:27:30 +03:00
|
|
|
return false;
|
|
|
|
|
2020-06-05 02:46:23 +03:00
|
|
|
p4dp = p4d_offset(pgdp, addr);
|
|
|
|
if (p4d_none(READ_ONCE(*p4dp)))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
pudp = pud_offset(p4dp, addr);
|
2018-02-15 14:14:56 +03:00
|
|
|
pud = READ_ONCE(*pudp);
|
|
|
|
if (pud_none(pud))
|
2016-08-24 20:27:30 +03:00
|
|
|
return false;
|
2018-02-15 14:14:56 +03:00
|
|
|
if (pud_sect(pud))
|
2016-08-24 20:27:30 +03:00
|
|
|
return true;
|
|
|
|
|
2018-02-15 14:14:56 +03:00
|
|
|
pmdp = pmd_offset(pudp, addr);
|
|
|
|
pmd = READ_ONCE(*pmdp);
|
|
|
|
if (pmd_none(pmd))
|
2016-08-24 20:27:30 +03:00
|
|
|
return false;
|
2018-02-15 14:14:56 +03:00
|
|
|
if (pmd_sect(pmd))
|
2016-08-24 20:27:30 +03:00
|
|
|
return true;
|
|
|
|
|
2018-02-15 14:14:56 +03:00
|
|
|
ptep = pte_offset_kernel(pmdp, addr);
|
|
|
|
return pte_valid(READ_ONCE(*ptep));
|
2016-08-24 20:27:30 +03:00
|
|
|
}
|