2007-12-14 04:35:10 +03:00
|
|
|
#ifndef __KVM_X86_MMU_H
|
|
|
|
#define __KVM_X86_MMU_H
|
|
|
|
|
2007-12-16 12:02:48 +03:00
|
|
|
#include <linux/kvm_host.h>
|
2009-12-07 13:16:48 +03:00
|
|
|
#include "kvm_cache_regs.h"
|
2007-12-14 04:35:10 +03:00
|
|
|
|
2008-04-25 06:17:08 +04:00
|
|
|
#define PT64_PT_BITS 9
|
|
|
|
#define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS)
|
|
|
|
#define PT32_PT_BITS 10
|
|
|
|
#define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS)
|
|
|
|
|
|
|
|
#define PT_WRITABLE_SHIFT 1
|
|
|
|
|
|
|
|
#define PT_PRESENT_MASK (1ULL << 0)
|
|
|
|
#define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
|
|
|
|
#define PT_USER_MASK (1ULL << 2)
|
|
|
|
#define PT_PWT_MASK (1ULL << 3)
|
|
|
|
#define PT_PCD_MASK (1ULL << 4)
|
2008-05-15 14:51:35 +04:00
|
|
|
#define PT_ACCESSED_SHIFT 5
|
|
|
|
#define PT_ACCESSED_MASK (1ULL << PT_ACCESSED_SHIFT)
|
2012-09-12 14:44:53 +04:00
|
|
|
#define PT_DIRTY_SHIFT 6
|
|
|
|
#define PT_DIRTY_MASK (1ULL << PT_DIRTY_SHIFT)
|
2012-09-12 21:46:56 +04:00
|
|
|
#define PT_PAGE_SIZE_SHIFT 7
|
|
|
|
#define PT_PAGE_SIZE_MASK (1ULL << PT_PAGE_SIZE_SHIFT)
|
2008-04-25 06:17:08 +04:00
|
|
|
#define PT_PAT_MASK (1ULL << 7)
|
|
|
|
#define PT_GLOBAL_MASK (1ULL << 8)
|
|
|
|
#define PT64_NX_SHIFT 63
|
|
|
|
#define PT64_NX_MASK (1ULL << PT64_NX_SHIFT)
|
|
|
|
|
|
|
|
#define PT_PAT_SHIFT 7
|
|
|
|
#define PT_DIR_PAT_SHIFT 12
|
|
|
|
#define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)
|
|
|
|
|
|
|
|
#define PT32_DIR_PSE36_SIZE 4
|
|
|
|
#define PT32_DIR_PSE36_SHIFT 13
|
|
|
|
#define PT32_DIR_PSE36_MASK \
|
|
|
|
(((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
|
|
|
|
|
|
|
|
#define PT64_ROOT_LEVEL 4
|
|
|
|
#define PT32_ROOT_LEVEL 2
|
|
|
|
#define PT32E_ROOT_LEVEL 3
|
|
|
|
|
2010-01-05 14:02:26 +03:00
|
|
|
#define PT_PDPE_LEVEL 3
|
|
|
|
#define PT_DIRECTORY_LEVEL 2
|
|
|
|
#define PT_PAGE_TABLE_LEVEL 1
|
|
|
|
|
2010-02-10 15:21:32 +03:00
|
|
|
#define PFERR_PRESENT_MASK (1U << 0)
|
|
|
|
#define PFERR_WRITE_MASK (1U << 1)
|
|
|
|
#define PFERR_USER_MASK (1U << 2)
|
|
|
|
#define PFERR_RSVD_MASK (1U << 3)
|
|
|
|
#define PFERR_FETCH_MASK (1U << 4)
|
|
|
|
|
2009-06-11 19:07:42 +04:00
|
|
|
int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4]);
|
2011-07-11 23:33:44 +04:00
|
|
|
void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask);
|
2013-06-07 12:51:25 +04:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Return values of handle_mmio_page_fault_common:
|
|
|
|
* RET_MMIO_PF_EMULATE: it is a real mmio page fault, emulate the instruction
|
2013-06-07 12:51:26 +04:00
|
|
|
* directly.
|
|
|
|
* RET_MMIO_PF_INVALID: invalid spte is detected then let the real page
|
|
|
|
* fault path update the mmio spte.
|
2013-06-07 12:51:25 +04:00
|
|
|
* RET_MMIO_PF_RETRY: let CPU fault again on the address.
|
|
|
|
* RET_MMIO_PF_BUG: bug is detected.
|
|
|
|
*/
|
|
|
|
enum {
|
|
|
|
RET_MMIO_PF_EMULATE = 1,
|
2013-06-07 12:51:26 +04:00
|
|
|
RET_MMIO_PF_INVALID = 2,
|
2013-06-07 12:51:25 +04:00
|
|
|
RET_MMIO_PF_RETRY = 0,
|
|
|
|
RET_MMIO_PF_BUG = -1
|
|
|
|
};
|
|
|
|
|
2011-07-11 23:33:44 +04:00
|
|
|
int handle_mmio_page_fault_common(struct kvm_vcpu *vcpu, u64 addr, bool direct);
|
2013-10-02 18:56:13 +04:00
|
|
|
void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *context);
|
|
|
|
void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *context,
|
2013-08-05 12:07:16 +04:00
|
|
|
bool execonly);
|
2009-06-11 19:07:42 +04:00
|
|
|
|
2010-08-20 05:11:05 +04:00
|
|
|
static inline unsigned int kvm_mmu_available_pages(struct kvm *kvm)
|
|
|
|
{
|
2013-03-13 05:36:43 +04:00
|
|
|
if (kvm->arch.n_max_mmu_pages > kvm->arch.n_used_mmu_pages)
|
|
|
|
return kvm->arch.n_max_mmu_pages -
|
|
|
|
kvm->arch.n_used_mmu_pages;
|
|
|
|
|
|
|
|
return 0;
|
2010-08-20 05:11:05 +04:00
|
|
|
}
|
|
|
|
|
2007-12-14 04:35:10 +03:00
|
|
|
static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
|
|
|
if (likely(vcpu->arch.mmu.root_hpa != INVALID_PAGE))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
return kvm_mmu_load(vcpu);
|
|
|
|
}
|
|
|
|
|
2009-06-10 15:12:05 +04:00
|
|
|
static inline int is_present_gpte(unsigned long pte)
|
2009-03-31 19:03:45 +04:00
|
|
|
{
|
|
|
|
return pte & PT_PRESENT_MASK;
|
|
|
|
}
|
|
|
|
|
2011-07-11 23:23:20 +04:00
|
|
|
static inline int is_writable_pte(unsigned long pte)
|
|
|
|
{
|
|
|
|
return pte & PT_WRITABLE_MASK;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool is_write_protection(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
|
|
|
return kvm_read_cr0_bits(vcpu, X86_CR0_WP);
|
|
|
|
}
|
|
|
|
|
KVM: MMU: Optimize pte permission checks
walk_addr_generic() permission checks are a maze of branchy code, which is
performed four times per lookup. It depends on the type of access, efer.nxe,
cr0.wp, cr4.smep, and in the near future, cr4.smap.
Optimize this away by precalculating all variants and storing them in a
bitmap. The bitmap is recalculated when rarely-changing variables change
(cr0, cr4) and is indexed by the often-changing variables (page fault error
code, pte access permissions).
The permission check is moved to the end of the loop, otherwise an SMEP
fault could be reported as a false positive, when PDE.U=1 but PTE.U=0.
Noted by Xiao Guangrong.
The result is short, branch-free code.
Reviewed-by: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Signed-off-by: Avi Kivity <avi@redhat.com>
2012-09-12 15:52:00 +04:00
|
|
|
/*
|
|
|
|
* Will a fault with a given page-fault error code (pfec) cause a permission
|
|
|
|
* fault with the given access (in ACC_* format)?
|
|
|
|
*/
|
|
|
|
static inline bool permission_fault(struct kvm_mmu *mmu, unsigned pte_access,
|
|
|
|
unsigned pfec)
|
2011-07-11 23:23:20 +04:00
|
|
|
{
|
KVM: MMU: Optimize pte permission checks
walk_addr_generic() permission checks are a maze of branchy code, which is
performed four times per lookup. It depends on the type of access, efer.nxe,
cr0.wp, cr4.smep, and in the near future, cr4.smap.
Optimize this away by precalculating all variants and storing them in a
bitmap. The bitmap is recalculated when rarely-changing variables change
(cr0, cr4) and is indexed by the often-changing variables (page fault error
code, pte access permissions).
The permission check is moved to the end of the loop, otherwise an SMEP
fault could be reported as a false positive, when PDE.U=1 but PTE.U=0.
Noted by Xiao Guangrong.
The result is short, branch-free code.
Reviewed-by: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Signed-off-by: Avi Kivity <avi@redhat.com>
2012-09-12 15:52:00 +04:00
|
|
|
return (mmu->permissions[pfec >> 1] >> pte_access) & 1;
|
2011-07-11 23:23:20 +04:00
|
|
|
}
|
KVM: MMU: Optimize pte permission checks
walk_addr_generic() permission checks are a maze of branchy code, which is
performed four times per lookup. It depends on the type of access, efer.nxe,
cr0.wp, cr4.smep, and in the near future, cr4.smap.
Optimize this away by precalculating all variants and storing them in a
bitmap. The bitmap is recalculated when rarely-changing variables change
(cr0, cr4) and is indexed by the often-changing variables (page fault error
code, pte access permissions).
The permission check is moved to the end of the loop, otherwise an SMEP
fault could be reported as a false positive, when PDE.U=1 but PTE.U=0.
Noted by Xiao Guangrong.
The result is short, branch-free code.
Reviewed-by: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Signed-off-by: Avi Kivity <avi@redhat.com>
2012-09-12 15:52:00 +04:00
|
|
|
|
2013-05-31 04:36:22 +04:00
|
|
|
void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm);
|
2007-12-14 04:35:10 +03:00
|
|
|
#endif
|