KVM: MMU: remove the arithmetic of parent pte rmap
Parent pte rmap and page rmap are very similar, so use the same arithmetic for them Signed-off-by: Xiao Guangrong <xiaoguangrong@cn.fujitsu.com> Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
This commit is contained in:
Xiao Guangrong
Avi Kivity
Родитель
53c07b1878
Коммит
67052b3508
|
|
@ -227,14 +227,10 @@ struct kvm_mmu_page {
|
|||
* in this shadow page.
|
||||
*/
|
||||
DECLARE_BITMAP(slot_bitmap, KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS);
|
||||
bool multimapped; /* More than one parent_pte? */
|
||||
bool unsync;
|
||||
int root_count; /* Currently serving as active root */
|
||||
unsigned int unsync_children;
|
||||
union {
|
||||
u64 *parent_pte; /* !multimapped */
|
||||
struct hlist_head parent_ptes; /* multimapped, kvm_pte_chain */
|
||||
};
|
||||
unsigned long parent_ptes; /* Reverse mapping for parent_pte */
|
||||
DECLARE_BITMAP(unsync_child_bitmap, 512);
|
||||
};
|
||||
|
||||
|
|
@ -346,7 +342,6 @@ struct kvm_vcpu_arch {
|
|||
* put it here to avoid allocation */
|
||||
struct kvm_pv_mmu_op_buffer mmu_op_buffer;
|
||||
|
||||
struct kvm_mmu_memory_cache mmu_pte_chain_cache;
|
||||
struct kvm_mmu_memory_cache mmu_pte_list_desc_cache;
|
||||
struct kvm_mmu_memory_cache mmu_page_cache;
|
||||
struct kvm_mmu_memory_cache mmu_page_header_cache;
|
||||
|
|
|
|||
|
|
@ -182,9 +182,6 @@ struct kvm_shadow_walk_iterator {
|
|||
shadow_walk_okay(&(_walker)); \
|
||||
shadow_walk_next(&(_walker)))
|
||||
|
||||
typedef void (*mmu_parent_walk_fn) (struct kvm_mmu_page *sp, u64 *spte);
|
||||
|
||||
static struct kmem_cache *pte_chain_cache;
|
||||
static struct kmem_cache *pte_list_desc_cache;
|
||||
static struct kmem_cache *mmu_page_header_cache;
|
||||
static struct percpu_counter kvm_total_used_mmu_pages;
|
||||
|
|
@ -397,12 +394,8 @@ static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
|
|||
{
|
||||
int r;
|
||||
|
||||
r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache,
|
||||
pte_chain_cache, 4);
|
||||
if (r)
|
||||
goto out;
|
||||
r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
|
||||
pte_list_desc_cache, 4 + PTE_PREFETCH_NUM);
|
||||
pte_list_desc_cache, 8 + PTE_PREFETCH_NUM);
|
||||
if (r)
|
||||
goto out;
|
||||
r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
|
||||
|
|
@ -416,8 +409,6 @@ out:
|
|||
|
||||
static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
mmu_free_memory_cache(&vcpu->arch.mmu_pte_chain_cache,
|
||||
pte_chain_cache);
|
||||
mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
|
||||
pte_list_desc_cache);
|
||||
mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
|
||||
|
|
@ -435,17 +426,6 @@ static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
|
|||
return p;
|
||||
}
|
||||
|
||||
static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_chain_cache,
|
||||
sizeof(struct kvm_pte_chain));
|
||||
}
|
||||
|
||||
static void mmu_free_pte_chain(struct kvm_pte_chain *pc)
|
||||
{
|
||||
kmem_cache_free(pte_chain_cache, pc);
|
||||
}
|
||||
|
||||
static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache,
|
||||
|
|
@ -721,6 +701,26 @@ static void pte_list_remove(u64 *spte, unsigned long *pte_list)
|
|||
}
|
||||
}
|
||||
|
||||
typedef void (*pte_list_walk_fn) (u64 *spte);
|
||||
static void pte_list_walk(unsigned long *pte_list, pte_list_walk_fn fn)
|
||||
{
|
||||
struct pte_list_desc *desc;
|
||||
int i;
|
||||
|
||||
if (!*pte_list)
|
||||
return;
|
||||
|
||||
if (!(*pte_list & 1))
|
||||
return fn((u64 *)*pte_list);
|
||||
|
||||
desc = (struct pte_list_desc *)(*pte_list & ~1ul);
|
||||
while (desc) {
|
||||
for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
|
||||
fn(desc->sptes[i]);
|
||||
desc = desc->more;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Take gfn and return the reverse mapping to it.
|
||||
*/
|
||||
|
|
@ -1069,12 +1069,27 @@ static unsigned kvm_page_table_hashfn(gfn_t gfn)
|
|||
return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
|
||||
}
|
||||
|
||||
static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
|
||||
struct kvm_mmu_page *sp, u64 *parent_pte)
|
||||
{
|
||||
if (!parent_pte)
|
||||
return;
|
||||
|
||||
pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
|
||||
}
|
||||
|
||||
static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
|
||||
u64 *parent_pte)
|
||||
{
|
||||
pte_list_remove(parent_pte, &sp->parent_ptes);
|
||||
}
|
||||
|
||||
static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
|
||||
u64 *parent_pte, int direct)
|
||||
{
|
||||
struct kvm_mmu_page *sp;
|
||||
|
||||
sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp);
|
||||
sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache,
|
||||
sizeof *sp);
|
||||
sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
|
||||
if (!direct)
|
||||
sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache,
|
||||
|
|
@ -1082,121 +1097,24 @@ static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
|
|||
set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
|
||||
list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
|
||||
bitmap_zero(sp->slot_bitmap, KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS);
|
||||
sp->multimapped = 0;
|
||||
sp->parent_pte = parent_pte;
|
||||
sp->parent_ptes = 0;
|
||||
mmu_page_add_parent_pte(vcpu, sp, parent_pte);
|
||||
kvm_mod_used_mmu_pages(vcpu->kvm, +1);
|
||||
return sp;
|
||||
}
|
||||
|
||||
static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
|
||||
struct kvm_mmu_page *sp, u64 *parent_pte)
|
||||
{
|
||||
struct kvm_pte_chain *pte_chain;
|
||||
struct hlist_node *node;
|
||||
int i;
|
||||
|
||||
if (!parent_pte)
|
||||
return;
|
||||
if (!sp->multimapped) {
|
||||
u64 *old = sp->parent_pte;
|
||||
|
||||
if (!old) {
|
||||
sp->parent_pte = parent_pte;
|
||||
return;
|
||||
}
|
||||
sp->multimapped = 1;
|
||||
pte_chain = mmu_alloc_pte_chain(vcpu);
|
||||
INIT_HLIST_HEAD(&sp->parent_ptes);
|
||||
hlist_add_head(&pte_chain->link, &sp->parent_ptes);
|
||||
pte_chain->parent_ptes[0] = old;
|
||||
}
|
||||
hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) {
|
||||
if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
|
||||
continue;
|
||||
for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
|
||||
if (!pte_chain->parent_ptes[i]) {
|
||||
pte_chain->parent_ptes[i] = parent_pte;
|
||||
return;
|
||||
}
|
||||
}
|
||||
pte_chain = mmu_alloc_pte_chain(vcpu);
|
||||
BUG_ON(!pte_chain);
|
||||
hlist_add_head(&pte_chain->link, &sp->parent_ptes);
|
||||
pte_chain->parent_ptes[0] = parent_pte;
|
||||
}
|
||||
|
||||
static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
|
||||
u64 *parent_pte)
|
||||
{
|
||||
struct kvm_pte_chain *pte_chain;
|
||||
struct hlist_node *node;
|
||||
int i;
|
||||
|
||||
if (!sp->multimapped) {
|
||||
BUG_ON(sp->parent_pte != parent_pte);
|
||||
sp->parent_pte = NULL;
|
||||
return;
|
||||
}
|
||||
hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
|
||||
for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
|
||||
if (!pte_chain->parent_ptes[i])
|
||||
break;
|
||||
if (pte_chain->parent_ptes[i] != parent_pte)
|
||||
continue;
|
||||
while (i + 1 < NR_PTE_CHAIN_ENTRIES
|
||||
&& pte_chain->parent_ptes[i + 1]) {
|
||||
pte_chain->parent_ptes[i]
|
||||
= pte_chain->parent_ptes[i + 1];
|
||||
++i;
|
||||
}
|
||||
pte_chain->parent_ptes[i] = NULL;
|
||||
if (i == 0) {
|
||||
hlist_del(&pte_chain->link);
|
||||
mmu_free_pte_chain(pte_chain);
|
||||
if (hlist_empty(&sp->parent_ptes)) {
|
||||
sp->multimapped = 0;
|
||||
sp->parent_pte = NULL;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
BUG();
|
||||
}
|
||||
|
||||
static void mmu_parent_walk(struct kvm_mmu_page *sp, mmu_parent_walk_fn fn)
|
||||
{
|
||||
struct kvm_pte_chain *pte_chain;
|
||||
struct hlist_node *node;
|
||||
struct kvm_mmu_page *parent_sp;
|
||||
int i;
|
||||
|
||||
if (!sp->multimapped && sp->parent_pte) {
|
||||
parent_sp = page_header(__pa(sp->parent_pte));
|
||||
fn(parent_sp, sp->parent_pte);
|
||||
return;
|
||||
}
|
||||
|
||||
hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
|
||||
for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
|
||||
u64 *spte = pte_chain->parent_ptes[i];
|
||||
|
||||
if (!spte)
|
||||
break;
|
||||
parent_sp = page_header(__pa(spte));
|
||||
fn(parent_sp, spte);
|
||||
}
|
||||
}
|
||||
|
||||
static void mark_unsync(struct kvm_mmu_page *sp, u64 *spte);
|
||||
static void mark_unsync(u64 *spte);
|
||||
static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
|
||||
{
|
||||
mmu_parent_walk(sp, mark_unsync);
|
||||
pte_list_walk(&sp->parent_ptes, mark_unsync);
|
||||
}
|
||||
|
||||
static void mark_unsync(struct kvm_mmu_page *sp, u64 *spte)
|
||||
static void mark_unsync(u64 *spte)
|
||||
{
|
||||
struct kvm_mmu_page *sp;
|
||||
unsigned int index;
|
||||
|
||||
sp = page_header(__pa(spte));
|
||||
index = spte - sp->spt;
|
||||
if (__test_and_set_bit(index, sp->unsync_child_bitmap))
|
||||
return;
|
||||
|
|
@ -1694,17 +1612,7 @@ static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
|
|||
{
|
||||
u64 *parent_pte;
|
||||
|
||||
while (sp->multimapped || sp->parent_pte) {
|
||||
if (!sp->multimapped)
|
||||
parent_pte = sp->parent_pte;
|
||||
else {
|
||||
struct kvm_pte_chain *chain;
|
||||
|
||||
chain = container_of(sp->parent_ptes.first,
|
||||
struct kvm_pte_chain, link);
|
||||
parent_pte = chain->parent_ptes[0];
|
||||
}
|
||||
BUG_ON(!parent_pte);
|
||||
while ((parent_pte = pte_list_next(&sp->parent_ptes, NULL))) {
|
||||
kvm_mmu_put_page(sp, parent_pte);
|
||||
__set_spte(parent_pte, shadow_trap_nonpresent_pte);
|
||||
}
|
||||
|
|
@ -3617,8 +3525,6 @@ static struct shrinker mmu_shrinker = {
|
|||
|
||||
static void mmu_destroy_caches(void)
|
||||
{
|
||||
if (pte_chain_cache)
|
||||
kmem_cache_destroy(pte_chain_cache);
|
||||
if (pte_list_desc_cache)
|
||||
kmem_cache_destroy(pte_list_desc_cache);
|
||||
if (mmu_page_header_cache)
|
||||
|
|
@ -3627,11 +3533,6 @@ static void mmu_destroy_caches(void)
|
|||
|
||||
int kvm_mmu_module_init(void)
|
||||
{
|
||||
pte_chain_cache = kmem_cache_create("kvm_pte_chain",
|
||||
sizeof(struct kvm_pte_chain),
|
||||
0, 0, NULL);
|
||||
if (!pte_chain_cache)
|
||||
goto nomem;
|
||||
pte_list_desc_cache = kmem_cache_create("pte_list_desc",
|
||||
sizeof(struct pte_list_desc),
|
||||
0, 0, NULL);
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче