From aca7f96600b170e470b3056aba0ed8d7df8d330d Mon Sep 17 00:00:00 2001 From: Anthony Liguori Date: Mon, 17 Sep 2007 14:57:49 -0500 Subject: [PATCH 001/249] KVM: x86 emulator: Add vmmcall/vmcall to x86_emulate (v3) Add vmmcall/vmcall to x86_emulate. Future patch will implement functionality for these instructions. Signed-off-by: Anthony Liguori Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index bd46de6bf891..84af9cc737fa 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1380,6 +1380,12 @@ twobyte_insn: u16 size; unsigned long address; + case 0: /* vmcall */ + if (modrm_mod != 3 || modrm_rm != 1) + goto cannot_emulate; + + /* nop */ + break; case 2: /* lgdt */ rc = read_descriptor(ctxt, ops, src.ptr, &size, &address, op_bytes); @@ -1387,12 +1393,17 @@ twobyte_insn: goto done; realmode_lgdt(ctxt->vcpu, size, address); break; - case 3: /* lidt */ - rc = read_descriptor(ctxt, ops, src.ptr, - &size, &address, op_bytes); - if (rc) - goto done; - realmode_lidt(ctxt->vcpu, size, address); + case 3: /* lidt/vmmcall */ + if (modrm_mod == 3 && modrm_rm == 1) { + /* nop */ + } else { + rc = read_descriptor(ctxt, ops, src.ptr, + &size, &address, + op_bytes); + if (rc) + goto done; + realmode_lidt(ctxt->vcpu, size, address); + } break; case 4: /* smsw */ if (modrm_mod != 3) From 7aa81cc04781b5b99a0647ec04533599d78cd219 Mon Sep 17 00:00:00 2001 From: Anthony Liguori Date: Mon, 17 Sep 2007 14:57:50 -0500 Subject: [PATCH 002/249] KVM: Refactor hypercall infrastructure (v3) This patch refactors the current hypercall infrastructure to better support live migration and SMP. It eliminates the hypercall page by trapping the UD exception that would occur if you used the wrong hypercall instruction for the underlying architecture and replacing it with the right one lazily. A fall-out of this patch is that the unhandled hypercalls no longer trap to userspace. There is very little reason though to use a hypercall to communicate with userspace as PIO or MMIO can be used. There is no code in tree that uses userspace hypercalls. [avi: fix #ud injection on vmx] Signed-off-by: Anthony Liguori Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 8 +- drivers/kvm/kvm_main.c | 156 +++++++++++--------------------------- drivers/kvm/svm.c | 19 ++++- drivers/kvm/vmx.c | 29 ++++++- drivers/kvm/x86_emulate.c | 11 ++- include/linux/kvm_para.h | 155 +++++++++++++++++++++++-------------- 6 files changed, 197 insertions(+), 181 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 3b0bc4bda5f2..da9c3aa1c08c 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -46,6 +46,7 @@ #define KVM_MAX_CPUID_ENTRIES 40 #define DE_VECTOR 0 +#define UD_VECTOR 6 #define NM_VECTOR 7 #define DF_VECTOR 8 #define TS_VECTOR 10 @@ -317,9 +318,6 @@ struct kvm_vcpu { unsigned long cr0; unsigned long cr2; unsigned long cr3; - gpa_t para_state_gpa; - struct page *para_state_page; - gpa_t hypercall_gpa; unsigned long cr4; unsigned long cr8; u64 pdptrs[4]; /* pae */ @@ -622,7 +620,9 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); int kvm_mmu_load(struct kvm_vcpu *vcpu); void kvm_mmu_unload(struct kvm_vcpu *vcpu); -int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run); +int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); + +int kvm_fix_hypercall(struct kvm_vcpu *vcpu); static inline void kvm_guest_enter(void) { diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index c0f372f1d761..1c662f63b7a9 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #include @@ -1362,51 +1363,61 @@ int kvm_emulate_halt(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_emulate_halt); -int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run) +int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) { - unsigned long nr, a0, a1, a2, a3, a4, a5, ret; + unsigned long nr, a0, a1, a2, a3, ret; kvm_x86_ops->cache_regs(vcpu); - ret = -KVM_EINVAL; -#ifdef CONFIG_X86_64 - if (is_long_mode(vcpu)) { - nr = vcpu->regs[VCPU_REGS_RAX]; - a0 = vcpu->regs[VCPU_REGS_RDI]; - a1 = vcpu->regs[VCPU_REGS_RSI]; - a2 = vcpu->regs[VCPU_REGS_RDX]; - a3 = vcpu->regs[VCPU_REGS_RCX]; - a4 = vcpu->regs[VCPU_REGS_R8]; - a5 = vcpu->regs[VCPU_REGS_R9]; - } else -#endif - { - nr = vcpu->regs[VCPU_REGS_RBX] & -1u; - a0 = vcpu->regs[VCPU_REGS_RAX] & -1u; - a1 = vcpu->regs[VCPU_REGS_RCX] & -1u; - a2 = vcpu->regs[VCPU_REGS_RDX] & -1u; - a3 = vcpu->regs[VCPU_REGS_RSI] & -1u; - a4 = vcpu->regs[VCPU_REGS_RDI] & -1u; - a5 = vcpu->regs[VCPU_REGS_RBP] & -1u; + + nr = vcpu->regs[VCPU_REGS_RAX]; + a0 = vcpu->regs[VCPU_REGS_RBX]; + a1 = vcpu->regs[VCPU_REGS_RCX]; + a2 = vcpu->regs[VCPU_REGS_RDX]; + a3 = vcpu->regs[VCPU_REGS_RSI]; + + if (!is_long_mode(vcpu)) { + nr &= 0xFFFFFFFF; + a0 &= 0xFFFFFFFF; + a1 &= 0xFFFFFFFF; + a2 &= 0xFFFFFFFF; + a3 &= 0xFFFFFFFF; } + switch (nr) { default: - run->hypercall.nr = nr; - run->hypercall.args[0] = a0; - run->hypercall.args[1] = a1; - run->hypercall.args[2] = a2; - run->hypercall.args[3] = a3; - run->hypercall.args[4] = a4; - run->hypercall.args[5] = a5; - run->hypercall.ret = ret; - run->hypercall.longmode = is_long_mode(vcpu); - kvm_x86_ops->decache_regs(vcpu); - return 0; + ret = -KVM_ENOSYS; + break; } vcpu->regs[VCPU_REGS_RAX] = ret; kvm_x86_ops->decache_regs(vcpu); - return 1; + return 0; +} +EXPORT_SYMBOL_GPL(kvm_emulate_hypercall); + +int kvm_fix_hypercall(struct kvm_vcpu *vcpu) +{ + char instruction[3]; + int ret = 0; + + mutex_lock(&vcpu->kvm->lock); + + /* + * Blow out the MMU to ensure that no other VCPU has an active mapping + * to ensure that the updated hypercall appears atomically across all + * VCPUs. + */ + kvm_mmu_zap_all(vcpu->kvm); + + kvm_x86_ops->cache_regs(vcpu); + kvm_x86_ops->patch_hypercall(vcpu, instruction); + if (emulator_write_emulated(vcpu->rip, instruction, 3, vcpu) + != X86EMUL_CONTINUE) + ret = -EFAULT; + + mutex_unlock(&vcpu->kvm->lock); + + return ret; } -EXPORT_SYMBOL_GPL(kvm_hypercall); static u64 mk_cr_64(u64 curr_cr, u32 new_val) { @@ -1474,75 +1485,6 @@ void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, } } -/* - * Register the para guest with the host: - */ -static int vcpu_register_para(struct kvm_vcpu *vcpu, gpa_t para_state_gpa) -{ - struct kvm_vcpu_para_state *para_state; - hpa_t para_state_hpa, hypercall_hpa; - struct page *para_state_page; - unsigned char *hypercall; - gpa_t hypercall_gpa; - - printk(KERN_DEBUG "kvm: guest trying to enter paravirtual mode\n"); - printk(KERN_DEBUG ".... para_state_gpa: %08Lx\n", para_state_gpa); - - /* - * Needs to be page aligned: - */ - if (para_state_gpa != PAGE_ALIGN(para_state_gpa)) - goto err_gp; - - para_state_hpa = gpa_to_hpa(vcpu, para_state_gpa); - printk(KERN_DEBUG ".... para_state_hpa: %08Lx\n", para_state_hpa); - if (is_error_hpa(para_state_hpa)) - goto err_gp; - - mark_page_dirty(vcpu->kvm, para_state_gpa >> PAGE_SHIFT); - para_state_page = pfn_to_page(para_state_hpa >> PAGE_SHIFT); - para_state = kmap(para_state_page); - - printk(KERN_DEBUG ".... guest version: %d\n", para_state->guest_version); - printk(KERN_DEBUG ".... size: %d\n", para_state->size); - - para_state->host_version = KVM_PARA_API_VERSION; - /* - * We cannot support guests that try to register themselves - * with a newer API version than the host supports: - */ - if (para_state->guest_version > KVM_PARA_API_VERSION) { - para_state->ret = -KVM_EINVAL; - goto err_kunmap_skip; - } - - hypercall_gpa = para_state->hypercall_gpa; - hypercall_hpa = gpa_to_hpa(vcpu, hypercall_gpa); - printk(KERN_DEBUG ".... hypercall_hpa: %08Lx\n", hypercall_hpa); - if (is_error_hpa(hypercall_hpa)) { - para_state->ret = -KVM_EINVAL; - goto err_kunmap_skip; - } - - printk(KERN_DEBUG "kvm: para guest successfully registered.\n"); - vcpu->para_state_page = para_state_page; - vcpu->para_state_gpa = para_state_gpa; - vcpu->hypercall_gpa = hypercall_gpa; - - mark_page_dirty(vcpu->kvm, hypercall_gpa >> PAGE_SHIFT); - hypercall = kmap_atomic(pfn_to_page(hypercall_hpa >> PAGE_SHIFT), - KM_USER1) + (hypercall_hpa & ~PAGE_MASK); - kvm_x86_ops->patch_hypercall(vcpu, hypercall); - kunmap_atomic(hypercall, KM_USER1); - - para_state->ret = 0; -err_kunmap_skip: - kunmap(para_state_page); - return 0; -err_gp: - return 1; -} - int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) { u64 data; @@ -1656,12 +1598,6 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) case MSR_IA32_MISC_ENABLE: vcpu->ia32_misc_enable_msr = data; break; - /* - * This is the 'probe whether the host is KVM' logic: - */ - case MSR_KVM_API_MAGIC: - return vcpu_register_para(vcpu, data); - default: pr_unimpl(vcpu, "unhandled wrmsr: 0x%x\n", msr); return 1; diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index ced4ac1955db..794d95416f7b 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -476,7 +476,8 @@ static void init_vmcb(struct vmcb *vmcb) INTERCEPT_DR5_MASK | INTERCEPT_DR7_MASK; - control->intercept_exceptions = 1 << PF_VECTOR; + control->intercept_exceptions = (1 << PF_VECTOR) | + (1 << UD_VECTOR); control->intercept = (1ULL << INTERCEPT_INTR) | @@ -979,6 +980,17 @@ static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) return 0; } +static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) +{ + int er; + + er = emulate_instruction(&svm->vcpu, kvm_run, 0, 0); + if (er != EMULATE_DONE) + inject_ud(&svm->vcpu); + + return 1; +} + static int nm_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR); @@ -1045,7 +1057,8 @@ static int vmmcall_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { svm->next_rip = svm->vmcb->save.rip + 3; skip_emulated_instruction(&svm->vcpu); - return kvm_hypercall(&svm->vcpu, kvm_run); + kvm_emulate_hypercall(&svm->vcpu); + return 1; } static int invalid_op_interception(struct vcpu_svm *svm, @@ -1241,6 +1254,7 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm, [SVM_EXIT_WRITE_DR3] = emulate_on_interception, [SVM_EXIT_WRITE_DR5] = emulate_on_interception, [SVM_EXIT_WRITE_DR7] = emulate_on_interception, + [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception, [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception, [SVM_EXIT_EXCP_BASE + NM_VECTOR] = nm_interception, [SVM_EXIT_INTR] = nop_on_interception, @@ -1675,7 +1689,6 @@ svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) hypercall[0] = 0x0f; hypercall[1] = 0x01; hypercall[2] = 0xd9; - hypercall[3] = 0xc3; } static void svm_check_processor_compat(void *rtn) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 5b397b6c9f93..47c827d3007c 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -164,6 +164,13 @@ static inline int is_no_device(u32 intr_info) (INTR_TYPE_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK); } +static inline int is_invalid_opcode(u32 intr_info) +{ + return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | + INTR_INFO_VALID_MASK)) == + (INTR_TYPE_EXCEPTION | UD_VECTOR | INTR_INFO_VALID_MASK); +} + static inline int is_external_interrupt(u32 intr_info) { return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK)) @@ -315,7 +322,7 @@ static void update_exception_bitmap(struct kvm_vcpu *vcpu) { u32 eb; - eb = 1u << PF_VECTOR; + eb = (1u << PF_VECTOR) | (1u << UD_VECTOR); if (!vcpu->fpu_active) eb |= 1u << NM_VECTOR; if (vcpu->guest_debug.enabled) @@ -560,6 +567,14 @@ static void vmx_inject_gp(struct kvm_vcpu *vcpu, unsigned error_code) INTR_INFO_VALID_MASK); } +static void vmx_inject_ud(struct kvm_vcpu *vcpu) +{ + vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, + UD_VECTOR | + INTR_TYPE_EXCEPTION | + INTR_INFO_VALID_MASK); +} + /* * Swap MSR entry in host/guest MSR entry array. */ @@ -1771,6 +1786,14 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) return 1; } + if (is_invalid_opcode(intr_info)) { + er = emulate_instruction(vcpu, kvm_run, 0, 0); + if (er != EMULATE_DONE) + vmx_inject_ud(vcpu); + + return 1; + } + error_code = 0; rip = vmcs_readl(GUEST_RIP); if (intr_info & INTR_INFO_DELIEVER_CODE_MASK) @@ -1873,7 +1896,6 @@ vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) hypercall[0] = 0x0f; hypercall[1] = 0x01; hypercall[2] = 0xc1; - hypercall[3] = 0xc3; } static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) @@ -2059,7 +2081,8 @@ static int handle_halt(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) static int handle_vmcall(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { skip_emulated_instruction(vcpu); - return kvm_hypercall(vcpu, kvm_run); + kvm_emulate_hypercall(vcpu); + return 1; } /* diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 84af9cc737fa..f12bc2c74040 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1384,7 +1384,11 @@ twobyte_insn: if (modrm_mod != 3 || modrm_rm != 1) goto cannot_emulate; - /* nop */ + rc = kvm_fix_hypercall(ctxt->vcpu); + if (rc) + goto done; + + kvm_emulate_hypercall(ctxt->vcpu); break; case 2: /* lgdt */ rc = read_descriptor(ctxt, ops, src.ptr, @@ -1395,7 +1399,10 @@ twobyte_insn: break; case 3: /* lidt/vmmcall */ if (modrm_mod == 3 && modrm_rm == 1) { - /* nop */ + rc = kvm_fix_hypercall(ctxt->vcpu); + if (rc) + goto done; + kvm_emulate_hypercall(ctxt->vcpu); } else { rc = read_descriptor(ctxt, ops, src.ptr, &size, &address, diff --git a/include/linux/kvm_para.h b/include/linux/kvm_para.h index 3b292565a693..cc5dfb433706 100644 --- a/include/linux/kvm_para.h +++ b/include/linux/kvm_para.h @@ -1,73 +1,110 @@ #ifndef __LINUX_KVM_PARA_H #define __LINUX_KVM_PARA_H -/* - * Guest OS interface for KVM paravirtualization +/* This CPUID returns the signature 'KVMKVMKVM' in ebx, ecx, and edx. It + * should be used to determine that a VM is running under KVM. + */ +#define KVM_CPUID_SIGNATURE 0x40000000 + +/* This CPUID returns a feature bitmap in eax. Before enabling a particular + * paravirtualization, the appropriate feature bit should be checked. + */ +#define KVM_CPUID_FEATURES 0x40000001 + +/* Return values for hypercalls */ +#define KVM_ENOSYS 1000 + +#ifdef __KERNEL__ +#include + +/* This instruction is vmcall. On non-VT architectures, it will generate a + * trap that we will then rewrite to the appropriate instruction. + */ +#define KVM_HYPERCALL ".byte 0x0f,0x01,0xc1" + +/* For KVM hypercalls, a three-byte sequence of either the vmrun or the vmmrun + * instruction. The hypervisor may replace it with something else but only the + * instructions are guaranteed to be supported. * - * Note: this interface is totally experimental, and is certain to change - * as we make progress. + * Up to four arguments may be passed in rbx, rcx, rdx, and rsi respectively. + * The hypercall number should be placed in rax and the return value will be + * placed in rax. No other registers will be clobbered unless explicited + * noted by the particular hypercall. */ -/* - * Per-VCPU descriptor area shared between guest and host. Writable to - * both guest and host. Registered with the host by the guest when - * a guest acknowledges paravirtual mode. - * - * NOTE: all addresses are guest-physical addresses (gpa), to make it - * easier for the hypervisor to map between the various addresses. - */ -struct kvm_vcpu_para_state { - /* - * API version information for compatibility. If there's any support - * mismatch (too old host trying to execute too new guest) then - * the host will deny entry into paravirtual mode. Any other - * combination (new host + old guest and new host + new guest) - * is supposed to work - new host versions will support all old - * guest API versions. - */ - u32 guest_version; - u32 host_version; - u32 size; - u32 ret; +static inline long kvm_hypercall0(unsigned int nr) +{ + long ret; + asm volatile(KVM_HYPERCALL + : "=a"(ret) + : "a"(nr)); + return ret; +} - /* - * The address of the vm exit instruction (VMCALL or VMMCALL), - * which the host will patch according to the CPU model the - * VM runs on: - */ - u64 hypercall_gpa; +static inline long kvm_hypercall1(unsigned int nr, unsigned long p1) +{ + long ret; + asm volatile(KVM_HYPERCALL + : "=a"(ret) + : "a"(nr), "b"(p1)); + return ret; +} -} __attribute__ ((aligned(PAGE_SIZE))); +static inline long kvm_hypercall2(unsigned int nr, unsigned long p1, + unsigned long p2) +{ + long ret; + asm volatile(KVM_HYPERCALL + : "=a"(ret) + : "a"(nr), "b"(p1), "c"(p2)); + return ret; +} -#define KVM_PARA_API_VERSION 1 +static inline long kvm_hypercall3(unsigned int nr, unsigned long p1, + unsigned long p2, unsigned long p3) +{ + long ret; + asm volatile(KVM_HYPERCALL + : "=a"(ret) + : "a"(nr), "b"(p1), "c"(p2), "d"(p3)); + return ret; +} -/* - * This is used for an RDMSR's ECX parameter to probe for a KVM host. - * Hopefully no CPU vendor will use up this number. This is placed well - * out of way of the typical space occupied by CPU vendors' MSR indices, - * and we think (or at least hope) it wont be occupied in the future - * either. - */ -#define MSR_KVM_API_MAGIC 0x87655678 +static inline long kvm_hypercall4(unsigned int nr, unsigned long p1, + unsigned long p2, unsigned long p3, + unsigned long p4) +{ + long ret; + asm volatile(KVM_HYPERCALL + : "=a"(ret) + : "a"(nr), "b"(p1), "c"(p2), "d"(p3), "S"(p4)); + return ret; +} -#define KVM_EINVAL 1 +static inline int kvm_para_available(void) +{ + unsigned int eax, ebx, ecx, edx; + char signature[13]; -/* - * Hypercall calling convention: - * - * Each hypercall may have 0-6 parameters. - * - * 64-bit hypercall index is in RAX, goes from 0 to __NR_hypercalls-1 - * - * 64-bit parameters 1-6 are in the standard gcc x86_64 calling convention - * order: RDI, RSI, RDX, RCX, R8, R9. - * - * 32-bit index is EBX, parameters are: EAX, ECX, EDX, ESI, EDI, EBP. - * (the first 3 are according to the gcc regparm calling convention) - * - * No registers are clobbered by the hypercall, except that the - * return value is in RAX. - */ -#define __NR_hypercalls 0 + cpuid(KVM_CPUID_SIGNATURE, &eax, &ebx, &ecx, &edx); + memcpy(signature + 0, &ebx, 4); + memcpy(signature + 4, &ecx, 4); + memcpy(signature + 8, &edx, 4); + signature[12] = 0; + + if (strcmp(signature, "KVMKVMKVM") == 0) + return 1; + + return 0; +} + +static inline int kvm_para_has_feature(unsigned int feature) +{ + if (cpuid_eax(KVM_CPUID_FEATURES) & (1UL << feature)) + return 1; + return 0; +} + +#endif #endif From a7ddce3afc8326870b9e5e02fa41e028bffb10a5 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Tue, 18 Sep 2007 11:26:38 +0200 Subject: [PATCH 003/249] KVM: x86 emulator: remove unused functions Remove #ifdef functions never used Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 39 --------------------------------------- 1 file changed, 39 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index f12bc2c74040..9ea82f84743e 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1639,42 +1639,3 @@ cannot_emulate: DPRINTF("Cannot emulate %02x\n", b); return -1; } - -#ifdef __XEN__ - -#include -#include - -int -x86_emulate_read_std(unsigned long addr, - unsigned long *val, - unsigned int bytes, struct x86_emulate_ctxt *ctxt) -{ - unsigned int rc; - - *val = 0; - - if ((rc = copy_from_user((void *)val, (void *)addr, bytes)) != 0) { - propagate_page_fault(addr + bytes - rc, 0); /* read fault */ - return X86EMUL_PROPAGATE_FAULT; - } - - return X86EMUL_CONTINUE; -} - -int -x86_emulate_write_std(unsigned long addr, - unsigned long val, - unsigned int bytes, struct x86_emulate_ctxt *ctxt) -{ - unsigned int rc; - - if ((rc = copy_to_user((void *)addr, (void *)&val, bytes)) != 0) { - propagate_page_fault(addr + bytes - rc, PGERR_write_access); - return X86EMUL_PROPAGATE_FAULT; - } - - return X86EMUL_CONTINUE; -} - -#endif From e4e03deda83b1f2fc37ccbfc1eef27e86e8ed4e9 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Tue, 18 Sep 2007 11:52:50 +0200 Subject: [PATCH 004/249] KVM: x86 emulator: move all x86_emulate_memop() to a structure Move all x86_emulate_memop() common variables between decode and execute to a structure decode_cache. This will help in later separating decode and emulate. struct decode_cache { u8 twobyte; u8 b; u8 lock_prefix; u8 rep_prefix; u8 op_bytes; u8 ad_bytes; struct operand src; struct operand dst; unsigned long *override_base; unsigned int d; unsigned long regs[NR_VCPU_REGS]; unsigned long eip; /* modrm */ u8 modrm; u8 modrm_mod; u8 modrm_reg; u8 modrm_rm; u8 use_modrm_ea; unsigned long modrm_ea; unsigned long modrm_val; }; Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 917 ++++++++++++++++++++------------------ drivers/kvm/x86_emulate.h | 34 ++ 2 files changed, 517 insertions(+), 434 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 9ea82f84743e..d7026cb8fc70 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -222,13 +222,6 @@ static u16 twobyte_table[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; -/* Type, address-of, and value of an instruction's operand. */ -struct operand { - enum { OP_REG, OP_MEM, OP_IMM } type; - unsigned int bytes; - unsigned long val, orig_val, *ptr; -}; - /* EFLAGS bit definitions. */ #define EFLG_OF (1<<11) #define EFLG_DF (1<<10) @@ -431,24 +424,26 @@ struct operand { /* Access/update address held in a register, based on addressing mode. */ #define address_mask(reg) \ - ((ad_bytes == sizeof(unsigned long)) ? \ - (reg) : ((reg) & ((1UL << (ad_bytes << 3)) - 1))) + ((c->ad_bytes == sizeof(unsigned long)) ? \ + (reg) : ((reg) & ((1UL << (c->ad_bytes << 3)) - 1))) #define register_address(base, reg) \ ((base) + address_mask(reg)) #define register_address_increment(reg, inc) \ do { \ /* signed type ensures sign extension to long */ \ int _inc = (inc); \ - if ( ad_bytes == sizeof(unsigned long) ) \ + if (c->ad_bytes == sizeof(unsigned long)) \ (reg) += _inc; \ else \ - (reg) = ((reg) & ~((1UL << (ad_bytes << 3)) - 1)) | \ - (((reg) + _inc) & ((1UL << (ad_bytes << 3)) - 1)); \ + (reg) = ((reg) & \ + ~((1UL << (c->ad_bytes << 3)) - 1)) | \ + (((reg) + _inc) & \ + ((1UL << (c->ad_bytes << 3)) - 1)); \ } while (0) #define JMP_REL(rel) \ do { \ - register_address_increment(_eip, rel); \ + register_address_increment(c->eip, rel); \ } while (0) /* @@ -524,39 +519,35 @@ static int test_cc(unsigned int condition, unsigned int flags) int x86_emulate_memop(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { - unsigned d; - u8 b, sib, twobyte = 0, rex_prefix = 0; - u8 modrm, modrm_mod = 0, modrm_reg = 0, modrm_rm = 0; - unsigned long *override_base = NULL; - unsigned int op_bytes, ad_bytes, lock_prefix = 0, rep_prefix = 0, i; + struct decode_cache *c = &ctxt->decode; + u8 sib, rex_prefix = 0; + unsigned int i; int rc = 0; - struct operand src, dst; unsigned long cr2 = ctxt->cr2; int mode = ctxt->mode; - unsigned long modrm_ea; - int use_modrm_ea, index_reg = 0, base_reg = 0, scale, rip_relative = 0; + int index_reg = 0, base_reg = 0, scale, rip_relative = 0; int no_wb = 0; u64 msr_data; /* Shadow copy of register state. Committed on successful emulation. */ - unsigned long _regs[NR_VCPU_REGS]; - unsigned long _eip = ctxt->vcpu->rip, _eflags = ctxt->eflags; - unsigned long modrm_val = 0; + unsigned long _eflags = ctxt->eflags; - memcpy(_regs, ctxt->vcpu->regs, sizeof _regs); + memset(c, 0, sizeof(struct decode_cache)); + c->eip = ctxt->vcpu->rip; + memcpy(c->regs, ctxt->vcpu->regs, sizeof c->regs); switch (mode) { case X86EMUL_MODE_REAL: case X86EMUL_MODE_PROT16: - op_bytes = ad_bytes = 2; + c->op_bytes = c->ad_bytes = 2; break; case X86EMUL_MODE_PROT32: - op_bytes = ad_bytes = 4; + c->op_bytes = c->ad_bytes = 4; break; #ifdef CONFIG_X86_64 case X86EMUL_MODE_PROT64: - op_bytes = 4; - ad_bytes = 8; + c->op_bytes = 4; + c->ad_bytes = 8; break; #endif default: @@ -565,40 +556,42 @@ x86_emulate_memop(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) /* Legacy prefixes. */ for (i = 0; i < 8; i++) { - switch (b = insn_fetch(u8, 1, _eip)) { + switch (c->b = insn_fetch(u8, 1, c->eip)) { case 0x66: /* operand-size override */ - op_bytes ^= 6; /* switch between 2/4 bytes */ + c->op_bytes ^= 6; /* switch between 2/4 bytes */ break; case 0x67: /* address-size override */ if (mode == X86EMUL_MODE_PROT64) - ad_bytes ^= 12; /* switch between 4/8 bytes */ + /* switch between 4/8 bytes */ + c->ad_bytes ^= 12; else - ad_bytes ^= 6; /* switch between 2/4 bytes */ + /* switch between 2/4 bytes */ + c->ad_bytes ^= 6; break; case 0x2e: /* CS override */ - override_base = &ctxt->cs_base; + c->override_base = &ctxt->cs_base; break; case 0x3e: /* DS override */ - override_base = &ctxt->ds_base; + c->override_base = &ctxt->ds_base; break; case 0x26: /* ES override */ - override_base = &ctxt->es_base; + c->override_base = &ctxt->es_base; break; case 0x64: /* FS override */ - override_base = &ctxt->fs_base; + c->override_base = &ctxt->fs_base; break; case 0x65: /* GS override */ - override_base = &ctxt->gs_base; + c->override_base = &ctxt->gs_base; break; case 0x36: /* SS override */ - override_base = &ctxt->ss_base; + c->override_base = &ctxt->ss_base; break; case 0xf0: /* LOCK */ - lock_prefix = 1; + c->lock_prefix = 1; break; case 0xf2: /* REPNE/REPNZ */ case 0xf3: /* REP/REPE/REPZ */ - rep_prefix = 1; + c->rep_prefix = 1; break; default: goto done_prefixes; @@ -608,177 +601,182 @@ x86_emulate_memop(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) done_prefixes: /* REX prefix. */ - if ((mode == X86EMUL_MODE_PROT64) && ((b & 0xf0) == 0x40)) { - rex_prefix = b; - if (b & 8) - op_bytes = 8; /* REX.W */ - modrm_reg = (b & 4) << 1; /* REX.R */ - index_reg = (b & 2) << 2; /* REX.X */ - modrm_rm = base_reg = (b & 1) << 3; /* REG.B */ - b = insn_fetch(u8, 1, _eip); + if ((mode == X86EMUL_MODE_PROT64) && ((c->b & 0xf0) == 0x40)) { + rex_prefix = c->b; + if (c->b & 8) + c->op_bytes = 8; /* REX.W */ + c->modrm_reg = (c->b & 4) << 1; /* REX.R */ + index_reg = (c->b & 2) << 2; /* REX.X */ + c->modrm_rm = base_reg = (c->b & 1) << 3; /* REG.B */ + c->b = insn_fetch(u8, 1, c->eip); } /* Opcode byte(s). */ - d = opcode_table[b]; - if (d == 0) { + c->d = opcode_table[c->b]; + if (c->d == 0) { /* Two-byte opcode? */ - if (b == 0x0f) { - twobyte = 1; - b = insn_fetch(u8, 1, _eip); - d = twobyte_table[b]; + if (c->b == 0x0f) { + c->twobyte = 1; + c->b = insn_fetch(u8, 1, c->eip); + c->d = twobyte_table[c->b]; } /* Unrecognised? */ - if (d == 0) + if (c->d == 0) goto cannot_emulate; } /* ModRM and SIB bytes. */ - if (d & ModRM) { - modrm = insn_fetch(u8, 1, _eip); - modrm_mod |= (modrm & 0xc0) >> 6; - modrm_reg |= (modrm & 0x38) >> 3; - modrm_rm |= (modrm & 0x07); - modrm_ea = 0; - use_modrm_ea = 1; + if (c->d & ModRM) { + c->modrm = insn_fetch(u8, 1, c->eip); + c->modrm_mod |= (c->modrm & 0xc0) >> 6; + c->modrm_reg |= (c->modrm & 0x38) >> 3; + c->modrm_rm |= (c->modrm & 0x07); + c->modrm_ea = 0; + c->use_modrm_ea = 1; - if (modrm_mod == 3) { - modrm_val = *(unsigned long *) - decode_register(modrm_rm, _regs, d & ByteOp); + if (c->modrm_mod == 3) { + c->modrm_val = *(unsigned long *) + decode_register(c->modrm_rm, c->regs, c->d & ByteOp); goto modrm_done; } - if (ad_bytes == 2) { - unsigned bx = _regs[VCPU_REGS_RBX]; - unsigned bp = _regs[VCPU_REGS_RBP]; - unsigned si = _regs[VCPU_REGS_RSI]; - unsigned di = _regs[VCPU_REGS_RDI]; + if (c->ad_bytes == 2) { + unsigned bx = c->regs[VCPU_REGS_RBX]; + unsigned bp = c->regs[VCPU_REGS_RBP]; + unsigned si = c->regs[VCPU_REGS_RSI]; + unsigned di = c->regs[VCPU_REGS_RDI]; /* 16-bit ModR/M decode. */ - switch (modrm_mod) { + switch (c->modrm_mod) { case 0: - if (modrm_rm == 6) - modrm_ea += insn_fetch(u16, 2, _eip); + if (c->modrm_rm == 6) + c->modrm_ea += + insn_fetch(u16, 2, c->eip); break; case 1: - modrm_ea += insn_fetch(s8, 1, _eip); + c->modrm_ea += insn_fetch(s8, 1, c->eip); break; case 2: - modrm_ea += insn_fetch(u16, 2, _eip); + c->modrm_ea += insn_fetch(u16, 2, c->eip); break; } - switch (modrm_rm) { + switch (c->modrm_rm) { case 0: - modrm_ea += bx + si; + c->modrm_ea += bx + si; break; case 1: - modrm_ea += bx + di; + c->modrm_ea += bx + di; break; case 2: - modrm_ea += bp + si; + c->modrm_ea += bp + si; break; case 3: - modrm_ea += bp + di; + c->modrm_ea += bp + di; break; case 4: - modrm_ea += si; + c->modrm_ea += si; break; case 5: - modrm_ea += di; + c->modrm_ea += di; break; case 6: - if (modrm_mod != 0) - modrm_ea += bp; + if (c->modrm_mod != 0) + c->modrm_ea += bp; break; case 7: - modrm_ea += bx; + c->modrm_ea += bx; break; } - if (modrm_rm == 2 || modrm_rm == 3 || - (modrm_rm == 6 && modrm_mod != 0)) - if (!override_base) - override_base = &ctxt->ss_base; - modrm_ea = (u16)modrm_ea; + if (c->modrm_rm == 2 || c->modrm_rm == 3 || + (c->modrm_rm == 6 && c->modrm_mod != 0)) + if (!c->override_base) + c->override_base = &ctxt->ss_base; + c->modrm_ea = (u16)c->modrm_ea; } else { /* 32/64-bit ModR/M decode. */ - switch (modrm_rm) { + switch (c->modrm_rm) { case 4: case 12: - sib = insn_fetch(u8, 1, _eip); + sib = insn_fetch(u8, 1, c->eip); index_reg |= (sib >> 3) & 7; base_reg |= sib & 7; scale = sib >> 6; switch (base_reg) { case 5: - if (modrm_mod != 0) - modrm_ea += _regs[base_reg]; + if (c->modrm_mod != 0) + c->modrm_ea += + c->regs[base_reg]; else - modrm_ea += insn_fetch(s32, 4, _eip); + c->modrm_ea += + insn_fetch(s32, 4, c->eip); break; default: - modrm_ea += _regs[base_reg]; + c->modrm_ea += c->regs[base_reg]; } switch (index_reg) { case 4: break; default: - modrm_ea += _regs[index_reg] << scale; + c->modrm_ea += + c->regs[index_reg] << scale; } break; case 5: - if (modrm_mod != 0) - modrm_ea += _regs[modrm_rm]; + if (c->modrm_mod != 0) + c->modrm_ea += c->regs[c->modrm_rm]; else if (mode == X86EMUL_MODE_PROT64) rip_relative = 1; break; default: - modrm_ea += _regs[modrm_rm]; + c->modrm_ea += c->regs[c->modrm_rm]; break; } - switch (modrm_mod) { + switch (c->modrm_mod) { case 0: - if (modrm_rm == 5) - modrm_ea += insn_fetch(s32, 4, _eip); + if (c->modrm_rm == 5) + c->modrm_ea += + insn_fetch(s32, 4, c->eip); break; case 1: - modrm_ea += insn_fetch(s8, 1, _eip); + c->modrm_ea += insn_fetch(s8, 1, c->eip); break; case 2: - modrm_ea += insn_fetch(s32, 4, _eip); + c->modrm_ea += insn_fetch(s32, 4, c->eip); break; } } - if (!override_base) - override_base = &ctxt->ds_base; + if (!c->override_base) + c->override_base = &ctxt->ds_base; if (mode == X86EMUL_MODE_PROT64 && - override_base != &ctxt->fs_base && - override_base != &ctxt->gs_base) - override_base = NULL; + c->override_base != &ctxt->fs_base && + c->override_base != &ctxt->gs_base) + c->override_base = NULL; - if (override_base) - modrm_ea += *override_base; + if (c->override_base) + c->modrm_ea += *c->override_base; if (rip_relative) { - modrm_ea += _eip; - switch (d & SrcMask) { + c->modrm_ea += c->eip; + switch (c->d & SrcMask) { case SrcImmByte: - modrm_ea += 1; + c->modrm_ea += 1; break; case SrcImm: - if (d & ByteOp) - modrm_ea += 1; + if (c->d & ByteOp) + c->modrm_ea += 1; else - if (op_bytes == 8) - modrm_ea += 4; + if (c->op_bytes == 8) + c->modrm_ea += 4; else - modrm_ea += op_bytes; + c->modrm_ea += c->op_bytes; } } - if (ad_bytes != 8) - modrm_ea = (u32)modrm_ea; - cr2 = modrm_ea; + if (c->ad_bytes != 8) + c->modrm_ea = (u32)c->modrm_ea; + cr2 = c->modrm_ea; modrm_done: ; } @@ -787,200 +785,210 @@ done_prefixes: * Decode and fetch the source operand: register, memory * or immediate. */ - switch (d & SrcMask) { + switch (c->d & SrcMask) { case SrcNone: break; case SrcReg: - src.type = OP_REG; - if (d & ByteOp) { - src.ptr = decode_register(modrm_reg, _regs, + c->src.type = OP_REG; + if (c->d & ByteOp) { + c->src.ptr = + decode_register(c->modrm_reg, c->regs, (rex_prefix == 0)); - src.val = src.orig_val = *(u8 *) src.ptr; - src.bytes = 1; + c->src.val = c->src.orig_val = *(u8 *)c->src.ptr; + c->src.bytes = 1; } else { - src.ptr = decode_register(modrm_reg, _regs, 0); - switch ((src.bytes = op_bytes)) { + c->src.ptr = + decode_register(c->modrm_reg, c->regs, 0); + switch ((c->src.bytes = c->op_bytes)) { case 2: - src.val = src.orig_val = *(u16 *) src.ptr; + c->src.val = c->src.orig_val = + *(u16 *) c->src.ptr; break; case 4: - src.val = src.orig_val = *(u32 *) src.ptr; + c->src.val = c->src.orig_val = + *(u32 *) c->src.ptr; break; case 8: - src.val = src.orig_val = *(u64 *) src.ptr; + c->src.val = c->src.orig_val = + *(u64 *) c->src.ptr; break; } } break; case SrcMem16: - src.bytes = 2; + c->src.bytes = 2; goto srcmem_common; case SrcMem32: - src.bytes = 4; + c->src.bytes = 4; goto srcmem_common; case SrcMem: - src.bytes = (d & ByteOp) ? 1 : op_bytes; + c->src.bytes = (c->d & ByteOp) ? 1 : + c->op_bytes; /* Don't fetch the address for invlpg: it could be unmapped. */ - if (twobyte && b == 0x01 && modrm_reg == 7) + if (c->twobyte && c->b == 0x01 + && c->modrm_reg == 7) break; srcmem_common: /* * For instructions with a ModR/M byte, switch to register * access if Mod = 3. */ - if ((d & ModRM) && modrm_mod == 3) { - src.type = OP_REG; + if ((c->d & ModRM) && c->modrm_mod == 3) { + c->src.type = OP_REG; break; } - src.type = OP_MEM; - src.ptr = (unsigned long *)cr2; - src.val = 0; - if ((rc = ops->read_emulated((unsigned long)src.ptr, - &src.val, src.bytes, ctxt->vcpu)) != 0) + c->src.type = OP_MEM; + c->src.ptr = (unsigned long *)cr2; + c->src.val = 0; + if ((rc = ops->read_emulated((unsigned long)c->src.ptr, + &c->src.val, + c->src.bytes, ctxt->vcpu)) != 0) goto done; - src.orig_val = src.val; + c->src.orig_val = c->src.val; break; case SrcImm: - src.type = OP_IMM; - src.ptr = (unsigned long *)_eip; - src.bytes = (d & ByteOp) ? 1 : op_bytes; - if (src.bytes == 8) - src.bytes = 4; + c->src.type = OP_IMM; + c->src.ptr = (unsigned long *)c->eip; + c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + if (c->src.bytes == 8) + c->src.bytes = 4; /* NB. Immediates are sign-extended as necessary. */ - switch (src.bytes) { + switch (c->src.bytes) { case 1: - src.val = insn_fetch(s8, 1, _eip); + c->src.val = insn_fetch(s8, 1, c->eip); break; case 2: - src.val = insn_fetch(s16, 2, _eip); + c->src.val = insn_fetch(s16, 2, c->eip); break; case 4: - src.val = insn_fetch(s32, 4, _eip); + c->src.val = insn_fetch(s32, 4, c->eip); break; } break; case SrcImmByte: - src.type = OP_IMM; - src.ptr = (unsigned long *)_eip; - src.bytes = 1; - src.val = insn_fetch(s8, 1, _eip); + c->src.type = OP_IMM; + c->src.ptr = (unsigned long *)c->eip; + c->src.bytes = 1; + c->src.val = insn_fetch(s8, 1, c->eip); break; } /* Decode and fetch the destination operand: register or memory. */ - switch (d & DstMask) { + switch (c->d & DstMask) { case ImplicitOps: /* Special instructions do their own operand decoding. */ goto special_insn; case DstReg: - dst.type = OP_REG; - if ((d & ByteOp) - && !(twobyte && (b == 0xb6 || b == 0xb7))) { - dst.ptr = decode_register(modrm_reg, _regs, + c->dst.type = OP_REG; + if ((c->d & ByteOp) + && !(c->twobyte && + (c->b == 0xb6 || c->b == 0xb7))) { + c->dst.ptr = + decode_register(c->modrm_reg, c->regs, (rex_prefix == 0)); - dst.val = *(u8 *) dst.ptr; - dst.bytes = 1; + c->dst.val = *(u8 *) c->dst.ptr; + c->dst.bytes = 1; } else { - dst.ptr = decode_register(modrm_reg, _regs, 0); - switch ((dst.bytes = op_bytes)) { + c->dst.ptr = + decode_register(c->modrm_reg, c->regs, 0); + switch ((c->dst.bytes = c->op_bytes)) { case 2: - dst.val = *(u16 *)dst.ptr; + c->dst.val = *(u16 *)c->dst.ptr; break; case 4: - dst.val = *(u32 *)dst.ptr; + c->dst.val = *(u32 *)c->dst.ptr; break; case 8: - dst.val = *(u64 *)dst.ptr; + c->dst.val = *(u64 *)c->dst.ptr; break; } } break; case DstMem: - dst.type = OP_MEM; - dst.ptr = (unsigned long *)cr2; - dst.bytes = (d & ByteOp) ? 1 : op_bytes; - dst.val = 0; - /* - * For instructions with a ModR/M byte, switch to register - * access if Mod = 3. - */ - if ((d & ModRM) && modrm_mod == 3) { - dst.type = OP_REG; + c->dst.type = OP_MEM; + c->dst.ptr = (unsigned long *)cr2; + c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->dst.val = 0; + if ((c->d & ModRM) && c->modrm_mod == 3) { + c->dst.type = OP_REG; break; } - if (d & BitOp) { - unsigned long mask = ~(dst.bytes * 8 - 1); + if (c->d & BitOp) { + unsigned long mask = ~(c->dst.bytes * 8 - 1); - dst.ptr = (void *)dst.ptr + (src.val & mask) / 8; + c->dst.ptr = (void *)c->dst.ptr + + (c->src.val & mask) / 8; } - if (!(d & Mov) && /* optimisation - avoid slow emulated read */ - ((rc = ops->read_emulated((unsigned long)dst.ptr, - &dst.val, dst.bytes, ctxt->vcpu)) != 0)) + if (!(c->d & Mov) && + /* optimisation - avoid slow emulated read */ + ((rc = ops->read_emulated((unsigned long)c->dst.ptr, + &c->dst.val, + c->dst.bytes, ctxt->vcpu)) != 0)) goto done; break; } - dst.orig_val = dst.val; + c->dst.orig_val = c->dst.val; - if (twobyte) + if (c->twobyte) goto twobyte_insn; - switch (b) { + switch (c->b) { case 0x00 ... 0x05: add: /* add */ - emulate_2op_SrcV("add", src, dst, _eflags); + emulate_2op_SrcV("add", c->src, c->dst, _eflags); break; case 0x08 ... 0x0d: or: /* or */ - emulate_2op_SrcV("or", src, dst, _eflags); + emulate_2op_SrcV("or", c->src, c->dst, _eflags); break; case 0x10 ... 0x15: adc: /* adc */ - emulate_2op_SrcV("adc", src, dst, _eflags); + emulate_2op_SrcV("adc", c->src, c->dst, _eflags); break; case 0x18 ... 0x1d: sbb: /* sbb */ - emulate_2op_SrcV("sbb", src, dst, _eflags); + emulate_2op_SrcV("sbb", c->src, c->dst, _eflags); break; case 0x20 ... 0x23: and: /* and */ - emulate_2op_SrcV("and", src, dst, _eflags); + emulate_2op_SrcV("and", c->src, c->dst, _eflags); break; case 0x24: /* and al imm8 */ - dst.type = OP_REG; - dst.ptr = &_regs[VCPU_REGS_RAX]; - dst.val = *(u8 *)dst.ptr; - dst.bytes = 1; - dst.orig_val = dst.val; + c->dst.type = OP_REG; + c->dst.ptr = &c->regs[VCPU_REGS_RAX]; + c->dst.val = *(u8 *)c->dst.ptr; + c->dst.bytes = 1; + c->dst.orig_val = c->dst.val; goto and; case 0x25: /* and ax imm16, or eax imm32 */ - dst.type = OP_REG; - dst.bytes = op_bytes; - dst.ptr = &_regs[VCPU_REGS_RAX]; - if (op_bytes == 2) - dst.val = *(u16 *)dst.ptr; + c->dst.type = OP_REG; + c->dst.bytes = c->op_bytes; + c->dst.ptr = &c->regs[VCPU_REGS_RAX]; + if (c->op_bytes == 2) + c->dst.val = *(u16 *)c->dst.ptr; else - dst.val = *(u32 *)dst.ptr; - dst.orig_val = dst.val; + c->dst.val = *(u32 *)c->dst.ptr; + c->dst.orig_val = c->dst.val; goto and; case 0x28 ... 0x2d: sub: /* sub */ - emulate_2op_SrcV("sub", src, dst, _eflags); + emulate_2op_SrcV("sub", c->src, c->dst, _eflags); break; case 0x30 ... 0x35: xor: /* xor */ - emulate_2op_SrcV("xor", src, dst, _eflags); + emulate_2op_SrcV("xor", c->src, c->dst, _eflags); break; case 0x38 ... 0x3d: cmp: /* cmp */ - emulate_2op_SrcV("cmp", src, dst, _eflags); + emulate_2op_SrcV("cmp", c->src, c->dst, _eflags); break; case 0x63: /* movsxd */ if (mode != X86EMUL_MODE_PROT64) goto cannot_emulate; - dst.val = (s32) src.val; + c->dst.val = (s32) c->src.val; break; case 0x80 ... 0x83: /* Grp1 */ - switch (modrm_reg) { + switch (c->modrm_reg) { case 0: goto add; case 1: @@ -1001,155 +1009,164 @@ done_prefixes: break; case 0x84 ... 0x85: test: /* test */ - emulate_2op_SrcV("test", src, dst, _eflags); + emulate_2op_SrcV("test", c->src, c->dst, _eflags); break; case 0x86 ... 0x87: /* xchg */ /* Write back the register source. */ - switch (dst.bytes) { + switch (c->dst.bytes) { case 1: - *(u8 *) src.ptr = (u8) dst.val; + *(u8 *) c->src.ptr = (u8) c->dst.val; break; case 2: - *(u16 *) src.ptr = (u16) dst.val; + *(u16 *) c->src.ptr = (u16) c->dst.val; break; case 4: - *src.ptr = (u32) dst.val; + *c->src.ptr = (u32) c->dst.val; break; /* 64b reg: zero-extend */ case 8: - *src.ptr = dst.val; + *c->src.ptr = c->dst.val; break; } /* * Write back the memory destination with implicit LOCK * prefix. */ - dst.val = src.val; - lock_prefix = 1; + c->dst.val = c->src.val; + c->lock_prefix = 1; break; case 0x88 ... 0x8b: /* mov */ goto mov; case 0x8d: /* lea r16/r32, m */ - dst.val = modrm_val; + c->dst.val = c->modrm_val; break; case 0x8f: /* pop (sole member of Grp1a) */ /* 64-bit mode: POP always pops a 64-bit operand. */ if (mode == X86EMUL_MODE_PROT64) - dst.bytes = 8; - if ((rc = ops->read_std(register_address(ctxt->ss_base, - _regs[VCPU_REGS_RSP]), - &dst.val, dst.bytes, ctxt->vcpu)) != 0) + c->dst.bytes = 8; + if ((rc = ops->read_std(register_address( + ctxt->ss_base, + c->regs[VCPU_REGS_RSP]), + &c->dst.val, + c->dst.bytes, + ctxt->vcpu)) != 0) goto done; - register_address_increment(_regs[VCPU_REGS_RSP], dst.bytes); + register_address_increment(c->regs[VCPU_REGS_RSP], + c->dst.bytes); break; case 0xa0 ... 0xa1: /* mov */ - dst.ptr = (unsigned long *)&_regs[VCPU_REGS_RAX]; - dst.val = src.val; - _eip += ad_bytes; /* skip src displacement */ + c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; + c->dst.val = c->src.val; + /* skip src displacement */ + c->eip += c->ad_bytes; break; case 0xa2 ... 0xa3: /* mov */ - dst.val = (unsigned long)_regs[VCPU_REGS_RAX]; - _eip += ad_bytes; /* skip dst displacement */ + c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX]; + /* skip c->dst displacement */ + c->eip += c->ad_bytes; break; case 0xc0 ... 0xc1: grp2: /* Grp2 */ - switch (modrm_reg) { + switch (c->modrm_reg) { case 0: /* rol */ - emulate_2op_SrcB("rol", src, dst, _eflags); + emulate_2op_SrcB("rol", c->src, c->dst, _eflags); break; case 1: /* ror */ - emulate_2op_SrcB("ror", src, dst, _eflags); + emulate_2op_SrcB("ror", c->src, c->dst, _eflags); break; case 2: /* rcl */ - emulate_2op_SrcB("rcl", src, dst, _eflags); + emulate_2op_SrcB("rcl", c->src, c->dst, _eflags); break; case 3: /* rcr */ - emulate_2op_SrcB("rcr", src, dst, _eflags); + emulate_2op_SrcB("rcr", c->src, c->dst, _eflags); break; case 4: /* sal/shl */ case 6: /* sal/shl */ - emulate_2op_SrcB("sal", src, dst, _eflags); + emulate_2op_SrcB("sal", c->src, c->dst, _eflags); break; case 5: /* shr */ - emulate_2op_SrcB("shr", src, dst, _eflags); + emulate_2op_SrcB("shr", c->src, c->dst, _eflags); break; case 7: /* sar */ - emulate_2op_SrcB("sar", src, dst, _eflags); + emulate_2op_SrcB("sar", c->src, c->dst, _eflags); break; } break; case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */ mov: - dst.val = src.val; + c->dst.val = c->src.val; break; case 0xd0 ... 0xd1: /* Grp2 */ - src.val = 1; + c->src.val = 1; goto grp2; case 0xd2 ... 0xd3: /* Grp2 */ - src.val = _regs[VCPU_REGS_RCX]; + c->src.val = c->regs[VCPU_REGS_RCX]; goto grp2; case 0xf6 ... 0xf7: /* Grp3 */ - switch (modrm_reg) { + switch (c->modrm_reg) { case 0 ... 1: /* test */ /* * Special case in Grp3: test has an immediate * source operand. */ - src.type = OP_IMM; - src.ptr = (unsigned long *)_eip; - src.bytes = (d & ByteOp) ? 1 : op_bytes; - if (src.bytes == 8) - src.bytes = 4; - switch (src.bytes) { + c->src.type = OP_IMM; + c->src.ptr = (unsigned long *)c->eip; + c->src.bytes = (c->d & ByteOp) ? 1 : + c->op_bytes; + if (c->src.bytes == 8) + c->src.bytes = 4; + switch (c->src.bytes) { case 1: - src.val = insn_fetch(s8, 1, _eip); + c->src.val = insn_fetch(s8, 1, c->eip); break; case 2: - src.val = insn_fetch(s16, 2, _eip); + c->src.val = insn_fetch(s16, 2, c->eip); break; case 4: - src.val = insn_fetch(s32, 4, _eip); + c->src.val = insn_fetch(s32, 4, c->eip); break; } goto test; case 2: /* not */ - dst.val = ~dst.val; + c->dst.val = ~c->dst.val; break; case 3: /* neg */ - emulate_1op("neg", dst, _eflags); + emulate_1op("neg", c->dst, _eflags); break; default: goto cannot_emulate; } break; case 0xfe ... 0xff: /* Grp4/Grp5 */ - switch (modrm_reg) { + switch (c->modrm_reg) { case 0: /* inc */ - emulate_1op("inc", dst, _eflags); + emulate_1op("inc", c->dst, _eflags); break; case 1: /* dec */ - emulate_1op("dec", dst, _eflags); + emulate_1op("dec", c->dst, _eflags); break; case 4: /* jmp abs */ - if (b == 0xff) - _eip = dst.val; + if (c->b == 0xff) + c->eip = c->dst.val; else goto cannot_emulate; break; case 6: /* push */ /* 64-bit mode: PUSH always pushes a 64-bit operand. */ if (mode == X86EMUL_MODE_PROT64) { - dst.bytes = 8; - if ((rc = ops->read_std((unsigned long)dst.ptr, - &dst.val, 8, - ctxt->vcpu)) != 0) + c->dst.bytes = 8; + if ((rc = ops->read_std( + (unsigned long)c->dst.ptr, + &c->dst.val, 8, + ctxt->vcpu)) != 0) goto done; } - register_address_increment(_regs[VCPU_REGS_RSP], - -dst.bytes); + register_address_increment(c->regs[VCPU_REGS_RSP], + -c->dst.bytes); if ((rc = ops->write_emulated( register_address(ctxt->ss_base, - _regs[VCPU_REGS_RSP]), - &dst.val, dst.bytes, ctxt->vcpu)) != 0) + c->regs[VCPU_REGS_RSP]), + &c->dst.val, + c->dst.bytes, ctxt->vcpu)) != 0) goto done; no_wb = 1; break; @@ -1161,34 +1178,40 @@ done_prefixes: writeback: if (!no_wb) { - switch (dst.type) { + switch (c->dst.type) { case OP_REG: - /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */ - switch (dst.bytes) { + /* The 4-byte case *is* correct: + * in 64-bit mode we zero-extend. + */ + switch (c->dst.bytes) { case 1: - *(u8 *)dst.ptr = (u8)dst.val; + *(u8 *)c->dst.ptr = (u8)c->dst.val; break; case 2: - *(u16 *)dst.ptr = (u16)dst.val; + *(u16 *)c->dst.ptr = (u16)c->dst.val; break; case 4: - *dst.ptr = (u32)dst.val; + *c->dst.ptr = (u32)c->dst.val; break; /* 64b: zero-ext */ case 8: - *dst.ptr = dst.val; + *c->dst.ptr = c->dst.val; break; } break; case OP_MEM: - if (lock_prefix) - rc = ops->cmpxchg_emulated((unsigned long)dst. - ptr, &dst.orig_val, - &dst.val, dst.bytes, - ctxt->vcpu); + if (c->lock_prefix) + rc = ops->cmpxchg_emulated( + (unsigned long)c->dst.ptr, + &c->dst.orig_val, + &c->dst.val, + c->dst.bytes, + ctxt->vcpu); else - rc = ops->write_emulated((unsigned long)dst.ptr, - &dst.val, dst.bytes, - ctxt->vcpu); + rc = ops->write_emulated( + (unsigned long)c->dst.ptr, + &c->dst.val, + c->dst.bytes, + ctxt->vcpu); if (rc != 0) goto done; default: @@ -1197,173 +1220,185 @@ writeback: } /* Commit shadow register state. */ - memcpy(ctxt->vcpu->regs, _regs, sizeof _regs); + memcpy(ctxt->vcpu->regs, c->regs, sizeof c->regs); ctxt->eflags = _eflags; - ctxt->vcpu->rip = _eip; + ctxt->vcpu->rip = c->eip; done: return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0; special_insn: - if (twobyte) + if (c->twobyte) goto twobyte_special_insn; - switch(b) { + switch (c->b) { case 0x50 ... 0x57: /* push reg */ - if (op_bytes == 2) - src.val = (u16) _regs[b & 0x7]; + if (c->op_bytes == 2) + c->src.val = (u16) c->regs[c->b & 0x7]; else - src.val = (u32) _regs[b & 0x7]; - dst.type = OP_MEM; - dst.bytes = op_bytes; - dst.val = src.val; - register_address_increment(_regs[VCPU_REGS_RSP], -op_bytes); - dst.ptr = (void *) register_address( - ctxt->ss_base, _regs[VCPU_REGS_RSP]); + c->src.val = (u32) c->regs[c->b & 0x7]; + c->dst.type = OP_MEM; + c->dst.bytes = c->op_bytes; + c->dst.val = c->src.val; + register_address_increment(c->regs[VCPU_REGS_RSP], + -c->op_bytes); + c->dst.ptr = (void *) register_address( + ctxt->ss_base, c->regs[VCPU_REGS_RSP]); break; case 0x58 ... 0x5f: /* pop reg */ - dst.ptr = (unsigned long *)&_regs[b & 0x7]; + c->dst.ptr = + (unsigned long *)&c->regs[c->b & 0x7]; pop_instruction: if ((rc = ops->read_std(register_address(ctxt->ss_base, - _regs[VCPU_REGS_RSP]), dst.ptr, op_bytes, ctxt->vcpu)) - != 0) + c->regs[VCPU_REGS_RSP]), c->dst.ptr, + c->op_bytes, ctxt->vcpu)) != 0) goto done; - register_address_increment(_regs[VCPU_REGS_RSP], op_bytes); + register_address_increment(c->regs[VCPU_REGS_RSP], + c->op_bytes); no_wb = 1; /* Disable writeback. */ break; case 0x6a: /* push imm8 */ - src.val = 0L; - src.val = insn_fetch(s8, 1, _eip); - push: - dst.type = OP_MEM; - dst.bytes = op_bytes; - dst.val = src.val; - register_address_increment(_regs[VCPU_REGS_RSP], -op_bytes); - dst.ptr = (void *) register_address(ctxt->ss_base, - _regs[VCPU_REGS_RSP]); + c->src.val = 0L; + c->src.val = insn_fetch(s8, 1, c->eip); +push: + c->dst.type = OP_MEM; + c->dst.bytes = c->op_bytes; + c->dst.val = c->src.val; + register_address_increment(c->regs[VCPU_REGS_RSP], + -c->op_bytes); + c->dst.ptr = (void *) register_address(ctxt->ss_base, + c->regs[VCPU_REGS_RSP]); break; case 0x6c: /* insb */ case 0x6d: /* insw/insd */ if (kvm_emulate_pio_string(ctxt->vcpu, NULL, - 1, /* in */ - (d & ByteOp) ? 1 : op_bytes, /* size */ - rep_prefix ? - address_mask(_regs[VCPU_REGS_RCX]) : 1, /* count */ - (_eflags & EFLG_DF), /* down */ + 1, + (c->d & ByteOp) ? 1 : c->op_bytes, + c->rep_prefix ? + address_mask(c->regs[VCPU_REGS_RCX]) : 1, + (_eflags & EFLG_DF), register_address(ctxt->es_base, - _regs[VCPU_REGS_RDI]), /* address */ - rep_prefix, - _regs[VCPU_REGS_RDX] /* port */ - ) == 0) + c->regs[VCPU_REGS_RDI]), + c->rep_prefix, + c->regs[VCPU_REGS_RDX]) == 0) return -1; return 0; case 0x6e: /* outsb */ case 0x6f: /* outsw/outsd */ if (kvm_emulate_pio_string(ctxt->vcpu, NULL, - 0, /* in */ - (d & ByteOp) ? 1 : op_bytes, /* size */ - rep_prefix ? - address_mask(_regs[VCPU_REGS_RCX]) : 1, /* count */ - (_eflags & EFLG_DF), /* down */ - register_address(override_base ? - *override_base : ctxt->ds_base, - _regs[VCPU_REGS_RSI]), /* address */ - rep_prefix, - _regs[VCPU_REGS_RDX] /* port */ - ) == 0) + 0, + (c->d & ByteOp) ? 1 : c->op_bytes, + c->rep_prefix ? + address_mask(c->regs[VCPU_REGS_RCX]) : 1, + (_eflags & EFLG_DF), + register_address(c->override_base ? + *c->override_base : + ctxt->ds_base, + c->regs[VCPU_REGS_RSI]), + c->rep_prefix, + c->regs[VCPU_REGS_RDX]) == 0) return -1; return 0; case 0x70 ... 0x7f: /* jcc (short) */ { - int rel = insn_fetch(s8, 1, _eip); + int rel = insn_fetch(s8, 1, c->eip); - if (test_cc(b, _eflags)) + if (test_cc(c->b, _eflags)) JMP_REL(rel); break; } case 0x9c: /* pushf */ - src.val = (unsigned long) _eflags; + c->src.val = (unsigned long) _eflags; goto push; case 0x9d: /* popf */ - dst.ptr = (unsigned long *) &_eflags; + c->dst.ptr = (unsigned long *) &_eflags; goto pop_instruction; case 0xc3: /* ret */ - dst.ptr = &_eip; + c->dst.ptr = &c->eip; goto pop_instruction; case 0xf4: /* hlt */ ctxt->vcpu->halt_request = 1; goto done; } - if (rep_prefix) { - if (_regs[VCPU_REGS_RCX] == 0) { - ctxt->vcpu->rip = _eip; + if (c->rep_prefix) { + if (c->regs[VCPU_REGS_RCX] == 0) { + ctxt->vcpu->rip = c->eip; goto done; } - _regs[VCPU_REGS_RCX]--; - _eip = ctxt->vcpu->rip; + c->regs[VCPU_REGS_RCX]--; + c->eip = ctxt->vcpu->rip; } - switch (b) { + switch (c->b) { case 0xa4 ... 0xa5: /* movs */ - dst.type = OP_MEM; - dst.bytes = (d & ByteOp) ? 1 : op_bytes; - dst.ptr = (unsigned long *)register_address(ctxt->es_base, - _regs[VCPU_REGS_RDI]); + c->dst.type = OP_MEM; + c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->dst.ptr = (unsigned long *)register_address( + ctxt->es_base, + c->regs[VCPU_REGS_RDI]); if ((rc = ops->read_emulated(register_address( - override_base ? *override_base : ctxt->ds_base, - _regs[VCPU_REGS_RSI]), &dst.val, dst.bytes, ctxt->vcpu)) != 0) + c->override_base ? *c->override_base : + ctxt->ds_base, + c->regs[VCPU_REGS_RSI]), + &c->dst.val, + c->dst.bytes, ctxt->vcpu)) != 0) goto done; - register_address_increment(_regs[VCPU_REGS_RSI], - (_eflags & EFLG_DF) ? -dst.bytes : dst.bytes); - register_address_increment(_regs[VCPU_REGS_RDI], - (_eflags & EFLG_DF) ? -dst.bytes : dst.bytes); + register_address_increment(c->regs[VCPU_REGS_RSI], + (_eflags & EFLG_DF) ? -c->dst.bytes + : c->dst.bytes); + register_address_increment(c->regs[VCPU_REGS_RDI], + (_eflags & EFLG_DF) ? -c->dst.bytes + : c->dst.bytes); break; case 0xa6 ... 0xa7: /* cmps */ DPRINTF("Urk! I don't handle CMPS.\n"); goto cannot_emulate; case 0xaa ... 0xab: /* stos */ - dst.type = OP_MEM; - dst.bytes = (d & ByteOp) ? 1 : op_bytes; - dst.ptr = (unsigned long *)cr2; - dst.val = _regs[VCPU_REGS_RAX]; - register_address_increment(_regs[VCPU_REGS_RDI], - (_eflags & EFLG_DF) ? -dst.bytes : dst.bytes); + c->dst.type = OP_MEM; + c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->dst.ptr = (unsigned long *)cr2; + c->dst.val = c->regs[VCPU_REGS_RAX]; + register_address_increment(c->regs[VCPU_REGS_RDI], + (_eflags & EFLG_DF) ? -c->dst.bytes + : c->dst.bytes); break; case 0xac ... 0xad: /* lods */ - dst.type = OP_REG; - dst.bytes = (d & ByteOp) ? 1 : op_bytes; - dst.ptr = (unsigned long *)&_regs[VCPU_REGS_RAX]; - if ((rc = ops->read_emulated(cr2, &dst.val, dst.bytes, + c->dst.type = OP_REG; + c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; + if ((rc = ops->read_emulated(cr2, &c->dst.val, + c->dst.bytes, ctxt->vcpu)) != 0) goto done; - register_address_increment(_regs[VCPU_REGS_RSI], - (_eflags & EFLG_DF) ? -dst.bytes : dst.bytes); + register_address_increment(c->regs[VCPU_REGS_RSI], + (_eflags & EFLG_DF) ? -c->dst.bytes + : c->dst.bytes); break; case 0xae ... 0xaf: /* scas */ DPRINTF("Urk! I don't handle SCAS.\n"); goto cannot_emulate; case 0xe8: /* call (near) */ { long int rel; - switch (op_bytes) { + switch (c->op_bytes) { case 2: - rel = insn_fetch(s16, 2, _eip); + rel = insn_fetch(s16, 2, c->eip); break; case 4: - rel = insn_fetch(s32, 4, _eip); + rel = insn_fetch(s32, 4, c->eip); break; case 8: - rel = insn_fetch(s64, 8, _eip); + rel = insn_fetch(s64, 8, c->eip); break; default: DPRINTF("Call: Invalid op_bytes\n"); goto cannot_emulate; } - src.val = (unsigned long) _eip; + c->src.val = (unsigned long) c->eip; JMP_REL(rel); - op_bytes = ad_bytes; + c->op_bytes = c->ad_bytes; goto push; } case 0xe9: /* jmp rel */ case 0xeb: /* jmp rel short */ - JMP_REL(src.val); + JMP_REL(c->src.val); no_wb = 1; /* Disable writeback. */ break; @@ -1372,16 +1407,16 @@ special_insn: goto writeback; twobyte_insn: - switch (b) { + switch (c->b) { case 0x01: /* lgdt, lidt, lmsw */ /* Disable writeback. */ no_wb = 1; - switch (modrm_reg) { + switch (c->modrm_reg) { u16 size; unsigned long address; case 0: /* vmcall */ - if (modrm_mod != 3 || modrm_rm != 1) + if (c->modrm_mod != 3 || c->modrm_rm != 1) goto cannot_emulate; rc = kvm_fix_hypercall(ctxt->vcpu); @@ -1391,37 +1426,37 @@ twobyte_insn: kvm_emulate_hypercall(ctxt->vcpu); break; case 2: /* lgdt */ - rc = read_descriptor(ctxt, ops, src.ptr, - &size, &address, op_bytes); + rc = read_descriptor(ctxt, ops, c->src.ptr, + &size, &address, c->op_bytes); if (rc) goto done; realmode_lgdt(ctxt->vcpu, size, address); break; case 3: /* lidt/vmmcall */ - if (modrm_mod == 3 && modrm_rm == 1) { + if (c->modrm_mod == 3 && c->modrm_rm == 1) { rc = kvm_fix_hypercall(ctxt->vcpu); if (rc) goto done; kvm_emulate_hypercall(ctxt->vcpu); } else { - rc = read_descriptor(ctxt, ops, src.ptr, + rc = read_descriptor(ctxt, ops, c->src.ptr, &size, &address, - op_bytes); + c->op_bytes); if (rc) goto done; realmode_lidt(ctxt->vcpu, size, address); } break; case 4: /* smsw */ - if (modrm_mod != 3) + if (c->modrm_mod != 3) goto cannot_emulate; - *(u16 *)&_regs[modrm_rm] + *(u16 *)&c->regs[c->modrm_rm] = realmode_get_cr(ctxt->vcpu, 0); break; case 6: /* lmsw */ - if (modrm_mod != 3) + if (c->modrm_mod != 3) goto cannot_emulate; - realmode_lmsw(ctxt->vcpu, (u16)modrm_val, &_eflags); + realmode_lmsw(ctxt->vcpu, (u16)c->modrm_val, &_eflags); break; case 7: /* invlpg*/ emulate_invlpg(ctxt->vcpu, cr2); @@ -1432,24 +1467,26 @@ twobyte_insn: break; case 0x21: /* mov from dr to reg */ no_wb = 1; - if (modrm_mod != 3) + if (c->modrm_mod != 3) goto cannot_emulate; - rc = emulator_get_dr(ctxt, modrm_reg, &_regs[modrm_rm]); + rc = emulator_get_dr(ctxt, c->modrm_reg, + &c->regs[c->modrm_rm]); break; case 0x23: /* mov from reg to dr */ no_wb = 1; - if (modrm_mod != 3) + if (c->modrm_mod != 3) goto cannot_emulate; - rc = emulator_set_dr(ctxt, modrm_reg, _regs[modrm_rm]); + rc = emulator_set_dr(ctxt, c->modrm_reg, + c->regs[c->modrm_rm]); break; case 0x40 ... 0x4f: /* cmov */ - dst.val = dst.orig_val = src.val; + c->dst.val = c->dst.orig_val = c->src.val; no_wb = 1; /* * First, assume we're decoding an even cmov opcode * (lsb == 0). */ - switch ((b & 15) >> 1) { + switch ((c->b & 15) >> 1) { case 0: /* cmovo */ no_wb = (_eflags & EFLG_OF) ? 0 : 1; break; @@ -1477,46 +1514,50 @@ twobyte_insn: break; } /* Odd cmov opcodes (lsb == 1) have inverted sense. */ - no_wb ^= b & 1; + no_wb ^= c->b & 1; break; case 0xa3: bt: /* bt */ - src.val &= (dst.bytes << 3) - 1; /* only subword offset */ - emulate_2op_SrcV_nobyte("bt", src, dst, _eflags); + /* only subword offset */ + c->src.val &= (c->dst.bytes << 3) - 1; + emulate_2op_SrcV_nobyte("bt", c->src, c->dst, _eflags); break; case 0xab: bts: /* bts */ - src.val &= (dst.bytes << 3) - 1; /* only subword offset */ - emulate_2op_SrcV_nobyte("bts", src, dst, _eflags); + /* only subword offset */ + c->src.val &= (c->dst.bytes << 3) - 1; + emulate_2op_SrcV_nobyte("bts", c->src, c->dst, _eflags); break; case 0xb0 ... 0xb1: /* cmpxchg */ /* * Save real source value, then compare EAX against * destination. */ - src.orig_val = src.val; - src.val = _regs[VCPU_REGS_RAX]; - emulate_2op_SrcV("cmp", src, dst, _eflags); + c->src.orig_val = c->src.val; + c->src.val = c->regs[VCPU_REGS_RAX]; + emulate_2op_SrcV("cmp", c->src, c->dst, _eflags); if (_eflags & EFLG_ZF) { /* Success: write back to memory. */ - dst.val = src.orig_val; + c->dst.val = c->src.orig_val; } else { /* Failure: write the value we saw to EAX. */ - dst.type = OP_REG; - dst.ptr = (unsigned long *)&_regs[VCPU_REGS_RAX]; + c->dst.type = OP_REG; + c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; } break; case 0xb3: btr: /* btr */ - src.val &= (dst.bytes << 3) - 1; /* only subword offset */ - emulate_2op_SrcV_nobyte("btr", src, dst, _eflags); + /* only subword offset */ + c->src.val &= (c->dst.bytes << 3) - 1; + emulate_2op_SrcV_nobyte("btr", c->src, c->dst, _eflags); break; case 0xb6 ... 0xb7: /* movzx */ - dst.bytes = op_bytes; - dst.val = (d & ByteOp) ? (u8) src.val : (u16) src.val; + c->dst.bytes = c->op_bytes; + c->dst.val = (c->d & ByteOp) ? (u8) c->src.val + : (u16) c->src.val; break; case 0xba: /* Grp8 */ - switch (modrm_reg & 3) { + switch (c->modrm_reg & 3) { case 0: goto bt; case 1: @@ -1529,16 +1570,19 @@ twobyte_insn: break; case 0xbb: btc: /* btc */ - src.val &= (dst.bytes << 3) - 1; /* only subword offset */ - emulate_2op_SrcV_nobyte("btc", src, dst, _eflags); + /* only subword offset */ + c->src.val &= (c->dst.bytes << 3) - 1; + emulate_2op_SrcV_nobyte("btc", c->src, c->dst, _eflags); break; case 0xbe ... 0xbf: /* movsx */ - dst.bytes = op_bytes; - dst.val = (d & ByteOp) ? (s8) src.val : (s16) src.val; + c->dst.bytes = c->op_bytes; + c->dst.val = (c->d & ByteOp) ? (s8) c->src.val : + (s16) c->src.val; break; case 0xc3: /* movnti */ - dst.bytes = op_bytes; - dst.val = (op_bytes == 4) ? (u32) src.val : (u64) src.val; + c->dst.bytes = c->op_bytes; + c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val : + (u64) c->src.val; break; } goto writeback; @@ -1546,7 +1590,7 @@ twobyte_insn: twobyte_special_insn: /* Disable writeback. */ no_wb = 1; - switch (b) { + switch (c->b) { case 0x06: emulate_clts(ctxt->vcpu); break; @@ -1558,56 +1602,59 @@ twobyte_special_insn: case 0x18: /* Grp16 (prefetch/nop) */ break; case 0x20: /* mov cr, reg */ - if (modrm_mod != 3) + if (c->modrm_mod != 3) goto cannot_emulate; - _regs[modrm_rm] = realmode_get_cr(ctxt->vcpu, modrm_reg); + c->regs[c->modrm_rm] = + realmode_get_cr(ctxt->vcpu, c->modrm_reg); break; case 0x22: /* mov reg, cr */ - if (modrm_mod != 3) + if (c->modrm_mod != 3) goto cannot_emulate; - realmode_set_cr(ctxt->vcpu, modrm_reg, modrm_val, &_eflags); + realmode_set_cr(ctxt->vcpu, + c->modrm_reg, c->modrm_val, &_eflags); break; case 0x30: /* wrmsr */ - msr_data = (u32)_regs[VCPU_REGS_RAX] - | ((u64)_regs[VCPU_REGS_RDX] << 32); - rc = kvm_set_msr(ctxt->vcpu, _regs[VCPU_REGS_RCX], msr_data); + msr_data = (u32)c->regs[VCPU_REGS_RAX] + | ((u64)c->regs[VCPU_REGS_RDX] << 32); + rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data); if (rc) { kvm_x86_ops->inject_gp(ctxt->vcpu, 0); - _eip = ctxt->vcpu->rip; + c->eip = ctxt->vcpu->rip; } rc = X86EMUL_CONTINUE; break; case 0x32: /* rdmsr */ - rc = kvm_get_msr(ctxt->vcpu, _regs[VCPU_REGS_RCX], &msr_data); + rc = kvm_get_msr(ctxt->vcpu, + c->regs[VCPU_REGS_RCX], &msr_data); if (rc) { kvm_x86_ops->inject_gp(ctxt->vcpu, 0); - _eip = ctxt->vcpu->rip; + c->eip = ctxt->vcpu->rip; } else { - _regs[VCPU_REGS_RAX] = (u32)msr_data; - _regs[VCPU_REGS_RDX] = msr_data >> 32; + c->regs[VCPU_REGS_RAX] = (u32)msr_data; + c->regs[VCPU_REGS_RDX] = msr_data >> 32; } rc = X86EMUL_CONTINUE; break; case 0x80 ... 0x8f: /* jnz rel, etc*/ { long int rel; - switch (op_bytes) { + switch (c->op_bytes) { case 2: - rel = insn_fetch(s16, 2, _eip); + rel = insn_fetch(s16, 2, c->eip); break; case 4: - rel = insn_fetch(s32, 4, _eip); + rel = insn_fetch(s32, 4, c->eip); break; case 8: - rel = insn_fetch(s64, 8, _eip); + rel = insn_fetch(s64, 8, c->eip); break; default: DPRINTF("jnz: Invalid op_bytes\n"); goto cannot_emulate; } - if (test_cc(b, _eflags)) + if (test_cc(c->b, _eflags)) JMP_REL(rel); break; } @@ -1617,14 +1664,16 @@ twobyte_special_insn: if ((rc = ops->read_emulated(cr2, &old, 8, ctxt->vcpu)) != 0) goto done; - if (((u32) (old >> 0) != (u32) _regs[VCPU_REGS_RAX]) || - ((u32) (old >> 32) != (u32) _regs[VCPU_REGS_RDX])) { - _regs[VCPU_REGS_RAX] = (u32) (old >> 0); - _regs[VCPU_REGS_RDX] = (u32) (old >> 32); + if (((u32) (old >> 0) != + (u32) c->regs[VCPU_REGS_RAX]) || + ((u32) (old >> 32) != + (u32) c->regs[VCPU_REGS_RDX])) { + c->regs[VCPU_REGS_RAX] = (u32) (old >> 0); + c->regs[VCPU_REGS_RDX] = (u32) (old >> 32); _eflags &= ~EFLG_ZF; } else { - new = ((u64)_regs[VCPU_REGS_RCX] << 32) - | (u32) _regs[VCPU_REGS_RBX]; + new = ((u64)c->regs[VCPU_REGS_RCX] << 32) + | (u32) c->regs[VCPU_REGS_RBX]; if ((rc = ops->cmpxchg_emulated(cr2, &old, &new, 8, ctxt->vcpu)) != 0) goto done; @@ -1636,6 +1685,6 @@ twobyte_special_insn: goto writeback; cannot_emulate: - DPRINTF("Cannot emulate %02x\n", b); + DPRINTF("Cannot emulate %02x\n", c->b); return -1; } diff --git a/drivers/kvm/x86_emulate.h b/drivers/kvm/x86_emulate.h index 92c73aa7f9ac..c354200d5834 100644 --- a/drivers/kvm/x86_emulate.h +++ b/drivers/kvm/x86_emulate.h @@ -112,6 +112,36 @@ struct x86_emulate_ops { }; +/* Type, address-of, and value of an instruction's operand. */ +struct operand { + enum { OP_REG, OP_MEM, OP_IMM } type; + unsigned int bytes; + unsigned long val, orig_val, *ptr; +}; + +struct decode_cache { + u8 twobyte; + u8 b; + u8 lock_prefix; + u8 rep_prefix; + u8 op_bytes; + u8 ad_bytes; + struct operand src; + struct operand dst; + unsigned long *override_base; + unsigned int d; + unsigned long regs[NR_VCPU_REGS]; + unsigned long eip; + /* modrm */ + u8 modrm; + u8 modrm_mod; + u8 modrm_reg; + u8 modrm_rm; + u8 use_modrm_ea; + unsigned long modrm_ea; + unsigned long modrm_val; +}; + struct x86_emulate_ctxt { /* Register state before/after emulation. */ struct kvm_vcpu *vcpu; @@ -129,6 +159,10 @@ struct x86_emulate_ctxt { unsigned long ss_base; unsigned long gs_base; unsigned long fs_base; + + /* decode cache */ + + struct decode_cache decode; }; /* Execution mode, passed to the emulator. */ From 8b4caf6650808024c37ec4b29cf81b308af998b1 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Tue, 18 Sep 2007 11:27:19 +0200 Subject: [PATCH 005/249] KVM: x86 emulator: move all decoding process to function x86_decode_insn() Split the decoding process into a new function x86_decode_insn(). Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 77 +++++++++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 24 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index d7026cb8fc70..540589c5d427 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -517,20 +517,16 @@ static int test_cc(unsigned int condition, unsigned int flags) } int -x86_emulate_memop(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) +x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { struct decode_cache *c = &ctxt->decode; u8 sib, rex_prefix = 0; unsigned int i; int rc = 0; - unsigned long cr2 = ctxt->cr2; int mode = ctxt->mode; int index_reg = 0, base_reg = 0, scale, rip_relative = 0; - int no_wb = 0; - u64 msr_data; /* Shadow copy of register state. Committed on successful emulation. */ - unsigned long _eflags = ctxt->eflags; memset(c, 0, sizeof(struct decode_cache)); c->eip = ctxt->vcpu->rip; @@ -622,8 +618,10 @@ done_prefixes: } /* Unrecognised? */ - if (c->d == 0) - goto cannot_emulate; + if (c->d == 0) { + DPRINTF("Cannot emulate %02x\n", c->b); + return -1; + } } /* ModRM and SIB bytes. */ @@ -776,7 +774,6 @@ done_prefixes: } if (c->ad_bytes != 8) c->modrm_ea = (u32)c->modrm_ea; - cr2 = c->modrm_ea; modrm_done: ; } @@ -838,13 +835,6 @@ done_prefixes: break; } c->src.type = OP_MEM; - c->src.ptr = (unsigned long *)cr2; - c->src.val = 0; - if ((rc = ops->read_emulated((unsigned long)c->src.ptr, - &c->src.val, - c->src.bytes, ctxt->vcpu)) != 0) - goto done; - c->src.orig_val = c->src.val; break; case SrcImm: c->src.type = OP_IMM; @@ -877,7 +867,7 @@ done_prefixes: switch (c->d & DstMask) { case ImplicitOps: /* Special instructions do their own operand decoding. */ - goto special_insn; + return 0; case DstReg: c->dst.type = OP_REG; if ((c->d & ByteOp) @@ -905,14 +895,54 @@ done_prefixes: } break; case DstMem: - c->dst.type = OP_MEM; - c->dst.ptr = (unsigned long *)cr2; - c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; - c->dst.val = 0; if ((c->d & ModRM) && c->modrm_mod == 3) { c->dst.type = OP_REG; break; } + c->dst.type = OP_MEM; + break; + } + +done: + return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0; +} + +int +x86_emulate_memop(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) +{ + unsigned long cr2 = ctxt->cr2; + int no_wb = 0; + u64 msr_data; + unsigned long _eflags = ctxt->eflags; + struct decode_cache *c = &ctxt->decode; + int rc; + + rc = x86_decode_insn(ctxt, ops); + if (rc) + return rc; + + if ((c->d & ModRM) && (c->modrm_mod != 3)) + cr2 = c->modrm_ea; + + if (c->src.type == OP_MEM) { + c->src.ptr = (unsigned long *)cr2; + c->src.val = 0; + if ((rc = ops->read_emulated((unsigned long)c->src.ptr, + &c->src.val, + c->src.bytes, + ctxt->vcpu)) != 0) + goto done; + c->src.orig_val = c->src.val; + } + + if ((c->d & DstMask) == ImplicitOps) + goto special_insn; + + + if (c->dst.type == OP_MEM) { + c->dst.ptr = (unsigned long *)cr2; + c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->dst.val = 0; if (c->d & BitOp) { unsigned long mask = ~(c->dst.bytes * 8 - 1); @@ -925,7 +955,6 @@ done_prefixes: &c->dst.val, c->dst.bytes, ctxt->vcpu)) != 0)) goto done; - break; } c->dst.orig_val = c->dst.val; @@ -983,7 +1012,7 @@ done_prefixes: emulate_2op_SrcV("cmp", c->src, c->dst, _eflags); break; case 0x63: /* movsxd */ - if (mode != X86EMUL_MODE_PROT64) + if (ctxt->mode != X86EMUL_MODE_PROT64) goto cannot_emulate; c->dst.val = (s32) c->src.val; break; @@ -1041,7 +1070,7 @@ done_prefixes: break; case 0x8f: /* pop (sole member of Grp1a) */ /* 64-bit mode: POP always pops a 64-bit operand. */ - if (mode == X86EMUL_MODE_PROT64) + if (ctxt->mode == X86EMUL_MODE_PROT64) c->dst.bytes = 8; if ((rc = ops->read_std(register_address( ctxt->ss_base, @@ -1152,7 +1181,7 @@ done_prefixes: break; case 6: /* push */ /* 64-bit mode: PUSH always pushes a 64-bit operand. */ - if (mode == X86EMUL_MODE_PROT64) { + if (ctxt->mode == X86EMUL_MODE_PROT64) { c->dst.bytes = 8; if ((rc = ops->read_std( (unsigned long)c->dst.ptr, From 1be3aa47182e94944e57b176a5c4ee4e74f1ce33 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Tue, 18 Sep 2007 11:27:27 +0200 Subject: [PATCH 006/249] KVM: emulate_instruction() calls now x86_decode_insn() and x86_emulate_insn() emulate_instruction() calls now x86_decode_insn() and x86_emulate_insn(). x86_emulate_insn() is x86_emulate_memop() without the decoding part. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 5 ++++- drivers/kvm/x86_emulate.c | 8 ++------ drivers/kvm/x86_emulate.h | 11 ++++------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 1c662f63b7a9..800ab5028ed3 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1287,7 +1287,10 @@ int emulate_instruction(struct kvm_vcpu *vcpu, vcpu->mmio_is_write = 0; vcpu->pio.string = 0; - r = x86_emulate_memop(&emulate_ctxt, &emulate_ops); + r = x86_decode_insn(&emulate_ctxt, &emulate_ops); + if (r == 0) + r = x86_emulate_insn(&emulate_ctxt, &emulate_ops); + if (vcpu->pio.string) return EMULATE_DO_MMIO; diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 540589c5d427..c191093982d8 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -908,18 +908,14 @@ done: } int -x86_emulate_memop(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) +x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { unsigned long cr2 = ctxt->cr2; int no_wb = 0; u64 msr_data; unsigned long _eflags = ctxt->eflags; struct decode_cache *c = &ctxt->decode; - int rc; - - rc = x86_decode_insn(ctxt, ops); - if (rc) - return rc; + int rc = 0; if ((c->d & ModRM) && (c->modrm_mod != 3)) cr2 = c->modrm_ea; diff --git a/drivers/kvm/x86_emulate.h b/drivers/kvm/x86_emulate.h index c354200d5834..28acad416bca 100644 --- a/drivers/kvm/x86_emulate.h +++ b/drivers/kvm/x86_emulate.h @@ -178,12 +178,9 @@ struct x86_emulate_ctxt { #define X86EMUL_MODE_HOST X86EMUL_MODE_PROT64 #endif -/* - * x86_emulate_memop: Emulate an instruction that faulted attempting to - * read/write a 'special' memory area. - * Returns -1 on failure, 0 on success. - */ -int x86_emulate_memop(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops); +int x86_decode_insn(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops); +int x86_emulate_insn(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops); #endif /* __X86_EMULATE_H__ */ From 3427318fd2244737a466a06a93c5fe579852f871 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Tue, 18 Sep 2007 11:27:37 +0200 Subject: [PATCH 007/249] KVM: Call x86_decode_insn() only when needed Move emulate_ctxt to kvm_vcpu to keep emulate context when we exit from kvm module. Call x86_decode_insn() only when needed. Modify x86_emulate_insn() to not modify the context if it must be re-entered. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 8 +++- drivers/kvm/kvm_main.c | 77 ++++++++++++++++++++++----------------- drivers/kvm/svm.c | 9 +++-- drivers/kvm/vmx.c | 9 +++-- drivers/kvm/x86_emulate.c | 24 ++++++++++-- 5 files changed, 82 insertions(+), 45 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index da9c3aa1c08c..e885b190b798 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -207,6 +207,8 @@ enum { VCPU_SREG_LDTR, }; +#include "x86_emulate.h" + struct kvm_pio_request { unsigned long count; int cur_count; @@ -380,6 +382,10 @@ struct kvm_vcpu { int cpuid_nent; struct kvm_cpuid_entry cpuid_entries[KVM_MAX_CPUID_ENTRIES]; + + /* emulate context */ + + struct x86_emulate_ctxt emulate_ctxt; }; struct kvm_mem_alias { @@ -555,7 +561,7 @@ enum emulation_result { }; int emulate_instruction(struct kvm_vcpu *vcpu, struct kvm_run *run, - unsigned long cr2, u16 error_code); + unsigned long cr2, u16 error_code, int no_decode); void kvm_report_emulation_failure(struct kvm_vcpu *cvpu, const char *context); void realmode_lgdt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); void realmode_lidt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 800ab5028ed3..710483669f34 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1251,45 +1251,56 @@ struct x86_emulate_ops emulate_ops = { int emulate_instruction(struct kvm_vcpu *vcpu, struct kvm_run *run, unsigned long cr2, - u16 error_code) + u16 error_code, + int no_decode) { - struct x86_emulate_ctxt emulate_ctxt; - int r; - int cs_db, cs_l; + int r = 0; vcpu->mmio_fault_cr2 = cr2; kvm_x86_ops->cache_regs(vcpu); - kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); - - emulate_ctxt.vcpu = vcpu; - emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu); - emulate_ctxt.cr2 = cr2; - emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM) - ? X86EMUL_MODE_REAL : cs_l - ? X86EMUL_MODE_PROT64 : cs_db - ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; - - if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) { - emulate_ctxt.cs_base = 0; - emulate_ctxt.ds_base = 0; - emulate_ctxt.es_base = 0; - emulate_ctxt.ss_base = 0; - } else { - emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS); - emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS); - emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES); - emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS); - } - - emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS); - emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS); - vcpu->mmio_is_write = 0; vcpu->pio.string = 0; - r = x86_decode_insn(&emulate_ctxt, &emulate_ops); + + if (!no_decode) { + int cs_db, cs_l; + kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); + + vcpu->emulate_ctxt.vcpu = vcpu; + vcpu->emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu); + vcpu->emulate_ctxt.cr2 = cr2; + vcpu->emulate_ctxt.mode = + (vcpu->emulate_ctxt.eflags & X86_EFLAGS_VM) + ? X86EMUL_MODE_REAL : cs_l + ? X86EMUL_MODE_PROT64 : cs_db + ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; + + if (vcpu->emulate_ctxt.mode == X86EMUL_MODE_PROT64) { + vcpu->emulate_ctxt.cs_base = 0; + vcpu->emulate_ctxt.ds_base = 0; + vcpu->emulate_ctxt.es_base = 0; + vcpu->emulate_ctxt.ss_base = 0; + } else { + vcpu->emulate_ctxt.cs_base = + get_segment_base(vcpu, VCPU_SREG_CS); + vcpu->emulate_ctxt.ds_base = + get_segment_base(vcpu, VCPU_SREG_DS); + vcpu->emulate_ctxt.es_base = + get_segment_base(vcpu, VCPU_SREG_ES); + vcpu->emulate_ctxt.ss_base = + get_segment_base(vcpu, VCPU_SREG_SS); + } + + vcpu->emulate_ctxt.gs_base = + get_segment_base(vcpu, VCPU_SREG_GS); + vcpu->emulate_ctxt.fs_base = + get_segment_base(vcpu, VCPU_SREG_FS); + + r = x86_decode_insn(&vcpu->emulate_ctxt, &emulate_ops); + } + if (r == 0) - r = x86_emulate_insn(&emulate_ctxt, &emulate_ops); + r = x86_emulate_insn(&vcpu->emulate_ctxt, &emulate_ops); if (vcpu->pio.string) return EMULATE_DO_MMIO; @@ -1313,7 +1324,7 @@ int emulate_instruction(struct kvm_vcpu *vcpu, } kvm_x86_ops->decache_regs(vcpu); - kvm_x86_ops->set_rflags(vcpu, emulate_ctxt.eflags); + kvm_x86_ops->set_rflags(vcpu, vcpu->emulate_ctxt.eflags); if (vcpu->mmio_is_write) { vcpu->mmio_needed = 0; @@ -2055,7 +2066,7 @@ static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) vcpu->mmio_read_completed = 1; vcpu->mmio_needed = 0; r = emulate_instruction(vcpu, kvm_run, - vcpu->mmio_fault_cr2, 0); + vcpu->mmio_fault_cr2, 0, 1); if (r == EMULATE_DO_MMIO) { /* * Read-modify-write. Back to userspace. diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 794d95416f7b..f268bd51f337 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -960,7 +960,7 @@ static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) return 1; } er = emulate_instruction(&svm->vcpu, kvm_run, fault_address, - error_code); + error_code, 0); mutex_unlock(&kvm->lock); switch (er) { @@ -984,7 +984,7 @@ static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { int er; - er = emulate_instruction(&svm->vcpu, kvm_run, 0, 0); + er = emulate_instruction(&svm->vcpu, kvm_run, 0, 0, 0); if (er != EMULATE_DONE) inject_ud(&svm->vcpu); @@ -1027,7 +1027,8 @@ static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) string = (io_info & SVM_IOIO_STR_MASK) != 0; if (string) { - if (emulate_instruction(&svm->vcpu, kvm_run, 0, 0) == EMULATE_DO_MMIO) + if (emulate_instruction(&svm->vcpu, + kvm_run, 0, 0, 0) == EMULATE_DO_MMIO) return 0; return 1; } @@ -1086,7 +1087,7 @@ static int cpuid_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) static int emulate_on_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { - if (emulate_instruction(&svm->vcpu, NULL, 0, 0) != EMULATE_DONE) + if (emulate_instruction(&svm->vcpu, NULL, 0, 0, 0) != EMULATE_DONE) pr_unimpl(&svm->vcpu, "%s: failed\n", __FUNCTION__); return 1; } diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 47c827d3007c..91768d5dbfb9 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1750,7 +1750,7 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu, * Cause the #SS fault with 0 error code in VM86 mode. */ if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) - if (emulate_instruction(vcpu, NULL, 0, 0) == EMULATE_DONE) + if (emulate_instruction(vcpu, NULL, 0, 0, 0) == EMULATE_DONE) return 1; return 0; } @@ -1787,7 +1787,7 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) } if (is_invalid_opcode(intr_info)) { - er = emulate_instruction(vcpu, kvm_run, 0, 0); + er = emulate_instruction(vcpu, kvm_run, 0, 0, 0); if (er != EMULATE_DONE) vmx_inject_ud(vcpu); @@ -1812,7 +1812,7 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) return 1; } - er = emulate_instruction(vcpu, kvm_run, cr2, error_code); + er = emulate_instruction(vcpu, kvm_run, cr2, error_code, 0); mutex_unlock(&vcpu->kvm->lock); switch (er) { @@ -1873,7 +1873,8 @@ static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) string = (exit_qualification & 16) != 0; if (string) { - if (emulate_instruction(vcpu, kvm_run, 0, 0) == EMULATE_DO_MMIO) + if (emulate_instruction(vcpu, + kvm_run, 0, 0, 0) == EMULATE_DO_MMIO) return 0; return 1; } diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index c191093982d8..dc9d2a870fbc 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -913,10 +913,19 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) unsigned long cr2 = ctxt->cr2; int no_wb = 0; u64 msr_data; + unsigned long saved_eip = 0; unsigned long _eflags = ctxt->eflags; struct decode_cache *c = &ctxt->decode; int rc = 0; + /* Shadow copy of register state. Committed on successful emulation. + * NOTE: we can copy them from vcpu as x86_decode_insn() doesn't + * modify them. + */ + + memcpy(c->regs, ctxt->vcpu->regs, sizeof c->regs); + saved_eip = c->eip; + if ((c->d & ModRM) && (c->modrm_mod != 3)) cr2 = c->modrm_ea; @@ -1250,7 +1259,11 @@ writeback: ctxt->vcpu->rip = c->eip; done: - return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0; + if (rc == X86EMUL_UNHANDLEABLE) { + c->eip = saved_eip; + return -1; + } + return 0; special_insn: if (c->twobyte) @@ -1305,8 +1318,10 @@ push: register_address(ctxt->es_base, c->regs[VCPU_REGS_RDI]), c->rep_prefix, - c->regs[VCPU_REGS_RDX]) == 0) + c->regs[VCPU_REGS_RDX]) == 0) { + c->eip = saved_eip; return -1; + } return 0; case 0x6e: /* outsb */ case 0x6f: /* outsw/outsd */ @@ -1321,8 +1336,10 @@ push: ctxt->ds_base, c->regs[VCPU_REGS_RSI]), c->rep_prefix, - c->regs[VCPU_REGS_RDX]) == 0) + c->regs[VCPU_REGS_RDX]) == 0) { + c->eip = saved_eip; return -1; + } return 0; case 0x70 ... 0x7f: /* jcc (short) */ { int rel = insn_fetch(s8, 1, c->eip); @@ -1711,5 +1728,6 @@ twobyte_special_insn: cannot_emulate: DPRINTF("Cannot emulate %02x\n", c->b); + c->eip = saved_eip; return -1; } From 51c6cf662b4b361a09fbd324f4c67875d9bcfbea Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 29 Aug 2007 03:48:05 +0300 Subject: [PATCH 008/249] KVM: VMX: Further reduce efer reloads KVM avoids reloading the efer msr when the difference between the guest and host values consist of the long mode bits (which are switched by hardware) and the NX bit (which is emulated by the KVM MMU). This patch also allows KVM to ignore SCE (syscall enable) when the guest is running in 32-bit mode. This is because the syscall instruction is not available in 32-bit mode on Intel processors, so the SCE bit is effectively meaningless. Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 61 ++++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 91768d5dbfb9..8eb49e055ec0 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -57,6 +57,7 @@ struct vcpu_vmx { u16 fs_sel, gs_sel, ldt_sel; int gs_ldt_reload_needed; int fs_reload_needed; + int guest_efer_loaded; }host_state; }; @@ -74,8 +75,6 @@ static DEFINE_PER_CPU(struct vmcs *, current_vmcs); static struct page *vmx_io_bitmap_a; static struct page *vmx_io_bitmap_b; -#define EFER_SAVE_RESTORE_BITS ((u64)EFER_SCE) - static struct vmcs_config { int size; int order; @@ -138,18 +137,6 @@ static void save_msrs(struct kvm_msr_entry *e, int n) rdmsrl(e[i].index, e[i].data); } -static inline u64 msr_efer_save_restore_bits(struct kvm_msr_entry msr) -{ - return (u64)msr.data & EFER_SAVE_RESTORE_BITS; -} - -static inline int msr_efer_need_save_restore(struct vcpu_vmx *vmx) -{ - int efer_offset = vmx->msr_offset_efer; - return msr_efer_save_restore_bits(vmx->host_msrs[efer_offset]) != - msr_efer_save_restore_bits(vmx->guest_msrs[efer_offset]); -} - static inline int is_page_fault(u32 intr_info) { return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | @@ -351,16 +338,42 @@ static void reload_tss(void) static void load_transition_efer(struct vcpu_vmx *vmx) { - u64 trans_efer; int efer_offset = vmx->msr_offset_efer; + u64 host_efer = vmx->host_msrs[efer_offset].data; + u64 guest_efer = vmx->guest_msrs[efer_offset].data; + u64 ignore_bits; - trans_efer = vmx->host_msrs[efer_offset].data; - trans_efer &= ~EFER_SAVE_RESTORE_BITS; - trans_efer |= msr_efer_save_restore_bits(vmx->guest_msrs[efer_offset]); - wrmsrl(MSR_EFER, trans_efer); + if (efer_offset < 0) + return; + /* + * NX is emulated; LMA and LME handled by hardware; SCE meaninless + * outside long mode + */ + ignore_bits = EFER_NX | EFER_SCE; +#ifdef CONFIG_X86_64 + ignore_bits |= EFER_LMA | EFER_LME; + /* SCE is meaningful only in long mode on Intel */ + if (guest_efer & EFER_LMA) + ignore_bits &= ~(u64)EFER_SCE; +#endif + if ((guest_efer & ~ignore_bits) == (host_efer & ~ignore_bits)) + return; + + vmx->host_state.guest_efer_loaded = 1; + guest_efer &= ~ignore_bits; + guest_efer |= host_efer & ignore_bits; + wrmsrl(MSR_EFER, guest_efer); vmx->vcpu.stat.efer_reload++; } +static void reload_host_efer(struct vcpu_vmx *vmx) +{ + if (vmx->host_state.guest_efer_loaded) { + vmx->host_state.guest_efer_loaded = 0; + load_msrs(vmx->host_msrs + vmx->msr_offset_efer, 1); + } +} + static void vmx_save_host_state(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -406,8 +419,7 @@ static void vmx_save_host_state(struct kvm_vcpu *vcpu) } #endif load_msrs(vmx->guest_msrs, vmx->save_nmsrs); - if (msr_efer_need_save_restore(vmx)) - load_transition_efer(vmx); + load_transition_efer(vmx); } static void vmx_load_host_state(struct vcpu_vmx *vmx) @@ -436,8 +448,7 @@ static void vmx_load_host_state(struct vcpu_vmx *vmx) reload_tss(); save_msrs(vmx->guest_msrs, vmx->save_nmsrs); load_msrs(vmx->host_msrs, vmx->save_nmsrs); - if (msr_efer_need_save_restore(vmx)) - load_msrs(vmx->host_msrs + vmx->msr_offset_efer, 1); + reload_host_efer(vmx); } /* @@ -727,8 +738,10 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) #ifdef CONFIG_X86_64 case MSR_EFER: ret = kvm_set_msr_common(vcpu, msr_index, data); - if (vmx->host_state.loaded) + if (vmx->host_state.loaded) { + reload_host_efer(vmx); load_transition_efer(vmx); + } break; case MSR_FS_BASE: vmcs_writel(GUEST_FS_BASE, data); From c7addb902054195b995114df154e061c7d604f69 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 16 Sep 2007 18:58:32 +0200 Subject: [PATCH 009/249] KVM: Allow not-present guest page faults to bypass kvm There are two classes of page faults trapped by kvm: - host page faults, where the fault is needed to allow kvm to install the shadow pte or update the guest accessed and dirty bits - guest page faults, where the guest has faulted and kvm simply injects the fault back into the guest to handle The second class, guest page faults, is pure overhead. We can eliminate some of it on vmx using the following evil trick: - when we set up a shadow page table entry, if the corresponding guest pte is not present, set up the shadow pte as not present - if the guest pte _is_ present, mark the shadow pte as present but also set one of the reserved bits in the shadow pte - tell the vmx hardware not to trap faults which have the present bit clear With this, normal page-not-present faults go directly to the guest, bypassing kvm entirely. Unfortunately, this trick only works on Intel hardware, as AMD lacks a way to discriminate among page faults based on error code. It is also a little risky since it uses reserved bits which might become unreserved in the future, so a module parameter is provided to disable it. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 ++ drivers/kvm/kvm_main.c | 4 +- drivers/kvm/mmu.c | 89 ++++++++++++++++++++++++++++++--------- drivers/kvm/paging_tmpl.h | 52 +++++++++++++++++------ drivers/kvm/vmx.c | 11 ++++- 5 files changed, 122 insertions(+), 37 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index e885b190b798..7de948e9e64e 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -150,6 +150,8 @@ struct kvm_mmu { int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err); void (*free)(struct kvm_vcpu *vcpu); gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva); + void (*prefetch_page)(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *page); hpa_t root_hpa; int root_level; int shadow_root_level; @@ -536,6 +538,7 @@ void kvm_mmu_module_exit(void); void kvm_mmu_destroy(struct kvm_vcpu *vcpu); int kvm_mmu_create(struct kvm_vcpu *vcpu); int kvm_mmu_setup(struct kvm_vcpu *vcpu); +void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte); int kvm_mmu_reset_context(struct kvm_vcpu *vcpu); void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 710483669f34..82cc7ae0fc83 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -3501,7 +3501,9 @@ int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, kvm_preempt_ops.sched_in = kvm_sched_in; kvm_preempt_ops.sched_out = kvm_sched_out; - return r; + kvm_mmu_set_nonpresent_ptes(0ull, 0ull); + + return 0; out_free: kmem_cache_destroy(kvm_vcpu_cache); diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index feb5ac986c5d..069ce83f018e 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -156,6 +156,16 @@ static struct kmem_cache *pte_chain_cache; static struct kmem_cache *rmap_desc_cache; static struct kmem_cache *mmu_page_header_cache; +static u64 __read_mostly shadow_trap_nonpresent_pte; +static u64 __read_mostly shadow_notrap_nonpresent_pte; + +void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte) +{ + shadow_trap_nonpresent_pte = trap_pte; + shadow_notrap_nonpresent_pte = notrap_pte; +} +EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes); + static int is_write_protection(struct kvm_vcpu *vcpu) { return vcpu->cr0 & X86_CR0_WP; @@ -176,6 +186,13 @@ static int is_present_pte(unsigned long pte) return pte & PT_PRESENT_MASK; } +static int is_shadow_present_pte(u64 pte) +{ + pte &= ~PT_SHADOW_IO_MARK; + return pte != shadow_trap_nonpresent_pte + && pte != shadow_notrap_nonpresent_pte; +} + static int is_writeble_pte(unsigned long pte) { return pte & PT_WRITABLE_MASK; @@ -450,7 +467,7 @@ static int is_empty_shadow_page(u64 *spt) u64 *end; for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++) - if (*pos != 0) { + if ((*pos & ~PT_SHADOW_IO_MARK) != shadow_trap_nonpresent_pte) { printk(KERN_ERR "%s: %p %llx\n", __FUNCTION__, pos, *pos); return 0; @@ -632,6 +649,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, page->gfn = gfn; page->role = role; hlist_add_head(&page->hash_link, bucket); + vcpu->mmu.prefetch_page(vcpu, page); if (!metaphysical) rmap_write_protect(vcpu, gfn); return page; @@ -648,9 +666,9 @@ static void kvm_mmu_page_unlink_children(struct kvm *kvm, if (page->role.level == PT_PAGE_TABLE_LEVEL) { for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { - if (pt[i] & PT_PRESENT_MASK) + if (is_shadow_present_pte(pt[i])) rmap_remove(&pt[i]); - pt[i] = 0; + pt[i] = shadow_trap_nonpresent_pte; } kvm_flush_remote_tlbs(kvm); return; @@ -659,8 +677,8 @@ static void kvm_mmu_page_unlink_children(struct kvm *kvm, for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { ent = pt[i]; - pt[i] = 0; - if (!(ent & PT_PRESENT_MASK)) + pt[i] = shadow_trap_nonpresent_pte; + if (!is_shadow_present_pte(ent)) continue; ent &= PT64_BASE_ADDR_MASK; mmu_page_remove_parent_pte(page_header(ent), &pt[i]); @@ -691,7 +709,7 @@ static void kvm_mmu_zap_page(struct kvm *kvm, } BUG_ON(!parent_pte); kvm_mmu_put_page(page, parent_pte); - set_shadow_pte(parent_pte, 0); + set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte); } kvm_mmu_page_unlink_children(kvm, page); if (!page->root_count) { @@ -798,7 +816,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) if (level == 1) { pte = table[index]; - if (is_present_pte(pte) && is_writeble_pte(pte)) + if (is_shadow_present_pte(pte) && is_writeble_pte(pte)) return 0; mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT); page_header_update_slot(vcpu->kvm, table, v); @@ -808,7 +826,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) return 0; } - if (table[index] == 0) { + if (table[index] == shadow_trap_nonpresent_pte) { struct kvm_mmu_page *new_table; gfn_t pseudo_gfn; @@ -829,6 +847,15 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) } } +static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *sp) +{ + int i; + + for (i = 0; i < PT64_ENT_PER_PAGE; ++i) + sp->spt[i] = shadow_trap_nonpresent_pte; +} + static void mmu_free_roots(struct kvm_vcpu *vcpu) { int i; @@ -943,6 +970,7 @@ static int nonpaging_init_context(struct kvm_vcpu *vcpu) context->page_fault = nonpaging_page_fault; context->gva_to_gpa = nonpaging_gva_to_gpa; context->free = nonpaging_free; + context->prefetch_page = nonpaging_prefetch_page; context->root_level = 0; context->shadow_root_level = PT32E_ROOT_LEVEL; context->root_hpa = INVALID_PAGE; @@ -989,6 +1017,7 @@ static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level) context->new_cr3 = paging_new_cr3; context->page_fault = paging64_page_fault; context->gva_to_gpa = paging64_gva_to_gpa; + context->prefetch_page = paging64_prefetch_page; context->free = paging_free; context->root_level = level; context->shadow_root_level = level; @@ -1009,6 +1038,7 @@ static int paging32_init_context(struct kvm_vcpu *vcpu) context->page_fault = paging32_page_fault; context->gva_to_gpa = paging32_gva_to_gpa; context->free = paging_free; + context->prefetch_page = paging32_prefetch_page; context->root_level = PT32_ROOT_LEVEL; context->shadow_root_level = PT32E_ROOT_LEVEL; context->root_hpa = INVALID_PAGE; @@ -1081,7 +1111,7 @@ static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *child; pte = *spte; - if (is_present_pte(pte)) { + if (is_shadow_present_pte(pte)) { if (page->role.level == PT_PAGE_TABLE_LEVEL) rmap_remove(spte); else { @@ -1089,22 +1119,25 @@ static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu, mmu_page_remove_parent_pte(child, spte); } } - set_shadow_pte(spte, 0); + set_shadow_pte(spte, shadow_trap_nonpresent_pte); kvm_flush_remote_tlbs(vcpu->kvm); } static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, u64 *spte, - const void *new, int bytes) + const void *new, int bytes, + int offset_in_pte) { if (page->role.level != PT_PAGE_TABLE_LEVEL) return; if (page->role.glevels == PT32_ROOT_LEVEL) - paging32_update_pte(vcpu, page, spte, new, bytes); + paging32_update_pte(vcpu, page, spte, new, bytes, + offset_in_pte); else - paging64_update_pte(vcpu, page, spte, new, bytes); + paging64_update_pte(vcpu, page, spte, new, bytes, + offset_in_pte); } void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, @@ -1126,6 +1159,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, int npte; pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes); + kvm_mmu_audit(vcpu, "pre pte write"); if (gfn == vcpu->last_pt_write_gfn) { ++vcpu->last_pt_write_count; if (vcpu->last_pt_write_count >= 3) @@ -1181,10 +1215,12 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, spte = &page->spt[page_offset / sizeof(*spte)]; while (npte--) { mmu_pte_write_zap_pte(vcpu, page, spte); - mmu_pte_write_new_pte(vcpu, page, spte, new, bytes); + mmu_pte_write_new_pte(vcpu, page, spte, new, bytes, + page_offset & (pte_size - 1)); ++spte; } } + kvm_mmu_audit(vcpu, "post pte write"); } int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) @@ -1359,22 +1395,33 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) { u64 ent = pt[i]; - if (!(ent & PT_PRESENT_MASK)) + if (ent == shadow_trap_nonpresent_pte) continue; va = canonicalize(va); - if (level > 1) + if (level > 1) { + if (ent == shadow_notrap_nonpresent_pte) + printk(KERN_ERR "audit: (%s) nontrapping pte" + " in nonleaf level: levels %d gva %lx" + " level %d pte %llx\n", audit_msg, + vcpu->mmu.root_level, va, level, ent); + audit_mappings_page(vcpu, ent, va, level - 1); - else { + } else { gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va); hpa_t hpa = gpa_to_hpa(vcpu, gpa); - if ((ent & PT_PRESENT_MASK) + if (is_shadow_present_pte(ent) && (ent & PT64_BASE_ADDR_MASK) != hpa) - printk(KERN_ERR "audit error: (%s) levels %d" - " gva %lx gpa %llx hpa %llx ent %llx\n", + printk(KERN_ERR "xx audit error: (%s) levels %d" + " gva %lx gpa %llx hpa %llx ent %llx %d\n", audit_msg, vcpu->mmu.root_level, - va, gpa, hpa, ent); + va, gpa, hpa, ent, is_shadow_present_pte(ent)); + else if (ent == shadow_notrap_nonpresent_pte + && !is_error_hpa(hpa)) + printk(KERN_ERR "audit: (%s) notrap shadow," + " valid guest gva %lx\n", audit_msg, va); + } } } diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 6b094b44f8fb..99ac9b15f773 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -31,6 +31,7 @@ #define PT_INDEX(addr, level) PT64_INDEX(addr, level) #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level) #define PT_LEVEL_MASK(level) PT64_LEVEL_MASK(level) + #define PT_LEVEL_BITS PT64_LEVEL_BITS #ifdef CONFIG_X86_64 #define PT_MAX_FULL_LEVELS 4 #else @@ -45,6 +46,7 @@ #define PT_INDEX(addr, level) PT32_INDEX(addr, level) #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level) #define PT_LEVEL_MASK(level) PT32_LEVEL_MASK(level) + #define PT_LEVEL_BITS PT32_LEVEL_BITS #define PT_MAX_FULL_LEVELS 2 #else #error Invalid PTTYPE value @@ -211,12 +213,12 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, { hpa_t paddr; int dirty = gpte & PT_DIRTY_MASK; - u64 spte = *shadow_pte; - int was_rmapped = is_rmap_pte(spte); + u64 spte; + int was_rmapped = is_rmap_pte(*shadow_pte); pgprintk("%s: spte %llx gpte %llx access %llx write_fault %d" " user_fault %d gfn %lx\n", - __FUNCTION__, spte, (u64)gpte, access_bits, + __FUNCTION__, *shadow_pte, (u64)gpte, access_bits, write_fault, user_fault, gfn); if (write_fault && !dirty) { @@ -236,7 +238,7 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, FNAME(mark_pagetable_dirty)(vcpu->kvm, walker); } - spte |= PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_DIRTY_MASK; + spte = PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_DIRTY_MASK; spte |= gpte & PT64_NX_MASK; if (!dirty) access_bits &= ~PT_WRITABLE_MASK; @@ -248,10 +250,8 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, spte |= PT_USER_MASK; if (is_error_hpa(paddr)) { - spte |= gaddr; - spte |= PT_SHADOW_IO_MARK; - spte &= ~PT_PRESENT_MASK; - set_shadow_pte(shadow_pte, spte); + set_shadow_pte(shadow_pte, + shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK); return; } @@ -286,6 +286,7 @@ unshadowed: if (access_bits & PT_WRITABLE_MASK) mark_page_dirty(vcpu->kvm, gaddr >> PAGE_SHIFT); + pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte); set_shadow_pte(shadow_pte, spte); page_header_update_slot(vcpu->kvm, shadow_pte, gaddr); if (!was_rmapped) @@ -304,14 +305,18 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, } static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, - u64 *spte, const void *pte, int bytes) + u64 *spte, const void *pte, int bytes, + int offset_in_pte) { pt_element_t gpte; - if (bytes < sizeof(pt_element_t)) - return; gpte = *(const pt_element_t *)pte; - if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) + if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) { + if (!offset_in_pte && !is_present_pte(gpte)) + set_shadow_pte(spte, shadow_notrap_nonpresent_pte); + return; + } + if (bytes < sizeof(pt_element_t)) return; pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); FNAME(set_pte)(vcpu, gpte, spte, PT_USER_MASK | PT_WRITABLE_MASK, 0, @@ -368,7 +373,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, unsigned hugepage_access = 0; shadow_ent = ((u64 *)__va(shadow_addr)) + index; - if (is_present_pte(*shadow_ent) || is_io_pte(*shadow_ent)) { + if (is_shadow_present_pte(*shadow_ent)) { if (level == PT_PAGE_TABLE_LEVEL) break; shadow_addr = *shadow_ent & PT64_BASE_ADDR_MASK; @@ -500,6 +505,26 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr) return gpa; } +static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *sp) +{ + int i; + pt_element_t *gpt; + + if (sp->role.metaphysical || PTTYPE == 32) { + nonpaging_prefetch_page(vcpu, sp); + return; + } + + gpt = kmap_atomic(gfn_to_page(vcpu->kvm, sp->gfn), KM_USER0); + for (i = 0; i < PT64_ENT_PER_PAGE; ++i) + if (is_present_pte(gpt[i])) + sp->spt[i] = shadow_trap_nonpresent_pte; + else + sp->spt[i] = shadow_notrap_nonpresent_pte; + kunmap_atomic(gpt, KM_USER0); +} + #undef pt_element_t #undef guest_walker #undef FNAME @@ -508,4 +533,5 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr) #undef SHADOW_PT_INDEX #undef PT_LEVEL_MASK #undef PT_DIR_BASE_ADDR_MASK +#undef PT_LEVEL_BITS #undef PT_MAX_FULL_LEVELS diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 8eb49e055ec0..27a3318fa6c2 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -33,6 +34,9 @@ MODULE_AUTHOR("Qumranet"); MODULE_LICENSE("GPL"); +static int bypass_guest_pf = 1; +module_param(bypass_guest_pf, bool, 0); + struct vmcs { u32 revision_id; u32 abort; @@ -1535,8 +1539,8 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) } vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control); - vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0); - vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0); + vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, !!bypass_guest_pf); + vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, !!bypass_guest_pf); vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */ vmcs_writel(HOST_CR0, read_cr0()); /* 22.2.3 */ @@ -2582,6 +2586,9 @@ static int __init vmx_init(void) if (r) goto out1; + if (bypass_guest_pf) + kvm_mmu_set_nonpresent_ptes(~0xffeull, 0ull); + return 0; out1: From 12b7d28fc102b772eb70f98491587ec5ee717baf Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 23 Sep 2007 14:10:49 +0200 Subject: [PATCH 010/249] KVM: MMU: Make flooding detection work when guest page faults are bypassed When we allow guest page faults to reach the guests directly, we lose the fault tracking which allows us to detect demand paging. So we provide an alternate mechnism by clearing the accessed bit when we set a pte, and checking it later to see if the guest actually used it. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/mmu.c | 21 ++++++++++++++++++++- drivers/kvm/paging_tmpl.h | 9 ++++++++- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 7de948e9e64e..08ffc829f07f 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -346,6 +346,7 @@ struct kvm_vcpu { gfn_t last_pt_write_gfn; int last_pt_write_count; + u64 *last_pte_updated; struct kvm_guest_debug guest_debug; diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 069ce83f018e..d347e895736e 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -692,6 +692,15 @@ static void kvm_mmu_put_page(struct kvm_mmu_page *page, mmu_page_remove_parent_pte(page, parent_pte); } +static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm) +{ + int i; + + for (i = 0; i < KVM_MAX_VCPUS; ++i) + if (kvm->vcpus[i]) + kvm->vcpus[i]->last_pte_updated = NULL; +} + static void kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *page) { @@ -717,6 +726,7 @@ static void kvm_mmu_zap_page(struct kvm *kvm, kvm_mmu_free_page(kvm, page); } else list_move(&page->link, &kvm->active_mmu_pages); + kvm_mmu_reset_last_pte_updated(kvm); } static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn) @@ -1140,6 +1150,13 @@ static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, offset_in_pte); } +static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu) +{ + u64 *spte = vcpu->last_pte_updated; + + return !!(spte && (*spte & PT_ACCESSED_MASK)); +} + void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, int bytes) { @@ -1160,13 +1177,15 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes); kvm_mmu_audit(vcpu, "pre pte write"); - if (gfn == vcpu->last_pt_write_gfn) { + if (gfn == vcpu->last_pt_write_gfn + && !last_updated_pte_accessed(vcpu)) { ++vcpu->last_pt_write_count; if (vcpu->last_pt_write_count >= 3) flooded = 1; } else { vcpu->last_pt_write_gfn = gfn; vcpu->last_pt_write_count = 1; + vcpu->last_pte_updated = NULL; } index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; bucket = &vcpu->kvm->mmu_page_hash[index]; diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 99ac9b15f773..be0f85231da9 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -238,7 +238,12 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, FNAME(mark_pagetable_dirty)(vcpu->kvm, walker); } - spte = PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_DIRTY_MASK; + /* + * We don't set the accessed bit, since we sometimes want to see + * whether the guest actually used the pte (in order to detect + * demand paging). + */ + spte = PT_PRESENT_MASK | PT_DIRTY_MASK; spte |= gpte & PT64_NX_MASK; if (!dirty) access_bits &= ~PT_WRITABLE_MASK; @@ -291,6 +296,8 @@ unshadowed: page_header_update_slot(vcpu->kvm, shadow_pte, gaddr); if (!was_rmapped) rmap_add(vcpu, shadow_pte); + if (!ptwrite || !*ptwrite) + vcpu->last_pte_updated = shadow_pte; } static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, From 217648638ccb62dfeea5ac5fe768539cdee61ed0 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Tue, 18 Sep 2007 14:05:16 -0500 Subject: [PATCH 011/249] KVM: MMU: Ignore reserved bits in cr3 in non-pae mode This patch removes the fault injected when the guest attempts to set reserved bits in cr3. X86 hardware doesn't generate a fault when setting reserved bits. The result of this patch is that vmware-server, running within a kvm guest, boots and runs memtest from an iso. Signed-off-by: Ryan Harper Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 82cc7ae0fc83..2d55bab41634 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -554,14 +554,11 @@ void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) inject_gp(vcpu); return; } - } else { - if (cr3 & CR3_NONPAE_RESERVED_BITS) { - printk(KERN_DEBUG - "set_cr3: #GP, reserved bits\n"); - inject_gp(vcpu); - return; - } } + /* + * We don't check reserved bits in nonpae mode, because + * this isn't enforced, and VMware depends on this. + */ } mutex_lock(&vcpu->kvm->lock); From 8cdbd2c9bf3108c41563eef586f22fa8b5174de0 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Mon, 24 Sep 2007 11:10:54 +0200 Subject: [PATCH 012/249] KVM: x86 emulator: split some decoding into functions for readability To improve readability, move push, writeback, and grp 1a/2/3/4/5/9 emulation parts into functions. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 451 ++++++++++++++++++++++---------------- 1 file changed, 266 insertions(+), 185 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index dc9d2a870fbc..a15609279283 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -907,6 +907,244 @@ done: return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0; } +static inline void emulate_push(struct x86_emulate_ctxt *ctxt) +{ + struct decode_cache *c = &ctxt->decode; + + c->dst.type = OP_MEM; + c->dst.bytes = c->op_bytes; + c->dst.val = c->src.val; + register_address_increment(c->regs[VCPU_REGS_RSP], -c->op_bytes); + c->dst.ptr = (void *) register_address(ctxt->ss_base, + c->regs[VCPU_REGS_RSP]); +} + +static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops) +{ + struct decode_cache *c = &ctxt->decode; + int rc; + + /* 64-bit mode: POP always pops a 64-bit operand. */ + + if (ctxt->mode == X86EMUL_MODE_PROT64) + c->dst.bytes = 8; + + rc = ops->read_std(register_address(ctxt->ss_base, + c->regs[VCPU_REGS_RSP]), + &c->dst.val, c->dst.bytes, ctxt->vcpu); + if (rc != 0) + return rc; + + register_address_increment(c->regs[VCPU_REGS_RSP], c->dst.bytes); + + return 0; +} + +static inline void emulate_grp2(struct decode_cache *c, unsigned long *_eflags) +{ + switch (c->modrm_reg) { + case 0: /* rol */ + emulate_2op_SrcB("rol", c->src, c->dst, *_eflags); + break; + case 1: /* ror */ + emulate_2op_SrcB("ror", c->src, c->dst, *_eflags); + break; + case 2: /* rcl */ + emulate_2op_SrcB("rcl", c->src, c->dst, *_eflags); + break; + case 3: /* rcr */ + emulate_2op_SrcB("rcr", c->src, c->dst, *_eflags); + break; + case 4: /* sal/shl */ + case 6: /* sal/shl */ + emulate_2op_SrcB("sal", c->src, c->dst, *_eflags); + break; + case 5: /* shr */ + emulate_2op_SrcB("shr", c->src, c->dst, *_eflags); + break; + case 7: /* sar */ + emulate_2op_SrcB("sar", c->src, c->dst, *_eflags); + break; + } +} + +static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops, + unsigned long *_eflags) +{ + struct decode_cache *c = &ctxt->decode; + int rc = 0; + + switch (c->modrm_reg) { + case 0 ... 1: /* test */ + /* + * Special case in Grp3: test has an immediate + * source operand. + */ + c->src.type = OP_IMM; + c->src.ptr = (unsigned long *)c->eip; + c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + if (c->src.bytes == 8) + c->src.bytes = 4; + switch (c->src.bytes) { + case 1: + c->src.val = insn_fetch(s8, 1, c->eip); + break; + case 2: + c->src.val = insn_fetch(s16, 2, c->eip); + break; + case 4: + c->src.val = insn_fetch(s32, 4, c->eip); + break; + } + emulate_2op_SrcV("test", c->src, c->dst, *_eflags); + break; + case 2: /* not */ + c->dst.val = ~c->dst.val; + break; + case 3: /* neg */ + emulate_1op("neg", c->dst, *_eflags); + break; + default: + DPRINTF("Cannot emulate %02x\n", c->b); + rc = X86EMUL_UNHANDLEABLE; + break; + } +done: + return rc; +} + +static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops, + unsigned long *_eflags, + int *no_wb) +{ + struct decode_cache *c = &ctxt->decode; + int rc; + + switch (c->modrm_reg) { + case 0: /* inc */ + emulate_1op("inc", c->dst, *_eflags); + break; + case 1: /* dec */ + emulate_1op("dec", c->dst, *_eflags); + break; + case 4: /* jmp abs */ + if (c->b == 0xff) + c->eip = c->dst.val; + else { + DPRINTF("Cannot emulate %02x\n", c->b); + return X86EMUL_UNHANDLEABLE; + } + break; + case 6: /* push */ + + /* 64-bit mode: PUSH always pushes a 64-bit operand. */ + + if (ctxt->mode == X86EMUL_MODE_PROT64) { + c->dst.bytes = 8; + rc = ops->read_std((unsigned long)c->dst.ptr, + &c->dst.val, 8, ctxt->vcpu); + if (rc != 0) + return rc; + } + register_address_increment(c->regs[VCPU_REGS_RSP], + -c->dst.bytes); + rc = ops->write_emulated(register_address(ctxt->ss_base, + c->regs[VCPU_REGS_RSP]), &c->dst.val, + c->dst.bytes, ctxt->vcpu); + if (rc != 0) + return rc; + *no_wb = 1; + break; + default: + DPRINTF("Cannot emulate %02x\n", c->b); + return X86EMUL_UNHANDLEABLE; + } + return 0; +} + +static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops, + unsigned long *_eflags, + unsigned long cr2) +{ + struct decode_cache *c = &ctxt->decode; + u64 old, new; + int rc; + + rc = ops->read_emulated(cr2, &old, 8, ctxt->vcpu); + if (rc != 0) + return rc; + + if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) || + ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) { + + c->regs[VCPU_REGS_RAX] = (u32) (old >> 0); + c->regs[VCPU_REGS_RDX] = (u32) (old >> 32); + *_eflags &= ~EFLG_ZF; + + } else { + new = ((u64)c->regs[VCPU_REGS_RCX] << 32) | + (u32) c->regs[VCPU_REGS_RBX]; + + rc = ops->cmpxchg_emulated(cr2, &old, &new, 8, ctxt->vcpu); + if (rc != 0) + return rc; + *_eflags |= EFLG_ZF; + } + return 0; +} + +static inline int writeback(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops) +{ + int rc; + struct decode_cache *c = &ctxt->decode; + + switch (c->dst.type) { + case OP_REG: + /* The 4-byte case *is* correct: + * in 64-bit mode we zero-extend. + */ + switch (c->dst.bytes) { + case 1: + *(u8 *)c->dst.ptr = (u8)c->dst.val; + break; + case 2: + *(u16 *)c->dst.ptr = (u16)c->dst.val; + break; + case 4: + *c->dst.ptr = (u32)c->dst.val; + break; /* 64b: zero-ext */ + case 8: + *c->dst.ptr = c->dst.val; + break; + } + break; + case OP_MEM: + if (c->lock_prefix) + rc = ops->cmpxchg_emulated( + (unsigned long)c->dst.ptr, + &c->dst.orig_val, + &c->dst.val, + c->dst.bytes, + ctxt->vcpu); + else + rc = ops->write_emulated( + (unsigned long)c->dst.ptr, + &c->dst.val, + c->dst.bytes, + ctxt->vcpu); + if (rc != 0) + return rc; + default: + break; + } + return 0; +} + int x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { @@ -1042,7 +1280,6 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) } break; case 0x84 ... 0x85: - test: /* test */ emulate_2op_SrcV("test", c->src, c->dst, _eflags); break; case 0x86 ... 0x87: /* xchg */ @@ -1074,18 +1311,9 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) c->dst.val = c->modrm_val; break; case 0x8f: /* pop (sole member of Grp1a) */ - /* 64-bit mode: POP always pops a 64-bit operand. */ - if (ctxt->mode == X86EMUL_MODE_PROT64) - c->dst.bytes = 8; - if ((rc = ops->read_std(register_address( - ctxt->ss_base, - c->regs[VCPU_REGS_RSP]), - &c->dst.val, - c->dst.bytes, - ctxt->vcpu)) != 0) + rc = emulate_grp1a(ctxt, ops); + if (rc != 0) goto done; - register_address_increment(c->regs[VCPU_REGS_RSP], - c->dst.bytes); break; case 0xa0 ... 0xa1: /* mov */ c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; @@ -1099,31 +1327,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) c->eip += c->ad_bytes; break; case 0xc0 ... 0xc1: - grp2: /* Grp2 */ - switch (c->modrm_reg) { - case 0: /* rol */ - emulate_2op_SrcB("rol", c->src, c->dst, _eflags); - break; - case 1: /* ror */ - emulate_2op_SrcB("ror", c->src, c->dst, _eflags); - break; - case 2: /* rcl */ - emulate_2op_SrcB("rcl", c->src, c->dst, _eflags); - break; - case 3: /* rcr */ - emulate_2op_SrcB("rcr", c->src, c->dst, _eflags); - break; - case 4: /* sal/shl */ - case 6: /* sal/shl */ - emulate_2op_SrcB("sal", c->src, c->dst, _eflags); - break; - case 5: /* shr */ - emulate_2op_SrcB("shr", c->src, c->dst, _eflags); - break; - case 7: /* sar */ - emulate_2op_SrcB("sar", c->src, c->dst, _eflags); - break; - } + emulate_grp2(c, &_eflags); break; case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */ mov: @@ -1131,126 +1335,29 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) break; case 0xd0 ... 0xd1: /* Grp2 */ c->src.val = 1; - goto grp2; + emulate_grp2(c, &_eflags); + break; case 0xd2 ... 0xd3: /* Grp2 */ c->src.val = c->regs[VCPU_REGS_RCX]; - goto grp2; + emulate_grp2(c, &_eflags); + break; case 0xf6 ... 0xf7: /* Grp3 */ - switch (c->modrm_reg) { - case 0 ... 1: /* test */ - /* - * Special case in Grp3: test has an immediate - * source operand. - */ - c->src.type = OP_IMM; - c->src.ptr = (unsigned long *)c->eip; - c->src.bytes = (c->d & ByteOp) ? 1 : - c->op_bytes; - if (c->src.bytes == 8) - c->src.bytes = 4; - switch (c->src.bytes) { - case 1: - c->src.val = insn_fetch(s8, 1, c->eip); - break; - case 2: - c->src.val = insn_fetch(s16, 2, c->eip); - break; - case 4: - c->src.val = insn_fetch(s32, 4, c->eip); - break; - } - goto test; - case 2: /* not */ - c->dst.val = ~c->dst.val; - break; - case 3: /* neg */ - emulate_1op("neg", c->dst, _eflags); - break; - default: - goto cannot_emulate; - } + rc = emulate_grp3(ctxt, ops, &_eflags); + if (rc != 0) + goto done; break; case 0xfe ... 0xff: /* Grp4/Grp5 */ - switch (c->modrm_reg) { - case 0: /* inc */ - emulate_1op("inc", c->dst, _eflags); - break; - case 1: /* dec */ - emulate_1op("dec", c->dst, _eflags); - break; - case 4: /* jmp abs */ - if (c->b == 0xff) - c->eip = c->dst.val; - else - goto cannot_emulate; - break; - case 6: /* push */ - /* 64-bit mode: PUSH always pushes a 64-bit operand. */ - if (ctxt->mode == X86EMUL_MODE_PROT64) { - c->dst.bytes = 8; - if ((rc = ops->read_std( - (unsigned long)c->dst.ptr, - &c->dst.val, 8, - ctxt->vcpu)) != 0) - goto done; - } - register_address_increment(c->regs[VCPU_REGS_RSP], - -c->dst.bytes); - if ((rc = ops->write_emulated( - register_address(ctxt->ss_base, - c->regs[VCPU_REGS_RSP]), - &c->dst.val, - c->dst.bytes, ctxt->vcpu)) != 0) - goto done; - no_wb = 1; - break; - default: - goto cannot_emulate; - } + rc = emulate_grp45(ctxt, ops, &_eflags, &no_wb); + if (rc != 0) + goto done; break; } writeback: if (!no_wb) { - switch (c->dst.type) { - case OP_REG: - /* The 4-byte case *is* correct: - * in 64-bit mode we zero-extend. - */ - switch (c->dst.bytes) { - case 1: - *(u8 *)c->dst.ptr = (u8)c->dst.val; - break; - case 2: - *(u16 *)c->dst.ptr = (u16)c->dst.val; - break; - case 4: - *c->dst.ptr = (u32)c->dst.val; - break; /* 64b: zero-ext */ - case 8: - *c->dst.ptr = c->dst.val; - break; - } - break; - case OP_MEM: - if (c->lock_prefix) - rc = ops->cmpxchg_emulated( - (unsigned long)c->dst.ptr, - &c->dst.orig_val, - &c->dst.val, - c->dst.bytes, - ctxt->vcpu); - else - rc = ops->write_emulated( - (unsigned long)c->dst.ptr, - &c->dst.val, - c->dst.bytes, - ctxt->vcpu); - if (rc != 0) - goto done; - default: - break; - } + rc = writeback(ctxt, ops); + if (rc != 0) + goto done; } /* Commit shadow register state. */ @@ -1283,8 +1390,7 @@ special_insn: ctxt->ss_base, c->regs[VCPU_REGS_RSP]); break; case 0x58 ... 0x5f: /* pop reg */ - c->dst.ptr = - (unsigned long *)&c->regs[c->b & 0x7]; + c->dst.ptr = (unsigned long *)&c->regs[c->b & 0x7]; pop_instruction: if ((rc = ops->read_std(register_address(ctxt->ss_base, c->regs[VCPU_REGS_RSP]), c->dst.ptr, @@ -1298,14 +1404,7 @@ special_insn: case 0x6a: /* push imm8 */ c->src.val = 0L; c->src.val = insn_fetch(s8, 1, c->eip); -push: - c->dst.type = OP_MEM; - c->dst.bytes = c->op_bytes; - c->dst.val = c->src.val; - register_address_increment(c->regs[VCPU_REGS_RSP], - -c->op_bytes); - c->dst.ptr = (void *) register_address(ctxt->ss_base, - c->regs[VCPU_REGS_RSP]); + emulate_push(ctxt); break; case 0x6c: /* insb */ case 0x6d: /* insw/insd */ @@ -1350,7 +1449,8 @@ push: } case 0x9c: /* pushf */ c->src.val = (unsigned long) _eflags; - goto push; + emulate_push(ctxt); + break; case 0x9d: /* popf */ c->dst.ptr = (unsigned long *) &_eflags; goto pop_instruction; @@ -1436,7 +1536,8 @@ push: c->src.val = (unsigned long) c->eip; JMP_REL(rel); c->op_bytes = c->ad_bytes; - goto push; + emulate_push(ctxt); + break; } case 0xe9: /* jmp rel */ case 0xeb: /* jmp rel short */ @@ -1511,8 +1612,7 @@ twobyte_insn: no_wb = 1; if (c->modrm_mod != 3) goto cannot_emulate; - rc = emulator_get_dr(ctxt, c->modrm_reg, - &c->regs[c->modrm_rm]); + rc = emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]); break; case 0x23: /* mov from reg to dr */ no_wb = 1; @@ -1668,8 +1768,7 @@ twobyte_special_insn: break; case 0x32: /* rdmsr */ - rc = kvm_get_msr(ctxt->vcpu, - c->regs[VCPU_REGS_RCX], &msr_data); + rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data); if (rc) { kvm_x86_ops->inject_gp(ctxt->vcpu, 0); c->eip = ctxt->vcpu->rip; @@ -1701,28 +1800,10 @@ twobyte_special_insn: break; } case 0xc7: /* Grp9 (cmpxchg8b) */ - { - u64 old, new; - if ((rc = ops->read_emulated(cr2, &old, 8, ctxt->vcpu)) - != 0) - goto done; - if (((u32) (old >> 0) != - (u32) c->regs[VCPU_REGS_RAX]) || - ((u32) (old >> 32) != - (u32) c->regs[VCPU_REGS_RDX])) { - c->regs[VCPU_REGS_RAX] = (u32) (old >> 0); - c->regs[VCPU_REGS_RDX] = (u32) (old >> 32); - _eflags &= ~EFLG_ZF; - } else { - new = ((u64)c->regs[VCPU_REGS_RCX] << 32) - | (u32) c->regs[VCPU_REGS_RBX]; - if ((rc = ops->cmpxchg_emulated(cr2, &old, - &new, 8, ctxt->vcpu)) != 0) - goto done; - _eflags |= EFLG_ZF; - } - break; - } + rc = emulate_grp9(ctxt, ops, &_eflags, cr2); + if (rc != 0) + goto done; + break; } goto writeback; From 05f086f87ef17da70d5f4e9b063e9ea31ab20f28 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Mon, 24 Sep 2007 11:10:55 +0200 Subject: [PATCH 013/249] KVM: x86 emulator: remove _eflags and use directly ctxt->eflags. Remove _eflags and use directly ctxt->eflags. Caching eflags is not needed as it is restored to vcpu by kvm_main.c:emulate_instruction() from ctxt->eflags only if emulation doesn't fail. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 121 +++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 62 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index a15609279283..76b813b6365c 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -941,37 +941,37 @@ static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt, return 0; } -static inline void emulate_grp2(struct decode_cache *c, unsigned long *_eflags) +static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt) { + struct decode_cache *c = &ctxt->decode; switch (c->modrm_reg) { case 0: /* rol */ - emulate_2op_SrcB("rol", c->src, c->dst, *_eflags); + emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags); break; case 1: /* ror */ - emulate_2op_SrcB("ror", c->src, c->dst, *_eflags); + emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags); break; case 2: /* rcl */ - emulate_2op_SrcB("rcl", c->src, c->dst, *_eflags); + emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags); break; case 3: /* rcr */ - emulate_2op_SrcB("rcr", c->src, c->dst, *_eflags); + emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags); break; case 4: /* sal/shl */ case 6: /* sal/shl */ - emulate_2op_SrcB("sal", c->src, c->dst, *_eflags); + emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags); break; case 5: /* shr */ - emulate_2op_SrcB("shr", c->src, c->dst, *_eflags); + emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags); break; case 7: /* sar */ - emulate_2op_SrcB("sar", c->src, c->dst, *_eflags); + emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags); break; } } static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops, - unsigned long *_eflags) + struct x86_emulate_ops *ops) { struct decode_cache *c = &ctxt->decode; int rc = 0; @@ -998,13 +998,13 @@ static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt, c->src.val = insn_fetch(s32, 4, c->eip); break; } - emulate_2op_SrcV("test", c->src, c->dst, *_eflags); + emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags); break; case 2: /* not */ c->dst.val = ~c->dst.val; break; case 3: /* neg */ - emulate_1op("neg", c->dst, *_eflags); + emulate_1op("neg", c->dst, ctxt->eflags); break; default: DPRINTF("Cannot emulate %02x\n", c->b); @@ -1017,7 +1017,6 @@ done: static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops, - unsigned long *_eflags, int *no_wb) { struct decode_cache *c = &ctxt->decode; @@ -1025,10 +1024,10 @@ static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt, switch (c->modrm_reg) { case 0: /* inc */ - emulate_1op("inc", c->dst, *_eflags); + emulate_1op("inc", c->dst, ctxt->eflags); break; case 1: /* dec */ - emulate_1op("dec", c->dst, *_eflags); + emulate_1op("dec", c->dst, ctxt->eflags); break; case 4: /* jmp abs */ if (c->b == 0xff) @@ -1067,7 +1066,6 @@ static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt, static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops, - unsigned long *_eflags, unsigned long cr2) { struct decode_cache *c = &ctxt->decode; @@ -1083,7 +1081,7 @@ static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt, c->regs[VCPU_REGS_RAX] = (u32) (old >> 0); c->regs[VCPU_REGS_RDX] = (u32) (old >> 32); - *_eflags &= ~EFLG_ZF; + ctxt->eflags &= ~EFLG_ZF; } else { new = ((u64)c->regs[VCPU_REGS_RCX] << 32) | @@ -1092,7 +1090,7 @@ static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt, rc = ops->cmpxchg_emulated(cr2, &old, &new, 8, ctxt->vcpu); if (rc != 0) return rc; - *_eflags |= EFLG_ZF; + ctxt->eflags |= EFLG_ZF; } return 0; } @@ -1152,7 +1150,6 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) int no_wb = 0; u64 msr_data; unsigned long saved_eip = 0; - unsigned long _eflags = ctxt->eflags; struct decode_cache *c = &ctxt->decode; int rc = 0; @@ -1207,23 +1204,23 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) switch (c->b) { case 0x00 ... 0x05: add: /* add */ - emulate_2op_SrcV("add", c->src, c->dst, _eflags); + emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags); break; case 0x08 ... 0x0d: or: /* or */ - emulate_2op_SrcV("or", c->src, c->dst, _eflags); + emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags); break; case 0x10 ... 0x15: adc: /* adc */ - emulate_2op_SrcV("adc", c->src, c->dst, _eflags); + emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags); break; case 0x18 ... 0x1d: sbb: /* sbb */ - emulate_2op_SrcV("sbb", c->src, c->dst, _eflags); + emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags); break; case 0x20 ... 0x23: and: /* and */ - emulate_2op_SrcV("and", c->src, c->dst, _eflags); + emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags); break; case 0x24: /* and al imm8 */ c->dst.type = OP_REG; @@ -1244,15 +1241,15 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) goto and; case 0x28 ... 0x2d: sub: /* sub */ - emulate_2op_SrcV("sub", c->src, c->dst, _eflags); + emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags); break; case 0x30 ... 0x35: xor: /* xor */ - emulate_2op_SrcV("xor", c->src, c->dst, _eflags); + emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags); break; case 0x38 ... 0x3d: cmp: /* cmp */ - emulate_2op_SrcV("cmp", c->src, c->dst, _eflags); + emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags); break; case 0x63: /* movsxd */ if (ctxt->mode != X86EMUL_MODE_PROT64) @@ -1280,7 +1277,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) } break; case 0x84 ... 0x85: - emulate_2op_SrcV("test", c->src, c->dst, _eflags); + emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags); break; case 0x86 ... 0x87: /* xchg */ /* Write back the register source. */ @@ -1327,7 +1324,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) c->eip += c->ad_bytes; break; case 0xc0 ... 0xc1: - emulate_grp2(c, &_eflags); + emulate_grp2(ctxt); break; case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */ mov: @@ -1335,19 +1332,19 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) break; case 0xd0 ... 0xd1: /* Grp2 */ c->src.val = 1; - emulate_grp2(c, &_eflags); + emulate_grp2(ctxt); break; case 0xd2 ... 0xd3: /* Grp2 */ c->src.val = c->regs[VCPU_REGS_RCX]; - emulate_grp2(c, &_eflags); + emulate_grp2(ctxt); break; case 0xf6 ... 0xf7: /* Grp3 */ - rc = emulate_grp3(ctxt, ops, &_eflags); + rc = emulate_grp3(ctxt, ops); if (rc != 0) goto done; break; case 0xfe ... 0xff: /* Grp4/Grp5 */ - rc = emulate_grp45(ctxt, ops, &_eflags, &no_wb); + rc = emulate_grp45(ctxt, ops, &no_wb); if (rc != 0) goto done; break; @@ -1362,7 +1359,6 @@ writeback: /* Commit shadow register state. */ memcpy(ctxt->vcpu->regs, c->regs, sizeof c->regs); - ctxt->eflags = _eflags; ctxt->vcpu->rip = c->eip; done: @@ -1413,7 +1409,7 @@ special_insn: (c->d & ByteOp) ? 1 : c->op_bytes, c->rep_prefix ? address_mask(c->regs[VCPU_REGS_RCX]) : 1, - (_eflags & EFLG_DF), + (ctxt->eflags & EFLG_DF), register_address(ctxt->es_base, c->regs[VCPU_REGS_RDI]), c->rep_prefix, @@ -1429,7 +1425,7 @@ special_insn: (c->d & ByteOp) ? 1 : c->op_bytes, c->rep_prefix ? address_mask(c->regs[VCPU_REGS_RCX]) : 1, - (_eflags & EFLG_DF), + (ctxt->eflags & EFLG_DF), register_address(c->override_base ? *c->override_base : ctxt->ds_base, @@ -1443,16 +1439,16 @@ special_insn: case 0x70 ... 0x7f: /* jcc (short) */ { int rel = insn_fetch(s8, 1, c->eip); - if (test_cc(c->b, _eflags)) + if (test_cc(c->b, ctxt->eflags)) JMP_REL(rel); break; } case 0x9c: /* pushf */ - c->src.val = (unsigned long) _eflags; + c->src.val = (unsigned long) ctxt->eflags; emulate_push(ctxt); break; case 0x9d: /* popf */ - c->dst.ptr = (unsigned long *) &_eflags; + c->dst.ptr = (unsigned long *) &ctxt->eflags; goto pop_instruction; case 0xc3: /* ret */ c->dst.ptr = &c->eip; @@ -1484,10 +1480,10 @@ special_insn: c->dst.bytes, ctxt->vcpu)) != 0) goto done; register_address_increment(c->regs[VCPU_REGS_RSI], - (_eflags & EFLG_DF) ? -c->dst.bytes + (ctxt->eflags & EFLG_DF) ? -c->dst.bytes : c->dst.bytes); register_address_increment(c->regs[VCPU_REGS_RDI], - (_eflags & EFLG_DF) ? -c->dst.bytes + (ctxt->eflags & EFLG_DF) ? -c->dst.bytes : c->dst.bytes); break; case 0xa6 ... 0xa7: /* cmps */ @@ -1499,7 +1495,7 @@ special_insn: c->dst.ptr = (unsigned long *)cr2; c->dst.val = c->regs[VCPU_REGS_RAX]; register_address_increment(c->regs[VCPU_REGS_RDI], - (_eflags & EFLG_DF) ? -c->dst.bytes + (ctxt->eflags & EFLG_DF) ? -c->dst.bytes : c->dst.bytes); break; case 0xac ... 0xad: /* lods */ @@ -1511,7 +1507,7 @@ special_insn: ctxt->vcpu)) != 0) goto done; register_address_increment(c->regs[VCPU_REGS_RSI], - (_eflags & EFLG_DF) ? -c->dst.bytes + (ctxt->eflags & EFLG_DF) ? -c->dst.bytes : c->dst.bytes); break; case 0xae ... 0xaf: /* scas */ @@ -1599,7 +1595,8 @@ twobyte_insn: case 6: /* lmsw */ if (c->modrm_mod != 3) goto cannot_emulate; - realmode_lmsw(ctxt->vcpu, (u16)c->modrm_val, &_eflags); + realmode_lmsw(ctxt->vcpu, (u16)c->modrm_val, + &ctxt->eflags); break; case 7: /* invlpg*/ emulate_invlpg(ctxt->vcpu, cr2); @@ -1630,29 +1627,29 @@ twobyte_insn: */ switch ((c->b & 15) >> 1) { case 0: /* cmovo */ - no_wb = (_eflags & EFLG_OF) ? 0 : 1; + no_wb = (ctxt->eflags & EFLG_OF) ? 0 : 1; break; case 1: /* cmovb/cmovc/cmovnae */ - no_wb = (_eflags & EFLG_CF) ? 0 : 1; + no_wb = (ctxt->eflags & EFLG_CF) ? 0 : 1; break; case 2: /* cmovz/cmove */ - no_wb = (_eflags & EFLG_ZF) ? 0 : 1; + no_wb = (ctxt->eflags & EFLG_ZF) ? 0 : 1; break; case 3: /* cmovbe/cmovna */ - no_wb = (_eflags & (EFLG_CF | EFLG_ZF)) ? 0 : 1; + no_wb = (ctxt->eflags & (EFLG_CF | EFLG_ZF)) ? 0 : 1; break; case 4: /* cmovs */ - no_wb = (_eflags & EFLG_SF) ? 0 : 1; + no_wb = (ctxt->eflags & EFLG_SF) ? 0 : 1; break; case 5: /* cmovp/cmovpe */ - no_wb = (_eflags & EFLG_PF) ? 0 : 1; + no_wb = (ctxt->eflags & EFLG_PF) ? 0 : 1; break; case 7: /* cmovle/cmovng */ - no_wb = (_eflags & EFLG_ZF) ? 0 : 1; + no_wb = (ctxt->eflags & EFLG_ZF) ? 0 : 1; /* fall through */ case 6: /* cmovl/cmovnge */ - no_wb &= (!(_eflags & EFLG_SF) != - !(_eflags & EFLG_OF)) ? 0 : 1; + no_wb &= (!(ctxt->eflags & EFLG_SF) != + !(ctxt->eflags & EFLG_OF)) ? 0 : 1; break; } /* Odd cmov opcodes (lsb == 1) have inverted sense. */ @@ -1662,13 +1659,13 @@ twobyte_insn: bt: /* bt */ /* only subword offset */ c->src.val &= (c->dst.bytes << 3) - 1; - emulate_2op_SrcV_nobyte("bt", c->src, c->dst, _eflags); + emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags); break; case 0xab: bts: /* bts */ /* only subword offset */ c->src.val &= (c->dst.bytes << 3) - 1; - emulate_2op_SrcV_nobyte("bts", c->src, c->dst, _eflags); + emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags); break; case 0xb0 ... 0xb1: /* cmpxchg */ /* @@ -1677,8 +1674,8 @@ twobyte_insn: */ c->src.orig_val = c->src.val; c->src.val = c->regs[VCPU_REGS_RAX]; - emulate_2op_SrcV("cmp", c->src, c->dst, _eflags); - if (_eflags & EFLG_ZF) { + emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags); + if (ctxt->eflags & EFLG_ZF) { /* Success: write back to memory. */ c->dst.val = c->src.orig_val; } else { @@ -1691,7 +1688,7 @@ twobyte_insn: btr: /* btr */ /* only subword offset */ c->src.val &= (c->dst.bytes << 3) - 1; - emulate_2op_SrcV_nobyte("btr", c->src, c->dst, _eflags); + emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags); break; case 0xb6 ... 0xb7: /* movzx */ c->dst.bytes = c->op_bytes; @@ -1714,7 +1711,7 @@ twobyte_insn: btc: /* btc */ /* only subword offset */ c->src.val &= (c->dst.bytes << 3) - 1; - emulate_2op_SrcV_nobyte("btc", c->src, c->dst, _eflags); + emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags); break; case 0xbe ... 0xbf: /* movsx */ c->dst.bytes = c->op_bytes; @@ -1753,7 +1750,7 @@ twobyte_special_insn: if (c->modrm_mod != 3) goto cannot_emulate; realmode_set_cr(ctxt->vcpu, - c->modrm_reg, c->modrm_val, &_eflags); + c->modrm_reg, c->modrm_val, &ctxt->eflags); break; case 0x30: /* wrmsr */ @@ -1795,12 +1792,12 @@ twobyte_special_insn: DPRINTF("jnz: Invalid op_bytes\n"); goto cannot_emulate; } - if (test_cc(c->b, _eflags)) + if (test_cc(c->b, ctxt->eflags)) JMP_REL(rel); break; } case 0xc7: /* Grp9 (cmpxchg8b) */ - rc = emulate_grp9(ctxt, ops, &_eflags, cr2); + rc = emulate_grp9(ctxt, ops, cr2); if (rc != 0) goto done; break; From a01af5ec5163cdc51c00f6dcb22d7766199377cf Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Mon, 24 Sep 2007 11:10:56 +0200 Subject: [PATCH 014/249] KVM: x86 emulator: Remove no_wb, use dst.type = OP_NONE instead Remove no_wb, use dst.type = OP_NONE instead, idea stollen from xen-3.1 Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 76 +++++++++++++-------------------------- drivers/kvm/x86_emulate.h | 2 +- 2 files changed, 25 insertions(+), 53 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 76b813b6365c..6161e3f66585 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1016,8 +1016,7 @@ done: } static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops, - int *no_wb) + struct x86_emulate_ops *ops) { struct decode_cache *c = &ctxt->decode; int rc; @@ -1055,7 +1054,7 @@ static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt, c->dst.bytes, ctxt->vcpu); if (rc != 0) return rc; - *no_wb = 1; + c->dst.type = OP_NONE; break; default: DPRINTF("Cannot emulate %02x\n", c->b); @@ -1137,6 +1136,10 @@ static inline int writeback(struct x86_emulate_ctxt *ctxt, ctxt->vcpu); if (rc != 0) return rc; + break; + case OP_NONE: + /* no writeback */ + break; default: break; } @@ -1147,7 +1150,6 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { unsigned long cr2 = ctxt->cr2; - int no_wb = 0; u64 msr_data; unsigned long saved_eip = 0; struct decode_cache *c = &ctxt->decode; @@ -1344,18 +1346,16 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) goto done; break; case 0xfe ... 0xff: /* Grp4/Grp5 */ - rc = emulate_grp45(ctxt, ops, &no_wb); + rc = emulate_grp45(ctxt, ops); if (rc != 0) goto done; break; } writeback: - if (!no_wb) { - rc = writeback(ctxt, ops); - if (rc != 0) - goto done; - } + rc = writeback(ctxt, ops); + if (rc != 0) + goto done; /* Commit shadow register state. */ memcpy(ctxt->vcpu->regs, c->regs, sizeof c->regs); @@ -1395,7 +1395,7 @@ special_insn: register_address_increment(c->regs[VCPU_REGS_RSP], c->op_bytes); - no_wb = 1; /* Disable writeback. */ + c->dst.type = OP_NONE; /* Disable writeback. */ break; case 0x6a: /* push imm8 */ c->src.val = 0L; @@ -1538,7 +1538,7 @@ special_insn: case 0xe9: /* jmp rel */ case 0xeb: /* jmp rel short */ JMP_REL(c->src.val); - no_wb = 1; /* Disable writeback. */ + c->dst.type = OP_NONE; /* Disable writeback. */ break; @@ -1548,8 +1548,6 @@ special_insn: twobyte_insn: switch (c->b) { case 0x01: /* lgdt, lidt, lmsw */ - /* Disable writeback. */ - no_wb = 1; switch (c->modrm_reg) { u16 size; unsigned long address; @@ -1604,56 +1602,30 @@ twobyte_insn: default: goto cannot_emulate; } + /* Disable writeback. */ + c->dst.type = OP_NONE; break; case 0x21: /* mov from dr to reg */ - no_wb = 1; if (c->modrm_mod != 3) goto cannot_emulate; rc = emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]); + if (rc) + goto cannot_emulate; + c->dst.type = OP_NONE; /* no writeback */ break; case 0x23: /* mov from reg to dr */ - no_wb = 1; if (c->modrm_mod != 3) goto cannot_emulate; rc = emulator_set_dr(ctxt, c->modrm_reg, c->regs[c->modrm_rm]); + if (rc) + goto cannot_emulate; + c->dst.type = OP_NONE; /* no writeback */ break; case 0x40 ... 0x4f: /* cmov */ c->dst.val = c->dst.orig_val = c->src.val; - no_wb = 1; - /* - * First, assume we're decoding an even cmov opcode - * (lsb == 0). - */ - switch ((c->b & 15) >> 1) { - case 0: /* cmovo */ - no_wb = (ctxt->eflags & EFLG_OF) ? 0 : 1; - break; - case 1: /* cmovb/cmovc/cmovnae */ - no_wb = (ctxt->eflags & EFLG_CF) ? 0 : 1; - break; - case 2: /* cmovz/cmove */ - no_wb = (ctxt->eflags & EFLG_ZF) ? 0 : 1; - break; - case 3: /* cmovbe/cmovna */ - no_wb = (ctxt->eflags & (EFLG_CF | EFLG_ZF)) ? 0 : 1; - break; - case 4: /* cmovs */ - no_wb = (ctxt->eflags & EFLG_SF) ? 0 : 1; - break; - case 5: /* cmovp/cmovpe */ - no_wb = (ctxt->eflags & EFLG_PF) ? 0 : 1; - break; - case 7: /* cmovle/cmovng */ - no_wb = (ctxt->eflags & EFLG_ZF) ? 0 : 1; - /* fall through */ - case 6: /* cmovl/cmovnge */ - no_wb &= (!(ctxt->eflags & EFLG_SF) != - !(ctxt->eflags & EFLG_OF)) ? 0 : 1; - break; - } - /* Odd cmov opcodes (lsb == 1) have inverted sense. */ - no_wb ^= c->b & 1; + if (!test_cc(c->b, ctxt->eflags)) + c->dst.type = OP_NONE; /* no writeback */ break; case 0xa3: bt: /* bt */ @@ -1727,8 +1699,6 @@ twobyte_insn: goto writeback; twobyte_special_insn: - /* Disable writeback. */ - no_wb = 1; switch (c->b) { case 0x06: emulate_clts(ctxt->vcpu); @@ -1802,6 +1772,8 @@ twobyte_special_insn: goto done; break; } + /* Disable writeback. */ + c->dst.type = OP_NONE; goto writeback; cannot_emulate: diff --git a/drivers/kvm/x86_emulate.h b/drivers/kvm/x86_emulate.h index 28acad416bca..f03b1287700a 100644 --- a/drivers/kvm/x86_emulate.h +++ b/drivers/kvm/x86_emulate.h @@ -114,7 +114,7 @@ struct x86_emulate_ops { /* Type, address-of, and value of an instruction's operand. */ struct operand { - enum { OP_REG, OP_MEM, OP_IMM } type; + enum { OP_REG, OP_MEM, OP_IMM, OP_NONE } type; unsigned int bytes; unsigned long val, orig_val, *ptr; }; From e4f8e03956de4b1a1dc4bbbdefa9c973d01ee91f Mon Sep 17 00:00:00 2001 From: Qing He Date: Mon, 24 Sep 2007 17:22:13 +0800 Subject: [PATCH 015/249] KVM: x86_emulator: no writeback for bt Signed-off-by: Qing He Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 6161e3f66585..0afe660aec4b 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1629,6 +1629,7 @@ twobyte_insn: break; case 0xa3: bt: /* bt */ + c->dst.type = OP_NONE; /* only subword offset */ c->src.val &= (c->dst.bytes << 3) - 1; emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags); From a22436b7b8ec9b14a0451d9ac0fdc9d370bd7800 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Mon, 24 Sep 2007 17:00:58 +0200 Subject: [PATCH 016/249] KVM: Purify x86_decode_insn() error case management The only valid case is on protected page access, other cases are errors. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 2d55bab41634..888b0924c17b 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1251,7 +1251,7 @@ int emulate_instruction(struct kvm_vcpu *vcpu, u16 error_code, int no_decode) { - int r = 0; + int r; vcpu->mmio_fault_cr2 = cr2; kvm_x86_ops->cache_regs(vcpu); @@ -1294,10 +1294,14 @@ int emulate_instruction(struct kvm_vcpu *vcpu, get_segment_base(vcpu, VCPU_SREG_FS); r = x86_decode_insn(&vcpu->emulate_ctxt, &emulate_ops); + if (r) { + if (kvm_mmu_unprotect_page_virt(vcpu, cr2)) + return EMULATE_DONE; + return EMULATE_FAIL; + } } - if (r == 0) - r = x86_emulate_insn(&vcpu->emulate_ctxt, &emulate_ops); + r = x86_emulate_insn(&vcpu->emulate_ctxt, &emulate_ops); if (vcpu->pio.string) return EMULATE_DO_MMIO; From b4c6abfef400c0f74d9b86a149a6719706cfdbbc Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Tue, 25 Sep 2007 13:36:40 +0200 Subject: [PATCH 017/249] KVM: x86 emulator: Any legacy prefix after a REX prefix nullifies its effect This patch modifies the management of REX prefix according behavior I saw in Xen 3.1. In Xen, this modification has been introduced by Jan Beulich. http://lists.xensource.com/archives/html/xen-changelog/2007-01/msg00081.html Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 0afe660aec4b..e294d8409571 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -521,7 +521,6 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { struct decode_cache *c = &ctxt->decode; u8 sib, rex_prefix = 0; - unsigned int i; int rc = 0; int mode = ctxt->mode; int index_reg = 0, base_reg = 0, scale, rip_relative = 0; @@ -551,7 +550,7 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) } /* Legacy prefixes. */ - for (i = 0; i < 8; i++) { + for (;;) { switch (c->b = insn_fetch(u8, 1, c->eip)) { case 0x66: /* operand-size override */ c->op_bytes ^= 6; /* switch between 2/4 bytes */ @@ -582,6 +581,11 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) case 0x36: /* SS override */ c->override_base = &ctxt->ss_base; break; + case 0x40 ... 0x4f: /* REX */ + if (mode != X86EMUL_MODE_PROT64) + goto done_prefixes; + rex_prefix = c->b; + continue; case 0xf0: /* LOCK */ c->lock_prefix = 1; break; @@ -592,19 +596,21 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) default: goto done_prefixes; } + + /* Any legacy prefix after a REX prefix nullifies its effect. */ + + rex_prefix = 0; } done_prefixes: /* REX prefix. */ - if ((mode == X86EMUL_MODE_PROT64) && ((c->b & 0xf0) == 0x40)) { - rex_prefix = c->b; - if (c->b & 8) + if (rex_prefix) { + if (rex_prefix & 8) c->op_bytes = 8; /* REX.W */ - c->modrm_reg = (c->b & 4) << 1; /* REX.R */ - index_reg = (c->b & 2) << 2; /* REX.X */ - c->modrm_rm = base_reg = (c->b & 1) << 3; /* REG.B */ - c->b = insn_fetch(u8, 1, c->eip); + c->modrm_reg = (rex_prefix & 4) << 1; /* REX.R */ + index_reg = (rex_prefix & 2) << 2; /* REX.X */ + c->modrm_rm = base_reg = (rex_prefix & 1) << 3; /* REG.B */ } /* Opcode byte(s). */ From eae5ecb5b9043812968fae7ad9d74bf5e7a50245 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 30 Sep 2007 10:50:12 +0200 Subject: [PATCH 018/249] KVM: VMX: Don't clear the vmcs if the vcpu is not loaded on any processor Noted by Eddie Dong. Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 27a3318fa6c2..cc2844203c24 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -225,7 +225,9 @@ static void __vcpu_clear(void *arg) static void vcpu_clear(struct vcpu_vmx *vmx) { - if (vmx->vcpu.cpu != raw_smp_processor_id() && vmx->vcpu.cpu != -1) + if (vmx->vcpu.cpu == -1) + return; + if (vmx->vcpu.cpu != raw_smp_processor_id()) smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, vmx, 0, 1); else From f566e09fc2c9f4164e1f0017c8c1c7a18bad7d72 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 30 Sep 2007 11:02:53 +0200 Subject: [PATCH 019/249] KVM: VMX: Simplify vcpu_clear() Now that smp_call_function_single() knows how to call a function on the current cpu, there's no need to check explicitly. Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index cc2844203c24..439873a25cca 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -227,11 +227,7 @@ static void vcpu_clear(struct vcpu_vmx *vmx) { if (vmx->vcpu.cpu == -1) return; - if (vmx->vcpu.cpu != raw_smp_processor_id()) - smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, - vmx, 0, 1); - else - __vcpu_clear(vmx); + smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, vmx, 0, 1); vmx->launched = 0; } From 290fc38da8187b53b78dd4d5ab27a20b88ef8b61 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Thu, 27 Sep 2007 14:11:22 +0200 Subject: [PATCH 020/249] KVM: Remove the usage of page->private field by rmap When kvm uses user-allocated pages in the future for the guest, we won't be able to use page->private for rmap, since page->rmap is reserved for the filesystem. So we move the rmap base pointers to the memory slot. A side effect of this is that we need to store the gfn of each gpte in the shadow pages, since the memory slot is addressed by gfn, instead of hfn like struct page. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 6 +- drivers/kvm/kvm_main.c | 11 +++- drivers/kvm/mmu.c | 122 ++++++++++++++++++++++---------------- drivers/kvm/paging_tmpl.h | 3 +- 4 files changed, 86 insertions(+), 56 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 08ffc829f07f..80cfb99fffe0 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -126,6 +126,8 @@ struct kvm_mmu_page { union kvm_mmu_page_role role; u64 *spt; + /* hold the gfn of each spte inside spt */ + gfn_t *gfns; unsigned long slot_bitmap; /* One bit set per slot which has memory * in this shadow page. */ @@ -159,7 +161,7 @@ struct kvm_mmu { u64 *pae_root; }; -#define KVM_NR_MEM_OBJS 20 +#define KVM_NR_MEM_OBJS 40 struct kvm_mmu_memory_cache { int nobjs; @@ -402,6 +404,7 @@ struct kvm_memory_slot { unsigned long npages; unsigned long flags; struct page **phys_mem; + unsigned long *rmap; unsigned long *dirty_bitmap; }; @@ -554,6 +557,7 @@ struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva); extern hpa_t bad_page_address; +gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn); void mark_page_dirty(struct kvm *kvm, gfn_t gfn); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 888b0924c17b..9510e2276ca3 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -309,6 +309,8 @@ static void kvm_free_physmem_slot(struct kvm_memory_slot *free, __free_page(free->phys_mem[i]); vfree(free->phys_mem); } + if (!dont || free->rmap != dont->rmap) + vfree(free->rmap); if (!dont || free->dirty_bitmap != dont->dirty_bitmap) vfree(free->dirty_bitmap); @@ -719,13 +721,18 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, if (!new.phys_mem) goto out_unlock; + new.rmap = vmalloc(npages * sizeof(struct page*)); + + if (!new.rmap) + goto out_unlock; + memset(new.phys_mem, 0, npages * sizeof(struct page *)); + memset(new.rmap, 0, npages * sizeof(*new.rmap)); for (i = 0; i < npages; ++i) { new.phys_mem[i] = alloc_page(GFP_HIGHUSER | __GFP_ZERO); if (!new.phys_mem[i]) goto out_unlock; - set_page_private(new.phys_mem[i],0); } } @@ -909,7 +916,7 @@ static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) return r; } -static gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) +gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) { int i; struct kvm_mem_alias *alias; diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index d347e895736e..72757db15065 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -276,7 +276,7 @@ static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu) rmap_desc_cache, 1); if (r) goto out; - r = mmu_topup_memory_cache_page(&vcpu->mmu_page_cache, 4); + r = mmu_topup_memory_cache_page(&vcpu->mmu_page_cache, 8); if (r) goto out; r = mmu_topup_memory_cache(&vcpu->mmu_page_header_cache, @@ -326,36 +326,53 @@ static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd) kfree(rd); } +/* + * Take gfn and return the reverse mapping to it. + * Note: gfn must be unaliased before this function get called + */ + +static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn) +{ + struct kvm_memory_slot *slot; + + slot = gfn_to_memslot(kvm, gfn); + return &slot->rmap[gfn - slot->base_gfn]; +} + /* * Reverse mapping data structures: * - * If page->private bit zero is zero, then page->private points to the - * shadow page table entry that points to page_address(page). + * If rmapp bit zero is zero, then rmapp point to the shadw page table entry + * that points to page_address(page). * - * If page->private bit zero is one, (then page->private & ~1) points - * to a struct kvm_rmap_desc containing more mappings. + * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc + * containing more mappings. */ -static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte) +static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) { - struct page *page; + struct kvm_mmu_page *page; struct kvm_rmap_desc *desc; + unsigned long *rmapp; int i; if (!is_rmap_pte(*spte)) return; - page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); - if (!page_private(page)) { + gfn = unalias_gfn(vcpu->kvm, gfn); + page = page_header(__pa(spte)); + page->gfns[spte - page->spt] = gfn; + rmapp = gfn_to_rmap(vcpu->kvm, gfn); + if (!*rmapp) { rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte); - set_page_private(page,(unsigned long)spte); - } else if (!(page_private(page) & 1)) { + *rmapp = (unsigned long)spte; + } else if (!(*rmapp & 1)) { rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte); desc = mmu_alloc_rmap_desc(vcpu); - desc->shadow_ptes[0] = (u64 *)page_private(page); + desc->shadow_ptes[0] = (u64 *)*rmapp; desc->shadow_ptes[1] = spte; - set_page_private(page,(unsigned long)desc | 1); + *rmapp = (unsigned long)desc | 1; } else { rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte); - desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul); + desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul); while (desc->shadow_ptes[RMAP_EXT-1] && desc->more) desc = desc->more; if (desc->shadow_ptes[RMAP_EXT-1]) { @@ -368,7 +385,7 @@ static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte) } } -static void rmap_desc_remove_entry(struct page *page, +static void rmap_desc_remove_entry(unsigned long *rmapp, struct kvm_rmap_desc *desc, int i, struct kvm_rmap_desc *prev_desc) @@ -382,44 +399,46 @@ static void rmap_desc_remove_entry(struct page *page, if (j != 0) return; if (!prev_desc && !desc->more) - set_page_private(page,(unsigned long)desc->shadow_ptes[0]); + *rmapp = (unsigned long)desc->shadow_ptes[0]; else if (prev_desc) prev_desc->more = desc->more; else - set_page_private(page,(unsigned long)desc->more | 1); + *rmapp = (unsigned long)desc->more | 1; mmu_free_rmap_desc(desc); } -static void rmap_remove(u64 *spte) +static void rmap_remove(struct kvm *kvm, u64 *spte) { - struct page *page; struct kvm_rmap_desc *desc; struct kvm_rmap_desc *prev_desc; + struct kvm_mmu_page *page; + unsigned long *rmapp; int i; if (!is_rmap_pte(*spte)) return; - page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); - if (!page_private(page)) { + page = page_header(__pa(spte)); + rmapp = gfn_to_rmap(kvm, page->gfns[spte - page->spt]); + if (!*rmapp) { printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte); BUG(); - } else if (!(page_private(page) & 1)) { + } else if (!(*rmapp & 1)) { rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte); - if ((u64 *)page_private(page) != spte) { + if ((u64 *)*rmapp != spte) { printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n", spte, *spte); BUG(); } - set_page_private(page,0); + *rmapp = 0; } else { rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte); - desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul); + desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul); prev_desc = NULL; while (desc) { for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) if (desc->shadow_ptes[i] == spte) { - rmap_desc_remove_entry(page, + rmap_desc_remove_entry(rmapp, desc, i, prev_desc); return; @@ -433,28 +452,25 @@ static void rmap_remove(u64 *spte) static void rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn) { - struct kvm *kvm = vcpu->kvm; - struct page *page; struct kvm_rmap_desc *desc; + unsigned long *rmapp; u64 *spte; - page = gfn_to_page(kvm, gfn); - BUG_ON(!page); + gfn = unalias_gfn(vcpu->kvm, gfn); + rmapp = gfn_to_rmap(vcpu->kvm, gfn); - while (page_private(page)) { - if (!(page_private(page) & 1)) - spte = (u64 *)page_private(page); + while (*rmapp) { + if (!(*rmapp & 1)) + spte = (u64 *)*rmapp; else { - desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul); + desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul); spte = desc->shadow_ptes[0]; } BUG_ON(!spte); - BUG_ON((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT - != page_to_pfn(page)); BUG_ON(!(*spte & PT_PRESENT_MASK)); BUG_ON(!(*spte & PT_WRITABLE_MASK)); rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte); - rmap_remove(spte); + rmap_remove(vcpu->kvm, spte); set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK); kvm_flush_remote_tlbs(vcpu->kvm); } @@ -482,6 +498,7 @@ static void kvm_mmu_free_page(struct kvm *kvm, ASSERT(is_empty_shadow_page(page_head->spt)); list_del(&page_head->link); __free_page(virt_to_page(page_head->spt)); + __free_page(virt_to_page(page_head->gfns)); kfree(page_head); ++kvm->n_free_mmu_pages; } @@ -502,6 +519,7 @@ static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, page = mmu_memory_cache_alloc(&vcpu->mmu_page_header_cache, sizeof *page); page->spt = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE); + page->gfns = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE); set_page_private(virt_to_page(page->spt), (unsigned long)page); list_add(&page->link, &vcpu->kvm->active_mmu_pages); ASSERT(is_empty_shadow_page(page->spt)); @@ -667,7 +685,7 @@ static void kvm_mmu_page_unlink_children(struct kvm *kvm, if (page->role.level == PT_PAGE_TABLE_LEVEL) { for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { if (is_shadow_present_pte(pt[i])) - rmap_remove(&pt[i]); + rmap_remove(kvm, &pt[i]); pt[i] = shadow_trap_nonpresent_pte; } kvm_flush_remote_tlbs(kvm); @@ -832,7 +850,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) page_header_update_slot(vcpu->kvm, table, v); table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK; - rmap_add(vcpu, &table[index]); + rmap_add(vcpu, &table[index], v >> PAGE_SHIFT); return 0; } @@ -1123,7 +1141,7 @@ static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu, pte = *spte; if (is_shadow_present_pte(pte)) { if (page->role.level == PT_PAGE_TABLE_LEVEL) - rmap_remove(spte); + rmap_remove(vcpu->kvm, spte); else { child = page_header(pte & PT64_BASE_ADDR_MASK); mmu_page_remove_parent_pte(child, spte); @@ -1340,7 +1358,7 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot) for (i = 0; i < PT64_ENT_PER_PAGE; ++i) /* avoid RMW */ if (pt[i] & PT_WRITABLE_MASK) { - rmap_remove(&pt[i]); + rmap_remove(kvm, &pt[i]); pt[i] &= ~PT_WRITABLE_MASK; } } @@ -1470,15 +1488,15 @@ static int count_rmaps(struct kvm_vcpu *vcpu) struct kvm_rmap_desc *d; for (j = 0; j < m->npages; ++j) { - struct page *page = m->phys_mem[j]; + unsigned long *rmapp = &m->rmap[j]; - if (!page->private) + if (!*rmapp) continue; - if (!(page->private & 1)) { + if (!(*rmapp & 1)) { ++nmaps; continue; } - d = (struct kvm_rmap_desc *)(page->private & ~1ul); + d = (struct kvm_rmap_desc *)(*rmapp & ~1ul); while (d) { for (k = 0; k < RMAP_EXT; ++k) if (d->shadow_ptes[k]) @@ -1530,18 +1548,18 @@ static void audit_rmap(struct kvm_vcpu *vcpu) static void audit_write_protection(struct kvm_vcpu *vcpu) { struct kvm_mmu_page *page; + struct kvm_memory_slot *slot; + unsigned long *rmapp; + gfn_t gfn; list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) { - hfn_t hfn; - struct page *pg; - if (page->role.metaphysical) continue; - hfn = gpa_to_hpa(vcpu, (gpa_t)page->gfn << PAGE_SHIFT) - >> PAGE_SHIFT; - pg = pfn_to_page(hfn); - if (pg->private) + slot = gfn_to_memslot(vcpu->kvm, page->gfn); + gfn = unalias_gfn(vcpu->kvm, page->gfn); + rmapp = &slot->rmap[gfn - slot->base_gfn]; + if (*rmapp) printk(KERN_ERR "%s: (%s) shadow page has writable" " mappings: gfn %lx role %x\n", __FUNCTION__, audit_msg, page->gfn, diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index be0f85231da9..fbe595f880af 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -295,7 +295,8 @@ unshadowed: set_shadow_pte(shadow_pte, spte); page_header_update_slot(vcpu->kvm, shadow_pte, gaddr); if (!was_rmapped) - rmap_add(vcpu, shadow_pte); + rmap_add(vcpu, shadow_pte, (gaddr & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT); if (!ptwrite || !*ptwrite) vcpu->last_pte_updated = shadow_pte; } From 195aefde9cc2cee38dd54ef92a866721fba4413e Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Mon, 1 Oct 2007 22:14:18 +0200 Subject: [PATCH 021/249] KVM: Add general accessors to read and write guest memory Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 9 +++ drivers/kvm/kvm_main.c | 160 +++++++++++++++++++++++++++++++++-------- drivers/kvm/vmx.c | 43 +++++------ 3 files changed, 158 insertions(+), 54 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 80cfb99fffe0..1965438f18cc 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -559,6 +559,15 @@ extern hpa_t bad_page_address; gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); +int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, + int len); +int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len); +int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, + int offset, int len); +int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, + unsigned long len); +int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len); +int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len); struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn); void mark_page_dirty(struct kvm *kvm, gfn_t gfn); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 9510e2276ca3..cac328b8421c 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -400,22 +400,16 @@ static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2; int i; - u64 *pdpt; int ret; - struct page *page; u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)]; mutex_lock(&vcpu->kvm->lock); - page = gfn_to_page(vcpu->kvm, pdpt_gfn); - if (!page) { + ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte, + offset * sizeof(u64), sizeof(pdpte)); + if (ret < 0) { ret = 0; goto out; } - - pdpt = kmap_atomic(page, KM_USER0); - memcpy(pdpte, pdpt+offset, sizeof(pdpte)); - kunmap_atomic(pdpt, KM_USER0); - for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { if ((pdpte[i] & 1) && (pdpte[i] & 0xfffffff0000001e6ull)) { ret = 0; @@ -962,6 +956,127 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) } EXPORT_SYMBOL_GPL(gfn_to_page); +static int next_segment(unsigned long len, int offset) +{ + if (len > PAGE_SIZE - offset) + return PAGE_SIZE - offset; + else + return len; +} + +int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, + int len) +{ + void *page_virt; + struct page *page; + + page = gfn_to_page(kvm, gfn); + if (!page) + return -EFAULT; + page_virt = kmap_atomic(page, KM_USER0); + + memcpy(data, page_virt + offset, len); + + kunmap_atomic(page_virt, KM_USER0); + return 0; +} +EXPORT_SYMBOL_GPL(kvm_read_guest_page); + +int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) +{ + gfn_t gfn = gpa >> PAGE_SHIFT; + int seg; + int offset = offset_in_page(gpa); + int ret; + + while ((seg = next_segment(len, offset)) != 0) { + ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); + if (ret < 0) + return ret; + offset = 0; + len -= seg; + data += seg; + ++gfn; + } + return 0; +} +EXPORT_SYMBOL_GPL(kvm_read_guest); + +int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, + int offset, int len) +{ + void *page_virt; + struct page *page; + + page = gfn_to_page(kvm, gfn); + if (!page) + return -EFAULT; + page_virt = kmap_atomic(page, KM_USER0); + + memcpy(page_virt + offset, data, len); + + kunmap_atomic(page_virt, KM_USER0); + mark_page_dirty(kvm, gfn); + return 0; +} +EXPORT_SYMBOL_GPL(kvm_write_guest_page); + +int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, + unsigned long len) +{ + gfn_t gfn = gpa >> PAGE_SHIFT; + int seg; + int offset = offset_in_page(gpa); + int ret; + + while ((seg = next_segment(len, offset)) != 0) { + ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); + if (ret < 0) + return ret; + offset = 0; + len -= seg; + data += seg; + ++gfn; + } + return 0; +} + +int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) +{ + void *page_virt; + struct page *page; + + page = gfn_to_page(kvm, gfn); + if (!page) + return -EFAULT; + page_virt = kmap_atomic(page, KM_USER0); + + memset(page_virt + offset, 0, len); + + kunmap_atomic(page_virt, KM_USER0); + return 0; +} +EXPORT_SYMBOL_GPL(kvm_clear_guest_page); + +int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) +{ + gfn_t gfn = gpa >> PAGE_SHIFT; + int seg; + int offset = offset_in_page(gpa); + int ret; + + while ((seg = next_segment(len, offset)) != 0) { + ret = kvm_clear_guest_page(kvm, gfn, offset, seg); + if (ret < 0) + return ret; + offset = 0; + len -= seg; + ++gfn; + } + return 0; +} +EXPORT_SYMBOL_GPL(kvm_clear_guest); + /* WARNING: Does not work on aliased pages. */ void mark_page_dirty(struct kvm *kvm, gfn_t gfn) { @@ -988,21 +1103,13 @@ int emulator_read_std(unsigned long addr, gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); unsigned offset = addr & (PAGE_SIZE-1); unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset); - unsigned long pfn; - struct page *page; - void *page_virt; + int ret; if (gpa == UNMAPPED_GVA) return X86EMUL_PROPAGATE_FAULT; - pfn = gpa >> PAGE_SHIFT; - page = gfn_to_page(vcpu->kvm, pfn); - if (!page) + ret = kvm_read_guest(vcpu->kvm, gpa, data, tocopy); + if (ret < 0) return X86EMUL_UNHANDLEABLE; - page_virt = kmap_atomic(page, KM_USER0); - - memcpy(data, page_virt + offset, tocopy); - - kunmap_atomic(page_virt, KM_USER0); bytes -= tocopy; data += tocopy; @@ -1095,19 +1202,12 @@ static int emulator_read_emulated(unsigned long addr, static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, const void *val, int bytes) { - struct page *page; - void *virt; + int ret; - if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT)) + ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes); + if (ret < 0) return 0; - page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); - if (!page) - return 0; - mark_page_dirty(vcpu->kvm, gpa >> PAGE_SHIFT); - virt = kmap_atomic(page, KM_USER0); kvm_mmu_pte_write(vcpu, gpa, val, bytes); - memcpy(virt + offset_in_page(gpa), val, bytes); - kunmap_atomic(virt, KM_USER0); return 1; } diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 439873a25cca..894fd45ecc98 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1387,33 +1387,28 @@ static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) static int init_rmode_tss(struct kvm* kvm) { - struct page *p1, *p2, *p3; gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT; - char *page; + u16 data = 0; + int r; - p1 = gfn_to_page(kvm, fn++); - p2 = gfn_to_page(kvm, fn++); - p3 = gfn_to_page(kvm, fn); - - if (!p1 || !p2 || !p3) { - kvm_printf(kvm,"%s: gfn_to_page failed\n", __FUNCTION__); + r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE); + if (r < 0) + return 0; + data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE; + r = kvm_write_guest_page(kvm, fn++, &data, 0x66, sizeof(u16)); + if (r < 0) + return 0; + r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE); + if (r < 0) + return 0; + r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE); + if (r < 0) + return 0; + data = ~0; + r = kvm_write_guest_page(kvm, fn, &data, RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1, + sizeof(u8)); + if (r < 0) return 0; - } - - page = kmap_atomic(p1, KM_USER0); - clear_page(page); - *(u16*)(page + 0x66) = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE; - kunmap_atomic(page, KM_USER0); - - page = kmap_atomic(p2, KM_USER0); - clear_page(page); - kunmap_atomic(page, KM_USER0); - - page = kmap_atomic(p3, KM_USER0); - clear_page(page); - *(page + RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1) = ~0; - kunmap_atomic(page, KM_USER0); - return 1; } From 82ce2c96831f049a37118733ced5c8f7c8848102 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Tue, 2 Oct 2007 18:52:55 +0200 Subject: [PATCH 022/249] KVM: Allow dynamic allocation of the mmu shadow cache size The user is now able to set how many mmu pages will be allocated to the guest. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 7 ++++++- drivers/kvm/kvm_main.c | 47 ++++++++++++++++++++++++++++++++++++++++++ drivers/kvm/mmu.c | 40 +++++++++++++++++++++++++++++++++-- include/linux/kvm.h | 3 +++ 4 files changed, 94 insertions(+), 3 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 1965438f18cc..9f10c373b74c 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -40,6 +40,8 @@ #define KVM_MAX_VCPUS 4 #define KVM_ALIAS_SLOTS 4 #define KVM_MEMORY_SLOTS 8 +#define KVM_PERMILLE_MMU_PAGES 20 +#define KVM_MIN_ALLOC_MMU_PAGES 64 #define KVM_NUM_MMU_PAGES 1024 #define KVM_MIN_FREE_MMU_PAGES 5 #define KVM_REFILL_PAGES 25 @@ -418,7 +420,9 @@ struct kvm { * Hash table of struct kvm_mmu_page. */ struct list_head active_mmu_pages; - int n_free_mmu_pages; + unsigned int n_free_mmu_pages; + unsigned int n_requested_mmu_pages; + unsigned int n_alloc_mmu_pages; struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES]; struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; unsigned long rmap_overflow; @@ -547,6 +551,7 @@ void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte); int kvm_mmu_reset_context(struct kvm_vcpu *vcpu); void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot); void kvm_mmu_zap_all(struct kvm *kvm); +void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages); hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa); #define HPA_MSB ((sizeof(hpa_t) * 8) - 1) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index cac328b8421c..2bb1f2f66efa 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -743,6 +743,24 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, if (mem->slot >= kvm->nmemslots) kvm->nmemslots = mem->slot + 1; + if (!kvm->n_requested_mmu_pages) { + unsigned int n_pages; + + if (npages) { + n_pages = npages * KVM_PERMILLE_MMU_PAGES / 1000; + kvm_mmu_change_mmu_pages(kvm, kvm->n_alloc_mmu_pages + + n_pages); + } else { + unsigned int nr_mmu_pages; + + n_pages = old.npages * KVM_PERMILLE_MMU_PAGES / 1000; + nr_mmu_pages = kvm->n_alloc_mmu_pages - n_pages; + nr_mmu_pages = max(nr_mmu_pages, + (unsigned int) KVM_MIN_ALLOC_MMU_PAGES); + kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); + } + } + *memslot = new; kvm_mmu_slot_remove_write_access(kvm, mem->slot); @@ -760,6 +778,26 @@ out: return r; } +static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, + u32 kvm_nr_mmu_pages) +{ + if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES) + return -EINVAL; + + mutex_lock(&kvm->lock); + + kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages); + kvm->n_requested_mmu_pages = kvm_nr_mmu_pages; + + mutex_unlock(&kvm->lock); + return 0; +} + +static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm) +{ + return kvm->n_alloc_mmu_pages; +} + /* * Get (and clear) the dirty memory log for a memory slot. */ @@ -3071,6 +3109,14 @@ static long kvm_vm_ioctl(struct file *filp, goto out; break; } + case KVM_SET_NR_MMU_PAGES: + r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg); + if (r) + goto out; + break; + case KVM_GET_NR_MMU_PAGES: + r = kvm_vm_ioctl_get_nr_mmu_pages(kvm); + break; case KVM_GET_DIRTY_LOG: { struct kvm_dirty_log log; @@ -3278,6 +3324,7 @@ static long kvm_dev_ioctl(struct file *filp, switch (ext) { case KVM_CAP_IRQCHIP: case KVM_CAP_HLT: + case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: r = 1; break; default: diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 72757db15065..6cda1feb9a95 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -747,6 +747,40 @@ static void kvm_mmu_zap_page(struct kvm *kvm, kvm_mmu_reset_last_pte_updated(kvm); } +/* + * Changing the number of mmu pages allocated to the vm + * Note: if kvm_nr_mmu_pages is too small, you will get dead lock + */ +void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages) +{ + /* + * If we set the number of mmu pages to be smaller be than the + * number of actived pages , we must to free some mmu pages before we + * change the value + */ + + if ((kvm->n_alloc_mmu_pages - kvm->n_free_mmu_pages) > + kvm_nr_mmu_pages) { + int n_used_mmu_pages = kvm->n_alloc_mmu_pages + - kvm->n_free_mmu_pages; + + while (n_used_mmu_pages > kvm_nr_mmu_pages) { + struct kvm_mmu_page *page; + + page = container_of(kvm->active_mmu_pages.prev, + struct kvm_mmu_page, link); + kvm_mmu_zap_page(kvm, page); + n_used_mmu_pages--; + } + kvm->n_free_mmu_pages = 0; + } + else + kvm->n_free_mmu_pages += kvm_nr_mmu_pages + - kvm->n_alloc_mmu_pages; + + kvm->n_alloc_mmu_pages = kvm_nr_mmu_pages; +} + static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn) { unsigned index; @@ -1297,8 +1331,10 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu) ASSERT(vcpu); - vcpu->kvm->n_free_mmu_pages = KVM_NUM_MMU_PAGES; - + if (vcpu->kvm->n_requested_mmu_pages) + vcpu->kvm->n_free_mmu_pages = vcpu->kvm->n_requested_mmu_pages; + else + vcpu->kvm->n_free_mmu_pages = vcpu->kvm->n_alloc_mmu_pages; /* * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64. * Therefore we need to allocate shadow page tables in the first diff --git a/include/linux/kvm.h b/include/linux/kvm.h index 057a7f34ee36..d2fd973d81fe 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -347,11 +347,14 @@ struct kvm_signal_mask { */ #define KVM_CAP_IRQCHIP 0 #define KVM_CAP_HLT 1 +#define KVM_CAP_MMU_SHADOW_CACHE_CONTROL 2 /* * ioctls for VM fds */ #define KVM_SET_MEMORY_REGION _IOW(KVMIO, 0x40, struct kvm_memory_region) +#define KVM_SET_NR_MMU_PAGES _IO(KVMIO, 0x44) +#define KVM_GET_NR_MMU_PAGES _IO(KVMIO, 0x45) /* * KVM_CREATE_VCPU receives as a parameter the vcpu slot, and returns * a vcpu fd. From d589444e924bc72e42fa94853f9096589d69374d Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 8 Oct 2007 10:48:30 +1000 Subject: [PATCH 023/249] KVM: Add kvm_free_lapic() to pair with kvm_create_lapic() Instead of the asymetry of kvm_free_apic, implement kvm_free_lapic(). And guess what? I found a minor bug: we don't need to hrtimer_cancel() from kvm_main.c, because we do that in kvm_free_apic(). Also: 1) kvm_vcpu_uninit should be the reverse order from kvm_vcpu_init. 2) Don't set apic->regs_page to zero before freeing apic. Signed-off-by: Rusty Russell Signed-off-by: Avi Kivity --- drivers/kvm/irq.h | 2 +- drivers/kvm/kvm_main.c | 4 +--- drivers/kvm/lapic.c | 19 +++++++++---------- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index 11fc014e2b30..508280ea6fcf 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -139,7 +139,7 @@ int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu); int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu); int kvm_create_lapic(struct kvm_vcpu *vcpu); void kvm_lapic_reset(struct kvm_vcpu *vcpu); -void kvm_free_apic(struct kvm_lapic *apic); +void kvm_free_lapic(struct kvm_vcpu *vcpu); u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu); void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8); void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 2bb1f2f66efa..b19734606cd9 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -268,10 +268,8 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_init); void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) { + kvm_free_lapic(vcpu); kvm_mmu_destroy(vcpu); - if (vcpu->apic) - hrtimer_cancel(&vcpu->apic->timer.dev); - kvm_free_apic(vcpu->apic); free_page((unsigned long)vcpu->pio_data); free_page((unsigned long)vcpu->run); } diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index 238fcad3cece..8e8dab08be58 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -762,19 +762,17 @@ static int apic_mmio_range(struct kvm_io_device *this, gpa_t addr) return ret; } -void kvm_free_apic(struct kvm_lapic *apic) +void kvm_free_lapic(struct kvm_vcpu *vcpu) { - if (!apic) + if (!vcpu->apic) return; - hrtimer_cancel(&apic->timer.dev); + hrtimer_cancel(&vcpu->apic->timer.dev); - if (apic->regs_page) { - __free_page(apic->regs_page); - apic->regs_page = 0; - } + if (vcpu->apic->regs_page) + __free_page(vcpu->apic->regs_page); - kfree(apic); + kfree(vcpu->apic); } /* @@ -962,7 +960,7 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu) if (apic->regs_page == NULL) { printk(KERN_ERR "malloc apic regs error for vcpu %x\n", vcpu->vcpu_id); - goto nomem; + goto nomem_free_apic; } apic->regs = page_address(apic->regs_page); memset(apic->regs, 0, PAGE_SIZE); @@ -980,8 +978,9 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu) apic->dev.private = apic; return 0; +nomem_free_apic: + kfree(apic); nomem: - kvm_free_apic(apic); return -ENOMEM; } EXPORT_SYMBOL_GPL(kvm_create_lapic); From 76fafa5e22bd82e92d2734852ba23f17322d675a Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 8 Oct 2007 10:50:48 +1000 Subject: [PATCH 024/249] KVM: Hoist kvm_create_lapic() into kvm_vcpu_init() Move kvm_create_lapic() into kvm_vcpu_init(), rather than having svm and vmx do it. And make it return the error rather than a fairly random -ENOMEM. This also solves the problem that neither svm.c nor vmx.c actually handles the error path properly. Signed-off-by: Rusty Russell Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 10 +++++++++- drivers/kvm/svm.c | 6 ------ drivers/kvm/vmx.c | 6 ------ 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index b19734606cd9..9ea9277014aa 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -255,14 +255,22 @@ int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) if (r < 0) goto fail_free_pio_data; + if (irqchip_in_kernel(kvm)) { + r = kvm_create_lapic(vcpu); + if (r < 0) + goto fail_mmu_destroy; + } + return 0; +fail_mmu_destroy: + kvm_mmu_destroy(vcpu); fail_free_pio_data: free_page((unsigned long)vcpu->pio_data); fail_free_run: free_page((unsigned long)vcpu->run); fail: - return -ENOMEM; + return r; } EXPORT_SYMBOL_GPL(kvm_vcpu_init); diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index f268bd51f337..fb2e591d5397 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -588,12 +588,6 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id) if (err) goto free_svm; - if (irqchip_in_kernel(kvm)) { - err = kvm_create_lapic(&svm->vcpu); - if (err < 0) - goto free_svm; - } - page = alloc_page(GFP_KERNEL); if (!page) { err = -ENOMEM; diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 894fd45ecc98..7b742901e783 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -2431,12 +2431,6 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) if (err) goto free_vcpu; - if (irqchip_in_kernel(kvm)) { - err = kvm_create_lapic(&vmx->vcpu); - if (err < 0) - goto free_vcpu; - } - vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!vmx->guest_msrs) { err = -ENOMEM; From 7e620d16b8838bc0ad5b27d2dd55796270cd588c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 8 Oct 2007 10:55:29 +1000 Subject: [PATCH 025/249] KVM: Remove gratuitous casts from lapic.c Since vcpu->apic is of the correct type, there's not need to cast. Signed-off-by: Rusty Russell Signed-off-by: Avi Kivity --- drivers/kvm/lapic.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index 8e8dab08be58..554e73ad33f0 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -172,7 +172,7 @@ static inline int apic_find_highest_irr(struct kvm_lapic *apic) int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = (struct kvm_lapic *)vcpu->apic; + struct kvm_lapic *apic = vcpu->apic; int highest_irr; if (!apic) @@ -783,7 +783,7 @@ void kvm_free_lapic(struct kvm_vcpu *vcpu) void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) { - struct kvm_lapic *apic = (struct kvm_lapic *)vcpu->apic; + struct kvm_lapic *apic = vcpu->apic; if (!apic) return; @@ -792,7 +792,7 @@ void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = (struct kvm_lapic *)vcpu->apic; + struct kvm_lapic *apic = vcpu->apic; u64 tpr; if (!apic) @@ -805,7 +805,7 @@ EXPORT_SYMBOL_GPL(kvm_lapic_get_cr8); void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) { - struct kvm_lapic *apic = (struct kvm_lapic *)vcpu->apic; + struct kvm_lapic *apic = vcpu->apic; if (!apic) { value |= MSR_IA32_APICBASE_BSP; @@ -882,7 +882,7 @@ EXPORT_SYMBOL_GPL(kvm_lapic_reset); int kvm_lapic_enabled(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = (struct kvm_lapic *)vcpu->apic; + struct kvm_lapic *apic = vcpu->apic; int ret = 0; if (!apic) From d77c26fce93d07802db97498959587eb9347b31d Mon Sep 17 00:00:00 2001 From: Mike Day Date: Mon, 8 Oct 2007 09:02:08 -0400 Subject: [PATCH 026/249] KVM: CodingStyle cleanup Signed-off-by: Mike D. Day Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 32 ++++++++--------- drivers/kvm/kvm_main.c | 58 +++++++++++++++--------------- drivers/kvm/lapic.c | 3 +- drivers/kvm/mmu.c | 10 +++--- drivers/kvm/paging_tmpl.h | 2 +- drivers/kvm/svm.c | 48 ++++++++++++------------- drivers/kvm/svm.h | 2 +- drivers/kvm/vmx.c | 60 ++++++++++++++++--------------- drivers/kvm/vmx.h | 8 ++--- drivers/kvm/x86_emulate.c | 76 +++++++++++++++++++-------------------- 10 files changed, 151 insertions(+), 148 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 9f10c373b74c..ec5b498945ae 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -528,7 +528,7 @@ extern struct kvm_x86_ops *kvm_x86_ops; if (printk_ratelimit()) \ printk(KERN_ERR "kvm: %i: cpu%i " fmt, \ current->tgid, (vcpu)->vcpu_id , ## __VA_ARGS__); \ - } while(0) + } while (0) #define kvm_printf(kvm, fmt ...) printk(KERN_DEBUG fmt) #define vcpu_printf(vcpu, fmt...) kvm_printf(vcpu->kvm, fmt) @@ -598,7 +598,7 @@ int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); struct x86_emulate_ctxt; -int kvm_emulate_pio (struct kvm_vcpu *vcpu, struct kvm_run *run, int in, +int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, int size, unsigned port); int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, int size, unsigned long count, int down, @@ -607,7 +607,7 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu); int kvm_emulate_halt(struct kvm_vcpu *vcpu); int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address); int emulate_clts(struct kvm_vcpu *vcpu); -int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, +int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest); int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value); @@ -631,7 +631,7 @@ void kvm_put_guest_fpu(struct kvm_vcpu *vcpu); void kvm_flush_remote_tlbs(struct kvm *kvm); int emulator_read_std(unsigned long addr, - void *val, + void *val, unsigned int bytes, struct kvm_vcpu *vcpu); int emulator_write_emulated(unsigned long addr, @@ -721,55 +721,55 @@ static inline struct kvm_mmu_page *page_header(hpa_t shadow_page) static inline u16 read_fs(void) { u16 seg; - asm ("mov %%fs, %0" : "=g"(seg)); + asm("mov %%fs, %0" : "=g"(seg)); return seg; } static inline u16 read_gs(void) { u16 seg; - asm ("mov %%gs, %0" : "=g"(seg)); + asm("mov %%gs, %0" : "=g"(seg)); return seg; } static inline u16 read_ldt(void) { u16 ldt; - asm ("sldt %0" : "=g"(ldt)); + asm("sldt %0" : "=g"(ldt)); return ldt; } static inline void load_fs(u16 sel) { - asm ("mov %0, %%fs" : : "rm"(sel)); + asm("mov %0, %%fs" : : "rm"(sel)); } static inline void load_gs(u16 sel) { - asm ("mov %0, %%gs" : : "rm"(sel)); + asm("mov %0, %%gs" : : "rm"(sel)); } #ifndef load_ldt static inline void load_ldt(u16 sel) { - asm ("lldt %0" : : "rm"(sel)); + asm("lldt %0" : : "rm"(sel)); } #endif static inline void get_idt(struct descriptor_table *table) { - asm ("sidt %0" : "=m"(*table)); + asm("sidt %0" : "=m"(*table)); } static inline void get_gdt(struct descriptor_table *table) { - asm ("sgdt %0" : "=m"(*table)); + asm("sgdt %0" : "=m"(*table)); } static inline unsigned long read_tr_base(void) { u16 tr; - asm ("str %0" : "=g"(tr)); + asm("str %0" : "=g"(tr)); return segment_base(tr); } @@ -785,17 +785,17 @@ static inline unsigned long read_msr(unsigned long msr) static inline void fx_save(struct i387_fxsave_struct *image) { - asm ("fxsave (%0)":: "r" (image)); + asm("fxsave (%0)":: "r" (image)); } static inline void fx_restore(struct i387_fxsave_struct *image) { - asm ("fxrstor (%0)":: "r" (image)); + asm("fxrstor (%0)":: "r" (image)); } static inline void fpu_init(void) { - asm ("finit"); + asm("finit"); } static inline u32 get_rdx_init_val(void) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 9ea9277014aa..a1983d2d5b8f 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -104,7 +104,7 @@ static struct dentry *debugfs_dir; #define EFER_RESERVED_BITS 0xfffffffffffff2fe #ifdef CONFIG_X86_64 -// LDT or TSS descriptor in the GDT. 16 bytes. +/* LDT or TSS descriptor in the GDT. 16 bytes. */ struct segment_descriptor_64 { struct segment_descriptor s; u32 base_higher; @@ -121,27 +121,27 @@ unsigned long segment_base(u16 selector) struct descriptor_table gdt; struct segment_descriptor *d; unsigned long table_base; - typedef unsigned long ul; unsigned long v; if (selector == 0) return 0; - asm ("sgdt %0" : "=m"(gdt)); + asm("sgdt %0" : "=m"(gdt)); table_base = gdt.base; if (selector & 4) { /* from ldt */ u16 ldt_selector; - asm ("sldt %0" : "=g"(ldt_selector)); + asm("sldt %0" : "=g"(ldt_selector)); table_base = segment_base(ldt_selector); } d = (struct segment_descriptor *)(table_base + (selector & ~7)); - v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24); + v = d->base_low | ((unsigned long)d->base_mid << 16) | + ((unsigned long)d->base_high << 24); #ifdef CONFIG_X86_64 - if (d->system == 0 - && (d->type == 2 || d->type == 9 || d->type == 11)) - v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32; + if (d->system == 0 && (d->type == 2 || d->type == 9 || d->type == 11)) + v |= ((unsigned long) \ + ((struct segment_descriptor_64 *)d)->base_higher) << 32; #endif return v; } @@ -721,7 +721,7 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, if (!new.phys_mem) goto out_unlock; - new.rmap = vmalloc(npages * sizeof(struct page*)); + new.rmap = vmalloc(npages * sizeof(struct page *)); if (!new.rmap) goto out_unlock; @@ -904,17 +904,17 @@ static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) r = 0; switch (chip->chip_id) { case KVM_IRQCHIP_PIC_MASTER: - memcpy (&chip->chip.pic, + memcpy(&chip->chip.pic, &pic_irqchip(kvm)->pics[0], sizeof(struct kvm_pic_state)); break; case KVM_IRQCHIP_PIC_SLAVE: - memcpy (&chip->chip.pic, + memcpy(&chip->chip.pic, &pic_irqchip(kvm)->pics[1], sizeof(struct kvm_pic_state)); break; case KVM_IRQCHIP_IOAPIC: - memcpy (&chip->chip.ioapic, + memcpy(&chip->chip.ioapic, ioapic_irqchip(kvm), sizeof(struct kvm_ioapic_state)); break; @@ -932,17 +932,17 @@ static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) r = 0; switch (chip->chip_id) { case KVM_IRQCHIP_PIC_MASTER: - memcpy (&pic_irqchip(kvm)->pics[0], + memcpy(&pic_irqchip(kvm)->pics[0], &chip->chip.pic, sizeof(struct kvm_pic_state)); break; case KVM_IRQCHIP_PIC_SLAVE: - memcpy (&pic_irqchip(kvm)->pics[1], + memcpy(&pic_irqchip(kvm)->pics[1], &chip->chip.pic, sizeof(struct kvm_pic_state)); break; case KVM_IRQCHIP_IOAPIC: - memcpy (ioapic_irqchip(kvm), + memcpy(ioapic_irqchip(kvm), &chip->chip.ioapic, sizeof(struct kvm_ioapic_state)); break; @@ -1341,7 +1341,7 @@ int emulate_clts(struct kvm_vcpu *vcpu) return X86EMUL_CONTINUE; } -int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest) +int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest) { struct kvm_vcpu *vcpu = ctxt->vcpu; @@ -1934,7 +1934,7 @@ static void pio_string_write(struct kvm_io_device *pio_dev, mutex_unlock(&vcpu->kvm->lock); } -int kvm_emulate_pio (struct kvm_vcpu *vcpu, struct kvm_run *run, int in, +int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, int size, unsigned port) { struct kvm_io_device *pio_dev; @@ -2089,7 +2089,7 @@ static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) int r; if (unlikely(vcpu->mp_state == VCPU_MP_STATE_SIPI_RECEIVED)) { - printk("vcpu %d received sipi with vector # %x\n", + pr_debug("vcpu %d received sipi with vector # %x\n", vcpu->vcpu_id, vcpu->sipi_vector); kvm_lapic_reset(vcpu); kvm_x86_ops->vcpu_reset(vcpu); @@ -2363,7 +2363,8 @@ static int kvm_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, sizeof sregs->interrupt_bitmap); pending_vec = kvm_x86_ops->get_irq(vcpu); if (pending_vec >= 0) - set_bit(pending_vec, (unsigned long *)sregs->interrupt_bitmap); + set_bit(pending_vec, + (unsigned long *)sregs->interrupt_bitmap); } else memcpy(sregs->interrupt_bitmap, vcpu->irq_pending, sizeof sregs->interrupt_bitmap); @@ -2436,7 +2437,8 @@ static int kvm_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, /* Only pending external irq is handled here */ if (pending_vec < max_bits) { kvm_x86_ops->set_irq(vcpu, pending_vec); - printk("Set back pending irq %d\n", pending_vec); + pr_debug("Set back pending irq %d\n", + pending_vec); } } @@ -3155,8 +3157,7 @@ static long kvm_vm_ioctl(struct file *filp, kvm->vpic = NULL; goto out; } - } - else + } else goto out; break; case KVM_IRQ_LINE: { @@ -3448,7 +3449,7 @@ static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, } static int kvm_reboot(struct notifier_block *notifier, unsigned long val, - void *v) + void *v) { if (val == SYS_RESTART) { /* @@ -3655,7 +3656,7 @@ int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, r = misc_register(&kvm_dev); if (r) { - printk (KERN_ERR "kvm: misc device register failed\n"); + printk(KERN_ERR "kvm: misc device register failed\n"); goto out_free; } @@ -3683,6 +3684,7 @@ out: kvm_x86_ops = NULL; return r; } +EXPORT_SYMBOL_GPL(kvm_init_x86); void kvm_exit_x86(void) { @@ -3696,6 +3698,7 @@ void kvm_exit_x86(void) kvm_x86_ops->hardware_unsetup(); kvm_x86_ops = NULL; } +EXPORT_SYMBOL_GPL(kvm_exit_x86); static __init int kvm_init(void) { @@ -3710,7 +3713,9 @@ static __init int kvm_init(void) kvm_init_msr_list(); - if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) { + bad_page = alloc_page(GFP_KERNEL); + + if (bad_page == NULL) { r = -ENOMEM; goto out; } @@ -3736,6 +3741,3 @@ static __exit void kvm_exit(void) module_init(kvm_init) module_exit(kvm_exit) - -EXPORT_SYMBOL_GPL(kvm_init_x86); -EXPORT_SYMBOL_GPL(kvm_exit_x86); diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index 554e73ad33f0..e15b42e48862 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -906,8 +906,7 @@ static int __apic_timer_fn(struct kvm_lapic *apic) wait_queue_head_t *q = &apic->vcpu->wq; atomic_inc(&apic->timer.pending); - if (waitqueue_active(q)) - { + if (waitqueue_active(q)) { apic->vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; wake_up_interruptible(q); } diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 6cda1feb9a95..ece0aa4e4c9f 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -90,7 +90,8 @@ static int dbg = 1; #define PT32_DIR_PSE36_SIZE 4 #define PT32_DIR_PSE36_SHIFT 13 -#define PT32_DIR_PSE36_MASK (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT) +#define PT32_DIR_PSE36_MASK \ + (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT) #define PT_FIRST_AVAIL_BITS_SHIFT 9 @@ -103,7 +104,7 @@ static int dbg = 1; #define PT64_LEVEL_BITS 9 #define PT64_LEVEL_SHIFT(level) \ - ( PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS ) + (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS) #define PT64_LEVEL_MASK(level) \ (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level)) @@ -115,7 +116,7 @@ static int dbg = 1; #define PT32_LEVEL_BITS 10 #define PT32_LEVEL_SHIFT(level) \ - ( PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS ) + (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS) #define PT32_LEVEL_MASK(level) \ (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level)) @@ -1489,7 +1490,8 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, printk(KERN_ERR "xx audit error: (%s) levels %d" " gva %lx gpa %llx hpa %llx ent %llx %d\n", audit_msg, vcpu->mmu.root_level, - va, gpa, hpa, ent, is_shadow_present_pte(ent)); + va, gpa, hpa, ent, + is_shadow_present_pte(ent)); else if (ent == shadow_notrap_nonpresent_pte && !is_error_hpa(hpa)) printk(KERN_ERR "audit: (%s) notrap shadow," diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index fbe595f880af..447d2c31f0cb 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -163,7 +163,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, walker->page = pfn_to_page(paddr >> PAGE_SHIFT); walker->table = kmap_atomic(walker->page, KM_USER0); --walker->level; - walker->table_gfn[walker->level - 1 ] = table_gfn; + walker->table_gfn[walker->level - 1] = table_gfn; pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, walker->level - 1, table_gfn); } diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index fb2e591d5397..7b21576b62bc 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -229,12 +229,11 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) printk(KERN_DEBUG "%s: NOP\n", __FUNCTION__); return; } - if (svm->next_rip - svm->vmcb->save.rip > MAX_INST_SIZE) { + if (svm->next_rip - svm->vmcb->save.rip > MAX_INST_SIZE) printk(KERN_ERR "%s: ip 0x%llx next 0x%llx\n", __FUNCTION__, svm->vmcb->save.rip, svm->next_rip); - } vcpu->rip = svm->vmcb->save.rip = svm->next_rip; svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK; @@ -312,7 +311,7 @@ static void svm_hardware_enable(void *garbage) svm_data->next_asid = svm_data->max_asid + 1; svm_features = cpuid_edx(SVM_CPUID_FUNC); - asm volatile ( "sgdt %0" : "=m"(gdt_descr) ); + asm volatile ("sgdt %0" : "=m"(gdt_descr)); gdt = (struct desc_struct *)gdt_descr.address; svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS); @@ -544,8 +543,7 @@ static void init_vmcb(struct vmcb *vmcb) init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16); save->efer = MSR_EFER_SVME_MASK; - - save->dr6 = 0xffff0ff0; + save->dr6 = 0xffff0ff0; save->dr7 = 0x400; save->rflags = 2; save->rip = 0x0000fff0; @@ -783,7 +781,7 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) svm->vmcb->save.efer |= KVM_EFER_LMA | KVM_EFER_LME; } - if (is_paging(vcpu) && !(cr0 & X86_CR0_PG) ) { + if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) { vcpu->shadow_efer &= ~KVM_EFER_LMA; svm->vmcb->save.efer &= ~(KVM_EFER_LMA | KVM_EFER_LME); } @@ -1010,7 +1008,7 @@ static int shutdown_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { - u32 io_info = svm->vmcb->control.exit_info_1; //address size bug? + u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ int size, down, in, string, rep; unsigned port; @@ -1316,7 +1314,7 @@ static void reload_tss(struct kvm_vcpu *vcpu) int cpu = raw_smp_processor_id(); struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu); - svm_data->tss_desc->type = 9; //available 32/64-bit TSS + svm_data->tss_desc->type = 9; /* available 32/64-bit TSS */ load_TR_desc(); } @@ -1434,9 +1432,9 @@ static void do_interrupt_requests(struct kvm_vcpu *vcpu, * Interrupts blocked. Wait for unblock. */ if (!svm->vcpu.interrupt_window_open && - (svm->vcpu.irq_summary || kvm_run->request_interrupt_window)) { + (svm->vcpu.irq_summary || kvm_run->request_interrupt_window)) control->intercept |= 1ULL << INTERCEPT_VINTR; - } else + else control->intercept &= ~(1ULL << INTERCEPT_VINTR); } @@ -1581,23 +1579,23 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) : : [svm]"a"(svm), [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)), - [rbx]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RBX])), - [rcx]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RCX])), - [rdx]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RDX])), - [rsi]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RSI])), - [rdi]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RDI])), - [rbp]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RBP])) + [rbx]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RBX])), + [rcx]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RCX])), + [rdx]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RDX])), + [rsi]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RSI])), + [rdi]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RDI])), + [rbp]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RBP])) #ifdef CONFIG_X86_64 - ,[r8 ]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R8])), - [r9 ]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R9 ])), - [r10]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R10])), - [r11]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R11])), - [r12]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R12])), - [r13]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R13])), - [r14]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R14])), - [r15]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R15])) + , [r8]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R8])), + [r9]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R9])), + [r10]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R10])), + [r11]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R11])), + [r12]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R12])), + [r13]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R13])), + [r14]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R14])), + [r15]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R15])) #endif - : "cc", "memory" ); + : "cc", "memory"); if ((svm->vmcb->save.dr7 & 0xff)) load_db_regs(svm->host_db_regs); diff --git a/drivers/kvm/svm.h b/drivers/kvm/svm.h index 3b1b0f35b6cb..5fa277c0187c 100644 --- a/drivers/kvm/svm.h +++ b/drivers/kvm/svm.h @@ -311,7 +311,7 @@ struct __attribute__ ((__packed__)) vmcb { #define SVM_EXIT_ERR -1 -#define SVM_CR0_SELECTIVE_MASK (1 << 3 | 1) // TS and MP +#define SVM_CR0_SELECTIVE_MASK (1 << 3 | 1) /* TS and MP */ #define SVM_VMLOAD ".byte 0x0f, 0x01, 0xda" #define SVM_VMRUN ".byte 0x0f, 0x01, 0xd8" diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 7b742901e783..6955580bb69e 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -62,7 +62,7 @@ struct vcpu_vmx { int gs_ldt_reload_needed; int fs_reload_needed; int guest_efer_loaded; - }host_state; + } host_state; }; @@ -271,7 +271,7 @@ static void vmcs_writel(unsigned long field, unsigned long value) u8 error; asm volatile (ASM_VMX_VMWRITE_RAX_RDX "; setna %0" - : "=q"(error) : "a"(value), "d"(field) : "cc" ); + : "=q"(error) : "a"(value), "d"(field) : "cc"); if (unlikely(error)) vmwrite_error(field, value); } @@ -415,10 +415,10 @@ static void vmx_save_host_state(struct kvm_vcpu *vcpu) #endif #ifdef CONFIG_X86_64 - if (is_long_mode(&vmx->vcpu)) { + if (is_long_mode(&vmx->vcpu)) save_msrs(vmx->host_msrs + vmx->msr_offset_kernel_gs_base, 1); - } + #endif load_msrs(vmx->guest_msrs, vmx->save_nmsrs); load_transition_efer(vmx); @@ -845,7 +845,7 @@ static int vmx_get_irq(struct kvm_vcpu *vcpu) if (is_external_interrupt(idtv_info_field)) return idtv_info_field & VECTORING_INFO_VECTOR_MASK; else - printk("pending exception: not handled yet\n"); + printk(KERN_DEBUG "pending exception: not handled yet\n"); } return -1; } @@ -893,7 +893,7 @@ static void hardware_disable(void *garbage) } static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt, - u32 msr, u32* result) + u32 msr, u32 *result) { u32 vmx_msr_low, vmx_msr_high; u32 ctl = ctl_min | ctl_opt; @@ -1102,7 +1102,7 @@ static void enter_pmode(struct kvm_vcpu *vcpu) vmcs_write32(GUEST_CS_AR_BYTES, 0x9b); } -static gva_t rmode_tss_base(struct kvm* kvm) +static gva_t rmode_tss_base(struct kvm *kvm) { gfn_t base_gfn = kvm->memslots[0].base_gfn + kvm->memslots[0].npages - 3; return base_gfn << PAGE_SHIFT; @@ -1385,7 +1385,7 @@ static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) vmcs_writel(GUEST_GDTR_BASE, dt->base); } -static int init_rmode_tss(struct kvm* kvm) +static int init_rmode_tss(struct kvm *kvm) { gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT; u16 data = 0; @@ -1494,7 +1494,7 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) vmcs_writel(GUEST_RIP, 0); vmcs_writel(GUEST_RSP, 0); - //todo: dr0 = dr1 = dr2 = dr3 = 0; dr6 = 0xffff0ff0 + /* todo: dr0 = dr1 = dr2 = dr3 = 0; dr6 = 0xffff0ff0 */ vmcs_writel(GUEST_DR7, 0x400); vmcs_writel(GUEST_GDTR_BASE, 0); @@ -1561,7 +1561,7 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) get_idt(&dt); vmcs_writel(HOST_IDTR_BASE, dt.base); /* 22.2.4 */ - asm ("mov $.Lkvm_vmx_return, %0" : "=r"(kvm_vmx_return)); + asm("mov $.Lkvm_vmx_return, %0" : "=r"(kvm_vmx_return)); vmcs_writel(HOST_RIP, kvm_vmx_return); /* 22.2.5 */ vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); @@ -1613,7 +1613,7 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) vmcs_writel(CR4_GUEST_HOST_MASK, KVM_GUEST_CR4_MASK); vmx->vcpu.cr0 = 0x60000010; - vmx_set_cr0(&vmx->vcpu, vmx->vcpu.cr0); // enter rmode + vmx_set_cr0(&vmx->vcpu, vmx->vcpu.cr0); /* enter rmode */ vmx_set_cr4(&vmx->vcpu, 0); #ifdef CONFIG_X86_64 vmx_set_efer(&vmx->vcpu, 0); @@ -1644,7 +1644,7 @@ static void inject_rmode_irq(struct kvm_vcpu *vcpu, int irq) u16 sp = vmcs_readl(GUEST_RSP); u32 ss_limit = vmcs_read32(GUEST_SS_LIMIT); - if (sp > ss_limit || sp < 6 ) { + if (sp > ss_limit || sp < 6) { vcpu_printf(vcpu, "%s: #SS, rsp 0x%lx ss 0x%lx limit 0x%x\n", __FUNCTION__, vmcs_readl(GUEST_RSP), @@ -1664,15 +1664,18 @@ static void inject_rmode_irq(struct kvm_vcpu *vcpu, int irq) ip = vmcs_readl(GUEST_RIP); - if (emulator_write_emulated(ss_base + sp - 2, &flags, 2, vcpu) != X86EMUL_CONTINUE || - emulator_write_emulated(ss_base + sp - 4, &cs, 2, vcpu) != X86EMUL_CONTINUE || - emulator_write_emulated(ss_base + sp - 6, &ip, 2, vcpu) != X86EMUL_CONTINUE) { + if (emulator_write_emulated( + ss_base + sp - 2, &flags, 2, vcpu) != X86EMUL_CONTINUE || + emulator_write_emulated( + ss_base + sp - 4, &cs, 2, vcpu) != X86EMUL_CONTINUE || + emulator_write_emulated( + ss_base + sp - 6, &ip, 2, vcpu) != X86EMUL_CONTINUE) { vcpu_printf(vcpu, "%s: write guest err\n", __FUNCTION__); return; } vmcs_writel(GUEST_RFLAGS, flags & - ~( X86_EFLAGS_IF | X86_EFLAGS_AC | X86_EFLAGS_TF)); + ~(X86_EFLAGS_IF | X86_EFLAGS_AC | X86_EFLAGS_TF)); vmcs_write16(GUEST_CS_SELECTOR, ent[1]) ; vmcs_writel(GUEST_CS_BASE, ent[1] << 4); vmcs_writel(GUEST_RIP, ent[0]); @@ -1777,10 +1780,9 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) intr_info = vmcs_read32(VM_EXIT_INTR_INFO); if ((vect_info & VECTORING_INFO_VALID_MASK) && - !is_page_fault(intr_info)) { + !is_page_fault(intr_info)) printk(KERN_ERR "%s: unexpected, vectoring info 0x%x " "intr info 0x%x\n", __FUNCTION__, vect_info, intr_info); - } if (!irqchip_in_kernel(vcpu->kvm) && is_external_interrupt(vect_info)) { int irq = vect_info & VECTORING_INFO_VECTOR_MASK; @@ -1831,7 +1833,7 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) case EMULATE_DO_MMIO: ++vcpu->stat.mmio_exits; return 0; - case EMULATE_FAIL: + case EMULATE_FAIL: kvm_report_emulation_failure(vcpu, "pagetable"); break; default: @@ -1849,7 +1851,8 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) return 1; } - if ((intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK)) == (INTR_TYPE_EXCEPTION | 1)) { + if ((intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK)) == + (INTR_TYPE_EXCEPTION | 1)) { kvm_run->exit_reason = KVM_EXIT_DEBUG; return 0; } @@ -2138,8 +2141,8 @@ static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) return 0; } - if ( (vectoring_info & VECTORING_INFO_VALID_MASK) && - exit_reason != EXIT_REASON_EXCEPTION_NMI ) + if ((vectoring_info & VECTORING_INFO_VALID_MASK) && + exit_reason != EXIT_REASON_EXCEPTION_NMI) printk(KERN_WARNING "%s: unexpected, valid vectoring info and " "exit reason is 0x%x\n", __FUNCTION__, exit_reason); if (exit_reason < kvm_vmx_max_exit_handlers @@ -2238,7 +2241,7 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) */ vmcs_writel(HOST_CR0, read_cr0()); - asm ( + asm( /* Store host registers */ #ifdef CONFIG_X86_64 "push %%rax; push %%rbx; push %%rdx;" @@ -2342,8 +2345,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) [rdi]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RDI])), [rbp]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RBP])), #ifdef CONFIG_X86_64 - [r8 ]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R8 ])), - [r9 ]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R9 ])), + [r8]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R8])), + [r9]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R9])), [r10]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R10])), [r11]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R11])), [r12]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R12])), @@ -2352,11 +2355,12 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) [r15]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R15])), #endif [cr2]"i"(offsetof(struct kvm_vcpu, cr2)) - : "cc", "memory" ); + : "cc", "memory"); - vcpu->interrupt_window_open = (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0; + vcpu->interrupt_window_open = + (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0; - asm ("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS)); + asm("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS)); vmx->launched = 1; intr_info = vmcs_read32(VM_EXIT_INTR_INFO); diff --git a/drivers/kvm/vmx.h b/drivers/kvm/vmx.h index fd4e14666088..270d477a2aa6 100644 --- a/drivers/kvm/vmx.h +++ b/drivers/kvm/vmx.h @@ -234,9 +234,9 @@ enum vmcs_field { /* * Exit Qualifications for MOV for Control Register Access */ -#define CONTROL_REG_ACCESS_NUM 0x7 /* 2:0, number of control register */ +#define CONTROL_REG_ACCESS_NUM 0x7 /* 2:0, number of control reg.*/ #define CONTROL_REG_ACCESS_TYPE 0x30 /* 5:4, access type */ -#define CONTROL_REG_ACCESS_REG 0xf00 /* 10:8, general purpose register */ +#define CONTROL_REG_ACCESS_REG 0xf00 /* 10:8, general purpose reg. */ #define LMSW_SOURCE_DATA_SHIFT 16 #define LMSW_SOURCE_DATA (0xFFFF << LMSW_SOURCE_DATA_SHIFT) /* 16:31 lmsw source */ #define REG_EAX (0 << 8) @@ -259,11 +259,11 @@ enum vmcs_field { /* * Exit Qualifications for MOV for Debug Register Access */ -#define DEBUG_REG_ACCESS_NUM 0x7 /* 2:0, number of debug register */ +#define DEBUG_REG_ACCESS_NUM 0x7 /* 2:0, number of debug reg. */ #define DEBUG_REG_ACCESS_TYPE 0x10 /* 4, direction of access */ #define TYPE_MOV_TO_DR (0 << 4) #define TYPE_MOV_FROM_DR (1 << 4) -#define DEBUG_REG_ACCESS_REG 0xf00 /* 11:8, general purpose register */ +#define DEBUG_REG_ACCESS_REG 0xf00 /* 11:8, general purpose reg. */ /* segment AR */ diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index e294d8409571..75fd23bade9c 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -23,7 +23,7 @@ #include #include #include -#define DPRINTF(_f, _a ...) printf( _f , ## _a ) +#define DPRINTF(_f, _a ...) printf(_f , ## _a) #else #include "kvm.h" #define DPRINTF(x...) do {} while (0) @@ -285,21 +285,21 @@ static u16 twobyte_table[256] = { switch ((_dst).bytes) { \ case 2: \ __asm__ __volatile__ ( \ - _PRE_EFLAGS("0","4","2") \ + _PRE_EFLAGS("0", "4", "2") \ _op"w %"_wx"3,%1; " \ - _POST_EFLAGS("0","4","2") \ + _POST_EFLAGS("0", "4", "2") \ : "=m" (_eflags), "=m" ((_dst).val), \ "=&r" (_tmp) \ - : _wy ((_src).val), "i" (EFLAGS_MASK) ); \ + : _wy ((_src).val), "i" (EFLAGS_MASK)); \ break; \ case 4: \ __asm__ __volatile__ ( \ - _PRE_EFLAGS("0","4","2") \ + _PRE_EFLAGS("0", "4", "2") \ _op"l %"_lx"3,%1; " \ - _POST_EFLAGS("0","4","2") \ + _POST_EFLAGS("0", "4", "2") \ : "=m" (_eflags), "=m" ((_dst).val), \ "=&r" (_tmp) \ - : _ly ((_src).val), "i" (EFLAGS_MASK) ); \ + : _ly ((_src).val), "i" (EFLAGS_MASK)); \ break; \ case 8: \ __emulate_2op_8byte(_op, _src, _dst, \ @@ -311,16 +311,15 @@ static u16 twobyte_table[256] = { #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \ do { \ unsigned long _tmp; \ - switch ( (_dst).bytes ) \ - { \ + switch ((_dst).bytes) { \ case 1: \ __asm__ __volatile__ ( \ - _PRE_EFLAGS("0","4","2") \ + _PRE_EFLAGS("0", "4", "2") \ _op"b %"_bx"3,%1; " \ - _POST_EFLAGS("0","4","2") \ + _POST_EFLAGS("0", "4", "2") \ : "=m" (_eflags), "=m" ((_dst).val), \ "=&r" (_tmp) \ - : _by ((_src).val), "i" (EFLAGS_MASK) ); \ + : _by ((_src).val), "i" (EFLAGS_MASK)); \ break; \ default: \ __emulate_2op_nobyte(_op, _src, _dst, _eflags, \ @@ -349,34 +348,33 @@ static u16 twobyte_table[256] = { do { \ unsigned long _tmp; \ \ - switch ( (_dst).bytes ) \ - { \ + switch ((_dst).bytes) { \ case 1: \ __asm__ __volatile__ ( \ - _PRE_EFLAGS("0","3","2") \ + _PRE_EFLAGS("0", "3", "2") \ _op"b %1; " \ - _POST_EFLAGS("0","3","2") \ + _POST_EFLAGS("0", "3", "2") \ : "=m" (_eflags), "=m" ((_dst).val), \ "=&r" (_tmp) \ - : "i" (EFLAGS_MASK) ); \ + : "i" (EFLAGS_MASK)); \ break; \ case 2: \ __asm__ __volatile__ ( \ - _PRE_EFLAGS("0","3","2") \ + _PRE_EFLAGS("0", "3", "2") \ _op"w %1; " \ - _POST_EFLAGS("0","3","2") \ + _POST_EFLAGS("0", "3", "2") \ : "=m" (_eflags), "=m" ((_dst).val), \ "=&r" (_tmp) \ - : "i" (EFLAGS_MASK) ); \ + : "i" (EFLAGS_MASK)); \ break; \ case 4: \ __asm__ __volatile__ ( \ - _PRE_EFLAGS("0","3","2") \ + _PRE_EFLAGS("0", "3", "2") \ _op"l %1; " \ - _POST_EFLAGS("0","3","2") \ + _POST_EFLAGS("0", "3", "2") \ : "=m" (_eflags), "=m" ((_dst).val), \ "=&r" (_tmp) \ - : "i" (EFLAGS_MASK) ); \ + : "i" (EFLAGS_MASK)); \ break; \ case 8: \ __emulate_1op_8byte(_op, _dst, _eflags); \ @@ -389,21 +387,21 @@ static u16 twobyte_table[256] = { #define __emulate_2op_8byte(_op, _src, _dst, _eflags, _qx, _qy) \ do { \ __asm__ __volatile__ ( \ - _PRE_EFLAGS("0","4","2") \ + _PRE_EFLAGS("0", "4", "2") \ _op"q %"_qx"3,%1; " \ - _POST_EFLAGS("0","4","2") \ + _POST_EFLAGS("0", "4", "2") \ : "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \ - : _qy ((_src).val), "i" (EFLAGS_MASK) ); \ + : _qy ((_src).val), "i" (EFLAGS_MASK)); \ } while (0) #define __emulate_1op_8byte(_op, _dst, _eflags) \ do { \ __asm__ __volatile__ ( \ - _PRE_EFLAGS("0","3","2") \ + _PRE_EFLAGS("0", "3", "2") \ _op"q %1; " \ - _POST_EFLAGS("0","3","2") \ + _POST_EFLAGS("0", "3", "2") \ : "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \ - : "i" (EFLAGS_MASK) ); \ + : "i" (EFLAGS_MASK)); \ } while (0) #elif defined(__i386__) @@ -415,8 +413,8 @@ static u16 twobyte_table[256] = { #define insn_fetch(_type, _size, _eip) \ ({ unsigned long _x; \ rc = ops->read_std((unsigned long)(_eip) + ctxt->cs_base, &_x, \ - (_size), ctxt->vcpu); \ - if ( rc != 0 ) \ + (_size), ctxt->vcpu); \ + if (rc != 0) \ goto done; \ (_eip) += (_size); \ (_type)_x; \ @@ -780,7 +778,7 @@ done_prefixes: } if (c->ad_bytes != 8) c->modrm_ea = (u32)c->modrm_ea; - modrm_done: +modrm_done: ; } @@ -828,10 +826,9 @@ done_prefixes: c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; /* Don't fetch the address for invlpg: it could be unmapped. */ - if (c->twobyte && c->b == 0x01 - && c->modrm_reg == 7) + if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7) break; - srcmem_common: + srcmem_common: /* * For instructions with a ModR/M byte, switch to register * access if Mod = 3. @@ -1175,10 +1172,11 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) if (c->src.type == OP_MEM) { c->src.ptr = (unsigned long *)cr2; c->src.val = 0; - if ((rc = ops->read_emulated((unsigned long)c->src.ptr, - &c->src.val, - c->src.bytes, - ctxt->vcpu)) != 0) + rc = ops->read_emulated((unsigned long)c->src.ptr, + &c->src.val, + c->src.bytes, + ctxt->vcpu); + if (rc != 0) goto done; c->src.orig_val = c->src.val; } From 6fc138d2278078990f597cb1f62fde9e5b458f96 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Tue, 9 Oct 2007 19:20:39 +0200 Subject: [PATCH 027/249] KVM: Support assigning userspace memory to the guest Instead of having the kernel allocate memory to the guest, let userspace allocate it and pass the address to the kernel. This is required for s390 support, but also enables features like memory sharing and using hugetlbfs backed memory. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/kvm_main.c | 81 ++++++++++++++++++++++++++++++++++++------ include/linux/kvm.h | 12 +++++++ 3 files changed, 83 insertions(+), 11 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index ec5b498945ae..3eaed4dafec7 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -408,6 +408,7 @@ struct kvm_memory_slot { struct page **phys_mem; unsigned long *rmap; unsigned long *dirty_bitmap; + int user_alloc; /* user allocated memory */ }; struct kvm { diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index a1983d2d5b8f..22b143feb66d 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -300,19 +301,40 @@ static struct kvm *kvm_create_vm(void) return kvm; } +static void kvm_free_userspace_physmem(struct kvm_memory_slot *free) +{ + int i; + + for (i = 0; i < free->npages; ++i) { + if (free->phys_mem[i]) { + if (!PageReserved(free->phys_mem[i])) + SetPageDirty(free->phys_mem[i]); + page_cache_release(free->phys_mem[i]); + } + } +} + +static void kvm_free_kernel_physmem(struct kvm_memory_slot *free) +{ + int i; + + for (i = 0; i < free->npages; ++i) + if (free->phys_mem[i]) + __free_page(free->phys_mem[i]); +} + /* * Free any memory in @free but not in @dont. */ static void kvm_free_physmem_slot(struct kvm_memory_slot *free, struct kvm_memory_slot *dont) { - int i; - if (!dont || free->phys_mem != dont->phys_mem) if (free->phys_mem) { - for (i = 0; i < free->npages; ++i) - if (free->phys_mem[i]) - __free_page(free->phys_mem[i]); + if (free->user_alloc) + kvm_free_userspace_physmem(free); + else + kvm_free_kernel_physmem(free); vfree(free->phys_mem); } if (!dont || free->rmap != dont->rmap) @@ -652,7 +674,9 @@ EXPORT_SYMBOL_GPL(fx_init); * Discontiguous memory is allowed, mostly for framebuffers. */ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, - struct kvm_memory_region *mem) + struct + kvm_userspace_memory_region *mem, + int user_alloc) { int r; gfn_t base_gfn; @@ -728,11 +752,27 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, memset(new.phys_mem, 0, npages * sizeof(struct page *)); memset(new.rmap, 0, npages * sizeof(*new.rmap)); - for (i = 0; i < npages; ++i) { - new.phys_mem[i] = alloc_page(GFP_HIGHUSER - | __GFP_ZERO); - if (!new.phys_mem[i]) + if (user_alloc) { + unsigned long pages_num; + + new.user_alloc = 1; + down_read(¤t->mm->mmap_sem); + + pages_num = get_user_pages(current, current->mm, + mem->userspace_addr, + npages, 1, 1, new.phys_mem, + NULL); + + up_read(¤t->mm->mmap_sem); + if (pages_num != npages) goto out_unlock; + } else { + for (i = 0; i < npages; ++i) { + new.phys_mem[i] = alloc_page(GFP_HIGHUSER + | __GFP_ZERO); + if (!new.phys_mem[i]) + goto out_unlock; + } } } @@ -3108,11 +3148,29 @@ static long kvm_vm_ioctl(struct file *filp, break; case KVM_SET_MEMORY_REGION: { struct kvm_memory_region kvm_mem; + struct kvm_userspace_memory_region kvm_userspace_mem; r = -EFAULT; if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem)) goto out; - r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_mem); + kvm_userspace_mem.slot = kvm_mem.slot; + kvm_userspace_mem.flags = kvm_mem.flags; + kvm_userspace_mem.guest_phys_addr = kvm_mem.guest_phys_addr; + kvm_userspace_mem.memory_size = kvm_mem.memory_size; + r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 0); + if (r) + goto out; + break; + } + case KVM_SET_USER_MEMORY_REGION: { + struct kvm_userspace_memory_region kvm_userspace_mem; + + r = -EFAULT; + if (copy_from_user(&kvm_userspace_mem, argp, + sizeof kvm_userspace_mem)) + goto out; + + r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1); if (r) goto out; break; @@ -3332,6 +3390,7 @@ static long kvm_dev_ioctl(struct file *filp, case KVM_CAP_IRQCHIP: case KVM_CAP_HLT: case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: + case KVM_CAP_USER_MEMORY: r = 1; break; default: diff --git a/include/linux/kvm.h b/include/linux/kvm.h index d2fd973d81fe..971f465e26fa 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -23,6 +23,15 @@ struct kvm_memory_region { __u64 memory_size; /* bytes */ }; +/* for KVM_SET_USER_MEMORY_REGION */ +struct kvm_userspace_memory_region { + __u32 slot; + __u32 flags; + __u64 guest_phys_addr; + __u64 memory_size; /* bytes */ + __u64 userspace_addr; /* start of the userspace allocated memory */ +}; + /* for kvm_memory_region::flags */ #define KVM_MEM_LOG_DIRTY_PAGES 1UL @@ -348,6 +357,7 @@ struct kvm_signal_mask { #define KVM_CAP_IRQCHIP 0 #define KVM_CAP_HLT 1 #define KVM_CAP_MMU_SHADOW_CACHE_CONTROL 2 +#define KVM_CAP_USER_MEMORY 3 /* * ioctls for VM fds @@ -355,6 +365,8 @@ struct kvm_signal_mask { #define KVM_SET_MEMORY_REGION _IOW(KVMIO, 0x40, struct kvm_memory_region) #define KVM_SET_NR_MMU_PAGES _IO(KVMIO, 0x44) #define KVM_GET_NR_MMU_PAGES _IO(KVMIO, 0x45) +#define KVM_SET_USER_MEMORY_REGION _IOW(KVMIO, 0x46,\ + struct kvm_userspace_memory_region) /* * KVM_CREATE_VCPU receives as a parameter the vcpu slot, and returns * a vcpu fd. From 043405e10001fe7aae60c46a57189515f13a6468 Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Wed, 10 Oct 2007 17:16:19 +0200 Subject: [PATCH 028/249] KVM: Move x86 msr handling to new files x86.[ch] Signed-off-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/Makefile | 2 +- drivers/kvm/kvm.h | 4 ++ drivers/kvm/kvm_main.c | 69 ++-------------------------- drivers/kvm/x86.c | 102 +++++++++++++++++++++++++++++++++++++++++ drivers/kvm/x86.h | 16 +++++++ 5 files changed, 126 insertions(+), 67 deletions(-) create mode 100644 drivers/kvm/x86.c create mode 100644 drivers/kvm/x86.h diff --git a/drivers/kvm/Makefile b/drivers/kvm/Makefile index e5a8f4d3e973..cf18ad46e987 100644 --- a/drivers/kvm/Makefile +++ b/drivers/kvm/Makefile @@ -2,7 +2,7 @@ # Makefile for Kernel-based Virtual Machine module # -kvm-objs := kvm_main.o mmu.o x86_emulate.o i8259.o irq.o lapic.o ioapic.o +kvm-objs := kvm_main.o x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o ioapic.o obj-$(CONFIG_KVM) += kvm.o kvm-intel-objs = vmx.o obj-$(CONFIG_KVM_INTEL) += kvm-intel.o diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 3eaed4dafec7..9c9c1d7f5b30 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -653,6 +653,10 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); int kvm_fix_hypercall(struct kvm_vcpu *vcpu); +long kvm_arch_dev_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg); +__init void kvm_arch_init(void); + static inline void kvm_guest_enter(void) { current->flags |= PF_VCPU; diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 22b143feb66d..ec696887b222 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -16,6 +16,7 @@ */ #include "kvm.h" +#include "x86.h" #include "x86_emulate.h" #include "segment_descriptor.h" #include "irq.h" @@ -2507,43 +2508,6 @@ void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) } EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits); -/* - * List of msr numbers which we expose to userspace through KVM_GET_MSRS - * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. - * - * This list is modified at module load time to reflect the - * capabilities of the host cpu. - */ -static u32 msrs_to_save[] = { - MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, - MSR_K6_STAR, -#ifdef CONFIG_X86_64 - MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR, -#endif - MSR_IA32_TIME_STAMP_COUNTER, -}; - -static unsigned num_msrs_to_save; - -static u32 emulated_msrs[] = { - MSR_IA32_MISC_ENABLE, -}; - -static __init void kvm_init_msr_list(void) -{ - u32 dummy[2]; - unsigned i, j; - - for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) { - if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0) - continue; - if (j < i) - msrs_to_save[j] = msrs_to_save[i]; - j++; - } - num_msrs_to_save = j; -} - /* * Adapt set_msr() to msr_io()'s calling convention */ @@ -3356,33 +3320,6 @@ static long kvm_dev_ioctl(struct file *filp, goto out; r = kvm_dev_ioctl_create_vm(); break; - case KVM_GET_MSR_INDEX_LIST: { - struct kvm_msr_list __user *user_msr_list = argp; - struct kvm_msr_list msr_list; - unsigned n; - - r = -EFAULT; - if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list)) - goto out; - n = msr_list.nmsrs; - msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs); - if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list)) - goto out; - r = -E2BIG; - if (n < num_msrs_to_save) - goto out; - r = -EFAULT; - if (copy_to_user(user_msr_list->indices, &msrs_to_save, - num_msrs_to_save * sizeof(u32))) - goto out; - if (copy_to_user(user_msr_list->indices - + num_msrs_to_save * sizeof(u32), - &emulated_msrs, - ARRAY_SIZE(emulated_msrs) * sizeof(u32))) - goto out; - r = 0; - break; - } case KVM_CHECK_EXTENSION: { int ext = (long)argp; @@ -3406,7 +3343,7 @@ static long kvm_dev_ioctl(struct file *filp, r = 2 * PAGE_SIZE; break; default: - ; + return kvm_arch_dev_ioctl(filp, ioctl, arg); } out: return r; @@ -3770,7 +3707,7 @@ static __init int kvm_init(void) kvm_init_debug(); - kvm_init_msr_list(); + kvm_arch_init(); bad_page = alloc_page(GFP_KERNEL); diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c new file mode 100644 index 000000000000..437902cf178d --- /dev/null +++ b/drivers/kvm/x86.c @@ -0,0 +1,102 @@ +/* + * Kernel-based Virtual Machine driver for Linux + * + * derived from drivers/kvm/kvm_main.c + * + * Copyright (C) 2006 Qumranet, Inc. + * + * Authors: + * Avi Kivity + * Yaniv Kamay + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + */ + +#include "x86.h" + +#include + +/* + * List of msr numbers which we expose to userspace through KVM_GET_MSRS + * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. + * + * This list is modified at module load time to reflect the + * capabilities of the host cpu. + */ +static u32 msrs_to_save[] = { + MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, + MSR_K6_STAR, +#ifdef CONFIG_X86_64 + MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR, +#endif + MSR_IA32_TIME_STAMP_COUNTER, +}; + +static unsigned num_msrs_to_save; + +static u32 emulated_msrs[] = { + MSR_IA32_MISC_ENABLE, +}; + +long kvm_arch_dev_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg) +{ + void __user *argp = (void __user *)arg; + long r; + + switch (ioctl) { + case KVM_GET_MSR_INDEX_LIST: { + struct kvm_msr_list __user *user_msr_list = argp; + struct kvm_msr_list msr_list; + unsigned n; + + r = -EFAULT; + if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list)) + goto out; + n = msr_list.nmsrs; + msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs); + if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list)) + goto out; + r = -E2BIG; + if (n < num_msrs_to_save) + goto out; + r = -EFAULT; + if (copy_to_user(user_msr_list->indices, &msrs_to_save, + num_msrs_to_save * sizeof(u32))) + goto out; + if (copy_to_user(user_msr_list->indices + + num_msrs_to_save * sizeof(u32), + &emulated_msrs, + ARRAY_SIZE(emulated_msrs) * sizeof(u32))) + goto out; + r = 0; + break; + } + default: + r = -EINVAL; + } +out: + return r; +} + +static __init void kvm_init_msr_list(void) +{ + u32 dummy[2]; + unsigned i, j; + + for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) { + if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0) + continue; + if (j < i) + msrs_to_save[j] = msrs_to_save[i]; + j++; + } + num_msrs_to_save = j; +} + +__init void kvm_arch_init(void) +{ + kvm_init_msr_list(); +} diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h new file mode 100644 index 000000000000..1e2f71bd805d --- /dev/null +++ b/drivers/kvm/x86.h @@ -0,0 +1,16 @@ +#/* + * Kernel-based Virtual Machine driver for Linux + * + * This header defines architecture specific interfaces, x86 version + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + */ + +#ifndef KVM_X86_H +#define KVM_X86_H + +#include "kvm.h" + +#endif From f67a46f4aa1212b38696ac6b6a82b4323cea61aa Mon Sep 17 00:00:00 2001 From: Anthony Liguori Date: Wed, 10 Oct 2007 19:25:50 -0500 Subject: [PATCH 029/249] KVM: MMU: Clean up MMU functions to take struct kvm when appropriate Some of the MMU functions take a struct kvm_vcpu even though they affect all VCPUs. This patch cleans up some of them to instead take a struct kvm. This makes things a bit more clear. The main thing that was confusing me was whether certain functions need to be called on all VCPUs. Signed-off-by: Anthony Liguori Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 18 +++++++++--------- drivers/kvm/paging_tmpl.h | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index ece0aa4e4c9f..a5ca9457e929 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -606,7 +606,7 @@ static void mmu_page_remove_parent_pte(struct kvm_mmu_page *page, BUG(); } -static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm_vcpu *vcpu, +static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn) { unsigned index; @@ -616,7 +616,7 @@ static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm_vcpu *vcpu, pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn); index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; - bucket = &vcpu->kvm->mmu_page_hash[index]; + bucket = &kvm->mmu_page_hash[index]; hlist_for_each_entry(page, node, bucket, hash_link) if (page->gfn == gfn && !page->role.metaphysical) { pgprintk("%s: found role %x\n", @@ -782,7 +782,7 @@ void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages) kvm->n_alloc_mmu_pages = kvm_nr_mmu_pages; } -static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn) +static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) { unsigned index; struct hlist_head *bucket; @@ -793,25 +793,25 @@ static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn) pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn); r = 0; index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; - bucket = &vcpu->kvm->mmu_page_hash[index]; + bucket = &kvm->mmu_page_hash[index]; hlist_for_each_entry_safe(page, node, n, bucket, hash_link) if (page->gfn == gfn && !page->role.metaphysical) { pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn, page->role.word); - kvm_mmu_zap_page(vcpu->kvm, page); + kvm_mmu_zap_page(kvm, page); r = 1; } return r; } -static void mmu_unshadow(struct kvm_vcpu *vcpu, gfn_t gfn) +static void mmu_unshadow(struct kvm *kvm, gfn_t gfn) { struct kvm_mmu_page *page; - while ((page = kvm_mmu_lookup_page(vcpu, gfn)) != NULL) { + while ((page = kvm_mmu_lookup_page(kvm, gfn)) != NULL) { pgprintk("%s: zap %lx %x\n", __FUNCTION__, gfn, page->role.word); - kvm_mmu_zap_page(vcpu->kvm, page); + kvm_mmu_zap_page(kvm, page); } } @@ -1299,7 +1299,7 @@ int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) { gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva); - return kvm_mmu_unprotect_page(vcpu, gpa >> PAGE_SHIFT); + return kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT); } void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 447d2c31f0cb..4f6edf85d13f 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -268,11 +268,11 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, spte |= PT_WRITABLE_MASK; if (user_fault) { - mmu_unshadow(vcpu, gfn); + mmu_unshadow(vcpu->kvm, gfn); goto unshadowed; } - shadow = kvm_mmu_lookup_page(vcpu, gfn); + shadow = kvm_mmu_lookup_page(vcpu->kvm, gfn); if (shadow) { pgprintk("%s: found shadow page for %lx, marking ro\n", __FUNCTION__, gfn); From 4a4c99248713e878e1e2880015d01049aec805f3 Mon Sep 17 00:00:00 2001 From: Anthony Liguori Date: Wed, 10 Oct 2007 20:08:41 -0500 Subject: [PATCH 030/249] KVM: MMU: More struct kvm_vcpu -> struct kvm cleanups This time, the biggest change is gpa_to_hpa. The translation of GPA to HPA does not depend on the VCPU state unlike GVA to GPA so there's no need to pass in the kvm_vcpu. Signed-off-by: Anthony Liguori Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 2 +- drivers/kvm/mmu.c | 26 +++++++++++++------------- drivers/kvm/paging_tmpl.h | 6 +++--- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 9c9c1d7f5b30..d56962d49aa6 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -554,7 +554,7 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot); void kvm_mmu_zap_all(struct kvm *kvm); void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages); -hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa); +hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa); #define HPA_MSB ((sizeof(hpa_t) * 8) - 1) #define HPA_ERR_MASK ((hpa_t)1 << HPA_MSB) static inline int is_error_hpa(hpa_t hpa) { return hpa >> HPA_MSB; } diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index a5ca9457e929..d046ba807763 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -451,14 +451,14 @@ static void rmap_remove(struct kvm *kvm, u64 *spte) } } -static void rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn) +static void rmap_write_protect(struct kvm *kvm, u64 gfn) { struct kvm_rmap_desc *desc; unsigned long *rmapp; u64 *spte; - gfn = unalias_gfn(vcpu->kvm, gfn); - rmapp = gfn_to_rmap(vcpu->kvm, gfn); + gfn = unalias_gfn(kvm, gfn); + rmapp = gfn_to_rmap(kvm, gfn); while (*rmapp) { if (!(*rmapp & 1)) @@ -471,9 +471,9 @@ static void rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn) BUG_ON(!(*spte & PT_PRESENT_MASK)); BUG_ON(!(*spte & PT_WRITABLE_MASK)); rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte); - rmap_remove(vcpu->kvm, spte); + rmap_remove(kvm, spte); set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK); - kvm_flush_remote_tlbs(vcpu->kvm); + kvm_flush_remote_tlbs(kvm); } } @@ -670,7 +670,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, hlist_add_head(&page->hash_link, bucket); vcpu->mmu.prefetch_page(vcpu, page); if (!metaphysical) - rmap_write_protect(vcpu, gfn); + rmap_write_protect(vcpu->kvm, gfn); return page; } @@ -823,19 +823,19 @@ static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa) __set_bit(slot, &page_head->slot_bitmap); } -hpa_t safe_gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa) +hpa_t safe_gpa_to_hpa(struct kvm *kvm, gpa_t gpa) { - hpa_t hpa = gpa_to_hpa(vcpu, gpa); + hpa_t hpa = gpa_to_hpa(kvm, gpa); return is_error_hpa(hpa) ? bad_page_address | (gpa & ~PAGE_MASK): hpa; } -hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa) +hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa) { struct page *page; ASSERT((gpa & HPA_ERR_MASK) == 0); - page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); + page = gfn_to_page(kvm, gpa >> PAGE_SHIFT); if (!page) return gpa | HPA_ERR_MASK; return ((hpa_t)page_to_pfn(page) << PAGE_SHIFT) @@ -848,7 +848,7 @@ hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva) if (gpa == UNMAPPED_GVA) return UNMAPPED_GVA; - return gpa_to_hpa(vcpu, gpa); + return gpa_to_hpa(vcpu->kvm, gpa); } struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva) @@ -857,7 +857,7 @@ struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva) if (gpa == UNMAPPED_GVA) return NULL; - return pfn_to_page(gpa_to_hpa(vcpu, gpa) >> PAGE_SHIFT); + return pfn_to_page(gpa_to_hpa(vcpu->kvm, gpa) >> PAGE_SHIFT); } static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) @@ -1012,7 +1012,7 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, ASSERT(VALID_PAGE(vcpu->mmu.root_hpa)); - paddr = gpa_to_hpa(vcpu , addr & PT64_BASE_ADDR_MASK); + paddr = gpa_to_hpa(vcpu->kvm, addr & PT64_BASE_ADDR_MASK); if (is_error_hpa(paddr)) return 1; diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 4f6edf85d13f..8e1e4ca6ea4e 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -103,7 +103,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, walker->level - 1, table_gfn); slot = gfn_to_memslot(vcpu->kvm, table_gfn); - hpa = safe_gpa_to_hpa(vcpu, root & PT64_BASE_ADDR_MASK); + hpa = safe_gpa_to_hpa(vcpu->kvm, root & PT64_BASE_ADDR_MASK); walker->page = pfn_to_page(hpa >> PAGE_SHIFT); walker->table = kmap_atomic(walker->page, KM_USER0); @@ -159,7 +159,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, walker->inherited_ar &= walker->table[index]; table_gfn = (*ptep & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; kunmap_atomic(walker->table, KM_USER0); - paddr = safe_gpa_to_hpa(vcpu, table_gfn << PAGE_SHIFT); + paddr = safe_gpa_to_hpa(vcpu->kvm, table_gfn << PAGE_SHIFT); walker->page = pfn_to_page(paddr >> PAGE_SHIFT); walker->table = kmap_atomic(walker->page, KM_USER0); --walker->level; @@ -248,7 +248,7 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, if (!dirty) access_bits &= ~PT_WRITABLE_MASK; - paddr = gpa_to_hpa(vcpu, gaddr & PT64_BASE_ADDR_MASK); + paddr = gpa_to_hpa(vcpu->kvm, gaddr & PT64_BASE_ADDR_MASK); spte |= PT_PRESENT_MASK; if (access_bits & PT_USER_MASK) From e3c5e7ec9efe0ebd47fa812cc86f01c51905edf6 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 11 Oct 2007 12:32:30 +0200 Subject: [PATCH 031/249] KVM: Move guest pte dirty bit management to the guest pagetable walker This is more consistent with the accessed bit management, and makes the dirty bit available earlier for other purposes. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 5 +++++ drivers/kvm/paging_tmpl.h | 31 ++++++++----------------------- 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index d046ba807763..e6616a6c9cef 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -199,6 +199,11 @@ static int is_writeble_pte(unsigned long pte) return pte & PT_WRITABLE_MASK; } +static int is_dirty_pte(unsigned long pte) +{ + return pte & PT_DIRTY_MASK; +} + static int is_io_pte(unsigned long pte) { return pte & PT_SHADOW_IO_MARK; diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 8e1e4ca6ea4e..da36e48fd026 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -144,6 +144,10 @@ static int FNAME(walk_addr)(struct guest_walker *walker, if (walker->level == PT_PAGE_TABLE_LEVEL) { walker->gfn = (*ptep & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; + if (write_fault && !is_dirty_pte(*ptep)) { + mark_page_dirty(vcpu->kvm, table_gfn); + *ptep |= PT_DIRTY_MASK; + } break; } @@ -153,6 +157,10 @@ static int FNAME(walk_addr)(struct guest_walker *walker, walker->gfn = (*ptep & PT_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT; walker->gfn += PT_INDEX(addr, PT_PAGE_TABLE_LEVEL); + if (write_fault && !is_dirty_pte(*ptep)) { + mark_page_dirty(vcpu->kvm, table_gfn); + *ptep |= PT_DIRTY_MASK; + } break; } @@ -194,12 +202,6 @@ err: return 0; } -static void FNAME(mark_pagetable_dirty)(struct kvm *kvm, - struct guest_walker *walker) -{ - mark_page_dirty(kvm, walker->table_gfn[walker->level - 1]); -} - static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, u64 *shadow_pte, gpa_t gaddr, @@ -221,23 +223,6 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, __FUNCTION__, *shadow_pte, (u64)gpte, access_bits, write_fault, user_fault, gfn); - if (write_fault && !dirty) { - pt_element_t *guest_ent, *tmp = NULL; - - if (walker->ptep) - guest_ent = walker->ptep; - else { - tmp = kmap_atomic(walker->page, KM_USER0); - guest_ent = &tmp[walker->index]; - } - - *guest_ent |= PT_DIRTY_MASK; - if (!walker->ptep) - kunmap_atomic(tmp, KM_USER0); - dirty = 1; - FNAME(mark_pagetable_dirty)(vcpu->kvm, walker); - } - /* * We don't set the accessed bit, since we sometimes want to see * whether the guest actually used the pte (in order to detect From c22e3514fceb2f514093ce1d19a2f660ac7347ae Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 11 Oct 2007 15:08:41 +0200 Subject: [PATCH 032/249] KVM: MMU: Fix nx access bit for huge pages We must set the bit before the shift, otherwise the wrong bit gets set. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index da36e48fd026..e07cb2e006c2 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -382,9 +382,9 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, metaphysical = 1; hugepage_access = walker->pte; hugepage_access &= PT_USER_MASK | PT_WRITABLE_MASK; + hugepage_access >>= PT_WRITABLE_SHIFT; if (walker->pte & PT64_NX_MASK) hugepage_access |= (1 << 2); - hugepage_access >>= PT_WRITABLE_SHIFT; table_gfn = (walker->pte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; } else { From cc70e7374df1e3a56d718e8ca330619f316511a6 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 11 Oct 2007 15:12:24 +0200 Subject: [PATCH 033/249] KVM: MMU: Disable write access on clean large pages By forcing clean huge pages to be read-only, we have separate roles for the shadow of a clean large page and the shadow of a dirty large page. This is necessary because different ptes will be instantiated for the two cases, even for read faults. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index e07cb2e006c2..4538b1533d5b 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -382,6 +382,8 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, metaphysical = 1; hugepage_access = walker->pte; hugepage_access &= PT_USER_MASK | PT_WRITABLE_MASK; + if (!is_dirty_pte(walker->pte)) + hugepage_access &= ~PT_WRITABLE_MASK; hugepage_access >>= PT_WRITABLE_SHIFT; if (walker->pte & PT64_NX_MASK) hugepage_access |= (1 << 2); From 6bfccdc9ae55bb2e0be2e015d7d1ba061f4071c5 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 11 Oct 2007 15:13:49 +0200 Subject: [PATCH 034/249] KVM: MMU: Instantiate real-mode shadows as user writable shadows This is consistent with real-mode permissions. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index e6616a6c9cef..f52604a7f9e8 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -902,7 +902,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) >> PAGE_SHIFT; new_table = kvm_mmu_get_page(vcpu, pseudo_gfn, v, level - 1, - 1, 0, &table[index]); + 1, 3, &table[index]); if (!new_table) { pgprintk("nonpaging_map: ENOMEM\n"); return -ENOMEM; From 5df34a86f917024b67f9e7c850153390973cdfe3 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 11 Oct 2007 15:22:59 +0200 Subject: [PATCH 035/249] KVM: MMU: Move dirty bit updates to a separate function Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 4538b1533d5b..a0f84a5379a4 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -69,6 +69,17 @@ struct guest_walker { u32 error_code; }; +static void FNAME(update_dirty_bit)(struct kvm_vcpu *vcpu, + int write_fault, + pt_element_t *ptep, + gfn_t table_gfn) +{ + if (write_fault && !is_dirty_pte(*ptep)) { + mark_page_dirty(vcpu->kvm, table_gfn); + *ptep |= PT_DIRTY_MASK; + } +} + /* * Fetch a guest pte for a guest virtual address */ @@ -144,10 +155,8 @@ static int FNAME(walk_addr)(struct guest_walker *walker, if (walker->level == PT_PAGE_TABLE_LEVEL) { walker->gfn = (*ptep & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; - if (write_fault && !is_dirty_pte(*ptep)) { - mark_page_dirty(vcpu->kvm, table_gfn); - *ptep |= PT_DIRTY_MASK; - } + FNAME(update_dirty_bit)(vcpu, write_fault, ptep, + table_gfn); break; } @@ -157,10 +166,8 @@ static int FNAME(walk_addr)(struct guest_walker *walker, walker->gfn = (*ptep & PT_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT; walker->gfn += PT_INDEX(addr, PT_PAGE_TABLE_LEVEL); - if (write_fault && !is_dirty_pte(*ptep)) { - mark_page_dirty(vcpu->kvm, table_gfn); - *ptep |= PT_DIRTY_MASK; - } + FNAME(update_dirty_bit)(vcpu, write_fault, ptep, + table_gfn); break; } From c4fcc2724628c6548748ec80a90b548fc300e81f Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 11 Oct 2007 15:30:21 +0200 Subject: [PATCH 036/249] KVM: MMU: When updating the dirty bit, inform the mmu about it Since the mmu uses different shadow pages for dirty large pages and clean large pages, this allows the mmu to drop ptes that are now invalid. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index a0f84a5379a4..a9e687b5c1e9 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -74,9 +74,14 @@ static void FNAME(update_dirty_bit)(struct kvm_vcpu *vcpu, pt_element_t *ptep, gfn_t table_gfn) { + gpa_t pte_gpa; + if (write_fault && !is_dirty_pte(*ptep)) { mark_page_dirty(vcpu->kvm, table_gfn); *ptep |= PT_DIRTY_MASK; + pte_gpa = ((gpa_t)table_gfn << PAGE_SHIFT); + pte_gpa += offset_in_page(ptep); + kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)ptep, sizeof(*ptep)); } } From 313a3dc75da20630e549441932a7654223f8d72a Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Thu, 11 Oct 2007 19:16:52 +0200 Subject: [PATCH 037/249] KVM: Portability: split kvm_vcpu_ioctl This patch splits kvm_vcpu_ioctl into archtecture independent parts, and x86 specific parts which go to kvm_arch_vcpu_ioctl in x86.c. Common ioctls for all architectures are: KVM_RUN, KVM_GET/SET_(S-)REGS, KVM_TRANSLATE, KVM_INTERRUPT, KVM_DEBUG_GUEST, KVM_SET_SIGNAL_MASK, KVM_GET/SET_FPU Note that some PPC chips don't have an FPU, so we might need an #ifdef around KVM_GET/SET_FPU one day. x86 specific ioctls are: KVM_GET/SET_LAPIC, KVM_SET_CPUID, KVM_GET/SET_MSRS An interresting aspect is vcpu_load/vcpu_put. We now have a common vcpu_load/put which does the preemption stuff, and an architecture specific kvm_arch_vcpu_load/put. In the x86 case, this one calls the vmx/svm function defined in kvm_x86_ops. Signed-off-by: Carsten Otte Reviewed-by: Christian Borntraeger Reviewed-by: Christian Ehrhardt Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 9 ++ drivers/kvm/kvm_main.c | 200 ++----------------------------------- drivers/kvm/x86.c | 219 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 234 insertions(+), 194 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index d56962d49aa6..1edf8a5e365e 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -537,6 +537,10 @@ extern struct kvm_x86_ops *kvm_x86_ops; int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id); void kvm_vcpu_uninit(struct kvm_vcpu *vcpu); +void vcpu_load(struct kvm_vcpu *vcpu); +void vcpu_put(struct kvm_vcpu *vcpu); + + int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, struct module *module); void kvm_exit_x86(void); @@ -655,6 +659,11 @@ int kvm_fix_hypercall(struct kvm_vcpu *vcpu); long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); +long kvm_arch_vcpu_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg); +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); + __init void kvm_arch_init(void); static inline void kvm_guest_enter(void) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index ec696887b222..5fd2864b7811 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -90,8 +90,6 @@ static struct kvm_stats_debugfs_item { static struct dentry *debugfs_dir; -#define MAX_IO_MSRS 256 - #define CR0_RESERVED_BITS \ (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \ @@ -179,21 +177,21 @@ EXPORT_SYMBOL_GPL(kvm_put_guest_fpu); /* * Switches to specified vcpu, until a matching vcpu_put() */ -static void vcpu_load(struct kvm_vcpu *vcpu) +void vcpu_load(struct kvm_vcpu *vcpu) { int cpu; mutex_lock(&vcpu->mutex); cpu = get_cpu(); preempt_notifier_register(&vcpu->preempt_notifier); - kvm_x86_ops->vcpu_load(vcpu, cpu); + kvm_arch_vcpu_load(vcpu, cpu); put_cpu(); } -static void vcpu_put(struct kvm_vcpu *vcpu) +void vcpu_put(struct kvm_vcpu *vcpu) { preempt_disable(); - kvm_x86_ops->vcpu_put(vcpu); + kvm_arch_vcpu_put(vcpu); preempt_notifier_unregister(&vcpu->preempt_notifier); preempt_enable(); mutex_unlock(&vcpu->mutex); @@ -2508,86 +2506,6 @@ void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) } EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits); -/* - * Adapt set_msr() to msr_io()'s calling convention - */ -static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) -{ - return kvm_set_msr(vcpu, index, *data); -} - -/* - * Read or write a bunch of msrs. All parameters are kernel addresses. - * - * @return number of msrs set successfully. - */ -static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs, - struct kvm_msr_entry *entries, - int (*do_msr)(struct kvm_vcpu *vcpu, - unsigned index, u64 *data)) -{ - int i; - - vcpu_load(vcpu); - - for (i = 0; i < msrs->nmsrs; ++i) - if (do_msr(vcpu, entries[i].index, &entries[i].data)) - break; - - vcpu_put(vcpu); - - return i; -} - -/* - * Read or write a bunch of msrs. Parameters are user addresses. - * - * @return number of msrs set successfully. - */ -static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs, - int (*do_msr)(struct kvm_vcpu *vcpu, - unsigned index, u64 *data), - int writeback) -{ - struct kvm_msrs msrs; - struct kvm_msr_entry *entries; - int r, n; - unsigned size; - - r = -EFAULT; - if (copy_from_user(&msrs, user_msrs, sizeof msrs)) - goto out; - - r = -E2BIG; - if (msrs.nmsrs >= MAX_IO_MSRS) - goto out; - - r = -ENOMEM; - size = sizeof(struct kvm_msr_entry) * msrs.nmsrs; - entries = vmalloc(size); - if (!entries) - goto out; - - r = -EFAULT; - if (copy_from_user(entries, user_msrs->entries, size)) - goto out_free; - - r = n = __msr_io(vcpu, &msrs, entries, do_msr); - if (r < 0) - goto out_free; - - r = -EFAULT; - if (writeback && copy_to_user(user_msrs->entries, entries, size)) - goto out_free; - - r = n; - -out_free: - vfree(entries); -out: - return r; -} - /* * Translate a guest virtual address to a guest physical address. */ @@ -2761,48 +2679,6 @@ free_vcpu: return r; } -static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) -{ - u64 efer; - int i; - struct kvm_cpuid_entry *e, *entry; - - rdmsrl(MSR_EFER, efer); - entry = NULL; - for (i = 0; i < vcpu->cpuid_nent; ++i) { - e = &vcpu->cpuid_entries[i]; - if (e->function == 0x80000001) { - entry = e; - break; - } - } - if (entry && (entry->edx & (1 << 20)) && !(efer & EFER_NX)) { - entry->edx &= ~(1 << 20); - printk(KERN_INFO "kvm: guest NX capability removed\n"); - } -} - -static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, - struct kvm_cpuid *cpuid, - struct kvm_cpuid_entry __user *entries) -{ - int r; - - r = -E2BIG; - if (cpuid->nent > KVM_MAX_CPUID_ENTRIES) - goto out; - r = -EFAULT; - if (copy_from_user(&vcpu->cpuid_entries, entries, - cpuid->nent * sizeof(struct kvm_cpuid_entry))) - goto out; - vcpu->cpuid_nent = cpuid->nent; - cpuid_fix_nx_cap(vcpu); - return 0; - -out: - return r; -} - static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) { if (sigset) { @@ -2875,33 +2751,12 @@ static int kvm_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) return 0; } -static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, - struct kvm_lapic_state *s) -{ - vcpu_load(vcpu); - memcpy(s->regs, vcpu->apic->regs, sizeof *s); - vcpu_put(vcpu); - - return 0; -} - -static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, - struct kvm_lapic_state *s) -{ - vcpu_load(vcpu); - memcpy(vcpu->apic->regs, s->regs, sizeof *s); - kvm_apic_post_state_restore(vcpu); - vcpu_put(vcpu); - - return 0; -} - static long kvm_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { struct kvm_vcpu *vcpu = filp->private_data; void __user *argp = (void __user *)arg; - int r = -EINVAL; + int r; switch (ioctl) { case KVM_RUN: @@ -2999,24 +2854,6 @@ static long kvm_vcpu_ioctl(struct file *filp, r = 0; break; } - case KVM_GET_MSRS: - r = msr_io(vcpu, argp, kvm_get_msr, 1); - break; - case KVM_SET_MSRS: - r = msr_io(vcpu, argp, do_set_msr, 0); - break; - case KVM_SET_CPUID: { - struct kvm_cpuid __user *cpuid_arg = argp; - struct kvm_cpuid cpuid; - - r = -EFAULT; - if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) - goto out; - r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries); - if (r) - goto out; - break; - } case KVM_SET_SIGNAL_MASK: { struct kvm_signal_mask __user *sigmask_arg = argp; struct kvm_signal_mask kvm_sigmask; @@ -3065,33 +2902,8 @@ static long kvm_vcpu_ioctl(struct file *filp, r = 0; break; } - case KVM_GET_LAPIC: { - struct kvm_lapic_state lapic; - - memset(&lapic, 0, sizeof lapic); - r = kvm_vcpu_ioctl_get_lapic(vcpu, &lapic); - if (r) - goto out; - r = -EFAULT; - if (copy_to_user(argp, &lapic, sizeof lapic)) - goto out; - r = 0; - break; - } - case KVM_SET_LAPIC: { - struct kvm_lapic_state lapic; - - r = -EFAULT; - if (copy_from_user(&lapic, argp, sizeof lapic)) - goto out; - r = kvm_vcpu_ioctl_set_lapic(vcpu, &lapic);; - if (r) - goto out; - r = 0; - break; - } default: - ; + r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); } out: return r; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 437902cf178d..1fe209dd4caf 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -14,10 +14,18 @@ * */ +#include "kvm.h" #include "x86.h" +#include "irq.h" + +#include +#include +#include #include +#define MAX_IO_MSRS 256 + /* * List of msr numbers which we expose to userspace through KVM_GET_MSRS * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. @@ -40,6 +48,86 @@ static u32 emulated_msrs[] = { MSR_IA32_MISC_ENABLE, }; +/* + * Adapt set_msr() to msr_io()'s calling convention + */ +static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) +{ + return kvm_set_msr(vcpu, index, *data); +} + +/* + * Read or write a bunch of msrs. All parameters are kernel addresses. + * + * @return number of msrs set successfully. + */ +static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs, + struct kvm_msr_entry *entries, + int (*do_msr)(struct kvm_vcpu *vcpu, + unsigned index, u64 *data)) +{ + int i; + + vcpu_load(vcpu); + + for (i = 0; i < msrs->nmsrs; ++i) + if (do_msr(vcpu, entries[i].index, &entries[i].data)) + break; + + vcpu_put(vcpu); + + return i; +} + +/* + * Read or write a bunch of msrs. Parameters are user addresses. + * + * @return number of msrs set successfully. + */ +static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs, + int (*do_msr)(struct kvm_vcpu *vcpu, + unsigned index, u64 *data), + int writeback) +{ + struct kvm_msrs msrs; + struct kvm_msr_entry *entries; + int r, n; + unsigned size; + + r = -EFAULT; + if (copy_from_user(&msrs, user_msrs, sizeof msrs)) + goto out; + + r = -E2BIG; + if (msrs.nmsrs >= MAX_IO_MSRS) + goto out; + + r = -ENOMEM; + size = sizeof(struct kvm_msr_entry) * msrs.nmsrs; + entries = vmalloc(size); + if (!entries) + goto out; + + r = -EFAULT; + if (copy_from_user(entries, user_msrs->entries, size)) + goto out_free; + + r = n = __msr_io(vcpu, &msrs, entries, do_msr); + if (r < 0) + goto out_free; + + r = -EFAULT; + if (writeback && copy_to_user(user_msrs->entries, entries, size)) + goto out_free; + + r = n; + +out_free: + vfree(entries); +out: + return r; +} + long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -81,6 +169,137 @@ out: return r; } +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + kvm_x86_ops->vcpu_load(vcpu, cpu); +} + +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) +{ + kvm_x86_ops->vcpu_put(vcpu); +} + +static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) +{ + u64 efer; + int i; + struct kvm_cpuid_entry *e, *entry; + + rdmsrl(MSR_EFER, efer); + entry = NULL; + for (i = 0; i < vcpu->cpuid_nent; ++i) { + e = &vcpu->cpuid_entries[i]; + if (e->function == 0x80000001) { + entry = e; + break; + } + } + if (entry && (entry->edx & (1 << 20)) && !(efer & EFER_NX)) { + entry->edx &= ~(1 << 20); + printk(KERN_INFO "kvm: guest NX capability removed\n"); + } +} + +static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, + struct kvm_cpuid *cpuid, + struct kvm_cpuid_entry __user *entries) +{ + int r; + + r = -E2BIG; + if (cpuid->nent > KVM_MAX_CPUID_ENTRIES) + goto out; + r = -EFAULT; + if (copy_from_user(&vcpu->cpuid_entries, entries, + cpuid->nent * sizeof(struct kvm_cpuid_entry))) + goto out; + vcpu->cpuid_nent = cpuid->nent; + cpuid_fix_nx_cap(vcpu); + return 0; + +out: + return r; +} + +static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, + struct kvm_lapic_state *s) +{ + vcpu_load(vcpu); + memcpy(s->regs, vcpu->apic->regs, sizeof *s); + vcpu_put(vcpu); + + return 0; +} + +static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, + struct kvm_lapic_state *s) +{ + vcpu_load(vcpu); + memcpy(vcpu->apic->regs, s->regs, sizeof *s); + kvm_apic_post_state_restore(vcpu); + vcpu_put(vcpu); + + return 0; +} + +long kvm_arch_vcpu_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + void __user *argp = (void __user *)arg; + int r; + + switch (ioctl) { + case KVM_GET_LAPIC: { + struct kvm_lapic_state lapic; + + memset(&lapic, 0, sizeof lapic); + r = kvm_vcpu_ioctl_get_lapic(vcpu, &lapic); + if (r) + goto out; + r = -EFAULT; + if (copy_to_user(argp, &lapic, sizeof lapic)) + goto out; + r = 0; + break; + } + case KVM_SET_LAPIC: { + struct kvm_lapic_state lapic; + + r = -EFAULT; + if (copy_from_user(&lapic, argp, sizeof lapic)) + goto out; + r = kvm_vcpu_ioctl_set_lapic(vcpu, &lapic);; + if (r) + goto out; + r = 0; + break; + } + case KVM_SET_CPUID: { + struct kvm_cpuid __user *cpuid_arg = argp; + struct kvm_cpuid cpuid; + + r = -EFAULT; + if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) + goto out; + r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries); + if (r) + goto out; + break; + } + case KVM_GET_MSRS: + r = msr_io(vcpu, argp, kvm_get_msr, 1); + break; + case KVM_SET_MSRS: + r = msr_io(vcpu, argp, do_set_msr, 0); + break; + default: + r = -EINVAL; + } +out: + return r; +} + static __init void kvm_init_msr_list(void) { u32 dummy[2]; From e4d47f404b59adf22f639fedc943db99f090f9ee Mon Sep 17 00:00:00 2001 From: Qing He Date: Mon, 24 Sep 2007 17:39:41 +0800 Subject: [PATCH 038/249] KVM: apic round robin cleanup If no apic is enabled in the bitmap of an interrupt delivery with delivery mode of lowest priority, a warning should be reported rather than select a fallback vcpu Signed-off-by: Qing He Signed-off-by: Eddie (Yaozu) Dong Signed-off-by: Avi Kivity --- drivers/kvm/lapic.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index e15b42e48862..8840f9dc0bca 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -395,10 +395,9 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, unsigned long bitmap) { - int vcpu_id; int last; int next; - struct kvm_lapic *apic; + struct kvm_lapic *apic = NULL; last = kvm->round_robin_prev_vcpu; next = last; @@ -415,14 +414,8 @@ struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, } while (next != last); kvm->round_robin_prev_vcpu = next; - if (!apic) { - vcpu_id = ffs(bitmap) - 1; - if (vcpu_id < 0) { - vcpu_id = 0; - printk(KERN_DEBUG "vcpu not ready for apic_round_robin\n"); - } - apic = kvm->vcpus[vcpu_id]->apic; - } + if (!apic) + printk(KERN_DEBUG "vcpu not ready for apic_round_robin\n"); return apic; } From e25e3ed56fce591e09477b0ce043d90f97988f37 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Fri, 12 Oct 2007 11:01:59 +0200 Subject: [PATCH 039/249] KVM: Add some \n in ioapic_debug() Add new-line at end of debug strings. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/ioapic.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/drivers/kvm/ioapic.c b/drivers/kvm/ioapic.c index c7992e667fdb..8503d99b8339 100644 --- a/drivers/kvm/ioapic.c +++ b/drivers/kvm/ioapic.c @@ -40,8 +40,11 @@ #include #include #include "irq.h" -/* #define ioapic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */ +#if 0 +#define ioapic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) +#else #define ioapic_debug(fmt, arg...) +#endif static void ioapic_deliver(struct kvm_ioapic *vioapic, int irq); static unsigned long ioapic_read_indirect(struct kvm_ioapic *ioapic, @@ -113,7 +116,7 @@ static void ioapic_write_indirect(struct kvm_ioapic *ioapic, u32 val) default: index = (ioapic->ioregsel - 0x10) >> 1; - ioapic_debug("change redir index %x val %x", index, val); + ioapic_debug("change redir index %x val %x\n", index, val); if (index >= IOAPIC_NUM_PINS) return; if (ioapic->ioregsel & 1) { @@ -134,7 +137,7 @@ static void ioapic_inj_irq(struct kvm_ioapic *ioapic, struct kvm_lapic *target, u8 vector, u8 trig_mode, u8 delivery_mode) { - ioapic_debug("irq %d trig %d deliv %d", vector, trig_mode, + ioapic_debug("irq %d trig %d deliv %d\n", vector, trig_mode, delivery_mode); ASSERT((delivery_mode == dest_Fixed) || @@ -151,7 +154,7 @@ static u32 ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest, struct kvm *kvm = ioapic->kvm; struct kvm_vcpu *vcpu; - ioapic_debug("dest %d dest_mode %d", dest, dest_mode); + ioapic_debug("dest %d dest_mode %d\n", dest, dest_mode); if (dest_mode == 0) { /* Physical mode. */ if (dest == 0xFF) { /* Broadcast. */ @@ -179,7 +182,7 @@ static u32 ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest, kvm_apic_match_logical_addr(vcpu->apic, dest)) mask |= 1 << vcpu->vcpu_id; } - ioapic_debug("mask %x", mask); + ioapic_debug("mask %x\n", mask); return mask; } @@ -196,12 +199,12 @@ static void ioapic_deliver(struct kvm_ioapic *ioapic, int irq) int vcpu_id; ioapic_debug("dest=%x dest_mode=%x delivery_mode=%x " - "vector=%x trig_mode=%x", + "vector=%x trig_mode=%x\n", dest, dest_mode, delivery_mode, vector, trig_mode); deliver_bitmask = ioapic_get_delivery_bitmask(ioapic, dest, dest_mode); if (!deliver_bitmask) { - ioapic_debug("no target on destination"); + ioapic_debug("no target on destination\n"); return; } @@ -214,7 +217,7 @@ static void ioapic_deliver(struct kvm_ioapic *ioapic, int irq) trig_mode, delivery_mode); else ioapic_debug("null round robin: " - "mask=%x vector=%x delivery_mode=%x", + "mask=%x vector=%x delivery_mode=%x\n", deliver_bitmask, vector, dest_LowestPrio); break; case dest_Fixed: @@ -304,7 +307,7 @@ static void ioapic_mmio_read(struct kvm_io_device *this, gpa_t addr, int len, struct kvm_ioapic *ioapic = (struct kvm_ioapic *)this->private; u32 result; - ioapic_debug("addr %lx", (unsigned long)addr); + ioapic_debug("addr %lx\n", (unsigned long)addr); ASSERT(!(addr & 0xf)); /* check alignment */ addr &= 0xff; @@ -341,8 +344,8 @@ static void ioapic_mmio_write(struct kvm_io_device *this, gpa_t addr, int len, struct kvm_ioapic *ioapic = (struct kvm_ioapic *)this->private; u32 data; - ioapic_debug("ioapic_mmio_write addr=%lx len=%d val=%p\n", - addr, len, val); + ioapic_debug("ioapic_mmio_write addr=%p len=%d val=%p\n", + (void*)addr, len, val); ASSERT(!(addr & 0xf)); /* check alignment */ if (len == 4 || len == 8) data = *(u32 *) val; From ab6ef34b906546d85d92dbc3e0fb4e01cce05f62 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 16 Oct 2007 16:23:22 +0200 Subject: [PATCH 040/249] KVM: Move apic timer interrupt backlog processing to common code Beside the obvious goodness of making code more common, this prevents a livelock with the next patch which moves interrupt injection out of the critical section. Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 2 ++ drivers/kvm/svm.c | 1 - drivers/kvm/vmx.c | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 5fd2864b7811..4c96817929fd 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -2144,6 +2144,8 @@ again: if (unlikely(r)) goto out; + kvm_inject_pending_timer_irqs(vcpu); + preempt_disable(); kvm_x86_ops->prepare_guest_switch(vcpu); diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 7b21576b62bc..ef068d2dddd7 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -1355,7 +1355,6 @@ static void svm_intr_assist(struct kvm_vcpu *vcpu) struct vmcb *vmcb = svm->vmcb; int intr_vector = -1; - kvm_inject_pending_timer_irqs(vcpu); if ((vmcb->control.exit_int_info & SVM_EVTINJ_VALID) && ((vmcb->control.exit_int_info & SVM_EVTINJ_TYPE_MASK) == 0)) { intr_vector = vmcb->control.exit_int_info & diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 6955580bb69e..9f77ddbeb025 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -2191,7 +2191,6 @@ static void vmx_intr_assist(struct kvm_vcpu *vcpu) int has_ext_irq, interrupt_window_open; int vector; - kvm_inject_pending_timer_irqs(vcpu); update_tpr_threshold(vcpu); has_ext_irq = kvm_cpu_has_interrupt(vcpu); From 3176bc3e59cc4e30ac10888f08124b31969a7a88 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 16 Oct 2007 17:22:08 +0200 Subject: [PATCH 041/249] KVM: Rename KVM_TLB_FLUSH to KVM_REQ_TLB_FLUSH We now have a new namespace, KVM_REQ_*, for bits in vcpu->requests. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 2 +- drivers/kvm/kvm_main.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 1edf8a5e365e..6ae7b6332e32 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -67,7 +67,7 @@ /* * vcpu->requests bit members */ -#define KVM_TLB_FLUSH 0 +#define KVM_REQ_TLB_FLUSH 0 /* * Address types: diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 4c96817929fd..1879b409bed2 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -212,7 +212,7 @@ void kvm_flush_remote_tlbs(struct kvm *kvm) vcpu = kvm->vcpus[i]; if (!vcpu) continue; - if (test_and_set_bit(KVM_TLB_FLUSH, &vcpu->requests)) + if (test_and_set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests)) continue; cpu = vcpu->cpu; if (cpu != -1 && cpu != raw_smp_processor_id()) @@ -2171,7 +2171,7 @@ again: kvm_guest_enter(); if (vcpu->requests) - if (test_and_clear_bit(KVM_TLB_FLUSH, &vcpu->requests)) + if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests)) kvm_x86_ops->tlb_flush(vcpu); kvm_x86_ops->run(vcpu, kvm_run); From d77a25074a8f845401f0eb786ebb8996e45d9e22 Mon Sep 17 00:00:00 2001 From: Nitin A Kamble Date: Fri, 12 Oct 2007 17:40:33 -0700 Subject: [PATCH 042/249] KVM: x86 emulator: Implement emulation of instruction: inc & dec Instructions: inc r16/r32 (opcode 0x40-0x47) dec r16/r32 (opcode 0x48-0x4f) Signed-off-by: Nitin A Kamble Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 75fd23bade9c..988c6498640f 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -96,8 +96,12 @@ static u8 opcode_table[256] = { ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, 0, 0, 0, 0, - /* 0x40 - 0x4F */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* 0x40 - 0x47 */ + ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + /* 0x48 - 0x4F */ + ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, /* 0x50 - 0x57 */ ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, @@ -1376,6 +1380,18 @@ special_insn: if (c->twobyte) goto twobyte_special_insn; switch (c->b) { + case 0x40 ... 0x47: /* inc r16/r32 */ + c->dst.bytes = c->op_bytes; + c->dst.ptr = (unsigned long *)&c->regs[c->b & 0x7]; + c->dst.val = *c->dst.ptr; + emulate_1op("inc", c->dst, ctxt->eflags); + break; + case 0x48 ... 0x4f: /* dec r16/r32 */ + c->dst.bytes = c->op_bytes; + c->dst.ptr = (unsigned long *)&c->regs[c->b & 0x7]; + c->dst.val = *c->dst.ptr; + emulate_1op("dec", c->dst, ctxt->eflags); + break; case 0x50 ... 0x57: /* push reg */ if (c->op_bytes == 2) c->src.val = (u16) c->regs[c->b & 0x7]; From 42bf3f0a1f5a25be26e6bb53162cdee82410310d Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 17 Oct 2007 12:18:47 +0200 Subject: [PATCH 043/249] KVM: MMU: Simplify page table walker Simplify the walker level loop not to carry so much information from one loop to the next. In addition to being complex, this made kmap_atomic() critical sections difficult to manage. As a result of this change, kmap_atomic() sections are limited to actually touching the guest pte, which allows the other functions called from the walker to do sleepy operations. This will happen when we enable swapping. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 124 +++++++++++++++----------------------- 1 file changed, 48 insertions(+), 76 deletions(-) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index a9e687b5c1e9..bab1b7f8d705 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -59,32 +59,12 @@ struct guest_walker { int level; gfn_t table_gfn[PT_MAX_FULL_LEVELS]; - pt_element_t *table; pt_element_t pte; - pt_element_t *ptep; - struct page *page; - int index; pt_element_t inherited_ar; gfn_t gfn; u32 error_code; }; -static void FNAME(update_dirty_bit)(struct kvm_vcpu *vcpu, - int write_fault, - pt_element_t *ptep, - gfn_t table_gfn) -{ - gpa_t pte_gpa; - - if (write_fault && !is_dirty_pte(*ptep)) { - mark_page_dirty(vcpu->kvm, table_gfn); - *ptep |= PT_DIRTY_MASK; - pte_gpa = ((gpa_t)table_gfn << PAGE_SHIFT); - pte_gpa += offset_in_page(ptep); - kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)ptep, sizeof(*ptep)); - } -} - /* * Fetch a guest pte for a guest virtual address */ @@ -94,105 +74,99 @@ static int FNAME(walk_addr)(struct guest_walker *walker, { hpa_t hpa; struct kvm_memory_slot *slot; - pt_element_t *ptep; - pt_element_t root; + struct page *page; + pt_element_t *table; + pt_element_t pte; gfn_t table_gfn; + unsigned index; + gpa_t pte_gpa; pgprintk("%s: addr %lx\n", __FUNCTION__, addr); walker->level = vcpu->mmu.root_level; - walker->table = NULL; - walker->page = NULL; - walker->ptep = NULL; - root = vcpu->cr3; + pte = vcpu->cr3; #if PTTYPE == 64 if (!is_long_mode(vcpu)) { - walker->ptep = &vcpu->pdptrs[(addr >> 30) & 3]; - root = *walker->ptep; - walker->pte = root; - if (!(root & PT_PRESENT_MASK)) + pte = vcpu->pdptrs[(addr >> 30) & 3]; + if (!is_present_pte(pte)) goto not_present; --walker->level; } #endif - table_gfn = (root & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT; - walker->table_gfn[walker->level - 1] = table_gfn; - pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, - walker->level - 1, table_gfn); - slot = gfn_to_memslot(vcpu->kvm, table_gfn); - hpa = safe_gpa_to_hpa(vcpu->kvm, root & PT64_BASE_ADDR_MASK); - walker->page = pfn_to_page(hpa >> PAGE_SHIFT); - walker->table = kmap_atomic(walker->page, KM_USER0); - ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) || (vcpu->cr3 & CR3_NONPAE_RESERVED_BITS) == 0); walker->inherited_ar = PT_USER_MASK | PT_WRITABLE_MASK; for (;;) { - int index = PT_INDEX(addr, walker->level); - hpa_t paddr; + index = PT_INDEX(addr, walker->level); - ptep = &walker->table[index]; - walker->index = index; - ASSERT(((unsigned long)walker->table & PAGE_MASK) == - ((unsigned long)ptep & PAGE_MASK)); + table_gfn = (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT; + walker->table_gfn[walker->level - 1] = table_gfn; + pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, + walker->level - 1, table_gfn); - if (!is_present_pte(*ptep)) + slot = gfn_to_memslot(vcpu->kvm, table_gfn); + hpa = safe_gpa_to_hpa(vcpu->kvm, pte & PT64_BASE_ADDR_MASK); + page = pfn_to_page(hpa >> PAGE_SHIFT); + + table = kmap_atomic(page, KM_USER0); + pte = table[index]; + kunmap_atomic(table, KM_USER0); + + if (!is_present_pte(pte)) goto not_present; - if (write_fault && !is_writeble_pte(*ptep)) + if (write_fault && !is_writeble_pte(pte)) if (user_fault || is_write_protection(vcpu)) goto access_error; - if (user_fault && !(*ptep & PT_USER_MASK)) + if (user_fault && !(pte & PT_USER_MASK)) goto access_error; #if PTTYPE == 64 - if (fetch_fault && is_nx(vcpu) && (*ptep & PT64_NX_MASK)) + if (fetch_fault && is_nx(vcpu) && (pte & PT64_NX_MASK)) goto access_error; #endif - if (!(*ptep & PT_ACCESSED_MASK)) { + if (!(pte & PT_ACCESSED_MASK)) { mark_page_dirty(vcpu->kvm, table_gfn); - *ptep |= PT_ACCESSED_MASK; + pte |= PT_ACCESSED_MASK; + table = kmap_atomic(page, KM_USER0); + table[index] = pte; + kunmap_atomic(table, KM_USER0); } if (walker->level == PT_PAGE_TABLE_LEVEL) { - walker->gfn = (*ptep & PT_BASE_ADDR_MASK) - >> PAGE_SHIFT; - FNAME(update_dirty_bit)(vcpu, write_fault, ptep, - table_gfn); + walker->gfn = (pte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; break; } if (walker->level == PT_DIRECTORY_LEVEL - && (*ptep & PT_PAGE_SIZE_MASK) + && (pte & PT_PAGE_SIZE_MASK) && (PTTYPE == 64 || is_pse(vcpu))) { - walker->gfn = (*ptep & PT_DIR_BASE_ADDR_MASK) + walker->gfn = (pte & PT_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT; walker->gfn += PT_INDEX(addr, PT_PAGE_TABLE_LEVEL); - FNAME(update_dirty_bit)(vcpu, write_fault, ptep, - table_gfn); break; } - walker->inherited_ar &= walker->table[index]; - table_gfn = (*ptep & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; - kunmap_atomic(walker->table, KM_USER0); - paddr = safe_gpa_to_hpa(vcpu->kvm, table_gfn << PAGE_SHIFT); - walker->page = pfn_to_page(paddr >> PAGE_SHIFT); - walker->table = kmap_atomic(walker->page, KM_USER0); + walker->inherited_ar &= pte; --walker->level; - walker->table_gfn[walker->level - 1] = table_gfn; - pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, - walker->level - 1, table_gfn); } - walker->pte = *ptep; - if (walker->page) - walker->ptep = NULL; - if (walker->table) - kunmap_atomic(walker->table, KM_USER0); - pgprintk("%s: pte %llx\n", __FUNCTION__, (u64)*ptep); + + if (write_fault && !is_dirty_pte(pte)) { + mark_page_dirty(vcpu->kvm, table_gfn); + pte |= PT_DIRTY_MASK; + table = kmap_atomic(page, KM_USER0); + table[index] = pte; + kunmap_atomic(table, KM_USER0); + pte_gpa = table_gfn << PAGE_SHIFT; + pte_gpa += index * sizeof(pt_element_t); + kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte)); + } + + walker->pte = pte; + pgprintk("%s: pte %llx\n", __FUNCTION__, (u64)pte); return 1; not_present: @@ -209,8 +183,6 @@ err: walker->error_code |= PFERR_USER_MASK; if (fetch_fault) walker->error_code |= PFERR_FETCH_MASK; - if (walker->table) - kunmap_atomic(walker->table, KM_USER0); return 0; } From b284be576471b1521dd04c0845beedb790cbb9d7 Mon Sep 17 00:00:00 2001 From: Nitin A Kamble Date: Tue, 16 Oct 2007 18:23:27 -0700 Subject: [PATCH 044/249] KVM: x86 emulator: cmc, clc, cli, sti Instruction: cmc, clc, cli, sti opcodes: 0xf5, 0xf8, 0xfa, 0xfb respectively. [avi: fix reference to EFLG_IF which is not defined anywhere] Signed-off-by: Nitin A Kamble Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 988c6498640f..e962de331606 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -161,10 +161,10 @@ static u8 opcode_table[256] = { ImplicitOps, SrcImm|ImplicitOps, 0, SrcImmByte|ImplicitOps, 0, 0, 0, 0, /* 0xF0 - 0xF7 */ 0, 0, 0, 0, - ImplicitOps, 0, + ImplicitOps, ImplicitOps, ByteOp | DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM, /* 0xF8 - 0xFF */ - 0, 0, 0, 0, + ImplicitOps, 0, ImplicitOps, ImplicitOps, 0, 0, ByteOp | DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM }; @@ -1476,6 +1476,23 @@ special_insn: case 0xf4: /* hlt */ ctxt->vcpu->halt_request = 1; goto done; + case 0xf5: /* cmc */ + /* complement carry flag from eflags reg */ + ctxt->eflags ^= EFLG_CF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0xf8: /* clc */ + ctxt->eflags &= ~EFLG_CF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0xfa: /* cli */ + ctxt->eflags &= ~X86_EFLAGS_IF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0xfb: /* sti */ + ctxt->eflags |= X86_EFLAGS_IF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; } if (c->rep_prefix) { if (c->regs[VCPU_REGS_RCX] == 0) { From 98348e9507ace5fda95432ff8ca23f13e7f66176 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Tue, 16 Oct 2007 14:42:30 +0200 Subject: [PATCH 045/249] KVM: MMU: Add rmap_next(), a helper for walking kvm rmaps Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 45 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index f52604a7f9e8..14e54e331f50 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -456,28 +456,53 @@ static void rmap_remove(struct kvm *kvm, u64 *spte) } } -static void rmap_write_protect(struct kvm *kvm, u64 gfn) +static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte) { struct kvm_rmap_desc *desc; + struct kvm_rmap_desc *prev_desc; + u64 *prev_spte; + int i; + + if (!*rmapp) + return NULL; + else if (!(*rmapp & 1)) { + if (!spte) + return (u64 *)*rmapp; + return NULL; + } + desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul); + prev_desc = NULL; + prev_spte = NULL; + while (desc) { + for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) { + if (prev_spte == spte) + return desc->shadow_ptes[i]; + prev_spte = desc->shadow_ptes[i]; + } + desc = desc->more; + } + return NULL; +} + +static void rmap_write_protect(struct kvm *kvm, u64 gfn) +{ unsigned long *rmapp; u64 *spte; + u64 *prev_spte; gfn = unalias_gfn(kvm, gfn); rmapp = gfn_to_rmap(kvm, gfn); - while (*rmapp) { - if (!(*rmapp & 1)) - spte = (u64 *)*rmapp; - else { - desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul); - spte = desc->shadow_ptes[0]; - } + spte = rmap_next(kvm, rmapp, NULL); + while (spte) { BUG_ON(!spte); BUG_ON(!(*spte & PT_PRESENT_MASK)); BUG_ON(!(*spte & PT_WRITABLE_MASK)); rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte); - rmap_remove(kvm, spte); - set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK); + prev_spte = spte; + spte = rmap_next(kvm, rmapp, spte); + rmap_remove(kvm, prev_spte); + set_shadow_pte(prev_spte, *prev_spte & ~PT_WRITABLE_MASK); kvm_flush_remote_tlbs(kvm); } } From 9647c14c98687d0abf5197e74b9d1448ab6ebb95 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Tue, 16 Oct 2007 14:43:46 +0200 Subject: [PATCH 046/249] KVM: MMU: Keep a reverse mapping of non-writable translations The current kvm mmu only reverse maps writable translation. This is used to write-protect a page in case it becomes a pagetable. But with swapping support, we need a reverse mapping of read-only pages as well: when we evict a page, we need to remove any mapping to it, whether writable or not. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 14e54e331f50..bbf5eb427dc6 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -211,8 +211,8 @@ static int is_io_pte(unsigned long pte) static int is_rmap_pte(u64 pte) { - return (pte & (PT_WRITABLE_MASK | PT_PRESENT_MASK)) - == (PT_WRITABLE_MASK | PT_PRESENT_MASK); + return pte != shadow_trap_nonpresent_pte + && pte != shadow_notrap_nonpresent_pte; } static void set_shadow_pte(u64 *sptep, u64 spte) @@ -488,7 +488,6 @@ static void rmap_write_protect(struct kvm *kvm, u64 gfn) { unsigned long *rmapp; u64 *spte; - u64 *prev_spte; gfn = unalias_gfn(kvm, gfn); rmapp = gfn_to_rmap(kvm, gfn); @@ -497,13 +496,11 @@ static void rmap_write_protect(struct kvm *kvm, u64 gfn) while (spte) { BUG_ON(!spte); BUG_ON(!(*spte & PT_PRESENT_MASK)); - BUG_ON(!(*spte & PT_WRITABLE_MASK)); rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte); - prev_spte = spte; - spte = rmap_next(kvm, rmapp, spte); - rmap_remove(kvm, prev_spte); - set_shadow_pte(prev_spte, *prev_spte & ~PT_WRITABLE_MASK); + if (is_writeble_pte(*spte)) + set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK); kvm_flush_remote_tlbs(kvm); + spte = rmap_next(kvm, rmapp, spte); } } @@ -908,14 +905,18 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) table = __va(table_addr); if (level == 1) { + int was_rmapped; + pte = table[index]; + was_rmapped = is_rmap_pte(pte); if (is_shadow_present_pte(pte) && is_writeble_pte(pte)) return 0; mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT); page_header_update_slot(vcpu->kvm, table, v); table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK; - rmap_add(vcpu, &table[index], v >> PAGE_SHIFT); + if (!was_rmapped) + rmap_add(vcpu, &table[index], v >> PAGE_SHIFT); return 0; } @@ -1424,10 +1425,8 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot) pt = page->spt; for (i = 0; i < PT64_ENT_PER_PAGE; ++i) /* avoid RMW */ - if (pt[i] & PT_WRITABLE_MASK) { - rmap_remove(kvm, &pt[i]); + if (pt[i] & PT_WRITABLE_MASK) pt[i] &= ~PT_WRITABLE_MASK; - } } } From cea7bb21280e3a825e64b54740edc5d3e6e4193c Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Wed, 17 Oct 2007 19:17:48 +0200 Subject: [PATCH 047/249] KVM: MMU: Make gfn_to_page() always safe In case the page is not present in the guest memory map, return a dummy page the guest can scribble on. This simplifies error checking in its users. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 ++- drivers/kvm/kvm_main.c | 26 ++++++++++++++------------ drivers/kvm/mmu.c | 16 +++++----------- drivers/kvm/paging_tmpl.h | 7 ++----- 4 files changed, 23 insertions(+), 29 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 6ae7b6332e32..0c17c76d030f 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -565,8 +565,9 @@ static inline int is_error_hpa(hpa_t hpa) { return hpa >> HPA_MSB; } hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva); struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva); -extern hpa_t bad_page_address; +extern struct page *bad_page; +int is_error_page(struct page *page); gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 1879b409bed2..47000be25479 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -993,6 +993,12 @@ static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) return r; } +int is_error_page(struct page *page) +{ + return page == bad_page; +} +EXPORT_SYMBOL_GPL(is_error_page); + gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) { int i; @@ -1034,7 +1040,7 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) gfn = unalias_gfn(kvm, gfn); slot = __gfn_to_memslot(kvm, gfn); if (!slot) - return NULL; + return bad_page; return slot->phys_mem[gfn - slot->base_gfn]; } EXPORT_SYMBOL_GPL(gfn_to_page); @@ -1054,7 +1060,7 @@ int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, struct page *page; page = gfn_to_page(kvm, gfn); - if (!page) + if (is_error_page(page)) return -EFAULT; page_virt = kmap_atomic(page, KM_USER0); @@ -1092,7 +1098,7 @@ int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, struct page *page; page = gfn_to_page(kvm, gfn); - if (!page) + if (is_error_page(page)) return -EFAULT; page_virt = kmap_atomic(page, KM_USER0); @@ -1130,7 +1136,7 @@ int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) struct page *page; page = gfn_to_page(kvm, gfn); - if (!page) + if (is_error_page(page)) return -EFAULT; page_virt = kmap_atomic(page, KM_USER0); @@ -3068,7 +3074,7 @@ static struct page *kvm_vm_nopage(struct vm_area_struct *vma, pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; page = gfn_to_page(kvm, pgoff); - if (!page) + if (is_error_page(page)) return NOPAGE_SIGBUS; get_page(page); if (type != NULL) @@ -3383,7 +3389,7 @@ static struct sys_device kvm_sysdev = { .cls = &kvm_sysdev_class, }; -hpa_t bad_page_address; +struct page *bad_page; static inline struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) @@ -3512,7 +3518,6 @@ EXPORT_SYMBOL_GPL(kvm_exit_x86); static __init int kvm_init(void) { - static struct page *bad_page; int r; r = kvm_mmu_module_init(); @@ -3523,16 +3528,13 @@ static __init int kvm_init(void) kvm_arch_init(); - bad_page = alloc_page(GFP_KERNEL); + bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (bad_page == NULL) { r = -ENOMEM; goto out; } - bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT; - memset(__va(bad_page_address), 0, PAGE_SIZE); - return 0; out: @@ -3545,7 +3547,7 @@ out4: static __exit void kvm_exit(void) { kvm_exit_debug(); - __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT)); + __free_page(bad_page); kvm_mmu_module_exit(); } diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index bbf5eb427dc6..2ad14fbdcfa0 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -850,23 +850,17 @@ static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa) __set_bit(slot, &page_head->slot_bitmap); } -hpa_t safe_gpa_to_hpa(struct kvm *kvm, gpa_t gpa) -{ - hpa_t hpa = gpa_to_hpa(kvm, gpa); - - return is_error_hpa(hpa) ? bad_page_address | (gpa & ~PAGE_MASK): hpa; -} - hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa) { struct page *page; + hpa_t hpa; ASSERT((gpa & HPA_ERR_MASK) == 0); page = gfn_to_page(kvm, gpa >> PAGE_SHIFT); - if (!page) - return gpa | HPA_ERR_MASK; - return ((hpa_t)page_to_pfn(page) << PAGE_SHIFT) - | (gpa & (PAGE_SIZE-1)); + hpa = ((hpa_t)page_to_pfn(page) << PAGE_SHIFT) | (gpa & (PAGE_SIZE-1)); + if (is_error_page(page)) + return hpa | HPA_ERR_MASK; + return hpa; } hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index bab1b7f8d705..572e5b6d9a7a 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -72,8 +72,6 @@ static int FNAME(walk_addr)(struct guest_walker *walker, struct kvm_vcpu *vcpu, gva_t addr, int write_fault, int user_fault, int fetch_fault) { - hpa_t hpa; - struct kvm_memory_slot *slot; struct page *page; pt_element_t *table; pt_element_t pte; @@ -105,9 +103,8 @@ static int FNAME(walk_addr)(struct guest_walker *walker, pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, walker->level - 1, table_gfn); - slot = gfn_to_memslot(vcpu->kvm, table_gfn); - hpa = safe_gpa_to_hpa(vcpu->kvm, pte & PT64_BASE_ADDR_MASK); - page = pfn_to_page(hpa >> PAGE_SHIFT); + page = gfn_to_page(vcpu->kvm, (pte & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT); table = kmap_atomic(page, KM_USER0); pte = table[index]; From 8a7ae055f3533b520401c170ac55e30628b34df5 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Thu, 18 Oct 2007 11:09:33 +0200 Subject: [PATCH 048/249] KVM: MMU: Partial swapping of guest memory This allows guest memory to be swapped. Pages which are currently mapped via shadow page tables are pinned into memory, but all other pages can be freely swapped. The patch makes gfn_to_page() elevate the page's reference count, and introduces kvm_release_page() that pairs with it. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 2 + drivers/kvm/kvm_main.c | 83 ++++++++++++++++++++++----------------- drivers/kvm/mmu.c | 14 ++++++- drivers/kvm/paging_tmpl.h | 26 ++++++++++-- 4 files changed, 84 insertions(+), 41 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 0c17c76d030f..df0711ce9baa 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -409,6 +409,7 @@ struct kvm_memory_slot { unsigned long *rmap; unsigned long *dirty_bitmap; int user_alloc; /* user allocated memory */ + unsigned long userspace_addr; }; struct kvm { @@ -570,6 +571,7 @@ extern struct page *bad_page; int is_error_page(struct page *page); gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); +void kvm_release_page(struct page *page); int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, int len); int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 47000be25479..f86a47c2f255 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -300,19 +300,6 @@ static struct kvm *kvm_create_vm(void) return kvm; } -static void kvm_free_userspace_physmem(struct kvm_memory_slot *free) -{ - int i; - - for (i = 0; i < free->npages; ++i) { - if (free->phys_mem[i]) { - if (!PageReserved(free->phys_mem[i])) - SetPageDirty(free->phys_mem[i]); - page_cache_release(free->phys_mem[i]); - } - } -} - static void kvm_free_kernel_physmem(struct kvm_memory_slot *free) { int i; @@ -330,9 +317,7 @@ static void kvm_free_physmem_slot(struct kvm_memory_slot *free, { if (!dont || free->phys_mem != dont->phys_mem) if (free->phys_mem) { - if (free->user_alloc) - kvm_free_userspace_physmem(free); - else + if (!free->user_alloc) kvm_free_kernel_physmem(free); vfree(free->phys_mem); } @@ -361,7 +346,7 @@ static void free_pio_guest_pages(struct kvm_vcpu *vcpu) for (i = 0; i < ARRAY_SIZE(vcpu->pio.guest_pages); ++i) if (vcpu->pio.guest_pages[i]) { - __free_page(vcpu->pio.guest_pages[i]); + kvm_release_page(vcpu->pio.guest_pages[i]); vcpu->pio.guest_pages[i] = NULL; } } @@ -752,19 +737,8 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, memset(new.phys_mem, 0, npages * sizeof(struct page *)); memset(new.rmap, 0, npages * sizeof(*new.rmap)); if (user_alloc) { - unsigned long pages_num; - new.user_alloc = 1; - down_read(¤t->mm->mmap_sem); - - pages_num = get_user_pages(current, current->mm, - mem->userspace_addr, - npages, 1, 1, new.phys_mem, - NULL); - - up_read(¤t->mm->mmap_sem); - if (pages_num != npages) - goto out_unlock; + new.userspace_addr = mem->userspace_addr; } else { for (i = 0; i < npages; ++i) { new.phys_mem[i] = alloc_page(GFP_HIGHUSER @@ -1039,12 +1013,39 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) gfn = unalias_gfn(kvm, gfn); slot = __gfn_to_memslot(kvm, gfn); - if (!slot) + if (!slot) { + get_page(bad_page); return bad_page; + } + if (slot->user_alloc) { + struct page *page[1]; + int npages; + + down_read(¤t->mm->mmap_sem); + npages = get_user_pages(current, current->mm, + slot->userspace_addr + + (gfn - slot->base_gfn) * PAGE_SIZE, 1, + 1, 1, page, NULL); + up_read(¤t->mm->mmap_sem); + if (npages != 1) { + get_page(bad_page); + return bad_page; + } + return page[0]; + } + get_page(slot->phys_mem[gfn - slot->base_gfn]); return slot->phys_mem[gfn - slot->base_gfn]; } EXPORT_SYMBOL_GPL(gfn_to_page); +void kvm_release_page(struct page *page) +{ + if (!PageReserved(page)) + SetPageDirty(page); + put_page(page); +} +EXPORT_SYMBOL_GPL(kvm_release_page); + static int next_segment(unsigned long len, int offset) { if (len > PAGE_SIZE - offset) @@ -1060,13 +1061,16 @@ int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, struct page *page; page = gfn_to_page(kvm, gfn); - if (is_error_page(page)) + if (is_error_page(page)) { + kvm_release_page(page); return -EFAULT; + } page_virt = kmap_atomic(page, KM_USER0); memcpy(data, page_virt + offset, len); kunmap_atomic(page_virt, KM_USER0); + kvm_release_page(page); return 0; } EXPORT_SYMBOL_GPL(kvm_read_guest_page); @@ -1098,14 +1102,17 @@ int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, struct page *page; page = gfn_to_page(kvm, gfn); - if (is_error_page(page)) + if (is_error_page(page)) { + kvm_release_page(page); return -EFAULT; + } page_virt = kmap_atomic(page, KM_USER0); memcpy(page_virt + offset, data, len); kunmap_atomic(page_virt, KM_USER0); mark_page_dirty(kvm, gfn); + kvm_release_page(page); return 0; } EXPORT_SYMBOL_GPL(kvm_write_guest_page); @@ -1136,13 +1143,16 @@ int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) struct page *page; page = gfn_to_page(kvm, gfn); - if (is_error_page(page)) + if (is_error_page(page)) { + kvm_release_page(page); return -EFAULT; + } page_virt = kmap_atomic(page, KM_USER0); memset(page_virt + offset, 0, len); kunmap_atomic(page_virt, KM_USER0); + kvm_release_page(page); return 0; } EXPORT_SYMBOL_GPL(kvm_clear_guest_page); @@ -2070,8 +2080,6 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, for (i = 0; i < nr_pages; ++i) { mutex_lock(&vcpu->kvm->lock); page = gva_to_page(vcpu, address + i * PAGE_SIZE); - if (page) - get_page(page); vcpu->pio.guest_pages[i] = page; mutex_unlock(&vcpu->kvm->lock); if (!page) { @@ -3074,9 +3082,10 @@ static struct page *kvm_vm_nopage(struct vm_area_struct *vma, pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; page = gfn_to_page(kvm, pgoff); - if (is_error_page(page)) + if (is_error_page(page)) { + kvm_release_page(page); return NOPAGE_SIGBUS; - get_page(page); + } if (type != NULL) *type = VM_FAULT_MINOR; diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 2ad14fbdcfa0..5d7af4bde595 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -425,6 +425,8 @@ static void rmap_remove(struct kvm *kvm, u64 *spte) if (!is_rmap_pte(*spte)) return; page = page_header(__pa(spte)); + kvm_release_page(pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> + PAGE_SHIFT)); rmapp = gfn_to_rmap(kvm, page->gfns[spte - page->spt]); if (!*rmapp) { printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte); @@ -911,6 +913,8 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) PT_USER_MASK; if (!was_rmapped) rmap_add(vcpu, &table[index], v >> PAGE_SHIFT); + else + kvm_release_page(pfn_to_page(p >> PAGE_SHIFT)); return 0; } @@ -925,6 +929,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) 1, 3, &table[index]); if (!new_table) { pgprintk("nonpaging_map: ENOMEM\n"); + kvm_release_page(pfn_to_page(p >> PAGE_SHIFT)); return -ENOMEM; } @@ -1039,8 +1044,11 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, paddr = gpa_to_hpa(vcpu->kvm, addr & PT64_BASE_ADDR_MASK); - if (is_error_hpa(paddr)) + if (is_error_hpa(paddr)) { + kvm_release_page(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT)); return 1; + } return nonpaging_map(vcpu, addr & PAGE_MASK, paddr); } @@ -1507,6 +1515,7 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, } else { gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va); hpa_t hpa = gpa_to_hpa(vcpu, gpa); + struct page *page; if (is_shadow_present_pte(ent) && (ent & PT64_BASE_ADDR_MASK) != hpa) @@ -1519,6 +1528,9 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, && !is_error_hpa(hpa)) printk(KERN_ERR "audit: (%s) notrap shadow," " valid guest gva %lx\n", audit_msg, va); + page = pfn_to_page((gpa & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT); + kvm_release_page(page); } } diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 572e5b6d9a7a..0f0266af3f68 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -72,7 +72,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, struct kvm_vcpu *vcpu, gva_t addr, int write_fault, int user_fault, int fetch_fault) { - struct page *page; + struct page *page = NULL; pt_element_t *table; pt_element_t pte; gfn_t table_gfn; @@ -149,6 +149,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, walker->inherited_ar &= pte; --walker->level; + kvm_release_page(page); } if (write_fault && !is_dirty_pte(pte)) { @@ -162,6 +163,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte)); } + kvm_release_page(page); walker->pte = pte; pgprintk("%s: pte %llx\n", __FUNCTION__, (u64)pte); return 1; @@ -180,6 +182,8 @@ err: walker->error_code |= PFERR_USER_MASK; if (fetch_fault) walker->error_code |= PFERR_FETCH_MASK; + if (page) + kvm_release_page(page); return 0; } @@ -223,6 +227,8 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, if (is_error_hpa(paddr)) { set_shadow_pte(shadow_pte, shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK); + kvm_release_page(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT)); return; } @@ -260,9 +266,20 @@ unshadowed: pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte); set_shadow_pte(shadow_pte, spte); page_header_update_slot(vcpu->kvm, shadow_pte, gaddr); - if (!was_rmapped) + if (!was_rmapped) { rmap_add(vcpu, shadow_pte, (gaddr & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); + if (!is_rmap_pte(*shadow_pte)) { + struct page *page; + + page = pfn_to_page((paddr & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT); + kvm_release_page(page); + } + } + else + kvm_release_page(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT)); if (!ptwrite || !*ptwrite) vcpu->last_pte_updated = shadow_pte; } @@ -486,19 +503,22 @@ static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, { int i; pt_element_t *gpt; + struct page *page; if (sp->role.metaphysical || PTTYPE == 32) { nonpaging_prefetch_page(vcpu, sp); return; } - gpt = kmap_atomic(gfn_to_page(vcpu->kvm, sp->gfn), KM_USER0); + page = gfn_to_page(vcpu->kvm, sp->gfn); + gpt = kmap_atomic(page, KM_USER0); for (i = 0; i < PT64_ENT_PER_PAGE; ++i) if (is_present_pte(gpt[i])) sp->spt[i] = shadow_trap_nonpresent_pte; else sp->spt[i] = shadow_notrap_nonpresent_pte; kunmap_atomic(gpt, KM_USER0); + kvm_release_page(page); } #undef pt_element_t From e56a7a28e21aa2d1af659d8f38952411ce9ae40e Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Thu, 18 Oct 2007 14:39:10 +0200 Subject: [PATCH 049/249] KVM: Use virtual cpu accounting if available for guest times. ppc and s390 offer the possibility to track process times precisely by looking at cpu timer on every context switch, irq, softirq etc. We can use that infrastructure as well for guest time accounting. We need to account the used time before we change the state. This patch adds a call to account_system_vtime to kvm_guest_enter and kvm_guest exit. If CONFIG_VIRT_CPU_ACCOUNTING is not set, account_system_vtime is defined in hardirq.h as an empty function, which means this patch does not change the behaviour on other platforms. I compile tested this patch on x86 and function tested the patch on s390. Signed-off-by: Christian Borntraeger Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index df0711ce9baa..e8a21e8cb2ad 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -671,11 +672,13 @@ __init void kvm_arch_init(void); static inline void kvm_guest_enter(void) { + account_system_vtime(current); current->flags |= PF_VCPU; } static inline void kvm_guest_exit(void) { + account_system_vtime(current); current->flags &= ~PF_VCPU; } From 8d4e1288ebb753d3140d81cb349f22b0a6829a4a Mon Sep 17 00:00:00 2001 From: Anthony Liguori Date: Thu, 18 Oct 2007 09:59:34 -0500 Subject: [PATCH 050/249] KVM: Allocate userspace memory for older userspace Allocate a userspace buffer for older userspaces. Also eliminate phys_mem buffer. The memset() in kvmctl really kills initial memory usage but swapping works even with old userspaces. A side effect is that maximum guest side is reduced for older userspace on i386. Signed-off-by: Anthony Liguori Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 2 -- drivers/kvm/kvm_main.c | 81 +++++++++++++++--------------------------- 2 files changed, 29 insertions(+), 54 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index e8a21e8cb2ad..eb006ed696c1 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -406,10 +406,8 @@ struct kvm_memory_slot { gfn_t base_gfn; unsigned long npages; unsigned long flags; - struct page **phys_mem; unsigned long *rmap; unsigned long *dirty_bitmap; - int user_alloc; /* user allocated memory */ unsigned long userspace_addr; }; diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index f86a47c2f255..8f6c21d02656 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include @@ -300,36 +301,21 @@ static struct kvm *kvm_create_vm(void) return kvm; } -static void kvm_free_kernel_physmem(struct kvm_memory_slot *free) -{ - int i; - - for (i = 0; i < free->npages; ++i) - if (free->phys_mem[i]) - __free_page(free->phys_mem[i]); -} - /* * Free any memory in @free but not in @dont. */ static void kvm_free_physmem_slot(struct kvm_memory_slot *free, struct kvm_memory_slot *dont) { - if (!dont || free->phys_mem != dont->phys_mem) - if (free->phys_mem) { - if (!free->user_alloc) - kvm_free_kernel_physmem(free); - vfree(free->phys_mem); - } if (!dont || free->rmap != dont->rmap) vfree(free->rmap); if (!dont || free->dirty_bitmap != dont->dirty_bitmap) vfree(free->dirty_bitmap); - free->phys_mem = NULL; free->npages = 0; free->dirty_bitmap = NULL; + free->rmap = NULL; } static void kvm_free_physmem(struct kvm *kvm) @@ -712,10 +698,6 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, goto out_unlock; } - /* Deallocate if slot is being removed */ - if (!npages) - new.phys_mem = NULL; - /* Free page dirty bitmap if unneeded */ if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) new.dirty_bitmap = NULL; @@ -723,29 +705,27 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, r = -ENOMEM; /* Allocate if a slot is being created */ - if (npages && !new.phys_mem) { - new.phys_mem = vmalloc(npages * sizeof(struct page *)); - - if (!new.phys_mem) - goto out_unlock; - + if (npages && !new.rmap) { new.rmap = vmalloc(npages * sizeof(struct page *)); if (!new.rmap) goto out_unlock; - memset(new.phys_mem, 0, npages * sizeof(struct page *)); memset(new.rmap, 0, npages * sizeof(*new.rmap)); - if (user_alloc) { - new.user_alloc = 1; + + if (user_alloc) new.userspace_addr = mem->userspace_addr; - } else { - for (i = 0; i < npages; ++i) { - new.phys_mem[i] = alloc_page(GFP_HIGHUSER - | __GFP_ZERO); - if (!new.phys_mem[i]) - goto out_unlock; - } + else { + down_write(¤t->mm->mmap_sem); + new.userspace_addr = do_mmap(NULL, 0, + npages * PAGE_SIZE, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, + 0); + up_write(¤t->mm->mmap_sem); + + if (IS_ERR((void *)new.userspace_addr)) + goto out_unlock; } } @@ -1010,6 +990,8 @@ struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) { struct kvm_memory_slot *slot; + struct page *page[1]; + int npages; gfn = unalias_gfn(kvm, gfn); slot = __gfn_to_memslot(kvm, gfn); @@ -1017,24 +999,19 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) get_page(bad_page); return bad_page; } - if (slot->user_alloc) { - struct page *page[1]; - int npages; - down_read(¤t->mm->mmap_sem); - npages = get_user_pages(current, current->mm, - slot->userspace_addr - + (gfn - slot->base_gfn) * PAGE_SIZE, 1, - 1, 1, page, NULL); - up_read(¤t->mm->mmap_sem); - if (npages != 1) { - get_page(bad_page); - return bad_page; - } - return page[0]; + down_read(¤t->mm->mmap_sem); + npages = get_user_pages(current, current->mm, + slot->userspace_addr + + (gfn - slot->base_gfn) * PAGE_SIZE, 1, + 1, 1, page, NULL); + up_read(¤t->mm->mmap_sem); + if (npages != 1) { + get_page(bad_page); + return bad_page; } - get_page(slot->phys_mem[gfn - slot->base_gfn]); - return slot->phys_mem[gfn - slot->base_gfn]; + + return page[0]; } EXPORT_SYMBOL_GPL(gfn_to_page); From 34c16eecf78ed4cf01f39ac7211f5b57942ec899 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Sat, 20 Oct 2007 15:34:38 +0800 Subject: [PATCH 051/249] KVM: Portability: Split kvm_vcpu into arch dependent and independent parts (part 1) First step to split kvm_vcpu. Currently, we just use an macro to define the common fields in kvm_vcpu for all archs, and all archs need to define its own kvm_vcpu struct. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/ioapic.c | 2 + drivers/kvm/irq.c | 1 + drivers/kvm/kvm.h | 154 +++++++------------------------------- drivers/kvm/kvm_main.c | 4 +- drivers/kvm/lapic.c | 2 + drivers/kvm/mmu.c | 1 + drivers/kvm/svm.c | 2 +- drivers/kvm/vmx.c | 1 + drivers/kvm/x86.h | 117 +++++++++++++++++++++++++++++ drivers/kvm/x86_emulate.c | 1 + 10 files changed, 155 insertions(+), 130 deletions(-) diff --git a/drivers/kvm/ioapic.c b/drivers/kvm/ioapic.c index 8503d99b8339..e14b7c724e67 100644 --- a/drivers/kvm/ioapic.c +++ b/drivers/kvm/ioapic.c @@ -27,6 +27,8 @@ */ #include "kvm.h" +#include "x86.h" + #include #include #include diff --git a/drivers/kvm/irq.c b/drivers/kvm/irq.c index 7628c7ff628f..59b47c55fc76 100644 --- a/drivers/kvm/irq.c +++ b/drivers/kvm/irq.c @@ -22,6 +22,7 @@ #include #include "kvm.h" +#include "x86.h" #include "irq.h" /* diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index eb006ed696c1..db18d278c1c0 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -308,93 +308,37 @@ struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr); void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev); -struct kvm_vcpu { - struct kvm *kvm; - struct preempt_notifier preempt_notifier; - int vcpu_id; - struct mutex mutex; - int cpu; - u64 host_tsc; - struct kvm_run *run; - int interrupt_window_open; - int guest_mode; - unsigned long requests; - unsigned long irq_summary; /* bit vector: 1 per word in irq_pending */ - DECLARE_BITMAP(irq_pending, KVM_NR_INTERRUPTS); - unsigned long regs[NR_VCPU_REGS]; /* for rsp: vcpu_load_rsp_rip() */ - unsigned long rip; /* needs vcpu_load_rsp_rip() */ - - unsigned long cr0; - unsigned long cr2; - unsigned long cr3; - unsigned long cr4; - unsigned long cr8; - u64 pdptrs[4]; /* pae */ - u64 shadow_efer; - u64 apic_base; - struct kvm_lapic *apic; /* kernel irqchip context */ -#define VCPU_MP_STATE_RUNNABLE 0 -#define VCPU_MP_STATE_UNINITIALIZED 1 -#define VCPU_MP_STATE_INIT_RECEIVED 2 -#define VCPU_MP_STATE_SIPI_RECEIVED 3 -#define VCPU_MP_STATE_HALTED 4 - int mp_state; - int sipi_vector; - u64 ia32_misc_enable_msr; - - struct kvm_mmu mmu; - - struct kvm_mmu_memory_cache mmu_pte_chain_cache; - struct kvm_mmu_memory_cache mmu_rmap_desc_cache; - struct kvm_mmu_memory_cache mmu_page_cache; - struct kvm_mmu_memory_cache mmu_page_header_cache; - - gfn_t last_pt_write_gfn; - int last_pt_write_count; - u64 *last_pte_updated; - - struct kvm_guest_debug guest_debug; - - struct i387_fxsave_struct host_fx_image; - struct i387_fxsave_struct guest_fx_image; - int fpu_active; - int guest_fpu_loaded; - - int mmio_needed; - int mmio_read_completed; - int mmio_is_write; - int mmio_size; - unsigned char mmio_data[8]; +#ifdef CONFIG_HAS_IOMEM +#define KVM_VCPU_MMIO \ + int mmio_needed; \ + int mmio_read_completed; \ + int mmio_is_write; \ + int mmio_size; \ + unsigned char mmio_data[8]; \ gpa_t mmio_phys_addr; - gva_t mmio_fault_cr2; - struct kvm_pio_request pio; - void *pio_data; - wait_queue_head_t wq; - int sigset_active; - sigset_t sigset; +#else +#define KVM_VCPU_MMIO - struct kvm_stat stat; +#endif - struct { - int active; - u8 save_iopl; - struct kvm_save_segment { - u16 selector; - unsigned long base; - u32 limit; - u32 ar; - } tr, es, ds, fs, gs; - } rmode; - int halt_request; /* real mode on Intel only */ - - int cpuid_nent; - struct kvm_cpuid_entry cpuid_entries[KVM_MAX_CPUID_ENTRIES]; - - /* emulate context */ - - struct x86_emulate_ctxt emulate_ctxt; -}; +#define KVM_VCPU_COMM \ + struct kvm *kvm; \ + struct preempt_notifier preempt_notifier; \ + int vcpu_id; \ + struct mutex mutex; \ + int cpu; \ + struct kvm_run *run; \ + int guest_mode; \ + unsigned long requests; \ + struct kvm_guest_debug guest_debug; \ + int fpu_active; \ + int guest_fpu_loaded; \ + wait_queue_head_t wq; \ + int sigset_active; \ + sigset_t sigset; \ + struct kvm_stat stat; \ + KVM_VCPU_MMIO struct kvm_mem_alias { gfn_t base_gfn; @@ -680,50 +624,6 @@ static inline void kvm_guest_exit(void) current->flags &= ~PF_VCPU; } -static inline int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, - u32 error_code) -{ - return vcpu->mmu.page_fault(vcpu, gva, error_code); -} - -static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) -{ - if (unlikely(vcpu->kvm->n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES)) - __kvm_mmu_free_some_pages(vcpu); -} - -static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) -{ - if (likely(vcpu->mmu.root_hpa != INVALID_PAGE)) - return 0; - - return kvm_mmu_load(vcpu); -} - -static inline int is_long_mode(struct kvm_vcpu *vcpu) -{ -#ifdef CONFIG_X86_64 - return vcpu->shadow_efer & EFER_LME; -#else - return 0; -#endif -} - -static inline int is_pae(struct kvm_vcpu *vcpu) -{ - return vcpu->cr4 & X86_CR4_PAE; -} - -static inline int is_pse(struct kvm_vcpu *vcpu) -{ - return vcpu->cr4 & X86_CR4_PSE; -} - -static inline int is_paging(struct kvm_vcpu *vcpu) -{ - return vcpu->cr0 & X86_CR0_PG; -} - static inline int memslot_id(struct kvm *kvm, struct kvm_memory_slot *slot) { return slot - kvm->memslots; diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 8f6c21d02656..0b23657f434c 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -2244,7 +2244,7 @@ static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) if (r) goto out; } - +#if CONFIG_HAS_IOMEM if (vcpu->mmio_needed) { memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8); vcpu->mmio_read_completed = 1; @@ -2259,7 +2259,7 @@ static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) goto out; } } - +#endif if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) { kvm_x86_ops->cache_regs(vcpu); vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret; diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index 8840f9dc0bca..64f74bd7093a 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -18,6 +18,8 @@ */ #include "kvm.h" +#include "x86.h" + #include #include #include diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 5d7af4bde595..d9c5950cfae1 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -19,6 +19,7 @@ #include "vmx.h" #include "kvm.h" +#include "x86.h" #include #include diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index ef068d2dddd7..035c8e6898f1 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -13,7 +13,7 @@ * the COPYING file in the top-level directory. * */ - +#include "x86.h" #include "kvm_svm.h" #include "x86_emulate.h" #include "irq.h" diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 9f77ddbeb025..87ff35128825 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -16,6 +16,7 @@ */ #include "kvm.h" +#include "x86.h" #include "x86_emulate.h" #include "irq.h" #include "vmx.h" diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 1e2f71bd805d..01452b552db3 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -13,4 +13,121 @@ #include "kvm.h" +#include +#include + +#include +#include + +struct kvm_vcpu { + KVM_VCPU_COMM; + u64 host_tsc; + int interrupt_window_open; + unsigned long irq_summary; /* bit vector: 1 per word in irq_pending */ + DECLARE_BITMAP(irq_pending, KVM_NR_INTERRUPTS); + unsigned long regs[NR_VCPU_REGS]; /* for rsp: vcpu_load_rsp_rip() */ + unsigned long rip; /* needs vcpu_load_rsp_rip() */ + + unsigned long cr0; + unsigned long cr2; + unsigned long cr3; + unsigned long cr4; + unsigned long cr8; + u64 pdptrs[4]; /* pae */ + u64 shadow_efer; + u64 apic_base; + struct kvm_lapic *apic; /* kernel irqchip context */ +#define VCPU_MP_STATE_RUNNABLE 0 +#define VCPU_MP_STATE_UNINITIALIZED 1 +#define VCPU_MP_STATE_INIT_RECEIVED 2 +#define VCPU_MP_STATE_SIPI_RECEIVED 3 +#define VCPU_MP_STATE_HALTED 4 + int mp_state; + int sipi_vector; + u64 ia32_misc_enable_msr; + + struct kvm_mmu mmu; + + struct kvm_mmu_memory_cache mmu_pte_chain_cache; + struct kvm_mmu_memory_cache mmu_rmap_desc_cache; + struct kvm_mmu_memory_cache mmu_page_cache; + struct kvm_mmu_memory_cache mmu_page_header_cache; + + gfn_t last_pt_write_gfn; + int last_pt_write_count; + u64 *last_pte_updated; + + + struct i387_fxsave_struct host_fx_image; + struct i387_fxsave_struct guest_fx_image; + + gva_t mmio_fault_cr2; + struct kvm_pio_request pio; + void *pio_data; + + struct { + int active; + u8 save_iopl; + struct kvm_save_segment { + u16 selector; + unsigned long base; + u32 limit; + u32 ar; + } tr, es, ds, fs, gs; + } rmode; + int halt_request; /* real mode on Intel only */ + + int cpuid_nent; + struct kvm_cpuid_entry cpuid_entries[KVM_MAX_CPUID_ENTRIES]; + + /* emulate context */ + + struct x86_emulate_ctxt emulate_ctxt; +}; + +static inline int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, + u32 error_code) +{ + return vcpu->mmu.page_fault(vcpu, gva, error_code); +} + +static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) +{ + if (unlikely(vcpu->kvm->n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES)) + __kvm_mmu_free_some_pages(vcpu); +} + +static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) +{ + if (likely(vcpu->mmu.root_hpa != INVALID_PAGE)) + return 0; + + return kvm_mmu_load(vcpu); +} + +static inline int is_long_mode(struct kvm_vcpu *vcpu) +{ +#ifdef CONFIG_X86_64 + return vcpu->shadow_efer & EFER_LME; +#else + return 0; +#endif +} + +static inline int is_pae(struct kvm_vcpu *vcpu) +{ + return vcpu->cr4 & X86_CR4_PAE; +} + +static inline int is_pse(struct kvm_vcpu *vcpu) +{ + return vcpu->cr4 & X86_CR4_PSE; +} + +static inline int is_paging(struct kvm_vcpu *vcpu) +{ + return vcpu->cr0 & X86_CR0_PG; +} + + #endif diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index e962de331606..73e3580c88e4 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -26,6 +26,7 @@ #define DPRINTF(_f, _a ...) printf(_f , ## _a) #else #include "kvm.h" +#include "x86.h" #define DPRINTF(x...) do {} while (0) #endif #include "x86_emulate.h" From e00c8cf29b9798eb9918469b0cce1766e0ae40d7 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 21 Oct 2007 11:00:39 +0200 Subject: [PATCH 052/249] KVM: Move vmx_vcpu_reset() out of vmx_vcpu_setup() Split guest reset code out of vmx_vcpu_setup(). Besides being cleaner, this moves the realmode tss setup (which can sleep) outside vmx_vcpu_setup() (which is executed with preemption enabled). [izik: remove unused variable] Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 2 +- drivers/kvm/kvm_main.c | 8 +- drivers/kvm/svm.c | 4 +- drivers/kvm/vmx.c | 178 ++++++++++++++++++++--------------------- 4 files changed, 99 insertions(+), 93 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index db18d278c1c0..f7181a407be1 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -412,7 +412,7 @@ struct kvm_x86_ops { /* Create, but do not attach this VCPU */ struct kvm_vcpu *(*vcpu_create)(struct kvm *kvm, unsigned id); void (*vcpu_free)(struct kvm_vcpu *vcpu); - void (*vcpu_reset)(struct kvm_vcpu *vcpu); + int (*vcpu_reset)(struct kvm_vcpu *vcpu); void (*prepare_guest_switch)(struct kvm_vcpu *vcpu); void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 0b23657f434c..a6e3165043d1 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -2122,7 +2122,9 @@ static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) pr_debug("vcpu %d received sipi with vector # %x\n", vcpu->vcpu_id, vcpu->sipi_vector); kvm_lapic_reset(vcpu); - kvm_x86_ops->vcpu_reset(vcpu); + r = kvm_x86_ops->vcpu_reset(vcpu); + if (r) + return r; vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; } @@ -2637,7 +2639,9 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n) BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF); vcpu_load(vcpu); - r = kvm_mmu_setup(vcpu); + r = kvm_x86_ops->vcpu_reset(vcpu); + if (r == 0) + r = kvm_mmu_setup(vcpu); vcpu_put(vcpu); if (r < 0) goto free_vcpu; diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 035c8e6898f1..953c111544e1 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -557,7 +557,7 @@ static void init_vmcb(struct vmcb *vmcb) /* rdx = ?? */ } -static void svm_vcpu_reset(struct kvm_vcpu *vcpu) +static int svm_vcpu_reset(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); @@ -568,6 +568,8 @@ static void svm_vcpu_reset(struct kvm_vcpu *vcpu) svm->vmcb->save.cs.base = svm->vcpu.sipi_vector << 12; svm->vmcb->save.cs.selector = svm->vcpu.sipi_vector << 8; } + + return 0; } static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 87ff35128825..768ea88beb45 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1433,92 +1433,15 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) unsigned long a; struct descriptor_table dt; int i; - int ret = 0; unsigned long kvm_vmx_return; - u64 msr; u32 exec_control; - if (!init_rmode_tss(vmx->vcpu.kvm)) { - ret = -ENOMEM; - goto out; - } - - vmx->vcpu.rmode.active = 0; - - vmx->vcpu.regs[VCPU_REGS_RDX] = get_rdx_init_val(); - set_cr8(&vmx->vcpu, 0); - msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE; - if (vmx->vcpu.vcpu_id == 0) - msr |= MSR_IA32_APICBASE_BSP; - kvm_set_apic_base(&vmx->vcpu, msr); - - fx_init(&vmx->vcpu); - - /* - * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode - * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4. Sigh. - */ - if (vmx->vcpu.vcpu_id == 0) { - vmcs_write16(GUEST_CS_SELECTOR, 0xf000); - vmcs_writel(GUEST_CS_BASE, 0x000f0000); - } else { - vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.sipi_vector << 8); - vmcs_writel(GUEST_CS_BASE, vmx->vcpu.sipi_vector << 12); - } - vmcs_write32(GUEST_CS_LIMIT, 0xffff); - vmcs_write32(GUEST_CS_AR_BYTES, 0x9b); - - seg_setup(VCPU_SREG_DS); - seg_setup(VCPU_SREG_ES); - seg_setup(VCPU_SREG_FS); - seg_setup(VCPU_SREG_GS); - seg_setup(VCPU_SREG_SS); - - vmcs_write16(GUEST_TR_SELECTOR, 0); - vmcs_writel(GUEST_TR_BASE, 0); - vmcs_write32(GUEST_TR_LIMIT, 0xffff); - vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); - - vmcs_write16(GUEST_LDTR_SELECTOR, 0); - vmcs_writel(GUEST_LDTR_BASE, 0); - vmcs_write32(GUEST_LDTR_LIMIT, 0xffff); - vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082); - - vmcs_write32(GUEST_SYSENTER_CS, 0); - vmcs_writel(GUEST_SYSENTER_ESP, 0); - vmcs_writel(GUEST_SYSENTER_EIP, 0); - - vmcs_writel(GUEST_RFLAGS, 0x02); - if (vmx->vcpu.vcpu_id == 0) - vmcs_writel(GUEST_RIP, 0xfff0); - else - vmcs_writel(GUEST_RIP, 0); - vmcs_writel(GUEST_RSP, 0); - - /* todo: dr0 = dr1 = dr2 = dr3 = 0; dr6 = 0xffff0ff0 */ - vmcs_writel(GUEST_DR7, 0x400); - - vmcs_writel(GUEST_GDTR_BASE, 0); - vmcs_write32(GUEST_GDTR_LIMIT, 0xffff); - - vmcs_writel(GUEST_IDTR_BASE, 0); - vmcs_write32(GUEST_IDTR_LIMIT, 0xffff); - - vmcs_write32(GUEST_ACTIVITY_STATE, 0); - vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); - vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0); - /* I/O */ vmcs_write64(IO_BITMAP_A, page_to_phys(vmx_io_bitmap_a)); vmcs_write64(IO_BITMAP_B, page_to_phys(vmx_io_bitmap_b)); - guest_write_tsc(0); - vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */ - /* Special registers */ - vmcs_write64(GUEST_IA32_DEBUGCTL, 0); - /* Control */ vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, vmcs_config.pin_based_exec_ctrl); @@ -1593,13 +1516,100 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) ++vmx->nmsrs; } - setup_msrs(vmx); - vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl); /* 22.2.1, 20.8.1 */ vmcs_write32(VM_ENTRY_CONTROLS, vmcs_config.vmentry_ctrl); + vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL); + vmcs_writel(CR4_GUEST_HOST_MASK, KVM_GUEST_CR4_MASK); + + return 0; +} + +static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + u64 msr; + int ret; + + if (!init_rmode_tss(vmx->vcpu.kvm)) { + ret = -ENOMEM; + goto out; + } + + vmx->vcpu.rmode.active = 0; + + vmx->vcpu.regs[VCPU_REGS_RDX] = get_rdx_init_val(); + set_cr8(&vmx->vcpu, 0); + msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE; + if (vmx->vcpu.vcpu_id == 0) + msr |= MSR_IA32_APICBASE_BSP; + kvm_set_apic_base(&vmx->vcpu, msr); + + fx_init(&vmx->vcpu); + + /* + * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode + * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4. Sigh. + */ + if (vmx->vcpu.vcpu_id == 0) { + vmcs_write16(GUEST_CS_SELECTOR, 0xf000); + vmcs_writel(GUEST_CS_BASE, 0x000f0000); + } else { + vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.sipi_vector << 8); + vmcs_writel(GUEST_CS_BASE, vmx->vcpu.sipi_vector << 12); + } + vmcs_write32(GUEST_CS_LIMIT, 0xffff); + vmcs_write32(GUEST_CS_AR_BYTES, 0x9b); + + seg_setup(VCPU_SREG_DS); + seg_setup(VCPU_SREG_ES); + seg_setup(VCPU_SREG_FS); + seg_setup(VCPU_SREG_GS); + seg_setup(VCPU_SREG_SS); + + vmcs_write16(GUEST_TR_SELECTOR, 0); + vmcs_writel(GUEST_TR_BASE, 0); + vmcs_write32(GUEST_TR_LIMIT, 0xffff); + vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); + + vmcs_write16(GUEST_LDTR_SELECTOR, 0); + vmcs_writel(GUEST_LDTR_BASE, 0); + vmcs_write32(GUEST_LDTR_LIMIT, 0xffff); + vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082); + + vmcs_write32(GUEST_SYSENTER_CS, 0); + vmcs_writel(GUEST_SYSENTER_ESP, 0); + vmcs_writel(GUEST_SYSENTER_EIP, 0); + + vmcs_writel(GUEST_RFLAGS, 0x02); + if (vmx->vcpu.vcpu_id == 0) + vmcs_writel(GUEST_RIP, 0xfff0); + else + vmcs_writel(GUEST_RIP, 0); + vmcs_writel(GUEST_RSP, 0); + + /* todo: dr0 = dr1 = dr2 = dr3 = 0; dr6 = 0xffff0ff0 */ + vmcs_writel(GUEST_DR7, 0x400); + + vmcs_writel(GUEST_GDTR_BASE, 0); + vmcs_write32(GUEST_GDTR_LIMIT, 0xffff); + + vmcs_writel(GUEST_IDTR_BASE, 0); + vmcs_write32(GUEST_IDTR_LIMIT, 0xffff); + + vmcs_write32(GUEST_ACTIVITY_STATE, 0); + vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); + vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0); + + guest_write_tsc(0); + + /* Special registers */ + vmcs_write64(GUEST_IA32_DEBUGCTL, 0); + + setup_msrs(vmx); + vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */ #ifdef CONFIG_X86_64 @@ -1610,9 +1620,6 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) vmcs_write32(TPR_THRESHOLD, 0); #endif - vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL); - vmcs_writel(CR4_GUEST_HOST_MASK, KVM_GUEST_CR4_MASK); - vmx->vcpu.cr0 = 0x60000010; vmx_set_cr0(&vmx->vcpu, vmx->vcpu.cr0); /* enter rmode */ vmx_set_cr4(&vmx->vcpu, 0); @@ -1628,13 +1635,6 @@ out: return ret; } -static void vmx_vcpu_reset(struct kvm_vcpu *vcpu) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - - vmx_vcpu_setup(vmx); -} - static void inject_rmode_irq(struct kvm_vcpu *vcpu, int irq) { u16 ent[2]; From 60395224d94945544f1f9dce5566981844bf0e77 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 21 Oct 2007 11:03:36 +0200 Subject: [PATCH 053/249] KVM: Add a might_sleep() annotation to gfn_to_page() This will help trap accesses to guest memory in atomic context. Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index a6e3165043d1..453e98e251da 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -993,6 +993,8 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) struct page *page[1]; int npages; + might_sleep(); + gfn = unalias_gfn(kvm, gfn); slot = __gfn_to_memslot(kvm, gfn); if (!slot) { From 2fcceae145645ec6a4fab0a806bb674d7484d97b Mon Sep 17 00:00:00 2001 From: Eddie Dong Date: Wed, 10 Oct 2007 12:14:25 +0200 Subject: [PATCH 054/249] KVM: Export PIC reset for kernel device reset Signed-off-by: Yaozu (Eddie) Dong Signed-off-by: Avi Kivity --- drivers/kvm/i8259.c | 6 ++---- drivers/kvm/irq.h | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/kvm/i8259.c b/drivers/kvm/i8259.c index a679157bc599..f0dc2eeadf0b 100644 --- a/drivers/kvm/i8259.c +++ b/drivers/kvm/i8259.c @@ -181,10 +181,8 @@ int kvm_pic_read_irq(struct kvm_pic *s) return intno; } -static void pic_reset(void *opaque) +void kvm_pic_reset(struct kvm_kpic_state *s) { - struct kvm_kpic_state *s = opaque; - s->last_irr = 0; s->irr = 0; s->imr = 0; @@ -209,7 +207,7 @@ static void pic_ioport_write(void *opaque, u32 addr, u32 val) addr &= 1; if (addr == 0) { if (val & 0x10) { - pic_reset(s); /* init */ + kvm_pic_reset(s); /* init */ /* * deassert a pending interrupt */ diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index 508280ea6fcf..e08ae1002bd8 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -139,6 +139,7 @@ int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu); int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu); int kvm_create_lapic(struct kvm_vcpu *vcpu); void kvm_lapic_reset(struct kvm_vcpu *vcpu); +void kvm_pic_reset(struct kvm_kpic_state *s); void kvm_free_lapic(struct kvm_vcpu *vcpu); u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu); void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8); From 8c392696e749171531d155ea5cefdfc1c970fd18 Mon Sep 17 00:00:00 2001 From: Eddie Dong Date: Wed, 10 Oct 2007 12:15:54 +0200 Subject: [PATCH 055/249] KVM: Split IOAPIC reset function and export for kernel RESET Signed-off-by: Yaozu (Eddie) Dong Signed-off-by: Avi Kivity --- drivers/kvm/ioapic.c | 17 +++++++++++++---- drivers/kvm/irq.h | 1 + 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/kvm/ioapic.c b/drivers/kvm/ioapic.c index e14b7c724e67..cf1d50b3549b 100644 --- a/drivers/kvm/ioapic.c +++ b/drivers/kvm/ioapic.c @@ -371,18 +371,27 @@ static void ioapic_mmio_write(struct kvm_io_device *this, gpa_t addr, int len, } } +void kvm_ioapic_reset(struct kvm_ioapic *ioapic) +{ + int i; + + for (i = 0; i < IOAPIC_NUM_PINS; i++) + ioapic->redirtbl[i].fields.mask = 1; + ioapic->base_address = IOAPIC_DEFAULT_BASE_ADDRESS; + ioapic->ioregsel = 0; + ioapic->irr = 0; + ioapic->id = 0; +} + int kvm_ioapic_init(struct kvm *kvm) { struct kvm_ioapic *ioapic; - int i; ioapic = kzalloc(sizeof(struct kvm_ioapic), GFP_KERNEL); if (!ioapic) return -ENOMEM; kvm->vioapic = ioapic; - for (i = 0; i < IOAPIC_NUM_PINS; i++) - ioapic->redirtbl[i].fields.mask = 1; - ioapic->base_address = IOAPIC_DEFAULT_BASE_ADDRESS; + kvm_ioapic_reset(ioapic); ioapic->dev.read = ioapic_mmio_read; ioapic->dev.write = ioapic_mmio_write; ioapic->dev.in_range = ioapic_in_range; diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index e08ae1002bd8..5ad3cfd7622d 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -140,6 +140,7 @@ int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu); int kvm_create_lapic(struct kvm_vcpu *vcpu); void kvm_lapic_reset(struct kvm_vcpu *vcpu); void kvm_pic_reset(struct kvm_kpic_state *s); +void kvm_ioapic_reset(struct kvm_ioapic *ioapic); void kvm_free_lapic(struct kvm_vcpu *vcpu); u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu); void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8); From 5f43238d036fb30e73563e81e42d9c6f1de5551a Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Thu, 11 Oct 2007 15:34:17 +0200 Subject: [PATCH 056/249] KVM: Per-architecture hypercall definitions Currently kvm provides hypercalls only for x86* architectures. To provide hypercall infrastructure for other kvm architectures I split kvm_para.h into a generic header file and architecture specific definitions. Signed-off-by: Christian Borntraeger Signed-off-by: Avi Kivity --- include/asm-x86/kvm_para.h | 105 +++++++++++++++++++++++++++++++++++++ include/linux/kvm_para.h | 105 +++++-------------------------------- 2 files changed, 117 insertions(+), 93 deletions(-) create mode 100644 include/asm-x86/kvm_para.h diff --git a/include/asm-x86/kvm_para.h b/include/asm-x86/kvm_para.h new file mode 100644 index 000000000000..c6f3fd8d8c53 --- /dev/null +++ b/include/asm-x86/kvm_para.h @@ -0,0 +1,105 @@ +#ifndef __X86_KVM_PARA_H +#define __X86_KVM_PARA_H + +/* This CPUID returns the signature 'KVMKVMKVM' in ebx, ecx, and edx. It + * should be used to determine that a VM is running under KVM. + */ +#define KVM_CPUID_SIGNATURE 0x40000000 + +/* This CPUID returns a feature bitmap in eax. Before enabling a particular + * paravirtualization, the appropriate feature bit should be checked. + */ +#define KVM_CPUID_FEATURES 0x40000001 + +#ifdef __KERNEL__ +#include + +/* This instruction is vmcall. On non-VT architectures, it will generate a + * trap that we will then rewrite to the appropriate instruction. + */ +#define KVM_HYPERCALL ".byte 0x0f,0x01,0xc1" + +/* For KVM hypercalls, a three-byte sequence of either the vmrun or the vmmrun + * instruction. The hypervisor may replace it with something else but only the + * instructions are guaranteed to be supported. + * + * Up to four arguments may be passed in rbx, rcx, rdx, and rsi respectively. + * The hypercall number should be placed in rax and the return value will be + * placed in rax. No other registers will be clobbered unless explicited + * noted by the particular hypercall. + */ + +static inline long kvm_hypercall0(unsigned int nr) +{ + long ret; + asm volatile(KVM_HYPERCALL + : "=a"(ret) + : "a"(nr)); + return ret; +} + +static inline long kvm_hypercall1(unsigned int nr, unsigned long p1) +{ + long ret; + asm volatile(KVM_HYPERCALL + : "=a"(ret) + : "a"(nr), "b"(p1)); + return ret; +} + +static inline long kvm_hypercall2(unsigned int nr, unsigned long p1, + unsigned long p2) +{ + long ret; + asm volatile(KVM_HYPERCALL + : "=a"(ret) + : "a"(nr), "b"(p1), "c"(p2)); + return ret; +} + +static inline long kvm_hypercall3(unsigned int nr, unsigned long p1, + unsigned long p2, unsigned long p3) +{ + long ret; + asm volatile(KVM_HYPERCALL + : "=a"(ret) + : "a"(nr), "b"(p1), "c"(p2), "d"(p3)); + return ret; +} + +static inline long kvm_hypercall4(unsigned int nr, unsigned long p1, + unsigned long p2, unsigned long p3, + unsigned long p4) +{ + long ret; + asm volatile(KVM_HYPERCALL + : "=a"(ret) + : "a"(nr), "b"(p1), "c"(p2), "d"(p3), "S"(p4)); + return ret; +} + +static inline int kvm_para_available(void) +{ + unsigned int eax, ebx, ecx, edx; + char signature[13]; + + cpuid(KVM_CPUID_SIGNATURE, &eax, &ebx, &ecx, &edx); + memcpy(signature + 0, &ebx, 4); + memcpy(signature + 4, &ecx, 4); + memcpy(signature + 8, &edx, 4); + signature[12] = 0; + + if (strcmp(signature, "KVMKVMKVM") == 0) + return 1; + + return 0; +} + +static inline unsigned int kvm_arch_para_features(void) +{ + return cpuid_eax(KVM_CPUID_FEATURES); +} + +#endif + +#endif diff --git a/include/linux/kvm_para.h b/include/linux/kvm_para.h index cc5dfb433706..e4db25ffdb52 100644 --- a/include/linux/kvm_para.h +++ b/include/linux/kvm_para.h @@ -1,110 +1,29 @@ #ifndef __LINUX_KVM_PARA_H #define __LINUX_KVM_PARA_H -/* This CPUID returns the signature 'KVMKVMKVM' in ebx, ecx, and edx. It - * should be used to determine that a VM is running under KVM. +/* + * This header file provides a method for making a hypercall to the host + * Architectures should define: + * - kvm_hypercall0, kvm_hypercall1... + * - kvm_arch_para_features + * - kvm_para_available */ -#define KVM_CPUID_SIGNATURE 0x40000000 - -/* This CPUID returns a feature bitmap in eax. Before enabling a particular - * paravirtualization, the appropriate feature bit should be checked. - */ -#define KVM_CPUID_FEATURES 0x40000001 /* Return values for hypercalls */ #define KVM_ENOSYS 1000 #ifdef __KERNEL__ -#include - -/* This instruction is vmcall. On non-VT architectures, it will generate a - * trap that we will then rewrite to the appropriate instruction. +/* + * hypercalls use architecture specific */ -#define KVM_HYPERCALL ".byte 0x0f,0x01,0xc1" - -/* For KVM hypercalls, a three-byte sequence of either the vmrun or the vmmrun - * instruction. The hypervisor may replace it with something else but only the - * instructions are guaranteed to be supported. - * - * Up to four arguments may be passed in rbx, rcx, rdx, and rsi respectively. - * The hypercall number should be placed in rax and the return value will be - * placed in rax. No other registers will be clobbered unless explicited - * noted by the particular hypercall. - */ - -static inline long kvm_hypercall0(unsigned int nr) -{ - long ret; - asm volatile(KVM_HYPERCALL - : "=a"(ret) - : "a"(nr)); - return ret; -} - -static inline long kvm_hypercall1(unsigned int nr, unsigned long p1) -{ - long ret; - asm volatile(KVM_HYPERCALL - : "=a"(ret) - : "a"(nr), "b"(p1)); - return ret; -} - -static inline long kvm_hypercall2(unsigned int nr, unsigned long p1, - unsigned long p2) -{ - long ret; - asm volatile(KVM_HYPERCALL - : "=a"(ret) - : "a"(nr), "b"(p1), "c"(p2)); - return ret; -} - -static inline long kvm_hypercall3(unsigned int nr, unsigned long p1, - unsigned long p2, unsigned long p3) -{ - long ret; - asm volatile(KVM_HYPERCALL - : "=a"(ret) - : "a"(nr), "b"(p1), "c"(p2), "d"(p3)); - return ret; -} - -static inline long kvm_hypercall4(unsigned int nr, unsigned long p1, - unsigned long p2, unsigned long p3, - unsigned long p4) -{ - long ret; - asm volatile(KVM_HYPERCALL - : "=a"(ret) - : "a"(nr), "b"(p1), "c"(p2), "d"(p3), "S"(p4)); - return ret; -} - -static inline int kvm_para_available(void) -{ - unsigned int eax, ebx, ecx, edx; - char signature[13]; - - cpuid(KVM_CPUID_SIGNATURE, &eax, &ebx, &ecx, &edx); - memcpy(signature + 0, &ebx, 4); - memcpy(signature + 4, &ecx, 4); - memcpy(signature + 8, &edx, 4); - signature[12] = 0; - - if (strcmp(signature, "KVMKVMKVM") == 0) - return 1; - - return 0; -} +#include static inline int kvm_para_has_feature(unsigned int feature) { - if (cpuid_eax(KVM_CPUID_FEATURES) & (1UL << feature)) + if (kvm_arch_para_features() & (1UL << feature)) return 1; return 0; } +#endif /* __KERNEL__ */ +#endif /* __LINUX_KVM_PARA_H */ -#endif - -#endif From 80b14b5b32cb0a98234283daf9b5a6643e1a1ef3 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Thu, 25 Oct 2007 11:54:04 +0200 Subject: [PATCH 057/249] KVM: Unmap kernel-allocated memory on slot destruction kvm_vm_ioctl_set_memory_region() is able to remove memory in addition to adding it. Therefore when using kernel swapping support for old userspaces, we need to munmap the memory if the user request to remove it Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/kvm_main.c | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index f7181a407be1..12de42c789a2 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -353,6 +353,7 @@ struct kvm_memory_slot { unsigned long *rmap; unsigned long *dirty_bitmap; unsigned long userspace_addr; + int user_alloc; }; struct kvm { diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 453e98e251da..ab2c77c9ecef 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -713,6 +713,7 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, memset(new.rmap, 0, npages * sizeof(*new.rmap)); + new.user_alloc = user_alloc; if (user_alloc) new.userspace_addr = mem->userspace_addr; else { @@ -727,6 +728,19 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, if (IS_ERR((void *)new.userspace_addr)) goto out_unlock; } + } else { + if (!old.user_alloc && old.rmap) { + int ret; + + down_write(¤t->mm->mmap_sem); + ret = do_munmap(current->mm, old.userspace_addr, + old.npages * PAGE_SIZE); + up_write(¤t->mm->mmap_sem); + if (ret < 0) + printk(KERN_WARNING + "kvm_vm_ioctl_set_memory_region: " + "failed to munmap memory\n"); + } } /* Allocate page dirty bitmap if needed */ From 210c7c4d7f200f00ec34960a8c96bb990cbd266d Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Wed, 24 Oct 2007 23:52:57 +0200 Subject: [PATCH 058/249] KVM: Export memory slot allocation mechanism Remove kvm memory slot allocation mechanism from the ioctl and put it to exported function. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 +++ drivers/kvm/kvm_main.c | 17 +++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 12de42c789a2..f3dda088e34b 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -513,6 +513,9 @@ struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva); extern struct page *bad_page; int is_error_page(struct page *page); +int kvm_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + int user_alloc); gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); void kvm_release_page(struct page *page); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index ab2c77c9ecef..5113cbf75674 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -643,10 +643,9 @@ EXPORT_SYMBOL_GPL(fx_init); * * Discontiguous memory is allowed, mostly for framebuffers. */ -static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, - struct - kvm_userspace_memory_region *mem, - int user_alloc) +int kvm_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + int user_alloc) { int r; gfn_t base_gfn; @@ -789,6 +788,16 @@ out_unlock: kvm_free_physmem_slot(&new, &old); out: return r; + +} +EXPORT_SYMBOL_GPL(kvm_set_memory_region); + +static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, + struct + kvm_userspace_memory_region *mem, + int user_alloc) +{ + return kvm_set_memory_region(kvm, mem, user_alloc); } static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, From e0d62c7f48605119a7f9fa632e77561c89928963 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Wed, 24 Oct 2007 23:57:46 +0200 Subject: [PATCH 059/249] KVM: Add kernel-internal memory slots Reserve a few memory slots for kernel internal use. This is good for case you have to register memory region and you want to be sure it was not registered from userspace, and for case you want to register a memory region that won't be seen from userspace. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 6 +++++- drivers/kvm/kvm_main.c | 22 +++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index f3dda088e34b..82c3b13e5d0a 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -41,6 +41,8 @@ #define KVM_MAX_VCPUS 4 #define KVM_ALIAS_SLOTS 4 #define KVM_MEMORY_SLOTS 8 +/* memory slots that does not exposed to userspace */ +#define KVM_PRIVATE_MEM_SLOTS 4 #define KVM_PERMILLE_MMU_PAGES 20 #define KVM_MIN_ALLOC_MMU_PAGES 64 #define KVM_NUM_MMU_PAGES 1024 @@ -361,7 +363,8 @@ struct kvm { int naliases; struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS]; int nmemslots; - struct kvm_memory_slot memslots[KVM_MEMORY_SLOTS]; + struct kvm_memory_slot memslots[KVM_MEMORY_SLOTS + + KVM_PRIVATE_MEM_SLOTS]; /* * Hash table of struct kvm_mmu_page. */ @@ -529,6 +532,7 @@ int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len); int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len); struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn); +int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn); void mark_page_dirty(struct kvm *kvm, gfn_t gfn); enum emulation_result { diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 5113cbf75674..a044856bd02b 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -660,7 +660,7 @@ int kvm_set_memory_region(struct kvm *kvm, goto out; if (mem->guest_phys_addr & (PAGE_SIZE - 1)) goto out; - if (mem->slot >= KVM_MEMORY_SLOTS) + if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS) goto out; if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) goto out; @@ -797,6 +797,8 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, kvm_userspace_memory_region *mem, int user_alloc) { + if (mem->slot >= KVM_MEMORY_SLOTS) + return -EINVAL; return kvm_set_memory_region(kvm, mem, user_alloc); } @@ -1010,6 +1012,22 @@ struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) return __gfn_to_memslot(kvm, gfn); } +int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) +{ + int i; + + gfn = unalias_gfn(kvm, gfn); + for (i = 0; i < KVM_MEMORY_SLOTS; ++i) { + struct kvm_memory_slot *memslot = &kvm->memslots[i]; + + if (gfn >= memslot->base_gfn + && gfn < memslot->base_gfn + memslot->npages) + return 1; + } + return 0; +} +EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); + struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) { struct kvm_memory_slot *slot; @@ -3087,6 +3105,8 @@ static struct page *kvm_vm_nopage(struct vm_area_struct *vma, struct page *page; pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; + if (!kvm_is_visible_gfn(kvm, pgoff)) + return NOPAGE_SIGBUS; page = gfn_to_page(kvm, pgoff); if (is_error_page(page)) { kvm_release_page(page); From cbc9402297b9a233981f74587786364cda21c771 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Thu, 25 Oct 2007 00:29:55 +0200 Subject: [PATCH 060/249] KVM: Add ioctl to tss address from userspace, Currently kvm has a wart in that it requires three extra pages for use as a tss when emulating real mode on Intel. This patch moves the allocation internally, only requiring userspace to tell us where in the physical address space we can place the tss. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 +++ drivers/kvm/kvm_main.c | 16 ++++++++++++++++ drivers/kvm/svm.c | 7 +++++++ drivers/kvm/vmx.c | 27 +++++++++++++++++++++++++-- include/linux/kvm.h | 2 ++ 5 files changed, 53 insertions(+), 2 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 82c3b13e5d0a..3d07d9b1b815 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -382,6 +382,7 @@ struct kvm { struct kvm_pic *vpic; struct kvm_ioapic *vioapic; int round_robin_prev_vcpu; + unsigned int tss_addr; }; static inline struct kvm_pic *pic_irqchip(struct kvm *kvm) @@ -467,6 +468,8 @@ struct kvm_x86_ops { void (*inject_pending_irq)(struct kvm_vcpu *vcpu); void (*inject_pending_vectors)(struct kvm_vcpu *vcpu, struct kvm_run *run); + + int (*set_tss_addr)(struct kvm *kvm, unsigned int addr); }; extern struct kvm_x86_ops *kvm_x86_ops; diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index a044856bd02b..50fd040b9ea9 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -2661,6 +2661,16 @@ static int create_vcpu_fd(struct kvm_vcpu *vcpu) return fd; } +static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr) +{ + int ret; + + if (addr > (unsigned int)(-3 * PAGE_SIZE)) + return -1; + ret = kvm_x86_ops->set_tss_addr(kvm, addr); + return ret; +} + /* * Creates some virtual cpus. Good luck creating more than one. */ @@ -2957,6 +2967,11 @@ static long kvm_vm_ioctl(struct file *filp, int r = -EINVAL; switch (ioctl) { + case KVM_SET_TSS_ADDR: + r = kvm_vm_ioctl_set_tss_addr(kvm, arg); + if (r < 0) + goto out; + break; case KVM_CREATE_VCPU: r = kvm_vm_ioctl_create_vcpu(kvm, arg); if (r < 0) @@ -3183,6 +3198,7 @@ static long kvm_dev_ioctl(struct file *filp, case KVM_CAP_HLT: case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: case KVM_CAP_USER_MEMORY: + case KVM_CAP_SET_TSS_ADDR: r = 1; break; default: diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 953c111544e1..beb1e684b68a 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -1439,6 +1439,11 @@ static void do_interrupt_requests(struct kvm_vcpu *vcpu, control->intercept &= ~(1ULL << INTERCEPT_VINTR); } +static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr) +{ + return 0; +} + static void save_db_regs(unsigned long *db_regs) { asm volatile ("mov %%dr0, %0" : "=r"(db_regs[0])); @@ -1744,6 +1749,8 @@ static struct kvm_x86_ops svm_x86_ops = { .set_irq = svm_set_irq, .inject_pending_irq = svm_intr_assist, .inject_pending_vectors = do_interrupt_requests, + + .set_tss_addr = svm_set_tss_addr, }; static int __init svm_init(void) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 768ea88beb45..46b29184a4d8 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1105,8 +1105,12 @@ static void enter_pmode(struct kvm_vcpu *vcpu) static gva_t rmode_tss_base(struct kvm *kvm) { - gfn_t base_gfn = kvm->memslots[0].base_gfn + kvm->memslots[0].npages - 3; - return base_gfn << PAGE_SHIFT; + if (!kvm->tss_addr) { + gfn_t base_gfn = kvm->memslots[0].base_gfn + + kvm->memslots[0].npages - 3; + return base_gfn << PAGE_SHIFT; + } + return kvm->tss_addr; } static void fix_rmode_seg(int seg, struct kvm_save_segment *save) @@ -1735,6 +1739,23 @@ static void do_interrupt_requests(struct kvm_vcpu *vcpu, vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control); } +static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr) +{ + int ret; + struct kvm_userspace_memory_region tss_mem = { + .slot = 8, + .guest_phys_addr = addr, + .memory_size = PAGE_SIZE * 3, + .flags = 0, + }; + + ret = kvm_set_memory_region(kvm, &tss_mem, 0); + if (ret) + return ret; + kvm->tss_addr = addr; + return 0; +} + static void kvm_guest_debug_pre(struct kvm_vcpu *vcpu) { struct kvm_guest_debug *dbg = &vcpu->guest_debug; @@ -2543,6 +2564,8 @@ static struct kvm_x86_ops vmx_x86_ops = { .set_irq = vmx_inject_irq, .inject_pending_irq = vmx_intr_assist, .inject_pending_vectors = do_interrupt_requests, + + .set_tss_addr = vmx_set_tss_addr, }; static int __init vmx_init(void) diff --git a/include/linux/kvm.h b/include/linux/kvm.h index 971f465e26fa..71d33d62944f 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -358,6 +358,7 @@ struct kvm_signal_mask { #define KVM_CAP_HLT 1 #define KVM_CAP_MMU_SHADOW_CACHE_CONTROL 2 #define KVM_CAP_USER_MEMORY 3 +#define KVM_CAP_SET_TSS_ADDR 4 /* * ioctls for VM fds @@ -367,6 +368,7 @@ struct kvm_signal_mask { #define KVM_GET_NR_MMU_PAGES _IO(KVMIO, 0x45) #define KVM_SET_USER_MEMORY_REGION _IOW(KVMIO, 0x46,\ struct kvm_userspace_memory_region) +#define KVM_SET_TSS_ADDR _IO(KVMIO, 0x47) /* * KVM_CREATE_VCPU receives as a parameter the vcpu slot, and returns * a vcpu fd. From c20363006af64cf397519da5e984b18e6bdffd82 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Thu, 25 Oct 2007 14:18:52 +0200 Subject: [PATCH 061/249] KVM: VMX: Let gcc to choose which registers to save (x86_64) This patch lets GCC to determine which registers to save when we switch to/from a VCPU in the case of intel x86_64. * Original code saves following registers: rax, rbx, rcx, rdx, rsi, rdi, rbp, r8, r9, r10, r11, r12, r13, r14, r15 * Patched code: - informs GCC that we modify following registers using the clobber description: rbx, rdi, rsi, r8, r9, r10, r11, r12, r13, r14, r15 - doesn't save rax because it is an output operand (vmx->fail) - cannot put rcx in clobber description because it is an input operand, but as we modify it and we want to keep its value (vcpu), we must save it (pop/push) - rbp is saved (pop/push) because GCC seems to ignore its use in the clobber description. - rdx is saved (pop/push) because it is reserved by GCC (REGPARM) and cannot be put in the clobber description. - line "mov (%%rsp), %3 \n\t" has been removed because %3 is rcx and rcx is restored just after. - line ASM_VMX_VMWRITE_RSP_RDX() is moved out of the ifdef/else/endif Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 46b29184a4d8..56c9bcc82836 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -2265,16 +2265,12 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) asm( /* Store host registers */ #ifdef CONFIG_X86_64 - "push %%rax; push %%rbx; push %%rdx;" - "push %%rsi; push %%rdi; push %%rbp;" - "push %%r8; push %%r9; push %%r10; push %%r11;" - "push %%r12; push %%r13; push %%r14; push %%r15;" + "push %%rdx; push %%rbp;" "push %%rcx \n\t" - ASM_VMX_VMWRITE_RSP_RDX "\n\t" #else "pusha; push %%ecx \n\t" - ASM_VMX_VMWRITE_RSP_RDX "\n\t" #endif + ASM_VMX_VMWRITE_RSP_RDX "\n\t" /* Check if vmlaunch of vmresume is needed */ "cmp $0, %1 \n\t" /* Load guest registers. Don't clobber flags. */ @@ -2333,12 +2329,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) "mov %%r15, %c[r15](%3) \n\t" "mov %%cr2, %%rax \n\t" "mov %%rax, %c[cr2](%3) \n\t" - "mov (%%rsp), %3 \n\t" - "pop %%rcx; pop %%r15; pop %%r14; pop %%r13; pop %%r12;" - "pop %%r11; pop %%r10; pop %%r9; pop %%r8;" - "pop %%rbp; pop %%rdi; pop %%rsi;" - "pop %%rdx; pop %%rbx; pop %%rax \n\t" + "pop %%rcx; pop %%rbp; pop %%rdx \n\t" #else "xchg %3, (%%esp) \n\t" "mov %%eax, %c[rax](%3) \n\t" @@ -2376,7 +2368,12 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) [r15]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R15])), #endif [cr2]"i"(offsetof(struct kvm_vcpu, cr2)) - : "cc", "memory"); + : "cc", "memory" +#ifdef CONFIG_X86_64 + , "rbx", "rdi", "rsi" + , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" +#endif + ); vcpu->interrupt_window_open = (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0; From ff593e5abebd899b0b41c11280e2fbeff4103375 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Thu, 25 Oct 2007 14:18:55 +0200 Subject: [PATCH 062/249] KVM: VMX: Let gcc to choose which registers to save (i386) This patch lets GCC to determine which registers to save when we switch to/from a VCPU in the case of intel i386. * Original code saves following registers: eax, ebx, ecx, edx, edi, esi, ebp (using popa) * Patched code: - informs GCC that we modify following registers using the clobber description: ebx, edi, rsi - doesn't save eax because it is an output operand (vmx->fail) - cannot put ecx in clobber description because it is an input operand, but as we modify it and we want to keep its value (vcpu), we must save it (pop/push) - ebp is saved (pop/push) because GCC seems to ignore its use the clobber description. - edx is saved (pop/push) because it is reserved by GCC (REGPARM) and cannot be put in the clobber description. - line "mov (%%esp), %3 \n\t" has been removed because %3 is ecx and ecx is restored just after. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 56c9bcc82836..2d7d638d72d0 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -2268,7 +2268,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) "push %%rdx; push %%rbp;" "push %%rcx \n\t" #else - "pusha; push %%ecx \n\t" + "push %%edx; push %%ebp;" + "push %%ecx \n\t" #endif ASM_VMX_VMWRITE_RSP_RDX "\n\t" /* Check if vmlaunch of vmresume is needed */ @@ -2342,9 +2343,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) "mov %%ebp, %c[rbp](%3) \n\t" "mov %%cr2, %%eax \n\t" "mov %%eax, %c[cr2](%3) \n\t" - "mov (%%esp), %3 \n\t" - "pop %%ecx; popa \n\t" + "pop %%ecx; pop %%ebp; pop %%edx \n\t" #endif "setbe %0 \n\t" : "=q" (vmx->fail) @@ -2372,6 +2372,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) #ifdef CONFIG_X86_64 , "rbx", "rdi", "rsi" , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" +#else + , "ebx", "edi", "rsi" #endif ); From 54a08c0449683f763dc69e653d17e8c120e16299 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Thu, 25 Oct 2007 14:18:53 +0200 Subject: [PATCH 063/249] KVM: SVM: Let gcc to choose which registers to save (x86_64) This patch lets GCC to determine which registers to save when we switch to/from a VCPU in the case of AMD x86_64. * Original code saves following registers: rbx, rcx, rdx, rsi, rdi, rbp, r8, r9, r10, r11, r12, r13, r14, r15 * Patched code: - informs GCC that we modify following registers using the clobber description: rbx, rcx, rdx, rsi, rdi r8, r9, r10, r11, r12, r13, r14, r15 - rbp is saved (pop/push) because GCC seems to ignore its use in the clobber description. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index beb1e684b68a..debd0912a016 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -1499,10 +1499,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) asm volatile ( #ifdef CONFIG_X86_64 - "push %%rbx; push %%rcx; push %%rdx;" - "push %%rsi; push %%rdi; push %%rbp;" - "push %%r8; push %%r9; push %%r10; push %%r11;" - "push %%r12; push %%r13; push %%r14; push %%r15;" + "push %%rbp; \n\t" #else "push %%ebx; push %%ecx; push %%edx;" "push %%esi; push %%edi; push %%ebp;" @@ -1567,10 +1564,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) "mov %%r14, %c[r14](%[svm]) \n\t" "mov %%r15, %c[r15](%[svm]) \n\t" - "pop %%r15; pop %%r14; pop %%r13; pop %%r12;" - "pop %%r11; pop %%r10; pop %%r9; pop %%r8;" - "pop %%rbp; pop %%rdi; pop %%rsi;" - "pop %%rdx; pop %%rcx; pop %%rbx; \n\t" + "pop %%rbp; \n\t" #else "mov %%ebx, %c[rbx](%[svm]) \n\t" "mov %%ecx, %c[rcx](%[svm]) \n\t" @@ -1601,7 +1595,12 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) [r14]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R14])), [r15]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R15])) #endif - : "cc", "memory"); + : "cc", "memory" +#ifdef CONFIG_X86_64 + , "rbx", "rcx", "rdx", "rsi", "rdi" + , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15" +#endif + ); if ((svm->vmcb->save.dr7 & 0xff)) load_db_regs(svm->host_db_regs); From fe7935d49fbe33308c1b5f0e35137989da851010 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Thu, 25 Oct 2007 14:18:54 +0200 Subject: [PATCH 064/249] KVM: SVM: Let gcc to choose which registers to save (i386) This patch lets GCC to determine which registers to save when we switch to/from a VCPU in the case of AMD i386 * Original code saves following registers: ebx, ecx, edx, esi, edi, ebp * Patched code: - informs GCC that we modify following registers using the clobber description: ebx, ecx, edx, esi, edi - rbp is saved (pop/push) because GCC seems to ignore its use in the clobber description. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index debd0912a016..fe2a5e8b0524 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -1501,8 +1501,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) #ifdef CONFIG_X86_64 "push %%rbp; \n\t" #else - "push %%ebx; push %%ecx; push %%edx;" - "push %%esi; push %%edi; push %%ebp;" + "push %%ebp; \n\t" #endif #ifdef CONFIG_X86_64 @@ -1573,8 +1572,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) "mov %%edi, %c[rdi](%[svm]) \n\t" "mov %%ebp, %c[rbp](%[svm]) \n\t" - "pop %%ebp; pop %%edi; pop %%esi;" - "pop %%edx; pop %%ecx; pop %%ebx; \n\t" + "pop %%ebp; \n\t" #endif : : [svm]"a"(svm), @@ -1599,6 +1597,8 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) #ifdef CONFIG_X86_64 , "rbx", "rcx", "rdx", "rsi", "rdi" , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15" +#else + , "ebx", "ecx", "edx" , "esi", "edi" #endif ); From c7e75a3db4ecd952e7a5562cea1b27007bf0c01c Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 28 Oct 2007 16:34:25 +0200 Subject: [PATCH 065/249] KVM: x86 emulator: don't depend on cr2 for mov abs emulation The 'mov abs' instruction family (opcodes 0xa0 - 0xa3) still depends on cr2 provided by the page fault handler. This is wrong for several reasons: - if an instruction accessed misaligned data that crosses a page boundary, and if the fault happened on the second page, cr2 will point at the second page, not the data itself. - if we're emulating in real mode, or due to a FlexPriority exit, there is no cr2 generated. So, this change adds decoding for this instruction form and drops reliance on cr2. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 50 +++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 73e3580c88e4..087a8208f873 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -63,8 +63,9 @@ /* Destination is only written; never read. */ #define Mov (1<<7) #define BitOp (1<<8) +#define MemAbs (1<<9) /* Memory operand is absolute displacement */ -static u8 opcode_table[256] = { +static u16 opcode_table[256] = { /* 0x00 - 0x07 */ ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, @@ -134,8 +135,8 @@ static u8 opcode_table[256] = { /* 0x90 - 0x9F */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps, ImplicitOps, 0, 0, /* 0xA0 - 0xA7 */ - ByteOp | DstReg | SrcMem | Mov, DstReg | SrcMem | Mov, - ByteOp | DstMem | SrcReg | Mov, DstMem | SrcReg | Mov, + ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs, + ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs, ByteOp | ImplicitOps | Mov, ImplicitOps | Mov, ByteOp | ImplicitOps, ImplicitOps, /* 0xA8 - 0xAF */ @@ -755,16 +756,6 @@ done_prefixes: break; } } - if (!c->override_base) - c->override_base = &ctxt->ds_base; - if (mode == X86EMUL_MODE_PROT64 && - c->override_base != &ctxt->fs_base && - c->override_base != &ctxt->gs_base) - c->override_base = NULL; - - if (c->override_base) - c->modrm_ea += *c->override_base; - if (rip_relative) { c->modrm_ea += c->eip; switch (c->d & SrcMask) { @@ -781,12 +772,35 @@ done_prefixes: c->modrm_ea += c->op_bytes; } } - if (c->ad_bytes != 8) - c->modrm_ea = (u32)c->modrm_ea; modrm_done: ; + } else if (c->d & MemAbs) { + switch (c->ad_bytes) { + case 2: + c->modrm_ea = insn_fetch(u16, 2, c->eip); + break; + case 4: + c->modrm_ea = insn_fetch(u32, 4, c->eip); + break; + case 8: + c->modrm_ea = insn_fetch(u64, 8, c->eip); + break; + } + } + if (!c->override_base) + c->override_base = &ctxt->ds_base; + if (mode == X86EMUL_MODE_PROT64 && + c->override_base != &ctxt->fs_base && + c->override_base != &ctxt->gs_base) + c->override_base = NULL; + + if (c->override_base) + c->modrm_ea += *c->override_base; + + if (c->ad_bytes != 8) + c->modrm_ea = (u32)c->modrm_ea; /* * Decode and fetch the source operand: register, memory * or immediate. @@ -1171,7 +1185,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) memcpy(c->regs, ctxt->vcpu->regs, sizeof c->regs); saved_eip = c->eip; - if ((c->d & ModRM) && (c->modrm_mod != 3)) + if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs)) cr2 = c->modrm_ea; if (c->src.type == OP_MEM) { @@ -1326,13 +1340,9 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) case 0xa0 ... 0xa1: /* mov */ c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; c->dst.val = c->src.val; - /* skip src displacement */ - c->eip += c->ad_bytes; break; case 0xa2 ... 0xa3: /* mov */ c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX]; - /* skip c->dst displacement */ - c->eip += c->ad_bytes; break; case 0xc0 ... 0xc1: emulate_grp2(ctxt); From 3067714cf59bd4a6dbf788b709485bc62c1ff845 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 28 Oct 2007 18:48:59 +0200 Subject: [PATCH 066/249] KVM: Move page fault processing to common code The code that dispatches the page fault and emulates if we failed to map is duplicated across vmx and svm. Merge it to simplify further bugfixing. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 36 ++++++++++++++++++++++++++++++++++++ drivers/kvm/svm.c | 33 +-------------------------------- drivers/kvm/vmx.c | 29 +---------------------------- drivers/kvm/x86.h | 6 +----- 4 files changed, 39 insertions(+), 65 deletions(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index d9c5950cfae1..ace3cb86214b 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -1347,6 +1347,42 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) } } +int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) +{ + int r; + enum emulation_result er; + + mutex_lock(&vcpu->kvm->lock); + r = vcpu->mmu.page_fault(vcpu, cr2, error_code); + if (r < 0) + goto out; + + if (!r) { + r = 1; + goto out; + } + + er = emulate_instruction(vcpu, vcpu->run, cr2, error_code, 0); + mutex_unlock(&vcpu->kvm->lock); + + switch (er) { + case EMULATE_DONE: + return 1; + case EMULATE_DO_MMIO: + ++vcpu->stat.mmio_exits; + return 0; + case EMULATE_FAIL: + kvm_report_emulation_failure(vcpu, "pagetable"); + return 1; + default: + BUG(); + } +out: + mutex_unlock(&vcpu->kvm->lock); + return r; +} +EXPORT_SYMBOL_GPL(kvm_mmu_page_fault); + static void free_mmu_pages(struct kvm_vcpu *vcpu) { struct kvm_mmu_page *page; diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index fe2a5e8b0524..97863f8fd001 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -933,45 +933,14 @@ static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) struct kvm *kvm = svm->vcpu.kvm; u64 fault_address; u32 error_code; - enum emulation_result er; - int r; if (!irqchip_in_kernel(kvm) && is_external_interrupt(exit_int_info)) push_irq(&svm->vcpu, exit_int_info & SVM_EVTINJ_VEC_MASK); - mutex_lock(&kvm->lock); - fault_address = svm->vmcb->control.exit_info_2; error_code = svm->vmcb->control.exit_info_1; - r = kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code); - if (r < 0) { - mutex_unlock(&kvm->lock); - return r; - } - if (!r) { - mutex_unlock(&kvm->lock); - return 1; - } - er = emulate_instruction(&svm->vcpu, kvm_run, fault_address, - error_code, 0); - mutex_unlock(&kvm->lock); - - switch (er) { - case EMULATE_DONE: - return 1; - case EMULATE_DO_MMIO: - ++svm->vcpu.stat.mmio_exits; - return 0; - case EMULATE_FAIL: - kvm_report_emulation_failure(&svm->vcpu, "pagetable"); - break; - default: - BUG(); - } - - kvm_run->exit_reason = KVM_EXIT_UNKNOWN; - return 0; + return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code); } static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 2d7d638d72d0..7fe834cb0d81 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1796,7 +1796,6 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) unsigned long cr2, rip; u32 vect_info; enum emulation_result er; - int r; vect_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); intr_info = vmcs_read32(VM_EXIT_INTR_INFO); @@ -1834,33 +1833,7 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); if (is_page_fault(intr_info)) { cr2 = vmcs_readl(EXIT_QUALIFICATION); - - mutex_lock(&vcpu->kvm->lock); - r = kvm_mmu_page_fault(vcpu, cr2, error_code); - if (r < 0) { - mutex_unlock(&vcpu->kvm->lock); - return r; - } - if (!r) { - mutex_unlock(&vcpu->kvm->lock); - return 1; - } - - er = emulate_instruction(vcpu, kvm_run, cr2, error_code, 0); - mutex_unlock(&vcpu->kvm->lock); - - switch (er) { - case EMULATE_DONE: - return 1; - case EMULATE_DO_MMIO: - ++vcpu->stat.mmio_exits; - return 0; - case EMULATE_FAIL: - kvm_report_emulation_failure(vcpu, "pagetable"); - break; - default: - BUG(); - } + return kvm_mmu_page_fault(vcpu, cr2, error_code); } if (vcpu->rmode.active && diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 01452b552db3..20da8e9751c0 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -85,11 +85,7 @@ struct kvm_vcpu { struct x86_emulate_ctxt emulate_ctxt; }; -static inline int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, - u32 error_code) -{ - return vcpu->mmu.page_fault(vcpu, gva, error_code); -} +int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code); static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) { From b733bfb524af69612f85c36a511f0109c5e3fe8d Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 28 Oct 2007 18:52:05 +0200 Subject: [PATCH 067/249] KVM: MMU: Topup the mmu memory preallocation caches before emulating an insn Emulation may cause a shadow pte to be instantiated, which requires memory resources. Make sure the caches are filled to avoid an oops. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index ace3cb86214b..9be54a5e858e 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -1362,6 +1362,10 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) goto out; } + r = mmu_topup_memory_caches(vcpu); + if (r) + goto out; + er = emulate_instruction(vcpu, vcpu->run, cr2, error_code, 0); mutex_unlock(&vcpu->kvm->lock); From 1fe779f8eccd16e527315e1bafd2b3a876ff2489 Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Mon, 29 Oct 2007 16:08:35 +0100 Subject: [PATCH 068/249] KVM: Portability: Split kvm_vm_ioctl v3 This patch splits kvm_vm_ioctl into archtecture independent parts, and x86 specific parts which go to kvm_arch_vcpu_ioctl in x86.c. The patch is unchanged since last submission. Common ioctls for all architectures are: KVM_CREATE_VCPU, KVM_GET_DIRTY_LOG, KVM_SET_USER_MEMORY_REGION x86 specific ioctls are: KVM_SET_MEMORY_REGION, KVM_GET/SET_NR_MMU_PAGES, KVM_SET_MEMORY_ALIAS, KVM_CREATE_IRQCHIP, KVM_CREATE_IRQ_LINE, KVM_GET/SET_IRQCHIP KVM_SET_TSS_ADDR Signed-off-by: Carsten Otte Reviewed-by: Christian Borntraeger Acked-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 7 ++ drivers/kvm/kvm_main.c | 255 +--------------------------------------- drivers/kvm/x86.c | 258 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 271 insertions(+), 249 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 3d07d9b1b815..516f79ffd126 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -620,6 +620,13 @@ long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); +int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, + struct + kvm_userspace_memory_region *mem, + int user_alloc); +long kvm_arch_vm_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg); +void kvm_arch_destroy_vm(struct kvm *kvm); __init void kvm_arch_init(void); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 50fd040b9ea9..c632e3a3b514 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -792,36 +792,16 @@ out: } EXPORT_SYMBOL_GPL(kvm_set_memory_region); -static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, - struct - kvm_userspace_memory_region *mem, - int user_alloc) +int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, + struct + kvm_userspace_memory_region *mem, + int user_alloc) { if (mem->slot >= KVM_MEMORY_SLOTS) return -EINVAL; return kvm_set_memory_region(kvm, mem, user_alloc); } -static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, - u32 kvm_nr_mmu_pages) -{ - if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES) - return -EINVAL; - - mutex_lock(&kvm->lock); - - kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages); - kvm->n_requested_mmu_pages = kvm_nr_mmu_pages; - - mutex_unlock(&kvm->lock); - return 0; -} - -static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm) -{ - return kvm->n_alloc_mmu_pages; -} - /* * Get (and clear) the dirty memory log for a memory slot. */ @@ -867,111 +847,6 @@ out: return r; } -/* - * Set a new alias region. Aliases map a portion of physical memory into - * another portion. This is useful for memory windows, for example the PC - * VGA region. - */ -static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm, - struct kvm_memory_alias *alias) -{ - int r, n; - struct kvm_mem_alias *p; - - r = -EINVAL; - /* General sanity checks */ - if (alias->memory_size & (PAGE_SIZE - 1)) - goto out; - if (alias->guest_phys_addr & (PAGE_SIZE - 1)) - goto out; - if (alias->slot >= KVM_ALIAS_SLOTS) - goto out; - if (alias->guest_phys_addr + alias->memory_size - < alias->guest_phys_addr) - goto out; - if (alias->target_phys_addr + alias->memory_size - < alias->target_phys_addr) - goto out; - - mutex_lock(&kvm->lock); - - p = &kvm->aliases[alias->slot]; - p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT; - p->npages = alias->memory_size >> PAGE_SHIFT; - p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT; - - for (n = KVM_ALIAS_SLOTS; n > 0; --n) - if (kvm->aliases[n - 1].npages) - break; - kvm->naliases = n; - - kvm_mmu_zap_all(kvm); - - mutex_unlock(&kvm->lock); - - return 0; - -out: - return r; -} - -static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) -{ - int r; - - r = 0; - switch (chip->chip_id) { - case KVM_IRQCHIP_PIC_MASTER: - memcpy(&chip->chip.pic, - &pic_irqchip(kvm)->pics[0], - sizeof(struct kvm_pic_state)); - break; - case KVM_IRQCHIP_PIC_SLAVE: - memcpy(&chip->chip.pic, - &pic_irqchip(kvm)->pics[1], - sizeof(struct kvm_pic_state)); - break; - case KVM_IRQCHIP_IOAPIC: - memcpy(&chip->chip.ioapic, - ioapic_irqchip(kvm), - sizeof(struct kvm_ioapic_state)); - break; - default: - r = -EINVAL; - break; - } - return r; -} - -static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) -{ - int r; - - r = 0; - switch (chip->chip_id) { - case KVM_IRQCHIP_PIC_MASTER: - memcpy(&pic_irqchip(kvm)->pics[0], - &chip->chip.pic, - sizeof(struct kvm_pic_state)); - break; - case KVM_IRQCHIP_PIC_SLAVE: - memcpy(&pic_irqchip(kvm)->pics[1], - &chip->chip.pic, - sizeof(struct kvm_pic_state)); - break; - case KVM_IRQCHIP_IOAPIC: - memcpy(ioapic_irqchip(kvm), - &chip->chip.ioapic, - sizeof(struct kvm_ioapic_state)); - break; - default: - r = -EINVAL; - break; - } - kvm_pic_update_irq(pic_irqchip(kvm)); - return r; -} - int is_error_page(struct page *page) { return page == bad_page; @@ -2661,16 +2536,6 @@ static int create_vcpu_fd(struct kvm_vcpu *vcpu) return fd; } -static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr) -{ - int ret; - - if (addr > (unsigned int)(-3 * PAGE_SIZE)) - return -1; - ret = kvm_x86_ops->set_tss_addr(kvm, addr); - return ret; -} - /* * Creates some virtual cpus. Good luck creating more than one. */ @@ -2964,35 +2829,14 @@ static long kvm_vm_ioctl(struct file *filp, { struct kvm *kvm = filp->private_data; void __user *argp = (void __user *)arg; - int r = -EINVAL; + int r; switch (ioctl) { - case KVM_SET_TSS_ADDR: - r = kvm_vm_ioctl_set_tss_addr(kvm, arg); - if (r < 0) - goto out; - break; case KVM_CREATE_VCPU: r = kvm_vm_ioctl_create_vcpu(kvm, arg); if (r < 0) goto out; break; - case KVM_SET_MEMORY_REGION: { - struct kvm_memory_region kvm_mem; - struct kvm_userspace_memory_region kvm_userspace_mem; - - r = -EFAULT; - if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem)) - goto out; - kvm_userspace_mem.slot = kvm_mem.slot; - kvm_userspace_mem.flags = kvm_mem.flags; - kvm_userspace_mem.guest_phys_addr = kvm_mem.guest_phys_addr; - kvm_userspace_mem.memory_size = kvm_mem.memory_size; - r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 0); - if (r) - goto out; - break; - } case KVM_SET_USER_MEMORY_REGION: { struct kvm_userspace_memory_region kvm_userspace_mem; @@ -3006,14 +2850,6 @@ static long kvm_vm_ioctl(struct file *filp, goto out; break; } - case KVM_SET_NR_MMU_PAGES: - r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg); - if (r) - goto out; - break; - case KVM_GET_NR_MMU_PAGES: - r = kvm_vm_ioctl_get_nr_mmu_pages(kvm); - break; case KVM_GET_DIRTY_LOG: { struct kvm_dirty_log log; @@ -3025,87 +2861,8 @@ static long kvm_vm_ioctl(struct file *filp, goto out; break; } - case KVM_SET_MEMORY_ALIAS: { - struct kvm_memory_alias alias; - - r = -EFAULT; - if (copy_from_user(&alias, argp, sizeof alias)) - goto out; - r = kvm_vm_ioctl_set_memory_alias(kvm, &alias); - if (r) - goto out; - break; - } - case KVM_CREATE_IRQCHIP: - r = -ENOMEM; - kvm->vpic = kvm_create_pic(kvm); - if (kvm->vpic) { - r = kvm_ioapic_init(kvm); - if (r) { - kfree(kvm->vpic); - kvm->vpic = NULL; - goto out; - } - } else - goto out; - break; - case KVM_IRQ_LINE: { - struct kvm_irq_level irq_event; - - r = -EFAULT; - if (copy_from_user(&irq_event, argp, sizeof irq_event)) - goto out; - if (irqchip_in_kernel(kvm)) { - mutex_lock(&kvm->lock); - if (irq_event.irq < 16) - kvm_pic_set_irq(pic_irqchip(kvm), - irq_event.irq, - irq_event.level); - kvm_ioapic_set_irq(kvm->vioapic, - irq_event.irq, - irq_event.level); - mutex_unlock(&kvm->lock); - r = 0; - } - break; - } - case KVM_GET_IRQCHIP: { - /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ - struct kvm_irqchip chip; - - r = -EFAULT; - if (copy_from_user(&chip, argp, sizeof chip)) - goto out; - r = -ENXIO; - if (!irqchip_in_kernel(kvm)) - goto out; - r = kvm_vm_ioctl_get_irqchip(kvm, &chip); - if (r) - goto out; - r = -EFAULT; - if (copy_to_user(argp, &chip, sizeof chip)) - goto out; - r = 0; - break; - } - case KVM_SET_IRQCHIP: { - /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ - struct kvm_irqchip chip; - - r = -EFAULT; - if (copy_from_user(&chip, argp, sizeof chip)) - goto out; - r = -ENXIO; - if (!irqchip_in_kernel(kvm)) - goto out; - r = kvm_vm_ioctl_set_irqchip(kvm, &chip); - if (r) - goto out; - r = 0; - break; - } default: - ; + r = kvm_arch_vm_ioctl(filp, ioctl, arg); } out: return r; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 1fe209dd4caf..b84cb6707f78 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -300,6 +300,264 @@ out: return r; } +static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr) +{ + int ret; + + if (addr > (unsigned int)(-3 * PAGE_SIZE)) + return -1; + ret = kvm_x86_ops->set_tss_addr(kvm, addr); + return ret; +} + +static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, + u32 kvm_nr_mmu_pages) +{ + if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES) + return -EINVAL; + + mutex_lock(&kvm->lock); + + kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages); + kvm->n_requested_mmu_pages = kvm_nr_mmu_pages; + + mutex_unlock(&kvm->lock); + return 0; +} + +static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm) +{ + return kvm->n_alloc_mmu_pages; +} + +/* + * Set a new alias region. Aliases map a portion of physical memory into + * another portion. This is useful for memory windows, for example the PC + * VGA region. + */ +static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm, + struct kvm_memory_alias *alias) +{ + int r, n; + struct kvm_mem_alias *p; + + r = -EINVAL; + /* General sanity checks */ + if (alias->memory_size & (PAGE_SIZE - 1)) + goto out; + if (alias->guest_phys_addr & (PAGE_SIZE - 1)) + goto out; + if (alias->slot >= KVM_ALIAS_SLOTS) + goto out; + if (alias->guest_phys_addr + alias->memory_size + < alias->guest_phys_addr) + goto out; + if (alias->target_phys_addr + alias->memory_size + < alias->target_phys_addr) + goto out; + + mutex_lock(&kvm->lock); + + p = &kvm->aliases[alias->slot]; + p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT; + p->npages = alias->memory_size >> PAGE_SHIFT; + p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT; + + for (n = KVM_ALIAS_SLOTS; n > 0; --n) + if (kvm->aliases[n - 1].npages) + break; + kvm->naliases = n; + + kvm_mmu_zap_all(kvm); + + mutex_unlock(&kvm->lock); + + return 0; + +out: + return r; +} + +static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) +{ + int r; + + r = 0; + switch (chip->chip_id) { + case KVM_IRQCHIP_PIC_MASTER: + memcpy(&chip->chip.pic, + &pic_irqchip(kvm)->pics[0], + sizeof(struct kvm_pic_state)); + break; + case KVM_IRQCHIP_PIC_SLAVE: + memcpy(&chip->chip.pic, + &pic_irqchip(kvm)->pics[1], + sizeof(struct kvm_pic_state)); + break; + case KVM_IRQCHIP_IOAPIC: + memcpy(&chip->chip.ioapic, + ioapic_irqchip(kvm), + sizeof(struct kvm_ioapic_state)); + break; + default: + r = -EINVAL; + break; + } + return r; +} + +static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) +{ + int r; + + r = 0; + switch (chip->chip_id) { + case KVM_IRQCHIP_PIC_MASTER: + memcpy(&pic_irqchip(kvm)->pics[0], + &chip->chip.pic, + sizeof(struct kvm_pic_state)); + break; + case KVM_IRQCHIP_PIC_SLAVE: + memcpy(&pic_irqchip(kvm)->pics[1], + &chip->chip.pic, + sizeof(struct kvm_pic_state)); + break; + case KVM_IRQCHIP_IOAPIC: + memcpy(ioapic_irqchip(kvm), + &chip->chip.ioapic, + sizeof(struct kvm_ioapic_state)); + break; + default: + r = -EINVAL; + break; + } + kvm_pic_update_irq(pic_irqchip(kvm)); + return r; +} + +long kvm_arch_vm_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg) +{ + struct kvm *kvm = filp->private_data; + void __user *argp = (void __user *)arg; + int r = -EINVAL; + + switch (ioctl) { + case KVM_SET_TSS_ADDR: + r = kvm_vm_ioctl_set_tss_addr(kvm, arg); + if (r < 0) + goto out; + break; + case KVM_SET_MEMORY_REGION: { + struct kvm_memory_region kvm_mem; + struct kvm_userspace_memory_region kvm_userspace_mem; + + r = -EFAULT; + if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem)) + goto out; + kvm_userspace_mem.slot = kvm_mem.slot; + kvm_userspace_mem.flags = kvm_mem.flags; + kvm_userspace_mem.guest_phys_addr = kvm_mem.guest_phys_addr; + kvm_userspace_mem.memory_size = kvm_mem.memory_size; + r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 0); + if (r) + goto out; + break; + } + case KVM_SET_NR_MMU_PAGES: + r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg); + if (r) + goto out; + break; + case KVM_GET_NR_MMU_PAGES: + r = kvm_vm_ioctl_get_nr_mmu_pages(kvm); + break; + case KVM_SET_MEMORY_ALIAS: { + struct kvm_memory_alias alias; + + r = -EFAULT; + if (copy_from_user(&alias, argp, sizeof alias)) + goto out; + r = kvm_vm_ioctl_set_memory_alias(kvm, &alias); + if (r) + goto out; + break; + } + case KVM_CREATE_IRQCHIP: + r = -ENOMEM; + kvm->vpic = kvm_create_pic(kvm); + if (kvm->vpic) { + r = kvm_ioapic_init(kvm); + if (r) { + kfree(kvm->vpic); + kvm->vpic = NULL; + goto out; + } + } else + goto out; + break; + case KVM_IRQ_LINE: { + struct kvm_irq_level irq_event; + + r = -EFAULT; + if (copy_from_user(&irq_event, argp, sizeof irq_event)) + goto out; + if (irqchip_in_kernel(kvm)) { + mutex_lock(&kvm->lock); + if (irq_event.irq < 16) + kvm_pic_set_irq(pic_irqchip(kvm), + irq_event.irq, + irq_event.level); + kvm_ioapic_set_irq(kvm->vioapic, + irq_event.irq, + irq_event.level); + mutex_unlock(&kvm->lock); + r = 0; + } + break; + } + case KVM_GET_IRQCHIP: { + /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ + struct kvm_irqchip chip; + + r = -EFAULT; + if (copy_from_user(&chip, argp, sizeof chip)) + goto out; + r = -ENXIO; + if (!irqchip_in_kernel(kvm)) + goto out; + r = kvm_vm_ioctl_get_irqchip(kvm, &chip); + if (r) + goto out; + r = -EFAULT; + if (copy_to_user(argp, &chip, sizeof chip)) + goto out; + r = 0; + break; + } + case KVM_SET_IRQCHIP: { + /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ + struct kvm_irqchip chip; + + r = -EFAULT; + if (copy_from_user(&chip, argp, sizeof chip)) + goto out; + r = -ENXIO; + if (!irqchip_in_kernel(kvm)) + goto out; + r = kvm_vm_ioctl_set_irqchip(kvm, &chip); + if (r) + goto out; + r = 0; + break; + } + default: + ; + } +out: + return r; +} + static __init void kvm_init_msr_list(void) { u32 dummy[2]; From 5fb76f9be1a050a25e21a44ab2003c9d36a72a77 Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Mon, 29 Oct 2007 16:08:51 +0100 Subject: [PATCH 069/249] KVM: Portability: Move memory segmentation to x86.c This patch moves the definition of segment_descriptor_64 for AMD64 and EM64T from kvm_main.c to segment_descriptor.h. It also adds a proper #ifndef...#define...#endif around that header file. The implementation of segment_base is moved from kvm_main.c to x86.c. Signed-off-by: Carsten Otte Reviewed-by: Christian Borntraeger Acked-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 42 -------------------------------- drivers/kvm/segment_descriptor.h | 12 +++++++++ drivers/kvm/x86.c | 33 +++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 42 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index c632e3a3b514..cf163bddfbd3 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -18,7 +18,6 @@ #include "kvm.h" #include "x86.h" #include "x86_emulate.h" -#include "segment_descriptor.h" #include "irq.h" #include @@ -104,50 +103,9 @@ static struct dentry *debugfs_dir; #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) #define EFER_RESERVED_BITS 0xfffffffffffff2fe -#ifdef CONFIG_X86_64 -/* LDT or TSS descriptor in the GDT. 16 bytes. */ -struct segment_descriptor_64 { - struct segment_descriptor s; - u32 base_higher; - u32 pad_zero; -}; - -#endif - static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, unsigned long arg); -unsigned long segment_base(u16 selector) -{ - struct descriptor_table gdt; - struct segment_descriptor *d; - unsigned long table_base; - unsigned long v; - - if (selector == 0) - return 0; - - asm("sgdt %0" : "=m"(gdt)); - table_base = gdt.base; - - if (selector & 4) { /* from ldt */ - u16 ldt_selector; - - asm("sldt %0" : "=g"(ldt_selector)); - table_base = segment_base(ldt_selector); - } - d = (struct segment_descriptor *)(table_base + (selector & ~7)); - v = d->base_low | ((unsigned long)d->base_mid << 16) | - ((unsigned long)d->base_high << 24); -#ifdef CONFIG_X86_64 - if (d->system == 0 && (d->type == 2 || d->type == 9 || d->type == 11)) - v |= ((unsigned long) \ - ((struct segment_descriptor_64 *)d)->base_higher) << 32; -#endif - return v; -} -EXPORT_SYMBOL_GPL(segment_base); - static inline int valid_vcpu(int n) { return likely(n >= 0 && n < KVM_MAX_VCPUS); diff --git a/drivers/kvm/segment_descriptor.h b/drivers/kvm/segment_descriptor.h index 71fdf458619a..56fc4c873389 100644 --- a/drivers/kvm/segment_descriptor.h +++ b/drivers/kvm/segment_descriptor.h @@ -1,3 +1,6 @@ +#ifndef __SEGMENT_DESCRIPTOR_H +#define __SEGMENT_DESCRIPTOR_H + struct segment_descriptor { u16 limit_low; u16 base_low; @@ -14,4 +17,13 @@ struct segment_descriptor { u8 base_high; } __attribute__((packed)); +#ifdef CONFIG_X86_64 +/* LDT or TSS descriptor in the GDT. 16 bytes. */ +struct segment_descriptor_64 { + struct segment_descriptor s; + u32 base_higher; + u32 pad_zero; +}; +#endif +#endif diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index b84cb6707f78..5a959220410a 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -16,16 +16,49 @@ #include "kvm.h" #include "x86.h" +#include "segment_descriptor.h" #include "irq.h" #include #include #include +#include #include #define MAX_IO_MSRS 256 +unsigned long segment_base(u16 selector) +{ + struct descriptor_table gdt; + struct segment_descriptor *d; + unsigned long table_base; + unsigned long v; + + if (selector == 0) + return 0; + + asm("sgdt %0" : "=m"(gdt)); + table_base = gdt.base; + + if (selector & 4) { /* from ldt */ + u16 ldt_selector; + + asm("sldt %0" : "=g"(ldt_selector)); + table_base = segment_base(ldt_selector); + } + d = (struct segment_descriptor *)(table_base + (selector & ~7)); + v = d->base_low | ((unsigned long)d->base_mid << 16) | + ((unsigned long)d->base_high << 24); +#ifdef CONFIG_X86_64 + if (d->system == 0 && (d->type == 2 || d->type == 9 || d->type == 11)) + v |= ((unsigned long) \ + ((struct segment_descriptor_64 *)d)->base_higher) << 32; +#endif + return v; +} +EXPORT_SYMBOL_GPL(segment_base); + /* * List of msr numbers which we expose to userspace through KVM_GET_MSRS * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. From 6866b83ed7240bf4a7c50836ee10f61c8534503f Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Mon, 29 Oct 2007 16:09:10 +0100 Subject: [PATCH 070/249] KVM: Portability: move get/set_apic_base to x86.c This patch moves the implementation of get_apic_base and set_apic_base from kvm_main.c to x86.c Signed-off-by: Carsten Otte Reviewed-by: Christian Borntraeger Acked-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 19 ------------------- drivers/kvm/x86.c | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index cf163bddfbd3..e8972a82544b 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -556,25 +556,6 @@ unsigned long get_cr8(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(get_cr8); -u64 kvm_get_apic_base(struct kvm_vcpu *vcpu) -{ - if (irqchip_in_kernel(vcpu->kvm)) - return vcpu->apic_base; - else - return vcpu->apic_base; -} -EXPORT_SYMBOL_GPL(kvm_get_apic_base); - -void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data) -{ - /* TODO: reserve bits check */ - if (irqchip_in_kernel(vcpu->kvm)) - kvm_lapic_set_base(vcpu, data); - else - vcpu->apic_base = data; -} -EXPORT_SYMBOL_GPL(kvm_set_apic_base); - void fx_init(struct kvm_vcpu *vcpu) { unsigned after_mxcsr_mask; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 5a959220410a..c26e3715bee8 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -59,6 +59,25 @@ unsigned long segment_base(u16 selector) } EXPORT_SYMBOL_GPL(segment_base); +u64 kvm_get_apic_base(struct kvm_vcpu *vcpu) +{ + if (irqchip_in_kernel(vcpu->kvm)) + return vcpu->apic_base; + else + return vcpu->apic_base; +} +EXPORT_SYMBOL_GPL(kvm_get_apic_base); + +void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data) +{ + /* TODO: reserve bits check */ + if (irqchip_in_kernel(vcpu->kvm)) + kvm_lapic_set_base(vcpu, data); + else + vcpu->apic_base = data; +} +EXPORT_SYMBOL_GPL(kvm_set_apic_base); + /* * List of msr numbers which we expose to userspace through KVM_GET_MSRS * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. From a03490ed29d2771c675d4d9c0ffe22e19a1757f3 Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Mon, 29 Oct 2007 16:09:35 +0100 Subject: [PATCH 071/249] KVM: Portability: Move control register helper functions to x86.c This patch moves the definitions of CR0_RESERVED_BITS, CR4_RESERVED_BITS, and CR8_RESERVED_BITS along with the following functions from kvm_main.c to x86.c: set_cr0(), set_cr3(), set_cr4(), set_cr8(), get_cr8(), lmsw(), load_pdptrs() The static function wrapper inject_gp is duplicated in kvm_main.c and x86.c for now, the version in kvm_main.c should disappear once the last user of it is gone too. The function load_pdptrs is no longer static, and now defined in x86.h for the time being, until the last user of it is gone from kvm_main.c. Signed-off-by: Carsten Otte Reviewed-by: Christian Borntraeger Acked-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 219 ---------------------------------------- drivers/kvm/x86.c | 224 +++++++++++++++++++++++++++++++++++++++++ drivers/kvm/x86.h | 2 +- 3 files changed, 225 insertions(+), 220 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index e8972a82544b..8f7125710d02 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -90,17 +90,6 @@ static struct kvm_stats_debugfs_item { static struct dentry *debugfs_dir; -#define CR0_RESERVED_BITS \ - (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ - | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \ - | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG)) -#define CR4_RESERVED_BITS \ - (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\ - | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \ - | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR \ - | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE)) - -#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) #define EFER_RESERVED_BITS 0xfffffffffffff2fe static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, @@ -348,214 +337,6 @@ static void inject_gp(struct kvm_vcpu *vcpu) kvm_x86_ops->inject_gp(vcpu, 0); } -/* - * Load the pae pdptrs. Return true is they are all valid. - */ -static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) -{ - gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; - unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2; - int i; - int ret; - u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)]; - - mutex_lock(&vcpu->kvm->lock); - ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte, - offset * sizeof(u64), sizeof(pdpte)); - if (ret < 0) { - ret = 0; - goto out; - } - for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { - if ((pdpte[i] & 1) && (pdpte[i] & 0xfffffff0000001e6ull)) { - ret = 0; - goto out; - } - } - ret = 1; - - memcpy(vcpu->pdptrs, pdpte, sizeof(vcpu->pdptrs)); -out: - mutex_unlock(&vcpu->kvm->lock); - - return ret; -} - -void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) -{ - if (cr0 & CR0_RESERVED_BITS) { - printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n", - cr0, vcpu->cr0); - inject_gp(vcpu); - return; - } - - if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) { - printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n"); - inject_gp(vcpu); - return; - } - - if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) { - printk(KERN_DEBUG "set_cr0: #GP, set PG flag " - "and a clear PE flag\n"); - inject_gp(vcpu); - return; - } - - if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { -#ifdef CONFIG_X86_64 - if ((vcpu->shadow_efer & EFER_LME)) { - int cs_db, cs_l; - - if (!is_pae(vcpu)) { - printk(KERN_DEBUG "set_cr0: #GP, start paging " - "in long mode while PAE is disabled\n"); - inject_gp(vcpu); - return; - } - kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); - if (cs_l) { - printk(KERN_DEBUG "set_cr0: #GP, start paging " - "in long mode while CS.L == 1\n"); - inject_gp(vcpu); - return; - - } - } else -#endif - if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) { - printk(KERN_DEBUG "set_cr0: #GP, pdptrs " - "reserved bits\n"); - inject_gp(vcpu); - return; - } - - } - - kvm_x86_ops->set_cr0(vcpu, cr0); - vcpu->cr0 = cr0; - - mutex_lock(&vcpu->kvm->lock); - kvm_mmu_reset_context(vcpu); - mutex_unlock(&vcpu->kvm->lock); - return; -} -EXPORT_SYMBOL_GPL(set_cr0); - -void lmsw(struct kvm_vcpu *vcpu, unsigned long msw) -{ - set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f)); -} -EXPORT_SYMBOL_GPL(lmsw); - -void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) -{ - if (cr4 & CR4_RESERVED_BITS) { - printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n"); - inject_gp(vcpu); - return; - } - - if (is_long_mode(vcpu)) { - if (!(cr4 & X86_CR4_PAE)) { - printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while " - "in long mode\n"); - inject_gp(vcpu); - return; - } - } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE) - && !load_pdptrs(vcpu, vcpu->cr3)) { - printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n"); - inject_gp(vcpu); - return; - } - - if (cr4 & X86_CR4_VMXE) { - printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n"); - inject_gp(vcpu); - return; - } - kvm_x86_ops->set_cr4(vcpu, cr4); - vcpu->cr4 = cr4; - mutex_lock(&vcpu->kvm->lock); - kvm_mmu_reset_context(vcpu); - mutex_unlock(&vcpu->kvm->lock); -} -EXPORT_SYMBOL_GPL(set_cr4); - -void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) -{ - if (is_long_mode(vcpu)) { - if (cr3 & CR3_L_MODE_RESERVED_BITS) { - printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n"); - inject_gp(vcpu); - return; - } - } else { - if (is_pae(vcpu)) { - if (cr3 & CR3_PAE_RESERVED_BITS) { - printk(KERN_DEBUG - "set_cr3: #GP, reserved bits\n"); - inject_gp(vcpu); - return; - } - if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) { - printk(KERN_DEBUG "set_cr3: #GP, pdptrs " - "reserved bits\n"); - inject_gp(vcpu); - return; - } - } - /* - * We don't check reserved bits in nonpae mode, because - * this isn't enforced, and VMware depends on this. - */ - } - - mutex_lock(&vcpu->kvm->lock); - /* - * Does the new cr3 value map to physical memory? (Note, we - * catch an invalid cr3 even in real-mode, because it would - * cause trouble later on when we turn on paging anyway.) - * - * A real CPU would silently accept an invalid cr3 and would - * attempt to use it - with largely undefined (and often hard - * to debug) behavior on the guest side. - */ - if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT))) - inject_gp(vcpu); - else { - vcpu->cr3 = cr3; - vcpu->mmu.new_cr3(vcpu); - } - mutex_unlock(&vcpu->kvm->lock); -} -EXPORT_SYMBOL_GPL(set_cr3); - -void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) -{ - if (cr8 & CR8_RESERVED_BITS) { - printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8); - inject_gp(vcpu); - return; - } - if (irqchip_in_kernel(vcpu->kvm)) - kvm_lapic_set_tpr(vcpu, cr8); - else - vcpu->cr8 = cr8; -} -EXPORT_SYMBOL_GPL(set_cr8); - -unsigned long get_cr8(struct kvm_vcpu *vcpu) -{ - if (irqchip_in_kernel(vcpu->kvm)) - return kvm_lapic_get_cr8(vcpu); - else - return vcpu->cr8; -} -EXPORT_SYMBOL_GPL(get_cr8); - void fx_init(struct kvm_vcpu *vcpu) { unsigned after_mxcsr_mask; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index c26e3715bee8..a728af8a83e5 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -27,6 +27,17 @@ #include #define MAX_IO_MSRS 256 +#define CR0_RESERVED_BITS \ + (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ + | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \ + | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG)) +#define CR4_RESERVED_BITS \ + (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\ + | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \ + | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR \ + | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE)) + +#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) unsigned long segment_base(u16 selector) { @@ -78,6 +89,219 @@ void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data) } EXPORT_SYMBOL_GPL(kvm_set_apic_base); +static void inject_gp(struct kvm_vcpu *vcpu) +{ + kvm_x86_ops->inject_gp(vcpu, 0); +} + +/* + * Load the pae pdptrs. Return true is they are all valid. + */ +int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) +{ + gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; + unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2; + int i; + int ret; + u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)]; + + mutex_lock(&vcpu->kvm->lock); + ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte, + offset * sizeof(u64), sizeof(pdpte)); + if (ret < 0) { + ret = 0; + goto out; + } + for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { + if ((pdpte[i] & 1) && (pdpte[i] & 0xfffffff0000001e6ull)) { + ret = 0; + goto out; + } + } + ret = 1; + + memcpy(vcpu->pdptrs, pdpte, sizeof(vcpu->pdptrs)); +out: + mutex_unlock(&vcpu->kvm->lock); + + return ret; +} + +void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) +{ + if (cr0 & CR0_RESERVED_BITS) { + printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n", + cr0, vcpu->cr0); + inject_gp(vcpu); + return; + } + + if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) { + printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n"); + inject_gp(vcpu); + return; + } + + if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) { + printk(KERN_DEBUG "set_cr0: #GP, set PG flag " + "and a clear PE flag\n"); + inject_gp(vcpu); + return; + } + + if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { +#ifdef CONFIG_X86_64 + if ((vcpu->shadow_efer & EFER_LME)) { + int cs_db, cs_l; + + if (!is_pae(vcpu)) { + printk(KERN_DEBUG "set_cr0: #GP, start paging " + "in long mode while PAE is disabled\n"); + inject_gp(vcpu); + return; + } + kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); + if (cs_l) { + printk(KERN_DEBUG "set_cr0: #GP, start paging " + "in long mode while CS.L == 1\n"); + inject_gp(vcpu); + return; + + } + } else +#endif + if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) { + printk(KERN_DEBUG "set_cr0: #GP, pdptrs " + "reserved bits\n"); + inject_gp(vcpu); + return; + } + + } + + kvm_x86_ops->set_cr0(vcpu, cr0); + vcpu->cr0 = cr0; + + mutex_lock(&vcpu->kvm->lock); + kvm_mmu_reset_context(vcpu); + mutex_unlock(&vcpu->kvm->lock); + return; +} +EXPORT_SYMBOL_GPL(set_cr0); + +void lmsw(struct kvm_vcpu *vcpu, unsigned long msw) +{ + set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f)); +} +EXPORT_SYMBOL_GPL(lmsw); + +void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) +{ + if (cr4 & CR4_RESERVED_BITS) { + printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n"); + inject_gp(vcpu); + return; + } + + if (is_long_mode(vcpu)) { + if (!(cr4 & X86_CR4_PAE)) { + printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while " + "in long mode\n"); + inject_gp(vcpu); + return; + } + } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE) + && !load_pdptrs(vcpu, vcpu->cr3)) { + printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n"); + inject_gp(vcpu); + return; + } + + if (cr4 & X86_CR4_VMXE) { + printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n"); + inject_gp(vcpu); + return; + } + kvm_x86_ops->set_cr4(vcpu, cr4); + vcpu->cr4 = cr4; + mutex_lock(&vcpu->kvm->lock); + kvm_mmu_reset_context(vcpu); + mutex_unlock(&vcpu->kvm->lock); +} +EXPORT_SYMBOL_GPL(set_cr4); + +void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) +{ + if (is_long_mode(vcpu)) { + if (cr3 & CR3_L_MODE_RESERVED_BITS) { + printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n"); + inject_gp(vcpu); + return; + } + } else { + if (is_pae(vcpu)) { + if (cr3 & CR3_PAE_RESERVED_BITS) { + printk(KERN_DEBUG + "set_cr3: #GP, reserved bits\n"); + inject_gp(vcpu); + return; + } + if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) { + printk(KERN_DEBUG "set_cr3: #GP, pdptrs " + "reserved bits\n"); + inject_gp(vcpu); + return; + } + } + /* + * We don't check reserved bits in nonpae mode, because + * this isn't enforced, and VMware depends on this. + */ + } + + mutex_lock(&vcpu->kvm->lock); + /* + * Does the new cr3 value map to physical memory? (Note, we + * catch an invalid cr3 even in real-mode, because it would + * cause trouble later on when we turn on paging anyway.) + * + * A real CPU would silently accept an invalid cr3 and would + * attempt to use it - with largely undefined (and often hard + * to debug) behavior on the guest side. + */ + if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT))) + inject_gp(vcpu); + else { + vcpu->cr3 = cr3; + vcpu->mmu.new_cr3(vcpu); + } + mutex_unlock(&vcpu->kvm->lock); +} +EXPORT_SYMBOL_GPL(set_cr3); + +void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) +{ + if (cr8 & CR8_RESERVED_BITS) { + printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8); + inject_gp(vcpu); + return; + } + if (irqchip_in_kernel(vcpu->kvm)) + kvm_lapic_set_tpr(vcpu, cr8); + else + vcpu->cr8 = cr8; +} +EXPORT_SYMBOL_GPL(set_cr8); + +unsigned long get_cr8(struct kvm_vcpu *vcpu) +{ + if (irqchip_in_kernel(vcpu->kvm)) + return kvm_lapic_get_cr8(vcpu); + else + return vcpu->cr8; +} +EXPORT_SYMBOL_GPL(get_cr8); + /* * List of msr numbers which we expose to userspace through KVM_GET_MSRS * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 20da8e9751c0..5592456c36ad 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -125,5 +125,5 @@ static inline int is_paging(struct kvm_vcpu *vcpu) return vcpu->cr0 & X86_CR0_PG; } - +int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); #endif From f78e0e2ee498e8f847500b565792c7d7634dcf54 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Mon, 29 Oct 2007 09:40:42 +0800 Subject: [PATCH 072/249] KVM: VMX: Enable memory mapped TPR shadow (FlexPriority) This patch based on CR8/TPR patch, and enable the TPR shadow (FlexPriority) for 32bit Windows. Since TPR is accessed very frequently by 32bit Windows, especially SMP guest, with FlexPriority enabled, we saw significant performance gain. Signed-off-by: Sheng Yang Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 4 ++ drivers/kvm/kvm_main.c | 56 ++++++++++++++------ drivers/kvm/vmx.c | 117 ++++++++++++++++++++++++++++++++++++----- drivers/kvm/vmx.h | 5 ++ 4 files changed, 152 insertions(+), 30 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 516f79ffd126..22317d6f66ae 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -383,6 +383,7 @@ struct kvm { struct kvm_ioapic *vioapic; int round_robin_prev_vcpu; unsigned int tss_addr; + struct page *apic_access_page; }; static inline struct kvm_pic *pic_irqchip(struct kvm *kvm) @@ -522,6 +523,9 @@ int is_error_page(struct page *page); int kvm_set_memory_region(struct kvm *kvm, struct kvm_userspace_memory_region *mem, int user_alloc); +int __kvm_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + int user_alloc); gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); void kvm_release_page(struct page *page); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 8f7125710d02..ac5ed00e9065 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -362,10 +362,12 @@ EXPORT_SYMBOL_GPL(fx_init); * space. * * Discontiguous memory is allowed, mostly for framebuffers. + * + * Must be called holding kvm->lock. */ -int kvm_set_memory_region(struct kvm *kvm, - struct kvm_userspace_memory_region *mem, - int user_alloc) +int __kvm_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + int user_alloc) { int r; gfn_t base_gfn; @@ -392,8 +394,6 @@ int kvm_set_memory_region(struct kvm *kvm, if (!npages) mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES; - mutex_lock(&kvm->lock); - new = old = *memslot; new.base_gfn = base_gfn; @@ -403,7 +403,7 @@ int kvm_set_memory_region(struct kvm *kvm, /* Disallow changing a memory slot's size. */ r = -EINVAL; if (npages && old.npages && npages != old.npages) - goto out_unlock; + goto out_free; /* Check for overlaps */ r = -EEXIST; @@ -414,7 +414,7 @@ int kvm_set_memory_region(struct kvm *kvm, continue; if (!((base_gfn + npages <= s->base_gfn) || (base_gfn >= s->base_gfn + s->npages))) - goto out_unlock; + goto out_free; } /* Free page dirty bitmap if unneeded */ @@ -428,7 +428,7 @@ int kvm_set_memory_region(struct kvm *kvm, new.rmap = vmalloc(npages * sizeof(struct page *)); if (!new.rmap) - goto out_unlock; + goto out_free; memset(new.rmap, 0, npages * sizeof(*new.rmap)); @@ -445,7 +445,7 @@ int kvm_set_memory_region(struct kvm *kvm, up_write(¤t->mm->mmap_sem); if (IS_ERR((void *)new.userspace_addr)) - goto out_unlock; + goto out_free; } } else { if (!old.user_alloc && old.rmap) { @@ -468,7 +468,7 @@ int kvm_set_memory_region(struct kvm *kvm, new.dirty_bitmap = vmalloc(dirty_bytes); if (!new.dirty_bitmap) - goto out_unlock; + goto out_free; memset(new.dirty_bitmap, 0, dirty_bytes); } @@ -498,18 +498,28 @@ int kvm_set_memory_region(struct kvm *kvm, kvm_mmu_slot_remove_write_access(kvm, mem->slot); kvm_flush_remote_tlbs(kvm); - mutex_unlock(&kvm->lock); - kvm_free_physmem_slot(&old, &new); return 0; -out_unlock: - mutex_unlock(&kvm->lock); +out_free: kvm_free_physmem_slot(&new, &old); out: return r; } +EXPORT_SYMBOL_GPL(__kvm_set_memory_region); + +int kvm_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + int user_alloc) +{ + int r; + + mutex_lock(&kvm->lock); + r = __kvm_set_memory_region(kvm, mem, user_alloc); + mutex_unlock(&kvm->lock); + return r; +} EXPORT_SYMBOL_GPL(kvm_set_memory_region); int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, @@ -888,14 +898,21 @@ static int emulator_read_emulated(unsigned long addr, memcpy(val, vcpu->mmio_data, bytes); vcpu->mmio_read_completed = 0; return X86EMUL_CONTINUE; - } else if (emulator_read_std(addr, val, bytes, vcpu) - == X86EMUL_CONTINUE) - return X86EMUL_CONTINUE; + } gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + + /* For APIC access vmexit */ + if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) + goto mmio; + + if (emulator_read_std(addr, val, bytes, vcpu) + == X86EMUL_CONTINUE) + return X86EMUL_CONTINUE; if (gpa == UNMAPPED_GVA) return X86EMUL_PROPAGATE_FAULT; +mmio: /* * Is this MMIO handled locally? */ @@ -938,9 +955,14 @@ static int emulator_write_emulated_onepage(unsigned long addr, return X86EMUL_PROPAGATE_FAULT; } + /* For APIC access vmexit */ + if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) + goto mmio; + if (emulator_write_phys(vcpu, gpa, val, bytes)) return X86EMUL_CONTINUE; +mmio: /* * Is this MMIO handled locally? */ diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 7fe834cb0d81..eca422e9506d 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -86,6 +86,7 @@ static struct vmcs_config { u32 revision_id; u32 pin_based_exec_ctrl; u32 cpu_based_exec_ctrl; + u32 cpu_based_2nd_exec_ctrl; u32 vmexit_ctrl; u32 vmentry_ctrl; } vmcs_config; @@ -179,6 +180,29 @@ static inline int vm_need_tpr_shadow(struct kvm *kvm) return ((cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm))); } +static inline int cpu_has_secondary_exec_ctrls(void) +{ + return (vmcs_config.cpu_based_exec_ctrl & + CPU_BASED_ACTIVATE_SECONDARY_CONTROLS); +} + +static inline int vm_need_secondary_exec_ctrls(struct kvm *kvm) +{ + return ((cpu_has_secondary_exec_ctrls()) && (irqchip_in_kernel(kvm))); +} + +static inline int cpu_has_vmx_virtualize_apic_accesses(void) +{ + return (vmcs_config.cpu_based_2nd_exec_ctrl & + SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES); +} + +static inline int vm_need_virtualize_apic_accesses(struct kvm *kvm) +{ + return ((cpu_has_vmx_virtualize_apic_accesses()) && + (irqchip_in_kernel(kvm))); +} + static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr) { int i; @@ -918,6 +942,7 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) u32 min, opt; u32 _pin_based_exec_control = 0; u32 _cpu_based_exec_control = 0; + u32 _cpu_based_2nd_exec_control = 0; u32 _vmexit_control = 0; u32 _vmentry_control = 0; @@ -935,11 +960,8 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MOV_DR_EXITING | CPU_BASED_USE_TSC_OFFSETING; -#ifdef CONFIG_X86_64 - opt = CPU_BASED_TPR_SHADOW; -#else - opt = 0; -#endif + opt = CPU_BASED_TPR_SHADOW | + CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS, &_cpu_based_exec_control) < 0) return -EIO; @@ -948,6 +970,18 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING & ~CPU_BASED_CR8_STORE_EXITING; #endif + if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) { + min = 0; + opt = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; + if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS2, + &_cpu_based_2nd_exec_control) < 0) + return -EIO; + } +#ifndef CONFIG_X86_64 + if (!(_cpu_based_2nd_exec_control & + SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) + _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW; +#endif min = 0; #ifdef CONFIG_X86_64 @@ -985,6 +1019,7 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control; vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control; + vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control; vmcs_conf->vmexit_ctrl = _vmexit_control; vmcs_conf->vmentry_ctrl = _vmentry_control; @@ -1427,6 +1462,27 @@ static void seg_setup(int seg) vmcs_write32(sf->ar_bytes, 0x93); } +static int alloc_apic_access_page(struct kvm *kvm) +{ + struct kvm_userspace_memory_region kvm_userspace_mem; + int r = 0; + + mutex_lock(&kvm->lock); + if (kvm->apic_access_page) + goto out; + kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT; + kvm_userspace_mem.flags = 0; + kvm_userspace_mem.guest_phys_addr = 0xfee00000ULL; + kvm_userspace_mem.memory_size = PAGE_SIZE; + r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0); + if (r) + goto out; + kvm->apic_access_page = gfn_to_page(kvm, 0xfee00); +out: + mutex_unlock(&kvm->lock); + return r; +} + /* * Sets up the vmcs for emulated real mode. */ @@ -1458,8 +1514,14 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) CPU_BASED_CR8_LOAD_EXITING; #endif } + if (!vm_need_secondary_exec_ctrls(vmx->vcpu.kvm)) + exec_control &= ~CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control); + if (vm_need_secondary_exec_ctrls(vmx->vcpu.kvm)) + vmcs_write32(SECONDARY_VM_EXEC_CONTROL, + vmcs_config.cpu_based_2nd_exec_ctrl); + vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, !!bypass_guest_pf); vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, !!bypass_guest_pf); vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */ @@ -1528,6 +1590,10 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL); vmcs_writel(CR4_GUEST_HOST_MASK, KVM_GUEST_CR4_MASK); + if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm)) + if (alloc_apic_access_page(vmx->vcpu.kvm) != 0) + return -ENOMEM; + return 0; } @@ -1616,13 +1682,17 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */ -#ifdef CONFIG_X86_64 - vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); - if (vm_need_tpr_shadow(vmx->vcpu.kvm)) - vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, - page_to_phys(vmx->vcpu.apic->regs_page)); - vmcs_write32(TPR_THRESHOLD, 0); -#endif + if (cpu_has_vmx_tpr_shadow()) { + vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); + if (vm_need_tpr_shadow(vmx->vcpu.kvm)) + vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, + page_to_phys(vmx->vcpu.apic->regs_page)); + vmcs_write32(TPR_THRESHOLD, 0); + } + + if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm)) + vmcs_write64(APIC_ACCESS_ADDR, + page_to_phys(vmx->vcpu.kvm->apic_access_page)); vmx->vcpu.cr0 = 0x60000010; vmx_set_cr0(&vmx->vcpu, vmx->vcpu.cr0); /* enter rmode */ @@ -2094,6 +2164,26 @@ static int handle_vmcall(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) return 1; } +static int handle_apic_access(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + u64 exit_qualification; + enum emulation_result er; + unsigned long offset; + + exit_qualification = vmcs_read64(EXIT_QUALIFICATION); + offset = exit_qualification & 0xffful; + + er = emulate_instruction(vcpu, kvm_run, 0, 0, 0); + + if (er != EMULATE_DONE) { + printk(KERN_ERR + "Fail to handle apic access vmexit! Offset is 0x%lx\n", + offset); + return -ENOTSUPP; + } + return 1; +} + /* * The exit handlers return 1 if the exit was handled fully and guest execution * may resume. Otherwise they set the kvm_run parameter to indicate what needs @@ -2113,7 +2203,8 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu, [EXIT_REASON_PENDING_INTERRUPT] = handle_interrupt_window, [EXIT_REASON_HLT] = handle_halt, [EXIT_REASON_VMCALL] = handle_vmcall, - [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold + [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold, + [EXIT_REASON_APIC_ACCESS] = handle_apic_access, }; static const int kvm_vmx_max_exit_handlers = diff --git a/drivers/kvm/vmx.h b/drivers/kvm/vmx.h index 270d477a2aa6..c84bd3733153 100644 --- a/drivers/kvm/vmx.h +++ b/drivers/kvm/vmx.h @@ -89,6 +89,8 @@ enum vmcs_field { TSC_OFFSET_HIGH = 0x00002011, VIRTUAL_APIC_PAGE_ADDR = 0x00002012, VIRTUAL_APIC_PAGE_ADDR_HIGH = 0x00002013, + APIC_ACCESS_ADDR = 0x00002014, + APIC_ACCESS_ADDR_HIGH = 0x00002015, VMCS_LINK_POINTER = 0x00002800, VMCS_LINK_POINTER_HIGH = 0x00002801, GUEST_IA32_DEBUGCTL = 0x00002802, @@ -214,6 +216,7 @@ enum vmcs_field { #define EXIT_REASON_MSR_WRITE 32 #define EXIT_REASON_MWAIT_INSTRUCTION 36 #define EXIT_REASON_TPR_BELOW_THRESHOLD 43 +#define EXIT_REASON_APIC_ACCESS 44 /* * Interruption-information format @@ -307,4 +310,6 @@ enum vmcs_field { #define MSR_IA32_FEATURE_CONTROL_LOCKED 0x1 #define MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED 0x4 +#define APIC_ACCESS_PAGE_PRIVATE_MEMSLOT 9 + #endif From aab61cc0d28f6fab0c2c9137d95dea54c7dbcf46 Mon Sep 17 00:00:00 2001 From: Anthony Liguori Date: Mon, 29 Oct 2007 15:15:20 -0500 Subject: [PATCH 073/249] KVM: Fix gfn_to_page() acquiring mmap_sem twice KVM's nopage handler calls gfn_to_page() which acquires the mmap_sem when calling out to get_user_pages(). nopage handlers are already invoked with the mmap_sem held though. Introduce a __gfn_to_page() for use by the nopage handler which requires the lock to already be held. This was noticed by tglx. Signed-off-by: Anthony Liguori Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index ac5ed00e9065..f439e45233f2 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -633,7 +633,10 @@ int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) } EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); -struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) +/* + * Requires current->mm->mmap_sem to be held + */ +static struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn) { struct kvm_memory_slot *slot; struct page *page[1]; @@ -648,12 +651,10 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) return bad_page; } - down_read(¤t->mm->mmap_sem); npages = get_user_pages(current, current->mm, slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE, 1, 1, 1, page, NULL); - up_read(¤t->mm->mmap_sem); if (npages != 1) { get_page(bad_page); return bad_page; @@ -661,6 +662,18 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) return page[0]; } + +struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) +{ + struct page *page; + + down_read(¤t->mm->mmap_sem); + page = __gfn_to_page(kvm, gfn); + up_read(¤t->mm->mmap_sem); + + return page; +} + EXPORT_SYMBOL_GPL(gfn_to_page); void kvm_release_page(struct page *page) @@ -2621,7 +2634,8 @@ static struct page *kvm_vm_nopage(struct vm_area_struct *vma, pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; if (!kvm_is_visible_gfn(kvm, pgoff)) return NOPAGE_SIGBUS; - page = gfn_to_page(kvm, pgoff); + /* current->mm->mmap_sem is already held so call lockless version */ + page = __gfn_to_page(kvm, pgoff); if (is_error_page(page)) { kvm_release_page(page); return NOPAGE_SIGBUS; From 15c4a6406f6c40632260861e1db7c539e79dcf1a Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Tue, 30 Oct 2007 18:44:17 +0100 Subject: [PATCH 074/249] KVM: Portability: Move kvm_get/set_msr[_common] to x86.c This patch moves the implementation of the functions of kvm_get/set_msr, kvm_get/set_msr_common, and set_efer from kvm_main.c to x86.c. The definition of EFER_RESERVED_BITS is moved too. Signed-off-by: Carsten Otte Acked-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 133 ---------------------------------------- drivers/kvm/x86.c | 134 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 133 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index f439e45233f2..7acf4cb07793 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -90,8 +90,6 @@ static struct kvm_stats_debugfs_item { static struct dentry *debugfs_dir; -#define EFER_RESERVED_BITS 0xfffffffffffff2fe - static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, unsigned long arg); @@ -1356,137 +1354,6 @@ void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, } } -int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) -{ - u64 data; - - switch (msr) { - case 0xc0010010: /* SYSCFG */ - case 0xc0010015: /* HWCR */ - case MSR_IA32_PLATFORM_ID: - case MSR_IA32_P5_MC_ADDR: - case MSR_IA32_P5_MC_TYPE: - case MSR_IA32_MC0_CTL: - case MSR_IA32_MCG_STATUS: - case MSR_IA32_MCG_CAP: - case MSR_IA32_MC0_MISC: - case MSR_IA32_MC0_MISC+4: - case MSR_IA32_MC0_MISC+8: - case MSR_IA32_MC0_MISC+12: - case MSR_IA32_MC0_MISC+16: - case MSR_IA32_UCODE_REV: - case MSR_IA32_PERF_STATUS: - case MSR_IA32_EBL_CR_POWERON: - /* MTRR registers */ - case 0xfe: - case 0x200 ... 0x2ff: - data = 0; - break; - case 0xcd: /* fsb frequency */ - data = 3; - break; - case MSR_IA32_APICBASE: - data = kvm_get_apic_base(vcpu); - break; - case MSR_IA32_MISC_ENABLE: - data = vcpu->ia32_misc_enable_msr; - break; -#ifdef CONFIG_X86_64 - case MSR_EFER: - data = vcpu->shadow_efer; - break; -#endif - default: - pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr); - return 1; - } - *pdata = data; - return 0; -} -EXPORT_SYMBOL_GPL(kvm_get_msr_common); - -/* - * Reads an msr value (of 'msr_index') into 'pdata'. - * Returns 0 on success, non-0 otherwise. - * Assumes vcpu_load() was already called. - */ -int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) -{ - return kvm_x86_ops->get_msr(vcpu, msr_index, pdata); -} - -#ifdef CONFIG_X86_64 - -static void set_efer(struct kvm_vcpu *vcpu, u64 efer) -{ - if (efer & EFER_RESERVED_BITS) { - printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n", - efer); - inject_gp(vcpu); - return; - } - - if (is_paging(vcpu) - && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) { - printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n"); - inject_gp(vcpu); - return; - } - - kvm_x86_ops->set_efer(vcpu, efer); - - efer &= ~EFER_LMA; - efer |= vcpu->shadow_efer & EFER_LMA; - - vcpu->shadow_efer = efer; -} - -#endif - -int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) -{ - switch (msr) { -#ifdef CONFIG_X86_64 - case MSR_EFER: - set_efer(vcpu, data); - break; -#endif - case MSR_IA32_MC0_STATUS: - pr_unimpl(vcpu, "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n", - __FUNCTION__, data); - break; - case MSR_IA32_MCG_STATUS: - pr_unimpl(vcpu, "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n", - __FUNCTION__, data); - break; - case MSR_IA32_UCODE_REV: - case MSR_IA32_UCODE_WRITE: - case 0x200 ... 0x2ff: /* MTRRs */ - break; - case MSR_IA32_APICBASE: - kvm_set_apic_base(vcpu, data); - break; - case MSR_IA32_MISC_ENABLE: - vcpu->ia32_misc_enable_msr = data; - break; - default: - pr_unimpl(vcpu, "unhandled wrmsr: 0x%x\n", msr); - return 1; - } - return 0; -} -EXPORT_SYMBOL_GPL(kvm_set_msr_common); - -/* - * Writes msr value into into the appropriate "register". - * Returns 0 on success, non-0 otherwise. - * Assumes vcpu_load() was already called. - */ -int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) -{ - return kvm_x86_ops->set_msr(vcpu, msr_index, data); -} - void kvm_resched(struct kvm_vcpu *vcpu) { if (!need_resched()) diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index a728af8a83e5..786274347512 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -38,6 +38,7 @@ | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE)) #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) +#define EFER_RESERVED_BITS 0xfffffffffffff2fe unsigned long segment_base(u16 selector) { @@ -324,6 +325,44 @@ static u32 emulated_msrs[] = { MSR_IA32_MISC_ENABLE, }; +#ifdef CONFIG_X86_64 + +static void set_efer(struct kvm_vcpu *vcpu, u64 efer) +{ + if (efer & EFER_RESERVED_BITS) { + printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n", + efer); + inject_gp(vcpu); + return; + } + + if (is_paging(vcpu) + && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) { + printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n"); + inject_gp(vcpu); + return; + } + + kvm_x86_ops->set_efer(vcpu, efer); + + efer &= ~EFER_LMA; + efer |= vcpu->shadow_efer & EFER_LMA; + + vcpu->shadow_efer = efer; +} + +#endif + +/* + * Writes msr value into into the appropriate "register". + * Returns 0 on success, non-0 otherwise. + * Assumes vcpu_load() was already called. + */ +int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) +{ + return kvm_x86_ops->set_msr(vcpu, msr_index, data); +} + /* * Adapt set_msr() to msr_io()'s calling convention */ @@ -332,6 +371,101 @@ static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) return kvm_set_msr(vcpu, index, *data); } + +int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) +{ + switch (msr) { +#ifdef CONFIG_X86_64 + case MSR_EFER: + set_efer(vcpu, data); + break; +#endif + case MSR_IA32_MC0_STATUS: + pr_unimpl(vcpu, "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n", + __FUNCTION__, data); + break; + case MSR_IA32_MCG_STATUS: + pr_unimpl(vcpu, "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n", + __FUNCTION__, data); + break; + case MSR_IA32_UCODE_REV: + case MSR_IA32_UCODE_WRITE: + case 0x200 ... 0x2ff: /* MTRRs */ + break; + case MSR_IA32_APICBASE: + kvm_set_apic_base(vcpu, data); + break; + case MSR_IA32_MISC_ENABLE: + vcpu->ia32_misc_enable_msr = data; + break; + default: + pr_unimpl(vcpu, "unhandled wrmsr: 0x%x\n", msr); + return 1; + } + return 0; +} +EXPORT_SYMBOL_GPL(kvm_set_msr_common); + + +/* + * Reads an msr value (of 'msr_index') into 'pdata'. + * Returns 0 on success, non-0 otherwise. + * Assumes vcpu_load() was already called. + */ +int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) +{ + return kvm_x86_ops->get_msr(vcpu, msr_index, pdata); +} + +int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) +{ + u64 data; + + switch (msr) { + case 0xc0010010: /* SYSCFG */ + case 0xc0010015: /* HWCR */ + case MSR_IA32_PLATFORM_ID: + case MSR_IA32_P5_MC_ADDR: + case MSR_IA32_P5_MC_TYPE: + case MSR_IA32_MC0_CTL: + case MSR_IA32_MCG_STATUS: + case MSR_IA32_MCG_CAP: + case MSR_IA32_MC0_MISC: + case MSR_IA32_MC0_MISC+4: + case MSR_IA32_MC0_MISC+8: + case MSR_IA32_MC0_MISC+12: + case MSR_IA32_MC0_MISC+16: + case MSR_IA32_UCODE_REV: + case MSR_IA32_PERF_STATUS: + case MSR_IA32_EBL_CR_POWERON: + /* MTRR registers */ + case 0xfe: + case 0x200 ... 0x2ff: + data = 0; + break; + case 0xcd: /* fsb frequency */ + data = 3; + break; + case MSR_IA32_APICBASE: + data = kvm_get_apic_base(vcpu); + break; + case MSR_IA32_MISC_ENABLE: + data = vcpu->ia32_misc_enable_msr; + break; +#ifdef CONFIG_X86_64 + case MSR_EFER: + data = vcpu->shadow_efer; + break; +#endif + default: + pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr); + return 1; + } + *pdata = data; + return 0; +} +EXPORT_SYMBOL_GPL(kvm_get_msr_common); + /* * Read or write a bunch of msrs. All parameters are kernel addresses. * From bbd9b64e37aff5aa715ec5e168425790f5983bf1 Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Tue, 30 Oct 2007 18:44:21 +0100 Subject: [PATCH 075/249] KVM: Portability: Move x86 emulation and mmio device hook to x86.c This patch moves the following functions to from kvm_main.c to x86.c: emulator_read/write_std, vcpu_find_pervcpu_dev, vcpu_find_mmio_dev, emulator_read/write_emulated, emulator_write_phys, emulator_write_emulated_onepage, emulator_cmpxchg_emulated, get_setment_base, emulate_invlpg, emulate_clts, emulator_get/set_dr, kvm_report_emulation_failure, emulate_instruction The following data type is moved to x86.c: struct x86_emulate_ops emulate_ops Signed-off-by: Carsten Otte Acked-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 357 ---------------------------------------- drivers/kvm/x86.c | 358 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 358 insertions(+), 357 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 7acf4cb07793..1a56d76560de 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -827,369 +827,12 @@ void mark_page_dirty(struct kvm *kvm, gfn_t gfn) } } -int emulator_read_std(unsigned long addr, - void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - void *data = val; - - while (bytes) { - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); - unsigned offset = addr & (PAGE_SIZE-1); - unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset); - int ret; - - if (gpa == UNMAPPED_GVA) - return X86EMUL_PROPAGATE_FAULT; - ret = kvm_read_guest(vcpu->kvm, gpa, data, tocopy); - if (ret < 0) - return X86EMUL_UNHANDLEABLE; - - bytes -= tocopy; - data += tocopy; - addr += tocopy; - } - - return X86EMUL_CONTINUE; -} -EXPORT_SYMBOL_GPL(emulator_read_std); - -static int emulator_write_std(unsigned long addr, - const void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - pr_unimpl(vcpu, "emulator_write_std: addr %lx n %d\n", addr, bytes); - return X86EMUL_UNHANDLEABLE; -} - -/* - * Only apic need an MMIO device hook, so shortcut now.. - */ -static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu, - gpa_t addr) -{ - struct kvm_io_device *dev; - - if (vcpu->apic) { - dev = &vcpu->apic->dev; - if (dev->in_range(dev, addr)) - return dev; - } - return NULL; -} - -static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu, - gpa_t addr) -{ - struct kvm_io_device *dev; - - dev = vcpu_find_pervcpu_dev(vcpu, addr); - if (dev == NULL) - dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr); - return dev; -} - static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu, gpa_t addr) { return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr); } -static int emulator_read_emulated(unsigned long addr, - void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - struct kvm_io_device *mmio_dev; - gpa_t gpa; - - if (vcpu->mmio_read_completed) { - memcpy(val, vcpu->mmio_data, bytes); - vcpu->mmio_read_completed = 0; - return X86EMUL_CONTINUE; - } - - gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); - - /* For APIC access vmexit */ - if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) - goto mmio; - - if (emulator_read_std(addr, val, bytes, vcpu) - == X86EMUL_CONTINUE) - return X86EMUL_CONTINUE; - if (gpa == UNMAPPED_GVA) - return X86EMUL_PROPAGATE_FAULT; - -mmio: - /* - * Is this MMIO handled locally? - */ - mmio_dev = vcpu_find_mmio_dev(vcpu, gpa); - if (mmio_dev) { - kvm_iodevice_read(mmio_dev, gpa, bytes, val); - return X86EMUL_CONTINUE; - } - - vcpu->mmio_needed = 1; - vcpu->mmio_phys_addr = gpa; - vcpu->mmio_size = bytes; - vcpu->mmio_is_write = 0; - - return X86EMUL_UNHANDLEABLE; -} - -static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, - const void *val, int bytes) -{ - int ret; - - ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes); - if (ret < 0) - return 0; - kvm_mmu_pte_write(vcpu, gpa, val, bytes); - return 1; -} - -static int emulator_write_emulated_onepage(unsigned long addr, - const void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - struct kvm_io_device *mmio_dev; - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); - - if (gpa == UNMAPPED_GVA) { - kvm_x86_ops->inject_page_fault(vcpu, addr, 2); - return X86EMUL_PROPAGATE_FAULT; - } - - /* For APIC access vmexit */ - if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) - goto mmio; - - if (emulator_write_phys(vcpu, gpa, val, bytes)) - return X86EMUL_CONTINUE; - -mmio: - /* - * Is this MMIO handled locally? - */ - mmio_dev = vcpu_find_mmio_dev(vcpu, gpa); - if (mmio_dev) { - kvm_iodevice_write(mmio_dev, gpa, bytes, val); - return X86EMUL_CONTINUE; - } - - vcpu->mmio_needed = 1; - vcpu->mmio_phys_addr = gpa; - vcpu->mmio_size = bytes; - vcpu->mmio_is_write = 1; - memcpy(vcpu->mmio_data, val, bytes); - - return X86EMUL_CONTINUE; -} - -int emulator_write_emulated(unsigned long addr, - const void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - /* Crossing a page boundary? */ - if (((addr + bytes - 1) ^ addr) & PAGE_MASK) { - int rc, now; - - now = -addr & ~PAGE_MASK; - rc = emulator_write_emulated_onepage(addr, val, now, vcpu); - if (rc != X86EMUL_CONTINUE) - return rc; - addr += now; - val += now; - bytes -= now; - } - return emulator_write_emulated_onepage(addr, val, bytes, vcpu); -} -EXPORT_SYMBOL_GPL(emulator_write_emulated); - -static int emulator_cmpxchg_emulated(unsigned long addr, - const void *old, - const void *new, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - static int reported; - - if (!reported) { - reported = 1; - printk(KERN_WARNING "kvm: emulating exchange as write\n"); - } - return emulator_write_emulated(addr, new, bytes, vcpu); -} - -static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg) -{ - return kvm_x86_ops->get_segment_base(vcpu, seg); -} - -int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address) -{ - return X86EMUL_CONTINUE; -} - -int emulate_clts(struct kvm_vcpu *vcpu) -{ - kvm_x86_ops->set_cr0(vcpu, vcpu->cr0 & ~X86_CR0_TS); - return X86EMUL_CONTINUE; -} - -int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest) -{ - struct kvm_vcpu *vcpu = ctxt->vcpu; - - switch (dr) { - case 0 ... 3: - *dest = kvm_x86_ops->get_dr(vcpu, dr); - return X86EMUL_CONTINUE; - default: - pr_unimpl(vcpu, "%s: unexpected dr %u\n", __FUNCTION__, dr); - return X86EMUL_UNHANDLEABLE; - } -} - -int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value) -{ - unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U; - int exception; - - kvm_x86_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception); - if (exception) { - /* FIXME: better handling */ - return X86EMUL_UNHANDLEABLE; - } - return X86EMUL_CONTINUE; -} - -void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context) -{ - static int reported; - u8 opcodes[4]; - unsigned long rip = vcpu->rip; - unsigned long rip_linear; - - rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS); - - if (reported) - return; - - emulator_read_std(rip_linear, (void *)opcodes, 4, vcpu); - - printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n", - context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]); - reported = 1; -} -EXPORT_SYMBOL_GPL(kvm_report_emulation_failure); - -struct x86_emulate_ops emulate_ops = { - .read_std = emulator_read_std, - .write_std = emulator_write_std, - .read_emulated = emulator_read_emulated, - .write_emulated = emulator_write_emulated, - .cmpxchg_emulated = emulator_cmpxchg_emulated, -}; - -int emulate_instruction(struct kvm_vcpu *vcpu, - struct kvm_run *run, - unsigned long cr2, - u16 error_code, - int no_decode) -{ - int r; - - vcpu->mmio_fault_cr2 = cr2; - kvm_x86_ops->cache_regs(vcpu); - - vcpu->mmio_is_write = 0; - vcpu->pio.string = 0; - - if (!no_decode) { - int cs_db, cs_l; - kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); - - vcpu->emulate_ctxt.vcpu = vcpu; - vcpu->emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu); - vcpu->emulate_ctxt.cr2 = cr2; - vcpu->emulate_ctxt.mode = - (vcpu->emulate_ctxt.eflags & X86_EFLAGS_VM) - ? X86EMUL_MODE_REAL : cs_l - ? X86EMUL_MODE_PROT64 : cs_db - ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; - - if (vcpu->emulate_ctxt.mode == X86EMUL_MODE_PROT64) { - vcpu->emulate_ctxt.cs_base = 0; - vcpu->emulate_ctxt.ds_base = 0; - vcpu->emulate_ctxt.es_base = 0; - vcpu->emulate_ctxt.ss_base = 0; - } else { - vcpu->emulate_ctxt.cs_base = - get_segment_base(vcpu, VCPU_SREG_CS); - vcpu->emulate_ctxt.ds_base = - get_segment_base(vcpu, VCPU_SREG_DS); - vcpu->emulate_ctxt.es_base = - get_segment_base(vcpu, VCPU_SREG_ES); - vcpu->emulate_ctxt.ss_base = - get_segment_base(vcpu, VCPU_SREG_SS); - } - - vcpu->emulate_ctxt.gs_base = - get_segment_base(vcpu, VCPU_SREG_GS); - vcpu->emulate_ctxt.fs_base = - get_segment_base(vcpu, VCPU_SREG_FS); - - r = x86_decode_insn(&vcpu->emulate_ctxt, &emulate_ops); - if (r) { - if (kvm_mmu_unprotect_page_virt(vcpu, cr2)) - return EMULATE_DONE; - return EMULATE_FAIL; - } - } - - r = x86_emulate_insn(&vcpu->emulate_ctxt, &emulate_ops); - - if (vcpu->pio.string) - return EMULATE_DO_MMIO; - - if ((r || vcpu->mmio_is_write) && run) { - run->exit_reason = KVM_EXIT_MMIO; - run->mmio.phys_addr = vcpu->mmio_phys_addr; - memcpy(run->mmio.data, vcpu->mmio_data, 8); - run->mmio.len = vcpu->mmio_size; - run->mmio.is_write = vcpu->mmio_is_write; - } - - if (r) { - if (kvm_mmu_unprotect_page_virt(vcpu, cr2)) - return EMULATE_DONE; - if (!vcpu->mmio_needed) { - kvm_report_emulation_failure(vcpu, "mmio"); - return EMULATE_FAIL; - } - return EMULATE_DO_MMIO; - } - - kvm_x86_ops->decache_regs(vcpu); - kvm_x86_ops->set_rflags(vcpu, vcpu->emulate_ctxt.eflags); - - if (vcpu->mmio_is_write) { - vcpu->mmio_needed = 0; - return EMULATE_DO_MMIO; - } - - return EMULATE_DONE; -} -EXPORT_SYMBOL_GPL(emulate_instruction); - /* * The vCPU has executed a HLT instruction with in-kernel mode enabled. */ diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 786274347512..fe3733d8ece5 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -983,6 +983,364 @@ static __init void kvm_init_msr_list(void) num_msrs_to_save = j; } +/* + * Only apic need an MMIO device hook, so shortcut now.. + */ +static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu, + gpa_t addr) +{ + struct kvm_io_device *dev; + + if (vcpu->apic) { + dev = &vcpu->apic->dev; + if (dev->in_range(dev, addr)) + return dev; + } + return NULL; +} + + +static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu, + gpa_t addr) +{ + struct kvm_io_device *dev; + + dev = vcpu_find_pervcpu_dev(vcpu, addr); + if (dev == NULL) + dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr); + return dev; +} + +int emulator_read_std(unsigned long addr, + void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu) +{ + void *data = val; + + while (bytes) { + gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + unsigned offset = addr & (PAGE_SIZE-1); + unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset); + int ret; + + if (gpa == UNMAPPED_GVA) + return X86EMUL_PROPAGATE_FAULT; + ret = kvm_read_guest(vcpu->kvm, gpa, data, tocopy); + if (ret < 0) + return X86EMUL_UNHANDLEABLE; + + bytes -= tocopy; + data += tocopy; + addr += tocopy; + } + + return X86EMUL_CONTINUE; +} +EXPORT_SYMBOL_GPL(emulator_read_std); + +static int emulator_write_std(unsigned long addr, + const void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu) +{ + pr_unimpl(vcpu, "emulator_write_std: addr %lx n %d\n", addr, bytes); + return X86EMUL_UNHANDLEABLE; +} + +static int emulator_read_emulated(unsigned long addr, + void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu) +{ + struct kvm_io_device *mmio_dev; + gpa_t gpa; + + if (vcpu->mmio_read_completed) { + memcpy(val, vcpu->mmio_data, bytes); + vcpu->mmio_read_completed = 0; + return X86EMUL_CONTINUE; + } + + gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + + /* For APIC access vmexit */ + if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) + goto mmio; + + if (emulator_read_std(addr, val, bytes, vcpu) + == X86EMUL_CONTINUE) + return X86EMUL_CONTINUE; + if (gpa == UNMAPPED_GVA) + return X86EMUL_PROPAGATE_FAULT; + +mmio: + /* + * Is this MMIO handled locally? + */ + mmio_dev = vcpu_find_mmio_dev(vcpu, gpa); + if (mmio_dev) { + kvm_iodevice_read(mmio_dev, gpa, bytes, val); + return X86EMUL_CONTINUE; + } + + vcpu->mmio_needed = 1; + vcpu->mmio_phys_addr = gpa; + vcpu->mmio_size = bytes; + vcpu->mmio_is_write = 0; + + return X86EMUL_UNHANDLEABLE; +} + +static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, + const void *val, int bytes) +{ + int ret; + + ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes); + if (ret < 0) + return 0; + kvm_mmu_pte_write(vcpu, gpa, val, bytes); + return 1; +} + +static int emulator_write_emulated_onepage(unsigned long addr, + const void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu) +{ + struct kvm_io_device *mmio_dev; + gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + + if (gpa == UNMAPPED_GVA) { + kvm_x86_ops->inject_page_fault(vcpu, addr, 2); + return X86EMUL_PROPAGATE_FAULT; + } + + /* For APIC access vmexit */ + if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) + goto mmio; + + if (emulator_write_phys(vcpu, gpa, val, bytes)) + return X86EMUL_CONTINUE; + +mmio: + /* + * Is this MMIO handled locally? + */ + mmio_dev = vcpu_find_mmio_dev(vcpu, gpa); + if (mmio_dev) { + kvm_iodevice_write(mmio_dev, gpa, bytes, val); + return X86EMUL_CONTINUE; + } + + vcpu->mmio_needed = 1; + vcpu->mmio_phys_addr = gpa; + vcpu->mmio_size = bytes; + vcpu->mmio_is_write = 1; + memcpy(vcpu->mmio_data, val, bytes); + + return X86EMUL_CONTINUE; +} + +int emulator_write_emulated(unsigned long addr, + const void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu) +{ + /* Crossing a page boundary? */ + if (((addr + bytes - 1) ^ addr) & PAGE_MASK) { + int rc, now; + + now = -addr & ~PAGE_MASK; + rc = emulator_write_emulated_onepage(addr, val, now, vcpu); + if (rc != X86EMUL_CONTINUE) + return rc; + addr += now; + val += now; + bytes -= now; + } + return emulator_write_emulated_onepage(addr, val, bytes, vcpu); +} +EXPORT_SYMBOL_GPL(emulator_write_emulated); + +static int emulator_cmpxchg_emulated(unsigned long addr, + const void *old, + const void *new, + unsigned int bytes, + struct kvm_vcpu *vcpu) +{ + static int reported; + + if (!reported) { + reported = 1; + printk(KERN_WARNING "kvm: emulating exchange as write\n"); + } + return emulator_write_emulated(addr, new, bytes, vcpu); +} + +static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg) +{ + return kvm_x86_ops->get_segment_base(vcpu, seg); +} + +int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address) +{ + return X86EMUL_CONTINUE; +} + +int emulate_clts(struct kvm_vcpu *vcpu) +{ + kvm_x86_ops->set_cr0(vcpu, vcpu->cr0 & ~X86_CR0_TS); + return X86EMUL_CONTINUE; +} + +int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest) +{ + struct kvm_vcpu *vcpu = ctxt->vcpu; + + switch (dr) { + case 0 ... 3: + *dest = kvm_x86_ops->get_dr(vcpu, dr); + return X86EMUL_CONTINUE; + default: + pr_unimpl(vcpu, "%s: unexpected dr %u\n", __FUNCTION__, dr); + return X86EMUL_UNHANDLEABLE; + } +} + +int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value) +{ + unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U; + int exception; + + kvm_x86_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception); + if (exception) { + /* FIXME: better handling */ + return X86EMUL_UNHANDLEABLE; + } + return X86EMUL_CONTINUE; +} + +void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context) +{ + static int reported; + u8 opcodes[4]; + unsigned long rip = vcpu->rip; + unsigned long rip_linear; + + rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS); + + if (reported) + return; + + emulator_read_std(rip_linear, (void *)opcodes, 4, vcpu); + + printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n", + context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]); + reported = 1; +} +EXPORT_SYMBOL_GPL(kvm_report_emulation_failure); + +struct x86_emulate_ops emulate_ops = { + .read_std = emulator_read_std, + .write_std = emulator_write_std, + .read_emulated = emulator_read_emulated, + .write_emulated = emulator_write_emulated, + .cmpxchg_emulated = emulator_cmpxchg_emulated, +}; + +int emulate_instruction(struct kvm_vcpu *vcpu, + struct kvm_run *run, + unsigned long cr2, + u16 error_code, + int no_decode) +{ + int r; + + vcpu->mmio_fault_cr2 = cr2; + kvm_x86_ops->cache_regs(vcpu); + + vcpu->mmio_is_write = 0; + vcpu->pio.string = 0; + + if (!no_decode) { + int cs_db, cs_l; + kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); + + vcpu->emulate_ctxt.vcpu = vcpu; + vcpu->emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu); + vcpu->emulate_ctxt.cr2 = cr2; + vcpu->emulate_ctxt.mode = + (vcpu->emulate_ctxt.eflags & X86_EFLAGS_VM) + ? X86EMUL_MODE_REAL : cs_l + ? X86EMUL_MODE_PROT64 : cs_db + ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; + + if (vcpu->emulate_ctxt.mode == X86EMUL_MODE_PROT64) { + vcpu->emulate_ctxt.cs_base = 0; + vcpu->emulate_ctxt.ds_base = 0; + vcpu->emulate_ctxt.es_base = 0; + vcpu->emulate_ctxt.ss_base = 0; + } else { + vcpu->emulate_ctxt.cs_base = + get_segment_base(vcpu, VCPU_SREG_CS); + vcpu->emulate_ctxt.ds_base = + get_segment_base(vcpu, VCPU_SREG_DS); + vcpu->emulate_ctxt.es_base = + get_segment_base(vcpu, VCPU_SREG_ES); + vcpu->emulate_ctxt.ss_base = + get_segment_base(vcpu, VCPU_SREG_SS); + } + + vcpu->emulate_ctxt.gs_base = + get_segment_base(vcpu, VCPU_SREG_GS); + vcpu->emulate_ctxt.fs_base = + get_segment_base(vcpu, VCPU_SREG_FS); + + r = x86_decode_insn(&vcpu->emulate_ctxt, &emulate_ops); + if (r) { + if (kvm_mmu_unprotect_page_virt(vcpu, cr2)) + return EMULATE_DONE; + return EMULATE_FAIL; + } + } + + r = x86_emulate_insn(&vcpu->emulate_ctxt, &emulate_ops); + + if (vcpu->pio.string) + return EMULATE_DO_MMIO; + + if ((r || vcpu->mmio_is_write) && run) { + run->exit_reason = KVM_EXIT_MMIO; + run->mmio.phys_addr = vcpu->mmio_phys_addr; + memcpy(run->mmio.data, vcpu->mmio_data, 8); + run->mmio.len = vcpu->mmio_size; + run->mmio.is_write = vcpu->mmio_is_write; + } + + if (r) { + if (kvm_mmu_unprotect_page_virt(vcpu, cr2)) + return EMULATE_DONE; + if (!vcpu->mmio_needed) { + kvm_report_emulation_failure(vcpu, "mmio"); + return EMULATE_FAIL; + } + return EMULATE_DO_MMIO; + } + + kvm_x86_ops->decache_regs(vcpu); + kvm_x86_ops->set_rflags(vcpu, vcpu->emulate_ctxt.eflags); + + if (vcpu->mmio_is_write) { + vcpu->mmio_needed = 0; + return EMULATE_DO_MMIO; + } + + return EMULATE_DONE; +} +EXPORT_SYMBOL_GPL(emulate_instruction); + __init void kvm_arch_init(void) { kvm_init_msr_list(); From de7d789acd7f373268194bb48dc0690c975ab8e6 Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Tue, 30 Oct 2007 18:44:25 +0100 Subject: [PATCH 076/249] KVM: Portability: Move pio emulation functions to x86.c This patch moves implementation of the following functions from kvm_main.c to x86.c: free_pio_guest_pages, vcpu_find_pio_dev, pio_copy_data, complete_pio, kernel_pio, pio_string_write, kvm_emulate_pio, kvm_emulate_pio_string The function inject_gp, which was duplicated by yesterday's patch series, is removed from kvm_main.c now because it is not needed anymore. Signed-off-by: Carsten Otte Acked-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 248 ----------------------------------------- drivers/kvm/x86.c | 243 ++++++++++++++++++++++++++++++++++++++++ drivers/kvm/x86.h | 1 + 3 files changed, 244 insertions(+), 248 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 1a56d76560de..2025cdfb4593 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -271,17 +271,6 @@ static void kvm_free_physmem(struct kvm *kvm) kvm_free_physmem_slot(&kvm->memslots[i], NULL); } -static void free_pio_guest_pages(struct kvm_vcpu *vcpu) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(vcpu->pio.guest_pages); ++i) - if (vcpu->pio.guest_pages[i]) { - kvm_release_page(vcpu->pio.guest_pages[i]); - vcpu->pio.guest_pages[i] = NULL; - } -} - static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) { vcpu_load(vcpu); @@ -330,11 +319,6 @@ static int kvm_vm_release(struct inode *inode, struct file *filp) return 0; } -static void inject_gp(struct kvm_vcpu *vcpu) -{ - kvm_x86_ops->inject_gp(vcpu, 0); -} - void fx_init(struct kvm_vcpu *vcpu) { unsigned after_mxcsr_mask; @@ -827,12 +811,6 @@ void mark_page_dirty(struct kvm *kvm, gfn_t gfn) } } -static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu, - gpa_t addr) -{ - return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr); -} - /* * The vCPU has executed a HLT instruction with in-kernel mode enabled. */ @@ -1042,232 +1020,6 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); -static int pio_copy_data(struct kvm_vcpu *vcpu) -{ - void *p = vcpu->pio_data; - void *q; - unsigned bytes; - int nr_pages = vcpu->pio.guest_pages[1] ? 2 : 1; - - q = vmap(vcpu->pio.guest_pages, nr_pages, VM_READ|VM_WRITE, - PAGE_KERNEL); - if (!q) { - free_pio_guest_pages(vcpu); - return -ENOMEM; - } - q += vcpu->pio.guest_page_offset; - bytes = vcpu->pio.size * vcpu->pio.cur_count; - if (vcpu->pio.in) - memcpy(q, p, bytes); - else - memcpy(p, q, bytes); - q -= vcpu->pio.guest_page_offset; - vunmap(q); - free_pio_guest_pages(vcpu); - return 0; -} - -static int complete_pio(struct kvm_vcpu *vcpu) -{ - struct kvm_pio_request *io = &vcpu->pio; - long delta; - int r; - - kvm_x86_ops->cache_regs(vcpu); - - if (!io->string) { - if (io->in) - memcpy(&vcpu->regs[VCPU_REGS_RAX], vcpu->pio_data, - io->size); - } else { - if (io->in) { - r = pio_copy_data(vcpu); - if (r) { - kvm_x86_ops->cache_regs(vcpu); - return r; - } - } - - delta = 1; - if (io->rep) { - delta *= io->cur_count; - /* - * The size of the register should really depend on - * current address size. - */ - vcpu->regs[VCPU_REGS_RCX] -= delta; - } - if (io->down) - delta = -delta; - delta *= io->size; - if (io->in) - vcpu->regs[VCPU_REGS_RDI] += delta; - else - vcpu->regs[VCPU_REGS_RSI] += delta; - } - - kvm_x86_ops->decache_regs(vcpu); - - io->count -= io->cur_count; - io->cur_count = 0; - - return 0; -} - -static void kernel_pio(struct kvm_io_device *pio_dev, - struct kvm_vcpu *vcpu, - void *pd) -{ - /* TODO: String I/O for in kernel device */ - - mutex_lock(&vcpu->kvm->lock); - if (vcpu->pio.in) - kvm_iodevice_read(pio_dev, vcpu->pio.port, - vcpu->pio.size, - pd); - else - kvm_iodevice_write(pio_dev, vcpu->pio.port, - vcpu->pio.size, - pd); - mutex_unlock(&vcpu->kvm->lock); -} - -static void pio_string_write(struct kvm_io_device *pio_dev, - struct kvm_vcpu *vcpu) -{ - struct kvm_pio_request *io = &vcpu->pio; - void *pd = vcpu->pio_data; - int i; - - mutex_lock(&vcpu->kvm->lock); - for (i = 0; i < io->cur_count; i++) { - kvm_iodevice_write(pio_dev, io->port, - io->size, - pd); - pd += io->size; - } - mutex_unlock(&vcpu->kvm->lock); -} - -int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, - int size, unsigned port) -{ - struct kvm_io_device *pio_dev; - - vcpu->run->exit_reason = KVM_EXIT_IO; - vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; - vcpu->run->io.size = vcpu->pio.size = size; - vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; - vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = 1; - vcpu->run->io.port = vcpu->pio.port = port; - vcpu->pio.in = in; - vcpu->pio.string = 0; - vcpu->pio.down = 0; - vcpu->pio.guest_page_offset = 0; - vcpu->pio.rep = 0; - - kvm_x86_ops->cache_regs(vcpu); - memcpy(vcpu->pio_data, &vcpu->regs[VCPU_REGS_RAX], 4); - kvm_x86_ops->decache_regs(vcpu); - - kvm_x86_ops->skip_emulated_instruction(vcpu); - - pio_dev = vcpu_find_pio_dev(vcpu, port); - if (pio_dev) { - kernel_pio(pio_dev, vcpu, vcpu->pio_data); - complete_pio(vcpu); - return 1; - } - return 0; -} -EXPORT_SYMBOL_GPL(kvm_emulate_pio); - -int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, - int size, unsigned long count, int down, - gva_t address, int rep, unsigned port) -{ - unsigned now, in_page; - int i, ret = 0; - int nr_pages = 1; - struct page *page; - struct kvm_io_device *pio_dev; - - vcpu->run->exit_reason = KVM_EXIT_IO; - vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; - vcpu->run->io.size = vcpu->pio.size = size; - vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; - vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = count; - vcpu->run->io.port = vcpu->pio.port = port; - vcpu->pio.in = in; - vcpu->pio.string = 1; - vcpu->pio.down = down; - vcpu->pio.guest_page_offset = offset_in_page(address); - vcpu->pio.rep = rep; - - if (!count) { - kvm_x86_ops->skip_emulated_instruction(vcpu); - return 1; - } - - if (!down) - in_page = PAGE_SIZE - offset_in_page(address); - else - in_page = offset_in_page(address) + size; - now = min(count, (unsigned long)in_page / size); - if (!now) { - /* - * String I/O straddles page boundary. Pin two guest pages - * so that we satisfy atomicity constraints. Do just one - * transaction to avoid complexity. - */ - nr_pages = 2; - now = 1; - } - if (down) { - /* - * String I/O in reverse. Yuck. Kill the guest, fix later. - */ - pr_unimpl(vcpu, "guest string pio down\n"); - inject_gp(vcpu); - return 1; - } - vcpu->run->io.count = now; - vcpu->pio.cur_count = now; - - if (vcpu->pio.cur_count == vcpu->pio.count) - kvm_x86_ops->skip_emulated_instruction(vcpu); - - for (i = 0; i < nr_pages; ++i) { - mutex_lock(&vcpu->kvm->lock); - page = gva_to_page(vcpu, address + i * PAGE_SIZE); - vcpu->pio.guest_pages[i] = page; - mutex_unlock(&vcpu->kvm->lock); - if (!page) { - inject_gp(vcpu); - free_pio_guest_pages(vcpu); - return 1; - } - } - - pio_dev = vcpu_find_pio_dev(vcpu, port); - if (!vcpu->pio.in) { - /* string PIO write */ - ret = pio_copy_data(vcpu); - if (ret >= 0 && pio_dev) { - pio_string_write(pio_dev, vcpu); - complete_pio(vcpu); - if (vcpu->pio.count == 0) - ret = 1; - } - } else if (pio_dev) - pr_unimpl(vcpu, "no string pio read support yet, " - "port %x size %d count %ld\n", - port, size, count); - - return ret; -} -EXPORT_SYMBOL_GPL(kvm_emulate_pio_string); - /* * Check if userspace requested an interrupt window, and that the * interrupt window is open. diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index fe3733d8ece5..f75e7d7d9ead 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1341,6 +1341,249 @@ int emulate_instruction(struct kvm_vcpu *vcpu, } EXPORT_SYMBOL_GPL(emulate_instruction); +static void free_pio_guest_pages(struct kvm_vcpu *vcpu) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(vcpu->pio.guest_pages); ++i) + if (vcpu->pio.guest_pages[i]) { + kvm_release_page(vcpu->pio.guest_pages[i]); + vcpu->pio.guest_pages[i] = NULL; + } +} + +static int pio_copy_data(struct kvm_vcpu *vcpu) +{ + void *p = vcpu->pio_data; + void *q; + unsigned bytes; + int nr_pages = vcpu->pio.guest_pages[1] ? 2 : 1; + + q = vmap(vcpu->pio.guest_pages, nr_pages, VM_READ|VM_WRITE, + PAGE_KERNEL); + if (!q) { + free_pio_guest_pages(vcpu); + return -ENOMEM; + } + q += vcpu->pio.guest_page_offset; + bytes = vcpu->pio.size * vcpu->pio.cur_count; + if (vcpu->pio.in) + memcpy(q, p, bytes); + else + memcpy(p, q, bytes); + q -= vcpu->pio.guest_page_offset; + vunmap(q); + free_pio_guest_pages(vcpu); + return 0; +} + +int complete_pio(struct kvm_vcpu *vcpu) +{ + struct kvm_pio_request *io = &vcpu->pio; + long delta; + int r; + + kvm_x86_ops->cache_regs(vcpu); + + if (!io->string) { + if (io->in) + memcpy(&vcpu->regs[VCPU_REGS_RAX], vcpu->pio_data, + io->size); + } else { + if (io->in) { + r = pio_copy_data(vcpu); + if (r) { + kvm_x86_ops->cache_regs(vcpu); + return r; + } + } + + delta = 1; + if (io->rep) { + delta *= io->cur_count; + /* + * The size of the register should really depend on + * current address size. + */ + vcpu->regs[VCPU_REGS_RCX] -= delta; + } + if (io->down) + delta = -delta; + delta *= io->size; + if (io->in) + vcpu->regs[VCPU_REGS_RDI] += delta; + else + vcpu->regs[VCPU_REGS_RSI] += delta; + } + + kvm_x86_ops->decache_regs(vcpu); + + io->count -= io->cur_count; + io->cur_count = 0; + + return 0; +} + +static void kernel_pio(struct kvm_io_device *pio_dev, + struct kvm_vcpu *vcpu, + void *pd) +{ + /* TODO: String I/O for in kernel device */ + + mutex_lock(&vcpu->kvm->lock); + if (vcpu->pio.in) + kvm_iodevice_read(pio_dev, vcpu->pio.port, + vcpu->pio.size, + pd); + else + kvm_iodevice_write(pio_dev, vcpu->pio.port, + vcpu->pio.size, + pd); + mutex_unlock(&vcpu->kvm->lock); +} + +static void pio_string_write(struct kvm_io_device *pio_dev, + struct kvm_vcpu *vcpu) +{ + struct kvm_pio_request *io = &vcpu->pio; + void *pd = vcpu->pio_data; + int i; + + mutex_lock(&vcpu->kvm->lock); + for (i = 0; i < io->cur_count; i++) { + kvm_iodevice_write(pio_dev, io->port, + io->size, + pd); + pd += io->size; + } + mutex_unlock(&vcpu->kvm->lock); +} + +static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu, + gpa_t addr) +{ + return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr); +} + +int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, + int size, unsigned port) +{ + struct kvm_io_device *pio_dev; + + vcpu->run->exit_reason = KVM_EXIT_IO; + vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; + vcpu->run->io.size = vcpu->pio.size = size; + vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; + vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = 1; + vcpu->run->io.port = vcpu->pio.port = port; + vcpu->pio.in = in; + vcpu->pio.string = 0; + vcpu->pio.down = 0; + vcpu->pio.guest_page_offset = 0; + vcpu->pio.rep = 0; + + kvm_x86_ops->cache_regs(vcpu); + memcpy(vcpu->pio_data, &vcpu->regs[VCPU_REGS_RAX], 4); + kvm_x86_ops->decache_regs(vcpu); + + kvm_x86_ops->skip_emulated_instruction(vcpu); + + pio_dev = vcpu_find_pio_dev(vcpu, port); + if (pio_dev) { + kernel_pio(pio_dev, vcpu, vcpu->pio_data); + complete_pio(vcpu); + return 1; + } + return 0; +} +EXPORT_SYMBOL_GPL(kvm_emulate_pio); + +int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, + int size, unsigned long count, int down, + gva_t address, int rep, unsigned port) +{ + unsigned now, in_page; + int i, ret = 0; + int nr_pages = 1; + struct page *page; + struct kvm_io_device *pio_dev; + + vcpu->run->exit_reason = KVM_EXIT_IO; + vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; + vcpu->run->io.size = vcpu->pio.size = size; + vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; + vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = count; + vcpu->run->io.port = vcpu->pio.port = port; + vcpu->pio.in = in; + vcpu->pio.string = 1; + vcpu->pio.down = down; + vcpu->pio.guest_page_offset = offset_in_page(address); + vcpu->pio.rep = rep; + + if (!count) { + kvm_x86_ops->skip_emulated_instruction(vcpu); + return 1; + } + + if (!down) + in_page = PAGE_SIZE - offset_in_page(address); + else + in_page = offset_in_page(address) + size; + now = min(count, (unsigned long)in_page / size); + if (!now) { + /* + * String I/O straddles page boundary. Pin two guest pages + * so that we satisfy atomicity constraints. Do just one + * transaction to avoid complexity. + */ + nr_pages = 2; + now = 1; + } + if (down) { + /* + * String I/O in reverse. Yuck. Kill the guest, fix later. + */ + pr_unimpl(vcpu, "guest string pio down\n"); + inject_gp(vcpu); + return 1; + } + vcpu->run->io.count = now; + vcpu->pio.cur_count = now; + + if (vcpu->pio.cur_count == vcpu->pio.count) + kvm_x86_ops->skip_emulated_instruction(vcpu); + + for (i = 0; i < nr_pages; ++i) { + mutex_lock(&vcpu->kvm->lock); + page = gva_to_page(vcpu, address + i * PAGE_SIZE); + vcpu->pio.guest_pages[i] = page; + mutex_unlock(&vcpu->kvm->lock); + if (!page) { + inject_gp(vcpu); + free_pio_guest_pages(vcpu); + return 1; + } + } + + pio_dev = vcpu_find_pio_dev(vcpu, port); + if (!vcpu->pio.in) { + /* string PIO write */ + ret = pio_copy_data(vcpu); + if (ret >= 0 && pio_dev) { + pio_string_write(pio_dev, vcpu); + complete_pio(vcpu); + if (vcpu->pio.count == 0) + ret = 1; + } + } else if (pio_dev) + pr_unimpl(vcpu, "no string pio read support yet, " + "port %x size %d count %ld\n", + port, size, count); + + return ret; +} +EXPORT_SYMBOL_GPL(kvm_emulate_pio_string); + __init void kvm_arch_init(void) { kvm_init_msr_list(); diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 5592456c36ad..663b822b4ddb 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -126,4 +126,5 @@ static inline int is_paging(struct kvm_vcpu *vcpu) } int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); +int complete_pio(struct kvm_vcpu *vcpu); #endif From 3c118e24af821d68dca0ba81e9499820c840c133 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 31 Oct 2007 10:27:04 +0200 Subject: [PATCH 077/249] KVM: x86 emulator: Extract the common code of SrcReg and DstReg Share the common parts of SrcReg and DstReg decoding. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 80 +++++++++++++++------------------------ 1 file changed, 31 insertions(+), 49 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 087a8208f873..58ceb6616364 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -520,6 +520,34 @@ static int test_cc(unsigned int condition, unsigned int flags) return (!!rc ^ (condition & 1)); } +static void decode_register_operand(struct operand *op, + struct decode_cache *c, + int highbyte_regs, + int inhibit_bytereg) +{ + op->type = OP_REG; + if ((c->d & ByteOp) && !inhibit_bytereg) { + op->ptr = decode_register(c->modrm_reg, c->regs, highbyte_regs); + op->val = *(u8 *)op->ptr; + op->bytes = 1; + } else { + op->ptr = decode_register(c->modrm_reg, c->regs, 0); + op->bytes = c->op_bytes; + switch (op->bytes) { + case 2: + op->val = *(u16 *)op->ptr; + break; + case 4: + op->val = *(u32 *)op->ptr; + break; + case 8: + op->val = *(u64 *) op->ptr; + break; + } + } + op->orig_val = op->val; +} + int x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { @@ -809,31 +837,7 @@ modrm_done: case SrcNone: break; case SrcReg: - c->src.type = OP_REG; - if (c->d & ByteOp) { - c->src.ptr = - decode_register(c->modrm_reg, c->regs, - (rex_prefix == 0)); - c->src.val = c->src.orig_val = *(u8 *)c->src.ptr; - c->src.bytes = 1; - } else { - c->src.ptr = - decode_register(c->modrm_reg, c->regs, 0); - switch ((c->src.bytes = c->op_bytes)) { - case 2: - c->src.val = c->src.orig_val = - *(u16 *) c->src.ptr; - break; - case 4: - c->src.val = c->src.orig_val = - *(u32 *) c->src.ptr; - break; - case 8: - c->src.val = c->src.orig_val = - *(u64 *) c->src.ptr; - break; - } - } + decode_register_operand(&c->src, c, rex_prefix == 0, 0); break; case SrcMem16: c->src.bytes = 2; @@ -891,30 +895,8 @@ modrm_done: /* Special instructions do their own operand decoding. */ return 0; case DstReg: - c->dst.type = OP_REG; - if ((c->d & ByteOp) - && !(c->twobyte && - (c->b == 0xb6 || c->b == 0xb7))) { - c->dst.ptr = - decode_register(c->modrm_reg, c->regs, - (rex_prefix == 0)); - c->dst.val = *(u8 *) c->dst.ptr; - c->dst.bytes = 1; - } else { - c->dst.ptr = - decode_register(c->modrm_reg, c->regs, 0); - switch ((c->dst.bytes = c->op_bytes)) { - case 2: - c->dst.val = *(u16 *)c->dst.ptr; - break; - case 4: - c->dst.val = *(u32 *)c->dst.ptr; - break; - case 8: - c->dst.val = *(u64 *)c->dst.ptr; - break; - } - } + decode_register_operand(&c->dst, c, rex_prefix == 0, + c->twobyte && (c->b == 0xb6 || c->b == 0xb7)); break; case DstMem: if ((c->d & ModRM) && c->modrm_mod == 3) { From 33615aa956521923eab0552994b5961cd3034042 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 31 Oct 2007 11:15:56 +0200 Subject: [PATCH 078/249] KVM: x86 emulator: centralize decoding of one-byte register access insns Instructions like 'inc reg' that have the register operand encoded in the opcode are currently specially decoded. Extend decode_register_operand() to handle that case, indicated by having DstReg or SrcReg without ModRM. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 103 +++++++++++++++++--------------------- drivers/kvm/x86_emulate.h | 1 + 2 files changed, 47 insertions(+), 57 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 58ceb6616364..884e4a2e47e6 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -99,17 +99,13 @@ static u16 opcode_table[256] = { ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, 0, 0, 0, 0, /* 0x40 - 0x47 */ - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, /* 0x48 - 0x4F */ - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, /* 0x50 - 0x57 */ - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + SrcReg, SrcReg, SrcReg, SrcReg, SrcReg, SrcReg, SrcReg, SrcReg, /* 0x58 - 0x5F */ - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, /* 0x60 - 0x67 */ 0, 0, 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ , 0, 0, 0, 0, @@ -525,13 +521,17 @@ static void decode_register_operand(struct operand *op, int highbyte_regs, int inhibit_bytereg) { + unsigned reg = c->modrm_reg; + + if (!(c->d & ModRM)) + reg = (c->b & 7) | ((c->rex_prefix & 1) << 3); op->type = OP_REG; if ((c->d & ByteOp) && !inhibit_bytereg) { - op->ptr = decode_register(c->modrm_reg, c->regs, highbyte_regs); + op->ptr = decode_register(reg, c->regs, highbyte_regs); op->val = *(u8 *)op->ptr; op->bytes = 1; } else { - op->ptr = decode_register(c->modrm_reg, c->regs, 0); + op->ptr = decode_register(reg, c->regs, 0); op->bytes = c->op_bytes; switch (op->bytes) { case 2: @@ -552,7 +552,7 @@ int x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { struct decode_cache *c = &ctxt->decode; - u8 sib, rex_prefix = 0; + u8 sib; int rc = 0; int mode = ctxt->mode; int index_reg = 0, base_reg = 0, scale, rip_relative = 0; @@ -616,7 +616,7 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) case 0x40 ... 0x4f: /* REX */ if (mode != X86EMUL_MODE_PROT64) goto done_prefixes; - rex_prefix = c->b; + c->rex_prefix = c->b; continue; case 0xf0: /* LOCK */ c->lock_prefix = 1; @@ -631,18 +631,18 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) /* Any legacy prefix after a REX prefix nullifies its effect. */ - rex_prefix = 0; + c->rex_prefix = 0; } done_prefixes: /* REX prefix. */ - if (rex_prefix) { - if (rex_prefix & 8) + if (c->rex_prefix) { + if (c->rex_prefix & 8) c->op_bytes = 8; /* REX.W */ - c->modrm_reg = (rex_prefix & 4) << 1; /* REX.R */ - index_reg = (rex_prefix & 2) << 2; /* REX.X */ - c->modrm_rm = base_reg = (rex_prefix & 1) << 3; /* REG.B */ + c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */ + index_reg = (c->rex_prefix & 2) << 2; /* REX.X */ + c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */ } /* Opcode byte(s). */ @@ -837,7 +837,7 @@ modrm_done: case SrcNone: break; case SrcReg: - decode_register_operand(&c->src, c, rex_prefix == 0, 0); + decode_register_operand(&c->src, c, c->rex_prefix == 0, 0); break; case SrcMem16: c->src.bytes = 2; @@ -895,7 +895,7 @@ modrm_done: /* Special instructions do their own operand decoding. */ return 0; case DstReg: - decode_register_operand(&c->dst, c, rex_prefix == 0, + decode_register_operand(&c->dst, c, c->rex_prefix == 0, c->twobyte && (c->b == 0xb6 || c->b == 0xb7)); break; case DstMem: @@ -1258,6 +1258,32 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) cmp: /* cmp */ emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags); break; + case 0x40 ... 0x47: /* inc r16/r32 */ + emulate_1op("inc", c->dst, ctxt->eflags); + break; + case 0x48 ... 0x4f: /* dec r16/r32 */ + emulate_1op("dec", c->dst, ctxt->eflags); + break; + case 0x50 ... 0x57: /* push reg */ + c->dst.type = OP_MEM; + c->dst.bytes = c->op_bytes; + c->dst.val = c->src.val; + register_address_increment(c->regs[VCPU_REGS_RSP], + -c->op_bytes); + c->dst.ptr = (void *) register_address( + ctxt->ss_base, c->regs[VCPU_REGS_RSP]); + break; + case 0x58 ... 0x5f: /* pop reg */ + pop_instruction: + if ((rc = ops->read_std(register_address(ctxt->ss_base, + c->regs[VCPU_REGS_RSP]), c->dst.ptr, + c->op_bytes, ctxt->vcpu)) != 0) + goto done; + + register_address_increment(c->regs[VCPU_REGS_RSP], + c->op_bytes); + c->dst.type = OP_NONE; /* Disable writeback. */ + break; case 0x63: /* movsxd */ if (ctxt->mode != X86EMUL_MODE_PROT64) goto cannot_emulate; @@ -1373,43 +1399,6 @@ special_insn: if (c->twobyte) goto twobyte_special_insn; switch (c->b) { - case 0x40 ... 0x47: /* inc r16/r32 */ - c->dst.bytes = c->op_bytes; - c->dst.ptr = (unsigned long *)&c->regs[c->b & 0x7]; - c->dst.val = *c->dst.ptr; - emulate_1op("inc", c->dst, ctxt->eflags); - break; - case 0x48 ... 0x4f: /* dec r16/r32 */ - c->dst.bytes = c->op_bytes; - c->dst.ptr = (unsigned long *)&c->regs[c->b & 0x7]; - c->dst.val = *c->dst.ptr; - emulate_1op("dec", c->dst, ctxt->eflags); - break; - case 0x50 ... 0x57: /* push reg */ - if (c->op_bytes == 2) - c->src.val = (u16) c->regs[c->b & 0x7]; - else - c->src.val = (u32) c->regs[c->b & 0x7]; - c->dst.type = OP_MEM; - c->dst.bytes = c->op_bytes; - c->dst.val = c->src.val; - register_address_increment(c->regs[VCPU_REGS_RSP], - -c->op_bytes); - c->dst.ptr = (void *) register_address( - ctxt->ss_base, c->regs[VCPU_REGS_RSP]); - break; - case 0x58 ... 0x5f: /* pop reg */ - c->dst.ptr = (unsigned long *)&c->regs[c->b & 0x7]; - pop_instruction: - if ((rc = ops->read_std(register_address(ctxt->ss_base, - c->regs[VCPU_REGS_RSP]), c->dst.ptr, - c->op_bytes, ctxt->vcpu)) != 0) - goto done; - - register_address_increment(c->regs[VCPU_REGS_RSP], - c->op_bytes); - c->dst.type = OP_NONE; /* Disable writeback. */ - break; case 0x6a: /* push imm8 */ c->src.val = 0L; c->src.val = insn_fetch(s8, 1, c->eip); diff --git a/drivers/kvm/x86_emulate.h b/drivers/kvm/x86_emulate.h index f03b1287700a..e34868b42c33 100644 --- a/drivers/kvm/x86_emulate.h +++ b/drivers/kvm/x86_emulate.h @@ -126,6 +126,7 @@ struct decode_cache { u8 rep_prefix; u8 op_bytes; u8 ad_bytes; + u8 rex_prefix; struct operand src; struct operand dst; unsigned long *override_base; From 9f1ef3f8f5f8ade2561d969181b152c225b82a3e Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 31 Oct 2007 11:21:06 +0200 Subject: [PATCH 079/249] KVM: Simplify decode_register_operand() calling convention Now that rex_prefix is part of the decode cache, there is no need to pass it along. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 884e4a2e47e6..b352a6c4d4a2 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -518,10 +518,10 @@ static int test_cc(unsigned int condition, unsigned int flags) static void decode_register_operand(struct operand *op, struct decode_cache *c, - int highbyte_regs, int inhibit_bytereg) { unsigned reg = c->modrm_reg; + int highbyte_regs = c->rex_prefix == 0; if (!(c->d & ModRM)) reg = (c->b & 7) | ((c->rex_prefix & 1) << 3); @@ -837,7 +837,7 @@ modrm_done: case SrcNone: break; case SrcReg: - decode_register_operand(&c->src, c, c->rex_prefix == 0, 0); + decode_register_operand(&c->src, c, 0); break; case SrcMem16: c->src.bytes = 2; @@ -895,7 +895,7 @@ modrm_done: /* Special instructions do their own operand decoding. */ return 0; case DstReg: - decode_register_operand(&c->dst, c, c->rex_prefix == 0, + decode_register_operand(&c->dst, c, c->twobyte && (c->b == 0xb6 || c->b == 0xb7)); break; case DstMem: From 3b6fff198c2ae3ca87ee8c9103d52ac9e8f40dd5 Mon Sep 17 00:00:00 2001 From: Uri Lublin Date: Tue, 30 Oct 2007 10:42:09 +0200 Subject: [PATCH 080/249] KVM: Make mark_page_dirty() work for aliased pages too. Recommended by Izik Eidus. Signed-off-by: Uri Lublin Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 2025cdfb4593..cac66d505f57 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -796,11 +796,11 @@ int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) } EXPORT_SYMBOL_GPL(kvm_clear_guest); -/* WARNING: Does not work on aliased pages. */ void mark_page_dirty(struct kvm *kvm, gfn_t gfn) { struct kvm_memory_slot *memslot; + gfn = unalias_gfn(kvm, gfn); memslot = __gfn_to_memslot(kvm, gfn); if (memslot && memslot->dirty_bitmap) { unsigned long rel_gfn = gfn - memslot->base_gfn; From 1c73ef66502311d9aff7fed7d7f970288329c6cb Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 1 Nov 2007 06:31:28 +0200 Subject: [PATCH 081/249] KVM: x86 emulator: Hoist modrm and abs decoding into separate functions Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 337 ++++++++++++++++++++------------------ 1 file changed, 177 insertions(+), 160 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index b352a6c4d4a2..64888a6edc27 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -548,14 +548,182 @@ static void decode_register_operand(struct operand *op, op->orig_val = op->val; } +static int decode_modrm(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops) +{ + struct decode_cache *c = &ctxt->decode; + u8 sib; + int index_reg = 0, base_reg = 0, scale, rip_relative = 0; + int rc = 0; + + if (c->rex_prefix) { + c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */ + index_reg = (c->rex_prefix & 2) << 2; /* REX.X */ + c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */ + } + + c->modrm = insn_fetch(u8, 1, c->eip); + c->modrm_mod |= (c->modrm & 0xc0) >> 6; + c->modrm_reg |= (c->modrm & 0x38) >> 3; + c->modrm_rm |= (c->modrm & 0x07); + c->modrm_ea = 0; + c->use_modrm_ea = 1; + + if (c->modrm_mod == 3) { + c->modrm_val = *(unsigned long *) + decode_register(c->modrm_rm, c->regs, c->d & ByteOp); + return rc; + } + + if (c->ad_bytes == 2) { + unsigned bx = c->regs[VCPU_REGS_RBX]; + unsigned bp = c->regs[VCPU_REGS_RBP]; + unsigned si = c->regs[VCPU_REGS_RSI]; + unsigned di = c->regs[VCPU_REGS_RDI]; + + /* 16-bit ModR/M decode. */ + switch (c->modrm_mod) { + case 0: + if (c->modrm_rm == 6) + c->modrm_ea += insn_fetch(u16, 2, c->eip); + break; + case 1: + c->modrm_ea += insn_fetch(s8, 1, c->eip); + break; + case 2: + c->modrm_ea += insn_fetch(u16, 2, c->eip); + break; + } + switch (c->modrm_rm) { + case 0: + c->modrm_ea += bx + si; + break; + case 1: + c->modrm_ea += bx + di; + break; + case 2: + c->modrm_ea += bp + si; + break; + case 3: + c->modrm_ea += bp + di; + break; + case 4: + c->modrm_ea += si; + break; + case 5: + c->modrm_ea += di; + break; + case 6: + if (c->modrm_mod != 0) + c->modrm_ea += bp; + break; + case 7: + c->modrm_ea += bx; + break; + } + if (c->modrm_rm == 2 || c->modrm_rm == 3 || + (c->modrm_rm == 6 && c->modrm_mod != 0)) + if (!c->override_base) + c->override_base = &ctxt->ss_base; + c->modrm_ea = (u16)c->modrm_ea; + } else { + /* 32/64-bit ModR/M decode. */ + switch (c->modrm_rm) { + case 4: + case 12: + sib = insn_fetch(u8, 1, c->eip); + index_reg |= (sib >> 3) & 7; + base_reg |= sib & 7; + scale = sib >> 6; + + switch (base_reg) { + case 5: + if (c->modrm_mod != 0) + c->modrm_ea += c->regs[base_reg]; + else + c->modrm_ea += + insn_fetch(s32, 4, c->eip); + break; + default: + c->modrm_ea += c->regs[base_reg]; + } + switch (index_reg) { + case 4: + break; + default: + c->modrm_ea += c->regs[index_reg] << scale; + } + break; + case 5: + if (c->modrm_mod != 0) + c->modrm_ea += c->regs[c->modrm_rm]; + else if (ctxt->mode == X86EMUL_MODE_PROT64) + rip_relative = 1; + break; + default: + c->modrm_ea += c->regs[c->modrm_rm]; + break; + } + switch (c->modrm_mod) { + case 0: + if (c->modrm_rm == 5) + c->modrm_ea += insn_fetch(s32, 4, c->eip); + break; + case 1: + c->modrm_ea += insn_fetch(s8, 1, c->eip); + break; + case 2: + c->modrm_ea += insn_fetch(s32, 4, c->eip); + break; + } + } + if (rip_relative) { + c->modrm_ea += c->eip; + switch (c->d & SrcMask) { + case SrcImmByte: + c->modrm_ea += 1; + break; + case SrcImm: + if (c->d & ByteOp) + c->modrm_ea += 1; + else + if (c->op_bytes == 8) + c->modrm_ea += 4; + else + c->modrm_ea += c->op_bytes; + } + } +done: + return rc; +} + +static int decode_abs(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops) +{ + struct decode_cache *c = &ctxt->decode; + int rc = 0; + + switch (c->ad_bytes) { + case 2: + c->modrm_ea = insn_fetch(u16, 2, c->eip); + break; + case 4: + c->modrm_ea = insn_fetch(u32, 4, c->eip); + break; + case 8: + c->modrm_ea = insn_fetch(u64, 8, c->eip); + break; + } +done: + return rc; +} + int x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { struct decode_cache *c = &ctxt->decode; - u8 sib; int rc = 0; int mode = ctxt->mode; - int index_reg = 0, base_reg = 0, scale, rip_relative = 0; /* Shadow copy of register state. Committed on successful emulation. */ @@ -637,13 +805,9 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) done_prefixes: /* REX prefix. */ - if (c->rex_prefix) { + if (c->rex_prefix) if (c->rex_prefix & 8) c->op_bytes = 8; /* REX.W */ - c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */ - index_reg = (c->rex_prefix & 2) << 2; /* REX.X */ - c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */ - } /* Opcode byte(s). */ c->d = opcode_table[c->b]; @@ -663,159 +827,12 @@ done_prefixes: } /* ModRM and SIB bytes. */ - if (c->d & ModRM) { - c->modrm = insn_fetch(u8, 1, c->eip); - c->modrm_mod |= (c->modrm & 0xc0) >> 6; - c->modrm_reg |= (c->modrm & 0x38) >> 3; - c->modrm_rm |= (c->modrm & 0x07); - c->modrm_ea = 0; - c->use_modrm_ea = 1; - - if (c->modrm_mod == 3) { - c->modrm_val = *(unsigned long *) - decode_register(c->modrm_rm, c->regs, c->d & ByteOp); - goto modrm_done; - } - - if (c->ad_bytes == 2) { - unsigned bx = c->regs[VCPU_REGS_RBX]; - unsigned bp = c->regs[VCPU_REGS_RBP]; - unsigned si = c->regs[VCPU_REGS_RSI]; - unsigned di = c->regs[VCPU_REGS_RDI]; - - /* 16-bit ModR/M decode. */ - switch (c->modrm_mod) { - case 0: - if (c->modrm_rm == 6) - c->modrm_ea += - insn_fetch(u16, 2, c->eip); - break; - case 1: - c->modrm_ea += insn_fetch(s8, 1, c->eip); - break; - case 2: - c->modrm_ea += insn_fetch(u16, 2, c->eip); - break; - } - switch (c->modrm_rm) { - case 0: - c->modrm_ea += bx + si; - break; - case 1: - c->modrm_ea += bx + di; - break; - case 2: - c->modrm_ea += bp + si; - break; - case 3: - c->modrm_ea += bp + di; - break; - case 4: - c->modrm_ea += si; - break; - case 5: - c->modrm_ea += di; - break; - case 6: - if (c->modrm_mod != 0) - c->modrm_ea += bp; - break; - case 7: - c->modrm_ea += bx; - break; - } - if (c->modrm_rm == 2 || c->modrm_rm == 3 || - (c->modrm_rm == 6 && c->modrm_mod != 0)) - if (!c->override_base) - c->override_base = &ctxt->ss_base; - c->modrm_ea = (u16)c->modrm_ea; - } else { - /* 32/64-bit ModR/M decode. */ - switch (c->modrm_rm) { - case 4: - case 12: - sib = insn_fetch(u8, 1, c->eip); - index_reg |= (sib >> 3) & 7; - base_reg |= sib & 7; - scale = sib >> 6; - - switch (base_reg) { - case 5: - if (c->modrm_mod != 0) - c->modrm_ea += - c->regs[base_reg]; - else - c->modrm_ea += - insn_fetch(s32, 4, c->eip); - break; - default: - c->modrm_ea += c->regs[base_reg]; - } - switch (index_reg) { - case 4: - break; - default: - c->modrm_ea += - c->regs[index_reg] << scale; - - } - break; - case 5: - if (c->modrm_mod != 0) - c->modrm_ea += c->regs[c->modrm_rm]; - else if (mode == X86EMUL_MODE_PROT64) - rip_relative = 1; - break; - default: - c->modrm_ea += c->regs[c->modrm_rm]; - break; - } - switch (c->modrm_mod) { - case 0: - if (c->modrm_rm == 5) - c->modrm_ea += - insn_fetch(s32, 4, c->eip); - break; - case 1: - c->modrm_ea += insn_fetch(s8, 1, c->eip); - break; - case 2: - c->modrm_ea += insn_fetch(s32, 4, c->eip); - break; - } - } - if (rip_relative) { - c->modrm_ea += c->eip; - switch (c->d & SrcMask) { - case SrcImmByte: - c->modrm_ea += 1; - break; - case SrcImm: - if (c->d & ByteOp) - c->modrm_ea += 1; - else - if (c->op_bytes == 8) - c->modrm_ea += 4; - else - c->modrm_ea += c->op_bytes; - } - } -modrm_done: - ; - } else if (c->d & MemAbs) { - switch (c->ad_bytes) { - case 2: - c->modrm_ea = insn_fetch(u16, 2, c->eip); - break; - case 4: - c->modrm_ea = insn_fetch(u32, 4, c->eip); - break; - case 8: - c->modrm_ea = insn_fetch(u64, 8, c->eip); - break; - } - - } + if (c->d & ModRM) + rc = decode_modrm(ctxt, ops); + else if (c->d & MemAbs) + rc = decode_abs(ctxt, ops); + if (rc) + goto done; if (!c->override_base) c->override_base = &ctxt->ds_base; From 417bc3041f5e66df1ce7f03d8fc481c3b12f250a Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Wed, 31 Oct 2007 17:24:23 -0500 Subject: [PATCH 082/249] KVM: Portability: Make exported debugfs data architecture-specific Signed-off-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 7 +++++++ drivers/kvm/kvm_main.c | 25 ------------------------- drivers/kvm/x86.c | 22 ++++++++++++++++++++++ 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 22317d6f66ae..d030a82966f5 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -761,4 +761,11 @@ static inline u32 get_rdx_init_val(void) #define TSS_REDIRECTION_SIZE (256 / 8) #define RMODE_TSS_SIZE (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) +struct kvm_stats_debugfs_item { + const char *name; + int offset; + struct dentry *dentry; +}; +extern struct kvm_stats_debugfs_item debugfs_entries[]; + #endif diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index cac66d505f57..58a5f399ad85 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -63,31 +63,6 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_cache); static __read_mostly struct preempt_ops kvm_preempt_ops; -#define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x) - -static struct kvm_stats_debugfs_item { - const char *name; - int offset; - struct dentry *dentry; -} debugfs_entries[] = { - { "pf_fixed", STAT_OFFSET(pf_fixed) }, - { "pf_guest", STAT_OFFSET(pf_guest) }, - { "tlb_flush", STAT_OFFSET(tlb_flush) }, - { "invlpg", STAT_OFFSET(invlpg) }, - { "exits", STAT_OFFSET(exits) }, - { "io_exits", STAT_OFFSET(io_exits) }, - { "mmio_exits", STAT_OFFSET(mmio_exits) }, - { "signal_exits", STAT_OFFSET(signal_exits) }, - { "irq_window", STAT_OFFSET(irq_window_exits) }, - { "halt_exits", STAT_OFFSET(halt_exits) }, - { "halt_wakeup", STAT_OFFSET(halt_wakeup) }, - { "request_irq", STAT_OFFSET(request_irq_exits) }, - { "irq_exits", STAT_OFFSET(irq_exits) }, - { "light_exits", STAT_OFFSET(light_exits) }, - { "efer_reload", STAT_OFFSET(efer_reload) }, - { NULL } -}; - static struct dentry *debugfs_dir; static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index f75e7d7d9ead..c1f10e58f4d2 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -40,6 +40,28 @@ #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) #define EFER_RESERVED_BITS 0xfffffffffffff2fe +#define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x) + +struct kvm_stats_debugfs_item debugfs_entries[] = { + { "pf_fixed", STAT_OFFSET(pf_fixed) }, + { "pf_guest", STAT_OFFSET(pf_guest) }, + { "tlb_flush", STAT_OFFSET(tlb_flush) }, + { "invlpg", STAT_OFFSET(invlpg) }, + { "exits", STAT_OFFSET(exits) }, + { "io_exits", STAT_OFFSET(io_exits) }, + { "mmio_exits", STAT_OFFSET(mmio_exits) }, + { "signal_exits", STAT_OFFSET(signal_exits) }, + { "irq_window", STAT_OFFSET(irq_window_exits) }, + { "halt_exits", STAT_OFFSET(halt_exits) }, + { "halt_wakeup", STAT_OFFSET(halt_wakeup) }, + { "request_irq", STAT_OFFSET(request_irq_exits) }, + { "irq_exits", STAT_OFFSET(irq_exits) }, + { "light_exits", STAT_OFFSET(light_exits) }, + { "efer_reload", STAT_OFFSET(efer_reload) }, + { NULL } +}; + + unsigned long segment_base(u16 selector) { struct descriptor_table gdt; From 8776e5194f7bb847906e3561c4dba12ed66ebfb6 Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Wed, 31 Oct 2007 17:24:24 -0500 Subject: [PATCH 083/249] KVM: Portability: Move x86 instruction emulation code to x86.c Signed-off-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/kvm_main.c | 177 +---------------------------------------- drivers/kvm/x86.c | 175 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 177 insertions(+), 176 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index d030a82966f5..ef2a6a8328ea 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -591,6 +591,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data); void fx_init(struct kvm_vcpu *vcpu); +void kvm_vcpu_block(struct kvm_vcpu *vcpu); void kvm_resched(struct kvm_vcpu *vcpu); void kvm_load_guest_fpu(struct kvm_vcpu *vcpu); void kvm_put_guest_fpu(struct kvm_vcpu *vcpu); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 58a5f399ad85..57573ebf02ba 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -789,7 +789,7 @@ void mark_page_dirty(struct kvm *kvm, gfn_t gfn) /* * The vCPU has executed a HLT instruction with in-kernel mode enabled. */ -static void kvm_vcpu_block(struct kvm_vcpu *vcpu) +void kvm_vcpu_block(struct kvm_vcpu *vcpu) { DECLARE_WAITQUEUE(wait, current); @@ -812,144 +812,6 @@ static void kvm_vcpu_block(struct kvm_vcpu *vcpu) remove_wait_queue(&vcpu->wq, &wait); } -int kvm_emulate_halt(struct kvm_vcpu *vcpu) -{ - ++vcpu->stat.halt_exits; - if (irqchip_in_kernel(vcpu->kvm)) { - vcpu->mp_state = VCPU_MP_STATE_HALTED; - kvm_vcpu_block(vcpu); - if (vcpu->mp_state != VCPU_MP_STATE_RUNNABLE) - return -EINTR; - return 1; - } else { - vcpu->run->exit_reason = KVM_EXIT_HLT; - return 0; - } -} -EXPORT_SYMBOL_GPL(kvm_emulate_halt); - -int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) -{ - unsigned long nr, a0, a1, a2, a3, ret; - - kvm_x86_ops->cache_regs(vcpu); - - nr = vcpu->regs[VCPU_REGS_RAX]; - a0 = vcpu->regs[VCPU_REGS_RBX]; - a1 = vcpu->regs[VCPU_REGS_RCX]; - a2 = vcpu->regs[VCPU_REGS_RDX]; - a3 = vcpu->regs[VCPU_REGS_RSI]; - - if (!is_long_mode(vcpu)) { - nr &= 0xFFFFFFFF; - a0 &= 0xFFFFFFFF; - a1 &= 0xFFFFFFFF; - a2 &= 0xFFFFFFFF; - a3 &= 0xFFFFFFFF; - } - - switch (nr) { - default: - ret = -KVM_ENOSYS; - break; - } - vcpu->regs[VCPU_REGS_RAX] = ret; - kvm_x86_ops->decache_regs(vcpu); - return 0; -} -EXPORT_SYMBOL_GPL(kvm_emulate_hypercall); - -int kvm_fix_hypercall(struct kvm_vcpu *vcpu) -{ - char instruction[3]; - int ret = 0; - - mutex_lock(&vcpu->kvm->lock); - - /* - * Blow out the MMU to ensure that no other VCPU has an active mapping - * to ensure that the updated hypercall appears atomically across all - * VCPUs. - */ - kvm_mmu_zap_all(vcpu->kvm); - - kvm_x86_ops->cache_regs(vcpu); - kvm_x86_ops->patch_hypercall(vcpu, instruction); - if (emulator_write_emulated(vcpu->rip, instruction, 3, vcpu) - != X86EMUL_CONTINUE) - ret = -EFAULT; - - mutex_unlock(&vcpu->kvm->lock); - - return ret; -} - -static u64 mk_cr_64(u64 curr_cr, u32 new_val) -{ - return (curr_cr & ~((1ULL << 32) - 1)) | new_val; -} - -void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base) -{ - struct descriptor_table dt = { limit, base }; - - kvm_x86_ops->set_gdt(vcpu, &dt); -} - -void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base) -{ - struct descriptor_table dt = { limit, base }; - - kvm_x86_ops->set_idt(vcpu, &dt); -} - -void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw, - unsigned long *rflags) -{ - lmsw(vcpu, msw); - *rflags = kvm_x86_ops->get_rflags(vcpu); -} - -unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr) -{ - kvm_x86_ops->decache_cr4_guest_bits(vcpu); - switch (cr) { - case 0: - return vcpu->cr0; - case 2: - return vcpu->cr2; - case 3: - return vcpu->cr3; - case 4: - return vcpu->cr4; - default: - vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr); - return 0; - } -} - -void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, - unsigned long *rflags) -{ - switch (cr) { - case 0: - set_cr0(vcpu, mk_cr_64(vcpu->cr0, val)); - *rflags = kvm_x86_ops->get_rflags(vcpu); - break; - case 2: - vcpu->cr2 = val; - break; - case 3: - set_cr3(vcpu, val); - break; - case 4: - set_cr4(vcpu, mk_cr_64(vcpu->cr4, val)); - break; - default: - vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr); - } -} - void kvm_resched(struct kvm_vcpu *vcpu) { if (!need_resched()) @@ -958,43 +820,6 @@ void kvm_resched(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_resched); -void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) -{ - int i; - u32 function; - struct kvm_cpuid_entry *e, *best; - - kvm_x86_ops->cache_regs(vcpu); - function = vcpu->regs[VCPU_REGS_RAX]; - vcpu->regs[VCPU_REGS_RAX] = 0; - vcpu->regs[VCPU_REGS_RBX] = 0; - vcpu->regs[VCPU_REGS_RCX] = 0; - vcpu->regs[VCPU_REGS_RDX] = 0; - best = NULL; - for (i = 0; i < vcpu->cpuid_nent; ++i) { - e = &vcpu->cpuid_entries[i]; - if (e->function == function) { - best = e; - break; - } - /* - * Both basic or both extended? - */ - if (((e->function ^ function) & 0x80000000) == 0) - if (!best || e->function > best->function) - best = e; - } - if (best) { - vcpu->regs[VCPU_REGS_RAX] = best->eax; - vcpu->regs[VCPU_REGS_RBX] = best->ebx; - vcpu->regs[VCPU_REGS_RCX] = best->ecx; - vcpu->regs[VCPU_REGS_RDX] = best->edx; - } - kvm_x86_ops->decache_regs(vcpu); - kvm_x86_ops->skip_emulated_instruction(vcpu); -} -EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); - /* * Check if userspace requested an interrupt window, and that the * interrupt window is open. diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index c1f10e58f4d2..2cf7ebab50f4 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1610,3 +1610,178 @@ __init void kvm_arch_init(void) { kvm_init_msr_list(); } + +int kvm_emulate_halt(struct kvm_vcpu *vcpu) +{ + ++vcpu->stat.halt_exits; + if (irqchip_in_kernel(vcpu->kvm)) { + vcpu->mp_state = VCPU_MP_STATE_HALTED; + kvm_vcpu_block(vcpu); + if (vcpu->mp_state != VCPU_MP_STATE_RUNNABLE) + return -EINTR; + return 1; + } else { + vcpu->run->exit_reason = KVM_EXIT_HLT; + return 0; + } +} +EXPORT_SYMBOL_GPL(kvm_emulate_halt); + +int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) +{ + unsigned long nr, a0, a1, a2, a3, ret; + + kvm_x86_ops->cache_regs(vcpu); + + nr = vcpu->regs[VCPU_REGS_RAX]; + a0 = vcpu->regs[VCPU_REGS_RBX]; + a1 = vcpu->regs[VCPU_REGS_RCX]; + a2 = vcpu->regs[VCPU_REGS_RDX]; + a3 = vcpu->regs[VCPU_REGS_RSI]; + + if (!is_long_mode(vcpu)) { + nr &= 0xFFFFFFFF; + a0 &= 0xFFFFFFFF; + a1 &= 0xFFFFFFFF; + a2 &= 0xFFFFFFFF; + a3 &= 0xFFFFFFFF; + } + + switch (nr) { + default: + ret = -KVM_ENOSYS; + break; + } + vcpu->regs[VCPU_REGS_RAX] = ret; + kvm_x86_ops->decache_regs(vcpu); + return 0; +} +EXPORT_SYMBOL_GPL(kvm_emulate_hypercall); + +int kvm_fix_hypercall(struct kvm_vcpu *vcpu) +{ + char instruction[3]; + int ret = 0; + + mutex_lock(&vcpu->kvm->lock); + + /* + * Blow out the MMU to ensure that no other VCPU has an active mapping + * to ensure that the updated hypercall appears atomically across all + * VCPUs. + */ + kvm_mmu_zap_all(vcpu->kvm); + + kvm_x86_ops->cache_regs(vcpu); + kvm_x86_ops->patch_hypercall(vcpu, instruction); + if (emulator_write_emulated(vcpu->rip, instruction, 3, vcpu) + != X86EMUL_CONTINUE) + ret = -EFAULT; + + mutex_unlock(&vcpu->kvm->lock); + + return ret; +} + +static u64 mk_cr_64(u64 curr_cr, u32 new_val) +{ + return (curr_cr & ~((1ULL << 32) - 1)) | new_val; +} + +void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base) +{ + struct descriptor_table dt = { limit, base }; + + kvm_x86_ops->set_gdt(vcpu, &dt); +} + +void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base) +{ + struct descriptor_table dt = { limit, base }; + + kvm_x86_ops->set_idt(vcpu, &dt); +} + +void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw, + unsigned long *rflags) +{ + lmsw(vcpu, msw); + *rflags = kvm_x86_ops->get_rflags(vcpu); +} + +unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr) +{ + kvm_x86_ops->decache_cr4_guest_bits(vcpu); + switch (cr) { + case 0: + return vcpu->cr0; + case 2: + return vcpu->cr2; + case 3: + return vcpu->cr3; + case 4: + return vcpu->cr4; + default: + vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr); + return 0; + } +} + +void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, + unsigned long *rflags) +{ + switch (cr) { + case 0: + set_cr0(vcpu, mk_cr_64(vcpu->cr0, val)); + *rflags = kvm_x86_ops->get_rflags(vcpu); + break; + case 2: + vcpu->cr2 = val; + break; + case 3: + set_cr3(vcpu, val); + break; + case 4: + set_cr4(vcpu, mk_cr_64(vcpu->cr4, val)); + break; + default: + vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr); + } +} + +void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) +{ + int i; + u32 function; + struct kvm_cpuid_entry *e, *best; + + kvm_x86_ops->cache_regs(vcpu); + function = vcpu->regs[VCPU_REGS_RAX]; + vcpu->regs[VCPU_REGS_RAX] = 0; + vcpu->regs[VCPU_REGS_RBX] = 0; + vcpu->regs[VCPU_REGS_RCX] = 0; + vcpu->regs[VCPU_REGS_RDX] = 0; + best = NULL; + for (i = 0; i < vcpu->cpuid_nent; ++i) { + e = &vcpu->cpuid_entries[i]; + if (e->function == function) { + best = e; + break; + } + /* + * Both basic or both extended? + */ + if (((e->function ^ function) & 0x80000000) == 0) + if (!best || e->function > best->function) + best = e; + } + if (best) { + vcpu->regs[VCPU_REGS_RAX] = best->eax; + vcpu->regs[VCPU_REGS_RBX] = best->ebx; + vcpu->regs[VCPU_REGS_RCX] = best->ecx; + vcpu->regs[VCPU_REGS_RDX] = best->edx; + } + kvm_x86_ops->decache_regs(vcpu); + kvm_x86_ops->skip_emulated_instruction(vcpu); +} +EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); From d075206073286dca84768137af0a0bf3d11f0663 Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Wed, 31 Oct 2007 17:24:25 -0500 Subject: [PATCH 084/249] KVM: Portability: Move x86 FPU handling to x86.c Signed-off-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 ++ drivers/kvm/kvm_main.c | 107 +---------------------------------------- drivers/kvm/x86.c | 103 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 105 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index ef2a6a8328ea..469ca42c2a19 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -633,6 +633,9 @@ long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); void kvm_arch_destroy_vm(struct kvm *kvm); +int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); +int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); + __init void kvm_arch_init(void); static inline void kvm_guest_enter(void) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 57573ebf02ba..7230f48ba08e 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -73,28 +73,6 @@ static inline int valid_vcpu(int n) return likely(n >= 0 && n < KVM_MAX_VCPUS); } -void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) -{ - if (!vcpu->fpu_active || vcpu->guest_fpu_loaded) - return; - - vcpu->guest_fpu_loaded = 1; - fx_save(&vcpu->host_fx_image); - fx_restore(&vcpu->guest_fx_image); -} -EXPORT_SYMBOL_GPL(kvm_load_guest_fpu); - -void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) -{ - if (!vcpu->guest_fpu_loaded) - return; - - vcpu->guest_fpu_loaded = 0; - fx_save(&vcpu->guest_fx_image); - fx_restore(&vcpu->host_fx_image); -} -EXPORT_SYMBOL_GPL(kvm_put_guest_fpu); - /* * Switches to specified vcpu, until a matching vcpu_put() */ @@ -294,26 +272,6 @@ static int kvm_vm_release(struct inode *inode, struct file *filp) return 0; } -void fx_init(struct kvm_vcpu *vcpu) -{ - unsigned after_mxcsr_mask; - - /* Initialize guest FPU by resetting ours and saving into guest's */ - preempt_disable(); - fx_save(&vcpu->host_fx_image); - fpu_init(); - fx_save(&vcpu->guest_fx_image); - fx_restore(&vcpu->host_fx_image); - preempt_enable(); - - vcpu->cr0 |= X86_CR0_ET; - after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space); - vcpu->guest_fx_image.mxcsr = 0x1f80; - memset((void *)&vcpu->guest_fx_image + after_mxcsr_mask, - 0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask); -} -EXPORT_SYMBOL_GPL(fx_init); - /* * Allocate some memory and give it an address in the guest physical address * space. @@ -1422,67 +1380,6 @@ static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) return 0; } -/* - * fxsave fpu state. Taken from x86_64/processor.h. To be killed when - * we have asm/x86/processor.h - */ -struct fxsave { - u16 cwd; - u16 swd; - u16 twd; - u16 fop; - u64 rip; - u64 rdp; - u32 mxcsr; - u32 mxcsr_mask; - u32 st_space[32]; /* 8*16 bytes for each FP-reg = 128 bytes */ -#ifdef CONFIG_X86_64 - u32 xmm_space[64]; /* 16*16 bytes for each XMM-reg = 256 bytes */ -#else - u32 xmm_space[32]; /* 8*16 bytes for each XMM-reg = 128 bytes */ -#endif -}; - -static int kvm_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) -{ - struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image; - - vcpu_load(vcpu); - - memcpy(fpu->fpr, fxsave->st_space, 128); - fpu->fcw = fxsave->cwd; - fpu->fsw = fxsave->swd; - fpu->ftwx = fxsave->twd; - fpu->last_opcode = fxsave->fop; - fpu->last_ip = fxsave->rip; - fpu->last_dp = fxsave->rdp; - memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space); - - vcpu_put(vcpu); - - return 0; -} - -static int kvm_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) -{ - struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image; - - vcpu_load(vcpu); - - memcpy(fxsave->st_space, fpu->fpr, 128); - fxsave->cwd = fpu->fcw; - fxsave->swd = fpu->fsw; - fxsave->twd = fpu->ftwx; - fxsave->fop = fpu->last_opcode; - fxsave->rip = fpu->last_ip; - fxsave->rdp = fpu->last_dp; - memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space); - - vcpu_put(vcpu); - - return 0; -} - static long kvm_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -1613,7 +1510,7 @@ static long kvm_vcpu_ioctl(struct file *filp, struct kvm_fpu fpu; memset(&fpu, 0, sizeof fpu); - r = kvm_vcpu_ioctl_get_fpu(vcpu, &fpu); + r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, &fpu); if (r) goto out; r = -EFAULT; @@ -1628,7 +1525,7 @@ static long kvm_vcpu_ioctl(struct file *filp, r = -EFAULT; if (copy_from_user(&fpu, argp, sizeof fpu)) goto out; - r = kvm_vcpu_ioctl_set_fpu(vcpu, &fpu); + r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, &fpu); if (r) goto out; r = 0; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 2cf7ebab50f4..ef1661f10b48 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1785,3 +1785,106 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) kvm_x86_ops->skip_emulated_instruction(vcpu); } EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); + +/* + * fxsave fpu state. Taken from x86_64/processor.h. To be killed when + * we have asm/x86/processor.h + */ +struct fxsave { + u16 cwd; + u16 swd; + u16 twd; + u16 fop; + u64 rip; + u64 rdp; + u32 mxcsr; + u32 mxcsr_mask; + u32 st_space[32]; /* 8*16 bytes for each FP-reg = 128 bytes */ +#ifdef CONFIG_X86_64 + u32 xmm_space[64]; /* 16*16 bytes for each XMM-reg = 256 bytes */ +#else + u32 xmm_space[32]; /* 8*16 bytes for each XMM-reg = 128 bytes */ +#endif +}; + +int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image; + + vcpu_load(vcpu); + + memcpy(fpu->fpr, fxsave->st_space, 128); + fpu->fcw = fxsave->cwd; + fpu->fsw = fxsave->swd; + fpu->ftwx = fxsave->twd; + fpu->last_opcode = fxsave->fop; + fpu->last_ip = fxsave->rip; + fpu->last_dp = fxsave->rdp; + memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space); + + vcpu_put(vcpu); + + return 0; +} + +int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image; + + vcpu_load(vcpu); + + memcpy(fxsave->st_space, fpu->fpr, 128); + fxsave->cwd = fpu->fcw; + fxsave->swd = fpu->fsw; + fxsave->twd = fpu->ftwx; + fxsave->fop = fpu->last_opcode; + fxsave->rip = fpu->last_ip; + fxsave->rdp = fpu->last_dp; + memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space); + + vcpu_put(vcpu); + + return 0; +} + +void fx_init(struct kvm_vcpu *vcpu) +{ + unsigned after_mxcsr_mask; + + /* Initialize guest FPU by resetting ours and saving into guest's */ + preempt_disable(); + fx_save(&vcpu->host_fx_image); + fpu_init(); + fx_save(&vcpu->guest_fx_image); + fx_restore(&vcpu->host_fx_image); + preempt_enable(); + + vcpu->cr0 |= X86_CR0_ET; + after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space); + vcpu->guest_fx_image.mxcsr = 0x1f80; + memset((void *)&vcpu->guest_fx_image + after_mxcsr_mask, + 0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask); +} +EXPORT_SYMBOL_GPL(fx_init); + +void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) +{ + if (!vcpu->fpu_active || vcpu->guest_fpu_loaded) + return; + + vcpu->guest_fpu_loaded = 1; + fx_save(&vcpu->host_fx_image); + fx_restore(&vcpu->guest_fx_image); +} +EXPORT_SYMBOL_GPL(kvm_load_guest_fpu); + +void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) +{ + if (!vcpu->guest_fpu_loaded) + return; + + vcpu->guest_fpu_loaded = 0; + fx_save(&vcpu->guest_fx_image); + fx_restore(&vcpu->host_fx_image); +} +EXPORT_SYMBOL_GPL(kvm_put_guest_fpu); From b6c7a5dccf9471f4891df722dbd0700ce56eb2e2 Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Thu, 1 Nov 2007 14:16:10 -0500 Subject: [PATCH 085/249] KVM: Portability: Move x86 vcpu ioctl handlers to x86.c Signed-off-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 10 + drivers/kvm/kvm_main.c | 442 +---------------------------------------- drivers/kvm/x86.c | 427 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 443 insertions(+), 436 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 469ca42c2a19..3f5ffc37480d 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -636,6 +636,16 @@ void kvm_arch_destroy_vm(struct kvm *kvm); int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); +int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); +int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); +int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs); +int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs); +int kvm_arch_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu, + struct kvm_debug_guest *dbg); +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run); + __init void kvm_arch_init(void); static inline void kvm_guest_enter(void) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 7230f48ba08e..8665531d9287 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -778,422 +778,6 @@ void kvm_resched(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_resched); -/* - * Check if userspace requested an interrupt window, and that the - * interrupt window is open. - * - * No need to exit to userspace if we already have an interrupt queued. - */ -static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu, - struct kvm_run *kvm_run) -{ - return (!vcpu->irq_summary && - kvm_run->request_interrupt_window && - vcpu->interrupt_window_open && - (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF)); -} - -static void post_kvm_run_save(struct kvm_vcpu *vcpu, - struct kvm_run *kvm_run) -{ - kvm_run->if_flag = (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF) != 0; - kvm_run->cr8 = get_cr8(vcpu); - kvm_run->apic_base = kvm_get_apic_base(vcpu); - if (irqchip_in_kernel(vcpu->kvm)) - kvm_run->ready_for_interrupt_injection = 1; - else - kvm_run->ready_for_interrupt_injection = - (vcpu->interrupt_window_open && - vcpu->irq_summary == 0); -} - -static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - int r; - - if (unlikely(vcpu->mp_state == VCPU_MP_STATE_SIPI_RECEIVED)) { - pr_debug("vcpu %d received sipi with vector # %x\n", - vcpu->vcpu_id, vcpu->sipi_vector); - kvm_lapic_reset(vcpu); - r = kvm_x86_ops->vcpu_reset(vcpu); - if (r) - return r; - vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; - } - -preempted: - if (vcpu->guest_debug.enabled) - kvm_x86_ops->guest_debug_pre(vcpu); - -again: - r = kvm_mmu_reload(vcpu); - if (unlikely(r)) - goto out; - - kvm_inject_pending_timer_irqs(vcpu); - - preempt_disable(); - - kvm_x86_ops->prepare_guest_switch(vcpu); - kvm_load_guest_fpu(vcpu); - - local_irq_disable(); - - if (signal_pending(current)) { - local_irq_enable(); - preempt_enable(); - r = -EINTR; - kvm_run->exit_reason = KVM_EXIT_INTR; - ++vcpu->stat.signal_exits; - goto out; - } - - if (irqchip_in_kernel(vcpu->kvm)) - kvm_x86_ops->inject_pending_irq(vcpu); - else if (!vcpu->mmio_read_completed) - kvm_x86_ops->inject_pending_vectors(vcpu, kvm_run); - - vcpu->guest_mode = 1; - kvm_guest_enter(); - - if (vcpu->requests) - if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests)) - kvm_x86_ops->tlb_flush(vcpu); - - kvm_x86_ops->run(vcpu, kvm_run); - - vcpu->guest_mode = 0; - local_irq_enable(); - - ++vcpu->stat.exits; - - /* - * We must have an instruction between local_irq_enable() and - * kvm_guest_exit(), so the timer interrupt isn't delayed by - * the interrupt shadow. The stat.exits increment will do nicely. - * But we need to prevent reordering, hence this barrier(): - */ - barrier(); - - kvm_guest_exit(); - - preempt_enable(); - - /* - * Profile KVM exit RIPs: - */ - if (unlikely(prof_on == KVM_PROFILING)) { - kvm_x86_ops->cache_regs(vcpu); - profile_hit(KVM_PROFILING, (void *)vcpu->rip); - } - - r = kvm_x86_ops->handle_exit(kvm_run, vcpu); - - if (r > 0) { - if (dm_request_for_irq_injection(vcpu, kvm_run)) { - r = -EINTR; - kvm_run->exit_reason = KVM_EXIT_INTR; - ++vcpu->stat.request_irq_exits; - goto out; - } - if (!need_resched()) { - ++vcpu->stat.light_exits; - goto again; - } - } - -out: - if (r > 0) { - kvm_resched(vcpu); - goto preempted; - } - - post_kvm_run_save(vcpu, kvm_run); - - return r; -} - - -static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - int r; - sigset_t sigsaved; - - vcpu_load(vcpu); - - if (unlikely(vcpu->mp_state == VCPU_MP_STATE_UNINITIALIZED)) { - kvm_vcpu_block(vcpu); - vcpu_put(vcpu); - return -EAGAIN; - } - - if (vcpu->sigset_active) - sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); - - /* re-sync apic's tpr */ - if (!irqchip_in_kernel(vcpu->kvm)) - set_cr8(vcpu, kvm_run->cr8); - - if (vcpu->pio.cur_count) { - r = complete_pio(vcpu); - if (r) - goto out; - } -#if CONFIG_HAS_IOMEM - if (vcpu->mmio_needed) { - memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8); - vcpu->mmio_read_completed = 1; - vcpu->mmio_needed = 0; - r = emulate_instruction(vcpu, kvm_run, - vcpu->mmio_fault_cr2, 0, 1); - if (r == EMULATE_DO_MMIO) { - /* - * Read-modify-write. Back to userspace. - */ - r = 0; - goto out; - } - } -#endif - if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) { - kvm_x86_ops->cache_regs(vcpu); - vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret; - kvm_x86_ops->decache_regs(vcpu); - } - - r = __vcpu_run(vcpu, kvm_run); - -out: - if (vcpu->sigset_active) - sigprocmask(SIG_SETMASK, &sigsaved, NULL); - - vcpu_put(vcpu); - return r; -} - -static int kvm_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, - struct kvm_regs *regs) -{ - vcpu_load(vcpu); - - kvm_x86_ops->cache_regs(vcpu); - - regs->rax = vcpu->regs[VCPU_REGS_RAX]; - regs->rbx = vcpu->regs[VCPU_REGS_RBX]; - regs->rcx = vcpu->regs[VCPU_REGS_RCX]; - regs->rdx = vcpu->regs[VCPU_REGS_RDX]; - regs->rsi = vcpu->regs[VCPU_REGS_RSI]; - regs->rdi = vcpu->regs[VCPU_REGS_RDI]; - regs->rsp = vcpu->regs[VCPU_REGS_RSP]; - regs->rbp = vcpu->regs[VCPU_REGS_RBP]; -#ifdef CONFIG_X86_64 - regs->r8 = vcpu->regs[VCPU_REGS_R8]; - regs->r9 = vcpu->regs[VCPU_REGS_R9]; - regs->r10 = vcpu->regs[VCPU_REGS_R10]; - regs->r11 = vcpu->regs[VCPU_REGS_R11]; - regs->r12 = vcpu->regs[VCPU_REGS_R12]; - regs->r13 = vcpu->regs[VCPU_REGS_R13]; - regs->r14 = vcpu->regs[VCPU_REGS_R14]; - regs->r15 = vcpu->regs[VCPU_REGS_R15]; -#endif - - regs->rip = vcpu->rip; - regs->rflags = kvm_x86_ops->get_rflags(vcpu); - - /* - * Don't leak debug flags in case they were set for guest debugging - */ - if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep) - regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF); - - vcpu_put(vcpu); - - return 0; -} - -static int kvm_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, - struct kvm_regs *regs) -{ - vcpu_load(vcpu); - - vcpu->regs[VCPU_REGS_RAX] = regs->rax; - vcpu->regs[VCPU_REGS_RBX] = regs->rbx; - vcpu->regs[VCPU_REGS_RCX] = regs->rcx; - vcpu->regs[VCPU_REGS_RDX] = regs->rdx; - vcpu->regs[VCPU_REGS_RSI] = regs->rsi; - vcpu->regs[VCPU_REGS_RDI] = regs->rdi; - vcpu->regs[VCPU_REGS_RSP] = regs->rsp; - vcpu->regs[VCPU_REGS_RBP] = regs->rbp; -#ifdef CONFIG_X86_64 - vcpu->regs[VCPU_REGS_R8] = regs->r8; - vcpu->regs[VCPU_REGS_R9] = regs->r9; - vcpu->regs[VCPU_REGS_R10] = regs->r10; - vcpu->regs[VCPU_REGS_R11] = regs->r11; - vcpu->regs[VCPU_REGS_R12] = regs->r12; - vcpu->regs[VCPU_REGS_R13] = regs->r13; - vcpu->regs[VCPU_REGS_R14] = regs->r14; - vcpu->regs[VCPU_REGS_R15] = regs->r15; -#endif - - vcpu->rip = regs->rip; - kvm_x86_ops->set_rflags(vcpu, regs->rflags); - - kvm_x86_ops->decache_regs(vcpu); - - vcpu_put(vcpu); - - return 0; -} - -static void get_segment(struct kvm_vcpu *vcpu, - struct kvm_segment *var, int seg) -{ - return kvm_x86_ops->get_segment(vcpu, var, seg); -} - -static int kvm_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, - struct kvm_sregs *sregs) -{ - struct descriptor_table dt; - int pending_vec; - - vcpu_load(vcpu); - - get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); - get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); - get_segment(vcpu, &sregs->es, VCPU_SREG_ES); - get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); - get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); - get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); - - get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); - get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); - - kvm_x86_ops->get_idt(vcpu, &dt); - sregs->idt.limit = dt.limit; - sregs->idt.base = dt.base; - kvm_x86_ops->get_gdt(vcpu, &dt); - sregs->gdt.limit = dt.limit; - sregs->gdt.base = dt.base; - - kvm_x86_ops->decache_cr4_guest_bits(vcpu); - sregs->cr0 = vcpu->cr0; - sregs->cr2 = vcpu->cr2; - sregs->cr3 = vcpu->cr3; - sregs->cr4 = vcpu->cr4; - sregs->cr8 = get_cr8(vcpu); - sregs->efer = vcpu->shadow_efer; - sregs->apic_base = kvm_get_apic_base(vcpu); - - if (irqchip_in_kernel(vcpu->kvm)) { - memset(sregs->interrupt_bitmap, 0, - sizeof sregs->interrupt_bitmap); - pending_vec = kvm_x86_ops->get_irq(vcpu); - if (pending_vec >= 0) - set_bit(pending_vec, - (unsigned long *)sregs->interrupt_bitmap); - } else - memcpy(sregs->interrupt_bitmap, vcpu->irq_pending, - sizeof sregs->interrupt_bitmap); - - vcpu_put(vcpu); - - return 0; -} - -static void set_segment(struct kvm_vcpu *vcpu, - struct kvm_segment *var, int seg) -{ - return kvm_x86_ops->set_segment(vcpu, var, seg); -} - -static int kvm_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, - struct kvm_sregs *sregs) -{ - int mmu_reset_needed = 0; - int i, pending_vec, max_bits; - struct descriptor_table dt; - - vcpu_load(vcpu); - - dt.limit = sregs->idt.limit; - dt.base = sregs->idt.base; - kvm_x86_ops->set_idt(vcpu, &dt); - dt.limit = sregs->gdt.limit; - dt.base = sregs->gdt.base; - kvm_x86_ops->set_gdt(vcpu, &dt); - - vcpu->cr2 = sregs->cr2; - mmu_reset_needed |= vcpu->cr3 != sregs->cr3; - vcpu->cr3 = sregs->cr3; - - set_cr8(vcpu, sregs->cr8); - - mmu_reset_needed |= vcpu->shadow_efer != sregs->efer; -#ifdef CONFIG_X86_64 - kvm_x86_ops->set_efer(vcpu, sregs->efer); -#endif - kvm_set_apic_base(vcpu, sregs->apic_base); - - kvm_x86_ops->decache_cr4_guest_bits(vcpu); - - mmu_reset_needed |= vcpu->cr0 != sregs->cr0; - vcpu->cr0 = sregs->cr0; - kvm_x86_ops->set_cr0(vcpu, sregs->cr0); - - mmu_reset_needed |= vcpu->cr4 != sregs->cr4; - kvm_x86_ops->set_cr4(vcpu, sregs->cr4); - if (!is_long_mode(vcpu) && is_pae(vcpu)) - load_pdptrs(vcpu, vcpu->cr3); - - if (mmu_reset_needed) - kvm_mmu_reset_context(vcpu); - - if (!irqchip_in_kernel(vcpu->kvm)) { - memcpy(vcpu->irq_pending, sregs->interrupt_bitmap, - sizeof vcpu->irq_pending); - vcpu->irq_summary = 0; - for (i = 0; i < ARRAY_SIZE(vcpu->irq_pending); ++i) - if (vcpu->irq_pending[i]) - __set_bit(i, &vcpu->irq_summary); - } else { - max_bits = (sizeof sregs->interrupt_bitmap) << 3; - pending_vec = find_first_bit( - (const unsigned long *)sregs->interrupt_bitmap, - max_bits); - /* Only pending external irq is handled here */ - if (pending_vec < max_bits) { - kvm_x86_ops->set_irq(vcpu, pending_vec); - pr_debug("Set back pending irq %d\n", - pending_vec); - } - } - - set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); - set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); - set_segment(vcpu, &sregs->es, VCPU_SREG_ES); - set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); - set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); - set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); - - set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); - set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); - - vcpu_put(vcpu); - - return 0; -} - -void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) -{ - struct kvm_segment cs; - - get_segment(vcpu, &cs, VCPU_SREG_CS); - *db = cs.db; - *l = cs.l; -} -EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits); - /* * Translate a guest virtual address to a guest physical address. */ @@ -1233,20 +817,6 @@ static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, return 0; } -static int kvm_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu, - struct kvm_debug_guest *dbg) -{ - int r; - - vcpu_load(vcpu); - - r = kvm_x86_ops->set_guest_debug(vcpu, dbg); - - vcpu_put(vcpu); - - return r; -} - static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma, unsigned long address, int *type) @@ -1392,13 +962,13 @@ static long kvm_vcpu_ioctl(struct file *filp, r = -EINVAL; if (arg) goto out; - r = kvm_vcpu_ioctl_run(vcpu, vcpu->run); + r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); break; case KVM_GET_REGS: { struct kvm_regs kvm_regs; memset(&kvm_regs, 0, sizeof kvm_regs); - r = kvm_vcpu_ioctl_get_regs(vcpu, &kvm_regs); + r = kvm_arch_vcpu_ioctl_get_regs(vcpu, &kvm_regs); if (r) goto out; r = -EFAULT; @@ -1413,7 +983,7 @@ static long kvm_vcpu_ioctl(struct file *filp, r = -EFAULT; if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs)) goto out; - r = kvm_vcpu_ioctl_set_regs(vcpu, &kvm_regs); + r = kvm_arch_vcpu_ioctl_set_regs(vcpu, &kvm_regs); if (r) goto out; r = 0; @@ -1423,7 +993,7 @@ static long kvm_vcpu_ioctl(struct file *filp, struct kvm_sregs kvm_sregs; memset(&kvm_sregs, 0, sizeof kvm_sregs); - r = kvm_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs); + r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs); if (r) goto out; r = -EFAULT; @@ -1438,7 +1008,7 @@ static long kvm_vcpu_ioctl(struct file *filp, r = -EFAULT; if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs)) goto out; - r = kvm_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs); + r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs); if (r) goto out; r = 0; @@ -1477,7 +1047,7 @@ static long kvm_vcpu_ioctl(struct file *filp, r = -EFAULT; if (copy_from_user(&dbg, argp, sizeof dbg)) goto out; - r = kvm_vcpu_ioctl_debug_guest(vcpu, &dbg); + r = kvm_arch_vcpu_ioctl_debug_guest(vcpu, &dbg); if (r) goto out; r = 0; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index ef1661f10b48..394da6605364 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1786,6 +1786,433 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); +/* + * Check if userspace requested an interrupt window, and that the + * interrupt window is open. + * + * No need to exit to userspace if we already have an interrupt queued. + */ +static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu, + struct kvm_run *kvm_run) +{ + return (!vcpu->irq_summary && + kvm_run->request_interrupt_window && + vcpu->interrupt_window_open && + (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF)); +} + +static void post_kvm_run_save(struct kvm_vcpu *vcpu, + struct kvm_run *kvm_run) +{ + kvm_run->if_flag = (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF) != 0; + kvm_run->cr8 = get_cr8(vcpu); + kvm_run->apic_base = kvm_get_apic_base(vcpu); + if (irqchip_in_kernel(vcpu->kvm)) + kvm_run->ready_for_interrupt_injection = 1; + else + kvm_run->ready_for_interrupt_injection = + (vcpu->interrupt_window_open && + vcpu->irq_summary == 0); +} + +static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + int r; + + if (unlikely(vcpu->mp_state == VCPU_MP_STATE_SIPI_RECEIVED)) { + pr_debug("vcpu %d received sipi with vector # %x\n", + vcpu->vcpu_id, vcpu->sipi_vector); + kvm_lapic_reset(vcpu); + r = kvm_x86_ops->vcpu_reset(vcpu); + if (r) + return r; + vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; + } + +preempted: + if (vcpu->guest_debug.enabled) + kvm_x86_ops->guest_debug_pre(vcpu); + +again: + r = kvm_mmu_reload(vcpu); + if (unlikely(r)) + goto out; + + kvm_inject_pending_timer_irqs(vcpu); + + preempt_disable(); + + kvm_x86_ops->prepare_guest_switch(vcpu); + kvm_load_guest_fpu(vcpu); + + local_irq_disable(); + + if (signal_pending(current)) { + local_irq_enable(); + preempt_enable(); + r = -EINTR; + kvm_run->exit_reason = KVM_EXIT_INTR; + ++vcpu->stat.signal_exits; + goto out; + } + + if (irqchip_in_kernel(vcpu->kvm)) + kvm_x86_ops->inject_pending_irq(vcpu); + else if (!vcpu->mmio_read_completed) + kvm_x86_ops->inject_pending_vectors(vcpu, kvm_run); + + vcpu->guest_mode = 1; + kvm_guest_enter(); + + if (vcpu->requests) + if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests)) + kvm_x86_ops->tlb_flush(vcpu); + + kvm_x86_ops->run(vcpu, kvm_run); + + vcpu->guest_mode = 0; + local_irq_enable(); + + ++vcpu->stat.exits; + + /* + * We must have an instruction between local_irq_enable() and + * kvm_guest_exit(), so the timer interrupt isn't delayed by + * the interrupt shadow. The stat.exits increment will do nicely. + * But we need to prevent reordering, hence this barrier(): + */ + barrier(); + + kvm_guest_exit(); + + preempt_enable(); + + /* + * Profile KVM exit RIPs: + */ + if (unlikely(prof_on == KVM_PROFILING)) { + kvm_x86_ops->cache_regs(vcpu); + profile_hit(KVM_PROFILING, (void *)vcpu->rip); + } + + r = kvm_x86_ops->handle_exit(kvm_run, vcpu); + + if (r > 0) { + if (dm_request_for_irq_injection(vcpu, kvm_run)) { + r = -EINTR; + kvm_run->exit_reason = KVM_EXIT_INTR; + ++vcpu->stat.request_irq_exits; + goto out; + } + if (!need_resched()) { + ++vcpu->stat.light_exits; + goto again; + } + } + +out: + if (r > 0) { + kvm_resched(vcpu); + goto preempted; + } + + post_kvm_run_save(vcpu, kvm_run); + + return r; +} + +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + int r; + sigset_t sigsaved; + + vcpu_load(vcpu); + + if (unlikely(vcpu->mp_state == VCPU_MP_STATE_UNINITIALIZED)) { + kvm_vcpu_block(vcpu); + vcpu_put(vcpu); + return -EAGAIN; + } + + if (vcpu->sigset_active) + sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); + + /* re-sync apic's tpr */ + if (!irqchip_in_kernel(vcpu->kvm)) + set_cr8(vcpu, kvm_run->cr8); + + if (vcpu->pio.cur_count) { + r = complete_pio(vcpu); + if (r) + goto out; + } +#if CONFIG_HAS_IOMEM + if (vcpu->mmio_needed) { + memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8); + vcpu->mmio_read_completed = 1; + vcpu->mmio_needed = 0; + r = emulate_instruction(vcpu, kvm_run, + vcpu->mmio_fault_cr2, 0, 1); + if (r == EMULATE_DO_MMIO) { + /* + * Read-modify-write. Back to userspace. + */ + r = 0; + goto out; + } + } +#endif + if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) { + kvm_x86_ops->cache_regs(vcpu); + vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret; + kvm_x86_ops->decache_regs(vcpu); + } + + r = __vcpu_run(vcpu, kvm_run); + +out: + if (vcpu->sigset_active) + sigprocmask(SIG_SETMASK, &sigsaved, NULL); + + vcpu_put(vcpu); + return r; +} + +int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + vcpu_load(vcpu); + + kvm_x86_ops->cache_regs(vcpu); + + regs->rax = vcpu->regs[VCPU_REGS_RAX]; + regs->rbx = vcpu->regs[VCPU_REGS_RBX]; + regs->rcx = vcpu->regs[VCPU_REGS_RCX]; + regs->rdx = vcpu->regs[VCPU_REGS_RDX]; + regs->rsi = vcpu->regs[VCPU_REGS_RSI]; + regs->rdi = vcpu->regs[VCPU_REGS_RDI]; + regs->rsp = vcpu->regs[VCPU_REGS_RSP]; + regs->rbp = vcpu->regs[VCPU_REGS_RBP]; +#ifdef CONFIG_X86_64 + regs->r8 = vcpu->regs[VCPU_REGS_R8]; + regs->r9 = vcpu->regs[VCPU_REGS_R9]; + regs->r10 = vcpu->regs[VCPU_REGS_R10]; + regs->r11 = vcpu->regs[VCPU_REGS_R11]; + regs->r12 = vcpu->regs[VCPU_REGS_R12]; + regs->r13 = vcpu->regs[VCPU_REGS_R13]; + regs->r14 = vcpu->regs[VCPU_REGS_R14]; + regs->r15 = vcpu->regs[VCPU_REGS_R15]; +#endif + + regs->rip = vcpu->rip; + regs->rflags = kvm_x86_ops->get_rflags(vcpu); + + /* + * Don't leak debug flags in case they were set for guest debugging + */ + if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep) + regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF); + + vcpu_put(vcpu); + + return 0; +} + +int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + vcpu_load(vcpu); + + vcpu->regs[VCPU_REGS_RAX] = regs->rax; + vcpu->regs[VCPU_REGS_RBX] = regs->rbx; + vcpu->regs[VCPU_REGS_RCX] = regs->rcx; + vcpu->regs[VCPU_REGS_RDX] = regs->rdx; + vcpu->regs[VCPU_REGS_RSI] = regs->rsi; + vcpu->regs[VCPU_REGS_RDI] = regs->rdi; + vcpu->regs[VCPU_REGS_RSP] = regs->rsp; + vcpu->regs[VCPU_REGS_RBP] = regs->rbp; +#ifdef CONFIG_X86_64 + vcpu->regs[VCPU_REGS_R8] = regs->r8; + vcpu->regs[VCPU_REGS_R9] = regs->r9; + vcpu->regs[VCPU_REGS_R10] = regs->r10; + vcpu->regs[VCPU_REGS_R11] = regs->r11; + vcpu->regs[VCPU_REGS_R12] = regs->r12; + vcpu->regs[VCPU_REGS_R13] = regs->r13; + vcpu->regs[VCPU_REGS_R14] = regs->r14; + vcpu->regs[VCPU_REGS_R15] = regs->r15; +#endif + + vcpu->rip = regs->rip; + kvm_x86_ops->set_rflags(vcpu, regs->rflags); + + kvm_x86_ops->decache_regs(vcpu); + + vcpu_put(vcpu); + + return 0; +} + +static void get_segment(struct kvm_vcpu *vcpu, + struct kvm_segment *var, int seg) +{ + return kvm_x86_ops->get_segment(vcpu, var, seg); +} + +void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) +{ + struct kvm_segment cs; + + get_segment(vcpu, &cs, VCPU_SREG_CS); + *db = cs.db; + *l = cs.l; +} +EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits); + +int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + struct descriptor_table dt; + int pending_vec; + + vcpu_load(vcpu); + + get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); + get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); + get_segment(vcpu, &sregs->es, VCPU_SREG_ES); + get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); + get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); + get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); + + get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); + get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); + + kvm_x86_ops->get_idt(vcpu, &dt); + sregs->idt.limit = dt.limit; + sregs->idt.base = dt.base; + kvm_x86_ops->get_gdt(vcpu, &dt); + sregs->gdt.limit = dt.limit; + sregs->gdt.base = dt.base; + + kvm_x86_ops->decache_cr4_guest_bits(vcpu); + sregs->cr0 = vcpu->cr0; + sregs->cr2 = vcpu->cr2; + sregs->cr3 = vcpu->cr3; + sregs->cr4 = vcpu->cr4; + sregs->cr8 = get_cr8(vcpu); + sregs->efer = vcpu->shadow_efer; + sregs->apic_base = kvm_get_apic_base(vcpu); + + if (irqchip_in_kernel(vcpu->kvm)) { + memset(sregs->interrupt_bitmap, 0, + sizeof sregs->interrupt_bitmap); + pending_vec = kvm_x86_ops->get_irq(vcpu); + if (pending_vec >= 0) + set_bit(pending_vec, + (unsigned long *)sregs->interrupt_bitmap); + } else + memcpy(sregs->interrupt_bitmap, vcpu->irq_pending, + sizeof sregs->interrupt_bitmap); + + vcpu_put(vcpu); + + return 0; +} + +static void set_segment(struct kvm_vcpu *vcpu, + struct kvm_segment *var, int seg) +{ + return kvm_x86_ops->set_segment(vcpu, var, seg); +} + +int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + int mmu_reset_needed = 0; + int i, pending_vec, max_bits; + struct descriptor_table dt; + + vcpu_load(vcpu); + + dt.limit = sregs->idt.limit; + dt.base = sregs->idt.base; + kvm_x86_ops->set_idt(vcpu, &dt); + dt.limit = sregs->gdt.limit; + dt.base = sregs->gdt.base; + kvm_x86_ops->set_gdt(vcpu, &dt); + + vcpu->cr2 = sregs->cr2; + mmu_reset_needed |= vcpu->cr3 != sregs->cr3; + vcpu->cr3 = sregs->cr3; + + set_cr8(vcpu, sregs->cr8); + + mmu_reset_needed |= vcpu->shadow_efer != sregs->efer; +#ifdef CONFIG_X86_64 + kvm_x86_ops->set_efer(vcpu, sregs->efer); +#endif + kvm_set_apic_base(vcpu, sregs->apic_base); + + kvm_x86_ops->decache_cr4_guest_bits(vcpu); + + mmu_reset_needed |= vcpu->cr0 != sregs->cr0; + vcpu->cr0 = sregs->cr0; + kvm_x86_ops->set_cr0(vcpu, sregs->cr0); + + mmu_reset_needed |= vcpu->cr4 != sregs->cr4; + kvm_x86_ops->set_cr4(vcpu, sregs->cr4); + if (!is_long_mode(vcpu) && is_pae(vcpu)) + load_pdptrs(vcpu, vcpu->cr3); + + if (mmu_reset_needed) + kvm_mmu_reset_context(vcpu); + + if (!irqchip_in_kernel(vcpu->kvm)) { + memcpy(vcpu->irq_pending, sregs->interrupt_bitmap, + sizeof vcpu->irq_pending); + vcpu->irq_summary = 0; + for (i = 0; i < ARRAY_SIZE(vcpu->irq_pending); ++i) + if (vcpu->irq_pending[i]) + __set_bit(i, &vcpu->irq_summary); + } else { + max_bits = (sizeof sregs->interrupt_bitmap) << 3; + pending_vec = find_first_bit( + (const unsigned long *)sregs->interrupt_bitmap, + max_bits); + /* Only pending external irq is handled here */ + if (pending_vec < max_bits) { + kvm_x86_ops->set_irq(vcpu, pending_vec); + pr_debug("Set back pending irq %d\n", + pending_vec); + } + } + + set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); + set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); + set_segment(vcpu, &sregs->es, VCPU_SREG_ES); + set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); + set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); + set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); + + set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); + set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); + + vcpu_put(vcpu); + + return 0; +} + +int kvm_arch_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu, + struct kvm_debug_guest *dbg) +{ + int r; + + vcpu_load(vcpu); + + r = kvm_x86_ops->set_guest_debug(vcpu, dbg); + + vcpu_put(vcpu); + + return r; +} + /* * fxsave fpu state. Taken from x86_64/processor.h. To be killed when * we have asm/x86/processor.h From 12264760e46077a65c1240ac0b27dfa34b402158 Mon Sep 17 00:00:00 2001 From: Dor Laor Date: Wed, 7 Nov 2007 16:20:06 +0200 Subject: [PATCH 086/249] KVM: Add make_page_dirty() to kvm_clear_guest_page() Every write access to guest pages should be tracked. Signed-off-by: Dor Laor Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 8665531d9287..8d6e55f95fbb 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -706,6 +706,7 @@ int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) kunmap_atomic(page_virt, KM_USER0); kvm_release_page(page); + mark_page_dirty(kvm, gfn); return 0; } EXPORT_SYMBOL_GPL(kvm_clear_guest_page); From 9c5623e3e42e94927d02a6693875badf15692970 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 8 Nov 2007 18:19:20 +0200 Subject: [PATCH 087/249] KVM: VMX: Use vmx to inject real-mode interrupts Instead of injecting real-mode interrupts by writing the interrupt frame into guest memory, abuse vmx by injecting a software interrupt. We need to pretend the software interrupt instruction had a length > 0, so we have to adjust rip backward. This lets us not to mess with writing guest memory, which is complex and also sleeps. Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 53 ++++------------------------------------------- drivers/kvm/vmx.h | 1 + 2 files changed, 5 insertions(+), 49 deletions(-) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index eca422e9506d..d2c25e25d3aa 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1709,58 +1709,13 @@ out: return ret; } -static void inject_rmode_irq(struct kvm_vcpu *vcpu, int irq) -{ - u16 ent[2]; - u16 cs; - u16 ip; - unsigned long flags; - unsigned long ss_base = vmcs_readl(GUEST_SS_BASE); - u16 sp = vmcs_readl(GUEST_RSP); - u32 ss_limit = vmcs_read32(GUEST_SS_LIMIT); - - if (sp > ss_limit || sp < 6) { - vcpu_printf(vcpu, "%s: #SS, rsp 0x%lx ss 0x%lx limit 0x%x\n", - __FUNCTION__, - vmcs_readl(GUEST_RSP), - vmcs_readl(GUEST_SS_BASE), - vmcs_read32(GUEST_SS_LIMIT)); - return; - } - - if (emulator_read_std(irq * sizeof(ent), &ent, sizeof(ent), vcpu) != - X86EMUL_CONTINUE) { - vcpu_printf(vcpu, "%s: read guest err\n", __FUNCTION__); - return; - } - - flags = vmcs_readl(GUEST_RFLAGS); - cs = vmcs_readl(GUEST_CS_BASE) >> 4; - ip = vmcs_readl(GUEST_RIP); - - - if (emulator_write_emulated( - ss_base + sp - 2, &flags, 2, vcpu) != X86EMUL_CONTINUE || - emulator_write_emulated( - ss_base + sp - 4, &cs, 2, vcpu) != X86EMUL_CONTINUE || - emulator_write_emulated( - ss_base + sp - 6, &ip, 2, vcpu) != X86EMUL_CONTINUE) { - vcpu_printf(vcpu, "%s: write guest err\n", __FUNCTION__); - return; - } - - vmcs_writel(GUEST_RFLAGS, flags & - ~(X86_EFLAGS_IF | X86_EFLAGS_AC | X86_EFLAGS_TF)); - vmcs_write16(GUEST_CS_SELECTOR, ent[1]) ; - vmcs_writel(GUEST_CS_BASE, ent[1] << 4); - vmcs_writel(GUEST_RIP, ent[0]); - vmcs_writel(GUEST_RSP, (vmcs_readl(GUEST_RSP) & ~0xffff) | (sp - 6)); -} - static void vmx_inject_irq(struct kvm_vcpu *vcpu, int irq) { if (vcpu->rmode.active) { - inject_rmode_irq(vcpu, irq); + vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, + irq | INTR_TYPE_SOFT_INTR | INTR_INFO_VALID_MASK); + vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1); + vmcs_writel(GUEST_RIP, vmcs_readl(GUEST_RIP) - 1); return; } vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, diff --git a/drivers/kvm/vmx.h b/drivers/kvm/vmx.h index c84bd3733153..d757b36c2fbe 100644 --- a/drivers/kvm/vmx.h +++ b/drivers/kvm/vmx.h @@ -233,6 +233,7 @@ enum vmcs_field { #define INTR_TYPE_EXT_INTR (0 << 8) /* external interrupt */ #define INTR_TYPE_EXCEPTION (3 << 8) /* processor exception */ +#define INTR_TYPE_SOFT_INTR (4 << 8) /* software interrupt */ /* * Exit Qualifications for MOV for Control Register Access From 1155f76a8166ae6fc88e7d73eb6817eb9012d476 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 22 Nov 2007 11:30:47 +0200 Subject: [PATCH 088/249] KVM: VMX: Read & store IDT_VECTORING_INFO_FIELD We'll want to write to it in order to fix real-mode irq injection problems, but it is a read-only field. Storing it in a variable solves that issue. Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index d2c25e25d3aa..f045f4005eaa 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -48,6 +48,7 @@ struct vcpu_vmx { struct kvm_vcpu vcpu; int launched; u8 fail; + u32 idt_vectoring_info; struct kvm_msr_entry *guest_msrs; struct kvm_msr_entry *host_msrs; int nmsrs; @@ -863,9 +864,10 @@ static int set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg) static int vmx_get_irq(struct kvm_vcpu *vcpu) { + struct vcpu_vmx *vmx = to_vmx(vcpu); u32 idtv_info_field; - idtv_info_field = vmcs_read32(IDT_VECTORING_INFO_FIELD); + idtv_info_field = vmx->idt_vectoring_info; if (idtv_info_field & INTR_INFO_VALID_MASK) { if (is_external_interrupt(idtv_info_field)) return idtv_info_field & VECTORING_INFO_VECTOR_MASK; @@ -1817,12 +1819,13 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu, static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { + struct vcpu_vmx *vmx = to_vmx(vcpu); u32 intr_info, error_code; unsigned long cr2, rip; u32 vect_info; enum emulation_result er; - vect_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); + vect_info = vmx->idt_vectoring_info; intr_info = vmcs_read32(VM_EXIT_INTR_INFO); if ((vect_info & VECTORING_INFO_VALID_MASK) && @@ -2171,9 +2174,9 @@ static const int kvm_vmx_max_exit_handlers = */ static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) { - u32 vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); u32 exit_reason = vmcs_read32(VM_EXIT_REASON); struct vcpu_vmx *vmx = to_vmx(vcpu); + u32 vectoring_info = vmx->idt_vectoring_info; if (unlikely(vmx->fail)) { kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY; @@ -2228,6 +2231,7 @@ static void enable_irq_window(struct kvm_vcpu *vcpu) static void vmx_intr_assist(struct kvm_vcpu *vcpu) { + struct vcpu_vmx *vmx = to_vmx(vcpu); u32 idtv_info_field, intr_info_field; int has_ext_irq, interrupt_window_open; int vector; @@ -2236,7 +2240,7 @@ static void vmx_intr_assist(struct kvm_vcpu *vcpu) has_ext_irq = kvm_cpu_has_interrupt(vcpu); intr_info_field = vmcs_read32(VM_ENTRY_INTR_INFO_FIELD); - idtv_info_field = vmcs_read32(IDT_VECTORING_INFO_FIELD); + idtv_info_field = vmx->idt_vectoring_info; if (intr_info_field & INTR_INFO_VALID_MASK) { if (idtv_info_field & INTR_INFO_VALID_MASK) { /* TODO: fault when IDT_Vectoring */ @@ -2396,6 +2400,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) #endif ); + vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); + vcpu->interrupt_window_open = (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0; @@ -2413,7 +2419,8 @@ static void vmx_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr, u32 err_code) { - u32 vect_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); + struct vcpu_vmx *vmx = to_vmx(vcpu); + u32 vect_info = vmx->idt_vectoring_info; ++vcpu->stat.pf_guest; From 9c8cba3761d4741cfd53ecc40604fac01f52128a Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 22 Nov 2007 11:42:59 +0200 Subject: [PATCH 089/249] KVM: Fix faults during injection of real-mode interrupts If vmx fails to inject a real-mode interrupt while fetching the interrupt redirection table, it fails to record this in the vectoring information field. So we detect this condition and do it ourselves. Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 51 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index f045f4005eaa..26719aa9701e 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -65,7 +65,13 @@ struct vcpu_vmx { int fs_reload_needed; int guest_efer_loaded; } host_state; - + struct { + struct { + bool pending; + u8 vector; + unsigned rip; + } irq; + } rmode; }; static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu) @@ -1713,11 +1719,16 @@ out: static void vmx_inject_irq(struct kvm_vcpu *vcpu, int irq) { + struct vcpu_vmx *vmx = to_vmx(vcpu); + if (vcpu->rmode.active) { + vmx->rmode.irq.pending = true; + vmx->rmode.irq.vector = irq; + vmx->rmode.irq.rip = vmcs_readl(GUEST_RIP); vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, irq | INTR_TYPE_SOFT_INTR | INTR_INFO_VALID_MASK); vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1); - vmcs_writel(GUEST_RIP, vmcs_readl(GUEST_RIP) - 1); + vmcs_writel(GUEST_RIP, vmx->rmode.irq.rip - 1); return; } vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, @@ -2251,6 +2262,17 @@ static void vmx_intr_assist(struct kvm_vcpu *vcpu) return; } if (unlikely(idtv_info_field & INTR_INFO_VALID_MASK)) { + if ((idtv_info_field & VECTORING_INFO_TYPE_MASK) + == INTR_TYPE_EXT_INTR + && vcpu->rmode.active) { + u8 vect = idtv_info_field & VECTORING_INFO_VECTOR_MASK; + + vmx_inject_irq(vcpu, vect); + if (unlikely(has_ext_irq)) + enable_irq_window(vcpu); + return; + } + vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, idtv_info_field); vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, vmcs_read32(VM_EXIT_INSTRUCTION_LEN)); @@ -2275,6 +2297,29 @@ static void vmx_intr_assist(struct kvm_vcpu *vcpu) enable_irq_window(vcpu); } +/* + * Failure to inject an interrupt should give us the information + * in IDT_VECTORING_INFO_FIELD. However, if the failure occurs + * when fetching the interrupt redirection bitmap in the real-mode + * tss, this doesn't happen. So we do it ourselves. + */ +static void fixup_rmode_irq(struct vcpu_vmx *vmx) +{ + vmx->rmode.irq.pending = 0; + if (vmcs_readl(GUEST_RIP) + 1 != vmx->rmode.irq.rip) + return; + vmcs_writel(GUEST_RIP, vmx->rmode.irq.rip); + if (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK) { + vmx->idt_vectoring_info &= ~VECTORING_INFO_TYPE_MASK; + vmx->idt_vectoring_info |= INTR_TYPE_EXT_INTR; + return; + } + vmx->idt_vectoring_info = + VECTORING_INFO_VALID_MASK + | INTR_TYPE_EXT_INTR + | vmx->rmode.irq.vector; +} + static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -2401,6 +2446,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) ); vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); + if (vmx->rmode.irq.pending) + fixup_rmode_irq(vmx); vcpu->interrupt_window_open = (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0; From 8a70cc3d0f4877f862ac9cace2e61e4e5116b502 Mon Sep 17 00:00:00 2001 From: Eddie Dong Date: Sun, 11 Nov 2007 12:27:20 +0200 Subject: [PATCH 090/249] KVM: VMX: Comment VMX primary/secondary exec ctl definitions Add comments for secondary/primary Processor-Based VM-execution controls. Signed-off-by: Yaozu (Eddie) Dong Signed-off-by: Avi Kivity --- drivers/kvm/vmx.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/kvm/vmx.h b/drivers/kvm/vmx.h index d757b36c2fbe..6d32bc69285e 100644 --- a/drivers/kvm/vmx.h +++ b/drivers/kvm/vmx.h @@ -25,6 +25,9 @@ * */ +/* + * Definitions of Primary Processor-Based VM-Execution Controls. + */ #define CPU_BASED_VIRTUAL_INTR_PENDING 0x00000004 #define CPU_BASED_USE_TSC_OFFSETING 0x00000008 #define CPU_BASED_HLT_EXITING 0x00000080 @@ -42,6 +45,11 @@ #define CPU_BASED_MONITOR_EXITING 0x20000000 #define CPU_BASED_PAUSE_EXITING 0x40000000 #define CPU_BASED_ACTIVATE_SECONDARY_CONTROLS 0x80000000 +/* + * Definitions of Secondary Processor-Based VM-Execution Controls. + */ +#define SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES 0x00000001 + #define PIN_BASED_EXT_INTR_MASK 0x00000001 #define PIN_BASED_NMI_EXITING 0x00000008 @@ -54,8 +62,6 @@ #define VM_ENTRY_SMM 0x00000400 #define VM_ENTRY_DEACT_DUAL_MONITOR 0x00000800 -#define SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES 0x00000001 - /* VMCS Encodings */ enum vmcs_field { GUEST_ES_SELECTOR = 0x00000800, From e5edaa01c4cea5f60c617fac989c6458df0ecc4e Mon Sep 17 00:00:00 2001 From: Eddie Dong Date: Sun, 11 Nov 2007 12:28:35 +0200 Subject: [PATCH 091/249] KVM: VMX: wbinvd exiting Add wbinvd VM Exit support to prepare for pass-through device cache emulation and also enhance real time responsiveness. Signed-off-by: Yaozu (Eddie) Dong Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 11 ++++++++++- drivers/kvm/vmx.h | 2 ++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 26719aa9701e..c3bde7532714 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -980,7 +980,8 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) #endif if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) { min = 0; - opt = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; + opt = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | + SECONDARY_EXEC_WBINVD_EXITING; if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS2, &_cpu_based_2nd_exec_control) < 0) return -EIO; @@ -2133,6 +2134,13 @@ static int handle_vmcall(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) return 1; } +static int handle_wbinvd(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + skip_emulated_instruction(vcpu); + /* TODO: Add support for VT-d/pass-through device */ + return 1; +} + static int handle_apic_access(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { u64 exit_qualification; @@ -2174,6 +2182,7 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu, [EXIT_REASON_VMCALL] = handle_vmcall, [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold, [EXIT_REASON_APIC_ACCESS] = handle_apic_access, + [EXIT_REASON_WBINVD] = handle_wbinvd, }; static const int kvm_vmx_max_exit_handlers = diff --git a/drivers/kvm/vmx.h b/drivers/kvm/vmx.h index 6d32bc69285e..d52ae8d7303d 100644 --- a/drivers/kvm/vmx.h +++ b/drivers/kvm/vmx.h @@ -49,6 +49,7 @@ * Definitions of Secondary Processor-Based VM-Execution Controls. */ #define SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES 0x00000001 +#define SECONDARY_EXEC_WBINVD_EXITING 0x00000040 #define PIN_BASED_EXT_INTR_MASK 0x00000001 @@ -223,6 +224,7 @@ enum vmcs_field { #define EXIT_REASON_MWAIT_INSTRUCTION 36 #define EXIT_REASON_TPR_BELOW_THRESHOLD 43 #define EXIT_REASON_APIC_ACCESS 44 +#define EXIT_REASON_WBINVD 54 /* * Interruption-information format From 906e608b05b87b650e509491e41c89575358df5e Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Sun, 11 Nov 2007 14:48:17 +0200 Subject: [PATCH 092/249] KVM: x86 emulator: remove 8 bytes operands emulator for call near instruction it is removed beacuse it isnt supported on a real host Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 64888a6edc27..e6979475bee7 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1558,9 +1558,6 @@ special_insn: case 4: rel = insn_fetch(s32, 4, c->eip); break; - case 8: - rel = insn_fetch(s64, 8, c->eip); - break; default: DPRINTF("Call: Invalid op_bytes\n"); goto cannot_emulate; From 1a6f4d7fbd11e539630cd5637311a4e55fae60ef Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 11 Nov 2007 18:37:32 +0200 Subject: [PATCH 093/249] KVM: Simplify CPU_TASKS_FROZEN cpu notifier handling Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 8d6e55f95fbb..d395c987894e 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1335,21 +1335,19 @@ static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, { int cpu = (long)v; + val &= ~CPU_TASKS_FROZEN; switch (val) { case CPU_DYING: - case CPU_DYING_FROZEN: printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n", cpu); hardware_disable(NULL); break; case CPU_UP_CANCELED: - case CPU_UP_CANCELED_FROZEN: printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n", cpu); smp_call_function_single(cpu, hardware_disable, NULL, 0, 1); break; case CPU_ONLINE: - case CPU_ONLINE_FROZEN: printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n", cpu); smp_call_function_single(cpu, hardware_enable, NULL, 0, 1); From f9d46eb0e4a5b5e0926ca61c19f8c8bbb9496b28 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Sun, 11 Nov 2007 22:02:22 +0200 Subject: [PATCH 094/249] KVM: add kvm_is_error_hva() Check for the "error hva", an address outside the user address space that signals a bad gfn. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/kvm_main.c | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 3f5ffc37480d..64983240adca 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -520,6 +520,7 @@ struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva); extern struct page *bad_page; int is_error_page(struct page *page); +int kvm_is_error_hva(unsigned long addr); int kvm_set_memory_region(struct kvm *kvm, struct kvm_userspace_memory_region *mem, int user_alloc); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index d395c987894e..dfb65e2f87ee 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -498,6 +498,17 @@ int is_error_page(struct page *page) } EXPORT_SYMBOL_GPL(is_error_page); +static inline unsigned long bad_hva(void) +{ + return PAGE_OFFSET; +} + +int kvm_is_error_hva(unsigned long addr) +{ + return addr == bad_hva(); +} +EXPORT_SYMBOL_GPL(kvm_is_error_hva); + gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) { int i; From 539cb6608ca804e7805d8e88c83377d991a552b1 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Sun, 11 Nov 2007 22:05:04 +0200 Subject: [PATCH 095/249] KVM: introduce gfn_to_hva() Convert a guest frame number to the corresponding host virtual address. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index dfb65e2f87ee..04544aecf22f 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -559,28 +559,37 @@ int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) } EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); +static unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) +{ + struct kvm_memory_slot *slot; + + gfn = unalias_gfn(kvm, gfn); + slot = __gfn_to_memslot(kvm, gfn); + if (!slot) + return bad_hva(); + return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE); +} + /* * Requires current->mm->mmap_sem to be held */ static struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn) { - struct kvm_memory_slot *slot; struct page *page[1]; + unsigned long addr; int npages; might_sleep(); - gfn = unalias_gfn(kvm, gfn); - slot = __gfn_to_memslot(kvm, gfn); - if (!slot) { + addr = gfn_to_hva(kvm, gfn); + if (kvm_is_error_hva(addr)) { get_page(bad_page); return bad_page; } - npages = get_user_pages(current, current->mm, - slot->userspace_addr - + (gfn - slot->base_gfn) * PAGE_SIZE, 1, - 1, 1, page, NULL); + npages = get_user_pages(current, current->mm, addr, 1, 1, 1, page, + NULL); + if (npages != 1) { get_page(bad_page); return bad_page; From e0506bcba5992650b1190de9125f5963a30f32e2 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Sun, 11 Nov 2007 22:10:22 +0200 Subject: [PATCH 096/249] KVM: Change kvm_{read,write}_guest() to use copy_{from,to}_user() This changes kvm_write_guest_page/kvm_read_guest_page to use copy_to_user/read_from_user, as a result we get better speed and better dirty bit tracking. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 04544aecf22f..d68901bd0693 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -630,20 +630,15 @@ static int next_segment(unsigned long len, int offset) int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, int len) { - void *page_virt; - struct page *page; + int r; + unsigned long addr; - page = gfn_to_page(kvm, gfn); - if (is_error_page(page)) { - kvm_release_page(page); + addr = gfn_to_hva(kvm, gfn); + if (kvm_is_error_hva(addr)) + return -EFAULT; + r = copy_from_user(data, (void __user *)addr + offset, len); + if (r) return -EFAULT; - } - page_virt = kmap_atomic(page, KM_USER0); - - memcpy(data, page_virt + offset, len); - - kunmap_atomic(page_virt, KM_USER0); - kvm_release_page(page); return 0; } EXPORT_SYMBOL_GPL(kvm_read_guest_page); @@ -671,21 +666,16 @@ EXPORT_SYMBOL_GPL(kvm_read_guest); int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, int offset, int len) { - void *page_virt; - struct page *page; + int r; + unsigned long addr; - page = gfn_to_page(kvm, gfn); - if (is_error_page(page)) { - kvm_release_page(page); + addr = gfn_to_hva(kvm, gfn); + if (kvm_is_error_hva(addr)) + return -EFAULT; + r = copy_to_user((void __user *)addr + offset, data, len); + if (r) return -EFAULT; - } - page_virt = kmap_atomic(page, KM_USER0); - - memcpy(page_virt + offset, data, len); - - kunmap_atomic(page_virt, KM_USER0); mark_page_dirty(kvm, gfn); - kvm_release_page(page); return 0; } EXPORT_SYMBOL_GPL(kvm_write_guest_page); From d825ed0a97b8e82597dd652bfe51fed3fa9b3c9a Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Wed, 14 Nov 2007 20:08:51 +0800 Subject: [PATCH 097/249] KVM: Portability: Move some includes to x86.c Move some includes to x86.c from kvm_main.c, since the related functions have been moved to x86.c Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 2 -- drivers/kvm/x86.c | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index d68901bd0693..6aaba79f8c5a 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -17,7 +17,6 @@ #include "kvm.h" #include "x86.h" -#include "x86_emulate.h" #include "irq.h" #include @@ -44,7 +43,6 @@ #include #include -#include #include #include #include diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 394da6605364..4282a0fb2962 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -16,6 +16,7 @@ #include "kvm.h" #include "x86.h" +#include "x86_emulate.h" #include "segment_descriptor.h" #include "irq.h" @@ -25,6 +26,7 @@ #include #include +#include #define MAX_IO_MSRS 256 #define CR0_RESERVED_BITS \ From 97896d04a14669b146c17d779b81ec7a339deeb3 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Wed, 14 Nov 2007 20:09:30 +0800 Subject: [PATCH 098/249] KVM: Portability: Move kvm_x86_ops to x86.c Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 1 - drivers/kvm/x86.c | 2 ++ drivers/kvm/x86.h | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 6aaba79f8c5a..47a76c3a4c81 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -55,7 +55,6 @@ static LIST_HEAD(vm_list); static cpumask_t cpus_hardware_enabled; -struct kvm_x86_ops *kvm_x86_ops; struct kmem_cache *kvm_vcpu_cache; EXPORT_SYMBOL_GPL(kvm_vcpu_cache); diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 4282a0fb2962..2edc53ec8b3b 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -44,6 +44,8 @@ #define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x) +struct kvm_x86_ops *kvm_x86_ops; + struct kvm_stats_debugfs_item debugfs_entries[] = { { "pf_fixed", STAT_OFFSET(pf_fixed) }, { "pf_guest", STAT_OFFSET(pf_guest) }, diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 663b822b4ddb..ec32c26a5118 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -85,6 +85,8 @@ struct kvm_vcpu { struct x86_emulate_ctxt emulate_ctxt; }; +extern struct kvm_x86_ops *kvm_x86_ops; + int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code); static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) From e9b11c17552afe684e9e5d0444309a3ddf410116 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Wed, 14 Nov 2007 20:38:21 +0800 Subject: [PATCH 099/249] KVM: Portability: Add vcpu and hardware management arch hooks Add the following hooks: void decache_vcpus_on_cpu(int cpu); int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id); void kvm_arch_vcpu_destory(struct kvm_vcpu *vcpu); int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu); void kvm_arch_hardware_enable(void *garbage); void kvm_arch_hardware_disable(void *garbage); int kvm_arch_hardware_setup(void); void kvm_arch_hardware_unsetup(void); void kvm_arch_check_processor_compat(void *rtn); Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 19 +++++ drivers/kvm/kvm_main.c | 113 +++++------------------------ drivers/kvm/x86.c | 157 +++++++++++++++++++++++++++++++++++++++++ drivers/kvm/x86.h | 3 + 4 files changed, 197 insertions(+), 95 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 64983240adca..bca07c6d0460 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -492,6 +492,8 @@ void kvm_vcpu_uninit(struct kvm_vcpu *vcpu); void vcpu_load(struct kvm_vcpu *vcpu); void vcpu_put(struct kvm_vcpu *vcpu); +void decache_vcpus_on_cpu(int cpu); + int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, struct module *module); @@ -649,6 +651,23 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run); __init void kvm_arch_init(void); +int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu); +void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu); + +void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu); +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); +struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id); +void kvm_arch_vcpu_destory(struct kvm_vcpu *vcpu); + +int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu); +void kvm_arch_hardware_enable(void *garbage); +void kvm_arch_hardware_disable(void *garbage); +int kvm_arch_hardware_setup(void); +void kvm_arch_hardware_unsetup(void); +void kvm_arch_check_processor_compat(void *rtn); + + static inline void kvm_guest_enter(void) { account_system_vtime(current); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 47a76c3a4c81..7fdfed52dbe7 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -50,8 +50,8 @@ MODULE_AUTHOR("Qumranet"); MODULE_LICENSE("GPL"); -static DEFINE_SPINLOCK(kvm_lock); -static LIST_HEAD(vm_list); +DEFINE_SPINLOCK(kvm_lock); +LIST_HEAD(vm_list); static cpumask_t cpus_hardware_enabled; @@ -124,13 +124,8 @@ int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) mutex_init(&vcpu->mutex); vcpu->cpu = -1; - vcpu->mmu.root_hpa = INVALID_PAGE; vcpu->kvm = kvm; vcpu->vcpu_id = id; - if (!irqchip_in_kernel(kvm) || id == 0) - vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; - else - vcpu->mp_state = VCPU_MP_STATE_UNINITIALIZED; init_waitqueue_head(&vcpu->wq); page = alloc_page(GFP_KERNEL | __GFP_ZERO); @@ -140,29 +135,11 @@ int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) } vcpu->run = page_address(page); - page = alloc_page(GFP_KERNEL | __GFP_ZERO); - if (!page) { - r = -ENOMEM; - goto fail_free_run; - } - vcpu->pio_data = page_address(page); - - r = kvm_mmu_create(vcpu); + r = kvm_arch_vcpu_init(vcpu); if (r < 0) - goto fail_free_pio_data; - - if (irqchip_in_kernel(kvm)) { - r = kvm_create_lapic(vcpu); - if (r < 0) - goto fail_mmu_destroy; - } - + goto fail_free_run; return 0; -fail_mmu_destroy: - kvm_mmu_destroy(vcpu); -fail_free_pio_data: - free_page((unsigned long)vcpu->pio_data); fail_free_run: free_page((unsigned long)vcpu->run); fail: @@ -172,9 +149,7 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_init); void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) { - kvm_free_lapic(vcpu); - kvm_mmu_destroy(vcpu); - free_page((unsigned long)vcpu->pio_data); + kvm_arch_vcpu_uninit(vcpu); free_page((unsigned long)vcpu->run); } EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); @@ -240,7 +215,7 @@ static void kvm_free_vcpus(struct kvm *kvm) kvm_unload_vcpu_mmu(kvm->vcpus[i]); for (i = 0; i < KVM_MAX_VCPUS; ++i) { if (kvm->vcpus[i]) { - kvm_x86_ops->vcpu_free(kvm->vcpus[i]); + kvm_arch_vcpu_free(kvm->vcpus[i]); kvm->vcpus[i] = NULL; } } @@ -900,28 +875,17 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n) if (!valid_vcpu(n)) return -EINVAL; - vcpu = kvm_x86_ops->vcpu_create(kvm, n); + vcpu = kvm_arch_vcpu_create(kvm, n); if (IS_ERR(vcpu)) return PTR_ERR(vcpu); preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); - /* We do fxsave: this must be aligned. */ - BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF); - - vcpu_load(vcpu); - r = kvm_x86_ops->vcpu_reset(vcpu); - if (r == 0) - r = kvm_mmu_setup(vcpu); - vcpu_put(vcpu); - if (r < 0) - goto free_vcpu; - mutex_lock(&kvm->lock); if (kvm->vcpus[n]) { r = -EEXIST; mutex_unlock(&kvm->lock); - goto mmu_unload; + goto vcpu_destroy; } kvm->vcpus[n] = vcpu; mutex_unlock(&kvm->lock); @@ -936,14 +900,8 @@ unlink: mutex_lock(&kvm->lock); kvm->vcpus[n] = NULL; mutex_unlock(&kvm->lock); - -mmu_unload: - vcpu_load(vcpu); - kvm_mmu_unload(vcpu); - vcpu_put(vcpu); - -free_vcpu: - kvm_x86_ops->vcpu_free(vcpu); +vcpu_destroy: + kvm_arch_vcpu_destory(vcpu); return r; } @@ -1281,41 +1239,6 @@ static struct miscdevice kvm_dev = { &kvm_chardev_ops, }; -/* - * Make sure that a cpu that is being hot-unplugged does not have any vcpus - * cached on it. - */ -static void decache_vcpus_on_cpu(int cpu) -{ - struct kvm *vm; - struct kvm_vcpu *vcpu; - int i; - - spin_lock(&kvm_lock); - list_for_each_entry(vm, &vm_list, vm_list) - for (i = 0; i < KVM_MAX_VCPUS; ++i) { - vcpu = vm->vcpus[i]; - if (!vcpu) - continue; - /* - * If the vcpu is locked, then it is running on some - * other cpu and therefore it is not cached on the - * cpu in question. - * - * If it's not locked, check the last cpu it executed - * on. - */ - if (mutex_trylock(&vcpu->mutex)) { - if (vcpu->cpu == cpu) { - kvm_x86_ops->vcpu_decache(vcpu); - vcpu->cpu = -1; - } - mutex_unlock(&vcpu->mutex); - } - } - spin_unlock(&kvm_lock); -} - static void hardware_enable(void *junk) { int cpu = raw_smp_processor_id(); @@ -1323,7 +1246,7 @@ static void hardware_enable(void *junk) if (cpu_isset(cpu, cpus_hardware_enabled)) return; cpu_set(cpu, cpus_hardware_enabled); - kvm_x86_ops->hardware_enable(NULL); + kvm_arch_hardware_enable(NULL); } static void hardware_disable(void *junk) @@ -1334,7 +1257,7 @@ static void hardware_disable(void *junk) return; cpu_clear(cpu, cpus_hardware_enabled); decache_vcpus_on_cpu(cpu); - kvm_x86_ops->hardware_disable(NULL); + kvm_arch_hardware_disable(NULL); } static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, @@ -1500,7 +1423,7 @@ static void kvm_sched_in(struct preempt_notifier *pn, int cpu) { struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); - kvm_x86_ops->vcpu_load(vcpu, cpu); + kvm_arch_vcpu_load(vcpu, cpu); } static void kvm_sched_out(struct preempt_notifier *pn, @@ -1508,7 +1431,7 @@ static void kvm_sched_out(struct preempt_notifier *pn, { struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); - kvm_x86_ops->vcpu_put(vcpu); + kvm_arch_vcpu_put(vcpu); } int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, @@ -1533,13 +1456,13 @@ int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, kvm_x86_ops = ops; - r = kvm_x86_ops->hardware_setup(); + r = kvm_arch_hardware_setup(); if (r < 0) goto out; for_each_online_cpu(cpu) { smp_call_function_single(cpu, - kvm_x86_ops->check_processor_compatibility, + kvm_arch_check_processor_compat, &r, 0, 1); if (r < 0) goto out_free_0; @@ -1594,7 +1517,7 @@ out_free_2: out_free_1: on_each_cpu(hardware_disable, NULL, 0, 1); out_free_0: - kvm_x86_ops->hardware_unsetup(); + kvm_arch_hardware_unsetup(); out: kvm_x86_ops = NULL; return r; @@ -1610,7 +1533,7 @@ void kvm_exit_x86(void) unregister_reboot_notifier(&kvm_reboot_notifier); unregister_cpu_notifier(&kvm_cpu_notifier); on_each_cpu(hardware_disable, NULL, 0, 1); - kvm_x86_ops->hardware_unsetup(); + kvm_arch_hardware_unsetup(); kvm_x86_ops = NULL; } EXPORT_SYMBOL_GPL(kvm_exit_x86); diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 2edc53ec8b3b..4902b35060f5 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -564,6 +564,41 @@ out: return r; } +/* + * Make sure that a cpu that is being hot-unplugged does not have any vcpus + * cached on it. + */ +void decache_vcpus_on_cpu(int cpu) +{ + struct kvm *vm; + struct kvm_vcpu *vcpu; + int i; + + spin_lock(&kvm_lock); + list_for_each_entry(vm, &vm_list, vm_list) + for (i = 0; i < KVM_MAX_VCPUS; ++i) { + vcpu = vm->vcpus[i]; + if (!vcpu) + continue; + /* + * If the vcpu is locked, then it is running on some + * other cpu and therefore it is not cached on the + * cpu in question. + * + * If it's not locked, check the last cpu it executed + * on. + */ + if (mutex_trylock(&vcpu->mutex)) { + if (vcpu->cpu == cpu) { + kvm_x86_ops->vcpu_decache(vcpu); + vcpu->cpu = -1; + } + mutex_unlock(&vcpu->mutex); + } + } + spin_unlock(&kvm_lock); +} + long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -2319,3 +2354,125 @@ void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) fx_restore(&vcpu->host_fx_image); } EXPORT_SYMBOL_GPL(kvm_put_guest_fpu); + +void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) +{ + kvm_x86_ops->vcpu_free(vcpu); +} + +struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, + unsigned int id) +{ + int r; + struct kvm_vcpu *vcpu = kvm_x86_ops->vcpu_create(kvm, id); + + if (IS_ERR(vcpu)) { + r = -ENOMEM; + goto fail; + } + + /* We do fxsave: this must be aligned. */ + BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF); + + vcpu_load(vcpu); + r = kvm_arch_vcpu_reset(vcpu); + if (r == 0) + r = kvm_mmu_setup(vcpu); + vcpu_put(vcpu); + if (r < 0) + goto free_vcpu; + + return vcpu; +free_vcpu: + kvm_x86_ops->vcpu_free(vcpu); +fail: + return ERR_PTR(r); +} + +void kvm_arch_vcpu_destory(struct kvm_vcpu *vcpu) +{ + vcpu_load(vcpu); + kvm_mmu_unload(vcpu); + vcpu_put(vcpu); + + kvm_x86_ops->vcpu_free(vcpu); +} + +int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu) +{ + return kvm_x86_ops->vcpu_reset(vcpu); +} + +void kvm_arch_hardware_enable(void *garbage) +{ + kvm_x86_ops->hardware_enable(garbage); +} + +void kvm_arch_hardware_disable(void *garbage) +{ + kvm_x86_ops->hardware_disable(garbage); +} + +int kvm_arch_hardware_setup(void) +{ + return kvm_x86_ops->hardware_setup(); +} + +void kvm_arch_hardware_unsetup(void) +{ + kvm_x86_ops->hardware_unsetup(); +} + +void kvm_arch_check_processor_compat(void *rtn) +{ + kvm_x86_ops->check_processor_compatibility(rtn); +} + +int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) +{ + struct page *page; + struct kvm *kvm; + int r; + + BUG_ON(vcpu->kvm == NULL); + kvm = vcpu->kvm; + + vcpu->mmu.root_hpa = INVALID_PAGE; + if (!irqchip_in_kernel(kvm) || vcpu->vcpu_id == 0) + vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; + else + vcpu->mp_state = VCPU_MP_STATE_UNINITIALIZED; + + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) { + r = -ENOMEM; + goto fail; + } + vcpu->pio_data = page_address(page); + + r = kvm_mmu_create(vcpu); + if (r < 0) + goto fail_free_pio_data; + + if (irqchip_in_kernel(kvm)) { + r = kvm_create_lapic(vcpu); + if (r < 0) + goto fail_mmu_destroy; + } + + return 0; + +fail_mmu_destroy: + kvm_mmu_destroy(vcpu); +fail_free_pio_data: + free_page((unsigned long)vcpu->pio_data); +fail: + return r; +} + +void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) +{ + kvm_free_lapic(vcpu); + kvm_mmu_destroy(vcpu); + free_page((unsigned long)vcpu->pio_data); +} diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index ec32c26a5118..4df064100226 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -19,6 +19,9 @@ #include #include +extern spinlock_t kvm_lock; +extern struct list_head vm_list; + struct kvm_vcpu { KVM_VCPU_COMM; u64 host_tsc; From cb498ea2ce1d3f3c0bc0a2522241dca10263e437 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Wed, 14 Nov 2007 20:39:31 +0800 Subject: [PATCH 100/249] KVM: Portability: Combine kvm_init and kvm_init_x86 Will be called once arch module registers itself. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 4 +-- drivers/kvm/kvm_main.c | 61 +++++++++++++++--------------------------- drivers/kvm/svm.c | 4 +-- drivers/kvm/vmx.c | 4 +-- 4 files changed, 28 insertions(+), 45 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index bca07c6d0460..5e7be1504664 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -495,9 +495,9 @@ void vcpu_put(struct kvm_vcpu *vcpu); void decache_vcpus_on_cpu(int cpu); -int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, +int kvm_init(struct kvm_x86_ops *ops, unsigned int vcpu_size, struct module *module); -void kvm_exit_x86(void); +void kvm_exit(void); int kvm_mmu_module_init(void); void kvm_mmu_module_exit(void); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 7fdfed52dbe7..2b3736e7483c 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1434,12 +1434,27 @@ static void kvm_sched_out(struct preempt_notifier *pn, kvm_arch_vcpu_put(vcpu); } -int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, +int kvm_init(struct kvm_x86_ops *ops, unsigned int vcpu_size, struct module *module) { int r; int cpu; + r = kvm_mmu_module_init(); + if (r) + goto out4; + + kvm_init_debug(); + + kvm_arch_init(); + + bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO); + + if (bad_page == NULL) { + r = -ENOMEM; + goto out; + } + if (kvm_x86_ops) { printk(KERN_ERR "kvm: already loaded the other module\n"); return -EEXIST; @@ -1520,11 +1535,14 @@ out_free_0: kvm_arch_hardware_unsetup(); out: kvm_x86_ops = NULL; + kvm_exit_debug(); + kvm_mmu_module_exit(); +out4: return r; } -EXPORT_SYMBOL_GPL(kvm_init_x86); +EXPORT_SYMBOL_GPL(kvm_init); -void kvm_exit_x86(void) +void kvm_exit(void) { misc_deregister(&kvm_dev); kmem_cache_destroy(kvm_vcpu_cache); @@ -1535,43 +1553,8 @@ void kvm_exit_x86(void) on_each_cpu(hardware_disable, NULL, 0, 1); kvm_arch_hardware_unsetup(); kvm_x86_ops = NULL; -} -EXPORT_SYMBOL_GPL(kvm_exit_x86); - -static __init int kvm_init(void) -{ - int r; - - r = kvm_mmu_module_init(); - if (r) - goto out4; - - kvm_init_debug(); - - kvm_arch_init(); - - bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO); - - if (bad_page == NULL) { - r = -ENOMEM; - goto out; - } - - return 0; - -out: - kvm_exit_debug(); - kvm_mmu_module_exit(); -out4: - return r; -} - -static __exit void kvm_exit(void) -{ kvm_exit_debug(); __free_page(bad_page); kvm_mmu_module_exit(); } - -module_init(kvm_init) -module_exit(kvm_exit) +EXPORT_SYMBOL_GPL(kvm_exit); diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 97863f8fd001..0ccbef18476a 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -1723,13 +1723,13 @@ static struct kvm_x86_ops svm_x86_ops = { static int __init svm_init(void) { - return kvm_init_x86(&svm_x86_ops, sizeof(struct vcpu_svm), + return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm), THIS_MODULE); } static void __exit svm_exit(void) { - kvm_exit_x86(); + kvm_exit(); } module_init(svm_init) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index c3bde7532714..529162345c12 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -2677,7 +2677,7 @@ static int __init vmx_init(void) memset(iova, 0xff, PAGE_SIZE); kunmap(vmx_io_bitmap_b); - r = kvm_init_x86(&vmx_x86_ops, sizeof(struct vcpu_vmx), THIS_MODULE); + r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx), THIS_MODULE); if (r) goto out1; @@ -2698,7 +2698,7 @@ static void __exit vmx_exit(void) __free_page(vmx_io_bitmap_b); __free_page(vmx_io_bitmap_a); - kvm_exit_x86(); + kvm_exit(); } module_init(vmx_init) From f8c16bbaa9e14b309ffcf29cac0ea377a35b0dd2 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Wed, 14 Nov 2007 20:40:21 +0800 Subject: [PATCH 101/249] KVM: Portability: Move x86 specific code from kvm_init() to kvm_arch() Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 5 +++-- drivers/kvm/kvm_main.c | 26 ++++++-------------------- drivers/kvm/x86.c | 27 ++++++++++++++++++++++++++- 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 5e7be1504664..96d9c7da14c2 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -495,7 +495,7 @@ void vcpu_put(struct kvm_vcpu *vcpu); void decache_vcpus_on_cpu(int cpu); -int kvm_init(struct kvm_x86_ops *ops, unsigned int vcpu_size, +int kvm_init(void *opaque, unsigned int vcpu_size, struct module *module); void kvm_exit(void); @@ -649,7 +649,8 @@ int kvm_arch_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg); int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run); -__init void kvm_arch_init(void); +int kvm_arch_init(void *opaque); +void kvm_arch_exit(void); int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 2b3736e7483c..e0f20d020281 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1434,7 +1434,7 @@ static void kvm_sched_out(struct preempt_notifier *pn, kvm_arch_vcpu_put(vcpu); } -int kvm_init(struct kvm_x86_ops *ops, unsigned int vcpu_size, +int kvm_init(void *opaque, unsigned int vcpu_size, struct module *module) { int r; @@ -1446,7 +1446,9 @@ int kvm_init(struct kvm_x86_ops *ops, unsigned int vcpu_size, kvm_init_debug(); - kvm_arch_init(); + r = kvm_arch_init(opaque); + if (r) + goto out4; bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO); @@ -1455,22 +1457,6 @@ int kvm_init(struct kvm_x86_ops *ops, unsigned int vcpu_size, goto out; } - if (kvm_x86_ops) { - printk(KERN_ERR "kvm: already loaded the other module\n"); - return -EEXIST; - } - - if (!ops->cpu_has_kvm_support()) { - printk(KERN_ERR "kvm: no hardware support\n"); - return -EOPNOTSUPP; - } - if (ops->disabled_by_bios()) { - printk(KERN_ERR "kvm: disabled by bios\n"); - return -EOPNOTSUPP; - } - - kvm_x86_ops = ops; - r = kvm_arch_hardware_setup(); if (r < 0) goto out; @@ -1534,7 +1520,7 @@ out_free_1: out_free_0: kvm_arch_hardware_unsetup(); out: - kvm_x86_ops = NULL; + kvm_arch_exit(); kvm_exit_debug(); kvm_mmu_module_exit(); out4: @@ -1552,7 +1538,7 @@ void kvm_exit(void) unregister_cpu_notifier(&kvm_cpu_notifier); on_each_cpu(hardware_disable, NULL, 0, 1); kvm_arch_hardware_unsetup(); - kvm_x86_ops = NULL; + kvm_arch_exit(); kvm_exit_debug(); __free_page(bad_page); kvm_mmu_module_exit(); diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 4902b35060f5..bbfa810bf8bd 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1645,11 +1645,36 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, } EXPORT_SYMBOL_GPL(kvm_emulate_pio_string); -__init void kvm_arch_init(void) +int kvm_arch_init(void *opaque) { + struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque; + kvm_init_msr_list(); + + if (kvm_x86_ops) { + printk(KERN_ERR "kvm: already loaded the other module\n"); + return -EEXIST; + } + + if (!ops->cpu_has_kvm_support()) { + printk(KERN_ERR "kvm: no hardware support\n"); + return -EOPNOTSUPP; + } + if (ops->disabled_by_bios()) { + printk(KERN_ERR "kvm: disabled by bios\n"); + return -EOPNOTSUPP; + } + + kvm_x86_ops = ops; + + return 0; } +void kvm_arch_exit(void) +{ + kvm_x86_ops = NULL; + } + int kvm_emulate_halt(struct kvm_vcpu *vcpu) { ++vcpu->stat.halt_exits; From a7e6c88a78b7c3d67491bf90b2b95e66437dbdb6 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Thu, 15 Nov 2007 14:52:28 +0800 Subject: [PATCH 102/249] KVM: x86 emulator: modify 'lods', and 'stos' not to depend on CR2 The current 'lods' and 'stos' is depending on incoming CR2 rather than decode memory address from registers. Signed-off-by: Sheng Yang Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index e6979475bee7..8e2162fc6f70 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1528,7 +1528,9 @@ special_insn: case 0xaa ... 0xab: /* stos */ c->dst.type = OP_MEM; c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; - c->dst.ptr = (unsigned long *)cr2; + c->dst.ptr = (unsigned long *)register_address( + ctxt->es_base, + c->regs[VCPU_REGS_RDI]); c->dst.val = c->regs[VCPU_REGS_RAX]; register_address_increment(c->regs[VCPU_REGS_RDI], (ctxt->eflags & EFLG_DF) ? -c->dst.bytes @@ -1538,9 +1540,13 @@ special_insn: c->dst.type = OP_REG; c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; - if ((rc = ops->read_emulated(cr2, &c->dst.val, - c->dst.bytes, - ctxt->vcpu)) != 0) + if ((rc = ops->read_emulated(register_address( + c->override_base ? *c->override_base : + ctxt->ds_base, + c->regs[VCPU_REGS_RSI]), + &c->dst.val, + c->dst.bytes, + ctxt->vcpu)) != 0) goto done; register_address_increment(c->regs[VCPU_REGS_RSI], (ctxt->eflags & EFLG_DF) ? -c->dst.bytes From 018d00d2fef27fc65f2917bec8f72b93615c18e1 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Thu, 15 Nov 2007 23:07:47 +0800 Subject: [PATCH 103/249] KVM: Portability: move KVM_CHECK_EXTENSION Make KVM_CHECK_EXTENSION code into a function, all archs can define its capability independently. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 +++ drivers/kvm/kvm_main.c | 18 ++---------------- drivers/kvm/x86.c | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 96d9c7da14c2..a7be0733bc48 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -628,6 +628,9 @@ long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); + +int kvm_dev_ioctl_check_extension(long ext); + int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, struct kvm_userspace_memory_region *mem, diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index e0f20d020281..c782f27265de 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1198,23 +1198,9 @@ static long kvm_dev_ioctl(struct file *filp, goto out; r = kvm_dev_ioctl_create_vm(); break; - case KVM_CHECK_EXTENSION: { - int ext = (long)argp; - - switch (ext) { - case KVM_CAP_IRQCHIP: - case KVM_CAP_HLT: - case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: - case KVM_CAP_USER_MEMORY: - case KVM_CAP_SET_TSS_ADDR: - r = 1; - break; - default: - r = 0; - break; - } + case KVM_CHECK_EXTENSION: + r = kvm_dev_ioctl_check_extension((long)argp); break; - } case KVM_GET_VCPU_MMAP_SIZE: r = -EINVAL; if (arg) diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index bbfa810bf8bd..609792637fb4 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -599,6 +599,26 @@ void decache_vcpus_on_cpu(int cpu) spin_unlock(&kvm_lock); } +int kvm_dev_ioctl_check_extension(long ext) +{ + int r; + + switch (ext) { + case KVM_CAP_IRQCHIP: + case KVM_CAP_HLT: + case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: + case KVM_CAP_USER_MEMORY: + case KVM_CAP_SET_TSS_ADDR: + r = 1; + break; + default: + r = 0; + break; + } + return r; + +} + long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { From e08aa78ae5747c6e1dc525e8a40f23c2ea61d757 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 15 Nov 2007 18:06:18 +0200 Subject: [PATCH 104/249] KVM: VMX: Consolidate register usage in vmx_vcpu_run() We pass vcpu, vmx->fail, and vmx->launched to assembly code, but all three are fields within vmx. Consolidate by only passing in vmx and offsets for the rest. Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 146 +++++++++++++++++++++++----------------------- 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 529162345c12..84c77fed9d17 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -2350,36 +2350,36 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) #endif ASM_VMX_VMWRITE_RSP_RDX "\n\t" /* Check if vmlaunch of vmresume is needed */ - "cmp $0, %1 \n\t" + "cmpl $0, %c[launched](%0) \n\t" /* Load guest registers. Don't clobber flags. */ #ifdef CONFIG_X86_64 - "mov %c[cr2](%3), %%rax \n\t" + "mov %c[cr2](%0), %%rax \n\t" "mov %%rax, %%cr2 \n\t" - "mov %c[rax](%3), %%rax \n\t" - "mov %c[rbx](%3), %%rbx \n\t" - "mov %c[rdx](%3), %%rdx \n\t" - "mov %c[rsi](%3), %%rsi \n\t" - "mov %c[rdi](%3), %%rdi \n\t" - "mov %c[rbp](%3), %%rbp \n\t" - "mov %c[r8](%3), %%r8 \n\t" - "mov %c[r9](%3), %%r9 \n\t" - "mov %c[r10](%3), %%r10 \n\t" - "mov %c[r11](%3), %%r11 \n\t" - "mov %c[r12](%3), %%r12 \n\t" - "mov %c[r13](%3), %%r13 \n\t" - "mov %c[r14](%3), %%r14 \n\t" - "mov %c[r15](%3), %%r15 \n\t" - "mov %c[rcx](%3), %%rcx \n\t" /* kills %3 (rcx) */ + "mov %c[rax](%0), %%rax \n\t" + "mov %c[rbx](%0), %%rbx \n\t" + "mov %c[rdx](%0), %%rdx \n\t" + "mov %c[rsi](%0), %%rsi \n\t" + "mov %c[rdi](%0), %%rdi \n\t" + "mov %c[rbp](%0), %%rbp \n\t" + "mov %c[r8](%0), %%r8 \n\t" + "mov %c[r9](%0), %%r9 \n\t" + "mov %c[r10](%0), %%r10 \n\t" + "mov %c[r11](%0), %%r11 \n\t" + "mov %c[r12](%0), %%r12 \n\t" + "mov %c[r13](%0), %%r13 \n\t" + "mov %c[r14](%0), %%r14 \n\t" + "mov %c[r15](%0), %%r15 \n\t" + "mov %c[rcx](%0), %%rcx \n\t" /* kills %0 (rcx) */ #else - "mov %c[cr2](%3), %%eax \n\t" + "mov %c[cr2](%0), %%eax \n\t" "mov %%eax, %%cr2 \n\t" - "mov %c[rax](%3), %%eax \n\t" - "mov %c[rbx](%3), %%ebx \n\t" - "mov %c[rdx](%3), %%edx \n\t" - "mov %c[rsi](%3), %%esi \n\t" - "mov %c[rdi](%3), %%edi \n\t" - "mov %c[rbp](%3), %%ebp \n\t" - "mov %c[rcx](%3), %%ecx \n\t" /* kills %3 (ecx) */ + "mov %c[rax](%0), %%eax \n\t" + "mov %c[rbx](%0), %%ebx \n\t" + "mov %c[rdx](%0), %%edx \n\t" + "mov %c[rsi](%0), %%esi \n\t" + "mov %c[rdi](%0), %%edi \n\t" + "mov %c[rbp](%0), %%ebp \n\t" + "mov %c[rcx](%0), %%ecx \n\t" /* kills %0 (ecx) */ #endif /* Enter guest mode */ "jne .Llaunched \n\t" @@ -2389,62 +2389,62 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) ".Lkvm_vmx_return: " /* Save guest registers, load host registers, keep flags */ #ifdef CONFIG_X86_64 - "xchg %3, (%%rsp) \n\t" - "mov %%rax, %c[rax](%3) \n\t" - "mov %%rbx, %c[rbx](%3) \n\t" - "pushq (%%rsp); popq %c[rcx](%3) \n\t" - "mov %%rdx, %c[rdx](%3) \n\t" - "mov %%rsi, %c[rsi](%3) \n\t" - "mov %%rdi, %c[rdi](%3) \n\t" - "mov %%rbp, %c[rbp](%3) \n\t" - "mov %%r8, %c[r8](%3) \n\t" - "mov %%r9, %c[r9](%3) \n\t" - "mov %%r10, %c[r10](%3) \n\t" - "mov %%r11, %c[r11](%3) \n\t" - "mov %%r12, %c[r12](%3) \n\t" - "mov %%r13, %c[r13](%3) \n\t" - "mov %%r14, %c[r14](%3) \n\t" - "mov %%r15, %c[r15](%3) \n\t" + "xchg %0, (%%rsp) \n\t" + "mov %%rax, %c[rax](%0) \n\t" + "mov %%rbx, %c[rbx](%0) \n\t" + "pushq (%%rsp); popq %c[rcx](%0) \n\t" + "mov %%rdx, %c[rdx](%0) \n\t" + "mov %%rsi, %c[rsi](%0) \n\t" + "mov %%rdi, %c[rdi](%0) \n\t" + "mov %%rbp, %c[rbp](%0) \n\t" + "mov %%r8, %c[r8](%0) \n\t" + "mov %%r9, %c[r9](%0) \n\t" + "mov %%r10, %c[r10](%0) \n\t" + "mov %%r11, %c[r11](%0) \n\t" + "mov %%r12, %c[r12](%0) \n\t" + "mov %%r13, %c[r13](%0) \n\t" + "mov %%r14, %c[r14](%0) \n\t" + "mov %%r15, %c[r15](%0) \n\t" "mov %%cr2, %%rax \n\t" - "mov %%rax, %c[cr2](%3) \n\t" + "mov %%rax, %c[cr2](%0) \n\t" - "pop %%rcx; pop %%rbp; pop %%rdx \n\t" + "pop %%rbp; pop %%rbp; pop %%rdx \n\t" #else - "xchg %3, (%%esp) \n\t" - "mov %%eax, %c[rax](%3) \n\t" - "mov %%ebx, %c[rbx](%3) \n\t" - "pushl (%%esp); popl %c[rcx](%3) \n\t" - "mov %%edx, %c[rdx](%3) \n\t" - "mov %%esi, %c[rsi](%3) \n\t" - "mov %%edi, %c[rdi](%3) \n\t" - "mov %%ebp, %c[rbp](%3) \n\t" + "xchg %0, (%%esp) \n\t" + "mov %%eax, %c[rax](%0) \n\t" + "mov %%ebx, %c[rbx](%0) \n\t" + "pushl (%%esp); popl %c[rcx](%0) \n\t" + "mov %%edx, %c[rdx](%0) \n\t" + "mov %%esi, %c[rsi](%0) \n\t" + "mov %%edi, %c[rdi](%0) \n\t" + "mov %%ebp, %c[rbp](%0) \n\t" "mov %%cr2, %%eax \n\t" - "mov %%eax, %c[cr2](%3) \n\t" + "mov %%eax, %c[cr2](%0) \n\t" - "pop %%ecx; pop %%ebp; pop %%edx \n\t" + "pop %%ebp; pop %%ebp; pop %%edx \n\t" #endif - "setbe %0 \n\t" - : "=q" (vmx->fail) - : "r"(vmx->launched), "d"((unsigned long)HOST_RSP), - "c"(vcpu), - [rax]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RAX])), - [rbx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RBX])), - [rcx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RCX])), - [rdx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RDX])), - [rsi]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RSI])), - [rdi]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RDI])), - [rbp]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RBP])), + "setbe %c[fail](%0) \n\t" + : : "c"(vmx), "d"((unsigned long)HOST_RSP), + [launched]"i"(offsetof(struct vcpu_vmx, launched)), + [fail]"i"(offsetof(struct vcpu_vmx, fail)), + [rax]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RAX])), + [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RBX])), + [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RCX])), + [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RDX])), + [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RSI])), + [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RDI])), + [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RBP])), #ifdef CONFIG_X86_64 - [r8]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R8])), - [r9]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R9])), - [r10]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R10])), - [r11]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R11])), - [r12]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R12])), - [r13]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R13])), - [r14]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R14])), - [r15]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R15])), + [r8]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R8])), + [r9]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R9])), + [r10]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R10])), + [r11]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R11])), + [r12]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R12])), + [r13]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R13])), + [r14]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R14])), + [r15]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R15])), #endif - [cr2]"i"(offsetof(struct kvm_vcpu, cr2)) + [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.cr2)) : "cc", "memory" #ifdef CONFIG_X86_64 , "rbx", "rdi", "rsi" From 8b0067913d9d0439d4cf3c1f7314ba71fc7b2958 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 16 Nov 2007 13:05:55 +0800 Subject: [PATCH 105/249] KVM: Portability: Make kvm_vcpu_ioctl_translate arch dependent Move kvm_vcpu_ioctl_translate to arch, since mmu would be put under arch. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 +++ drivers/kvm/kvm_main.c | 24 +----------------------- drivers/kvm/x86.c | 22 ++++++++++++++++++++++ 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index a7be0733bc48..e34e246eafec 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -642,6 +642,9 @@ void kvm_arch_destroy_vm(struct kvm *kvm); int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); +int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, + struct kvm_translation *tr); + int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index c782f27265de..3946025f3d37 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -761,28 +761,6 @@ void kvm_resched(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_resched); -/* - * Translate a guest virtual address to a guest physical address. - */ -static int kvm_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, - struct kvm_translation *tr) -{ - unsigned long vaddr = tr->linear_address; - gpa_t gpa; - - vcpu_load(vcpu); - mutex_lock(&vcpu->kvm->lock); - gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr); - tr->physical_address = gpa; - tr->valid = gpa != UNMAPPED_GVA; - tr->writeable = 1; - tr->usermode = 0; - mutex_unlock(&vcpu->kvm->lock); - vcpu_put(vcpu); - - return 0; -} - static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq) { @@ -986,7 +964,7 @@ static long kvm_vcpu_ioctl(struct file *filp, r = -EFAULT; if (copy_from_user(&tr, argp, sizeof tr)) goto out; - r = kvm_vcpu_ioctl_translate(vcpu, &tr); + r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); if (r) goto out; r = -EFAULT; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 609792637fb4..f1746afab9e5 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -2318,6 +2318,28 @@ struct fxsave { #endif }; +/* + * Translate a guest virtual address to a guest physical address. + */ +int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, + struct kvm_translation *tr) +{ + unsigned long vaddr = tr->linear_address; + gpa_t gpa; + + vcpu_load(vcpu); + mutex_lock(&vcpu->kvm->lock); + gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr); + tr->physical_address = gpa; + tr->valid = gpa != UNMAPPED_GVA; + tr->writeable = 1; + tr->usermode = 0; + mutex_unlock(&vcpu->kvm->lock); + vcpu_put(vcpu); + + return 0; +} + int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) { struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image; From 56919c5c97aa07b150311c5c9802af07872ee2b0 Mon Sep 17 00:00:00 2001 From: Joe Perches Date: Mon, 12 Nov 2007 20:06:51 -0800 Subject: [PATCH 106/249] KVM: Remove ptr comparisons to 0 Fix sparse warnings "Using plain integer as NULL pointer" Signed-off-by: Joe Perches Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 2 +- drivers/kvm/kvm_main.c | 3 ++- drivers/kvm/svm.c | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index e34e246eafec..c4ad66bb5e08 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -398,7 +398,7 @@ static inline struct kvm_ioapic *ioapic_irqchip(struct kvm *kvm) static inline int irqchip_in_kernel(struct kvm *kvm) { - return pic_irqchip(kvm) != 0; + return pic_irqchip(kvm) != NULL; } struct descriptor_table { diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 3946025f3d37..7335435f528e 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1449,7 +1449,8 @@ int kvm_init(void *opaque, unsigned int vcpu_size, /* A kmem cache lets us meet the alignment requirements of fx_save. */ kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, - __alignof__(struct kvm_vcpu), 0, 0); + __alignof__(struct kvm_vcpu), + 0, NULL); if (!kvm_vcpu_cache) { r = -ENOMEM; goto out_free_4; diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 0ccbef18476a..a1a7f39ff3f1 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -1271,7 +1271,7 @@ static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) exit_code); if (exit_code >= ARRAY_SIZE(svm_exit_handlers) - || svm_exit_handlers[exit_code] == 0) { + || !svm_exit_handlers[exit_code]) { kvm_run->exit_reason = KVM_EXIT_UNKNOWN; kvm_run->hw.hardware_exit_reason = exit_code; return 0; From a16b043cc96db4a01abb337bef4a51cebcfcbb1b Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 16 Nov 2007 14:38:21 +0800 Subject: [PATCH 107/249] KVM: Remove __init attributes for kvm_init_debug and kvm_init_msr_list Since their callers are not declared with __init. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 2 +- drivers/kvm/x86.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 7335435f528e..e24489b409cf 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1332,7 +1332,7 @@ static u64 stat_get(void *_offset) DEFINE_SIMPLE_ATTRIBUTE(stat_fops, stat_get, NULL, "%llu\n"); -static __init void kvm_init_debug(void) +static void kvm_init_debug(void) { struct kvm_stats_debugfs_item *p; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index f1746afab9e5..abb7beeb8f54 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1049,7 +1049,7 @@ out: return r; } -static __init void kvm_init_msr_list(void) +static void kvm_init_msr_list(void) { u32 dummy[2]; unsigned i, j; From d19a9cd275b0fdc793d1bb8b644b7aad0517e4bc Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Sun, 18 Nov 2007 18:43:45 +0800 Subject: [PATCH 108/249] KVM: Portability: Add two hooks to handle kvm_create and destroy vm Add two arch hooks to handle kvm_create_vm and kvm destroy_vm. Now, just put io_bus init and destory in common. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 4 ++++ drivers/kvm/kvm_main.c | 42 ++++++------------------------------- drivers/kvm/x86.c | 47 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 36 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index c4ad66bb5e08..59e001c3ff09 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -674,6 +674,10 @@ int kvm_arch_hardware_setup(void); void kvm_arch_hardware_unsetup(void); void kvm_arch_check_processor_compat(void *rtn); +void kvm_free_physmem(struct kvm *kvm); + +struct kvm *kvm_arch_create_vm(void); +void kvm_arch_destroy_vm(struct kvm *kvm); static inline void kvm_guest_enter(void) { diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index e24489b409cf..bde3cf741892 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -156,18 +156,18 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); static struct kvm *kvm_create_vm(void) { - struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL); + struct kvm *kvm = kvm_arch_create_vm(); - if (!kvm) - return ERR_PTR(-ENOMEM); + if (IS_ERR(kvm)) + goto out; kvm_io_bus_init(&kvm->pio_bus); mutex_init(&kvm->lock); - INIT_LIST_HEAD(&kvm->active_mmu_pages); kvm_io_bus_init(&kvm->mmio_bus); spin_lock(&kvm_lock); list_add(&kvm->vm_list, &vm_list); spin_unlock(&kvm_lock); +out: return kvm; } @@ -188,7 +188,7 @@ static void kvm_free_physmem_slot(struct kvm_memory_slot *free, free->rmap = NULL; } -static void kvm_free_physmem(struct kvm *kvm) +void kvm_free_physmem(struct kvm *kvm) { int i; @@ -196,32 +196,6 @@ static void kvm_free_physmem(struct kvm *kvm) kvm_free_physmem_slot(&kvm->memslots[i], NULL); } -static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) -{ - vcpu_load(vcpu); - kvm_mmu_unload(vcpu); - vcpu_put(vcpu); -} - -static void kvm_free_vcpus(struct kvm *kvm) -{ - unsigned int i; - - /* - * Unpin any mmu pages first. - */ - for (i = 0; i < KVM_MAX_VCPUS; ++i) - if (kvm->vcpus[i]) - kvm_unload_vcpu_mmu(kvm->vcpus[i]); - for (i = 0; i < KVM_MAX_VCPUS; ++i) { - if (kvm->vcpus[i]) { - kvm_arch_vcpu_free(kvm->vcpus[i]); - kvm->vcpus[i] = NULL; - } - } - -} - static void kvm_destroy_vm(struct kvm *kvm) { spin_lock(&kvm_lock); @@ -229,11 +203,7 @@ static void kvm_destroy_vm(struct kvm *kvm) spin_unlock(&kvm_lock); kvm_io_bus_destroy(&kvm->pio_bus); kvm_io_bus_destroy(&kvm->mmio_bus); - kfree(kvm->vpic); - kfree(kvm->vioapic); - kvm_free_vcpus(kvm); - kvm_free_physmem(kvm); - kfree(kvm); + kvm_arch_destroy_vm(kvm); } static int kvm_vm_release(struct inode *inode, struct file *filp) diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index abb7beeb8f54..b7c72ac36735 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -2543,3 +2543,50 @@ void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) kvm_mmu_destroy(vcpu); free_page((unsigned long)vcpu->pio_data); } + +struct kvm *kvm_arch_create_vm(void) +{ + struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL); + + if (!kvm) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&kvm->active_mmu_pages); + + return kvm; +} + +static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) +{ + vcpu_load(vcpu); + kvm_mmu_unload(vcpu); + vcpu_put(vcpu); +} + +static void kvm_free_vcpus(struct kvm *kvm) +{ + unsigned int i; + + /* + * Unpin any mmu pages first. + */ + for (i = 0; i < KVM_MAX_VCPUS; ++i) + if (kvm->vcpus[i]) + kvm_unload_vcpu_mmu(kvm->vcpus[i]); + for (i = 0; i < KVM_MAX_VCPUS; ++i) { + if (kvm->vcpus[i]) { + kvm_arch_vcpu_free(kvm->vcpus[i]); + kvm->vcpus[i] = NULL; + } + } + +} + +void kvm_arch_destroy_vm(struct kvm *kvm) +{ + kfree(kvm->vpic); + kfree(kvm->vioapic); + kvm_free_vcpus(kvm); + kvm_free_physmem(kvm); + kfree(kvm); +} From e1beb1d37c3187aa12c6463bd15ba594e9986761 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 18 Nov 2007 13:50:24 +0200 Subject: [PATCH 109/249] KVM: Replace 'light_exits' stat with 'host_state_reload' This is a little more accurate (since it counts actual reloads, not potential reloads), and reverses the sense of the statistic to measure a bad event like most of the other stats (e.g. we want to minimize all counters). Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 2 +- drivers/kvm/svm.c | 1 + drivers/kvm/vmx.c | 1 + drivers/kvm/x86.c | 6 ++---- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 59e001c3ff09..04efe8833e09 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -246,7 +246,7 @@ struct kvm_stat { u32 halt_wakeup; u32 request_irq_exits; u32 irq_exits; - u32 light_exits; + u32 host_state_reload; u32 efer_reload; }; diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index a1a7f39ff3f1..0d32304944fc 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -654,6 +654,7 @@ static void svm_vcpu_put(struct kvm_vcpu *vcpu) struct vcpu_svm *svm = to_svm(vcpu); int i; + ++vcpu->stat.host_state_reload; for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 84c77fed9d17..7130f315afdf 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -463,6 +463,7 @@ static void vmx_load_host_state(struct vcpu_vmx *vmx) if (!vmx->host_state.loaded) return; + ++vmx->vcpu.stat.host_state_reload; vmx->host_state.loaded = 0; if (vmx->host_state.fs_reload_needed) load_fs(vmx->host_state.fs_sel); diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index b7c72ac36735..923dfd4e0afd 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -60,7 +60,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { { "halt_wakeup", STAT_OFFSET(halt_wakeup) }, { "request_irq", STAT_OFFSET(request_irq_exits) }, { "irq_exits", STAT_OFFSET(irq_exits) }, - { "light_exits", STAT_OFFSET(light_exits) }, + { "host_state_reload", STAT_OFFSET(host_state_reload) }, { "efer_reload", STAT_OFFSET(efer_reload) }, { NULL } }; @@ -1988,10 +1988,8 @@ again: ++vcpu->stat.request_irq_exits; goto out; } - if (!need_resched()) { - ++vcpu->stat.light_exits; + if (!need_resched()) goto again; - } } out: From f096ed8588b53bf802b84862c6c4d00d25d1ed72 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 18 Nov 2007 13:54:33 +0200 Subject: [PATCH 110/249] KVM: Add fpu_reload counter Measure the number of times we switch the fpu state. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/x86.c | 2 ++ 2 files changed, 3 insertions(+) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 04efe8833e09..a85c5903591f 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -248,6 +248,7 @@ struct kvm_stat { u32 irq_exits; u32 host_state_reload; u32 efer_reload; + u32 fpu_reload; }; struct kvm_io_device { diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 923dfd4e0afd..c1211e125b43 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -62,6 +62,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { { "irq_exits", STAT_OFFSET(irq_exits) }, { "host_state_reload", STAT_OFFSET(host_state_reload) }, { "efer_reload", STAT_OFFSET(efer_reload) }, + { "fpu_reload", STAT_OFFSET(fpu_reload) }, { NULL } }; @@ -2417,6 +2418,7 @@ void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) vcpu->guest_fpu_loaded = 0; fx_save(&vcpu->guest_fx_image); fx_restore(&vcpu->host_fx_image); + ++vcpu->stat.fpu_reload; } EXPORT_SYMBOL_GPL(kvm_put_guest_fpu); From f2b5756bb3fbdca912c4890e444c18650389d8ae Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 18 Nov 2007 15:17:51 +0200 Subject: [PATCH 111/249] KVM: Add instruction emulation statistics --- drivers/kvm/kvm.h | 2 ++ drivers/kvm/x86.c | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index a85c5903591f..5a8a9af3593a 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -249,6 +249,8 @@ struct kvm_stat { u32 host_state_reload; u32 efer_reload; u32 fpu_reload; + u32 insn_emulation; + u32 insn_emulation_fail; }; struct kvm_io_device { diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index c1211e125b43..a46b95b3651c 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -63,6 +63,8 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { { "host_state_reload", STAT_OFFSET(host_state_reload) }, { "efer_reload", STAT_OFFSET(efer_reload) }, { "fpu_reload", STAT_OFFSET(fpu_reload) }, + { "insn_emulation", STAT_OFFSET(insn_emulation) }, + { "insn_emulation_fail", STAT_OFFSET(insn_emulation_fail) }, { NULL } }; @@ -1381,7 +1383,9 @@ int emulate_instruction(struct kvm_vcpu *vcpu, get_segment_base(vcpu, VCPU_SREG_FS); r = x86_decode_insn(&vcpu->emulate_ctxt, &emulate_ops); + ++vcpu->stat.insn_emulation; if (r) { + ++vcpu->stat.insn_emulation_fail; if (kvm_mmu_unprotect_page_virt(vcpu, cr2)) return EMULATE_DONE; return EMULATE_FAIL; From ba1389b7a04de07e6231693b7ebb34f5b5d1a3e6 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 18 Nov 2007 16:24:12 +0200 Subject: [PATCH 112/249] KVM: Extend stats support for VM stats This is in addition to the current virtual cpu statistics. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 14 ++++++++++++-- drivers/kvm/kvm_main.c | 26 +++++++++++++++++++++++--- drivers/kvm/x86.c | 39 ++++++++++++++++++++------------------- 3 files changed, 55 insertions(+), 24 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 5a8a9af3593a..d3171f9c9c01 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -231,7 +231,7 @@ struct kvm_pio_request { int rep; }; -struct kvm_stat { +struct kvm_vcpu_stat { u32 pf_fixed; u32 pf_guest; u32 tlb_flush; @@ -342,7 +342,7 @@ void kvm_io_bus_register_dev(struct kvm_io_bus *bus, wait_queue_head_t wq; \ int sigset_active; \ sigset_t sigset; \ - struct kvm_stat stat; \ + struct kvm_vcpu_stat stat; \ KVM_VCPU_MMIO struct kvm_mem_alias { @@ -361,6 +361,9 @@ struct kvm_memory_slot { int user_alloc; }; +struct kvm_vm_stat { +}; + struct kvm { struct mutex lock; /* protects everything except vcpus */ int naliases; @@ -387,6 +390,7 @@ struct kvm { int round_robin_prev_vcpu; unsigned int tss_addr; struct page *apic_access_page; + struct kvm_vm_stat stat; }; static inline struct kvm_pic *pic_irqchip(struct kvm *kvm) @@ -809,9 +813,15 @@ static inline u32 get_rdx_init_val(void) #define TSS_REDIRECTION_SIZE (256 / 8) #define RMODE_TSS_SIZE (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) +enum kvm_stat_kind { + KVM_STAT_VM, + KVM_STAT_VCPU, +}; + struct kvm_stats_debugfs_item { const char *name; int offset; + enum kvm_stat_kind kind; struct dentry *dentry; }; extern struct kvm_stats_debugfs_item debugfs_entries[]; diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index bde3cf741892..b0b6ff2a8cbe 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1281,7 +1281,22 @@ static struct notifier_block kvm_cpu_notifier = { .priority = 20, /* must be > scheduler priority */ }; -static u64 stat_get(void *_offset) +static u64 vm_stat_get(void *_offset) +{ + unsigned offset = (long)_offset; + u64 total = 0; + struct kvm *kvm; + + spin_lock(&kvm_lock); + list_for_each_entry(kvm, &vm_list, vm_list) + total += *(u32 *)((void *)kvm + offset); + spin_unlock(&kvm_lock); + return total; +} + +DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n"); + +static u64 vcpu_stat_get(void *_offset) { unsigned offset = (long)_offset; u64 total = 0; @@ -1300,7 +1315,12 @@ static u64 stat_get(void *_offset) return total; } -DEFINE_SIMPLE_ATTRIBUTE(stat_fops, stat_get, NULL, "%llu\n"); +DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n"); + +static struct file_operations *stat_fops[] = { + [KVM_STAT_VCPU] = &vcpu_stat_fops, + [KVM_STAT_VM] = &vm_stat_fops, +}; static void kvm_init_debug(void) { @@ -1310,7 +1330,7 @@ static void kvm_init_debug(void) for (p = debugfs_entries; p->name; ++p) p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir, (void *)(long)p->offset, - &stat_fops); + stat_fops[p->kind]); } static void kvm_exit_debug(void) diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index a46b95b3651c..016abc3357e9 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -42,29 +42,30 @@ #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) #define EFER_RESERVED_BITS 0xfffffffffffff2fe -#define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x) +#define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM +#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU struct kvm_x86_ops *kvm_x86_ops; struct kvm_stats_debugfs_item debugfs_entries[] = { - { "pf_fixed", STAT_OFFSET(pf_fixed) }, - { "pf_guest", STAT_OFFSET(pf_guest) }, - { "tlb_flush", STAT_OFFSET(tlb_flush) }, - { "invlpg", STAT_OFFSET(invlpg) }, - { "exits", STAT_OFFSET(exits) }, - { "io_exits", STAT_OFFSET(io_exits) }, - { "mmio_exits", STAT_OFFSET(mmio_exits) }, - { "signal_exits", STAT_OFFSET(signal_exits) }, - { "irq_window", STAT_OFFSET(irq_window_exits) }, - { "halt_exits", STAT_OFFSET(halt_exits) }, - { "halt_wakeup", STAT_OFFSET(halt_wakeup) }, - { "request_irq", STAT_OFFSET(request_irq_exits) }, - { "irq_exits", STAT_OFFSET(irq_exits) }, - { "host_state_reload", STAT_OFFSET(host_state_reload) }, - { "efer_reload", STAT_OFFSET(efer_reload) }, - { "fpu_reload", STAT_OFFSET(fpu_reload) }, - { "insn_emulation", STAT_OFFSET(insn_emulation) }, - { "insn_emulation_fail", STAT_OFFSET(insn_emulation_fail) }, + { "pf_fixed", VCPU_STAT(pf_fixed) }, + { "pf_guest", VCPU_STAT(pf_guest) }, + { "tlb_flush", VCPU_STAT(tlb_flush) }, + { "invlpg", VCPU_STAT(invlpg) }, + { "exits", VCPU_STAT(exits) }, + { "io_exits", VCPU_STAT(io_exits) }, + { "mmio_exits", VCPU_STAT(mmio_exits) }, + { "signal_exits", VCPU_STAT(signal_exits) }, + { "irq_window", VCPU_STAT(irq_window_exits) }, + { "halt_exits", VCPU_STAT(halt_exits) }, + { "halt_wakeup", VCPU_STAT(halt_wakeup) }, + { "request_irq", VCPU_STAT(request_irq_exits) }, + { "irq_exits", VCPU_STAT(irq_exits) }, + { "host_state_reload", VCPU_STAT(host_state_reload) }, + { "efer_reload", VCPU_STAT(efer_reload) }, + { "fpu_reload", VCPU_STAT(fpu_reload) }, + { "insn_emulation", VCPU_STAT(insn_emulation) }, + { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) }, { NULL } }; From 4cee576493b6abc95cc7447a65f1b9d2b40b8f20 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 18 Nov 2007 16:37:07 +0200 Subject: [PATCH 113/249] KVM: MMU: Add some mmu statistics Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 6 ++++++ drivers/kvm/mmu.c | 9 ++++++++- drivers/kvm/x86.c | 6 ++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index d3171f9c9c01..bdcc44e6b6d7 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -362,6 +362,12 @@ struct kvm_memory_slot { }; struct kvm_vm_stat { + u32 mmu_shadow_zapped; + u32 mmu_pte_write; + u32 mmu_pte_updated; + u32 mmu_pde_zapped; + u32 mmu_flooded; + u32 mmu_recycled; }; struct kvm { diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 9be54a5e858e..87d8e70fe502 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -755,6 +755,7 @@ static void kvm_mmu_zap_page(struct kvm *kvm, { u64 *parent_pte; + ++kvm->stat.mmu_shadow_zapped; while (page->multimapped || page->parent_pte) { if (!page->multimapped) parent_pte = page->parent_pte; @@ -1226,9 +1227,12 @@ static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, const void *new, int bytes, int offset_in_pte) { - if (page->role.level != PT_PAGE_TABLE_LEVEL) + if (page->role.level != PT_PAGE_TABLE_LEVEL) { + ++vcpu->kvm->stat.mmu_pde_zapped; return; + } + ++vcpu->kvm->stat.mmu_pte_updated; if (page->role.glevels == PT32_ROOT_LEVEL) paging32_update_pte(vcpu, page, spte, new, bytes, offset_in_pte); @@ -1263,6 +1267,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, int npte; pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes); + ++vcpu->kvm->stat.mmu_pte_write; kvm_mmu_audit(vcpu, "pre pte write"); if (gfn == vcpu->last_pt_write_gfn && !last_updated_pte_accessed(vcpu)) { @@ -1296,6 +1301,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, pgprintk("misaligned: gpa %llx bytes %d role %x\n", gpa, bytes, page->role.word); kvm_mmu_zap_page(vcpu->kvm, page); + ++vcpu->kvm->stat.mmu_flooded; continue; } page_offset = offset; @@ -1344,6 +1350,7 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) page = container_of(vcpu->kvm->active_mmu_pages.prev, struct kvm_mmu_page, link); kvm_mmu_zap_page(vcpu->kvm, page); + ++vcpu->kvm->stat.mmu_recycled; } } diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 016abc3357e9..fdc7632d7620 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -66,6 +66,12 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { { "fpu_reload", VCPU_STAT(fpu_reload) }, { "insn_emulation", VCPU_STAT(insn_emulation) }, { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) }, + { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) }, + { "mmu_pte_write", VM_STAT(mmu_pte_write) }, + { "mmu_pte_updated", VM_STAT(mmu_pte_updated) }, + { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) }, + { "mmu_flooded", VM_STAT(mmu_flooded) }, + { "mmu_recycled", VM_STAT(mmu_recycled) }, { NULL } }; From 9327fd1195b77e67b86db8c1c1c4c4f7db009b00 Mon Sep 17 00:00:00 2001 From: Amit Shah Date: Thu, 15 Nov 2007 18:38:46 +0200 Subject: [PATCH 114/249] KVM: Make unloading of FPU state when putting vcpu arch-independent Instead of having each architecture do it individually, we do this in the arch-independent code (just x86 as of now). [avi: add svm to the mix, which was added to mainline during the 2.6.24-rc process] Signed-off-by: Amit Shah Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 1 - drivers/kvm/vmx.c | 1 - drivers/kvm/x86.c | 1 + 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 0d32304944fc..fb3721d88dbf 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -659,7 +659,6 @@ static void svm_vcpu_put(struct kvm_vcpu *vcpu) wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); rdtscll(vcpu->host_tsc); - kvm_put_guest_fpu(vcpu); } static void svm_vcpu_decache(struct kvm_vcpu *vcpu) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 7130f315afdf..0c082faaa6db 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -541,7 +541,6 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) static void vmx_vcpu_put(struct kvm_vcpu *vcpu) { vmx_load_host_state(to_vmx(vcpu)); - kvm_put_guest_fpu(vcpu); } static void vmx_fpu_activate(struct kvm_vcpu *vcpu) diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index fdc7632d7620..9618fcba887e 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -678,6 +678,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) { kvm_x86_ops->vcpu_put(vcpu); + kvm_put_guest_fpu(vcpu); } static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) From 5bb064dcdeb7ab341e2f8a3e2fc34faa63b1662c Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Sun, 18 Nov 2007 20:29:43 +0800 Subject: [PATCH 115/249] KVM: Portability: Move kvm_vcpu_ioctl_get_dirty_log to arch-specific file Meanwhile keep the interface in common, and leave as more logic in common as possible. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 5 +++++ drivers/kvm/kvm_main.c | 19 ++++--------------- drivers/kvm/x86.c | 31 +++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index bdcc44e6b6d7..c1aa84f7ca0a 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -644,6 +644,11 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); int kvm_dev_ioctl_check_extension(long ext); +int kvm_get_dirty_log(struct kvm *kvm, + struct kvm_dirty_log *log, int *is_dirty); +int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, + struct kvm_dirty_log *log); + int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, struct kvm_userspace_memory_region *mem, diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index b0b6ff2a8cbe..5f11e6b09458 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -389,19 +389,14 @@ int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, return kvm_set_memory_region(kvm, mem, user_alloc); } -/* - * Get (and clear) the dirty memory log for a memory slot. - */ -static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, - struct kvm_dirty_log *log) +int kvm_get_dirty_log(struct kvm *kvm, + struct kvm_dirty_log *log, int *is_dirty) { struct kvm_memory_slot *memslot; int r, i; int n; unsigned long any = 0; - mutex_lock(&kvm->lock); - r = -EINVAL; if (log->slot >= KVM_MEMORY_SLOTS) goto out; @@ -420,17 +415,11 @@ static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) goto out; - /* If nothing is dirty, don't bother messing with page tables. */ - if (any) { - kvm_mmu_slot_remove_write_access(kvm, log->slot); - kvm_flush_remote_tlbs(kvm); - memset(memslot->dirty_bitmap, 0, n); - } + if (any) + *is_dirty = 1; r = 0; - out: - mutex_unlock(&kvm->lock); return r; } diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 9618fcba887e..935e2769b787 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -937,6 +937,37 @@ static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) return r; } +/* + * Get (and clear) the dirty memory log for a memory slot. + */ +int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, + struct kvm_dirty_log *log) +{ + int r; + int n; + struct kvm_memory_slot *memslot; + int is_dirty = 0; + + mutex_lock(&kvm->lock); + + r = kvm_get_dirty_log(kvm, log, &is_dirty); + if (r) + goto out; + + /* If nothing is dirty, don't bother messing with page tables. */ + if (is_dirty) { + kvm_mmu_slot_remove_write_access(kvm, log->slot); + kvm_flush_remote_tlbs(kvm); + memslot = &kvm->memslots[log->slot]; + n = ALIGN(memslot->npages, BITS_PER_LONG) / 8; + memset(memslot->dirty_bitmap, 0, n); + } + r = 0; +out: + mutex_unlock(&kvm->lock); + return r; +} + long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { From 56c6d28a9afdca0d48dd618276e055f19c0306bb Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Sun, 18 Nov 2007 20:43:21 +0800 Subject: [PATCH 116/249] KVM: Portability: MMU initialization and teardown split Move out kvm_mmu init and exit functionality from kvm_main.c Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 8 -------- drivers/kvm/x86.c | 24 +++++++++++++++++++----- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 5f11e6b09458..595ed323fc53 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1383,10 +1383,6 @@ int kvm_init(void *opaque, unsigned int vcpu_size, int r; int cpu; - r = kvm_mmu_module_init(); - if (r) - goto out4; - kvm_init_debug(); r = kvm_arch_init(opaque); @@ -1446,8 +1442,6 @@ int kvm_init(void *opaque, unsigned int vcpu_size, kvm_preempt_ops.sched_in = kvm_sched_in; kvm_preempt_ops.sched_out = kvm_sched_out; - kvm_mmu_set_nonpresent_ptes(0ull, 0ull); - return 0; out_free: @@ -1466,7 +1460,6 @@ out_free_0: out: kvm_arch_exit(); kvm_exit_debug(); - kvm_mmu_module_exit(); out4: return r; } @@ -1485,6 +1478,5 @@ void kvm_exit(void) kvm_arch_exit(); kvm_exit_debug(); __free_page(bad_page); - kvm_mmu_module_exit(); } EXPORT_SYMBOL_GPL(kvm_exit); diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 935e2769b787..2257a0aab32f 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1711,33 +1711,47 @@ EXPORT_SYMBOL_GPL(kvm_emulate_pio_string); int kvm_arch_init(void *opaque) { + int r; struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque; + r = kvm_mmu_module_init(); + if (r) + goto out_fail; + kvm_init_msr_list(); if (kvm_x86_ops) { printk(KERN_ERR "kvm: already loaded the other module\n"); - return -EEXIST; + r = -EEXIST; + goto out; } if (!ops->cpu_has_kvm_support()) { printk(KERN_ERR "kvm: no hardware support\n"); - return -EOPNOTSUPP; + r = -EOPNOTSUPP; + goto out; } if (ops->disabled_by_bios()) { printk(KERN_ERR "kvm: disabled by bios\n"); - return -EOPNOTSUPP; + r = -EOPNOTSUPP; + goto out; } kvm_x86_ops = ops; - + kvm_mmu_set_nonpresent_ptes(0ull, 0ull); return 0; + +out: + kvm_mmu_module_exit(); +out_fail: + return r; } void kvm_arch_exit(void) { kvm_x86_ops = NULL; - } + kvm_mmu_module_exit(); +} int kvm_emulate_halt(struct kvm_vcpu *vcpu) { From cd6e8f87efb74d91100d84894021dd12fea1a8a5 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Mon, 19 Nov 2007 14:33:37 +0800 Subject: [PATCH 117/249] KVM: Portability: Move some macro definitions from kvm.h to x86.h Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 33 --------------------------------- drivers/kvm/x86.h | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index c1aa84f7ca0a..aceecf4e4f2d 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -20,24 +20,6 @@ #include #include -#define CR3_PAE_RESERVED_BITS ((X86_CR3_PWT | X86_CR3_PCD) - 1) -#define CR3_NONPAE_RESERVED_BITS ((PAGE_SIZE-1) & ~(X86_CR3_PWT | X86_CR3_PCD)) -#define CR3_L_MODE_RESERVED_BITS (CR3_NONPAE_RESERVED_BITS|0xFFFFFF0000000000ULL) - -#define KVM_GUEST_CR0_MASK \ - (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP | X86_CR0_NE \ - | X86_CR0_NW | X86_CR0_CD) -#define KVM_VM_CR0_ALWAYS_ON \ - (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP | X86_CR0_NE | X86_CR0_TS \ - | X86_CR0_MP) -#define KVM_GUEST_CR4_MASK \ - (X86_CR4_VME | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_PGE | X86_CR4_VMXE) -#define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE) -#define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE) - -#define INVALID_PAGE (~(hpa_t)0) -#define UNMAPPED_GVA (~(gpa_t)0) - #define KVM_MAX_VCPUS 4 #define KVM_ALIAS_SLOTS 4 #define KVM_MEMORY_SLOTS 8 @@ -50,21 +32,6 @@ #define KVM_REFILL_PAGES 25 #define KVM_MAX_CPUID_ENTRIES 40 -#define DE_VECTOR 0 -#define UD_VECTOR 6 -#define NM_VECTOR 7 -#define DF_VECTOR 8 -#define TS_VECTOR 10 -#define NP_VECTOR 11 -#define SS_VECTOR 12 -#define GP_VECTOR 13 -#define PF_VECTOR 14 - -#define SELECTOR_TI_MASK (1 << 2) -#define SELECTOR_RPL_MASK 0x03 - -#define IOPL_SHIFT 12 - #define KVM_PIO_PAGE_OFFSET 1 /* diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 4df064100226..ec1d66940c80 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -19,6 +19,39 @@ #include #include +#define CR3_PAE_RESERVED_BITS ((X86_CR3_PWT | X86_CR3_PCD) - 1) +#define CR3_NONPAE_RESERVED_BITS ((PAGE_SIZE-1) & ~(X86_CR3_PWT | X86_CR3_PCD)) +#define CR3_L_MODE_RESERVED_BITS (CR3_NONPAE_RESERVED_BITS|0xFFFFFF0000000000ULL) + +#define KVM_GUEST_CR0_MASK \ + (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP | X86_CR0_NE \ + | X86_CR0_NW | X86_CR0_CD) +#define KVM_VM_CR0_ALWAYS_ON \ + (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP | X86_CR0_NE | X86_CR0_TS \ + | X86_CR0_MP) +#define KVM_GUEST_CR4_MASK \ + (X86_CR4_VME | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_PGE | X86_CR4_VMXE) +#define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE) +#define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE) + +#define INVALID_PAGE (~(hpa_t)0) +#define UNMAPPED_GVA (~(gpa_t)0) + +#define DE_VECTOR 0 +#define UD_VECTOR 6 +#define NM_VECTOR 7 +#define DF_VECTOR 8 +#define TS_VECTOR 10 +#define NP_VECTOR 11 +#define SS_VECTOR 12 +#define GP_VECTOR 13 +#define PF_VECTOR 14 + +#define SELECTOR_TI_MASK (1 << 2) +#define SELECTOR_RPL_MASK 0x03 + +#define IOPL_SHIFT 12 + extern spinlock_t kvm_lock; extern struct list_head vm_list; From ea4a5ff80c24a25034a0c0468f6d1f47d1782076 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Mon, 19 Nov 2007 14:40:47 +0800 Subject: [PATCH 118/249] KVM: Portability: Move struct kvm_x86_ops definition to x86.h Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 69 ----------------------------------------------- drivers/kvm/x86.h | 67 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 69 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index aceecf4e4f2d..e4e1ff7e6f9e 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -386,75 +386,6 @@ struct descriptor_table { unsigned long base; } __attribute__((packed)); -struct kvm_x86_ops { - int (*cpu_has_kvm_support)(void); /* __init */ - int (*disabled_by_bios)(void); /* __init */ - void (*hardware_enable)(void *dummy); /* __init */ - void (*hardware_disable)(void *dummy); - void (*check_processor_compatibility)(void *rtn); - int (*hardware_setup)(void); /* __init */ - void (*hardware_unsetup)(void); /* __exit */ - - /* Create, but do not attach this VCPU */ - struct kvm_vcpu *(*vcpu_create)(struct kvm *kvm, unsigned id); - void (*vcpu_free)(struct kvm_vcpu *vcpu); - int (*vcpu_reset)(struct kvm_vcpu *vcpu); - - void (*prepare_guest_switch)(struct kvm_vcpu *vcpu); - void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); - void (*vcpu_put)(struct kvm_vcpu *vcpu); - void (*vcpu_decache)(struct kvm_vcpu *vcpu); - - int (*set_guest_debug)(struct kvm_vcpu *vcpu, - struct kvm_debug_guest *dbg); - void (*guest_debug_pre)(struct kvm_vcpu *vcpu); - int (*get_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata); - int (*set_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); - u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); - void (*get_segment)(struct kvm_vcpu *vcpu, - struct kvm_segment *var, int seg); - void (*set_segment)(struct kvm_vcpu *vcpu, - struct kvm_segment *var, int seg); - void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); - void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu); - void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); - void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); - void (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); - void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer); - void (*get_idt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); - void (*set_idt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); - void (*get_gdt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); - void (*set_gdt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); - unsigned long (*get_dr)(struct kvm_vcpu *vcpu, int dr); - void (*set_dr)(struct kvm_vcpu *vcpu, int dr, unsigned long value, - int *exception); - void (*cache_regs)(struct kvm_vcpu *vcpu); - void (*decache_regs)(struct kvm_vcpu *vcpu); - unsigned long (*get_rflags)(struct kvm_vcpu *vcpu); - void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); - - void (*tlb_flush)(struct kvm_vcpu *vcpu); - void (*inject_page_fault)(struct kvm_vcpu *vcpu, - unsigned long addr, u32 err_code); - - void (*inject_gp)(struct kvm_vcpu *vcpu, unsigned err_code); - - void (*run)(struct kvm_vcpu *vcpu, struct kvm_run *run); - int (*handle_exit)(struct kvm_run *run, struct kvm_vcpu *vcpu); - void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu); - void (*patch_hypercall)(struct kvm_vcpu *vcpu, - unsigned char *hypercall_addr); - int (*get_irq)(struct kvm_vcpu *vcpu); - void (*set_irq)(struct kvm_vcpu *vcpu, int vec); - void (*inject_pending_irq)(struct kvm_vcpu *vcpu); - void (*inject_pending_vectors)(struct kvm_vcpu *vcpu, - struct kvm_run *run); - - int (*set_tss_addr)(struct kvm *kvm, unsigned int addr); -}; - -extern struct kvm_x86_ops *kvm_x86_ops; - /* The guest did something we don't support. */ #define pr_unimpl(vcpu, fmt, ...) \ do { \ diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index ec1d66940c80..77b4092c0542 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -121,6 +121,73 @@ struct kvm_vcpu { struct x86_emulate_ctxt emulate_ctxt; }; +struct kvm_x86_ops { + int (*cpu_has_kvm_support)(void); /* __init */ + int (*disabled_by_bios)(void); /* __init */ + void (*hardware_enable)(void *dummy); /* __init */ + void (*hardware_disable)(void *dummy); + void (*check_processor_compatibility)(void *rtn); + int (*hardware_setup)(void); /* __init */ + void (*hardware_unsetup)(void); /* __exit */ + + /* Create, but do not attach this VCPU */ + struct kvm_vcpu *(*vcpu_create)(struct kvm *kvm, unsigned id); + void (*vcpu_free)(struct kvm_vcpu *vcpu); + int (*vcpu_reset)(struct kvm_vcpu *vcpu); + + void (*prepare_guest_switch)(struct kvm_vcpu *vcpu); + void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); + void (*vcpu_put)(struct kvm_vcpu *vcpu); + void (*vcpu_decache)(struct kvm_vcpu *vcpu); + + int (*set_guest_debug)(struct kvm_vcpu *vcpu, + struct kvm_debug_guest *dbg); + void (*guest_debug_pre)(struct kvm_vcpu *vcpu); + int (*get_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata); + int (*set_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); + u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); + void (*get_segment)(struct kvm_vcpu *vcpu, + struct kvm_segment *var, int seg); + void (*set_segment)(struct kvm_vcpu *vcpu, + struct kvm_segment *var, int seg); + void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); + void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu); + void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); + void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); + void (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); + void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer); + void (*get_idt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); + void (*set_idt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); + void (*get_gdt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); + void (*set_gdt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); + unsigned long (*get_dr)(struct kvm_vcpu *vcpu, int dr); + void (*set_dr)(struct kvm_vcpu *vcpu, int dr, unsigned long value, + int *exception); + void (*cache_regs)(struct kvm_vcpu *vcpu); + void (*decache_regs)(struct kvm_vcpu *vcpu); + unsigned long (*get_rflags)(struct kvm_vcpu *vcpu); + void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); + + void (*tlb_flush)(struct kvm_vcpu *vcpu); + void (*inject_page_fault)(struct kvm_vcpu *vcpu, + unsigned long addr, u32 err_code); + + void (*inject_gp)(struct kvm_vcpu *vcpu, unsigned err_code); + + void (*run)(struct kvm_vcpu *vcpu, struct kvm_run *run); + int (*handle_exit)(struct kvm_run *run, struct kvm_vcpu *vcpu); + void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu); + void (*patch_hypercall)(struct kvm_vcpu *vcpu, + unsigned char *hypercall_addr); + int (*get_irq)(struct kvm_vcpu *vcpu); + void (*set_irq)(struct kvm_vcpu *vcpu, int vec); + void (*inject_pending_irq)(struct kvm_vcpu *vcpu); + void (*inject_pending_vectors)(struct kvm_vcpu *vcpu, + struct kvm_run *run); + + int (*set_tss_addr)(struct kvm *kvm, unsigned int addr); +}; + extern struct kvm_x86_ops *kvm_x86_ops; int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code); From 2b3ccfa0c5c7738a08f473b5d00b78f87935de72 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Mon, 19 Nov 2007 14:56:05 +0800 Subject: [PATCH 119/249] KVM: Portability: Move vcpu regs enumeration definition to x86.h Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 35 ----------------------------------- drivers/kvm/x86.h | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index e4e1ff7e6f9e..1c4de50ac06a 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -150,41 +150,6 @@ struct kvm_guest_debug { int singlestep; }; -enum { - VCPU_REGS_RAX = 0, - VCPU_REGS_RCX = 1, - VCPU_REGS_RDX = 2, - VCPU_REGS_RBX = 3, - VCPU_REGS_RSP = 4, - VCPU_REGS_RBP = 5, - VCPU_REGS_RSI = 6, - VCPU_REGS_RDI = 7, -#ifdef CONFIG_X86_64 - VCPU_REGS_R8 = 8, - VCPU_REGS_R9 = 9, - VCPU_REGS_R10 = 10, - VCPU_REGS_R11 = 11, - VCPU_REGS_R12 = 12, - VCPU_REGS_R13 = 13, - VCPU_REGS_R14 = 14, - VCPU_REGS_R15 = 15, -#endif - NR_VCPU_REGS -}; - -enum { - VCPU_SREG_CS, - VCPU_SREG_DS, - VCPU_SREG_ES, - VCPU_SREG_FS, - VCPU_SREG_GS, - VCPU_SREG_SS, - VCPU_SREG_TR, - VCPU_SREG_LDTR, -}; - -#include "x86_emulate.h" - struct kvm_pio_request { unsigned long count; int cur_count; diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 77b4092c0542..77a4a4ab9391 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -55,6 +55,41 @@ extern spinlock_t kvm_lock; extern struct list_head vm_list; +enum { + VCPU_REGS_RAX = 0, + VCPU_REGS_RCX = 1, + VCPU_REGS_RDX = 2, + VCPU_REGS_RBX = 3, + VCPU_REGS_RSP = 4, + VCPU_REGS_RBP = 5, + VCPU_REGS_RSI = 6, + VCPU_REGS_RDI = 7, +#ifdef CONFIG_X86_64 + VCPU_REGS_R8 = 8, + VCPU_REGS_R9 = 9, + VCPU_REGS_R10 = 10, + VCPU_REGS_R11 = 11, + VCPU_REGS_R12 = 12, + VCPU_REGS_R13 = 13, + VCPU_REGS_R14 = 14, + VCPU_REGS_R15 = 15, +#endif + NR_VCPU_REGS +}; + +enum { + VCPU_SREG_CS, + VCPU_SREG_DS, + VCPU_SREG_ES, + VCPU_SREG_FS, + VCPU_SREG_GS, + VCPU_SREG_SS, + VCPU_SREG_TR, + VCPU_SREG_LDTR, +}; + +#include "x86_emulate.h" + struct kvm_vcpu { KVM_VCPU_COMM; u64 host_tsc; From ec6d273deb56b1607e2acaad3df4bca42f135cd7 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Mon, 19 Nov 2007 15:08:31 +0800 Subject: [PATCH 120/249] KVM: Move some static inline functions out from kvm.h into x86.h Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 109 --------------------------------------------- drivers/kvm/x86.h | 110 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 109 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 1c4de50ac06a..41f6ee2134de 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -577,115 +577,6 @@ static inline int memslot_id(struct kvm *kvm, struct kvm_memory_slot *slot) return slot - kvm->memslots; } -static inline struct kvm_mmu_page *page_header(hpa_t shadow_page) -{ - struct page *page = pfn_to_page(shadow_page >> PAGE_SHIFT); - - return (struct kvm_mmu_page *)page_private(page); -} - -static inline u16 read_fs(void) -{ - u16 seg; - asm("mov %%fs, %0" : "=g"(seg)); - return seg; -} - -static inline u16 read_gs(void) -{ - u16 seg; - asm("mov %%gs, %0" : "=g"(seg)); - return seg; -} - -static inline u16 read_ldt(void) -{ - u16 ldt; - asm("sldt %0" : "=g"(ldt)); - return ldt; -} - -static inline void load_fs(u16 sel) -{ - asm("mov %0, %%fs" : : "rm"(sel)); -} - -static inline void load_gs(u16 sel) -{ - asm("mov %0, %%gs" : : "rm"(sel)); -} - -#ifndef load_ldt -static inline void load_ldt(u16 sel) -{ - asm("lldt %0" : : "rm"(sel)); -} -#endif - -static inline void get_idt(struct descriptor_table *table) -{ - asm("sidt %0" : "=m"(*table)); -} - -static inline void get_gdt(struct descriptor_table *table) -{ - asm("sgdt %0" : "=m"(*table)); -} - -static inline unsigned long read_tr_base(void) -{ - u16 tr; - asm("str %0" : "=g"(tr)); - return segment_base(tr); -} - -#ifdef CONFIG_X86_64 -static inline unsigned long read_msr(unsigned long msr) -{ - u64 value; - - rdmsrl(msr, value); - return value; -} -#endif - -static inline void fx_save(struct i387_fxsave_struct *image) -{ - asm("fxsave (%0)":: "r" (image)); -} - -static inline void fx_restore(struct i387_fxsave_struct *image) -{ - asm("fxrstor (%0)":: "r" (image)); -} - -static inline void fpu_init(void) -{ - asm("finit"); -} - -static inline u32 get_rdx_init_val(void) -{ - return 0x600; /* P6 family */ -} - -#define ASM_VMX_VMCLEAR_RAX ".byte 0x66, 0x0f, 0xc7, 0x30" -#define ASM_VMX_VMLAUNCH ".byte 0x0f, 0x01, 0xc2" -#define ASM_VMX_VMRESUME ".byte 0x0f, 0x01, 0xc3" -#define ASM_VMX_VMPTRLD_RAX ".byte 0x0f, 0xc7, 0x30" -#define ASM_VMX_VMREAD_RDX_RAX ".byte 0x0f, 0x78, 0xd0" -#define ASM_VMX_VMWRITE_RAX_RDX ".byte 0x0f, 0x79, 0xd0" -#define ASM_VMX_VMWRITE_RSP_RDX ".byte 0x0f, 0x79, 0xd4" -#define ASM_VMX_VMXOFF ".byte 0x0f, 0x01, 0xc4" -#define ASM_VMX_VMXON_RAX ".byte 0xf3, 0x0f, 0xc7, 0x30" - -#define MSR_IA32_TIME_STAMP_COUNTER 0x010 - -#define TSS_IOPB_BASE_OFFSET 0x66 -#define TSS_BASE_SIZE 0x68 -#define TSS_IOPB_SIZE (65536 / 8) -#define TSS_REDIRECTION_SIZE (256 / 8) -#define RMODE_TSS_SIZE (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) enum kvm_stat_kind { KVM_STAT_VM, diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 77a4a4ab9391..f1c43cafb0ca 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -267,4 +267,114 @@ static inline int is_paging(struct kvm_vcpu *vcpu) int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); int complete_pio(struct kvm_vcpu *vcpu); + +static inline struct kvm_mmu_page *page_header(hpa_t shadow_page) +{ + struct page *page = pfn_to_page(shadow_page >> PAGE_SHIFT); + + return (struct kvm_mmu_page *)page_private(page); +} + +static inline u16 read_fs(void) +{ + u16 seg; + asm("mov %%fs, %0" : "=g"(seg)); + return seg; +} + +static inline u16 read_gs(void) +{ + u16 seg; + asm("mov %%gs, %0" : "=g"(seg)); + return seg; +} + +static inline u16 read_ldt(void) +{ + u16 ldt; + asm("sldt %0" : "=g"(ldt)); + return ldt; +} + +static inline void load_fs(u16 sel) +{ + asm("mov %0, %%fs" : : "rm"(sel)); +} + +static inline void load_gs(u16 sel) +{ + asm("mov %0, %%gs" : : "rm"(sel)); +} + +#ifndef load_ldt +static inline void load_ldt(u16 sel) +{ + asm("lldt %0" : : "rm"(sel)); +} +#endif + +static inline void get_idt(struct descriptor_table *table) +{ + asm("sidt %0" : "=m"(*table)); +} + +static inline void get_gdt(struct descriptor_table *table) +{ + asm("sgdt %0" : "=m"(*table)); +} + +static inline unsigned long read_tr_base(void) +{ + u16 tr; + asm("str %0" : "=g"(tr)); + return segment_base(tr); +} + +#ifdef CONFIG_X86_64 +static inline unsigned long read_msr(unsigned long msr) +{ + u64 value; + + rdmsrl(msr, value); + return value; +} +#endif + +static inline void fx_save(struct i387_fxsave_struct *image) +{ + asm("fxsave (%0)":: "r" (image)); +} + +static inline void fx_restore(struct i387_fxsave_struct *image) +{ + asm("fxrstor (%0)":: "r" (image)); +} + +static inline void fpu_init(void) +{ + asm("finit"); +} + +static inline u32 get_rdx_init_val(void) +{ + return 0x600; /* P6 family */ +} + +#define ASM_VMX_VMCLEAR_RAX ".byte 0x66, 0x0f, 0xc7, 0x30" +#define ASM_VMX_VMLAUNCH ".byte 0x0f, 0x01, 0xc2" +#define ASM_VMX_VMRESUME ".byte 0x0f, 0x01, 0xc3" +#define ASM_VMX_VMPTRLD_RAX ".byte 0x0f, 0xc7, 0x30" +#define ASM_VMX_VMREAD_RDX_RAX ".byte 0x0f, 0x78, 0xd0" +#define ASM_VMX_VMWRITE_RAX_RDX ".byte 0x0f, 0x79, 0xd0" +#define ASM_VMX_VMWRITE_RSP_RDX ".byte 0x0f, 0x79, 0xd4" +#define ASM_VMX_VMXOFF ".byte 0x0f, 0x01, 0xc4" +#define ASM_VMX_VMXON_RAX ".byte 0xf3, 0x0f, 0xc7, 0x30" + +#define MSR_IA32_TIME_STAMP_COUNTER 0x010 + +#define TSS_IOPB_BASE_OFFSET 0x66 +#define TSS_BASE_SIZE 0x68 +#define TSS_IOPB_SIZE (65536 / 8) +#define TSS_REDIRECTION_SIZE (256 / 8) +#define RMODE_TSS_SIZE (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) #endif From 54f1585a8d0131bc3993902d4441f4049daed0d5 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Mon, 19 Nov 2007 15:24:28 +0800 Subject: [PATCH 121/249] KVM: Portability: Move some function declarations to x86.h Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 84 ----------------------------------------------- drivers/kvm/x86.h | 84 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 84 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 41f6ee2134de..19014564ddbb 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -375,19 +375,6 @@ int kvm_init(void *opaque, unsigned int vcpu_size, struct module *module); void kvm_exit(void); -int kvm_mmu_module_init(void); -void kvm_mmu_module_exit(void); - -void kvm_mmu_destroy(struct kvm_vcpu *vcpu); -int kvm_mmu_create(struct kvm_vcpu *vcpu); -int kvm_mmu_setup(struct kvm_vcpu *vcpu); -void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte); - -int kvm_mmu_reset_context(struct kvm_vcpu *vcpu); -void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot); -void kvm_mmu_zap_all(struct kvm *kvm); -void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages); - hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa); #define HPA_MSB ((sizeof(hpa_t) * 8) - 1) #define HPA_ERR_MASK ((hpa_t)1 << HPA_MSB) @@ -421,83 +408,12 @@ struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn); int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn); void mark_page_dirty(struct kvm *kvm, gfn_t gfn); -enum emulation_result { - EMULATE_DONE, /* no further processing */ - EMULATE_DO_MMIO, /* kvm_run filled with mmio request */ - EMULATE_FAIL, /* can't emulate this instruction */ -}; - -int emulate_instruction(struct kvm_vcpu *vcpu, struct kvm_run *run, - unsigned long cr2, u16 error_code, int no_decode); -void kvm_report_emulation_failure(struct kvm_vcpu *cvpu, const char *context); -void realmode_lgdt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); -void realmode_lidt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); -void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw, - unsigned long *rflags); - -unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr); -void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long value, - unsigned long *rflags); -int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *data); -int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); - -struct x86_emulate_ctxt; - -int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, - int size, unsigned port); -int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, - int size, unsigned long count, int down, - gva_t address, int rep, unsigned port); -void kvm_emulate_cpuid(struct kvm_vcpu *vcpu); -int kvm_emulate_halt(struct kvm_vcpu *vcpu); -int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address); -int emulate_clts(struct kvm_vcpu *vcpu); -int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, - unsigned long *dest); -int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, - unsigned long value); - -void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); -void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr0); -void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr0); -void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr0); -unsigned long get_cr8(struct kvm_vcpu *vcpu); -void lmsw(struct kvm_vcpu *vcpu, unsigned long msw); -void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l); - -int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); -int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data); - -void fx_init(struct kvm_vcpu *vcpu); - void kvm_vcpu_block(struct kvm_vcpu *vcpu); void kvm_resched(struct kvm_vcpu *vcpu); void kvm_load_guest_fpu(struct kvm_vcpu *vcpu); void kvm_put_guest_fpu(struct kvm_vcpu *vcpu); void kvm_flush_remote_tlbs(struct kvm *kvm); -int emulator_read_std(unsigned long addr, - void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu); -int emulator_write_emulated(unsigned long addr, - const void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu); - -unsigned long segment_base(u16 selector); - -void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, - const u8 *new, int bytes); -int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva); -void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); -int kvm_mmu_load(struct kvm_vcpu *vcpu); -void kvm_mmu_unload(struct kvm_vcpu *vcpu); - -int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); - -int kvm_fix_hypercall(struct kvm_vcpu *vcpu); - long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); long kvm_arch_vcpu_ioctl(struct file *filp, diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index f1c43cafb0ca..90b791bf6287 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -225,6 +225,90 @@ struct kvm_x86_ops { extern struct kvm_x86_ops *kvm_x86_ops; +int kvm_mmu_module_init(void); +void kvm_mmu_module_exit(void); + +void kvm_mmu_destroy(struct kvm_vcpu *vcpu); +int kvm_mmu_create(struct kvm_vcpu *vcpu); +int kvm_mmu_setup(struct kvm_vcpu *vcpu); +void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte); + +int kvm_mmu_reset_context(struct kvm_vcpu *vcpu); +void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot); +void kvm_mmu_zap_all(struct kvm *kvm); +void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages); + +enum emulation_result { + EMULATE_DONE, /* no further processing */ + EMULATE_DO_MMIO, /* kvm_run filled with mmio request */ + EMULATE_FAIL, /* can't emulate this instruction */ +}; + +int emulate_instruction(struct kvm_vcpu *vcpu, struct kvm_run *run, + unsigned long cr2, u16 error_code, int no_decode); +void kvm_report_emulation_failure(struct kvm_vcpu *cvpu, const char *context); +void realmode_lgdt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); +void realmode_lidt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); +void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw, + unsigned long *rflags); + +unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr); +void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long value, + unsigned long *rflags); +int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *data); +int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); + +struct x86_emulate_ctxt; + +int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, + int size, unsigned port); +int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, + int size, unsigned long count, int down, + gva_t address, int rep, unsigned port); +void kvm_emulate_cpuid(struct kvm_vcpu *vcpu); +int kvm_emulate_halt(struct kvm_vcpu *vcpu); +int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address); +int emulate_clts(struct kvm_vcpu *vcpu); +int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, + unsigned long *dest); +int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, + unsigned long value); + +void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); +void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr0); +void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr0); +void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr0); +unsigned long get_cr8(struct kvm_vcpu *vcpu); +void lmsw(struct kvm_vcpu *vcpu, unsigned long msw); +void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l); + +int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); +int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data); + +void fx_init(struct kvm_vcpu *vcpu); + +int emulator_read_std(unsigned long addr, + void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu); +int emulator_write_emulated(unsigned long addr, + const void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu); + +unsigned long segment_base(u16 selector); + +void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, + const u8 *new, int bytes); +int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva); +void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); +int kvm_mmu_load(struct kvm_vcpu *vcpu); +void kvm_mmu_unload(struct kvm_vcpu *vcpu); + +int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); + +int kvm_fix_hypercall(struct kvm_vcpu *vcpu); + int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code); static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) From 15b00f32d53f31fe194b17ca0384a37756b55be2 Mon Sep 17 00:00:00 2001 From: Jan Kiszka Date: Mon, 19 Nov 2007 10:21:45 +0100 Subject: [PATCH 122/249] KVM: VMX: Force seg.base == (seg.sel << 4) in real mode Ensure that segment.base == segment.selector << 4 when entering the real mode on Intel so that the CPU will not bark at us. This fixes some old protected mode demo from http://www.x86.org/articles/pmbasics/tspec_a1_doc.htm. Signed-off-by: Jan Kiszka Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 0c082faaa6db..b4c0bdce7b34 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1165,7 +1165,8 @@ static void fix_rmode_seg(int seg, struct kvm_save_segment *save) save->base = vmcs_readl(sf->base); save->limit = vmcs_read32(sf->limit); save->ar = vmcs_read32(sf->ar_bytes); - vmcs_write16(sf->selector, vmcs_readl(sf->base) >> 4); + vmcs_write16(sf->selector, save->base >> 4); + vmcs_write32(sf->base, save->base & 0xfffff); vmcs_write32(sf->limit, 0xffff); vmcs_write32(sf->ar_bytes, 0xf3); } From ec8d4eaefc06636eb8fc83610c6605168796b4f7 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Mon, 19 Nov 2007 11:28:19 +0200 Subject: [PATCH 123/249] KVM: MMU: Change guest pte access to kvm_{read,write}_guest() Things are simpler and more regular this way. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 0f0266af3f68..be6640178f6b 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -72,7 +72,6 @@ static int FNAME(walk_addr)(struct guest_walker *walker, struct kvm_vcpu *vcpu, gva_t addr, int write_fault, int user_fault, int fetch_fault) { - struct page *page = NULL; pt_element_t *table; pt_element_t pte; gfn_t table_gfn; @@ -99,16 +98,13 @@ static int FNAME(walk_addr)(struct guest_walker *walker, index = PT_INDEX(addr, walker->level); table_gfn = (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT; + pte_gpa = table_gfn << PAGE_SHIFT; + pte_gpa += index * sizeof(pt_element_t); walker->table_gfn[walker->level - 1] = table_gfn; pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, walker->level - 1, table_gfn); - page = gfn_to_page(vcpu->kvm, (pte & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT); - - table = kmap_atomic(page, KM_USER0); - pte = table[index]; - kunmap_atomic(table, KM_USER0); + kvm_read_guest(vcpu->kvm, pte_gpa, &pte, sizeof(pte)); if (!is_present_pte(pte)) goto not_present; @@ -128,9 +124,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, if (!(pte & PT_ACCESSED_MASK)) { mark_page_dirty(vcpu->kvm, table_gfn); pte |= PT_ACCESSED_MASK; - table = kmap_atomic(page, KM_USER0); - table[index] = pte; - kunmap_atomic(table, KM_USER0); + kvm_write_guest(vcpu->kvm, pte_gpa, &pte, sizeof(pte)); } if (walker->level == PT_PAGE_TABLE_LEVEL) { @@ -149,21 +143,15 @@ static int FNAME(walk_addr)(struct guest_walker *walker, walker->inherited_ar &= pte; --walker->level; - kvm_release_page(page); } if (write_fault && !is_dirty_pte(pte)) { mark_page_dirty(vcpu->kvm, table_gfn); pte |= PT_DIRTY_MASK; - table = kmap_atomic(page, KM_USER0); - table[index] = pte; - kunmap_atomic(table, KM_USER0); - pte_gpa = table_gfn << PAGE_SHIFT; - pte_gpa += index * sizeof(pt_element_t); + kvm_write_guest(vcpu->kvm, pte_gpa, &pte, sizeof(pte)); kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte)); } - kvm_release_page(page); walker->pte = pte; pgprintk("%s: pte %llx\n", __FUNCTION__, (u64)pte); return 1; @@ -182,8 +170,6 @@ err: walker->error_code |= PFERR_USER_MASK; if (fetch_fault) walker->error_code |= PFERR_FETCH_MASK; - if (page) - kvm_release_page(page); return 0; } From 3e021bf5053bc45448e43cf1d7d020ddf1409714 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Mon, 19 Nov 2007 11:16:57 +0200 Subject: [PATCH 124/249] KVM: Simplify kvm_clear_guest_page() Use kvm_write_guest_page() with empty_zero_page, instead of doing kmap and memset. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 595ed323fc53..e83a178dc80f 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -46,6 +46,7 @@ #include #include #include +#include MODULE_AUTHOR("Qumranet"); MODULE_LICENSE("GPL"); @@ -633,22 +634,7 @@ int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) { - void *page_virt; - struct page *page; - - page = gfn_to_page(kvm, gfn); - if (is_error_page(page)) { - kvm_release_page(page); - return -EFAULT; - } - page_virt = kmap_atomic(page, KM_USER0); - - memset(page_virt + offset, 0, len); - - kunmap_atomic(page_virt, KM_USER0); - kvm_release_page(page); - mark_page_dirty(kvm, gfn); - return 0; + return kvm_write_guest_page(kvm, gfn, empty_zero_page, offset, len); } EXPORT_SYMBOL_GPL(kvm_clear_guest_page); From 971535ff65da2e967a1105a3b2fa3693fc83ca8f Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Mon, 19 Nov 2007 18:44:15 +0200 Subject: [PATCH 125/249] KVM: MMU: Remove unused variable Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index be6640178f6b..77a2b22492bf 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -72,7 +72,6 @@ static int FNAME(walk_addr)(struct guest_walker *walker, struct kvm_vcpu *vcpu, gva_t addr, int write_fault, int user_fault, int fetch_fault) { - pt_element_t *table; pt_element_t pte; gfn_t table_gfn; unsigned index; From 00fc9f5ae5988fb180f5a288451a741b35cc806e Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Mon, 19 Nov 2007 14:04:45 -0600 Subject: [PATCH 126/249] KVM: Remove unused "rmap_overflow" variable Signed-off-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 19014564ddbb..ba78a45811cd 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -318,7 +318,6 @@ struct kvm { unsigned int n_alloc_mmu_pages; struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES]; struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; - unsigned long rmap_overflow; struct list_head vm_list; struct file *filp; struct kvm_io_bus mmio_bus; From d40ccc6246ed19ca529043998942a6b450c0e498 Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Mon, 19 Nov 2007 14:04:43 -0600 Subject: [PATCH 127/249] KVM: Correct consistent typo: "destory" -> "destroy" Signed-off-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 2 +- drivers/kvm/kvm_main.c | 2 +- drivers/kvm/x86.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index ba78a45811cd..ccba9584355c 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -461,7 +461,7 @@ void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id); -void kvm_arch_vcpu_destory(struct kvm_vcpu *vcpu); +void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu); int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu); void kvm_arch_hardware_enable(void *garbage); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index e83a178dc80f..4e1bd9488470 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -824,7 +824,7 @@ unlink: kvm->vcpus[n] = NULL; mutex_unlock(&kvm->lock); vcpu_destroy: - kvm_arch_vcpu_destory(vcpu); + kvm_arch_vcpu_destroy(vcpu); return r; } diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 2257a0aab32f..5a1b72fbaeaa 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -2513,7 +2513,7 @@ fail: return ERR_PTR(r); } -void kvm_arch_vcpu_destory(struct kvm_vcpu *vcpu) +void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) { vcpu_load(vcpu); kvm_mmu_unload(vcpu); From 7faa8f6fcc119ce680a90a5f011841b466b1e161 Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Mon, 19 Nov 2007 14:04:44 -0600 Subject: [PATCH 128/249] KVM: Move misplaced comment Signed-off-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index ccba9584355c..52e80183e050 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -135,15 +135,15 @@ struct kvm_mmu { #define KVM_NR_MEM_OBJS 40 +/* + * We don't want allocation failures within the mmu code, so we preallocate + * enough memory for a single page fault in a cache. + */ struct kvm_mmu_memory_cache { int nobjs; void *objects[KVM_NR_MEM_OBJS]; }; -/* - * We don't want allocation failures within the mmu code, so we preallocate - * enough memory for a single page fault in a cache. - */ struct kvm_guest_debug { int enabled; unsigned long bp[4]; From f6a40e3bdf5fe0a7d7d7f2dbc5b10158fbdad968 Mon Sep 17 00:00:00 2001 From: Jerone Young Date: Mon, 19 Nov 2007 17:06:31 -0600 Subject: [PATCH 129/249] KVM: Portability: Move kvm_memory_alias to asm/kvm.h This patch moves sturct kvm_memory_alias from include/linux/kvm.h to include/asm-x86/kvm.h. Also have include/linux/kvm.h include include/asm/kvm.h. Signed-off-by: Jerone Young Signed-off-by: Avi Kivity --- include/asm-x86/kvm.h | 20 ++++++++++++++++++++ include/linux/kvm.h | 8 +------- 2 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 include/asm-x86/kvm.h diff --git a/include/asm-x86/kvm.h b/include/asm-x86/kvm.h new file mode 100644 index 000000000000..37cf8e995c83 --- /dev/null +++ b/include/asm-x86/kvm.h @@ -0,0 +1,20 @@ +#ifndef __LINUX_KVM_X86_H +#define __LINUX_KVM_X86_H + +/* + * KVM x86 specific structures and definitions + * + */ + +#include +#include + +struct kvm_memory_alias { + __u32 slot; /* this has a different namespace than memory slots */ + __u32 flags; + __u64 guest_phys_addr; + __u64 memory_size; + __u64 target_phys_addr; +}; + +#endif diff --git a/include/linux/kvm.h b/include/linux/kvm.h index 71d33d62944f..d09dd5dd5493 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -9,6 +9,7 @@ #include #include +#include #define KVM_API_VERSION 12 @@ -35,13 +36,6 @@ struct kvm_userspace_memory_region { /* for kvm_memory_region::flags */ #define KVM_MEM_LOG_DIRTY_PAGES 1UL -struct kvm_memory_alias { - __u32 slot; /* this has a different namespace than memory slots */ - __u32 flags; - __u64 guest_phys_addr; - __u64 memory_size; - __u64 target_phys_addr; -}; /* for KVM_IRQ_LINE */ struct kvm_irq_level { From da1386a5bc43fa0faf17089a68c2991cf1d526c8 Mon Sep 17 00:00:00 2001 From: Jerone Young Date: Mon, 19 Nov 2007 17:06:32 -0600 Subject: [PATCH 130/249] KVM: Portability: Move x86 pic strutctures This patch moves structures: kvm_pic_state kvm_ioapic_state to inclue/asm-x86/kvm.h. Signed-off-by: Jerone Young Signed-off-by: Avi Kivity --- include/asm-x86/kvm.h | 49 +++++++++++++++++++++++++++++++++++++++++++ include/linux/kvm.h | 48 ------------------------------------------ 2 files changed, 49 insertions(+), 48 deletions(-) diff --git a/include/asm-x86/kvm.h b/include/asm-x86/kvm.h index 37cf8e995c83..80752bccc756 100644 --- a/include/asm-x86/kvm.h +++ b/include/asm-x86/kvm.h @@ -17,4 +17,53 @@ struct kvm_memory_alias { __u64 target_phys_addr; }; +/* for KVM_GET_IRQCHIP and KVM_SET_IRQCHIP */ +struct kvm_pic_state { + __u8 last_irr; /* edge detection */ + __u8 irr; /* interrupt request register */ + __u8 imr; /* interrupt mask register */ + __u8 isr; /* interrupt service register */ + __u8 priority_add; /* highest irq priority */ + __u8 irq_base; + __u8 read_reg_select; + __u8 poll; + __u8 special_mask; + __u8 init_state; + __u8 auto_eoi; + __u8 rotate_on_auto_eoi; + __u8 special_fully_nested_mode; + __u8 init4; /* true if 4 byte init */ + __u8 elcr; /* PIIX edge/trigger selection */ + __u8 elcr_mask; +}; + +#define KVM_IOAPIC_NUM_PINS 24 +struct kvm_ioapic_state { + __u64 base_address; + __u32 ioregsel; + __u32 id; + __u32 irr; + __u32 pad; + union { + __u64 bits; + struct { + __u8 vector; + __u8 delivery_mode:3; + __u8 dest_mode:1; + __u8 delivery_status:1; + __u8 polarity:1; + __u8 remote_irr:1; + __u8 trig_mode:1; + __u8 mask:1; + __u8 reserve:7; + __u8 reserved[4]; + __u8 dest_id; + } fields; + } redirtbl[KVM_IOAPIC_NUM_PINS]; +}; + +#define KVM_IRQCHIP_PIC_MASTER 0 +#define KVM_IRQCHIP_PIC_SLAVE 1 +#define KVM_IRQCHIP_IOAPIC 2 + #endif diff --git a/include/linux/kvm.h b/include/linux/kvm.h index d09dd5dd5493..1779c3d3bdc9 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -48,54 +48,6 @@ struct kvm_irq_level { __u32 level; }; -/* for KVM_GET_IRQCHIP and KVM_SET_IRQCHIP */ -struct kvm_pic_state { - __u8 last_irr; /* edge detection */ - __u8 irr; /* interrupt request register */ - __u8 imr; /* interrupt mask register */ - __u8 isr; /* interrupt service register */ - __u8 priority_add; /* highest irq priority */ - __u8 irq_base; - __u8 read_reg_select; - __u8 poll; - __u8 special_mask; - __u8 init_state; - __u8 auto_eoi; - __u8 rotate_on_auto_eoi; - __u8 special_fully_nested_mode; - __u8 init4; /* true if 4 byte init */ - __u8 elcr; /* PIIX edge/trigger selection */ - __u8 elcr_mask; -}; - -#define KVM_IOAPIC_NUM_PINS 24 -struct kvm_ioapic_state { - __u64 base_address; - __u32 ioregsel; - __u32 id; - __u32 irr; - __u32 pad; - union { - __u64 bits; - struct { - __u8 vector; - __u8 delivery_mode:3; - __u8 dest_mode:1; - __u8 delivery_status:1; - __u8 polarity:1; - __u8 remote_irr:1; - __u8 trig_mode:1; - __u8 mask:1; - __u8 reserve:7; - __u8 reserved[4]; - __u8 dest_id; - } fields; - } redirtbl[KVM_IOAPIC_NUM_PINS]; -}; - -#define KVM_IRQCHIP_PIC_MASTER 0 -#define KVM_IRQCHIP_PIC_SLAVE 1 -#define KVM_IRQCHIP_IOAPIC 2 struct kvm_irqchip { __u32 chip_id; From 19d30b164417e7580d7f8c495a03137e1a45d4a7 Mon Sep 17 00:00:00 2001 From: Jerone Young Date: Mon, 19 Nov 2007 17:06:33 -0600 Subject: [PATCH 131/249] KVM: Portability: Move kvm_regs to This patch moves structure kvm_regs to include/asm-x86/kvm.h. Each architecture will need to create there own version of this structure. Signed-off-by: Jerone Young Signed-off-by: Avi Kivity --- include/asm-x86/kvm.h | 10 ++++++++++ include/linux/kvm.h | 9 --------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/include/asm-x86/kvm.h b/include/asm-x86/kvm.h index 80752bccc756..c83a2ff35bf1 100644 --- a/include/asm-x86/kvm.h +++ b/include/asm-x86/kvm.h @@ -66,4 +66,14 @@ struct kvm_ioapic_state { #define KVM_IRQCHIP_PIC_SLAVE 1 #define KVM_IRQCHIP_IOAPIC 2 +/* for KVM_GET_REGS and KVM_SET_REGS */ +struct kvm_regs { + /* out (KVM_GET_REGS) / in (KVM_SET_REGS) */ + __u64 rax, rbx, rcx, rdx; + __u64 rsi, rdi, rsp, rbp; + __u64 r8, r9, r10, r11; + __u64 r12, r13, r14, r15; + __u64 rip, rflags; +}; + #endif diff --git a/include/linux/kvm.h b/include/linux/kvm.h index 1779c3d3bdc9..0d83efc9462e 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -134,15 +134,6 @@ struct kvm_run { }; }; -/* for KVM_GET_REGS and KVM_SET_REGS */ -struct kvm_regs { - /* out (KVM_GET_REGS) / in (KVM_SET_REGS) */ - __u64 rax, rbx, rcx, rdx; - __u64 rsi, rdi, rsp, rbp; - __u64 r8, r9, r10, r11; - __u64 r12, r13, r14, r15; - __u64 rip, rflags; -}; /* for KVM_GET_FPU and KVM_SET_FPU */ struct kvm_fpu { From d9ecf9281069c4c8dd429e1eb61cb7e0f3dda6b4 Mon Sep 17 00:00:00 2001 From: Jerone Young Date: Mon, 19 Nov 2007 17:06:34 -0600 Subject: [PATCH 132/249] KVM: Portability: Move structure lapic_state to This patch moves structure lapic_state from include/linux/kvm.h to include/asm-x86/kvm.h Signed-off-by: Jerone Young Signed-off-by: Avi Kivity --- include/asm-x86/kvm.h | 6 ++++++ include/linux/kvm.h | 5 ----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/asm-x86/kvm.h b/include/asm-x86/kvm.h index c83a2ff35bf1..a2c65b5c4a85 100644 --- a/include/asm-x86/kvm.h +++ b/include/asm-x86/kvm.h @@ -76,4 +76,10 @@ struct kvm_regs { __u64 rip, rflags; }; +/* for KVM_GET_LAPIC and KVM_SET_LAPIC */ +#define KVM_APIC_REG_SIZE 0x400 +struct kvm_lapic_state { + char regs[KVM_APIC_REG_SIZE]; +}; + #endif diff --git a/include/linux/kvm.h b/include/linux/kvm.h index 0d83efc9462e..280ec0d85b4c 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -150,11 +150,6 @@ struct kvm_fpu { __u32 pad2; }; -/* for KVM_GET_LAPIC and KVM_SET_LAPIC */ -#define KVM_APIC_REG_SIZE 0x400 -struct kvm_lapic_state { - char regs[KVM_APIC_REG_SIZE]; -}; struct kvm_segment { __u64 base; From 3a56b20104f51bd6caf07e49fa1e3c09bee222e5 Mon Sep 17 00:00:00 2001 From: Jerone Young Date: Mon, 19 Nov 2007 17:06:35 -0600 Subject: [PATCH 133/249] KVM: Portability: Move kvm_segment & kvm_dtable structure to This patch moves structures: kvm_segment kvm_dtable from include/linux/kvm.h to include/asm-x86/kvm.h Signed-off-by: Jerone Young Signed-off-by: Avi Kivity --- include/asm-x86/kvm.h | 17 +++++++++++++++++ include/linux/kvm.h | 15 --------------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/include/asm-x86/kvm.h b/include/asm-x86/kvm.h index a2c65b5c4a85..644a325077f0 100644 --- a/include/asm-x86/kvm.h +++ b/include/asm-x86/kvm.h @@ -82,4 +82,21 @@ struct kvm_lapic_state { char regs[KVM_APIC_REG_SIZE]; }; +struct kvm_segment { + __u64 base; + __u32 limit; + __u16 selector; + __u8 type; + __u8 present, dpl, db, s, l, g, avl; + __u8 unusable; + __u8 padding; +}; + +struct kvm_dtable { + __u64 base; + __u16 limit; + __u16 padding[3]; +}; + + #endif diff --git a/include/linux/kvm.h b/include/linux/kvm.h index 280ec0d85b4c..442cb58bb73c 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -151,21 +151,6 @@ struct kvm_fpu { }; -struct kvm_segment { - __u64 base; - __u32 limit; - __u16 selector; - __u8 type; - __u8 present, dpl, db, s, l, g, avl; - __u8 unusable; - __u8 padding; -}; - -struct kvm_dtable { - __u64 base; - __u16 limit; - __u16 padding[3]; -}; /* for KVM_GET_SREGS and KVM_SET_SREGS */ struct kvm_sregs { From 244d57ece94b56692c61f2d507d1251b5844c550 Mon Sep 17 00:00:00 2001 From: Jerone Young Date: Mon, 19 Nov 2007 17:06:36 -0600 Subject: [PATCH 134/249] KVM: Portability: Move kvm_sregs and msr structures to Move structures: kvm_sregs kvm_msr_entry kvm_msrs kvm_msr_list from include/linux/kvm.h to include/asm-x86/kvm.h Signed-off-by: Avi Kivity --- include/asm-x86/kvm.h | 36 ++++++++++++++++++++++++++++++++++++ include/linux/kvm.h | 36 ------------------------------------ 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/include/asm-x86/kvm.h b/include/asm-x86/kvm.h index 644a325077f0..32c7dda61474 100644 --- a/include/asm-x86/kvm.h +++ b/include/asm-x86/kvm.h @@ -9,6 +9,9 @@ #include #include +/* Architectural interrupt line count. */ +#define KVM_NR_INTERRUPTS 256 + struct kvm_memory_alias { __u32 slot; /* this has a different namespace than memory slots */ __u32 flags; @@ -99,4 +102,37 @@ struct kvm_dtable { }; +/* for KVM_GET_SREGS and KVM_SET_SREGS */ +struct kvm_sregs { + /* out (KVM_GET_SREGS) / in (KVM_SET_SREGS) */ + struct kvm_segment cs, ds, es, fs, gs, ss; + struct kvm_segment tr, ldt; + struct kvm_dtable gdt, idt; + __u64 cr0, cr2, cr3, cr4, cr8; + __u64 efer; + __u64 apic_base; + __u64 interrupt_bitmap[(KVM_NR_INTERRUPTS + 63) / 64]; +}; + +struct kvm_msr_entry { + __u32 index; + __u32 reserved; + __u64 data; +}; + +/* for KVM_GET_MSRS and KVM_SET_MSRS */ +struct kvm_msrs { + __u32 nmsrs; /* number of msrs in entries */ + __u32 pad; + + struct kvm_msr_entry entries[0]; +}; + +/* for KVM_GET_MSR_INDEX_LIST */ +struct kvm_msr_list { + __u32 nmsrs; /* number of msrs in entries */ + __u32 indices[0]; +}; + + #endif diff --git a/include/linux/kvm.h b/include/linux/kvm.h index 442cb58bb73c..e6867aa6daf6 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -13,9 +13,6 @@ #define KVM_API_VERSION 12 -/* Architectural interrupt line count. */ -#define KVM_NR_INTERRUPTS 256 - /* for KVM_CREATE_MEMORY_REGION */ struct kvm_memory_region { __u32 slot; @@ -151,39 +148,6 @@ struct kvm_fpu { }; - -/* for KVM_GET_SREGS and KVM_SET_SREGS */ -struct kvm_sregs { - /* out (KVM_GET_SREGS) / in (KVM_SET_SREGS) */ - struct kvm_segment cs, ds, es, fs, gs, ss; - struct kvm_segment tr, ldt; - struct kvm_dtable gdt, idt; - __u64 cr0, cr2, cr3, cr4, cr8; - __u64 efer; - __u64 apic_base; - __u64 interrupt_bitmap[(KVM_NR_INTERRUPTS + 63) / 64]; -}; - -struct kvm_msr_entry { - __u32 index; - __u32 reserved; - __u64 data; -}; - -/* for KVM_GET_MSRS and KVM_SET_MSRS */ -struct kvm_msrs { - __u32 nmsrs; /* number of msrs in entries */ - __u32 pad; - - struct kvm_msr_entry entries[0]; -}; - -/* for KVM_GET_MSR_INDEX_LIST */ -struct kvm_msr_list { - __u32 nmsrs; /* number of msrs in entries */ - __u32 indices[0]; -}; - /* for KVM_TRANSLATE */ struct kvm_translation { /* in */ From a162dd58736d446f52ca45b170508c3a5868807b Mon Sep 17 00:00:00 2001 From: Jerone Young Date: Mon, 19 Nov 2007 17:06:37 -0600 Subject: [PATCH 135/249] KVM: Portability: Move cpuid structures to This patch moves structures: kvm_cpuid_entry kvm_cpuid from include/linux/kvm.h to include/asm-x86/kvm.h Signed-off-by: Jerone Young Signed-off-by: Avi Kivity --- include/asm-x86/kvm.h | 17 +++++++++++++++++ include/linux/kvm.h | 16 ---------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/include/asm-x86/kvm.h b/include/asm-x86/kvm.h index 32c7dda61474..4837d759bec0 100644 --- a/include/asm-x86/kvm.h +++ b/include/asm-x86/kvm.h @@ -135,4 +135,21 @@ struct kvm_msr_list { }; +struct kvm_cpuid_entry { + __u32 function; + __u32 eax; + __u32 ebx; + __u32 ecx; + __u32 edx; + __u32 padding; +}; + +/* for KVM_SET_CPUID */ +struct kvm_cpuid { + __u32 nent; + __u32 padding; + struct kvm_cpuid_entry entries[0]; +}; + + #endif diff --git a/include/linux/kvm.h b/include/linux/kvm.h index e6867aa6daf6..fd4f900fcce3 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -192,22 +192,6 @@ struct kvm_dirty_log { }; }; -struct kvm_cpuid_entry { - __u32 function; - __u32 eax; - __u32 ebx; - __u32 ecx; - __u32 edx; - __u32 padding; -}; - -/* for KVM_SET_CPUID */ -struct kvm_cpuid { - __u32 nent; - __u32 padding; - struct kvm_cpuid_entry entries[0]; -}; - /* for KVM_SET_SIGNAL_MASK */ struct kvm_signal_mask { __u32 len; From 2d2a7b9cff7f6324e69daa3ab76799641a94a0e8 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 20 Nov 2007 11:45:14 +0200 Subject: [PATCH 136/249] KVM: Export include/asm-x86/kvm.h Signed-off-by: Avi Kivity --- include/asm-x86/Kbuild | 1 + 1 file changed, 1 insertion(+) diff --git a/include/asm-x86/Kbuild b/include/asm-x86/Kbuild index e6189b229143..3c6f0f80e827 100644 --- a/include/asm-x86/Kbuild +++ b/include/asm-x86/Kbuild @@ -3,6 +3,7 @@ include include/asm-generic/Kbuild.asm header-y += boot.h header-y += bootparam.h header-y += debugreg.h +header-y += kvm.h header-y += ldt.h header-y += msr-index.h header-y += prctl.h From 2065b3727ecdb64450597d70f7e13af00b85dbd8 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Tue, 20 Nov 2007 11:30:04 +0200 Subject: [PATCH 137/249] KVM: MMU: Fix potential memory leak with smp real-mode When we map a page, we check whether some other vcpu mapped it for us and if so, bail out. But we should decrease the refcount on the page as we do so. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 87d8e70fe502..8add4d5c6840 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -907,8 +907,10 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) pte = table[index]; was_rmapped = is_rmap_pte(pte); - if (is_shadow_present_pte(pte) && is_writeble_pte(pte)) + if (is_shadow_present_pte(pte) && is_writeble_pte(pte)) { + kvm_release_page(pfn_to_page(p >> PAGE_SHIFT)); return 0; + } mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT); page_header_update_slot(vcpu->kvm, table, v); table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK | From b4231d61807cac8d9d257eb6979c1685fa9a171f Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Tue, 20 Nov 2007 11:49:33 +0200 Subject: [PATCH 138/249] KVM: MMU: Selectively set PageDirty when releasing guest memory Improve dirty bit setting for pages that kvm release, until now every page that we released we marked dirty, from now only pages that have potential to get dirty we mark dirty. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 ++- drivers/kvm/kvm_main.c | 12 +++++++++--- drivers/kvm/mmu.c | 23 +++++++++++++++-------- drivers/kvm/paging_tmpl.h | 12 ++++++------ drivers/kvm/x86.c | 2 +- 5 files changed, 33 insertions(+), 19 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 52e80183e050..c2acd74389fa 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -393,7 +393,8 @@ int __kvm_set_memory_region(struct kvm *kvm, int user_alloc); gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); -void kvm_release_page(struct page *page); +void kvm_release_page_clean(struct page *page); +void kvm_release_page_dirty(struct page *page); int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, int len); int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 4e1bd9488470..729573b844e5 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -543,13 +543,19 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) EXPORT_SYMBOL_GPL(gfn_to_page); -void kvm_release_page(struct page *page) +void kvm_release_page_clean(struct page *page) +{ + put_page(page); +} +EXPORT_SYMBOL_GPL(kvm_release_page_clean); + +void kvm_release_page_dirty(struct page *page) { if (!PageReserved(page)) SetPageDirty(page); put_page(page); } -EXPORT_SYMBOL_GPL(kvm_release_page); +EXPORT_SYMBOL_GPL(kvm_release_page_dirty); static int next_segment(unsigned long len, int offset) { @@ -1055,7 +1061,7 @@ static struct page *kvm_vm_nopage(struct vm_area_struct *vma, /* current->mm->mmap_sem is already held so call lockless version */ page = __gfn_to_page(kvm, pgoff); if (is_error_page(page)) { - kvm_release_page(page); + kvm_release_page_clean(page); return NOPAGE_SIGBUS; } if (type != NULL) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 8add4d5c6840..4624f3789b9a 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -420,14 +420,18 @@ static void rmap_remove(struct kvm *kvm, u64 *spte) struct kvm_rmap_desc *desc; struct kvm_rmap_desc *prev_desc; struct kvm_mmu_page *page; + struct page *release_page; unsigned long *rmapp; int i; if (!is_rmap_pte(*spte)) return; page = page_header(__pa(spte)); - kvm_release_page(pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> - PAGE_SHIFT)); + release_page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); + if (is_writeble_pte(*spte)) + kvm_release_page_dirty(release_page); + else + kvm_release_page_clean(release_page); rmapp = gfn_to_rmap(kvm, page->gfns[spte - page->spt]); if (!*rmapp) { printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte); @@ -893,7 +897,9 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) { int level = PT32E_ROOT_LEVEL; hpa_t table_addr = vcpu->mmu.root_hpa; + struct page *page; + page = pfn_to_page(p >> PAGE_SHIFT); for (; ; level--) { u32 index = PT64_INDEX(v, level); u64 *table; @@ -908,7 +914,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) pte = table[index]; was_rmapped = is_rmap_pte(pte); if (is_shadow_present_pte(pte) && is_writeble_pte(pte)) { - kvm_release_page(pfn_to_page(p >> PAGE_SHIFT)); + kvm_release_page_clean(page); return 0; } mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT); @@ -918,7 +924,8 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) if (!was_rmapped) rmap_add(vcpu, &table[index], v >> PAGE_SHIFT); else - kvm_release_page(pfn_to_page(p >> PAGE_SHIFT)); + kvm_release_page_clean(page); + return 0; } @@ -933,7 +940,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) 1, 3, &table[index]); if (!new_table) { pgprintk("nonpaging_map: ENOMEM\n"); - kvm_release_page(pfn_to_page(p >> PAGE_SHIFT)); + kvm_release_page_clean(page); return -ENOMEM; } @@ -1049,8 +1056,8 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, paddr = gpa_to_hpa(vcpu->kvm, addr & PT64_BASE_ADDR_MASK); if (is_error_hpa(paddr)) { - kvm_release_page(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT)); + kvm_release_page_clean(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT)); return 1; } @@ -1580,7 +1587,7 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, " valid guest gva %lx\n", audit_msg, va); page = pfn_to_page((gpa & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); - kvm_release_page(page); + kvm_release_page_clean(page); } } diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 77a2b22492bf..bf15d127a48f 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -212,8 +212,8 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, if (is_error_hpa(paddr)) { set_shadow_pte(shadow_pte, shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK); - kvm_release_page(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT)); + kvm_release_page_clean(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT)); return; } @@ -259,12 +259,12 @@ unshadowed: page = pfn_to_page((paddr & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); - kvm_release_page(page); + kvm_release_page_clean(page); } } else - kvm_release_page(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT)); + kvm_release_page_clean(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT)); if (!ptwrite || !*ptwrite) vcpu->last_pte_updated = shadow_pte; } @@ -503,7 +503,7 @@ static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, else sp->spt[i] = shadow_notrap_nonpresent_pte; kunmap_atomic(gpt, KM_USER0); - kvm_release_page(page); + kvm_release_page_clean(page); } #undef pt_element_t diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 5a1b72fbaeaa..6212984a2e6c 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1472,7 +1472,7 @@ static void free_pio_guest_pages(struct kvm_vcpu *vcpu) for (i = 0; i < ARRAY_SIZE(vcpu->pio.guest_pages); ++i) if (vcpu->pio.guest_pages[i]) { - kvm_release_page(vcpu->pio.guest_pages[i]); + kvm_release_page_dirty(vcpu->pio.guest_pages[i]); vcpu->pio.guest_pages[i] = NULL; } } From 93a0039c8d93074d5f92dfb69f6a7d453905d002 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 20 Nov 2007 12:49:31 +0200 Subject: [PATCH 139/249] KVM: x86 emulator: retire ->write_std() Theoretically used to acccess memory known to be ordinary RAM, it was never implemented. It is questionable whether it is possible to implement it correctly. Signed-off-by: Avi Kivity --- drivers/kvm/x86.c | 10 ---------- drivers/kvm/x86_emulate.h | 11 ----------- 2 files changed, 21 deletions(-) diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 6212984a2e6c..5a54e328088d 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1162,15 +1162,6 @@ int emulator_read_std(unsigned long addr, } EXPORT_SYMBOL_GPL(emulator_read_std); -static int emulator_write_std(unsigned long addr, - const void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - pr_unimpl(vcpu, "emulator_write_std: addr %lx n %d\n", addr, bytes); - return X86EMUL_UNHANDLEABLE; -} - static int emulator_read_emulated(unsigned long addr, void *val, unsigned int bytes, @@ -1367,7 +1358,6 @@ EXPORT_SYMBOL_GPL(kvm_report_emulation_failure); struct x86_emulate_ops emulate_ops = { .read_std = emulator_read_std, - .write_std = emulator_write_std, .read_emulated = emulator_read_emulated, .write_emulated = emulator_write_emulated, .cmpxchg_emulated = emulator_cmpxchg_emulated, diff --git a/drivers/kvm/x86_emulate.h b/drivers/kvm/x86_emulate.h index e34868b42c33..a62bf14bbf89 100644 --- a/drivers/kvm/x86_emulate.h +++ b/drivers/kvm/x86_emulate.h @@ -62,17 +62,6 @@ struct x86_emulate_ops { int (*read_std)(unsigned long addr, void *val, unsigned int bytes, struct kvm_vcpu *vcpu); - /* - * write_std: Write bytes of standard (non-emulated/special) memory. - * Used for stack operations, and others. - * @addr: [IN ] Linear address to which to write. - * @val: [IN ] Value to write to memory (low-order bytes used as - * required). - * @bytes: [IN ] Number of bytes to write to memory. - */ - int (*write_std)(unsigned long addr, const void *val, - unsigned int bytes, struct kvm_vcpu *vcpu); - /* * read_emulated: Read bytes from emulated/special memory area. * @addr: [IN ] Linear address from which to read. From 6226686954c4cce3d63ffe1777e60360fcbf0b83 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 20 Nov 2007 13:15:52 +0200 Subject: [PATCH 140/249] KVM: x86 emulator: prefetch up to 15 bytes of the instruction executed Instead of fetching one byte at a time, prefetch 15 bytes (or until the next page boundary) to avoid guest page table walks. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 38 ++++++++++++++++++++++++++++++++++++-- drivers/kvm/x86_emulate.h | 7 +++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 8e2162fc6f70..6e7f774d1751 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -414,8 +414,7 @@ static u16 twobyte_table[256] = { /* Fetch next part of the instruction being emulated. */ #define insn_fetch(_type, _size, _eip) \ ({ unsigned long _x; \ - rc = ops->read_std((unsigned long)(_eip) + ctxt->cs_base, &_x, \ - (_size), ctxt->vcpu); \ + rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \ if (rc != 0) \ goto done; \ (_eip) += (_size); \ @@ -446,6 +445,41 @@ static u16 twobyte_table[256] = { register_address_increment(c->eip, rel); \ } while (0) +static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops, + unsigned long linear, u8 *dest) +{ + struct fetch_cache *fc = &ctxt->decode.fetch; + int rc; + int size; + + if (linear < fc->start || linear >= fc->end) { + size = min(15UL, PAGE_SIZE - offset_in_page(linear)); + rc = ops->read_std(linear, fc->data, size, ctxt->vcpu); + if (rc) + return rc; + fc->start = linear; + fc->end = linear + size; + } + *dest = fc->data[linear - fc->start]; + return 0; +} + +static int do_insn_fetch(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops, + unsigned long eip, void *dest, unsigned size) +{ + int rc = 0; + + eip += ctxt->cs_base; + while (size--) { + rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++); + if (rc) + return rc; + } + return 0; +} + /* * Given the 'reg' portion of a ModRM byte, and a register block, return a * pointer into the block that addresses the relevant register. diff --git a/drivers/kvm/x86_emulate.h b/drivers/kvm/x86_emulate.h index a62bf14bbf89..4603b2bf3488 100644 --- a/drivers/kvm/x86_emulate.h +++ b/drivers/kvm/x86_emulate.h @@ -108,6 +108,12 @@ struct operand { unsigned long val, orig_val, *ptr; }; +struct fetch_cache { + u8 data[15]; + unsigned long start; + unsigned long end; +}; + struct decode_cache { u8 twobyte; u8 b; @@ -130,6 +136,7 @@ struct decode_cache { u8 use_modrm_ea; unsigned long modrm_ea; unsigned long modrm_val; + struct fetch_cache fetch; }; struct x86_emulate_ctxt { From 3ad82a7e874c5d6c4045090cc01d7794dd9eb21c Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Tue, 20 Nov 2007 13:11:38 +0800 Subject: [PATCH 141/249] KVM: Recalculate mmu pages needed for every memory region change Instead of incrementally changing the mmu cache size for every memory slot operation, recalculate it from scratch. This is simpler and safer. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 23 +++++------------------ drivers/kvm/mmu.c | 19 +++++++++++++++++++ drivers/kvm/x86.h | 1 + 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 729573b844e5..93ecafbfb1b6 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -333,26 +333,13 @@ int __kvm_set_memory_region(struct kvm *kvm, if (mem->slot >= kvm->nmemslots) kvm->nmemslots = mem->slot + 1; - if (!kvm->n_requested_mmu_pages) { - unsigned int n_pages; - - if (npages) { - n_pages = npages * KVM_PERMILLE_MMU_PAGES / 1000; - kvm_mmu_change_mmu_pages(kvm, kvm->n_alloc_mmu_pages + - n_pages); - } else { - unsigned int nr_mmu_pages; - - n_pages = old.npages * KVM_PERMILLE_MMU_PAGES / 1000; - nr_mmu_pages = kvm->n_alloc_mmu_pages - n_pages; - nr_mmu_pages = max(nr_mmu_pages, - (unsigned int) KVM_MIN_ALLOC_MMU_PAGES); - kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); - } - } - *memslot = new; + if (!kvm->n_requested_mmu_pages) { + unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm); + kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); + } + kvm_mmu_slot_remove_write_access(kvm, mem->slot); kvm_flush_remote_tlbs(kvm); diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 4624f3789b9a..101cd5377a89 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -1535,6 +1535,25 @@ nomem: return -ENOMEM; } +/* + * Caculate mmu pages needed for kvm. + */ +unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm) +{ + int i; + unsigned int nr_mmu_pages; + unsigned int nr_pages = 0; + + for (i = 0; i < kvm->nmemslots; i++) + nr_pages += kvm->memslots[i].npages; + + nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000; + nr_mmu_pages = max(nr_mmu_pages, + (unsigned int) KVM_MIN_ALLOC_MMU_PAGES); + + return nr_mmu_pages; +} + #ifdef AUDIT static const char *audit_msg; diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 90b791bf6287..71f2477d03fd 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -236,6 +236,7 @@ void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte); int kvm_mmu_reset_context(struct kvm_vcpu *vcpu); void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot); void kvm_mmu_zap_all(struct kvm *kvm); +unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm); void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages); enum emulation_result { From 0de10343b3ca7aa34dd606145748f73ed19f627e Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Tue, 20 Nov 2007 16:25:04 +0800 Subject: [PATCH 142/249] KVM: Portability: Split kvm_set_memory_region() to have an arch callout Moving !user_alloc case to kvm_arch to avoid unnecessary code logic in non-x86 platform. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 4 ++++ drivers/kvm/kvm_main.c | 38 +++++-------------------------- drivers/kvm/x86.c | 51 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 33 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index c2acd74389fa..49094a221f6a 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -391,6 +391,10 @@ int kvm_set_memory_region(struct kvm *kvm, int __kvm_set_memory_region(struct kvm *kvm, struct kvm_userspace_memory_region *mem, int user_alloc); +int kvm_arch_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + struct kvm_memory_slot old, + int user_alloc); gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); void kvm_release_page_clean(struct page *page); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 93ecafbfb1b6..9dd6ad3c6c7b 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -291,33 +291,7 @@ int __kvm_set_memory_region(struct kvm *kvm, memset(new.rmap, 0, npages * sizeof(*new.rmap)); new.user_alloc = user_alloc; - if (user_alloc) - new.userspace_addr = mem->userspace_addr; - else { - down_write(¤t->mm->mmap_sem); - new.userspace_addr = do_mmap(NULL, 0, - npages * PAGE_SIZE, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_ANONYMOUS, - 0); - up_write(¤t->mm->mmap_sem); - - if (IS_ERR((void *)new.userspace_addr)) - goto out_free; - } - } else { - if (!old.user_alloc && old.rmap) { - int ret; - - down_write(¤t->mm->mmap_sem); - ret = do_munmap(current->mm, old.userspace_addr, - old.npages * PAGE_SIZE); - up_write(¤t->mm->mmap_sem); - if (ret < 0) - printk(KERN_WARNING - "kvm_vm_ioctl_set_memory_region: " - "failed to munmap memory\n"); - } + new.userspace_addr = mem->userspace_addr; } /* Allocate page dirty bitmap if needed */ @@ -335,14 +309,12 @@ int __kvm_set_memory_region(struct kvm *kvm, *memslot = new; - if (!kvm->n_requested_mmu_pages) { - unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm); - kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); + r = kvm_arch_set_memory_region(kvm, mem, old, user_alloc); + if (r) { + *memslot = old; + goto out_free; } - kvm_mmu_slot_remove_write_access(kvm, mem->slot); - kvm_flush_remote_tlbs(kvm); - kvm_free_physmem_slot(&old, &new); return 0; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 5a54e328088d..6abb2ed1a908 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -2637,3 +2638,53 @@ void kvm_arch_destroy_vm(struct kvm *kvm) kvm_free_physmem(kvm); kfree(kvm); } + +int kvm_arch_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + struct kvm_memory_slot old, + int user_alloc) +{ + int npages = mem->memory_size >> PAGE_SHIFT; + struct kvm_memory_slot *memslot = &kvm->memslots[mem->slot]; + + /*To keep backward compatibility with older userspace, + *x86 needs to hanlde !user_alloc case. + */ + if (!user_alloc) { + if (npages && !old.rmap) { + down_write(¤t->mm->mmap_sem); + memslot->userspace_addr = do_mmap(NULL, 0, + npages * PAGE_SIZE, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, + 0); + up_write(¤t->mm->mmap_sem); + + if (IS_ERR((void *)memslot->userspace_addr)) + return PTR_ERR((void *)memslot->userspace_addr); + } else { + if (!old.user_alloc && old.rmap) { + int ret; + + down_write(¤t->mm->mmap_sem); + ret = do_munmap(current->mm, old.userspace_addr, + old.npages * PAGE_SIZE); + up_write(¤t->mm->mmap_sem); + if (ret < 0) + printk(KERN_WARNING + "kvm_vm_ioctl_set_memory_region: " + "failed to munmap memory\n"); + } + } + } + + if (!kvm->n_requested_mmu_pages) { + unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm); + kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); + } + + kvm_mmu_slot_remove_write_access(kvm, mem->slot); + kvm_flush_remote_tlbs(kvm); + + return 0; +} From 26e5215fdc6cf7b5a8b1269134095abbb7338b3c Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 20 Nov 2007 15:30:24 +0200 Subject: [PATCH 143/249] KVM: Split vcpu creation to avoid vcpu_load() before preemption setup Split kvm_arch_vcpu_create() into kvm_arch_vcpu_create() and kvm_arch_vcpu_setup(), enabling preemption notification between the two. This mean that we can now do vcpu_load() within kvm_arch_vcpu_setup(). Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/kvm_main.c | 4 ++++ drivers/kvm/x86.c | 16 +++++++--------- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 49094a221f6a..b65f5dee4b1b 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -466,6 +466,7 @@ void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id); +int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu); int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 9dd6ad3c6c7b..7939b5c9a4b4 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -769,6 +769,10 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n) preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); + r = kvm_arch_vcpu_setup(vcpu); + if (r) + goto vcpu_destroy; + mutex_lock(&kvm->lock); if (kvm->vcpus[n]) { r = -EEXIST; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 6abb2ed1a908..b482b6a8a828 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -2478,13 +2478,12 @@ void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id) { - int r; - struct kvm_vcpu *vcpu = kvm_x86_ops->vcpu_create(kvm, id); + return kvm_x86_ops->vcpu_create(kvm, id); +} - if (IS_ERR(vcpu)) { - r = -ENOMEM; - goto fail; - } +int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) +{ + int r; /* We do fxsave: this must be aligned. */ BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF); @@ -2497,11 +2496,10 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, if (r < 0) goto free_vcpu; - return vcpu; + return 0; free_vcpu: kvm_x86_ops->vcpu_free(vcpu); -fail: - return ERR_PTR(r); + return r; } void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) From e5a4c8cad952d6e060991841f3632578ab3ae133 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 20 Nov 2007 21:39:54 +0200 Subject: [PATCH 144/249] KVM: MMU: Implement guest page fault bypass for nonpae I spent an hour worrying why I see so many guest page faults on FC6 i386. Turns out bypass wasn't implemented for nonpae. Implement it so it doesn't happen again. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index bf15d127a48f..92b93134a2a6 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -486,19 +486,22 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr) static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) { - int i; + int i, offset = 0; pt_element_t *gpt; struct page *page; - if (sp->role.metaphysical || PTTYPE == 32) { + if (sp->role.metaphysical + || (PTTYPE == 32 && sp->role.level > PT_PAGE_TABLE_LEVEL)) { nonpaging_prefetch_page(vcpu, sp); return; } + if (PTTYPE == 32) + offset = sp->role.quadrant << PT64_LEVEL_BITS; page = gfn_to_page(vcpu->kvm, sp->gfn); gpt = kmap_atomic(page, KM_USER0); for (i = 0; i < PT64_ENT_PER_PAGE; ++i) - if (is_present_pte(gpt[i])) + if (is_present_pte(gpt[offset + i])) sp->spt[i] = shadow_trap_nonpresent_pte; else sp->spt[i] = shadow_notrap_nonpresent_pte; From 0f74a24c59b814c1c8085251cbea48d339f0c7c6 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 20 Nov 2007 23:01:14 +0200 Subject: [PATCH 145/249] KVM: Add statistic for remote tlb flushes Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/kvm_main.c | 3 +++ drivers/kvm/x86.c | 1 + 3 files changed, 5 insertions(+) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index b65f5dee4b1b..048849d97b35 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -300,6 +300,7 @@ struct kvm_vm_stat { u32 mmu_pde_zapped; u32 mmu_flooded; u32 mmu_recycled; + u32 remote_tlb_flush; }; struct kvm { diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 7939b5c9a4b4..aec6b67cfebb 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -115,6 +115,9 @@ void kvm_flush_remote_tlbs(struct kvm *kvm) if (cpu != -1 && cpu != raw_smp_processor_id()) cpu_set(cpu, cpus); } + if (cpus_empty(cpus)) + return; + ++kvm->stat.remote_tlb_flush; smp_call_function_mask(cpus, ack_flush, NULL, 1); } diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index b482b6a8a828..ac09f381f47f 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -73,6 +73,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) }, { "mmu_flooded", VM_STAT(mmu_flooded) }, { "mmu_recycled", VM_STAT(mmu_recycled) }, + { "remote_tlb_flush", VM_STAT(remote_tlb_flush) }, { NULL } }; From 79539cec0c3c38d35a1e3e5310d2c562ae6e82b8 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 02:06:21 +0200 Subject: [PATCH 146/249] KVM: MMU: Avoid unnecessary remote tlb flushes when guest updates a pte If all we're doing is increasing permissions on a pte (typical for demand paging), then there's not need to flush remote tlbs. Worst case they'll get a spurious page fault. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 101cd5377a89..281dd5f9310c 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -134,6 +134,8 @@ static int dbg = 1; #define PT32_DIR_BASE_ADDR_MASK \ (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1)) +#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \ + | PT64_NX_MASK) #define PFERR_PRESENT_MASK (1U << 0) #define PFERR_WRITE_MASK (1U << 1) @@ -1227,7 +1229,6 @@ static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu, } } set_shadow_pte(spte, shadow_trap_nonpresent_pte); - kvm_flush_remote_tlbs(vcpu->kvm); } static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, @@ -1250,6 +1251,27 @@ static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, offset_in_pte); } +static bool need_remote_flush(u64 old, u64 new) +{ + if (!is_shadow_present_pte(old)) + return false; + if (!is_shadow_present_pte(new)) + return true; + if ((old ^ new) & PT64_BASE_ADDR_MASK) + return true; + old ^= PT64_NX_MASK; + new ^= PT64_NX_MASK; + return (old & ~new & PT64_PERM_MASK) != 0; +} + +static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, u64 old, u64 new) +{ + if (need_remote_flush(old, new)) + kvm_flush_remote_tlbs(vcpu->kvm); + else + kvm_mmu_flush_tlb(vcpu); +} + static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu) { u64 *spte = vcpu->last_pte_updated; @@ -1265,6 +1287,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, struct hlist_node *node, *n; struct hlist_head *bucket; unsigned index; + u64 entry; u64 *spte; unsigned offset = offset_in_page(gpa); unsigned pte_size; @@ -1335,9 +1358,11 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, } spte = &page->spt[page_offset / sizeof(*spte)]; while (npte--) { + entry = *spte; mmu_pte_write_zap_pte(vcpu, page, spte); mmu_pte_write_new_pte(vcpu, page, spte, new, bytes, page_offset & (pte_size - 1)); + mmu_pte_write_flush_tlb(vcpu, entry, *spte); ++spte; } } From d835dfecd00fd770288dcd9a46c0e0966d526fdf Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 02:57:59 +0200 Subject: [PATCH 147/249] KVM: Don't bother the mmu if cr3 load doesn't change cr3 If the guest requests just a tlb flush, don't take the vm lock and drop the mmu context pointlessly. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 2 +- drivers/kvm/x86.c | 25 +++++++++++++++++++++++++ drivers/kvm/x86.h | 1 + 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 281dd5f9310c..346aa65a08d5 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -1086,7 +1086,7 @@ static int nonpaging_init_context(struct kvm_vcpu *vcpu) return 0; } -static void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu) +void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu) { ++vcpu->stat.tlb_flush; kvm_x86_ops->tlb_flush(vcpu); diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index ac09f381f47f..15e1203faef0 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -166,6 +166,26 @@ out: return ret; } +static bool pdptrs_changed(struct kvm_vcpu *vcpu) +{ + u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)]; + bool changed = true; + int r; + + if (is_long_mode(vcpu) || !is_pae(vcpu)) + return false; + + mutex_lock(&vcpu->kvm->lock); + r = kvm_read_guest(vcpu->kvm, vcpu->cr3 & ~31u, pdpte, sizeof(pdpte)); + if (r < 0) + goto out; + changed = memcmp(pdpte, vcpu->pdptrs, sizeof(pdpte)) != 0; +out: + mutex_unlock(&vcpu->kvm->lock); + + return changed; +} + void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) { if (cr0 & CR0_RESERVED_BITS) { @@ -271,6 +291,11 @@ EXPORT_SYMBOL_GPL(set_cr4); void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) { + if (cr3 == vcpu->cr3 && !pdptrs_changed(vcpu)) { + kvm_mmu_flush_tlb(vcpu); + return; + } + if (is_long_mode(vcpu)) { if (cr3 & CR3_L_MODE_RESERVED_BITS) { printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n"); diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 71f2477d03fd..b1528c9f566f 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -299,6 +299,7 @@ int emulator_write_emulated(unsigned long addr, unsigned long segment_base(u16 selector); +void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu); void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, int bytes); int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva); From b238f7bc2dd5e0e8ecbd28863d6d03e490c652ce Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Tue, 20 Nov 2007 12:02:12 +0200 Subject: [PATCH 148/249] KVM: MMU: Code cleanup Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 92b93134a2a6..6e013015f0a8 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -187,6 +187,7 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, int dirty = gpte & PT_DIRTY_MASK; u64 spte; int was_rmapped = is_rmap_pte(*shadow_pte); + struct page *page; pgprintk("%s: spte %llx gpte %llx access %llx write_fault %d" " user_fault %d gfn %lx\n", @@ -205,6 +206,12 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, paddr = gpa_to_hpa(vcpu->kvm, gaddr & PT64_BASE_ADDR_MASK); + /* + * the reason paddr get mask even that it isnt pte is beacuse the + * HPA_ERR_MASK bit might be used to signal error + */ + page = pfn_to_page((paddr & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); + spte |= PT_PRESENT_MASK; if (access_bits & PT_USER_MASK) spte |= PT_USER_MASK; @@ -212,8 +219,7 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, if (is_error_hpa(paddr)) { set_shadow_pte(shadow_pte, shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK); - kvm_release_page_clean(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT)); + kvm_release_page_clean(page); return; } @@ -254,17 +260,11 @@ unshadowed: if (!was_rmapped) { rmap_add(vcpu, shadow_pte, (gaddr & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); - if (!is_rmap_pte(*shadow_pte)) { - struct page *page; - - page = pfn_to_page((paddr & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT); + if (!is_rmap_pte(*shadow_pte)) kvm_release_page_clean(page); - } } else - kvm_release_page_clean(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT)); + kvm_release_page_clean(page); if (!ptwrite || !*ptwrite) vcpu->last_pte_updated = shadow_pte; } From 5fb07ddb183eb4728f240e09c3305d271ce61d48 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 12:35:07 +0200 Subject: [PATCH 149/249] KVM: MMU: Introduce and use gpte_to_gfn() Instead of repretitively open-coding this. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 6e013015f0a8..6f79ae87d0cf 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -52,6 +52,9 @@ #error Invalid PTTYPE value #endif +#define gpte_to_gfn FNAME(gpte_to_gfn) +#define gpte_to_gfn_pde FNAME(gpte_to_gfn_pde) + /* * The guest_walker structure emulates the behavior of the hardware page * table walker. @@ -65,6 +68,16 @@ struct guest_walker { u32 error_code; }; +static gfn_t gpte_to_gfn(pt_element_t gpte) +{ + return (gpte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; +} + +static gfn_t gpte_to_gfn_pde(pt_element_t gpte) +{ + return (gpte & PT_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT; +} + /* * Fetch a guest pte for a guest virtual address */ @@ -96,7 +109,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, for (;;) { index = PT_INDEX(addr, walker->level); - table_gfn = (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT; + table_gfn = gpte_to_gfn(pte); pte_gpa = table_gfn << PAGE_SHIFT; pte_gpa += index * sizeof(pt_element_t); walker->table_gfn[walker->level - 1] = table_gfn; @@ -127,15 +140,14 @@ static int FNAME(walk_addr)(struct guest_walker *walker, } if (walker->level == PT_PAGE_TABLE_LEVEL) { - walker->gfn = (pte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; + walker->gfn = gpte_to_gfn(pte); break; } if (walker->level == PT_DIRECTORY_LEVEL && (pte & PT_PAGE_SIZE_MASK) && (PTTYPE == 64 || is_pse(vcpu))) { - walker->gfn = (pte & PT_DIR_BASE_ADDR_MASK) - >> PAGE_SHIFT; + walker->gfn = gpte_to_gfn_pde(pte); walker->gfn += PT_INDEX(addr, PT_PAGE_TABLE_LEVEL); break; } @@ -296,8 +308,7 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, return; pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); FNAME(set_pte)(vcpu, gpte, spte, PT_USER_MASK | PT_WRITABLE_MASK, 0, - 0, NULL, NULL, - (gpte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT); + 0, NULL, NULL, gpte_to_gfn(gpte)); } static void FNAME(set_pde)(struct kvm_vcpu *vcpu, pt_element_t gpde, @@ -370,8 +381,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, hugepage_access >>= PT_WRITABLE_SHIFT; if (walker->pte & PT64_NX_MASK) hugepage_access |= (1 << 2); - table_gfn = (walker->pte & PT_BASE_ADDR_MASK) - >> PAGE_SHIFT; + table_gfn = gpte_to_gfn(walker->pte); } else { metaphysical = 0; table_gfn = walker->table_gfn[level - 2]; @@ -519,3 +529,5 @@ static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, #undef PT_DIR_BASE_ADDR_MASK #undef PT_LEVEL_BITS #undef PT_MAX_FULL_LEVELS +#undef gpte_to_gfn +#undef gpte_to_gfn_pde From da928521b7a095e85f6ac92a6c106bca7e17625f Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 13:54:47 +0200 Subject: [PATCH 150/249] KVM: MMU: Move pse36 handling to the guest walker Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 7 +++++++ drivers/kvm/paging_tmpl.h | 5 ++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 346aa65a08d5..a9fed594ac58 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -218,6 +218,13 @@ static int is_rmap_pte(u64 pte) && pte != shadow_notrap_nonpresent_pte; } +static gfn_t pse36_gfn_delta(u32 gpte) +{ + int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT; + + return (gpte & PT32_DIR_PSE36_MASK) << shift; +} + static void set_shadow_pte(u64 *sptep, u64 spte) { #ifdef CONFIG_X86_64 diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 6f79ae87d0cf..dceb4b9137a4 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -149,6 +149,8 @@ static int FNAME(walk_addr)(struct guest_walker *walker, && (PTTYPE == 64 || is_pse(vcpu))) { walker->gfn = gpte_to_gfn_pde(pte); walker->gfn += PT_INDEX(addr, PT_PAGE_TABLE_LEVEL); + if (PTTYPE == 32 && is_cpuid_PSE36()) + walker->gfn += pse36_gfn_delta(pte); break; } @@ -320,9 +322,6 @@ static void FNAME(set_pde)(struct kvm_vcpu *vcpu, pt_element_t gpde, access_bits &= gpde; gaddr = (gpa_t)gfn << PAGE_SHIFT; - if (PTTYPE == 32 && is_cpuid_PSE36()) - gaddr |= (gpde & PT32_DIR_PSE36_MASK) << - (32 - PT32_DIR_PSE36_SHIFT); FNAME(set_pte_common)(vcpu, shadow_pte, gaddr, gpde, access_bits, user_fault, write_fault, ptwrite, walker, gfn); From 4e542370c7570bdb582ab8c751ab899ff9e4214a Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 14:08:40 +0200 Subject: [PATCH 151/249] KVM: MMU: Remove extra gaddr parameter from set_pte_common() Similar information is available in the gfn parameter, so use that. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 1 + drivers/kvm/paging_tmpl.h | 29 +++++++++-------------------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index a9fed594ac58..c3362ba33b0a 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -29,6 +29,7 @@ #include #include +#include #undef MMU_DEBUG diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index dceb4b9137a4..cc373ed1f049 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -188,7 +188,6 @@ err: static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, u64 *shadow_pte, - gpa_t gaddr, pt_element_t gpte, u64 access_bits, int user_fault, @@ -197,7 +196,6 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, struct guest_walker *walker, gfn_t gfn) { - hpa_t paddr; int dirty = gpte & PT_DIRTY_MASK; u64 spte; int was_rmapped = is_rmap_pte(*shadow_pte); @@ -218,26 +216,20 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, if (!dirty) access_bits &= ~PT_WRITABLE_MASK; - paddr = gpa_to_hpa(vcpu->kvm, gaddr & PT64_BASE_ADDR_MASK); - - /* - * the reason paddr get mask even that it isnt pte is beacuse the - * HPA_ERR_MASK bit might be used to signal error - */ - page = pfn_to_page((paddr & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); + page = gfn_to_page(vcpu->kvm, gfn); spte |= PT_PRESENT_MASK; if (access_bits & PT_USER_MASK) spte |= PT_USER_MASK; - if (is_error_hpa(paddr)) { + if (is_error_page(page)) { set_shadow_pte(shadow_pte, shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK); kvm_release_page_clean(page); return; } - spte |= paddr; + spte |= page_to_phys(page); if ((access_bits & PT_WRITABLE_MASK) || (write_fault && !is_write_protection(vcpu) && !user_fault)) { @@ -266,14 +258,14 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, unshadowed: if (access_bits & PT_WRITABLE_MASK) - mark_page_dirty(vcpu->kvm, gaddr >> PAGE_SHIFT); + mark_page_dirty(vcpu->kvm, gfn); pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte); set_shadow_pte(shadow_pte, spte); - page_header_update_slot(vcpu->kvm, shadow_pte, gaddr); + page_header_update_slot(vcpu->kvm, shadow_pte, + (gpa_t)gfn << PAGE_SHIFT); if (!was_rmapped) { - rmap_add(vcpu, shadow_pte, (gaddr & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT); + rmap_add(vcpu, shadow_pte, gfn); if (!is_rmap_pte(*shadow_pte)) kvm_release_page_clean(page); } @@ -289,7 +281,7 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, struct guest_walker *walker, gfn_t gfn) { access_bits &= gpte; - FNAME(set_pte_common)(vcpu, shadow_pte, gpte & PT_BASE_ADDR_MASK, + FNAME(set_pte_common)(vcpu, shadow_pte, gpte, access_bits, user_fault, write_fault, ptwrite, walker, gfn); } @@ -318,11 +310,8 @@ static void FNAME(set_pde)(struct kvm_vcpu *vcpu, pt_element_t gpde, int user_fault, int write_fault, int *ptwrite, struct guest_walker *walker, gfn_t gfn) { - gpa_t gaddr; - access_bits &= gpde; - gaddr = (gpa_t)gfn << PAGE_SHIFT; - FNAME(set_pte_common)(vcpu, shadow_pte, gaddr, + FNAME(set_pte_common)(vcpu, shadow_pte, gpde, access_bits, user_fault, write_fault, ptwrite, walker, gfn); } From 050e64992f90abee4e1e77b57fe7abf1ba22d330 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 14:11:49 +0200 Subject: [PATCH 152/249] KVM: MMU: Remove set_pde() It is now identical to set_pte(). Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index cc373ed1f049..062f4f574649 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -305,17 +305,6 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, 0, NULL, NULL, gpte_to_gfn(gpte)); } -static void FNAME(set_pde)(struct kvm_vcpu *vcpu, pt_element_t gpde, - u64 *shadow_pte, u64 access_bits, - int user_fault, int write_fault, int *ptwrite, - struct guest_walker *walker, gfn_t gfn) -{ - access_bits &= gpde; - FNAME(set_pte_common)(vcpu, shadow_pte, - gpde, access_bits, user_fault, write_fault, - ptwrite, walker, gfn); -} - /* * Fetch a shadow pte for a specific level in the paging hierarchy. */ @@ -384,16 +373,10 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, prev_shadow_ent = shadow_ent; } - if (walker->level == PT_DIRECTORY_LEVEL) { - FNAME(set_pde)(vcpu, walker->pte, shadow_ent, - walker->inherited_ar, user_fault, write_fault, - ptwrite, walker, walker->gfn); - } else { - ASSERT(walker->level == PT_PAGE_TABLE_LEVEL); - FNAME(set_pte)(vcpu, walker->pte, shadow_ent, - walker->inherited_ar, user_fault, write_fault, - ptwrite, walker, walker->gfn); - } + FNAME(set_pte)(vcpu, walker->pte, shadow_ent, + walker->inherited_ar, user_fault, write_fault, + ptwrite, walker, walker->gfn); + return shadow_ent; } From 230c9a8f2354dc51f55bf81147f408d8fc2f3dd1 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 14:16:30 +0200 Subject: [PATCH 153/249] KVM: MMU: Merge set_pte() and set_pte_common() Since set_pte() is now the only caller of set_pte_common(), merge the two functions. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 062f4f574649..54a6ee802dc2 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -186,15 +186,11 @@ err: return 0; } -static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, - u64 *shadow_pte, - pt_element_t gpte, - u64 access_bits, - int user_fault, - int write_fault, - int *ptwrite, - struct guest_walker *walker, - gfn_t gfn) +static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, + u64 *shadow_pte, u64 access_bits, + int user_fault, int write_fault, + int *ptwrite, struct guest_walker *walker, + gfn_t gfn) { int dirty = gpte & PT_DIRTY_MASK; u64 spte; @@ -206,6 +202,7 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, __FUNCTION__, *shadow_pte, (u64)gpte, access_bits, write_fault, user_fault, gfn); + access_bits &= gpte; /* * We don't set the accessed bit, since we sometimes want to see * whether the guest actually used the pte (in order to detect @@ -275,17 +272,6 @@ unshadowed: vcpu->last_pte_updated = shadow_pte; } -static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, - u64 *shadow_pte, u64 access_bits, - int user_fault, int write_fault, int *ptwrite, - struct guest_walker *walker, gfn_t gfn) -{ - access_bits &= gpte; - FNAME(set_pte_common)(vcpu, shadow_pte, - gpte, access_bits, user_fault, write_fault, - ptwrite, walker, gfn); -} - static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, u64 *spte, const void *pte, int bytes, int offset_in_pte) From 38c335f1f57bd9edc3cd894e8f5f3fc133320ad3 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 14:20:22 +0200 Subject: [PATCH 154/249] KVM: MMU: Adjust page_header_update_slot() to accept a gfn instead of a gpa Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 7 ++++--- drivers/kvm/paging_tmpl.h | 3 +-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index c3362ba33b0a..1dcffc49eff8 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -860,9 +860,9 @@ static void mmu_unshadow(struct kvm *kvm, gfn_t gfn) } } -static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa) +static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn) { - int slot = memslot_id(kvm, gfn_to_memslot(kvm, gpa >> PAGE_SHIFT)); + int slot = memslot_id(kvm, gfn_to_memslot(kvm, gfn)); struct kvm_mmu_page *page_head = page_header(__pa(pte)); __set_bit(slot, &page_head->slot_bitmap); @@ -928,7 +928,8 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) return 0; } mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT); - page_header_update_slot(vcpu->kvm, table, v); + page_header_update_slot(vcpu->kvm, table, + v >> PAGE_SHIFT); table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK; if (!was_rmapped) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 54a6ee802dc2..a3da98bfd6a6 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -259,8 +259,7 @@ unshadowed: pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte); set_shadow_pte(shadow_pte, spte); - page_header_update_slot(vcpu->kvm, shadow_pte, - (gpa_t)gfn << PAGE_SHIFT); + page_header_update_slot(vcpu->kvm, shadow_pte, gfn); if (!was_rmapped) { rmap_add(vcpu, shadow_pte, gfn); if (!is_rmap_pte(*shadow_pte)) From 1755fbcc6666906cada43c1c839ca1fc6616d4a6 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 14:44:45 +0200 Subject: [PATCH 155/249] KVM: MMU: Introduce gfn_to_gpa() Converting a frame number to an address is tricky since the data type changes size. Introduce a function to do it. This fixes an actual bug when accessing guest ptes. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 4 ++++ drivers/kvm/paging_tmpl.h | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 048849d97b35..eda82cded88e 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -499,6 +499,10 @@ static inline int memslot_id(struct kvm *kvm, struct kvm_memory_slot *slot) return slot - kvm->memslots; } +static inline gpa_t gfn_to_gpa(gfn_t gfn) +{ + return (gpa_t)gfn << PAGE_SHIFT; +} enum kvm_stat_kind { KVM_STAT_VM, diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index a3da98bfd6a6..b24bc7c86078 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -110,7 +110,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, index = PT_INDEX(addr, walker->level); table_gfn = gpte_to_gfn(pte); - pte_gpa = table_gfn << PAGE_SHIFT; + pte_gpa = gfn_to_gpa(table_gfn); pte_gpa += index * sizeof(pt_element_t); walker->table_gfn[walker->level - 1] = table_gfn; pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, @@ -442,7 +442,7 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr) r = FNAME(walk_addr)(&walker, vcpu, vaddr, 0, 0, 0); if (r) { - gpa = (gpa_t)walker.gfn << PAGE_SHIFT; + gpa = gfn_to_gpa(walker.gfn); gpa |= vaddr & ~PAGE_MASK; } From 3f3e7124f66df2d36751b0c97eabe38a13752fff Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 14:54:16 +0200 Subject: [PATCH 156/249] KVM: MMU: Simplify nonpaging_map() Instead of passing an hpa, pass a regular struct page. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 1dcffc49eff8..1965185bbe23 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -903,13 +903,11 @@ static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) { } -static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) +static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, struct page *page) { int level = PT32E_ROOT_LEVEL; hpa_t table_addr = vcpu->mmu.root_hpa; - struct page *page; - page = pfn_to_page(p >> PAGE_SHIFT); for (; ; level--) { u32 index = PT64_INDEX(v, level); u64 *table; @@ -930,8 +928,9 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT); page_header_update_slot(vcpu->kvm, table, v >> PAGE_SHIFT); - table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK | - PT_USER_MASK; + table[index] = page_to_phys(page) + | PT_PRESENT_MASK | PT_WRITABLE_MASK + | PT_USER_MASK; if (!was_rmapped) rmap_add(vcpu, &table[index], v >> PAGE_SHIFT); else @@ -1050,10 +1049,9 @@ static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr) } static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, - u32 error_code) + u32 error_code) { - gpa_t addr = gva; - hpa_t paddr; + struct page *page; int r; r = mmu_topup_memory_caches(vcpu); @@ -1063,16 +1061,14 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, ASSERT(vcpu); ASSERT(VALID_PAGE(vcpu->mmu.root_hpa)); + page = gfn_to_page(vcpu->kvm, gva >> PAGE_SHIFT); - paddr = gpa_to_hpa(vcpu->kvm, addr & PT64_BASE_ADDR_MASK); - - if (is_error_hpa(paddr)) { - kvm_release_page_clean(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT)); + if (is_error_page(page)) { + kvm_release_page_clean(page); return 1; } - return nonpaging_map(vcpu, addr & PAGE_MASK, paddr); + return nonpaging_map(vcpu, gva & PAGE_MASK, page); } static void nonpaging_free(struct kvm_vcpu *vcpu) From 0d81f2966a61953c4cc7397342f5f907b3a6e257 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 14:57:44 +0200 Subject: [PATCH 157/249] KVM: MMU: Remove gva_to_hpa() No longer used. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 - drivers/kvm/mmu.c | 9 --------- 2 files changed, 10 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index eda82cded88e..31315bcf09c3 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -379,7 +379,6 @@ hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa); #define HPA_MSB ((sizeof(hpa_t) * 8) - 1) #define HPA_ERR_MASK ((hpa_t)1 << HPA_MSB) static inline int is_error_hpa(hpa_t hpa) { return hpa >> HPA_MSB; } -hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva); struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva); extern struct page *bad_page; diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 1965185bbe23..6aa0319ede4d 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -881,15 +881,6 @@ hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa) return hpa; } -hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva) -{ - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva); - - if (gpa == UNMAPPED_GVA) - return UNMAPPED_GVA; - return gpa_to_hpa(vcpu->kvm, gpa); -} - struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva) { gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva); From 1d28f5f4a4984be4cd7200ed512c94517c13e392 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 15:01:44 +0200 Subject: [PATCH 158/249] KVM: Remove gpa_to_hpa() Converting last uses along the way. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 - drivers/kvm/mmu.c | 21 +++------------------ 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 31315bcf09c3..1fd8158ced89 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -375,7 +375,6 @@ int kvm_init(void *opaque, unsigned int vcpu_size, struct module *module); void kvm_exit(void); -hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa); #define HPA_MSB ((sizeof(hpa_t) * 8) - 1) #define HPA_ERR_MASK ((hpa_t)1 << HPA_MSB) static inline int is_error_hpa(hpa_t hpa) { return hpa >> HPA_MSB; } diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 6aa0319ede4d..9b75b102b8d1 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -868,26 +868,13 @@ static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn) __set_bit(slot, &page_head->slot_bitmap); } -hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa) -{ - struct page *page; - hpa_t hpa; - - ASSERT((gpa & HPA_ERR_MASK) == 0); - page = gfn_to_page(kvm, gpa >> PAGE_SHIFT); - hpa = ((hpa_t)page_to_pfn(page) << PAGE_SHIFT) | (gpa & (PAGE_SIZE-1)); - if (is_error_page(page)) - return hpa | HPA_ERR_MASK; - return hpa; -} - struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva) { gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva); if (gpa == UNMAPPED_GVA) return NULL; - return pfn_to_page(gpa_to_hpa(vcpu->kvm, gpa) >> PAGE_SHIFT); + return gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); } static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) @@ -1611,8 +1598,8 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, audit_mappings_page(vcpu, ent, va, level - 1); } else { gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va); - hpa_t hpa = gpa_to_hpa(vcpu, gpa); - struct page *page; + struct page *page = gpa_to_page(vcpu, gpa); + hpa_t hpa = page_to_phys(page); if (is_shadow_present_pte(ent) && (ent & PT64_BASE_ADDR_MASK) != hpa) @@ -1625,8 +1612,6 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, && !is_error_hpa(hpa)) printk(KERN_ERR "audit: (%s) notrap shadow," " valid guest gva %lx\n", audit_msg, va); - page = pfn_to_page((gpa & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT); kvm_release_page_clean(page); } From 4db3531487bdde4027b701da7c1b8d094ae2cc91 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 15:28:32 +0200 Subject: [PATCH 159/249] KVM: MMU: Rename variables of type 'struct kvm_mmu_page *' These are traditionally named 'page', but even more traditionally, that name is reserved for variables that point to a 'struct page'. Rename them to 'sp' (for "shadow page"). Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 300 ++++++++++++++++++++++------------------------ 1 file changed, 146 insertions(+), 154 deletions(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 9b75b102b8d1..86896da3e837 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -367,7 +367,7 @@ static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn) */ static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) { - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; struct kvm_rmap_desc *desc; unsigned long *rmapp; int i; @@ -375,8 +375,8 @@ static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) if (!is_rmap_pte(*spte)) return; gfn = unalias_gfn(vcpu->kvm, gfn); - page = page_header(__pa(spte)); - page->gfns[spte - page->spt] = gfn; + sp = page_header(__pa(spte)); + sp->gfns[spte - sp->spt] = gfn; rmapp = gfn_to_rmap(vcpu->kvm, gfn); if (!*rmapp) { rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte); @@ -429,20 +429,20 @@ static void rmap_remove(struct kvm *kvm, u64 *spte) { struct kvm_rmap_desc *desc; struct kvm_rmap_desc *prev_desc; - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; struct page *release_page; unsigned long *rmapp; int i; if (!is_rmap_pte(*spte)) return; - page = page_header(__pa(spte)); + sp = page_header(__pa(spte)); release_page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); if (is_writeble_pte(*spte)) kvm_release_page_dirty(release_page); else kvm_release_page_clean(release_page); - rmapp = gfn_to_rmap(kvm, page->gfns[spte - page->spt]); + rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt]); if (!*rmapp) { printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte); BUG(); @@ -537,14 +537,13 @@ static int is_empty_shadow_page(u64 *spt) } #endif -static void kvm_mmu_free_page(struct kvm *kvm, - struct kvm_mmu_page *page_head) +static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp) { - ASSERT(is_empty_shadow_page(page_head->spt)); - list_del(&page_head->link); - __free_page(virt_to_page(page_head->spt)); - __free_page(virt_to_page(page_head->gfns)); - kfree(page_head); + ASSERT(is_empty_shadow_page(sp->spt)); + list_del(&sp->link); + __free_page(virt_to_page(sp->spt)); + __free_page(virt_to_page(sp->gfns)); + kfree(sp); ++kvm->n_free_mmu_pages; } @@ -556,27 +555,26 @@ static unsigned kvm_page_table_hashfn(gfn_t gfn) static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, u64 *parent_pte) { - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; if (!vcpu->kvm->n_free_mmu_pages) return NULL; - page = mmu_memory_cache_alloc(&vcpu->mmu_page_header_cache, - sizeof *page); - page->spt = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE); - page->gfns = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE); - set_page_private(virt_to_page(page->spt), (unsigned long)page); - list_add(&page->link, &vcpu->kvm->active_mmu_pages); - ASSERT(is_empty_shadow_page(page->spt)); - page->slot_bitmap = 0; - page->multimapped = 0; - page->parent_pte = parent_pte; + sp = mmu_memory_cache_alloc(&vcpu->mmu_page_header_cache, sizeof *sp); + sp->spt = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE); + sp->gfns = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE); + set_page_private(virt_to_page(sp->spt), (unsigned long)sp); + list_add(&sp->link, &vcpu->kvm->active_mmu_pages); + ASSERT(is_empty_shadow_page(sp->spt)); + sp->slot_bitmap = 0; + sp->multimapped = 0; + sp->parent_pte = parent_pte; --vcpu->kvm->n_free_mmu_pages; - return page; + return sp; } static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *page, u64 *parent_pte) + struct kvm_mmu_page *sp, u64 *parent_pte) { struct kvm_pte_chain *pte_chain; struct hlist_node *node; @@ -584,20 +582,20 @@ static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu, if (!parent_pte) return; - if (!page->multimapped) { - u64 *old = page->parent_pte; + if (!sp->multimapped) { + u64 *old = sp->parent_pte; if (!old) { - page->parent_pte = parent_pte; + sp->parent_pte = parent_pte; return; } - page->multimapped = 1; + sp->multimapped = 1; pte_chain = mmu_alloc_pte_chain(vcpu); - INIT_HLIST_HEAD(&page->parent_ptes); - hlist_add_head(&pte_chain->link, &page->parent_ptes); + INIT_HLIST_HEAD(&sp->parent_ptes); + hlist_add_head(&pte_chain->link, &sp->parent_ptes); pte_chain->parent_ptes[0] = old; } - hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link) { + hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) { if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1]) continue; for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) @@ -608,23 +606,23 @@ static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu, } pte_chain = mmu_alloc_pte_chain(vcpu); BUG_ON(!pte_chain); - hlist_add_head(&pte_chain->link, &page->parent_ptes); + hlist_add_head(&pte_chain->link, &sp->parent_ptes); pte_chain->parent_ptes[0] = parent_pte; } -static void mmu_page_remove_parent_pte(struct kvm_mmu_page *page, +static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp, u64 *parent_pte) { struct kvm_pte_chain *pte_chain; struct hlist_node *node; int i; - if (!page->multimapped) { - BUG_ON(page->parent_pte != parent_pte); - page->parent_pte = NULL; + if (!sp->multimapped) { + BUG_ON(sp->parent_pte != parent_pte); + sp->parent_pte = NULL; return; } - hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link) + hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) { if (!pte_chain->parent_ptes[i]) break; @@ -640,9 +638,9 @@ static void mmu_page_remove_parent_pte(struct kvm_mmu_page *page, if (i == 0) { hlist_del(&pte_chain->link); mmu_free_pte_chain(pte_chain); - if (hlist_empty(&page->parent_ptes)) { - page->multimapped = 0; - page->parent_pte = NULL; + if (hlist_empty(&sp->parent_ptes)) { + sp->multimapped = 0; + sp->parent_pte = NULL; } } return; @@ -650,22 +648,21 @@ static void mmu_page_remove_parent_pte(struct kvm_mmu_page *page, BUG(); } -static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, - gfn_t gfn) +static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn) { unsigned index; struct hlist_head *bucket; - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; struct hlist_node *node; pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn); index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; bucket = &kvm->mmu_page_hash[index]; - hlist_for_each_entry(page, node, bucket, hash_link) - if (page->gfn == gfn && !page->role.metaphysical) { + hlist_for_each_entry(sp, node, bucket, hash_link) + if (sp->gfn == gfn && !sp->role.metaphysical) { pgprintk("%s: found role %x\n", - __FUNCTION__, page->role.word); - return page; + __FUNCTION__, sp->role.word); + return sp; } return NULL; } @@ -682,7 +679,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, unsigned index; unsigned quadrant; struct hlist_head *bucket; - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; struct hlist_node *node; role.word = 0; @@ -699,35 +696,35 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, gfn, role.word); index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; bucket = &vcpu->kvm->mmu_page_hash[index]; - hlist_for_each_entry(page, node, bucket, hash_link) - if (page->gfn == gfn && page->role.word == role.word) { - mmu_page_add_parent_pte(vcpu, page, parent_pte); + hlist_for_each_entry(sp, node, bucket, hash_link) + if (sp->gfn == gfn && sp->role.word == role.word) { + mmu_page_add_parent_pte(vcpu, sp, parent_pte); pgprintk("%s: found\n", __FUNCTION__); - return page; + return sp; } - page = kvm_mmu_alloc_page(vcpu, parent_pte); - if (!page) - return page; + sp = kvm_mmu_alloc_page(vcpu, parent_pte); + if (!sp) + return sp; pgprintk("%s: adding gfn %lx role %x\n", __FUNCTION__, gfn, role.word); - page->gfn = gfn; - page->role = role; - hlist_add_head(&page->hash_link, bucket); - vcpu->mmu.prefetch_page(vcpu, page); + sp->gfn = gfn; + sp->role = role; + hlist_add_head(&sp->hash_link, bucket); + vcpu->mmu.prefetch_page(vcpu, sp); if (!metaphysical) rmap_write_protect(vcpu->kvm, gfn); - return page; + return sp; } static void kvm_mmu_page_unlink_children(struct kvm *kvm, - struct kvm_mmu_page *page) + struct kvm_mmu_page *sp) { unsigned i; u64 *pt; u64 ent; - pt = page->spt; + pt = sp->spt; - if (page->role.level == PT_PAGE_TABLE_LEVEL) { + if (sp->role.level == PT_PAGE_TABLE_LEVEL) { for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { if (is_shadow_present_pte(pt[i])) rmap_remove(kvm, &pt[i]); @@ -749,10 +746,9 @@ static void kvm_mmu_page_unlink_children(struct kvm *kvm, kvm_flush_remote_tlbs(kvm); } -static void kvm_mmu_put_page(struct kvm_mmu_page *page, - u64 *parent_pte) +static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte) { - mmu_page_remove_parent_pte(page, parent_pte); + mmu_page_remove_parent_pte(sp, parent_pte); } static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm) @@ -764,32 +760,31 @@ static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm) kvm->vcpus[i]->last_pte_updated = NULL; } -static void kvm_mmu_zap_page(struct kvm *kvm, - struct kvm_mmu_page *page) +static void kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp) { u64 *parent_pte; ++kvm->stat.mmu_shadow_zapped; - while (page->multimapped || page->parent_pte) { - if (!page->multimapped) - parent_pte = page->parent_pte; + while (sp->multimapped || sp->parent_pte) { + if (!sp->multimapped) + parent_pte = sp->parent_pte; else { struct kvm_pte_chain *chain; - chain = container_of(page->parent_ptes.first, + chain = container_of(sp->parent_ptes.first, struct kvm_pte_chain, link); parent_pte = chain->parent_ptes[0]; } BUG_ON(!parent_pte); - kvm_mmu_put_page(page, parent_pte); + kvm_mmu_put_page(sp, parent_pte); set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte); } - kvm_mmu_page_unlink_children(kvm, page); - if (!page->root_count) { - hlist_del(&page->hash_link); - kvm_mmu_free_page(kvm, page); + kvm_mmu_page_unlink_children(kvm, sp); + if (!sp->root_count) { + hlist_del(&sp->hash_link); + kvm_mmu_free_page(kvm, sp); } else - list_move(&page->link, &kvm->active_mmu_pages); + list_move(&sp->link, &kvm->active_mmu_pages); kvm_mmu_reset_last_pte_updated(kvm); } @@ -831,7 +826,7 @@ static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) { unsigned index; struct hlist_head *bucket; - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; struct hlist_node *node, *n; int r; @@ -839,11 +834,11 @@ static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) r = 0; index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; bucket = &kvm->mmu_page_hash[index]; - hlist_for_each_entry_safe(page, node, n, bucket, hash_link) - if (page->gfn == gfn && !page->role.metaphysical) { + hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) + if (sp->gfn == gfn && !sp->role.metaphysical) { pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn, - page->role.word); - kvm_mmu_zap_page(kvm, page); + sp->role.word); + kvm_mmu_zap_page(kvm, sp); r = 1; } return r; @@ -851,21 +846,20 @@ static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) static void mmu_unshadow(struct kvm *kvm, gfn_t gfn) { - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; - while ((page = kvm_mmu_lookup_page(kvm, gfn)) != NULL) { - pgprintk("%s: zap %lx %x\n", - __FUNCTION__, gfn, page->role.word); - kvm_mmu_zap_page(kvm, page); + while ((sp = kvm_mmu_lookup_page(kvm, gfn)) != NULL) { + pgprintk("%s: zap %lx %x\n", __FUNCTION__, gfn, sp->role.word); + kvm_mmu_zap_page(kvm, sp); } } static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn) { int slot = memslot_id(kvm, gfn_to_memslot(kvm, gfn)); - struct kvm_mmu_page *page_head = page_header(__pa(pte)); + struct kvm_mmu_page *sp = page_header(__pa(pte)); - __set_bit(slot, &page_head->slot_bitmap); + __set_bit(slot, &sp->slot_bitmap); } struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva) @@ -951,7 +945,7 @@ static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu, static void mmu_free_roots(struct kvm_vcpu *vcpu) { int i; - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; if (!VALID_PAGE(vcpu->mmu.root_hpa)) return; @@ -959,8 +953,8 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu) if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) { hpa_t root = vcpu->mmu.root_hpa; - page = page_header(root); - --page->root_count; + sp = page_header(root); + --sp->root_count; vcpu->mmu.root_hpa = INVALID_PAGE; return; } @@ -970,8 +964,8 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu) if (root) { root &= PT64_BASE_ADDR_MASK; - page = page_header(root); - --page->root_count; + sp = page_header(root); + --sp->root_count; } vcpu->mmu.pae_root[i] = INVALID_PAGE; } @@ -982,7 +976,7 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) { int i; gfn_t root_gfn; - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; root_gfn = vcpu->cr3 >> PAGE_SHIFT; @@ -991,10 +985,10 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) hpa_t root = vcpu->mmu.root_hpa; ASSERT(!VALID_PAGE(root)); - page = kvm_mmu_get_page(vcpu, root_gfn, 0, - PT64_ROOT_LEVEL, 0, 0, NULL); - root = __pa(page->spt); - ++page->root_count; + sp = kvm_mmu_get_page(vcpu, root_gfn, 0, + PT64_ROOT_LEVEL, 0, 0, NULL); + root = __pa(sp->spt); + ++sp->root_count; vcpu->mmu.root_hpa = root; return; } @@ -1011,11 +1005,11 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) root_gfn = vcpu->pdptrs[i] >> PAGE_SHIFT; } else if (vcpu->mmu.root_level == 0) root_gfn = 0; - page = kvm_mmu_get_page(vcpu, root_gfn, i << 30, - PT32_ROOT_LEVEL, !is_paging(vcpu), - 0, NULL); - root = __pa(page->spt); - ++page->root_count; + sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, + PT32_ROOT_LEVEL, !is_paging(vcpu), + 0, NULL); + root = __pa(sp->spt); + ++sp->root_count; vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK; } vcpu->mmu.root_hpa = __pa(vcpu->mmu.pae_root); @@ -1196,7 +1190,7 @@ void kvm_mmu_unload(struct kvm_vcpu *vcpu) } static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *page, + struct kvm_mmu_page *sp, u64 *spte) { u64 pte; @@ -1204,7 +1198,7 @@ static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu, pte = *spte; if (is_shadow_present_pte(pte)) { - if (page->role.level == PT_PAGE_TABLE_LEVEL) + if (sp->role.level == PT_PAGE_TABLE_LEVEL) rmap_remove(vcpu->kvm, spte); else { child = page_header(pte & PT64_BASE_ADDR_MASK); @@ -1215,23 +1209,21 @@ static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu, } static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *page, + struct kvm_mmu_page *sp, u64 *spte, const void *new, int bytes, int offset_in_pte) { - if (page->role.level != PT_PAGE_TABLE_LEVEL) { + if (sp->role.level != PT_PAGE_TABLE_LEVEL) { ++vcpu->kvm->stat.mmu_pde_zapped; return; } ++vcpu->kvm->stat.mmu_pte_updated; - if (page->role.glevels == PT32_ROOT_LEVEL) - paging32_update_pte(vcpu, page, spte, new, bytes, - offset_in_pte); + if (sp->role.glevels == PT32_ROOT_LEVEL) + paging32_update_pte(vcpu, sp, spte, new, bytes, offset_in_pte); else - paging64_update_pte(vcpu, page, spte, new, bytes, - offset_in_pte); + paging64_update_pte(vcpu, sp, spte, new, bytes, offset_in_pte); } static bool need_remote_flush(u64 old, u64 new) @@ -1266,7 +1258,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, int bytes) { gfn_t gfn = gpa >> PAGE_SHIFT; - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; struct hlist_node *node, *n; struct hlist_head *bucket; unsigned index; @@ -1296,10 +1288,10 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, } index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; bucket = &vcpu->kvm->mmu_page_hash[index]; - hlist_for_each_entry_safe(page, node, n, bucket, hash_link) { - if (page->gfn != gfn || page->role.metaphysical) + hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) { + if (sp->gfn != gfn || sp->role.metaphysical) continue; - pte_size = page->role.glevels == PT32_ROOT_LEVEL ? 4 : 8; + pte_size = sp->role.glevels == PT32_ROOT_LEVEL ? 4 : 8; misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1); misaligned |= bytes < 4; if (misaligned || flooded) { @@ -1314,15 +1306,15 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, * page. */ pgprintk("misaligned: gpa %llx bytes %d role %x\n", - gpa, bytes, page->role.word); - kvm_mmu_zap_page(vcpu->kvm, page); + gpa, bytes, sp->role.word); + kvm_mmu_zap_page(vcpu->kvm, sp); ++vcpu->kvm->stat.mmu_flooded; continue; } page_offset = offset; - level = page->role.level; + level = sp->role.level; npte = 1; - if (page->role.glevels == PT32_ROOT_LEVEL) { + if (sp->role.glevels == PT32_ROOT_LEVEL) { page_offset <<= 1; /* 32->64 */ /* * A 32-bit pde maps 4MB while the shadow pdes map @@ -1336,14 +1328,14 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, } quadrant = page_offset >> PAGE_SHIFT; page_offset &= ~PAGE_MASK; - if (quadrant != page->role.quadrant) + if (quadrant != sp->role.quadrant) continue; } - spte = &page->spt[page_offset / sizeof(*spte)]; + spte = &sp->spt[page_offset / sizeof(*spte)]; while (npte--) { entry = *spte; - mmu_pte_write_zap_pte(vcpu, page, spte); - mmu_pte_write_new_pte(vcpu, page, spte, new, bytes, + mmu_pte_write_zap_pte(vcpu, sp, spte); + mmu_pte_write_new_pte(vcpu, sp, spte, new, bytes, page_offset & (pte_size - 1)); mmu_pte_write_flush_tlb(vcpu, entry, *spte); ++spte; @@ -1362,11 +1354,11 @@ int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) { while (vcpu->kvm->n_free_mmu_pages < KVM_REFILL_PAGES) { - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; - page = container_of(vcpu->kvm->active_mmu_pages.prev, - struct kvm_mmu_page, link); - kvm_mmu_zap_page(vcpu->kvm, page); + sp = container_of(vcpu->kvm->active_mmu_pages.prev, + struct kvm_mmu_page, link); + kvm_mmu_zap_page(vcpu->kvm, sp); ++vcpu->kvm->stat.mmu_recycled; } } @@ -1413,12 +1405,12 @@ EXPORT_SYMBOL_GPL(kvm_mmu_page_fault); static void free_mmu_pages(struct kvm_vcpu *vcpu) { - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; while (!list_empty(&vcpu->kvm->active_mmu_pages)) { - page = container_of(vcpu->kvm->active_mmu_pages.next, - struct kvm_mmu_page, link); - kvm_mmu_zap_page(vcpu->kvm, page); + sp = container_of(vcpu->kvm->active_mmu_pages.next, + struct kvm_mmu_page, link); + kvm_mmu_zap_page(vcpu->kvm, sp); } free_page((unsigned long)vcpu->mmu.pae_root); } @@ -1480,16 +1472,16 @@ void kvm_mmu_destroy(struct kvm_vcpu *vcpu) void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot) { - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; - list_for_each_entry(page, &kvm->active_mmu_pages, link) { + list_for_each_entry(sp, &kvm->active_mmu_pages, link) { int i; u64 *pt; - if (!test_bit(slot, &page->slot_bitmap)) + if (!test_bit(slot, &sp->slot_bitmap)) continue; - pt = page->spt; + pt = sp->spt; for (i = 0; i < PT64_ENT_PER_PAGE; ++i) /* avoid RMW */ if (pt[i] & PT_WRITABLE_MASK) @@ -1499,10 +1491,10 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot) void kvm_mmu_zap_all(struct kvm *kvm) { - struct kvm_mmu_page *page, *node; + struct kvm_mmu_page *sp, *node; - list_for_each_entry_safe(page, node, &kvm->active_mmu_pages, link) - kvm_mmu_zap_page(kvm, page); + list_for_each_entry_safe(sp, node, &kvm->active_mmu_pages, link) + kvm_mmu_zap_page(kvm, sp); kvm_flush_remote_tlbs(kvm); } @@ -1668,13 +1660,13 @@ static int count_rmaps(struct kvm_vcpu *vcpu) static int count_writable_mappings(struct kvm_vcpu *vcpu) { int nmaps = 0; - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; int i; - list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) { - u64 *pt = page->spt; + list_for_each_entry(sp, &vcpu->kvm->active_mmu_pages, link) { + u64 *pt = sp->spt; - if (page->role.level != PT_PAGE_TABLE_LEVEL) + if (sp->role.level != PT_PAGE_TABLE_LEVEL) continue; for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { @@ -1702,23 +1694,23 @@ static void audit_rmap(struct kvm_vcpu *vcpu) static void audit_write_protection(struct kvm_vcpu *vcpu) { - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; struct kvm_memory_slot *slot; unsigned long *rmapp; gfn_t gfn; - list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) { - if (page->role.metaphysical) + list_for_each_entry(sp, &vcpu->kvm->active_mmu_pages, link) { + if (sp->role.metaphysical) continue; - slot = gfn_to_memslot(vcpu->kvm, page->gfn); - gfn = unalias_gfn(vcpu->kvm, page->gfn); + slot = gfn_to_memslot(vcpu->kvm, sp->gfn); + gfn = unalias_gfn(vcpu->kvm, sp->gfn); rmapp = &slot->rmap[gfn - slot->base_gfn]; if (*rmapp) printk(KERN_ERR "%s: (%s) shadow page has writable" " mappings: gfn %lx role %x\n", - __FUNCTION__, audit_msg, page->gfn, - page->role.word); + __FUNCTION__, audit_msg, sp->gfn, + sp->role.word); } } From 76c35c6e99cb46b936b88cc795c9c886e7fe7bd4 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 15:32:41 +0200 Subject: [PATCH 160/249] KVM: MMU: Rename 'release_page' Rename the awkwardly named variable. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 86896da3e837..0cb6580956ba 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -430,18 +430,18 @@ static void rmap_remove(struct kvm *kvm, u64 *spte) struct kvm_rmap_desc *desc; struct kvm_rmap_desc *prev_desc; struct kvm_mmu_page *sp; - struct page *release_page; + struct page *page; unsigned long *rmapp; int i; if (!is_rmap_pte(*spte)) return; sp = page_header(__pa(spte)); - release_page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); + page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); if (is_writeble_pte(*spte)) - kvm_release_page_dirty(release_page); + kvm_release_page_dirty(page); else - kvm_release_page_clean(release_page); + kvm_release_page_clean(page); rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt]); if (!*rmapp) { printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte); From 6d4e4c4fca5be806b888d606894d914847e82d78 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 16:41:05 +0200 Subject: [PATCH 161/249] KVM: Disallow fork() and similar games when using a VM We don't want the meaning of guest userspace changing under our feet. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/kvm_main.c | 9 +++++++++ kernel/fork.c | 1 + 3 files changed, 11 insertions(+) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 1fd8158ced89..be18620bd656 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -305,6 +305,7 @@ struct kvm_vm_stat { struct kvm { struct mutex lock; /* protects everything except vcpus */ + struct mm_struct *mm; /* userspace tied to this vm */ int naliases; struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS]; int nmemslots; diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index aec6b67cfebb..0efd759e585f 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -165,6 +165,8 @@ static struct kvm *kvm_create_vm(void) if (IS_ERR(kvm)) goto out; + kvm->mm = current->mm; + atomic_inc(&kvm->mm->mm_count); kvm_io_bus_init(&kvm->pio_bus); mutex_init(&kvm->lock); kvm_io_bus_init(&kvm->mmio_bus); @@ -202,12 +204,15 @@ void kvm_free_physmem(struct kvm *kvm) static void kvm_destroy_vm(struct kvm *kvm) { + struct mm_struct *mm = kvm->mm; + spin_lock(&kvm_lock); list_del(&kvm->vm_list); spin_unlock(&kvm_lock); kvm_io_bus_destroy(&kvm->pio_bus); kvm_io_bus_destroy(&kvm->mmio_bus); kvm_arch_destroy_vm(kvm); + mmdrop(mm); } static int kvm_vm_release(struct inode *inode, struct file *filp) @@ -818,6 +823,8 @@ static long kvm_vcpu_ioctl(struct file *filp, void __user *argp = (void __user *)arg; int r; + if (vcpu->kvm->mm != current->mm) + return -EIO; switch (ioctl) { case KVM_RUN: r = -EINVAL; @@ -976,6 +983,8 @@ static long kvm_vm_ioctl(struct file *filp, void __user *argp = (void __user *)arg; int r; + if (kvm->mm != current->mm) + return -EIO; switch (ioctl) { case KVM_CREATE_VCPU: r = kvm_vm_ioctl_create_vcpu(kvm, arg); diff --git a/kernel/fork.c b/kernel/fork.c index 314f5101d2b0..05e0b6f4365b 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -393,6 +393,7 @@ void fastcall __mmdrop(struct mm_struct *mm) destroy_context(mm); free_mm(mm); } +EXPORT_SYMBOL_GPL(__mmdrop); /* * Decrement the use count and release all resources for an mm. From 0771671749b59a507b6da4efb931c44d9691e248 Mon Sep 17 00:00:00 2001 From: Dan Kenigsberg Date: Wed, 21 Nov 2007 17:10:04 +0200 Subject: [PATCH 162/249] KVM: Enhance guest cpuid management The current cpuid management suffers from several problems, which inhibit passing through the host feature set to the guest: - No way to tell which features the host supports While some features can be supported with no changes to kvm, others need explicit support. That means kvm needs to vet the feature set before it is passed to the guest. - No support for indexed or stateful cpuid entries Some cpuid entries depend on ecx as well as on eax, or on internal state in the processor (running cpuid multiple times with the same input returns different output). The current cpuid machinery only supports keying on eax. - No support for save/restore/migrate The internal state above needs to be exposed to userspace so it can be saved or migrated. This patch adds extended cpuid support by means of three new ioctls: - KVM_GET_SUPPORTED_CPUID: get all cpuid entries the host (and kvm) supports - KVM_SET_CPUID2: sets the vcpu's cpuid table - KVM_GET_CPUID2: gets the vcpu's cpuid table, including hidden state [avi: fix original KVM_SET_CPUID not removing nx on non-nx hosts as it did before] Signed-off-by: Dan Kenigsberg Signed-off-by: Avi Kivity --- drivers/kvm/x86.c | 331 ++++++++++++++++++++++++++++++++++++++++-- drivers/kvm/x86.h | 2 +- include/asm-x86/kvm.h | 21 +++ include/linux/kvm.h | 4 + 4 files changed, 348 insertions(+), 10 deletions(-) diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 15e1203faef0..7237cb25f77d 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -646,6 +646,7 @@ int kvm_dev_ioctl_check_extension(long ext) case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: case KVM_CAP_USER_MEMORY: case KVM_CAP_SET_TSS_ADDR: + case KVM_CAP_EXT_CPUID: r = 1; break; default: @@ -708,13 +709,19 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) kvm_put_guest_fpu(vcpu); } -static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) +static int is_efer_nx(void) { u64 efer; - int i; - struct kvm_cpuid_entry *e, *entry; rdmsrl(MSR_EFER, efer); + return efer & EFER_NX; +} + +static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) +{ + int i; + struct kvm_cpuid_entry2 *e, *entry; + entry = NULL; for (i = 0; i < vcpu->cpuid_nent; ++i) { e = &vcpu->cpuid_entries[i]; @@ -723,15 +730,56 @@ static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) break; } } - if (entry && (entry->edx & (1 << 20)) && !(efer & EFER_NX)) { + if (entry && (entry->edx & (1 << 20)) && !is_efer_nx()) { entry->edx &= ~(1 << 20); printk(KERN_INFO "kvm: guest NX capability removed\n"); } } +/* when an old userspace process fills a new kernel module */ static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid *cpuid, struct kvm_cpuid_entry __user *entries) +{ + int r, i; + struct kvm_cpuid_entry *cpuid_entries; + + r = -E2BIG; + if (cpuid->nent > KVM_MAX_CPUID_ENTRIES) + goto out; + r = -ENOMEM; + cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry) * cpuid->nent); + if (!cpuid_entries) + goto out; + r = -EFAULT; + if (copy_from_user(cpuid_entries, entries, + cpuid->nent * sizeof(struct kvm_cpuid_entry))) + goto out_free; + for (i = 0; i < cpuid->nent; i++) { + vcpu->cpuid_entries[i].function = cpuid_entries[i].function; + vcpu->cpuid_entries[i].eax = cpuid_entries[i].eax; + vcpu->cpuid_entries[i].ebx = cpuid_entries[i].ebx; + vcpu->cpuid_entries[i].ecx = cpuid_entries[i].ecx; + vcpu->cpuid_entries[i].edx = cpuid_entries[i].edx; + vcpu->cpuid_entries[i].index = 0; + vcpu->cpuid_entries[i].flags = 0; + vcpu->cpuid_entries[i].padding[0] = 0; + vcpu->cpuid_entries[i].padding[1] = 0; + vcpu->cpuid_entries[i].padding[2] = 0; + } + vcpu->cpuid_nent = cpuid->nent; + cpuid_fix_nx_cap(vcpu); + r = 0; + +out_free: + vfree(cpuid_entries); +out: + return r; +} + +static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu, + struct kvm_cpuid2 *cpuid, + struct kvm_cpuid_entry2 __user *entries) { int r; @@ -740,16 +788,198 @@ static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, goto out; r = -EFAULT; if (copy_from_user(&vcpu->cpuid_entries, entries, - cpuid->nent * sizeof(struct kvm_cpuid_entry))) + cpuid->nent * sizeof(struct kvm_cpuid_entry2))) goto out; vcpu->cpuid_nent = cpuid->nent; - cpuid_fix_nx_cap(vcpu); return 0; out: return r; } +static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu, + struct kvm_cpuid2 *cpuid, + struct kvm_cpuid_entry2 __user *entries) +{ + int r; + + r = -E2BIG; + if (cpuid->nent < vcpu->cpuid_nent) + goto out; + r = -EFAULT; + if (copy_to_user(entries, &vcpu->cpuid_entries, + vcpu->cpuid_nent * sizeof(struct kvm_cpuid_entry2))) + goto out; + return 0; + +out: + cpuid->nent = vcpu->cpuid_nent; + return r; +} + +static inline u32 bit(int bitno) +{ + return 1 << (bitno & 31); +} + +static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function, + u32 index) +{ + entry->function = function; + entry->index = index; + cpuid_count(entry->function, entry->index, + &entry->eax, &entry->ebx, &entry->ecx, &entry->edx); + entry->flags = 0; +} + +static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function, + u32 index, int *nent, int maxnent) +{ + const u32 kvm_supported_word0_x86_features = bit(X86_FEATURE_FPU) | + bit(X86_FEATURE_VME) | bit(X86_FEATURE_DE) | + bit(X86_FEATURE_PSE) | bit(X86_FEATURE_TSC) | + bit(X86_FEATURE_MSR) | bit(X86_FEATURE_PAE) | + bit(X86_FEATURE_CX8) | bit(X86_FEATURE_APIC) | + bit(X86_FEATURE_SEP) | bit(X86_FEATURE_PGE) | + bit(X86_FEATURE_CMOV) | bit(X86_FEATURE_PSE36) | + bit(X86_FEATURE_CLFLSH) | bit(X86_FEATURE_MMX) | + bit(X86_FEATURE_FXSR) | bit(X86_FEATURE_XMM) | + bit(X86_FEATURE_XMM2) | bit(X86_FEATURE_SELFSNOOP); + const u32 kvm_supported_word1_x86_features = bit(X86_FEATURE_FPU) | + bit(X86_FEATURE_VME) | bit(X86_FEATURE_DE) | + bit(X86_FEATURE_PSE) | bit(X86_FEATURE_TSC) | + bit(X86_FEATURE_MSR) | bit(X86_FEATURE_PAE) | + bit(X86_FEATURE_CX8) | bit(X86_FEATURE_APIC) | + bit(X86_FEATURE_PGE) | + bit(X86_FEATURE_CMOV) | bit(X86_FEATURE_PSE36) | + bit(X86_FEATURE_MMX) | bit(X86_FEATURE_FXSR) | + bit(X86_FEATURE_SYSCALL) | + (bit(X86_FEATURE_NX) && is_efer_nx()) | +#ifdef CONFIG_X86_64 + bit(X86_FEATURE_LM) | +#endif + bit(X86_FEATURE_MMXEXT) | + bit(X86_FEATURE_3DNOWEXT) | + bit(X86_FEATURE_3DNOW); + const u32 kvm_supported_word3_x86_features = + bit(X86_FEATURE_XMM3) | bit(X86_FEATURE_CX16); + const u32 kvm_supported_word6_x86_features = + bit(X86_FEATURE_LAHF_LM) | bit(X86_FEATURE_CMP_LEGACY); + + /* all func 2 cpuid_count() should be called on the same cpu */ + get_cpu(); + do_cpuid_1_ent(entry, function, index); + ++*nent; + + switch (function) { + case 0: + entry->eax = min(entry->eax, (u32)0xb); + break; + case 1: + entry->edx &= kvm_supported_word0_x86_features; + entry->ecx &= kvm_supported_word3_x86_features; + break; + /* function 2 entries are STATEFUL. That is, repeated cpuid commands + * may return different values. This forces us to get_cpu() before + * issuing the first command, and also to emulate this annoying behavior + * in kvm_emulate_cpuid() using KVM_CPUID_FLAG_STATE_READ_NEXT */ + case 2: { + int t, times = entry->eax & 0xff; + + entry->flags |= KVM_CPUID_FLAG_STATEFUL_FUNC; + for (t = 1; t < times && *nent < maxnent; ++t) { + do_cpuid_1_ent(&entry[t], function, 0); + entry[t].flags |= KVM_CPUID_FLAG_STATEFUL_FUNC; + ++*nent; + } + break; + } + /* function 4 and 0xb have additional index. */ + case 4: { + int index, cache_type; + + entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; + /* read more entries until cache_type is zero */ + for (index = 1; *nent < maxnent; ++index) { + cache_type = entry[index - 1].eax & 0x1f; + if (!cache_type) + break; + do_cpuid_1_ent(&entry[index], function, index); + entry[index].flags |= + KVM_CPUID_FLAG_SIGNIFCANT_INDEX; + ++*nent; + } + break; + } + case 0xb: { + int index, level_type; + + entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; + /* read more entries until level_type is zero */ + for (index = 1; *nent < maxnent; ++index) { + level_type = entry[index - 1].ecx & 0xff; + if (!level_type) + break; + do_cpuid_1_ent(&entry[index], function, index); + entry[index].flags |= + KVM_CPUID_FLAG_SIGNIFCANT_INDEX; + ++*nent; + } + break; + } + case 0x80000000: + entry->eax = min(entry->eax, 0x8000001a); + break; + case 0x80000001: + entry->edx &= kvm_supported_word1_x86_features; + entry->ecx &= kvm_supported_word6_x86_features; + break; + } + put_cpu(); +} + +static int kvm_vm_ioctl_get_supported_cpuid(struct kvm *kvm, + struct kvm_cpuid2 *cpuid, + struct kvm_cpuid_entry2 __user *entries) +{ + struct kvm_cpuid_entry2 *cpuid_entries; + int limit, nent = 0, r = -E2BIG; + u32 func; + + if (cpuid->nent < 1) + goto out; + r = -ENOMEM; + cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry2) * cpuid->nent); + if (!cpuid_entries) + goto out; + + do_cpuid_ent(&cpuid_entries[0], 0, 0, &nent, cpuid->nent); + limit = cpuid_entries[0].eax; + for (func = 1; func <= limit && nent < cpuid->nent; ++func) + do_cpuid_ent(&cpuid_entries[nent], func, 0, + &nent, cpuid->nent); + r = -E2BIG; + if (nent >= cpuid->nent) + goto out_free; + + do_cpuid_ent(&cpuid_entries[nent], 0x80000000, 0, &nent, cpuid->nent); + limit = cpuid_entries[nent - 1].eax; + for (func = 0x80000001; func <= limit && nent < cpuid->nent; ++func) + do_cpuid_ent(&cpuid_entries[nent], func, 0, + &nent, cpuid->nent); + r = -EFAULT; + if (copy_to_user(entries, cpuid_entries, + nent * sizeof(struct kvm_cpuid_entry2))) + goto out_free; + cpuid->nent = nent; + r = 0; + +out_free: + vfree(cpuid_entries); +out: + return r; +} + static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) { @@ -816,6 +1046,36 @@ long kvm_arch_vcpu_ioctl(struct file *filp, goto out; break; } + case KVM_SET_CPUID2: { + struct kvm_cpuid2 __user *cpuid_arg = argp; + struct kvm_cpuid2 cpuid; + + r = -EFAULT; + if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) + goto out; + r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid, + cpuid_arg->entries); + if (r) + goto out; + break; + } + case KVM_GET_CPUID2: { + struct kvm_cpuid2 __user *cpuid_arg = argp; + struct kvm_cpuid2 cpuid; + + r = -EFAULT; + if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) + goto out; + r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid, + cpuid_arg->entries); + if (r) + goto out; + r = -EFAULT; + if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid)) + goto out; + r = 0; + break; + } case KVM_GET_MSRS: r = msr_io(vcpu, argp, kvm_get_msr, 1); break; @@ -1111,6 +1371,24 @@ long kvm_arch_vm_ioctl(struct file *filp, r = 0; break; } + case KVM_GET_SUPPORTED_CPUID: { + struct kvm_cpuid2 __user *cpuid_arg = argp; + struct kvm_cpuid2 cpuid; + + r = -EFAULT; + if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) + goto out; + r = kvm_vm_ioctl_get_supported_cpuid(kvm, &cpuid, + cpuid_arg->entries); + if (r) + goto out; + + r = -EFAULT; + if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid)) + goto out; + r = 0; + break; + } default: ; } @@ -1908,14 +2186,47 @@ void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, } } +static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i) +{ + struct kvm_cpuid_entry2 *e = &vcpu->cpuid_entries[i]; + int j, nent = vcpu->cpuid_nent; + + e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT; + /* when no next entry is found, the current entry[i] is reselected */ + for (j = i + 1; j == i; j = (j + 1) % nent) { + struct kvm_cpuid_entry2 *ej = &vcpu->cpuid_entries[j]; + if (ej->function == e->function) { + ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT; + return j; + } + } + return 0; /* silence gcc, even though control never reaches here */ +} + +/* find an entry with matching function, matching index (if needed), and that + * should be read next (if it's stateful) */ +static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e, + u32 function, u32 index) +{ + if (e->function != function) + return 0; + if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index) + return 0; + if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) && + !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT)) + return 0; + return 1; +} + void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) { int i; - u32 function; - struct kvm_cpuid_entry *e, *best; + u32 function, index; + struct kvm_cpuid_entry2 *e, *best; kvm_x86_ops->cache_regs(vcpu); function = vcpu->regs[VCPU_REGS_RAX]; + index = vcpu->regs[VCPU_REGS_RCX]; vcpu->regs[VCPU_REGS_RAX] = 0; vcpu->regs[VCPU_REGS_RBX] = 0; vcpu->regs[VCPU_REGS_RCX] = 0; @@ -1923,7 +2234,9 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) best = NULL; for (i = 0; i < vcpu->cpuid_nent; ++i) { e = &vcpu->cpuid_entries[i]; - if (e->function == function) { + if (is_matching_cpuid_entry(e, function, index)) { + if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) + move_to_next_stateful_cpuid_entry(vcpu, i); best = e; break; } diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index b1528c9f566f..78ab1e108d8b 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -149,7 +149,7 @@ struct kvm_vcpu { int halt_request; /* real mode on Intel only */ int cpuid_nent; - struct kvm_cpuid_entry cpuid_entries[KVM_MAX_CPUID_ENTRIES]; + struct kvm_cpuid_entry2 cpuid_entries[KVM_MAX_CPUID_ENTRIES]; /* emulate context */ diff --git a/include/asm-x86/kvm.h b/include/asm-x86/kvm.h index 4837d759bec0..17afa81d6d66 100644 --- a/include/asm-x86/kvm.h +++ b/include/asm-x86/kvm.h @@ -151,5 +151,26 @@ struct kvm_cpuid { struct kvm_cpuid_entry entries[0]; }; +struct kvm_cpuid_entry2 { + __u32 function; + __u32 index; + __u32 flags; + __u32 eax; + __u32 ebx; + __u32 ecx; + __u32 edx; + __u32 padding[3]; +}; + +#define KVM_CPUID_FLAG_SIGNIFCANT_INDEX 1 +#define KVM_CPUID_FLAG_STATEFUL_FUNC 2 +#define KVM_CPUID_FLAG_STATE_READ_NEXT 4 + +/* for KVM_SET_CPUID2 */ +struct kvm_cpuid2 { + __u32 nent; + __u32 padding; + struct kvm_cpuid_entry2 entries[0]; +}; #endif diff --git a/include/linux/kvm.h b/include/linux/kvm.h index fd4f900fcce3..b751552f2e30 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -224,6 +224,7 @@ struct kvm_signal_mask { #define KVM_CAP_MMU_SHADOW_CACHE_CONTROL 2 #define KVM_CAP_USER_MEMORY 3 #define KVM_CAP_SET_TSS_ADDR 4 +#define KVM_CAP_EXT_CPUID 5 /* * ioctls for VM fds @@ -241,6 +242,7 @@ struct kvm_signal_mask { #define KVM_CREATE_VCPU _IO(KVMIO, 0x41) #define KVM_GET_DIRTY_LOG _IOW(KVMIO, 0x42, struct kvm_dirty_log) #define KVM_SET_MEMORY_ALIAS _IOW(KVMIO, 0x43, struct kvm_memory_alias) +#define KVM_GET_SUPPORTED_CPUID _IOWR(KVMIO, 0x48, struct kvm_cpuid2) /* Device model IOC */ #define KVM_CREATE_IRQCHIP _IO(KVMIO, 0x60) #define KVM_IRQ_LINE _IOW(KVMIO, 0x61, struct kvm_irq_level) @@ -266,5 +268,7 @@ struct kvm_signal_mask { #define KVM_SET_FPU _IOW(KVMIO, 0x8d, struct kvm_fpu) #define KVM_GET_LAPIC _IOR(KVMIO, 0x8e, struct kvm_lapic_state) #define KVM_SET_LAPIC _IOW(KVMIO, 0x8f, struct kvm_lapic_state) +#define KVM_SET_CPUID2 _IOW(KVMIO, 0x90, struct kvm_cpuid2) +#define KVM_GET_CPUID2 _IOWR(KVMIO, 0x91, struct kvm_cpuid2) #endif From 83ff3b9d4a0ade8f8eb9757fd4b36c501ba161e4 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Wed, 21 Nov 2007 14:33:25 +0800 Subject: [PATCH 163/249] KVM: VMX: Remove the secondary execute control dependency on irqchip The state of SECONDARY_VM_EXEC_CONTROL shouldn't depend on in-kernel IRQ chip, this patch fix this. Signed-off-by: Sheng Yang Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index b4c0bdce7b34..fc5e7c8381ce 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -193,11 +193,6 @@ static inline int cpu_has_secondary_exec_ctrls(void) CPU_BASED_ACTIVATE_SECONDARY_CONTROLS); } -static inline int vm_need_secondary_exec_ctrls(struct kvm *kvm) -{ - return ((cpu_has_secondary_exec_ctrls()) && (irqchip_in_kernel(kvm))); -} - static inline int cpu_has_vmx_virtualize_apic_accesses(void) { return (vmcs_config.cpu_based_2nd_exec_ctrl & @@ -1524,13 +1519,15 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) CPU_BASED_CR8_LOAD_EXITING; #endif } - if (!vm_need_secondary_exec_ctrls(vmx->vcpu.kvm)) - exec_control &= ~CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control); - if (vm_need_secondary_exec_ctrls(vmx->vcpu.kvm)) - vmcs_write32(SECONDARY_VM_EXEC_CONTROL, - vmcs_config.cpu_based_2nd_exec_ctrl); + if (cpu_has_secondary_exec_ctrls()) { + exec_control = vmcs_config.cpu_based_2nd_exec_ctrl; + if (!vm_need_virtualize_apic_accesses(vmx->vcpu.kvm)) + exec_control &= + ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; + vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control); + } vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, !!bypass_guest_pf); vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, !!bypass_guest_pf); From e9f85cde99fa011db256f6683ba81d155e0d3f57 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Thu, 22 Nov 2007 11:20:33 +0800 Subject: [PATCH 164/249] KVM: Portability: Move unalias_gfn to arch dependent file Non-x86 archs don't need this mechanism. Move it to arch, and keep its interface in common. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 14 -------------- drivers/kvm/x86.c | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 0efd759e585f..721e660023f6 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -408,20 +408,6 @@ int kvm_is_error_hva(unsigned long addr) } EXPORT_SYMBOL_GPL(kvm_is_error_hva); -gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) -{ - int i; - struct kvm_mem_alias *alias; - - for (i = 0; i < kvm->naliases; ++i) { - alias = &kvm->aliases[i]; - if (gfn >= alias->base_gfn - && gfn < alias->base_gfn + alias->npages) - return alias->target_gfn + gfn - alias->base_gfn; - } - return gfn; -} - static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn) { int i; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 7237cb25f77d..2be59364a49d 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1119,6 +1119,20 @@ static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm) return kvm->n_alloc_mmu_pages; } +gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) +{ + int i; + struct kvm_mem_alias *alias; + + for (i = 0; i < kvm->naliases; ++i) { + alias = &kvm->aliases[i]; + if (gfn >= alias->base_gfn + && gfn < alias->base_gfn + alias->npages) + return alias->target_gfn + gfn - alias->base_gfn; + } + return gfn; +} + /* * Set a new alias region. Aliases map a portion of physical memory into * another portion. This is useful for memory windows, for example the PC From 90e0a28f6b7241c7793f2ebd540c349580170446 Mon Sep 17 00:00:00 2001 From: Guillaume Thouvenin Date: Thu, 22 Nov 2007 11:32:09 +0100 Subject: [PATCH 165/249] KVM: x86 emulator: Make a distinction between repeat prefixes F3 and F2 cmps and scas instructions accept repeat prefixes F3 and F2. So in order to emulate those prefixed instructions we need to be able to know if prefixes are REP/REPE/REPZ or REPNE/REPNZ. Currently kvm doesn't make this distinction. This patch introduces this distinction. Signed-off-by: Guillaume Thouvenin Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 4 +++- drivers/kvm/x86_emulate.h | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 6e7f774d1751..9f8d59a49313 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -824,8 +824,10 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) c->lock_prefix = 1; break; case 0xf2: /* REPNE/REPNZ */ + c->rep_prefix = REPNE_PREFIX; + break; case 0xf3: /* REP/REPE/REPZ */ - c->rep_prefix = 1; + c->rep_prefix = REPE_PREFIX; break; default: goto done_prefixes; diff --git a/drivers/kvm/x86_emulate.h b/drivers/kvm/x86_emulate.h index 4603b2bf3488..644086e354a7 100644 --- a/drivers/kvm/x86_emulate.h +++ b/drivers/kvm/x86_emulate.h @@ -162,6 +162,10 @@ struct x86_emulate_ctxt { struct decode_cache decode; }; +/* Repeat String Operation Prefix */ +#define REPE_PREFIX 1 +#define REPNE_PREFIX 2 + /* Execution mode, passed to the emulator. */ #define X86EMUL_MODE_REAL 0 /* Real mode. */ #define X86EMUL_MODE_PROT16 2 /* 16-bit protected mode. */ From f21b8bf4cc4091b23669987124fd13f758abf6d6 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 22 Nov 2007 14:16:12 +0200 Subject: [PATCH 166/249] KVM: x86 emulator: address size and operand size overrides are sticky Current implementation is to toggle, which is incorrect. Patch ported from corresponding Xen code. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 9f8d59a49313..3be506ac01a3 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -758,6 +758,7 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) struct decode_cache *c = &ctxt->decode; int rc = 0; int mode = ctxt->mode; + int def_op_bytes, def_ad_bytes; /* Shadow copy of register state. Committed on successful emulation. */ @@ -768,34 +769,38 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) switch (mode) { case X86EMUL_MODE_REAL: case X86EMUL_MODE_PROT16: - c->op_bytes = c->ad_bytes = 2; + def_op_bytes = def_ad_bytes = 2; break; case X86EMUL_MODE_PROT32: - c->op_bytes = c->ad_bytes = 4; + def_op_bytes = def_ad_bytes = 4; break; #ifdef CONFIG_X86_64 case X86EMUL_MODE_PROT64: - c->op_bytes = 4; - c->ad_bytes = 8; + def_op_bytes = 4; + def_ad_bytes = 8; break; #endif default: return -1; } + c->op_bytes = def_op_bytes; + c->ad_bytes = def_ad_bytes; + /* Legacy prefixes. */ for (;;) { switch (c->b = insn_fetch(u8, 1, c->eip)) { case 0x66: /* operand-size override */ - c->op_bytes ^= 6; /* switch between 2/4 bytes */ + /* switch between 2/4 bytes */ + c->op_bytes = def_op_bytes ^ 6; break; case 0x67: /* address-size override */ if (mode == X86EMUL_MODE_PROT64) /* switch between 4/8 bytes */ - c->ad_bytes ^= 12; + c->ad_bytes = def_ad_bytes ^ 12; else /* switch between 2/4 bytes */ - c->ad_bytes ^= 6; + c->ad_bytes = def_ad_bytes ^ 6; break; case 0x2e: /* CS override */ c->override_base = &ctxt->cs_base; From eb9774f0d675922d780f23cbb747f474b0573dea Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 25 Nov 2007 17:45:31 +0200 Subject: [PATCH 167/249] KVM: Remove misleading check for mmio during event injection mmio was already handled in kvm_arch_vcpu_ioctl_run(), so no need to check again. Signed-off-by: Avi Kivity --- drivers/kvm/x86.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 2be59364a49d..5e5815604cbd 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -2344,7 +2344,7 @@ again: if (irqchip_in_kernel(vcpu->kvm)) kvm_x86_ops->inject_pending_irq(vcpu); - else if (!vcpu->mmio_read_completed) + else kvm_x86_ops->inject_pending_vectors(vcpu, kvm_run); vcpu->guest_mode = 1; From 448353caeab1d9180f12194a80790859ddbeebd5 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Mon, 26 Nov 2007 14:08:14 +0200 Subject: [PATCH 168/249] KVM: MMU: mark pages that were inserted to the shadow pages table as accessed Mark guest pages as accessed when removed from the shadow page tables for better lru processing. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 0cb6580956ba..9b9d1b6eb2f8 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -438,6 +439,7 @@ static void rmap_remove(struct kvm *kvm, u64 *spte) return; sp = page_header(__pa(spte)); page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); + mark_page_accessed(page); if (is_writeble_pte(*spte)) kvm_release_page_dirty(page); else From e8d8d7fe8877c594c08f40cc7c013626cfe3e9cc Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Fri, 16 Nov 2007 16:29:15 +0800 Subject: [PATCH 169/249] KVM: x86 emulator: Rename 'cr2' to 'memop' Previous patches have removed the dependency on cr2; we can now stop passing it to the emulator and rename uses to 'memop'. Signed-off-by: Sheng Yang Signed-off-by: Avi Kivity --- drivers/kvm/x86.c | 1 - drivers/kvm/x86_emulate.c | 18 +++++++++--------- drivers/kvm/x86_emulate.h | 1 - 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 5e5815604cbd..12f1d6fc92bb 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1702,7 +1702,6 @@ int emulate_instruction(struct kvm_vcpu *vcpu, vcpu->emulate_ctxt.vcpu = vcpu; vcpu->emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu); - vcpu->emulate_ctxt.cr2 = cr2; vcpu->emulate_ctxt.mode = (vcpu->emulate_ctxt.eflags & X86_EFLAGS_VM) ? X86EMUL_MODE_REAL : cs_l diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 3be506ac01a3..22fdf0ac6615 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1127,13 +1127,13 @@ static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt, static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops, - unsigned long cr2) + unsigned long memop) { struct decode_cache *c = &ctxt->decode; u64 old, new; int rc; - rc = ops->read_emulated(cr2, &old, 8, ctxt->vcpu); + rc = ops->read_emulated(memop, &old, 8, ctxt->vcpu); if (rc != 0) return rc; @@ -1148,7 +1148,7 @@ static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt, new = ((u64)c->regs[VCPU_REGS_RCX] << 32) | (u32) c->regs[VCPU_REGS_RBX]; - rc = ops->cmpxchg_emulated(cr2, &old, &new, 8, ctxt->vcpu); + rc = ops->cmpxchg_emulated(memop, &old, &new, 8, ctxt->vcpu); if (rc != 0) return rc; ctxt->eflags |= EFLG_ZF; @@ -1211,7 +1211,7 @@ static inline int writeback(struct x86_emulate_ctxt *ctxt, int x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { - unsigned long cr2 = ctxt->cr2; + unsigned long memop = 0; u64 msr_data; unsigned long saved_eip = 0; struct decode_cache *c = &ctxt->decode; @@ -1226,10 +1226,10 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) saved_eip = c->eip; if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs)) - cr2 = c->modrm_ea; + memop = c->modrm_ea; if (c->src.type == OP_MEM) { - c->src.ptr = (unsigned long *)cr2; + c->src.ptr = (unsigned long *)memop; c->src.val = 0; rc = ops->read_emulated((unsigned long)c->src.ptr, &c->src.val, @@ -1245,7 +1245,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) if (c->dst.type == OP_MEM) { - c->dst.ptr = (unsigned long *)cr2; + c->dst.ptr = (unsigned long *)memop; c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; c->dst.val = 0; if (c->d & BitOp) { @@ -1677,7 +1677,7 @@ twobyte_insn: &ctxt->eflags); break; case 7: /* invlpg*/ - emulate_invlpg(ctxt->vcpu, cr2); + emulate_invlpg(ctxt->vcpu, memop); break; default: goto cannot_emulate; @@ -1848,7 +1848,7 @@ twobyte_special_insn: break; } case 0xc7: /* Grp9 (cmpxchg8b) */ - rc = emulate_grp9(ctxt, ops, cr2); + rc = emulate_grp9(ctxt, ops, memop); if (rc != 0) goto done; break; diff --git a/drivers/kvm/x86_emulate.h b/drivers/kvm/x86_emulate.h index 644086e354a7..7db91b9bdcd4 100644 --- a/drivers/kvm/x86_emulate.h +++ b/drivers/kvm/x86_emulate.h @@ -145,7 +145,6 @@ struct x86_emulate_ctxt { /* Linear faulting address (if emulating a page-faulting instruction). */ unsigned long eflags; - unsigned long cr2; /* Emulated execution mode, represented by an X86EMUL_MODE value. */ int mode; From d7e5117a2568f7407e98ca85155511ecfe4f0631 Mon Sep 17 00:00:00 2001 From: Guillaume Thouvenin Date: Mon, 26 Nov 2007 13:49:09 +0100 Subject: [PATCH 170/249] KVM: x86 emulator: cmps instruction Add emulation for the cmps instruction. This lets OpenBSD boot on kvm. Signed-off-by: Guillaume Thouvenin Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 58 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 22fdf0ac6615..84e536603f5a 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1535,10 +1535,31 @@ special_insn: break; } if (c->rep_prefix) { + /* All REP prefixes have the same first termination condition */ if (c->regs[VCPU_REGS_RCX] == 0) { ctxt->vcpu->rip = c->eip; goto done; } + /* The second termination condition only applies for REPE + * and REPNE. Test if the repeat string operation prefix is + * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the + * corresponding termination condition according to: + * - if REPE/REPZ and ZF = 0 then done + * - if REPNE/REPNZ and ZF = 1 then done + */ + if ((c->b == 0xa6) || (c->b == 0xa7) || + (c->b == 0xae) || (c->b == 0xaf)) { + if ((c->rep_prefix == REPE_PREFIX) && + ((ctxt->eflags & EFLG_ZF) == 0)) { + ctxt->vcpu->rip = c->eip; + goto done; + } + if ((c->rep_prefix == REPNE_PREFIX) && + ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) { + ctxt->vcpu->rip = c->eip; + goto done; + } + } c->regs[VCPU_REGS_RCX]--; c->eip = ctxt->vcpu->rip; } @@ -1564,8 +1585,41 @@ special_insn: : c->dst.bytes); break; case 0xa6 ... 0xa7: /* cmps */ - DPRINTF("Urk! I don't handle CMPS.\n"); - goto cannot_emulate; + c->src.type = OP_NONE; /* Disable writeback. */ + c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->src.ptr = (unsigned long *)register_address( + c->override_base ? *c->override_base : + ctxt->ds_base, + c->regs[VCPU_REGS_RSI]); + if ((rc = ops->read_emulated((unsigned long)c->src.ptr, + &c->src.val, + c->src.bytes, + ctxt->vcpu)) != 0) + goto done; + + c->dst.type = OP_NONE; /* Disable writeback. */ + c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->dst.ptr = (unsigned long *)register_address( + ctxt->es_base, + c->regs[VCPU_REGS_RDI]); + if ((rc = ops->read_emulated((unsigned long)c->dst.ptr, + &c->dst.val, + c->dst.bytes, + ctxt->vcpu)) != 0) + goto done; + + DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr); + + emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags); + + register_address_increment(c->regs[VCPU_REGS_RSI], + (ctxt->eflags & EFLG_DF) ? -c->src.bytes + : c->src.bytes); + register_address_increment(c->regs[VCPU_REGS_RDI], + (ctxt->eflags & EFLG_DF) ? -c->dst.bytes + : c->dst.bytes); + + break; case 0xaa ... 0xab: /* stos */ c->dst.type = OP_MEM; c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; From 51e296258cba41759ff0de5da110d8d52675ee67 Mon Sep 17 00:00:00 2001 From: Jerone Young Date: Mon, 26 Nov 2007 08:33:53 -0600 Subject: [PATCH 171/249] KVM: Add ifdef in irqchip struct for x86 only structures This patch fixes a small issue where sturctures: kvm_pic_state kvm_ioapic_state are defined inside x86 specific code and may or may not be defined in anyway for other architectures. The problem caused is one cannot compile userspace apps (ex. libkvm) for other archs since a size cannot be determined for these structures. Signed-off-by: Jerone Young Signed-off-by: Avi Kivity --- include/linux/Kbuild | 2 +- include/linux/kvm.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/include/linux/Kbuild b/include/linux/Kbuild index 27b9350052b4..c263a69b5680 100644 --- a/include/linux/Kbuild +++ b/include/linux/Kbuild @@ -100,7 +100,6 @@ header-y += iso_fs.h header-y += ixjuser.h header-y += jffs2.h header-y += keyctl.h -header-y += kvm.h header-y += limits.h header-y += lock_dlm_plock.h header-y += magic.h @@ -256,6 +255,7 @@ unifdef-y += kd.h unifdef-y += kernelcapi.h unifdef-y += kernel.h unifdef-y += keyboard.h +unifdef-y += kvm.h unifdef-y += llc.h unifdef-y += loop.h unifdef-y += lp.h diff --git a/include/linux/kvm.h b/include/linux/kvm.h index b751552f2e30..f0bebd61ff77 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -51,8 +51,10 @@ struct kvm_irqchip { __u32 pad; union { char dummy[512]; /* reserving space */ +#ifdef CONFIG_X86 struct kvm_pic_state pic; struct kvm_ioapic_state ioapic; +#endif } chip; }; From b9fa9d6bc6ac617170c4cf21ca764eb187618c0d Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 27 Nov 2007 19:05:37 +0200 Subject: [PATCH 172/249] KVM: x86 emulator: Move rep processing before instruction execution Currently rep processing is handled somewhere in the middle of instruction processing. Move it to a sensible place. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 70 ++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 84e536603f5a..53377f0b202e 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -64,6 +64,7 @@ #define Mov (1<<7) #define BitOp (1<<8) #define MemAbs (1<<9) /* Memory operand is absolute displacement */ +#define String (1<<10) /* String instruction (rep capable) */ static u16 opcode_table[256] = { /* 0x00 - 0x07 */ @@ -133,12 +134,12 @@ static u16 opcode_table[256] = { /* 0xA0 - 0xA7 */ ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs, ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs, - ByteOp | ImplicitOps | Mov, ImplicitOps | Mov, - ByteOp | ImplicitOps, ImplicitOps, + ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String, + ByteOp | ImplicitOps | String, ImplicitOps | String, /* 0xA8 - 0xAF */ - 0, 0, ByteOp | ImplicitOps | Mov, ImplicitOps | Mov, - ByteOp | ImplicitOps | Mov, ImplicitOps | Mov, - ByteOp | ImplicitOps, ImplicitOps, + 0, 0, ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String, + ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String, + ByteOp | ImplicitOps | String, ImplicitOps | String, /* 0xB0 - 0xBF */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0xC0 - 0xC7 */ @@ -1228,6 +1229,36 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs)) memop = c->modrm_ea; + if (c->rep_prefix && (c->d & String)) { + /* All REP prefixes have the same first termination condition */ + if (c->regs[VCPU_REGS_RCX] == 0) { + ctxt->vcpu->rip = c->eip; + goto done; + } + /* The second termination condition only applies for REPE + * and REPNE. Test if the repeat string operation prefix is + * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the + * corresponding termination condition according to: + * - if REPE/REPZ and ZF = 0 then done + * - if REPNE/REPNZ and ZF = 1 then done + */ + if ((c->b == 0xa6) || (c->b == 0xa7) || + (c->b == 0xae) || (c->b == 0xaf)) { + if ((c->rep_prefix == REPE_PREFIX) && + ((ctxt->eflags & EFLG_ZF) == 0)) { + ctxt->vcpu->rip = c->eip; + goto done; + } + if ((c->rep_prefix == REPNE_PREFIX) && + ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) { + ctxt->vcpu->rip = c->eip; + goto done; + } + } + c->regs[VCPU_REGS_RCX]--; + c->eip = ctxt->vcpu->rip; + } + if (c->src.type == OP_MEM) { c->src.ptr = (unsigned long *)memop; c->src.val = 0; @@ -1534,35 +1565,6 @@ special_insn: c->dst.type = OP_NONE; /* Disable writeback. */ break; } - if (c->rep_prefix) { - /* All REP prefixes have the same first termination condition */ - if (c->regs[VCPU_REGS_RCX] == 0) { - ctxt->vcpu->rip = c->eip; - goto done; - } - /* The second termination condition only applies for REPE - * and REPNE. Test if the repeat string operation prefix is - * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the - * corresponding termination condition according to: - * - if REPE/REPZ and ZF = 0 then done - * - if REPNE/REPNZ and ZF = 1 then done - */ - if ((c->b == 0xa6) || (c->b == 0xa7) || - (c->b == 0xae) || (c->b == 0xaf)) { - if ((c->rep_prefix == REPE_PREFIX) && - ((ctxt->eflags & EFLG_ZF) == 0)) { - ctxt->vcpu->rip = c->eip; - goto done; - } - if ((c->rep_prefix == REPNE_PREFIX) && - ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) { - ctxt->vcpu->rip = c->eip; - goto done; - } - } - c->regs[VCPU_REGS_RCX]--; - c->eip = ctxt->vcpu->rip; - } switch (c->b) { case 0xa4 ... 0xa5: /* movs */ c->dst.type = OP_MEM; From 111de5d60c589bca02b49dce076ca588618e0d1c Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 27 Nov 2007 19:14:21 +0200 Subject: [PATCH 173/249] KVM: x86 emulator: unify two switches The rep prefix cleanup left two switch () statements next to each other. Unify them. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 50 ++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 53377f0b202e..499d1ad42bc4 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1541,31 +1541,6 @@ special_insn: case 0x9d: /* popf */ c->dst.ptr = (unsigned long *) &ctxt->eflags; goto pop_instruction; - case 0xc3: /* ret */ - c->dst.ptr = &c->eip; - goto pop_instruction; - case 0xf4: /* hlt */ - ctxt->vcpu->halt_request = 1; - goto done; - case 0xf5: /* cmc */ - /* complement carry flag from eflags reg */ - ctxt->eflags ^= EFLG_CF; - c->dst.type = OP_NONE; /* Disable writeback. */ - break; - case 0xf8: /* clc */ - ctxt->eflags &= ~EFLG_CF; - c->dst.type = OP_NONE; /* Disable writeback. */ - break; - case 0xfa: /* cli */ - ctxt->eflags &= ~X86_EFLAGS_IF; - c->dst.type = OP_NONE; /* Disable writeback. */ - break; - case 0xfb: /* sti */ - ctxt->eflags |= X86_EFLAGS_IF; - c->dst.type = OP_NONE; /* Disable writeback. */ - break; - } - switch (c->b) { case 0xa4 ... 0xa5: /* movs */ c->dst.type = OP_MEM; c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; @@ -1652,6 +1627,9 @@ special_insn: case 0xae ... 0xaf: /* scas */ DPRINTF("Urk! I don't handle SCAS.\n"); goto cannot_emulate; + case 0xc3: /* ret */ + c->dst.ptr = &c->eip; + goto pop_instruction; case 0xe8: /* call (near) */ { long int rel; switch (c->op_bytes) { @@ -1676,8 +1654,26 @@ special_insn: JMP_REL(c->src.val); c->dst.type = OP_NONE; /* Disable writeback. */ break; - - + case 0xf4: /* hlt */ + ctxt->vcpu->halt_request = 1; + goto done; + case 0xf5: /* cmc */ + /* complement carry flag from eflags reg */ + ctxt->eflags ^= EFLG_CF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0xf8: /* clc */ + ctxt->eflags &= ~EFLG_CF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0xfa: /* cli */ + ctxt->eflags &= ~X86_EFLAGS_IF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0xfb: /* sti */ + ctxt->eflags |= X86_EFLAGS_IF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; } goto writeback; From 018a98db74d4cc8967cfc570197b14bcbdddb579 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 27 Nov 2007 19:30:56 +0200 Subject: [PATCH 174/249] KVM: x86 emulator: unify four switch statements into two Unify the special instruction switch with the regular instruction switch, and the two byte special instruction switch with the regular two byte instruction switch. That makes it much easier to find an instruction or the place an instruction needs to be added in. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 342 +++++++++++++++++++------------------- 1 file changed, 168 insertions(+), 174 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 499d1ad42bc4..3e3eba70d5ac 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1294,6 +1294,8 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) } c->dst.orig_val = c->dst.val; +special_insn: + if (c->twobyte) goto twobyte_insn; @@ -1378,6 +1380,52 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) goto cannot_emulate; c->dst.val = (s32) c->src.val; break; + case 0x6a: /* push imm8 */ + c->src.val = 0L; + c->src.val = insn_fetch(s8, 1, c->eip); + emulate_push(ctxt); + break; + case 0x6c: /* insb */ + case 0x6d: /* insw/insd */ + if (kvm_emulate_pio_string(ctxt->vcpu, NULL, + 1, + (c->d & ByteOp) ? 1 : c->op_bytes, + c->rep_prefix ? + address_mask(c->regs[VCPU_REGS_RCX]) : 1, + (ctxt->eflags & EFLG_DF), + register_address(ctxt->es_base, + c->regs[VCPU_REGS_RDI]), + c->rep_prefix, + c->regs[VCPU_REGS_RDX]) == 0) { + c->eip = saved_eip; + return -1; + } + return 0; + case 0x6e: /* outsb */ + case 0x6f: /* outsw/outsd */ + if (kvm_emulate_pio_string(ctxt->vcpu, NULL, + 0, + (c->d & ByteOp) ? 1 : c->op_bytes, + c->rep_prefix ? + address_mask(c->regs[VCPU_REGS_RCX]) : 1, + (ctxt->eflags & EFLG_DF), + register_address(c->override_base ? + *c->override_base : + ctxt->ds_base, + c->regs[VCPU_REGS_RSI]), + c->rep_prefix, + c->regs[VCPU_REGS_RDX]) == 0) { + c->eip = saved_eip; + return -1; + } + return 0; + case 0x70 ... 0x7f: /* jcc (short) */ { + int rel = insn_fetch(s8, 1, c->eip); + + if (test_cc(c->b, ctxt->eflags)) + JMP_REL(rel); + break; + } case 0x80 ... 0x83: /* Grp1 */ switch (c->modrm_reg) { case 0: @@ -1434,106 +1482,6 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) if (rc != 0) goto done; break; - case 0xa0 ... 0xa1: /* mov */ - c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; - c->dst.val = c->src.val; - break; - case 0xa2 ... 0xa3: /* mov */ - c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX]; - break; - case 0xc0 ... 0xc1: - emulate_grp2(ctxt); - break; - case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */ - mov: - c->dst.val = c->src.val; - break; - case 0xd0 ... 0xd1: /* Grp2 */ - c->src.val = 1; - emulate_grp2(ctxt); - break; - case 0xd2 ... 0xd3: /* Grp2 */ - c->src.val = c->regs[VCPU_REGS_RCX]; - emulate_grp2(ctxt); - break; - case 0xf6 ... 0xf7: /* Grp3 */ - rc = emulate_grp3(ctxt, ops); - if (rc != 0) - goto done; - break; - case 0xfe ... 0xff: /* Grp4/Grp5 */ - rc = emulate_grp45(ctxt, ops); - if (rc != 0) - goto done; - break; - } - -writeback: - rc = writeback(ctxt, ops); - if (rc != 0) - goto done; - - /* Commit shadow register state. */ - memcpy(ctxt->vcpu->regs, c->regs, sizeof c->regs); - ctxt->vcpu->rip = c->eip; - -done: - if (rc == X86EMUL_UNHANDLEABLE) { - c->eip = saved_eip; - return -1; - } - return 0; - -special_insn: - if (c->twobyte) - goto twobyte_special_insn; - switch (c->b) { - case 0x6a: /* push imm8 */ - c->src.val = 0L; - c->src.val = insn_fetch(s8, 1, c->eip); - emulate_push(ctxt); - break; - case 0x6c: /* insb */ - case 0x6d: /* insw/insd */ - if (kvm_emulate_pio_string(ctxt->vcpu, NULL, - 1, - (c->d & ByteOp) ? 1 : c->op_bytes, - c->rep_prefix ? - address_mask(c->regs[VCPU_REGS_RCX]) : 1, - (ctxt->eflags & EFLG_DF), - register_address(ctxt->es_base, - c->regs[VCPU_REGS_RDI]), - c->rep_prefix, - c->regs[VCPU_REGS_RDX]) == 0) { - c->eip = saved_eip; - return -1; - } - return 0; - case 0x6e: /* outsb */ - case 0x6f: /* outsw/outsd */ - if (kvm_emulate_pio_string(ctxt->vcpu, NULL, - 0, - (c->d & ByteOp) ? 1 : c->op_bytes, - c->rep_prefix ? - address_mask(c->regs[VCPU_REGS_RCX]) : 1, - (ctxt->eflags & EFLG_DF), - register_address(c->override_base ? - *c->override_base : - ctxt->ds_base, - c->regs[VCPU_REGS_RSI]), - c->rep_prefix, - c->regs[VCPU_REGS_RDX]) == 0) { - c->eip = saved_eip; - return -1; - } - return 0; - case 0x70 ... 0x7f: /* jcc (short) */ { - int rel = insn_fetch(s8, 1, c->eip); - - if (test_cc(c->b, ctxt->eflags)) - JMP_REL(rel); - break; - } case 0x9c: /* pushf */ c->src.val = (unsigned long) ctxt->eflags; emulate_push(ctxt); @@ -1541,6 +1489,13 @@ special_insn: case 0x9d: /* popf */ c->dst.ptr = (unsigned long *) &ctxt->eflags; goto pop_instruction; + case 0xa0 ... 0xa1: /* mov */ + c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; + c->dst.val = c->src.val; + break; + case 0xa2 ... 0xa3: /* mov */ + c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX]; + break; case 0xa4 ... 0xa5: /* movs */ c->dst.type = OP_MEM; c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; @@ -1627,9 +1582,24 @@ special_insn: case 0xae ... 0xaf: /* scas */ DPRINTF("Urk! I don't handle SCAS.\n"); goto cannot_emulate; + case 0xc0 ... 0xc1: + emulate_grp2(ctxt); + break; case 0xc3: /* ret */ c->dst.ptr = &c->eip; goto pop_instruction; + case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */ + mov: + c->dst.val = c->src.val; + break; + case 0xd0 ... 0xd1: /* Grp2 */ + c->src.val = 1; + emulate_grp2(ctxt); + break; + case 0xd2 ... 0xd3: /* Grp2 */ + c->src.val = c->regs[VCPU_REGS_RCX]; + emulate_grp2(ctxt); + break; case 0xe8: /* call (near) */ { long int rel; switch (c->op_bytes) { @@ -1662,6 +1632,11 @@ special_insn: ctxt->eflags ^= EFLG_CF; c->dst.type = OP_NONE; /* Disable writeback. */ break; + case 0xf6 ... 0xf7: /* Grp3 */ + rc = emulate_grp3(ctxt, ops); + if (rc != 0) + goto done; + break; case 0xf8: /* clc */ ctxt->eflags &= ~EFLG_CF; c->dst.type = OP_NONE; /* Disable writeback. */ @@ -1674,8 +1649,28 @@ special_insn: ctxt->eflags |= X86_EFLAGS_IF; c->dst.type = OP_NONE; /* Disable writeback. */ break; + case 0xfe ... 0xff: /* Grp4/Grp5 */ + rc = emulate_grp45(ctxt, ops); + if (rc != 0) + goto done; + break; } - goto writeback; + +writeback: + rc = writeback(ctxt, ops); + if (rc != 0) + goto done; + + /* Commit shadow register state. */ + memcpy(ctxt->vcpu->regs, c->regs, sizeof c->regs); + ctxt->vcpu->rip = c->eip; + +done: + if (rc == X86EMUL_UNHANDLEABLE) { + c->eip = saved_eip; + return -1; + } + return 0; twobyte_insn: switch (c->b) { @@ -1737,6 +1732,23 @@ twobyte_insn: /* Disable writeback. */ c->dst.type = OP_NONE; break; + case 0x06: + emulate_clts(ctxt->vcpu); + c->dst.type = OP_NONE; + break; + case 0x08: /* invd */ + case 0x09: /* wbinvd */ + case 0x0d: /* GrpP (prefetch) */ + case 0x18: /* Grp16 (prefetch/nop) */ + c->dst.type = OP_NONE; + break; + case 0x20: /* mov cr, reg */ + if (c->modrm_mod != 3) + goto cannot_emulate; + c->regs[c->modrm_rm] = + realmode_get_cr(ctxt->vcpu, c->modrm_reg); + c->dst.type = OP_NONE; /* no writeback */ + break; case 0x21: /* mov from dr to reg */ if (c->modrm_mod != 3) goto cannot_emulate; @@ -1745,6 +1757,13 @@ twobyte_insn: goto cannot_emulate; c->dst.type = OP_NONE; /* no writeback */ break; + case 0x22: /* mov reg, cr */ + if (c->modrm_mod != 3) + goto cannot_emulate; + realmode_set_cr(ctxt->vcpu, + c->modrm_reg, c->modrm_val, &ctxt->eflags); + c->dst.type = OP_NONE; + break; case 0x23: /* mov from reg to dr */ if (c->modrm_mod != 3) goto cannot_emulate; @@ -1754,11 +1773,58 @@ twobyte_insn: goto cannot_emulate; c->dst.type = OP_NONE; /* no writeback */ break; + case 0x30: + /* wrmsr */ + msr_data = (u32)c->regs[VCPU_REGS_RAX] + | ((u64)c->regs[VCPU_REGS_RDX] << 32); + rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data); + if (rc) { + kvm_x86_ops->inject_gp(ctxt->vcpu, 0); + c->eip = ctxt->vcpu->rip; + } + rc = X86EMUL_CONTINUE; + c->dst.type = OP_NONE; + break; + case 0x32: + /* rdmsr */ + rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data); + if (rc) { + kvm_x86_ops->inject_gp(ctxt->vcpu, 0); + c->eip = ctxt->vcpu->rip; + } else { + c->regs[VCPU_REGS_RAX] = (u32)msr_data; + c->regs[VCPU_REGS_RDX] = msr_data >> 32; + } + rc = X86EMUL_CONTINUE; + c->dst.type = OP_NONE; + break; case 0x40 ... 0x4f: /* cmov */ c->dst.val = c->dst.orig_val = c->src.val; if (!test_cc(c->b, ctxt->eflags)) c->dst.type = OP_NONE; /* no writeback */ break; + case 0x80 ... 0x8f: /* jnz rel, etc*/ { + long int rel; + + switch (c->op_bytes) { + case 2: + rel = insn_fetch(s16, 2, c->eip); + break; + case 4: + rel = insn_fetch(s32, 4, c->eip); + break; + case 8: + rel = insn_fetch(s64, 8, c->eip); + break; + default: + DPRINTF("jnz: Invalid op_bytes\n"); + goto cannot_emulate; + } + if (test_cc(c->b, ctxt->eflags)) + JMP_REL(rel); + c->dst.type = OP_NONE; + break; + } case 0xa3: bt: /* bt */ c->dst.type = OP_NONE; @@ -1828,85 +1894,13 @@ twobyte_insn: c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val : (u64) c->src.val; break; - } - goto writeback; - -twobyte_special_insn: - switch (c->b) { - case 0x06: - emulate_clts(ctxt->vcpu); - break; - case 0x08: /* invd */ - break; - case 0x09: /* wbinvd */ - break; - case 0x0d: /* GrpP (prefetch) */ - case 0x18: /* Grp16 (prefetch/nop) */ - break; - case 0x20: /* mov cr, reg */ - if (c->modrm_mod != 3) - goto cannot_emulate; - c->regs[c->modrm_rm] = - realmode_get_cr(ctxt->vcpu, c->modrm_reg); - break; - case 0x22: /* mov reg, cr */ - if (c->modrm_mod != 3) - goto cannot_emulate; - realmode_set_cr(ctxt->vcpu, - c->modrm_reg, c->modrm_val, &ctxt->eflags); - break; - case 0x30: - /* wrmsr */ - msr_data = (u32)c->regs[VCPU_REGS_RAX] - | ((u64)c->regs[VCPU_REGS_RDX] << 32); - rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data); - if (rc) { - kvm_x86_ops->inject_gp(ctxt->vcpu, 0); - c->eip = ctxt->vcpu->rip; - } - rc = X86EMUL_CONTINUE; - break; - case 0x32: - /* rdmsr */ - rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data); - if (rc) { - kvm_x86_ops->inject_gp(ctxt->vcpu, 0); - c->eip = ctxt->vcpu->rip; - } else { - c->regs[VCPU_REGS_RAX] = (u32)msr_data; - c->regs[VCPU_REGS_RDX] = msr_data >> 32; - } - rc = X86EMUL_CONTINUE; - break; - case 0x80 ... 0x8f: /* jnz rel, etc*/ { - long int rel; - - switch (c->op_bytes) { - case 2: - rel = insn_fetch(s16, 2, c->eip); - break; - case 4: - rel = insn_fetch(s32, 4, c->eip); - break; - case 8: - rel = insn_fetch(s64, 8, c->eip); - break; - default: - DPRINTF("jnz: Invalid op_bytes\n"); - goto cannot_emulate; - } - if (test_cc(c->b, ctxt->eflags)) - JMP_REL(rel); - break; - } case 0xc7: /* Grp9 (cmpxchg8b) */ rc = emulate_grp9(ctxt, ops, memop); if (rc != 0) goto done; + c->dst.type = OP_NONE; break; } - /* Disable writeback. */ - c->dst.type = OP_NONE; goto writeback; cannot_emulate: From f77bc6a420eba845605ff1d53cadf55f94c5e8b7 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Wed, 21 Nov 2007 04:36:41 +0800 Subject: [PATCH 175/249] KVM: Portability: Move KVM_INTERRUPT vcpu ioctl to x86.c Other archs doesn't need it. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 29 ----------------------------- drivers/kvm/x86.c | 29 +++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 721e660023f6..c74fb44dd87b 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -665,23 +665,6 @@ void kvm_resched(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_resched); -static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, - struct kvm_interrupt *irq) -{ - if (irq->irq < 0 || irq->irq >= 256) - return -EINVAL; - if (irqchip_in_kernel(vcpu->kvm)) - return -ENXIO; - vcpu_load(vcpu); - - set_bit(irq->irq, vcpu->irq_pending); - set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary); - - vcpu_put(vcpu); - - return 0; -} - static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma, unsigned long address, int *type) @@ -883,18 +866,6 @@ static long kvm_vcpu_ioctl(struct file *filp, r = 0; break; } - case KVM_INTERRUPT: { - struct kvm_interrupt irq; - - r = -EFAULT; - if (copy_from_user(&irq, argp, sizeof irq)) - goto out; - r = kvm_vcpu_ioctl_interrupt(vcpu, &irq); - if (r) - goto out; - r = 0; - break; - } case KVM_DEBUG_GUEST: { struct kvm_debug_guest dbg; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 12f1d6fc92bb..c9e4b67bfb1b 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1001,6 +1001,23 @@ static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, return 0; } +static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, + struct kvm_interrupt *irq) +{ + if (irq->irq < 0 || irq->irq >= 256) + return -EINVAL; + if (irqchip_in_kernel(vcpu->kvm)) + return -ENXIO; + vcpu_load(vcpu); + + set_bit(irq->irq, vcpu->irq_pending); + set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary); + + vcpu_put(vcpu); + + return 0; +} + long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -1034,6 +1051,18 @@ long kvm_arch_vcpu_ioctl(struct file *filp, r = 0; break; } + case KVM_INTERRUPT: { + struct kvm_interrupt irq; + + r = -EFAULT; + if (copy_from_user(&irq, argp, sizeof irq)) + goto out; + r = kvm_vcpu_ioctl_interrupt(vcpu, &irq); + if (r) + goto out; + r = 0; + break; + } case KVM_SET_CPUID: { struct kvm_cpuid __user *cpuid_arg = argp; struct kvm_cpuid cpuid; From d23087847184a7417fc69bdfaa8a32834b447bef Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Thu, 29 Nov 2007 15:35:39 +0800 Subject: [PATCH 176/249] KVM: Correct kvm_init() error paths not freeing bad_pge. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index c74fb44dd87b..058366f480dc 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1311,7 +1311,7 @@ int kvm_init(void *opaque, unsigned int vcpu_size, r = kvm_arch_init(opaque); if (r) - goto out4; + goto out_fail; bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO); @@ -1322,29 +1322,29 @@ int kvm_init(void *opaque, unsigned int vcpu_size, r = kvm_arch_hardware_setup(); if (r < 0) - goto out; + goto out_free_0; for_each_online_cpu(cpu) { smp_call_function_single(cpu, kvm_arch_check_processor_compat, &r, 0, 1); if (r < 0) - goto out_free_0; + goto out_free_1; } on_each_cpu(hardware_enable, NULL, 0, 1); r = register_cpu_notifier(&kvm_cpu_notifier); if (r) - goto out_free_1; + goto out_free_2; register_reboot_notifier(&kvm_reboot_notifier); r = sysdev_class_register(&kvm_sysdev_class); if (r) - goto out_free_2; + goto out_free_3; r = sysdev_register(&kvm_sysdev); if (r) - goto out_free_3; + goto out_free_4; /* A kmem cache lets us meet the alignment requirements of fx_save. */ kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, @@ -1352,7 +1352,7 @@ int kvm_init(void *opaque, unsigned int vcpu_size, 0, NULL); if (!kvm_vcpu_cache) { r = -ENOMEM; - goto out_free_4; + goto out_free_5; } kvm_chardev_ops.owner = module; @@ -1370,21 +1370,23 @@ int kvm_init(void *opaque, unsigned int vcpu_size, out_free: kmem_cache_destroy(kvm_vcpu_cache); -out_free_4: +out_free_5: sysdev_unregister(&kvm_sysdev); -out_free_3: +out_free_4: sysdev_class_unregister(&kvm_sysdev_class); -out_free_2: +out_free_3: unregister_reboot_notifier(&kvm_reboot_notifier); unregister_cpu_notifier(&kvm_cpu_notifier); -out_free_1: +out_free_2: on_each_cpu(hardware_disable, NULL, 0, 1); -out_free_0: +out_free_1: kvm_arch_hardware_unsetup(); +out_free_0: + __free_page(bad_page); out: kvm_arch_exit(); kvm_exit_debug(); -out4: +out_fail: return r; } EXPORT_SYMBOL_GPL(kvm_init); From fb56dbb31c4738a3918db81fd24da732ce3b4ae6 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 2 Dec 2007 10:50:06 +0200 Subject: [PATCH 177/249] KVM: Export include/linux/kvm.h only if $ARCH actually supports KVM Currently, make headers_check barfs due to , which includes, not existing. Rather than add a zillion s, export kvm.h only if the arch actually supports it. Signed-off-by: Avi Kivity --- arch/x86/Kconfig | 1 + drivers/kvm/Kconfig | 7 +++++-- include/linux/Kbuild | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index fb3eea3e38ee..d289cfcf92c4 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -107,6 +107,7 @@ config ARCH_SUPPORTS_OPROFILE bool default y +select HAVE_KVM config ZONE_DMA32 bool diff --git a/drivers/kvm/Kconfig b/drivers/kvm/Kconfig index 656920636cb2..c83e1c9b5129 100644 --- a/drivers/kvm/Kconfig +++ b/drivers/kvm/Kconfig @@ -1,9 +1,12 @@ # # KVM configuration # +config HAVE_KVM + bool + menuconfig VIRTUALIZATION bool "Virtualization" - depends on X86 + depends on HAVE_KVM || X86 default y ---help--- Say Y here to get to see options for using your Linux host to run other @@ -16,7 +19,7 @@ if VIRTUALIZATION config KVM tristate "Kernel-based Virtual Machine (KVM) support" - depends on X86 && EXPERIMENTAL + depends on HAVE_KVM && EXPERIMENTAL select PREEMPT_NOTIFIERS select ANON_INODES ---help--- diff --git a/include/linux/Kbuild b/include/linux/Kbuild index c263a69b5680..85b2482cc736 100644 --- a/include/linux/Kbuild +++ b/include/linux/Kbuild @@ -255,7 +255,7 @@ unifdef-y += kd.h unifdef-y += kernelcapi.h unifdef-y += kernel.h unifdef-y += keyboard.h -unifdef-y += kvm.h +unifdef-$(CONFIG_HAVE_KVM) += kvm.h unifdef-y += llc.h unifdef-y += loop.h unifdef-y += lp.h From 2b5203ee6828e9cabab1e1fb42e53d72ab9351a6 Mon Sep 17 00:00:00 2001 From: Carlo Marcelo Arenas Belon Date: Sat, 1 Dec 2007 06:17:11 -0600 Subject: [PATCH 178/249] KVM: SVM: Remove KVM specific defines for MSR_EFER This patch removes the KVM specific defines for MSR_EFER that were being used in the svm support file and migrates all references to use instead the ones from the kernel headers that are used everywhere else and that have the same values. Signed-off-by: Carlo Marcelo Arenas Belon Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index fb3721d88dbf..c75c6b65b651 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -42,9 +42,6 @@ MODULE_LICENSE("GPL"); #define SEG_TYPE_LDT 2 #define SEG_TYPE_BUSY_TSS16 3 -#define KVM_EFER_LMA (1 << 10) -#define KVM_EFER_LME (1 << 8) - #define SVM_FEATURE_NPT (1 << 0) #define SVM_FEATURE_LBRV (1 << 1) #define SVM_DEATURE_SVML (1 << 2) @@ -184,8 +181,8 @@ static inline void flush_guest_tlb(struct kvm_vcpu *vcpu) static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) { - if (!(efer & KVM_EFER_LMA)) - efer &= ~KVM_EFER_LME; + if (!(efer & EFER_LMA)) + efer &= ~EFER_LME; to_svm(vcpu)->vmcb->save.efer = efer | MSR_EFER_SVME_MASK; vcpu->shadow_efer = efer; @@ -777,15 +774,15 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) struct vcpu_svm *svm = to_svm(vcpu); #ifdef CONFIG_X86_64 - if (vcpu->shadow_efer & KVM_EFER_LME) { + if (vcpu->shadow_efer & EFER_LME) { if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { - vcpu->shadow_efer |= KVM_EFER_LMA; - svm->vmcb->save.efer |= KVM_EFER_LMA | KVM_EFER_LME; + vcpu->shadow_efer |= EFER_LMA; + svm->vmcb->save.efer |= EFER_LMA | EFER_LME; } if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) { - vcpu->shadow_efer &= ~KVM_EFER_LMA; - svm->vmcb->save.efer &= ~(KVM_EFER_LMA | KVM_EFER_LME); + vcpu->shadow_efer &= ~EFER_LMA; + svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME); } } #endif From 8be5453f95933340a42c6e7fc7b66f3bb786fddd Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Sun, 2 Dec 2007 22:35:57 +0800 Subject: [PATCH 179/249] KVM: Replace kvm_lapic with kvm_vcpu in ioapic/lapic interface This patch replaces lapic structure with kvm_vcpu in ioapic.c, making ioapic independent of the local apic, as required by ia64. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/ioapic.c | 18 ++++++++---------- drivers/kvm/irq.h | 5 +++-- drivers/kvm/lapic.c | 23 ++++++++++++++++++----- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/drivers/kvm/ioapic.c b/drivers/kvm/ioapic.c index cf1d50b3549b..3629867a76bc 100644 --- a/drivers/kvm/ioapic.c +++ b/drivers/kvm/ioapic.c @@ -136,7 +136,7 @@ static void ioapic_write_indirect(struct kvm_ioapic *ioapic, u32 val) } static void ioapic_inj_irq(struct kvm_ioapic *ioapic, - struct kvm_lapic *target, + struct kvm_vcpu *vcpu, u8 vector, u8 trig_mode, u8 delivery_mode) { ioapic_debug("irq %d trig %d deliv %d\n", vector, trig_mode, @@ -145,7 +145,7 @@ static void ioapic_inj_irq(struct kvm_ioapic *ioapic, ASSERT((delivery_mode == dest_Fixed) || (delivery_mode == dest_LowestPrio)); - kvm_apic_set_irq(target, vector, trig_mode); + kvm_apic_set_irq(vcpu, vector, trig_mode); } static u32 ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest, @@ -196,7 +196,6 @@ static void ioapic_deliver(struct kvm_ioapic *ioapic, int irq) u8 vector = ioapic->redirtbl[irq].fields.vector; u8 trig_mode = ioapic->redirtbl[irq].fields.trig_mode; u32 deliver_bitmask; - struct kvm_lapic *target; struct kvm_vcpu *vcpu; int vcpu_id; @@ -212,13 +211,13 @@ static void ioapic_deliver(struct kvm_ioapic *ioapic, int irq) switch (delivery_mode) { case dest_LowestPrio: - target = - kvm_apic_round_robin(ioapic->kvm, vector, deliver_bitmask); - if (target != NULL) - ioapic_inj_irq(ioapic, target, vector, + vcpu = kvm_get_lowest_prio_vcpu(ioapic->kvm, vector, + deliver_bitmask); + if (vcpu != NULL) + ioapic_inj_irq(ioapic, vcpu, vector, trig_mode, delivery_mode); else - ioapic_debug("null round robin: " + ioapic_debug("null lowest prio vcpu: " "mask=%x vector=%x delivery_mode=%x\n", deliver_bitmask, vector, dest_LowestPrio); break; @@ -229,8 +228,7 @@ static void ioapic_deliver(struct kvm_ioapic *ioapic, int irq) deliver_bitmask &= ~(1 << vcpu_id); vcpu = ioapic->kvm->vcpus[vcpu_id]; if (vcpu) { - target = vcpu->apic; - ioapic_inj_irq(ioapic, target, vector, + ioapic_inj_irq(ioapic, vcpu, vector, trig_mode, delivery_mode); } } diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index 5ad3cfd7622d..78a34dde1cb9 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -145,14 +145,15 @@ void kvm_free_lapic(struct kvm_vcpu *vcpu); u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu); void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8); void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value); -struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, + +struct kvm_vcpu *kvm_get_lowest_prio_vcpu(struct kvm *kvm, u8 vector, unsigned long bitmap); u64 kvm_get_apic_base(struct kvm_vcpu *vcpu); void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data); int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest); void kvm_ioapic_update_eoi(struct kvm *kvm, int vector); int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda); -int kvm_apic_set_irq(struct kvm_lapic *apic, u8 vec, u8 trig); +int kvm_apic_set_irq(struct kvm_vcpu *vcpu, u8 vec, u8 trig); void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu); int kvm_ioapic_init(struct kvm *kvm); void kvm_ioapic_set_irq(struct kvm_ioapic *ioapic, int irq, int level); diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index 64f74bd7093a..5efa6c0276c6 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -185,8 +185,10 @@ int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr); -int kvm_apic_set_irq(struct kvm_lapic *apic, u8 vec, u8 trig) +int kvm_apic_set_irq(struct kvm_vcpu *vcpu, u8 vec, u8 trig) { + struct kvm_lapic *apic = vcpu->apic; + if (!apic_test_and_set_irr(vec, apic)) { /* a new pending irq is set in IRR */ if (trig) @@ -394,7 +396,7 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, return result; } -struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, +static struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, unsigned long bitmap) { int last; @@ -422,6 +424,17 @@ struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, return apic; } +struct kvm_vcpu *kvm_get_lowest_prio_vcpu(struct kvm *kvm, u8 vector, + unsigned long bitmap) +{ + struct kvm_lapic *apic; + + apic = kvm_apic_round_robin(kvm, vector, bitmap); + if (apic) + return apic->vcpu; + return NULL; +} + static void apic_set_eoi(struct kvm_lapic *apic) { int vector = apic_find_highest_isr(apic); @@ -453,7 +466,7 @@ static void apic_send_ipi(struct kvm_lapic *apic) unsigned int delivery_mode = icr_low & APIC_MODE_MASK; unsigned int vector = icr_low & APIC_VECTOR_MASK; - struct kvm_lapic *target; + struct kvm_vcpu *target; struct kvm_vcpu *vcpu; unsigned long lpr_map = 0; int i; @@ -480,9 +493,9 @@ static void apic_send_ipi(struct kvm_lapic *apic) } if (delivery_mode == APIC_DM_LOWEST) { - target = kvm_apic_round_robin(vcpu->kvm, vector, lpr_map); + target = kvm_get_lowest_prio_vcpu(vcpu->kvm, vector, lpr_map); if (target != NULL) - __apic_accept_irq(target, delivery_mode, + __apic_accept_irq(target->apic, delivery_mode, vector, level, trig_mode); } } From 0c7ac28d3dd27d718493aae4bbc7de89a2d9d386 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Sun, 2 Dec 2007 22:49:09 +0800 Subject: [PATCH 180/249] KVM: Replace dest_Lowest_Prio and dest_Fixed with self-defined macros Change dest_Loest_Prio -> IOAPIC_LOWEST_PRIORITY dest_Fixed -> IOAPIC_FIXED the original names are x86 specific, while the ioapic code will be reused for ia64. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/ioapic.c | 13 +++++-------- drivers/kvm/irq.h | 8 ++++++++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/drivers/kvm/ioapic.c b/drivers/kvm/ioapic.c index 3629867a76bc..0feae6f7fb2d 100644 --- a/drivers/kvm/ioapic.c +++ b/drivers/kvm/ioapic.c @@ -36,11 +36,8 @@ #include #include #include -#include #include #include -#include -#include #include "irq.h" #if 0 #define ioapic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) @@ -142,8 +139,8 @@ static void ioapic_inj_irq(struct kvm_ioapic *ioapic, ioapic_debug("irq %d trig %d deliv %d\n", vector, trig_mode, delivery_mode); - ASSERT((delivery_mode == dest_Fixed) || - (delivery_mode == dest_LowestPrio)); + ASSERT((delivery_mode == IOAPIC_FIXED) || + (delivery_mode == IOAPIC_LOWEST_PRIORITY)); kvm_apic_set_irq(vcpu, vector, trig_mode); } @@ -210,7 +207,7 @@ static void ioapic_deliver(struct kvm_ioapic *ioapic, int irq) } switch (delivery_mode) { - case dest_LowestPrio: + case IOAPIC_LOWEST_PRIORITY: vcpu = kvm_get_lowest_prio_vcpu(ioapic->kvm, vector, deliver_bitmask); if (vcpu != NULL) @@ -219,9 +216,9 @@ static void ioapic_deliver(struct kvm_ioapic *ioapic, int irq) else ioapic_debug("null lowest prio vcpu: " "mask=%x vector=%x delivery_mode=%x\n", - deliver_bitmask, vector, dest_LowestPrio); + deliver_bitmask, vector, IOAPIC_LOWEST_PRIORITY); break; - case dest_Fixed: + case IOAPIC_FIXED: for (vcpu_id = 0; deliver_bitmask != 0; vcpu_id++) { if (!(deliver_bitmask & (1 << vcpu_id))) continue; diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index 78a34dde1cb9..75f5f18d801c 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -79,6 +79,14 @@ void kvm_pic_update_irq(struct kvm_pic *s); #define IOAPIC_REG_VERSION 0x01 #define IOAPIC_REG_ARB_ID 0x02 /* x86 IOAPIC only */ +/*ioapic delivery mode*/ +#define IOAPIC_FIXED 0x0 +#define IOAPIC_LOWEST_PRIORITY 0x1 +#define IOAPIC_PMI 0x2 +#define IOAPIC_NMI 0x4 +#define IOAPIC_INIT 0x5 +#define IOAPIC_EXTINT 0x7 + struct kvm_ioapic { u64 base_address; u32 ioregsel; From b1fd3d30ba46360fc94d40ee8dc1f7a4338b19be Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Sun, 2 Dec 2007 22:53:07 +0800 Subject: [PATCH 181/249] KVM: Extend ioapic code to support iosapic iosapic supports an additional mmio EOI register compared to ioapic. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/ioapic.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/kvm/ioapic.c b/drivers/kvm/ioapic.c index 0feae6f7fb2d..e7debfafca50 100644 --- a/drivers/kvm/ioapic.c +++ b/drivers/kvm/ioapic.c @@ -360,6 +360,11 @@ static void ioapic_mmio_write(struct kvm_io_device *this, gpa_t addr, int len, case IOAPIC_REG_WINDOW: ioapic_write_indirect(ioapic, data); break; +#ifdef CONFIG_IA64 + case IOAPIC_REG_EOI: + kvm_ioapic_update_eoi(ioapic, data); + break; +#endif default: break; From d77a39d982431ef9efc7e7a1690cee2ce252b95e Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Mon, 3 Dec 2007 15:30:23 -0600 Subject: [PATCH 182/249] KVM: Portability: Move address types to their own header file Signed-off-by: Hollis Blanchard Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 21 ++------------------- drivers/kvm/types.h | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 19 deletions(-) create mode 100644 drivers/kvm/types.h diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index be18620bd656..3b0ba5ab7104 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -20,6 +20,8 @@ #include #include +#include "types.h" + #define KVM_MAX_VCPUS 4 #define KVM_ALIAS_SLOTS 4 #define KVM_MEMORY_SLOTS 8 @@ -39,25 +41,6 @@ */ #define KVM_REQ_TLB_FLUSH 0 -/* - * Address types: - * - * gva - guest virtual address - * gpa - guest physical address - * gfn - guest frame number - * hva - host virtual address - * hpa - host physical address - * hfn - host frame number - */ - -typedef unsigned long gva_t; -typedef u64 gpa_t; -typedef unsigned long gfn_t; - -typedef unsigned long hva_t; -typedef u64 hpa_t; -typedef unsigned long hfn_t; - #define NR_PTE_CHAIN_ENTRIES 5 struct kvm_pte_chain { diff --git a/drivers/kvm/types.h b/drivers/kvm/types.h new file mode 100644 index 000000000000..6ad76231b8ff --- /dev/null +++ b/drivers/kvm/types.h @@ -0,0 +1,41 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ + +#ifndef __KVM_TYPES_H__ +#define __KVM_TYPES_H__ + +#include + +/* + * Address types: + * + * gva - guest virtual address + * gpa - guest physical address + * gfn - guest frame number + * hva - host virtual address + * hpa - host physical address + * hfn - host frame number + */ + +typedef unsigned long gva_t; +typedef u64 gpa_t; +typedef unsigned long gfn_t; + +typedef unsigned long hva_t; +typedef u64 hpa_t; +typedef unsigned long hfn_t; + +#endif /* __KVM_TYPES_H__ */ From e2174021cfa535dbcaef02dc6f2897019c30731d Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Mon, 3 Dec 2007 15:30:24 -0600 Subject: [PATCH 183/249] KVM: Portability: Move IO device definitions to its own header file Signed-off-by: Hollis Blanchard Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/iodev.h | 63 ++++++++++++++++++++++++++++++++++++++++++ drivers/kvm/irq.h | 1 + drivers/kvm/kvm.h | 42 ---------------------------- drivers/kvm/kvm_main.c | 1 + 4 files changed, 65 insertions(+), 42 deletions(-) create mode 100644 drivers/kvm/iodev.h diff --git a/drivers/kvm/iodev.h b/drivers/kvm/iodev.h new file mode 100644 index 000000000000..eb9e8a71843a --- /dev/null +++ b/drivers/kvm/iodev.h @@ -0,0 +1,63 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __KVM_IODEV_H__ +#define __KVM_IODEV_H__ + +#include "types.h" + +struct kvm_io_device { + void (*read)(struct kvm_io_device *this, + gpa_t addr, + int len, + void *val); + void (*write)(struct kvm_io_device *this, + gpa_t addr, + int len, + const void *val); + int (*in_range)(struct kvm_io_device *this, gpa_t addr); + void (*destructor)(struct kvm_io_device *this); + + void *private; +}; + +static inline void kvm_iodevice_read(struct kvm_io_device *dev, + gpa_t addr, + int len, + void *val) +{ + dev->read(dev, addr, len, val); +} + +static inline void kvm_iodevice_write(struct kvm_io_device *dev, + gpa_t addr, + int len, + const void *val) +{ + dev->write(dev, addr, len, val); +} + +static inline int kvm_iodevice_inrange(struct kvm_io_device *dev, gpa_t addr) +{ + return dev->in_range(dev, addr); +} + +static inline void kvm_iodevice_destructor(struct kvm_io_device *dev) +{ + if (dev->destructor) + dev->destructor(dev); +} + +#endif /* __KVM_IODEV_H__ */ diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index 75f5f18d801c..803b9c7db57b 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -23,6 +23,7 @@ #define __IRQ_H #include "kvm.h" +#include "iodev.h" typedef void irq_request_func(void *opaque, int level); diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 3b0ba5ab7104..a1b7d1c00bde 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -168,48 +168,6 @@ struct kvm_vcpu_stat { u32 insn_emulation_fail; }; -struct kvm_io_device { - void (*read)(struct kvm_io_device *this, - gpa_t addr, - int len, - void *val); - void (*write)(struct kvm_io_device *this, - gpa_t addr, - int len, - const void *val); - int (*in_range)(struct kvm_io_device *this, gpa_t addr); - void (*destructor)(struct kvm_io_device *this); - - void *private; -}; - -static inline void kvm_iodevice_read(struct kvm_io_device *dev, - gpa_t addr, - int len, - void *val) -{ - dev->read(dev, addr, len, val); -} - -static inline void kvm_iodevice_write(struct kvm_io_device *dev, - gpa_t addr, - int len, - const void *val) -{ - dev->write(dev, addr, len, val); -} - -static inline int kvm_iodevice_inrange(struct kvm_io_device *dev, gpa_t addr) -{ - return dev->in_range(dev, addr); -} - -static inline void kvm_iodevice_destructor(struct kvm_io_device *dev) -{ - if (dev->destructor) - dev->destructor(dev); -} - /* * It would be nice to use something smarter than a linear search, TBD... * Thankfully we dont expect many devices to register (famous last words :), diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 058366f480dc..14b376ead3da 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -18,6 +18,7 @@ #include "kvm.h" #include "x86.h" #include "irq.h" +#include "iodev.h" #include #include From e01a1b570f66ad318239517adbcc2cbe368d0a46 Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Mon, 3 Dec 2007 15:30:25 -0600 Subject: [PATCH 184/249] KVM: Portability: Stop including x86-specific headers in kvm_main.c Signed-off-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/i8259.c | 1 + drivers/kvm/irq.h | 7 ++++++- drivers/kvm/kvm.h | 9 ++++----- drivers/kvm/kvm_main.c | 3 --- drivers/kvm/x86.h | 8 ++++++++ 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/drivers/kvm/i8259.c b/drivers/kvm/i8259.c index f0dc2eeadf0b..b3cad632f3d5 100644 --- a/drivers/kvm/i8259.c +++ b/drivers/kvm/i8259.c @@ -27,6 +27,7 @@ */ #include #include "irq.h" +#include "kvm.h" /* * set irq level. If an edge is detected, then the IRR is set to 1 diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index 803b9c7db57b..730a87c173e5 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -22,9 +22,14 @@ #ifndef __IRQ_H #define __IRQ_H -#include "kvm.h" +#include +#include +#include #include "iodev.h" +struct kvm; +struct kvm_vcpu; + typedef void irq_request_func(void *opaque, int level); struct kvm_kpic_state { diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index a1b7d1c00bde..0d3555bf5333 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -288,11 +288,6 @@ static inline int irqchip_in_kernel(struct kvm *kvm) return pic_irqchip(kvm) != NULL; } -struct descriptor_table { - u16 limit; - unsigned long base; -} __attribute__((packed)); - /* The guest did something we don't support. */ #define pr_unimpl(vcpu, fmt, ...) \ do { \ @@ -457,4 +452,8 @@ struct kvm_stats_debugfs_item { }; extern struct kvm_stats_debugfs_item debugfs_entries[]; +#if defined(CONFIG_X86) +#include "x86.h" +#endif + #endif diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 14b376ead3da..0f11046b94fa 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -16,8 +16,6 @@ */ #include "kvm.h" -#include "x86.h" -#include "irq.h" #include "iodev.h" #include @@ -46,7 +44,6 @@ #include #include #include -#include #include MODULE_AUTHOR("Qumranet"); diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 78ab1e108d8b..4b7acc76bb34 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -12,6 +12,7 @@ #define KVM_X86_H #include "kvm.h" +#include "irq.h" #include #include @@ -19,6 +20,8 @@ #include #include +#include + #define CR3_PAE_RESERVED_BITS ((X86_CR3_PWT | X86_CR3_PCD) - 1) #define CR3_NONPAE_RESERVED_BITS ((PAGE_SIZE-1) & ~(X86_CR3_PWT | X86_CR3_PCD)) #define CR3_L_MODE_RESERVED_BITS (CR3_NONPAE_RESERVED_BITS|0xFFFFFF0000000000ULL) @@ -156,6 +159,11 @@ struct kvm_vcpu { struct x86_emulate_ctxt emulate_ctxt; }; +struct descriptor_table { + u16 limit; + unsigned long base; +} __attribute__((packed)); + struct kvm_x86_ops { int (*cpu_has_kvm_support)(void); /* __init */ int (*disabled_by_bios)(void); /* __init */ From 53e0aa7b65ec0d66e34e4d356a641c725cc2e519 Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Mon, 3 Dec 2007 16:15:26 -0600 Subject: [PATCH 185/249] KVM: Portability: Create kvm_arch_vcpu_runnable() function This abstracts the detail of x86 hlt and INIT modes into a function. Signed-off-by: Hollis Blanchard Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 3 +-- drivers/kvm/x86.h | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 0f11046b94fa..70664f020849 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -643,8 +643,7 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) */ while (!kvm_cpu_has_interrupt(vcpu) && !signal_pending(current) - && vcpu->mp_state != VCPU_MP_STATE_RUNNABLE - && vcpu->mp_state != VCPU_MP_STATE_SIPI_RECEIVED) { + && !kvm_arch_vcpu_runnable(vcpu)) { set_current_state(TASK_INTERRUPTIBLE); vcpu_put(vcpu); schedule(); diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 4b7acc76bb34..eed796402e3b 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -471,4 +471,11 @@ static inline u32 get_rdx_init_val(void) #define TSS_IOPB_SIZE (65536 / 8) #define TSS_REDIRECTION_SIZE (256 / 8) #define RMODE_TSS_SIZE (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) + +static inline int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) +{ + return vcpu->mp_state == VCPU_MP_STATE_RUNNABLE + || vcpu->mp_state == VCPU_MP_STATE_SIPI_RECEIVED; +} + #endif From e4a533a416936753882060bf743222576713fde8 Mon Sep 17 00:00:00 2001 From: "npiggin@suse.de" Date: Wed, 5 Dec 2007 18:15:52 +1100 Subject: [PATCH 186/249] KVM: Convert KVM from ->nopage() to ->fault() Signed-off-by: Nick Piggin Cc: kvm-devel@lists.sourceforge.net Cc: avi@qumranet.com Cc: linux-kernel@vger.kernel.org Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 42 +++++++++++++++--------------------------- 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 70664f020849..1a03b67eb921 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -662,30 +662,24 @@ void kvm_resched(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_resched); -static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma, - unsigned long address, - int *type) +static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf) { struct kvm_vcpu *vcpu = vma->vm_file->private_data; - unsigned long pgoff; struct page *page; - pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; - if (pgoff == 0) + if (vmf->pgoff == 0) page = virt_to_page(vcpu->run); - else if (pgoff == KVM_PIO_PAGE_OFFSET) + else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) page = virt_to_page(vcpu->pio_data); else - return NOPAGE_SIGBUS; + return VM_FAULT_SIGBUS; get_page(page); - if (type != NULL) - *type = VM_FAULT_MINOR; - - return page; + vmf->page = page; + return 0; } static struct vm_operations_struct kvm_vcpu_vm_ops = { - .nopage = kvm_vcpu_nopage, + .fault = kvm_vcpu_fault, }; static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) @@ -976,31 +970,25 @@ out: return r; } -static struct page *kvm_vm_nopage(struct vm_area_struct *vma, - unsigned long address, - int *type) +static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) { struct kvm *kvm = vma->vm_file->private_data; - unsigned long pgoff; struct page *page; - pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; - if (!kvm_is_visible_gfn(kvm, pgoff)) - return NOPAGE_SIGBUS; + if (!kvm_is_visible_gfn(kvm, vmf->pgoff)) + return VM_FAULT_SIGBUS; /* current->mm->mmap_sem is already held so call lockless version */ - page = __gfn_to_page(kvm, pgoff); + page = __gfn_to_page(kvm, vmf->pgoff); if (is_error_page(page)) { kvm_release_page_clean(page); - return NOPAGE_SIGBUS; + return VM_FAULT_SIGBUS; } - if (type != NULL) - *type = VM_FAULT_MINOR; - - return page; + vmf->page = page; + return 0; } static struct vm_operations_struct kvm_vm_vm_ops = { - .nopage = kvm_vm_nopage, + .fault = kvm_vm_fault, }; static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma) From 4bf8ed8dd2781a5e7603a83f8ee1d4f5aa04ebc4 Mon Sep 17 00:00:00 2001 From: Marcelo Tosatti Date: Tue, 4 Dec 2007 13:42:16 -0500 Subject: [PATCH 187/249] KVM: MMU: Remove unused prev_shadow_ent variable from fetch() Signed-off-by: Marcelo Tosatti Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index b24bc7c86078..3c40a598ceb8 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -300,7 +300,6 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, hpa_t shadow_addr; int level; u64 *shadow_ent; - u64 *prev_shadow_ent = NULL; if (!is_present_pte(walker->pte)) return NULL; @@ -326,7 +325,6 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, if (level == PT_PAGE_TABLE_LEVEL) break; shadow_addr = *shadow_ent & PT64_BASE_ADDR_MASK; - prev_shadow_ent = shadow_ent; continue; } @@ -355,7 +353,6 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, shadow_pte = shadow_addr | PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK; *shadow_ent = shadow_pte; - prev_shadow_ent = shadow_ent; } FNAME(set_pte)(vcpu, walker->pte, shadow_ent, From 298101da2f507c13eaf179ee4507a7c0fe3e7b06 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 25 Nov 2007 13:41:11 +0200 Subject: [PATCH 188/249] KVM: Generalize exception injection mechanism Instead of each subarch doing its own thing, add an API for queuing an injection, and manage failed exception injection centerally (i.e., if an inject failed due to a shadow page fault, we need to requeue it). Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 21 +++++++++++++++++++++ drivers/kvm/vmx.c | 20 ++++++++++++++++++++ drivers/kvm/x86.c | 33 ++++++++++++++++++++++++++++++++- drivers/kvm/x86.h | 13 +++++++++++++ 4 files changed, 86 insertions(+), 1 deletion(-) diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index c75c6b65b651..87072c647f28 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -188,6 +188,25 @@ static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) vcpu->shadow_efer = efer; } +static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, + bool has_error_code, u32 error_code) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + svm->vmcb->control.event_inj = nr + | SVM_EVTINJ_VALID + | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0) + | SVM_EVTINJ_TYPE_EXEPT; + svm->vmcb->control.event_inj_err = error_code; +} + +static bool svm_exception_injected(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + return !(svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID); +} + static void svm_inject_gp(struct kvm_vcpu *vcpu, unsigned error_code) { struct vcpu_svm *svm = to_svm(vcpu); @@ -1712,6 +1731,8 @@ static struct kvm_x86_ops svm_x86_ops = { .patch_hypercall = svm_patch_hypercall, .get_irq = svm_get_irq, .set_irq = svm_set_irq, + .queue_exception = svm_queue_exception, + .exception_injected = svm_exception_injected, .inject_pending_irq = svm_intr_assist, .inject_pending_vectors = do_interrupt_requests, diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index fc5e7c8381ce..f382956f176a 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -595,6 +595,24 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) vcpu->interrupt_window_open = 1; } +static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, + bool has_error_code, u32 error_code) +{ + vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, + nr | INTR_TYPE_EXCEPTION + | (has_error_code ? INTR_INFO_DELIEVER_CODE_MASK : 0) + | INTR_INFO_VALID_MASK); + if (has_error_code) + vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code); +} + +static bool vmx_exception_injected(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + + return !(vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK); +} + static void vmx_inject_gp(struct kvm_vcpu *vcpu, unsigned error_code) { printk(KERN_DEBUG "inject_general_protection: rip 0x%lx\n", @@ -2641,6 +2659,8 @@ static struct kvm_x86_ops vmx_x86_ops = { .patch_hypercall = vmx_patch_hypercall, .get_irq = vmx_get_irq, .set_irq = vmx_inject_irq, + .queue_exception = vmx_queue_exception, + .exception_injected = vmx_exception_injected, .inject_pending_irq = vmx_intr_assist, .inject_pending_vectors = do_interrupt_requests, diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index c9e4b67bfb1b..11440d12a2d3 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -133,6 +133,32 @@ static void inject_gp(struct kvm_vcpu *vcpu) kvm_x86_ops->inject_gp(vcpu, 0); } +void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) +{ + WARN_ON(vcpu->exception.pending); + vcpu->exception.pending = true; + vcpu->exception.has_error_code = false; + vcpu->exception.nr = nr; +} +EXPORT_SYMBOL_GPL(kvm_queue_exception); + +void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) +{ + WARN_ON(vcpu->exception.pending); + vcpu->exception.pending = true; + vcpu->exception.has_error_code = true; + vcpu->exception.nr = nr; + vcpu->exception.error_code = error_code; +} +EXPORT_SYMBOL_GPL(kvm_queue_exception_e); + +static void __queue_exception(struct kvm_vcpu *vcpu) +{ + kvm_x86_ops->queue_exception(vcpu, vcpu->exception.nr, + vcpu->exception.has_error_code, + vcpu->exception.error_code); +} + /* * Load the pae pdptrs. Return true is they are all valid. */ @@ -2370,7 +2396,9 @@ again: goto out; } - if (irqchip_in_kernel(vcpu->kvm)) + if (vcpu->exception.pending) + __queue_exception(vcpu); + else if (irqchip_in_kernel(vcpu->kvm)) kvm_x86_ops->inject_pending_irq(vcpu); else kvm_x86_ops->inject_pending_vectors(vcpu, kvm_run); @@ -2409,6 +2437,9 @@ again: profile_hit(KVM_PROFILING, (void *)vcpu->rip); } + if (vcpu->exception.pending && kvm_x86_ops->exception_injected(vcpu)) + vcpu->exception.pending = false; + r = kvm_x86_ops->handle_exit(kvm_run, vcpu); if (r > 0) { diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index eed796402e3b..1e71668694ea 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -139,6 +139,13 @@ struct kvm_vcpu { struct kvm_pio_request pio; void *pio_data; + struct kvm_queued_exception { + bool pending; + bool has_error_code; + u8 nr; + u32 error_code; + } exception; + struct { int active; u8 save_iopl; @@ -224,6 +231,9 @@ struct kvm_x86_ops { unsigned char *hypercall_addr); int (*get_irq)(struct kvm_vcpu *vcpu); void (*set_irq)(struct kvm_vcpu *vcpu, int vec); + void (*queue_exception)(struct kvm_vcpu *vcpu, unsigned nr, + bool has_error_code, u32 error_code); + bool (*exception_injected)(struct kvm_vcpu *vcpu); void (*inject_pending_irq)(struct kvm_vcpu *vcpu); void (*inject_pending_vectors)(struct kvm_vcpu *vcpu, struct kvm_run *run); @@ -294,6 +304,9 @@ void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l); int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data); +void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr); +void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); + void fx_init(struct kvm_vcpu *vcpu); int emulator_read_std(unsigned long addr, From c3c91fee5195ba5176a6da5ddc2a2822243eb79f Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 25 Nov 2007 14:04:58 +0200 Subject: [PATCH 189/249] KVM: Replace page fault injection by the generalized exception queue Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 2 +- drivers/kvm/svm.c | 35 ----------------------------------- drivers/kvm/vmx.c | 32 -------------------------------- drivers/kvm/x86.c | 17 ++++++++++++++++- drivers/kvm/x86.h | 4 ++-- 5 files changed, 19 insertions(+), 71 deletions(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 9b9d1b6eb2f8..62a74151f847 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -1081,7 +1081,7 @@ static void inject_page_fault(struct kvm_vcpu *vcpu, u64 addr, u32 err_code) { - kvm_x86_ops->inject_page_fault(vcpu, addr, err_code); + kvm_inject_page_fault(vcpu, addr, err_code); } static void paging_free(struct kvm_vcpu *vcpu) diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 87072c647f28..aa8e90b404a0 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -225,12 +225,6 @@ static void inject_ud(struct kvm_vcpu *vcpu) UD_VECTOR; } -static int is_page_fault(uint32_t info) -{ - info &= SVM_EVTINJ_VEC_MASK | SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID; - return info == (PF_VECTOR | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT); -} - static int is_external_interrupt(u32 info) { info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID; @@ -1624,34 +1618,6 @@ static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root) } } -static void svm_inject_page_fault(struct kvm_vcpu *vcpu, - unsigned long addr, - uint32_t err_code) -{ - struct vcpu_svm *svm = to_svm(vcpu); - uint32_t exit_int_info = svm->vmcb->control.exit_int_info; - - ++vcpu->stat.pf_guest; - - if (is_page_fault(exit_int_info)) { - - svm->vmcb->control.event_inj_err = 0; - svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | - SVM_EVTINJ_VALID_ERR | - SVM_EVTINJ_TYPE_EXEPT | - DF_VECTOR; - return; - } - vcpu->cr2 = addr; - svm->vmcb->save.cr2 = addr; - svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | - SVM_EVTINJ_VALID_ERR | - SVM_EVTINJ_TYPE_EXEPT | - PF_VECTOR; - svm->vmcb->control.event_inj_err = err_code; -} - - static int is_disabled(void) { u64 vm_cr; @@ -1721,7 +1687,6 @@ static struct kvm_x86_ops svm_x86_ops = { .set_rflags = svm_set_rflags, .tlb_flush = svm_flush_tlb, - .inject_page_fault = svm_inject_page_fault, .inject_gp = svm_inject_gp, diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index f382956f176a..be0b12e709e5 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -2487,37 +2487,6 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) asm("int $2"); } -static void vmx_inject_page_fault(struct kvm_vcpu *vcpu, - unsigned long addr, - u32 err_code) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - u32 vect_info = vmx->idt_vectoring_info; - - ++vcpu->stat.pf_guest; - - if (is_page_fault(vect_info)) { - printk(KERN_DEBUG "inject_page_fault: " - "double fault 0x%lx @ 0x%lx\n", - addr, vmcs_readl(GUEST_RIP)); - vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 0); - vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, - DF_VECTOR | - INTR_TYPE_EXCEPTION | - INTR_INFO_DELIEVER_CODE_MASK | - INTR_INFO_VALID_MASK); - return; - } - vcpu->cr2 = addr; - vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, err_code); - vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, - PF_VECTOR | - INTR_TYPE_EXCEPTION | - INTR_INFO_DELIEVER_CODE_MASK | - INTR_INFO_VALID_MASK); - -} - static void vmx_free_vmcs(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -2649,7 +2618,6 @@ static struct kvm_x86_ops vmx_x86_ops = { .set_rflags = vmx_set_rflags, .tlb_flush = vmx_flush_tlb, - .inject_page_fault = vmx_inject_page_fault, .inject_gp = vmx_inject_gp, diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 11440d12a2d3..dc007a32a883 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -142,6 +142,21 @@ void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) } EXPORT_SYMBOL_GPL(kvm_queue_exception); +void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr, + u32 error_code) +{ + ++vcpu->stat.pf_guest; + if (vcpu->exception.pending && vcpu->exception.nr == PF_VECTOR) { + printk(KERN_DEBUG "kvm: inject_page_fault:" + " double fault 0x%lx\n", addr); + vcpu->exception.nr = DF_VECTOR; + vcpu->exception.error_code = 0; + return; + } + vcpu->cr2 = addr; + kvm_queue_exception_e(vcpu, PF_VECTOR, error_code); +} + void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) { WARN_ON(vcpu->exception.pending); @@ -1601,7 +1616,7 @@ static int emulator_write_emulated_onepage(unsigned long addr, gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); if (gpa == UNMAPPED_GVA) { - kvm_x86_ops->inject_page_fault(vcpu, addr, 2); + kvm_inject_page_fault(vcpu, addr, 2); return X86EMUL_PROPAGATE_FAULT; } diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 1e71668694ea..d3ac4e2b3a41 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -219,8 +219,6 @@ struct kvm_x86_ops { void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); void (*tlb_flush)(struct kvm_vcpu *vcpu); - void (*inject_page_fault)(struct kvm_vcpu *vcpu, - unsigned long addr, u32 err_code); void (*inject_gp)(struct kvm_vcpu *vcpu, unsigned err_code); @@ -306,6 +304,8 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data); void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr); void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); +void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long cr2, + u32 error_code); void fx_init(struct kvm_vcpu *vcpu); From c1a5d4f990ce034bcb19aebbb910c07019e60f6b Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 25 Nov 2007 14:12:03 +0200 Subject: [PATCH 190/249] KVM: Replace #GP injection by the generalized exception queue Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 17 ++-------------- drivers/kvm/vmx.c | 18 ++-------------- drivers/kvm/x86.c | 43 +++++++++++++++++---------------------- drivers/kvm/x86.h | 7 +++++-- drivers/kvm/x86_emulate.c | 4 ++-- 5 files changed, 30 insertions(+), 59 deletions(-) diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index aa8e90b404a0..f9769338c621 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -207,17 +207,6 @@ static bool svm_exception_injected(struct kvm_vcpu *vcpu) return !(svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID); } -static void svm_inject_gp(struct kvm_vcpu *vcpu, unsigned error_code) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | - SVM_EVTINJ_VALID_ERR | - SVM_EVTINJ_TYPE_EXEPT | - GP_VECTOR; - svm->vmcb->control.event_inj_err = error_code; -} - static void inject_ud(struct kvm_vcpu *vcpu) { to_svm(vcpu)->vmcb->control.event_inj = SVM_EVTINJ_VALID | @@ -1115,7 +1104,7 @@ static int rdmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) u64 data; if (svm_get_msr(&svm->vcpu, ecx, &data)) - svm_inject_gp(&svm->vcpu, 0); + kvm_inject_gp(&svm->vcpu, 0); else { svm->vmcb->save.rax = data & 0xffffffff; svm->vcpu.regs[VCPU_REGS_RDX] = data >> 32; @@ -1176,7 +1165,7 @@ static int wrmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | ((u64)(svm->vcpu.regs[VCPU_REGS_RDX] & -1u) << 32); svm->next_rip = svm->vmcb->save.rip + 2; if (svm_set_msr(&svm->vcpu, ecx, data)) - svm_inject_gp(&svm->vcpu, 0); + kvm_inject_gp(&svm->vcpu, 0); else skip_emulated_instruction(&svm->vcpu); return 1; @@ -1688,8 +1677,6 @@ static struct kvm_x86_ops svm_x86_ops = { .tlb_flush = svm_flush_tlb, - .inject_gp = svm_inject_gp, - .run = svm_vcpu_run, .handle_exit = handle_exit, .skip_emulated_instruction = skip_emulated_instruction, diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index be0b12e709e5..3b3c5f7d2e7c 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -613,18 +613,6 @@ static bool vmx_exception_injected(struct kvm_vcpu *vcpu) return !(vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK); } -static void vmx_inject_gp(struct kvm_vcpu *vcpu, unsigned error_code) -{ - printk(KERN_DEBUG "inject_general_protection: rip 0x%lx\n", - vmcs_readl(GUEST_RIP)); - vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code); - vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, - GP_VECTOR | - INTR_TYPE_EXCEPTION | - INTR_INFO_DELIEVER_CODE_MASK | - INTR_INFO_VALID_MASK); -} - static void vmx_inject_ud(struct kvm_vcpu *vcpu) { vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, @@ -2083,7 +2071,7 @@ static int handle_rdmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) u64 data; if (vmx_get_msr(vcpu, ecx, &data)) { - vmx_inject_gp(vcpu, 0); + kvm_inject_gp(vcpu, 0); return 1; } @@ -2101,7 +2089,7 @@ static int handle_wrmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | ((u64)(vcpu->regs[VCPU_REGS_RDX] & -1u) << 32); if (vmx_set_msr(vcpu, ecx, data) != 0) { - vmx_inject_gp(vcpu, 0); + kvm_inject_gp(vcpu, 0); return 1; } @@ -2619,8 +2607,6 @@ static struct kvm_x86_ops vmx_x86_ops = { .tlb_flush = vmx_flush_tlb, - .inject_gp = vmx_inject_gp, - .run = vmx_vcpu_run, .handle_exit = kvm_handle_exit, .skip_emulated_instruction = skip_emulated_instruction, diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index dc007a32a883..6deb052b5f93 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -128,11 +128,6 @@ void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data) } EXPORT_SYMBOL_GPL(kvm_set_apic_base); -static void inject_gp(struct kvm_vcpu *vcpu) -{ - kvm_x86_ops->inject_gp(vcpu, 0); -} - void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) { WARN_ON(vcpu->exception.pending); @@ -232,20 +227,20 @@ void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) if (cr0 & CR0_RESERVED_BITS) { printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n", cr0, vcpu->cr0); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) { printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) { printk(KERN_DEBUG "set_cr0: #GP, set PG flag " "and a clear PE flag\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } @@ -257,14 +252,14 @@ void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) if (!is_pae(vcpu)) { printk(KERN_DEBUG "set_cr0: #GP, start paging " "in long mode while PAE is disabled\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); if (cs_l) { printk(KERN_DEBUG "set_cr0: #GP, start paging " "in long mode while CS.L == 1\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } @@ -273,7 +268,7 @@ void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) { printk(KERN_DEBUG "set_cr0: #GP, pdptrs " "reserved bits\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } @@ -299,7 +294,7 @@ void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) { if (cr4 & CR4_RESERVED_BITS) { printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } @@ -307,19 +302,19 @@ void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) if (!(cr4 & X86_CR4_PAE)) { printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while " "in long mode\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE) && !load_pdptrs(vcpu, vcpu->cr3)) { printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } if (cr4 & X86_CR4_VMXE) { printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } kvm_x86_ops->set_cr4(vcpu, cr4); @@ -340,7 +335,7 @@ void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) if (is_long_mode(vcpu)) { if (cr3 & CR3_L_MODE_RESERVED_BITS) { printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } } else { @@ -348,13 +343,13 @@ void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) if (cr3 & CR3_PAE_RESERVED_BITS) { printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) { printk(KERN_DEBUG "set_cr3: #GP, pdptrs " "reserved bits\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } } @@ -375,7 +370,7 @@ void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) * to debug) behavior on the guest side. */ if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT))) - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); else { vcpu->cr3 = cr3; vcpu->mmu.new_cr3(vcpu); @@ -388,7 +383,7 @@ void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) { if (cr8 & CR8_RESERVED_BITS) { printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } if (irqchip_in_kernel(vcpu->kvm)) @@ -436,14 +431,14 @@ static void set_efer(struct kvm_vcpu *vcpu, u64 efer) if (efer & EFER_RESERVED_BITS) { printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n", efer); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } if (is_paging(vcpu) && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) { printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } @@ -2047,7 +2042,7 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, * String I/O in reverse. Yuck. Kill the guest, fix later. */ pr_unimpl(vcpu, "guest string pio down\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return 1; } vcpu->run->io.count = now; @@ -2062,7 +2057,7 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, vcpu->pio.guest_pages[i] = page; mutex_unlock(&vcpu->kvm->lock); if (!page) { - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); free_pio_guest_pages(vcpu); return 1; } diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index d3ac4e2b3a41..78396d627be0 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -220,8 +220,6 @@ struct kvm_x86_ops { void (*tlb_flush)(struct kvm_vcpu *vcpu); - void (*inject_gp)(struct kvm_vcpu *vcpu, unsigned err_code); - void (*run)(struct kvm_vcpu *vcpu, struct kvm_run *run); int (*handle_exit)(struct kvm_run *run, struct kvm_vcpu *vcpu); void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu); @@ -467,6 +465,11 @@ static inline u32 get_rdx_init_val(void) return 0x600; /* P6 family */ } +static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code) +{ + kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); +} + #define ASM_VMX_VMCLEAR_RAX ".byte 0x66, 0x0f, 0xc7, 0x30" #define ASM_VMX_VMLAUNCH ".byte 0x0f, 0x01, 0xc2" #define ASM_VMX_VMRESUME ".byte 0x0f, 0x01, 0xc3" diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 3e3eba70d5ac..2e259a847697 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1779,7 +1779,7 @@ twobyte_insn: | ((u64)c->regs[VCPU_REGS_RDX] << 32); rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data); if (rc) { - kvm_x86_ops->inject_gp(ctxt->vcpu, 0); + kvm_inject_gp(ctxt->vcpu, 0); c->eip = ctxt->vcpu->rip; } rc = X86EMUL_CONTINUE; @@ -1789,7 +1789,7 @@ twobyte_insn: /* rdmsr */ rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data); if (rc) { - kvm_x86_ops->inject_gp(ctxt->vcpu, 0); + kvm_inject_gp(ctxt->vcpu, 0); c->eip = ctxt->vcpu->rip; } else { c->regs[VCPU_REGS_RAX] = (u32)msr_data; From 7ee5d940f5064a7a4f0e53a8ffe755bc26a8b0f1 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 25 Nov 2007 15:22:50 +0200 Subject: [PATCH 191/249] KVM: Use generalized exception queue for injecting #UD Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 12 ++---------- drivers/kvm/vmx.c | 11 +---------- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index f9769338c621..06beed7d4a08 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -207,13 +207,6 @@ static bool svm_exception_injected(struct kvm_vcpu *vcpu) return !(svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID); } -static void inject_ud(struct kvm_vcpu *vcpu) -{ - to_svm(vcpu)->vmcb->control.event_inj = SVM_EVTINJ_VALID | - SVM_EVTINJ_TYPE_EXEPT | - UD_VECTOR; -} - static int is_external_interrupt(u32 info) { info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID; @@ -948,8 +941,7 @@ static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) er = emulate_instruction(&svm->vcpu, kvm_run, 0, 0, 0); if (er != EMULATE_DONE) - inject_ud(&svm->vcpu); - + kvm_queue_exception(&svm->vcpu, UD_VECTOR); return 1; } @@ -1027,7 +1019,7 @@ static int vmmcall_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) static int invalid_op_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { - inject_ud(&svm->vcpu); + kvm_queue_exception(&svm->vcpu, UD_VECTOR); return 1; } diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 3b3c5f7d2e7c..3b44573c326e 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -613,14 +613,6 @@ static bool vmx_exception_injected(struct kvm_vcpu *vcpu) return !(vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK); } -static void vmx_inject_ud(struct kvm_vcpu *vcpu) -{ - vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, - UD_VECTOR | - INTR_TYPE_EXCEPTION | - INTR_INFO_VALID_MASK); -} - /* * Swap MSR entry in host/guest MSR entry array. */ @@ -1866,8 +1858,7 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) if (is_invalid_opcode(intr_info)) { er = emulate_instruction(vcpu, kvm_run, 0, 0, 0); if (er != EMULATE_DONE) - vmx_inject_ud(vcpu); - + kvm_queue_exception(vcpu, UD_VECTOR); return 1; } From e934c9c1c8742872a53efb84966d9c1d7b8c8e24 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 6 Dec 2007 16:15:02 +0200 Subject: [PATCH 192/249] KVM: x86 emulator: fix eflags preparation for emulation We prepare eflags for the emulated instruction, then clobber it with an 'andl'. Fix by popping eflags as the last thing in the sequence. Patch taken from Xen (16143:959b4b92b6bf) Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 2e259a847697..f423b0e327f4 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -256,21 +256,21 @@ static u16 twobyte_table[256] = { #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF) /* Before executing instruction: restore necessary bits in EFLAGS. */ -#define _PRE_EFLAGS(_sav, _msk, _tmp) \ - /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); */ \ - "push %"_sav"; " \ - "movl %"_msk",%"_LO32 _tmp"; " \ - "andl %"_LO32 _tmp",("_STK"); " \ - "pushf; " \ - "notl %"_LO32 _tmp"; " \ - "andl %"_LO32 _tmp",("_STK"); " \ - "pop %"_tmp"; " \ - "orl %"_LO32 _tmp",("_STK"); " \ - "popf; " \ - /* _sav &= ~msk; */ \ - "movl %"_msk",%"_LO32 _tmp"; " \ - "notl %"_LO32 _tmp"; " \ - "andl %"_LO32 _tmp",%"_sav"; " +#define _PRE_EFLAGS(_sav, _msk, _tmp) \ + /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \ + "movl %"_sav",%"_LO32 _tmp"; " \ + "push %"_tmp"; " \ + "push %"_tmp"; " \ + "movl %"_msk",%"_LO32 _tmp"; " \ + "andl %"_LO32 _tmp",("_STK"); " \ + "pushf; " \ + "notl %"_LO32 _tmp"; " \ + "andl %"_LO32 _tmp",("_STK"); " \ + "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \ + "pop %"_tmp"; " \ + "orl %"_LO32 _tmp",("_STK"); " \ + "popf; " \ + "pop %"_sav"; " /* After executing instruction: write-back necessary bits in EFLAGS. */ #define _POST_EFLAGS(_sav, _msk, _tmp) \ From e5314067f6a77688a3d36548e7618430ce4a6236 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 6 Dec 2007 16:32:45 +0200 Subject: [PATCH 193/249] KVM: VMX: Avoid exit when setting cr8 if the local apic is in the kernel With apic in userspace, we must exit to userspace after a cr8 write in order to update the tpr. But if the apic is in the kernel, the exit is unnecessary. Noticed by Joerg Roedel. Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 3b44573c326e..83084348581a 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1973,6 +1973,8 @@ static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) vcpu_load_rsp_rip(vcpu); set_cr8(vcpu, vcpu->regs[reg]); skip_emulated_instruction(vcpu); + if (irqchip_in_kernel(vcpu->kvm)) + return 1; kvm_run->exit_reason = KVM_EXIT_SET_TPR; return 0; }; From 152ff9be2ed11e76ed3014b94eacdd6bc9275518 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 6 Dec 2007 15:46:52 +0100 Subject: [PATCH 194/249] KVM: SVM: Emulate read/write access to cr8 This patch adds code to emulate the access to the cr8 register to the x86 instruction emulator in kvm. This is needed on svm, where there is no hardware decode for control register access. Signed-off-by: Joerg Roedel Signed-off-by: Markus Rechberger Signed-off-by: Avi Kivity --- drivers/kvm/x86.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 6deb052b5f93..9db4e3242b62 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -2236,6 +2236,8 @@ unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr) return vcpu->cr3; case 4: return vcpu->cr4; + case 8: + return get_cr8(vcpu); default: vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr); return 0; @@ -2259,6 +2261,9 @@ void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, case 4: set_cr4(vcpu, mk_cr_64(vcpu->cr4, val)); break; + case 8: + set_cr8(vcpu, val & 0xfUL); + break; default: vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr); } From 6e3d5dfbad4d8c29bdf8ed160f91f955f0efc12b Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 6 Dec 2007 18:14:14 +0200 Subject: [PATCH 195/249] KVM: x86 emulator: Fix stack instructions on 64-bit mode Stack instructions are always 64-bit on 64-bit mode; many of the emulated stack instructions did not take that into account. Fix by adding a 'Stack' bitflag and setting the operand size appropriately during the decode stage (except for 'push r/m', which is in a group with a few other instructions, so it gets its own treatment). This fixes random crashes on Vista x64. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index f423b0e327f4..0a6ab06fde01 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -65,6 +65,7 @@ #define BitOp (1<<8) #define MemAbs (1<<9) /* Memory operand is absolute displacement */ #define String (1<<10) /* String instruction (rep capable) */ +#define Stack (1<<11) /* Stack instruction (push/pop) */ static u16 opcode_table[256] = { /* 0x00 - 0x07 */ @@ -104,14 +105,16 @@ static u16 opcode_table[256] = { /* 0x48 - 0x4F */ DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, /* 0x50 - 0x57 */ - SrcReg, SrcReg, SrcReg, SrcReg, SrcReg, SrcReg, SrcReg, SrcReg, + SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, + SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, /* 0x58 - 0x5F */ - DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, + DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack, + DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack, /* 0x60 - 0x67 */ 0, 0, 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ , 0, 0, 0, 0, /* 0x68 - 0x6F */ - 0, 0, ImplicitOps|Mov, 0, + 0, 0, ImplicitOps | Mov | Stack, 0, SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* insb, insw/insd */ SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* outsb, outsw/outsd */ /* 0x70 - 0x77 */ @@ -128,9 +131,10 @@ static u16 opcode_table[256] = { /* 0x88 - 0x8F */ ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov, ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, - 0, ModRM | DstReg, 0, DstMem | SrcNone | ModRM | Mov, + 0, ModRM | DstReg, 0, DstMem | SrcNone | ModRM | Mov | Stack, /* 0x90 - 0x9F */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps, ImplicitOps, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, ImplicitOps | Stack, ImplicitOps | Stack, 0, 0, /* 0xA0 - 0xA7 */ ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs, ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs, @@ -144,7 +148,7 @@ static u16 opcode_table[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0xC0 - 0xC7 */ ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM, - 0, ImplicitOps, 0, 0, + 0, ImplicitOps | Stack, 0, 0, ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov, /* 0xC8 - 0xCF */ 0, 0, 0, 0, 0, 0, 0, 0, @@ -157,7 +161,8 @@ static u16 opcode_table[256] = { /* 0xE0 - 0xE7 */ 0, 0, 0, 0, 0, 0, 0, 0, /* 0xE8 - 0xEF */ - ImplicitOps, SrcImm|ImplicitOps, 0, SrcImmByte|ImplicitOps, 0, 0, 0, 0, + ImplicitOps | Stack, SrcImm|ImplicitOps, 0, SrcImmByte|ImplicitOps, + 0, 0, 0, 0, /* 0xF0 - 0xF7 */ 0, 0, 0, 0, ImplicitOps, ImplicitOps, @@ -868,6 +873,9 @@ done_prefixes: } } + if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack)) + c->op_bytes = 8; + /* ModRM and SIB bytes. */ if (c->d & ModRM) rc = decode_modrm(ctxt, ops); @@ -988,11 +996,6 @@ static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt, struct decode_cache *c = &ctxt->decode; int rc; - /* 64-bit mode: POP always pops a 64-bit operand. */ - - if (ctxt->mode == X86EMUL_MODE_PROT64) - c->dst.bytes = 8; - rc = ops->read_std(register_address(ctxt->ss_base, c->regs[VCPU_REGS_RSP]), &c->dst.val, c->dst.bytes, ctxt->vcpu); From 80a8119ca3f021037b8513d39dbb0ffd1af86b20 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 6 Dec 2007 19:50:00 +0200 Subject: [PATCH 196/249] KVM: SVM: Trap access to the cr8 register Later we may be able to use the virtual tpr feature, but for now, just trap it. Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 8 ++++++-- drivers/kvm/svm.h | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 06beed7d4a08..143d271d4431 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -449,11 +449,13 @@ static void init_vmcb(struct vmcb *vmcb) control->intercept_cr_read = INTERCEPT_CR0_MASK | INTERCEPT_CR3_MASK | - INTERCEPT_CR4_MASK; + INTERCEPT_CR4_MASK | + INTERCEPT_CR8_MASK; control->intercept_cr_write = INTERCEPT_CR0_MASK | INTERCEPT_CR3_MASK | - INTERCEPT_CR4_MASK; + INTERCEPT_CR4_MASK | + INTERCEPT_CR8_MASK; control->intercept_dr_read = INTERCEPT_DR0_MASK | INTERCEPT_DR1_MASK | @@ -1195,10 +1197,12 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm, [SVM_EXIT_READ_CR0] = emulate_on_interception, [SVM_EXIT_READ_CR3] = emulate_on_interception, [SVM_EXIT_READ_CR4] = emulate_on_interception, + [SVM_EXIT_READ_CR8] = emulate_on_interception, /* for now: */ [SVM_EXIT_WRITE_CR0] = emulate_on_interception, [SVM_EXIT_WRITE_CR3] = emulate_on_interception, [SVM_EXIT_WRITE_CR4] = emulate_on_interception, + [SVM_EXIT_WRITE_CR8] = emulate_on_interception, [SVM_EXIT_READ_DR0] = emulate_on_interception, [SVM_EXIT_READ_DR1] = emulate_on_interception, [SVM_EXIT_READ_DR2] = emulate_on_interception, diff --git a/drivers/kvm/svm.h b/drivers/kvm/svm.h index 5fa277c0187c..5fd50491b555 100644 --- a/drivers/kvm/svm.h +++ b/drivers/kvm/svm.h @@ -204,6 +204,7 @@ struct __attribute__ ((__packed__)) vmcb { #define INTERCEPT_CR0_MASK 1 #define INTERCEPT_CR3_MASK (1 << 3) #define INTERCEPT_CR4_MASK (1 << 4) +#define INTERCEPT_CR8_MASK (1 << 8) #define INTERCEPT_DR0_MASK 1 #define INTERCEPT_DR1_MASK (1 << 1) From b3e4e63fd9ee8cd5d9047b89e0d463d5c48ee5b5 Mon Sep 17 00:00:00 2001 From: Marcelo Tosatti Date: Fri, 7 Dec 2007 07:56:58 -0500 Subject: [PATCH 197/249] KVM: MMU: Use cmpxchg for pte updates on walk_addr() In preparation for multi-threaded guest pte walking, use cmpxchg() when updating guest pte's. This guarantees that the assignment of the dirty bit can't be lost if two CPU's are faulting the same address simultaneously. [avi: fix kunmap_atomic() parameters] Signed-off-by: Marcelo Tosatti Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 3c40a598ceb8..8086f827120f 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -34,7 +34,9 @@ #define PT_LEVEL_BITS PT64_LEVEL_BITS #ifdef CONFIG_X86_64 #define PT_MAX_FULL_LEVELS 4 + #define CMPXCHG cmpxchg #else + #define CMPXCHG cmpxchg64 #define PT_MAX_FULL_LEVELS 2 #endif #elif PTTYPE == 32 @@ -48,6 +50,7 @@ #define PT_LEVEL_MASK(level) PT32_LEVEL_MASK(level) #define PT_LEVEL_BITS PT32_LEVEL_BITS #define PT_MAX_FULL_LEVELS 2 + #define CMPXCHG cmpxchg #else #error Invalid PTTYPE value #endif @@ -78,6 +81,26 @@ static gfn_t gpte_to_gfn_pde(pt_element_t gpte) return (gpte & PT_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT; } +static bool FNAME(cmpxchg_gpte)(struct kvm *kvm, + gfn_t table_gfn, unsigned index, + pt_element_t orig_pte, pt_element_t new_pte) +{ + pt_element_t ret; + pt_element_t *table; + struct page *page; + + page = gfn_to_page(kvm, table_gfn); + table = kmap_atomic(page, KM_USER0); + + ret = CMPXCHG(&table[index], orig_pte, new_pte); + + kunmap_atomic(table, KM_USER0); + + kvm_release_page_dirty(page); + + return (ret != orig_pte); +} + /* * Fetch a guest pte for a guest virtual address */ @@ -91,6 +114,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, gpa_t pte_gpa; pgprintk("%s: addr %lx\n", __FUNCTION__, addr); +walk: walker->level = vcpu->mmu.root_level; pte = vcpu->cr3; #if PTTYPE == 64 @@ -135,8 +159,10 @@ static int FNAME(walk_addr)(struct guest_walker *walker, if (!(pte & PT_ACCESSED_MASK)) { mark_page_dirty(vcpu->kvm, table_gfn); + if (FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn, + index, pte, pte|PT_ACCESSED_MASK)) + goto walk; pte |= PT_ACCESSED_MASK; - kvm_write_guest(vcpu->kvm, pte_gpa, &pte, sizeof(pte)); } if (walker->level == PT_PAGE_TABLE_LEVEL) { @@ -159,9 +185,14 @@ static int FNAME(walk_addr)(struct guest_walker *walker, } if (write_fault && !is_dirty_pte(pte)) { + bool ret; + mark_page_dirty(vcpu->kvm, table_gfn); + ret = FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn, index, pte, + pte|PT_DIRTY_MASK); + if (ret) + goto walk; pte |= PT_DIRTY_MASK; - kvm_write_guest(vcpu->kvm, pte_gpa, &pte, sizeof(pte)); kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte)); } @@ -484,3 +515,4 @@ static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, #undef PT_MAX_FULL_LEVELS #undef gpte_to_gfn #undef gpte_to_gfn_pde +#undef CMPXCHG From fe135d2ceb3d7dc08151b3cbad96565d02cc8676 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 16:15:46 +0200 Subject: [PATCH 198/249] KVM: MMU: Simplify calculation of pte access The nx bit is awkwardly placed in the 63rd bit position; furthermore it has a reversed meaning compared to the other bits, which means we can't use a bitwise and to calculate compounded access masks. So, we simplify things by creating a new 3-bit exec/write/user access word, and doing all calculations in that. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 14 +++++++--- drivers/kvm/paging_tmpl.h | 58 ++++++++++++++++++++++----------------- 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 62a74151f847..f8a2137c64a2 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -85,7 +85,8 @@ static int dbg = 1; #define PT_PAGE_SIZE_MASK (1ULL << 7) #define PT_PAT_MASK (1ULL << 7) #define PT_GLOBAL_MASK (1ULL << 8) -#define PT64_NX_MASK (1ULL << 63) +#define PT64_NX_SHIFT 63 +#define PT64_NX_MASK (1ULL << PT64_NX_SHIFT) #define PT_PAT_SHIFT 7 #define PT_DIR_PAT_SHIFT 12 @@ -153,6 +154,11 @@ static int dbg = 1; #define RMAP_EXT 4 +#define ACC_EXEC_MASK 1 +#define ACC_WRITE_MASK PT_WRITABLE_MASK +#define ACC_USER_MASK PT_USER_MASK +#define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK) + struct kvm_rmap_desc { u64 *shadow_ptes[RMAP_EXT]; struct kvm_rmap_desc *more; @@ -921,7 +927,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, struct page *page) >> PAGE_SHIFT; new_table = kvm_mmu_get_page(vcpu, pseudo_gfn, v, level - 1, - 1, 3, &table[index]); + 1, ACC_ALL, &table[index]); if (!new_table) { pgprintk("nonpaging_map: ENOMEM\n"); kvm_release_page_clean(page); @@ -988,7 +994,7 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) ASSERT(!VALID_PAGE(root)); sp = kvm_mmu_get_page(vcpu, root_gfn, 0, - PT64_ROOT_LEVEL, 0, 0, NULL); + PT64_ROOT_LEVEL, 0, ACC_ALL, NULL); root = __pa(sp->spt); ++sp->root_count; vcpu->mmu.root_hpa = root; @@ -1009,7 +1015,7 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) root_gfn = 0; sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, PT32_ROOT_LEVEL, !is_paging(vcpu), - 0, NULL); + ACC_ALL, NULL); root = __pa(sp->spt); ++sp->root_count; vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK; diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 8086f827120f..7688cbf413c8 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -66,7 +66,8 @@ struct guest_walker { int level; gfn_t table_gfn[PT_MAX_FULL_LEVELS]; pt_element_t pte; - pt_element_t inherited_ar; + unsigned pt_access; + unsigned pte_access; gfn_t gfn; u32 error_code; }; @@ -110,7 +111,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, { pt_element_t pte; gfn_t table_gfn; - unsigned index; + unsigned index, pt_access, pte_access; gpa_t pte_gpa; pgprintk("%s: addr %lx\n", __FUNCTION__, addr); @@ -128,7 +129,7 @@ walk: ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) || (vcpu->cr3 & CR3_NONPAE_RESERVED_BITS) == 0); - walker->inherited_ar = PT_USER_MASK | PT_WRITABLE_MASK; + pt_access = ACC_ALL; for (;;) { index = PT_INDEX(addr, walker->level); @@ -165,6 +166,14 @@ walk: pte |= PT_ACCESSED_MASK; } + pte_access = pte & (PT_WRITABLE_MASK | PT_USER_MASK); + pte_access |= ACC_EXEC_MASK; +#if PTTYPE == 64 + if (is_nx(vcpu)) + pte_access &= ~(pte >> PT64_NX_SHIFT); +#endif + pte_access &= pt_access; + if (walker->level == PT_PAGE_TABLE_LEVEL) { walker->gfn = gpte_to_gfn(pte); break; @@ -180,7 +189,7 @@ walk: break; } - walker->inherited_ar &= pte; + pt_access = pte_access; --walker->level; } @@ -197,7 +206,10 @@ walk: } walker->pte = pte; - pgprintk("%s: pte %llx\n", __FUNCTION__, (u64)pte); + walker->pt_access = pt_access; + walker->pte_access = pte_access; + pgprintk("%s: pte %llx pte_access %x pt_access %x\n", + __FUNCTION__, (u64)pte, pt_access, pte_access); return 1; not_present: @@ -218,7 +230,8 @@ err: } static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, - u64 *shadow_pte, u64 access_bits, + u64 *shadow_pte, unsigned pt_access, + unsigned pte_access, int user_fault, int write_fault, int *ptwrite, struct guest_walker *walker, gfn_t gfn) @@ -228,12 +241,11 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, int was_rmapped = is_rmap_pte(*shadow_pte); struct page *page; - pgprintk("%s: spte %llx gpte %llx access %llx write_fault %d" + pgprintk("%s: spte %llx gpte %llx access %x write_fault %d" " user_fault %d gfn %lx\n", - __FUNCTION__, *shadow_pte, (u64)gpte, access_bits, + __FUNCTION__, *shadow_pte, (u64)gpte, pt_access, write_fault, user_fault, gfn); - access_bits &= gpte; /* * We don't set the accessed bit, since we sometimes want to see * whether the guest actually used the pte (in order to detect @@ -242,12 +254,12 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, spte = PT_PRESENT_MASK | PT_DIRTY_MASK; spte |= gpte & PT64_NX_MASK; if (!dirty) - access_bits &= ~PT_WRITABLE_MASK; + pte_access &= ~ACC_WRITE_MASK; page = gfn_to_page(vcpu->kvm, gfn); spte |= PT_PRESENT_MASK; - if (access_bits & PT_USER_MASK) + if (pte_access & ACC_USER_MASK) spte |= PT_USER_MASK; if (is_error_page(page)) { @@ -259,7 +271,7 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, spte |= page_to_phys(page); - if ((access_bits & PT_WRITABLE_MASK) + if ((pte_access & ACC_WRITE_MASK) || (write_fault && !is_write_protection(vcpu) && !user_fault)) { struct kvm_mmu_page *shadow; @@ -273,7 +285,7 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, if (shadow) { pgprintk("%s: found shadow page for %lx, marking ro\n", __FUNCTION__, gfn); - access_bits &= ~PT_WRITABLE_MASK; + pte_access &= ~ACC_WRITE_MASK; if (is_writeble_pte(spte)) { spte &= ~PT_WRITABLE_MASK; kvm_x86_ops->tlb_flush(vcpu); @@ -285,7 +297,7 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, unshadowed: - if (access_bits & PT_WRITABLE_MASK) + if (pte_access & ACC_WRITE_MASK) mark_page_dirty(vcpu->kvm, gfn); pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte); @@ -317,8 +329,8 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, if (bytes < sizeof(pt_element_t)) return; pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); - FNAME(set_pte)(vcpu, gpte, spte, PT_USER_MASK | PT_WRITABLE_MASK, 0, - 0, NULL, NULL, gpte_to_gfn(gpte)); + FNAME(set_pte)(vcpu, gpte, spte, ACC_ALL, ACC_ALL, + 0, 0, NULL, NULL, gpte_to_gfn(gpte)); } /* @@ -331,6 +343,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, hpa_t shadow_addr; int level; u64 *shadow_ent; + unsigned access = walker->pt_access; if (!is_present_pte(walker->pte)) return NULL; @@ -349,7 +362,6 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, u64 shadow_pte; int metaphysical; gfn_t table_gfn; - unsigned hugepage_access = 0; shadow_ent = ((u64 *)__va(shadow_addr)) + index; if (is_shadow_present_pte(*shadow_ent)) { @@ -365,20 +377,15 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, if (level - 1 == PT_PAGE_TABLE_LEVEL && walker->level == PT_DIRECTORY_LEVEL) { metaphysical = 1; - hugepage_access = walker->pte; - hugepage_access &= PT_USER_MASK | PT_WRITABLE_MASK; if (!is_dirty_pte(walker->pte)) - hugepage_access &= ~PT_WRITABLE_MASK; - hugepage_access >>= PT_WRITABLE_SHIFT; - if (walker->pte & PT64_NX_MASK) - hugepage_access |= (1 << 2); + access &= ~ACC_WRITE_MASK; table_gfn = gpte_to_gfn(walker->pte); } else { metaphysical = 0; table_gfn = walker->table_gfn[level - 2]; } shadow_page = kvm_mmu_get_page(vcpu, table_gfn, addr, level-1, - metaphysical, hugepage_access, + metaphysical, access, shadow_ent); shadow_addr = __pa(shadow_page->spt); shadow_pte = shadow_addr | PT_PRESENT_MASK | PT_ACCESSED_MASK @@ -387,7 +394,8 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, } FNAME(set_pte)(vcpu, walker->pte, shadow_ent, - walker->inherited_ar, user_fault, write_fault, + access, walker->pte_access & access, + user_fault, write_fault, ptwrite, walker, walker->gfn); return shadow_ent; From 8d87a03aea43535a92efbb180e0ceac94d4bb5db Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 16:37:36 +0200 Subject: [PATCH 199/249] KVM: MMU: Set nx bit correctly on shadow ptes While the page table walker correctly generates a guest page fault if a guest tries to execute a non-executable page, the shadow code does not mark it non-executable. This means that if a guest accesses an nx page first with a read access, then subsequent code fetch accesses will succeed. Fix by setting the nx bit on shadow ptes. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 7688cbf413c8..59ba752a6880 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -255,6 +255,8 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, spte |= gpte & PT64_NX_MASK; if (!dirty) pte_access &= ~ACC_WRITE_MASK; + if (!(pte_access & ACC_EXEC_MASK)) + spte |= PT64_NX_MASK; page = gfn_to_page(vcpu->kvm, gfn); From bedbe4ee86195dcd899577828714cc1413beb571 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 16:52:56 +0200 Subject: [PATCH 200/249] KVM: MMU: Move pte access calculation into a helper function Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 59ba752a6880..1fc4f9b321ee 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -102,6 +102,18 @@ static bool FNAME(cmpxchg_gpte)(struct kvm *kvm, return (ret != orig_pte); } +static unsigned FNAME(gpte_access)(struct kvm_vcpu *vcpu, pt_element_t gpte) +{ + unsigned access; + + access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK; +#if PTTYPE == 64 + if (is_nx(vcpu)) + access &= ~(gpte >> PT64_NX_SHIFT); +#endif + return access; +} + /* * Fetch a guest pte for a guest virtual address */ @@ -166,13 +178,7 @@ walk: pte |= PT_ACCESSED_MASK; } - pte_access = pte & (PT_WRITABLE_MASK | PT_USER_MASK); - pte_access |= ACC_EXEC_MASK; -#if PTTYPE == 64 - if (is_nx(vcpu)) - pte_access &= ~(pte >> PT64_NX_SHIFT); -#endif - pte_access &= pt_access; + pte_access = pt_access & FNAME(gpte_access)(vcpu, pte); if (walker->level == PT_PAGE_TABLE_LEVEL) { walker->gfn = gpte_to_gfn(pte); From 41074d07c78b086b18fc2af590caef05dbcca568 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 17:00:02 +0200 Subject: [PATCH 201/249] KVM: MMU: Fix inherited permissions for emulated guest pte updates When we emulate a guest pte write, we fail to apply the correct inherited permissions from the parent ptes. Now that we store inherited permissions in the shadow page, we can use that to update the pte permissions correctly. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 4 ++-- drivers/kvm/mmu.c | 4 ++-- drivers/kvm/paging_tmpl.h | 4 +++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 0d3555bf5333..ceefb4427dbd 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -55,7 +55,7 @@ struct kvm_pte_chain { * bits 4:7 - page table level for this shadow (1-4) * bits 8:9 - page table quadrant for 2-level guests * bit 16 - "metaphysical" - gfn is not a real page (huge page/real mode) - * bits 17:19 - "access" - the user, writable, and nx bits of a huge page pde + * bits 17:19 - common access permissions for all ptes in this shadow page */ union kvm_mmu_page_role { unsigned word; @@ -65,7 +65,7 @@ union kvm_mmu_page_role { unsigned quadrant : 2; unsigned pad_for_nice_hex_output : 6; unsigned metaphysical : 1; - unsigned hugepage_access : 3; + unsigned access : 3; }; }; diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index f8a2137c64a2..cace1e41b683 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -680,7 +680,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, gva_t gaddr, unsigned level, int metaphysical, - unsigned hugepage_access, + unsigned access, u64 *parent_pte) { union kvm_mmu_page_role role; @@ -694,7 +694,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, role.glevels = vcpu->mmu.root_level; role.level = level; role.metaphysical = metaphysical; - role.hugepage_access = hugepage_access; + role.access = access; if (vcpu->mmu.root_level <= PT32_ROOT_LEVEL) { quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level)); quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1; diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 1fc4f9b321ee..211fef83be5d 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -327,6 +327,7 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, int offset_in_pte) { pt_element_t gpte; + unsigned pte_access; gpte = *(const pt_element_t *)pte; if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) { @@ -337,7 +338,8 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, if (bytes < sizeof(pt_element_t)) return; pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); - FNAME(set_pte)(vcpu, gpte, spte, ACC_ALL, ACC_ALL, + pte_access = page->role.access & FNAME(gpte_access)(vcpu, gpte); + FNAME(set_pte)(vcpu, gpte, spte, page->role.access, pte_access, 0, 0, NULL, NULL, gpte_to_gfn(gpte)); } From b4ab019ce71efe30790b5aaa396549a16bb20608 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 17:27:52 +0200 Subject: [PATCH 202/249] KVM: MMU: No need to pick up nx bit from guest pte We already set it according to cumulative access permissions. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 211fef83be5d..179ce9e8145e 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -258,7 +258,6 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, * demand paging). */ spte = PT_PRESENT_MASK | PT_DIRTY_MASK; - spte |= gpte & PT64_NX_MASK; if (!dirty) pte_access &= ~ACC_WRITE_MASK; if (!(pte_access & ACC_EXEC_MASK)) From e3f955042219a595b1c3208e12c20860f23f72fb Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 17:32:30 +0200 Subject: [PATCH 203/249] KVM: MMU: Pass pte dirty flag to set_pte() instead of calculating it on-site This allows us to remove its dependency on pt_element_t. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 179ce9e8145e..ca8d51574118 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -235,14 +235,12 @@ err: return 0; } -static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, - u64 *shadow_pte, unsigned pt_access, - unsigned pte_access, - int user_fault, int write_fault, +static void FNAME(set_pte)(struct kvm_vcpu *vcpu, u64 *shadow_pte, + unsigned pt_access, unsigned pte_access, + int user_fault, int write_fault, int dirty, int *ptwrite, struct guest_walker *walker, gfn_t gfn) { - int dirty = gpte & PT_DIRTY_MASK; u64 spte; int was_rmapped = is_rmap_pte(*shadow_pte); struct page *page; @@ -338,8 +336,8 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, return; pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); pte_access = page->role.access & FNAME(gpte_access)(vcpu, gpte); - FNAME(set_pte)(vcpu, gpte, spte, page->role.access, pte_access, - 0, 0, NULL, NULL, gpte_to_gfn(gpte)); + FNAME(set_pte)(vcpu, spte, page->role.access, pte_access, 0, 0, + gpte & PT_DIRTY_MASK, NULL, NULL, gpte_to_gfn(gpte)); } /* @@ -402,9 +400,8 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, *shadow_ent = shadow_pte; } - FNAME(set_pte)(vcpu, walker->pte, shadow_ent, - access, walker->pte_access & access, - user_fault, write_fault, + FNAME(set_pte)(vcpu, shadow_ent, access, walker->pte_access & access, + user_fault, write_fault, walker->pte & PT_DIRTY_MASK, ptwrite, walker, walker->gfn); return shadow_ent; From 2fbf4cf13f777e1f61ee692fe67d16bddd747700 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 17:33:46 +0200 Subject: [PATCH 204/249] KVM: MMU: Remove walker argument to set_pte() Unused. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index ca8d51574118..2e5a80af22c9 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -238,8 +238,7 @@ err: static void FNAME(set_pte)(struct kvm_vcpu *vcpu, u64 *shadow_pte, unsigned pt_access, unsigned pte_access, int user_fault, int write_fault, int dirty, - int *ptwrite, struct guest_walker *walker, - gfn_t gfn) + int *ptwrite, gfn_t gfn) { u64 spte; int was_rmapped = is_rmap_pte(*shadow_pte); @@ -337,7 +336,7 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); pte_access = page->role.access & FNAME(gpte_access)(vcpu, gpte); FNAME(set_pte)(vcpu, spte, page->role.access, pte_access, 0, 0, - gpte & PT_DIRTY_MASK, NULL, NULL, gpte_to_gfn(gpte)); + gpte & PT_DIRTY_MASK, NULL, gpte_to_gfn(gpte)); } /* @@ -402,7 +401,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, FNAME(set_pte)(vcpu, shadow_ent, access, walker->pte_access & access, user_fault, write_fault, walker->pte & PT_DIRTY_MASK, - ptwrite, walker, walker->gfn); + ptwrite, walker->gfn); return shadow_ent; } From 1c4f1fd6d5692614e8dc75ee53f7be590f1e878b Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 17:40:31 +0200 Subject: [PATCH 205/249] KVM: MMU: Move set_pte() into guest paging mode independent code As set_pte() no longer references either a gpte or the guest walker, we can move it out of paging mode dependent code (which compiles twice and is generally nasty). Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 83 ++++++++++++++++++++++++++++++++++ drivers/kvm/paging_tmpl.h | 93 +++------------------------------------ 2 files changed, 88 insertions(+), 88 deletions(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index cace1e41b683..a91e05b42345 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -879,6 +879,89 @@ struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva) return gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); } +static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte, + unsigned pt_access, unsigned pte_access, + int user_fault, int write_fault, int dirty, + int *ptwrite, gfn_t gfn) +{ + u64 spte; + int was_rmapped = is_rmap_pte(*shadow_pte); + struct page *page; + + pgprintk("%s: spte %llx gpte %llx access %x write_fault %d" + " user_fault %d gfn %lx\n", + __FUNCTION__, *shadow_pte, (u64)gpte, pt_access, + write_fault, user_fault, gfn); + + /* + * We don't set the accessed bit, since we sometimes want to see + * whether the guest actually used the pte (in order to detect + * demand paging). + */ + spte = PT_PRESENT_MASK | PT_DIRTY_MASK; + if (!dirty) + pte_access &= ~ACC_WRITE_MASK; + if (!(pte_access & ACC_EXEC_MASK)) + spte |= PT64_NX_MASK; + + page = gfn_to_page(vcpu->kvm, gfn); + + spte |= PT_PRESENT_MASK; + if (pte_access & ACC_USER_MASK) + spte |= PT_USER_MASK; + + if (is_error_page(page)) { + set_shadow_pte(shadow_pte, + shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK); + kvm_release_page_clean(page); + return; + } + + spte |= page_to_phys(page); + + if ((pte_access & ACC_WRITE_MASK) + || (write_fault && !is_write_protection(vcpu) && !user_fault)) { + struct kvm_mmu_page *shadow; + + spte |= PT_WRITABLE_MASK; + if (user_fault) { + mmu_unshadow(vcpu->kvm, gfn); + goto unshadowed; + } + + shadow = kvm_mmu_lookup_page(vcpu->kvm, gfn); + if (shadow) { + pgprintk("%s: found shadow page for %lx, marking ro\n", + __FUNCTION__, gfn); + pte_access &= ~ACC_WRITE_MASK; + if (is_writeble_pte(spte)) { + spte &= ~PT_WRITABLE_MASK; + kvm_x86_ops->tlb_flush(vcpu); + } + if (write_fault) + *ptwrite = 1; + } + } + +unshadowed: + + if (pte_access & ACC_WRITE_MASK) + mark_page_dirty(vcpu->kvm, gfn); + + pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte); + set_shadow_pte(shadow_pte, spte); + page_header_update_slot(vcpu->kvm, shadow_pte, gfn); + if (!was_rmapped) { + rmap_add(vcpu, shadow_pte, gfn); + if (!is_rmap_pte(*shadow_pte)) + kvm_release_page_clean(page); + } + else + kvm_release_page_clean(page); + if (!ptwrite || !*ptwrite) + vcpu->last_pte_updated = shadow_pte; +} + static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) { } diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 2e5a80af22c9..3ab3fb635e16 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -235,89 +235,6 @@ err: return 0; } -static void FNAME(set_pte)(struct kvm_vcpu *vcpu, u64 *shadow_pte, - unsigned pt_access, unsigned pte_access, - int user_fault, int write_fault, int dirty, - int *ptwrite, gfn_t gfn) -{ - u64 spte; - int was_rmapped = is_rmap_pte(*shadow_pte); - struct page *page; - - pgprintk("%s: spte %llx gpte %llx access %x write_fault %d" - " user_fault %d gfn %lx\n", - __FUNCTION__, *shadow_pte, (u64)gpte, pt_access, - write_fault, user_fault, gfn); - - /* - * We don't set the accessed bit, since we sometimes want to see - * whether the guest actually used the pte (in order to detect - * demand paging). - */ - spte = PT_PRESENT_MASK | PT_DIRTY_MASK; - if (!dirty) - pte_access &= ~ACC_WRITE_MASK; - if (!(pte_access & ACC_EXEC_MASK)) - spte |= PT64_NX_MASK; - - page = gfn_to_page(vcpu->kvm, gfn); - - spte |= PT_PRESENT_MASK; - if (pte_access & ACC_USER_MASK) - spte |= PT_USER_MASK; - - if (is_error_page(page)) { - set_shadow_pte(shadow_pte, - shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK); - kvm_release_page_clean(page); - return; - } - - spte |= page_to_phys(page); - - if ((pte_access & ACC_WRITE_MASK) - || (write_fault && !is_write_protection(vcpu) && !user_fault)) { - struct kvm_mmu_page *shadow; - - spte |= PT_WRITABLE_MASK; - if (user_fault) { - mmu_unshadow(vcpu->kvm, gfn); - goto unshadowed; - } - - shadow = kvm_mmu_lookup_page(vcpu->kvm, gfn); - if (shadow) { - pgprintk("%s: found shadow page for %lx, marking ro\n", - __FUNCTION__, gfn); - pte_access &= ~ACC_WRITE_MASK; - if (is_writeble_pte(spte)) { - spte &= ~PT_WRITABLE_MASK; - kvm_x86_ops->tlb_flush(vcpu); - } - if (write_fault) - *ptwrite = 1; - } - } - -unshadowed: - - if (pte_access & ACC_WRITE_MASK) - mark_page_dirty(vcpu->kvm, gfn); - - pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte); - set_shadow_pte(shadow_pte, spte); - page_header_update_slot(vcpu->kvm, shadow_pte, gfn); - if (!was_rmapped) { - rmap_add(vcpu, shadow_pte, gfn); - if (!is_rmap_pte(*shadow_pte)) - kvm_release_page_clean(page); - } - else - kvm_release_page_clean(page); - if (!ptwrite || !*ptwrite) - vcpu->last_pte_updated = shadow_pte; -} - static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, u64 *spte, const void *pte, int bytes, int offset_in_pte) @@ -335,8 +252,8 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, return; pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); pte_access = page->role.access & FNAME(gpte_access)(vcpu, gpte); - FNAME(set_pte)(vcpu, spte, page->role.access, pte_access, 0, 0, - gpte & PT_DIRTY_MASK, NULL, gpte_to_gfn(gpte)); + mmu_set_spte(vcpu, spte, page->role.access, pte_access, 0, 0, + gpte & PT_DIRTY_MASK, NULL, gpte_to_gfn(gpte)); } /* @@ -399,9 +316,9 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, *shadow_ent = shadow_pte; } - FNAME(set_pte)(vcpu, shadow_ent, access, walker->pte_access & access, - user_fault, write_fault, walker->pte & PT_DIRTY_MASK, - ptwrite, walker->gfn); + mmu_set_spte(vcpu, shadow_ent, access, walker->pte_access & access, + user_fault, write_fault, walker->pte & PT_DIRTY_MASK, + ptwrite, walker->gfn); return shadow_ent; } From bc750ba860d978fcaac1e0db28774b1f38ae8193 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 18:39:41 +0200 Subject: [PATCH 206/249] KVM: MMU: Adjust mmu_set_spte() debug code for gpte removal Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index a91e05b42345..b4dd72645853 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -888,9 +888,9 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte, int was_rmapped = is_rmap_pte(*shadow_pte); struct page *page; - pgprintk("%s: spte %llx gpte %llx access %x write_fault %d" + pgprintk("%s: spte %llx access %x write_fault %d" " user_fault %d gfn %lx\n", - __FUNCTION__, *shadow_pte, (u64)gpte, pt_access, + __FUNCTION__, *shadow_pte, pt_access, write_fault, user_fault, gfn); /* From e833240f3c1b0b415efb14eaa102718769d5f063 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 18:43:00 +0200 Subject: [PATCH 207/249] KVM: MMU: Use mmu_set_spte() for real-mode shadows In addition to removing some duplicated code, this also handles the unlikely case of real-mode code updating a guest page table. This can happen when one vcpu (in real mode) touches a second vcpu's (in protected mode) page tables, or if a vcpu switches to real mode, touches page tables, and switches back. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 41 ++++++++++------------------------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index b4dd72645853..ba71e8d66761 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -966,40 +966,23 @@ static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) { } -static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, struct page *page) +static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) { int level = PT32E_ROOT_LEVEL; hpa_t table_addr = vcpu->mmu.root_hpa; + int pt_write = 0; for (; ; level--) { u32 index = PT64_INDEX(v, level); u64 *table; - u64 pte; ASSERT(VALID_PAGE(table_addr)); table = __va(table_addr); if (level == 1) { - int was_rmapped; - - pte = table[index]; - was_rmapped = is_rmap_pte(pte); - if (is_shadow_present_pte(pte) && is_writeble_pte(pte)) { - kvm_release_page_clean(page); - return 0; - } - mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT); - page_header_update_slot(vcpu->kvm, table, - v >> PAGE_SHIFT); - table[index] = page_to_phys(page) - | PT_PRESENT_MASK | PT_WRITABLE_MASK - | PT_USER_MASK; - if (!was_rmapped) - rmap_add(vcpu, &table[index], v >> PAGE_SHIFT); - else - kvm_release_page_clean(page); - - return 0; + mmu_set_spte(vcpu, &table[index], ACC_ALL, ACC_ALL, + 0, write, 1, &pt_write, gfn); + return pt_write || is_io_pte(table[index]); } if (table[index] == shadow_trap_nonpresent_pte) { @@ -1013,7 +996,6 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, struct page *page) 1, ACC_ALL, &table[index]); if (!new_table) { pgprintk("nonpaging_map: ENOMEM\n"); - kvm_release_page_clean(page); return -ENOMEM; } @@ -1114,9 +1096,10 @@ static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr) static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code) { - struct page *page; + gfn_t gfn; int r; + pgprintk("%s: gva %lx error %x\n", __FUNCTION__, gva, error_code); r = mmu_topup_memory_caches(vcpu); if (r) return r; @@ -1124,14 +1107,10 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, ASSERT(vcpu); ASSERT(VALID_PAGE(vcpu->mmu.root_hpa)); - page = gfn_to_page(vcpu->kvm, gva >> PAGE_SHIFT); + gfn = gva >> PAGE_SHIFT; - if (is_error_page(page)) { - kvm_release_page_clean(page); - return 1; - } - - return nonpaging_map(vcpu, gva & PAGE_MASK, page); + return nonpaging_map(vcpu, gva & PAGE_MASK, + error_code & PFERR_WRITE_MASK, gfn); } static void nonpaging_free(struct kvm_vcpu *vcpu) From 1d075434149c38d457c30d1f11d9c39210b0bb79 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 6 Dec 2007 21:02:25 +0100 Subject: [PATCH 208/249] KVM: SVM: Exit to userspace if write to cr8 and not using in-kernel apic With this patch KVM on SVM will exit to userspace if the guest writes to CR8 and the in-kernel APIC is disabled. Signed-off-by: Joerg Roedel Signed-off-by: Markus Rechberger Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 143d271d4431..442ca818c5a9 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -1048,6 +1048,15 @@ static int emulate_on_interception(struct vcpu_svm *svm, return 1; } +static int cr8_write_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) +{ + emulate_instruction(&svm->vcpu, NULL, 0, 0, 0); + if (irqchip_in_kernel(svm->vcpu.kvm)) + return 1; + kvm_run->exit_reason = KVM_EXIT_SET_TPR; + return 0; +} + static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data) { struct vcpu_svm *svm = to_svm(vcpu); @@ -1202,7 +1211,7 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm, [SVM_EXIT_WRITE_CR0] = emulate_on_interception, [SVM_EXIT_WRITE_CR3] = emulate_on_interception, [SVM_EXIT_WRITE_CR4] = emulate_on_interception, - [SVM_EXIT_WRITE_CR8] = emulate_on_interception, + [SVM_EXIT_WRITE_CR8] = cr8_write_interception, [SVM_EXIT_READ_DR0] = emulate_on_interception, [SVM_EXIT_READ_DR1] = emulate_on_interception, [SVM_EXIT_READ_DR2] = emulate_on_interception, From 7819026eefee53eaaac3fdce1a2f157c7ea943fe Mon Sep 17 00:00:00 2001 From: Marcelo Tosatti Date: Tue, 11 Dec 2007 19:12:27 -0500 Subject: [PATCH 209/249] KVM: MMU: Fix SMP shadow instantiation race There is a race where VCPU0 is shadowing a pagetable entry while VCPU1 is updating it, which results in a stale shadow copy. Fix that by comparing the contents of the cached guest pte with the current guest pte after write-protecting the guest pagetable. Signed-off-by: Marcelo Tosatti Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 12 ++++++++---- drivers/kvm/paging_tmpl.h | 29 +++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index ba71e8d66761..92ac0d1106b4 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -681,7 +681,8 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, unsigned level, int metaphysical, unsigned access, - u64 *parent_pte) + u64 *parent_pte, + bool *new_page) { union kvm_mmu_page_role role; unsigned index; @@ -720,6 +721,8 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, vcpu->mmu.prefetch_page(vcpu, sp); if (!metaphysical) rmap_write_protect(vcpu->kvm, gfn); + if (new_page) + *new_page = 1; return sp; } @@ -993,7 +996,8 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) >> PAGE_SHIFT; new_table = kvm_mmu_get_page(vcpu, pseudo_gfn, v, level - 1, - 1, ACC_ALL, &table[index]); + 1, ACC_ALL, &table[index], + NULL); if (!new_table) { pgprintk("nonpaging_map: ENOMEM\n"); return -ENOMEM; @@ -1059,7 +1063,7 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) ASSERT(!VALID_PAGE(root)); sp = kvm_mmu_get_page(vcpu, root_gfn, 0, - PT64_ROOT_LEVEL, 0, ACC_ALL, NULL); + PT64_ROOT_LEVEL, 0, ACC_ALL, NULL, NULL); root = __pa(sp->spt); ++sp->root_count; vcpu->mmu.root_hpa = root; @@ -1080,7 +1084,7 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) root_gfn = 0; sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, PT32_ROOT_LEVEL, !is_paging(vcpu), - ACC_ALL, NULL); + ACC_ALL, NULL, NULL); root = __pa(sp->spt); ++sp->root_count; vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK; diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 3ab3fb635e16..fb19596c9589 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -65,7 +65,8 @@ struct guest_walker { int level; gfn_t table_gfn[PT_MAX_FULL_LEVELS]; - pt_element_t pte; + pt_element_t ptes[PT_MAX_FULL_LEVELS]; + gpa_t pte_gpa[PT_MAX_FULL_LEVELS]; unsigned pt_access; unsigned pte_access; gfn_t gfn; @@ -150,6 +151,7 @@ walk: pte_gpa = gfn_to_gpa(table_gfn); pte_gpa += index * sizeof(pt_element_t); walker->table_gfn[walker->level - 1] = table_gfn; + walker->pte_gpa[walker->level - 1] = pte_gpa; pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, walker->level - 1, table_gfn); @@ -180,6 +182,8 @@ walk: pte_access = pt_access & FNAME(gpte_access)(vcpu, pte); + walker->ptes[walker->level - 1] = pte; + if (walker->level == PT_PAGE_TABLE_LEVEL) { walker->gfn = gpte_to_gfn(pte); break; @@ -209,9 +213,9 @@ walk: goto walk; pte |= PT_DIRTY_MASK; kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte)); + walker->ptes[walker->level - 1] = pte; } - walker->pte = pte; walker->pt_access = pt_access; walker->pte_access = pte_access; pgprintk("%s: pte %llx pte_access %x pt_access %x\n", @@ -268,7 +272,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, u64 *shadow_ent; unsigned access = walker->pt_access; - if (!is_present_pte(walker->pte)) + if (!is_present_pte(walker->ptes[walker->level - 1])) return NULL; shadow_addr = vcpu->mmu.root_hpa; @@ -285,6 +289,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, u64 shadow_pte; int metaphysical; gfn_t table_gfn; + bool new_page = 0; shadow_ent = ((u64 *)__va(shadow_addr)) + index; if (is_shadow_present_pte(*shadow_ent)) { @@ -300,16 +305,23 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, if (level - 1 == PT_PAGE_TABLE_LEVEL && walker->level == PT_DIRECTORY_LEVEL) { metaphysical = 1; - if (!is_dirty_pte(walker->pte)) + if (!is_dirty_pte(walker->ptes[level - 1])) access &= ~ACC_WRITE_MASK; - table_gfn = gpte_to_gfn(walker->pte); + table_gfn = gpte_to_gfn(walker->ptes[level - 1]); } else { metaphysical = 0; table_gfn = walker->table_gfn[level - 2]; } shadow_page = kvm_mmu_get_page(vcpu, table_gfn, addr, level-1, metaphysical, access, - shadow_ent); + shadow_ent, &new_page); + if (new_page && !metaphysical) { + pt_element_t curr_pte; + kvm_read_guest(vcpu->kvm, walker->pte_gpa[level - 2], + &curr_pte, sizeof(curr_pte)); + if (curr_pte != walker->ptes[level - 2]) + return NULL; + } shadow_addr = __pa(shadow_page->spt); shadow_pte = shadow_addr | PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK; @@ -317,7 +329,8 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, } mmu_set_spte(vcpu, shadow_ent, access, walker->pte_access & access, - user_fault, write_fault, walker->pte & PT_DIRTY_MASK, + user_fault, write_fault, + walker->ptes[walker->level-1] & PT_DIRTY_MASK, ptwrite, walker->gfn); return shadow_ent; @@ -382,7 +395,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, /* * mmio: emulate if accessible, otherwise its a guest fault. */ - if (is_io_pte(*shadow_pte)) + if (shadow_pte && is_io_pte(*shadow_pte)) return 1; ++vcpu->stat.pf_fixed; From 722f6ecbcf3e53809236b36e6dfd9042924a2d51 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 12 Dec 2007 12:37:24 +0100 Subject: [PATCH 210/249] KVM: LAPIC: minor debugging compile fix This patch fixes a compile error of the LAPIC code with APIC debugging enabled. Signed-off-by: Joerg Roedel Signed-off-by: Markus Rechberger Signed-off-by: Avi Kivity --- drivers/kvm/lapic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index 5efa6c0276c6..466c37f02e85 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -829,7 +829,7 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) /* with FSB delivery interrupt, we can restart APIC functionality */ apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is " - "0x%lx.\n", apic->apic_base, apic->base_address); + "0x%lx.\n", apic->vcpu->apic_base, apic->base_address); } From 62b9abaaf8423a3a38b938831b3b8431a6607ad8 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 11 Dec 2007 15:36:57 +0100 Subject: [PATCH 211/249] KVM: SVM: support writing 0 to K8 performance counter control registers This lets SVM ignore writes of the value 0 to the performance counter control registers. Thus enabling them will still fail in the guest, but a write of 0 which keeps them disabled is accepted. This is required to boot Windows Vista 64bit. [avi: avoid fall-thru in switch statement] Signed-off-by: Joerg Roedel Signed-off-by: Markus Rechberger Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 442ca818c5a9..ef21804a5c5c 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -1155,7 +1155,20 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data) case MSR_IA32_SYSENTER_ESP: svm->vmcb->save.sysenter_esp = data; break; + case MSR_K7_EVNTSEL0: + case MSR_K7_EVNTSEL1: + case MSR_K7_EVNTSEL2: + case MSR_K7_EVNTSEL3: + /* + * only support writing 0 to the performance counters for now + * to make Windows happy. Should be replaced by a real + * performance counter emulation later. + */ + if (data != 0) + goto unhandled; + break; default: + unhandled: return kvm_set_msr_common(vcpu, ecx, data); } return 0; From 2bacc55c7c3c61e356aef06b9a319b4cee90b519 Mon Sep 17 00:00:00 2001 From: Marcelo Tosatti Date: Wed, 12 Dec 2007 10:46:12 -0500 Subject: [PATCH 212/249] KVM: MMU: emulated cmpxchg8b should be atomic on i386 Emulate cmpxchg8b atomically on i386. This is required to avoid a guest pte walker from seeing a splitted write. [avi: make it compile] Signed-off-by: Marcelo Tosatti Signed-off-by: Avi Kivity --- drivers/kvm/x86.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 9db4e3242b62..3b79684a3c0c 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -1674,6 +1675,31 @@ static int emulator_cmpxchg_emulated(unsigned long addr, reported = 1; printk(KERN_WARNING "kvm: emulating exchange as write\n"); } +#ifndef CONFIG_X86_64 + /* guests cmpxchg8b have to be emulated atomically */ + if (bytes == 8) { + gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + struct page *page; + char *addr; + u64 val; + + if (gpa == UNMAPPED_GVA || + (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) + goto emul_write; + + if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK)) + goto emul_write; + + val = *(u64 *)new; + page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); + addr = kmap_atomic(page, KM_USER0); + set_64bit((u64 *)(addr + offset_in_page(gpa)), val); + kunmap_atomic(addr, KM_USER0); + kvm_release_page_dirty(page); + } +emul_write: +#endif + return emulator_write_emulated(addr, new, bytes, vcpu); } From 682c59a3f3f211ed555b17144f2d82eb8286a1db Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Tue, 11 Dec 2007 20:36:00 +0800 Subject: [PATCH 213/249] KVM: Portability: Move kvm{pic,ioapic} accesors to x86 specific code Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/irq.h | 18 ++++++++++++++++-- drivers/kvm/kvm.h | 18 +++--------------- drivers/kvm/x86.h | 1 - 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index 730a87c173e5..e88d93957d24 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -26,6 +26,7 @@ #include #include #include "iodev.h" +#include "x86.h" struct kvm; struct kvm_vcpu; @@ -63,8 +64,6 @@ struct kvm_pic { struct kvm_pic *kvm_create_pic(struct kvm *kvm); void kvm_pic_set_irq(void *opaque, int irq, int level); int kvm_pic_read_irq(struct kvm_pic *s); -int kvm_cpu_get_interrupt(struct kvm_vcpu *v); -int kvm_cpu_has_interrupt(struct kvm_vcpu *v); void kvm_pic_update_irq(struct kvm_pic *s); #define IOAPIC_NUM_PINS KVM_IOAPIC_NUM_PINS @@ -147,6 +146,21 @@ do { \ #define ASSERT(x) do { } while (0) #endif +static inline struct kvm_pic *pic_irqchip(struct kvm *kvm) +{ + return kvm->vpic; +} + +static inline struct kvm_ioapic *ioapic_irqchip(struct kvm *kvm) +{ + return kvm->vioapic; +} + +static inline int irqchip_in_kernel(struct kvm *kvm) +{ + return pic_irqchip(kvm) != NULL; +} + void kvm_vcpu_kick(struct kvm_vcpu *vcpu); int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu); int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu); diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index ceefb4427dbd..668a8300365d 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -273,21 +273,6 @@ struct kvm { struct kvm_vm_stat stat; }; -static inline struct kvm_pic *pic_irqchip(struct kvm *kvm) -{ - return kvm->vpic; -} - -static inline struct kvm_ioapic *ioapic_irqchip(struct kvm *kvm) -{ - return kvm->vioapic; -} - -static inline int irqchip_in_kernel(struct kvm *kvm) -{ - return pic_irqchip(kvm) != NULL; -} - /* The guest did something we don't support. */ #define pr_unimpl(vcpu, fmt, ...) \ do { \ @@ -417,6 +402,9 @@ void kvm_free_physmem(struct kvm *kvm); struct kvm *kvm_arch_create_vm(void); void kvm_arch_destroy_vm(struct kvm *kvm); +int kvm_cpu_get_interrupt(struct kvm_vcpu *v); +int kvm_cpu_has_interrupt(struct kvm_vcpu *v); + static inline void kvm_guest_enter(void) { account_system_vtime(current); diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 78396d627be0..0fc7020aa1a5 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -12,7 +12,6 @@ #define KVM_X86_H #include "kvm.h" -#include "irq.h" #include #include From ad312c7c79f781c822e37effe41307503a2bb85b Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Thu, 13 Dec 2007 23:50:52 +0800 Subject: [PATCH 214/249] KVM: Portability: Introduce kvm_vcpu_arch Move all the architecture-specific fields in kvm_vcpu into a new struct kvm_vcpu_arch. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/ioapic.c | 10 +- drivers/kvm/kvm_main.c | 2 +- drivers/kvm/lapic.c | 91 +++---- drivers/kvm/mmu.c | 142 +++++------ drivers/kvm/paging_tmpl.h | 16 +- drivers/kvm/svm.c | 122 ++++----- drivers/kvm/vmx.c | 213 ++++++++-------- drivers/kvm/x86.c | 520 +++++++++++++++++++------------------- drivers/kvm/x86.h | 25 +- drivers/kvm/x86_emulate.c | 24 +- 10 files changed, 585 insertions(+), 580 deletions(-) diff --git a/drivers/kvm/ioapic.c b/drivers/kvm/ioapic.c index e7debfafca50..04910368c251 100644 --- a/drivers/kvm/ioapic.c +++ b/drivers/kvm/ioapic.c @@ -158,7 +158,7 @@ static u32 ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest, if (dest_mode == 0) { /* Physical mode. */ if (dest == 0xFF) { /* Broadcast. */ for (i = 0; i < KVM_MAX_VCPUS; ++i) - if (kvm->vcpus[i] && kvm->vcpus[i]->apic) + if (kvm->vcpus[i] && kvm->vcpus[i]->arch.apic) mask |= 1 << i; return mask; } @@ -166,8 +166,8 @@ static u32 ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest, vcpu = kvm->vcpus[i]; if (!vcpu) continue; - if (kvm_apic_match_physical_addr(vcpu->apic, dest)) { - if (vcpu->apic) + if (kvm_apic_match_physical_addr(vcpu->arch.apic, dest)) { + if (vcpu->arch.apic) mask = 1 << i; break; } @@ -177,8 +177,8 @@ static u32 ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest, vcpu = kvm->vcpus[i]; if (!vcpu) continue; - if (vcpu->apic && - kvm_apic_match_logical_addr(vcpu->apic, dest)) + if (vcpu->arch.apic && + kvm_apic_match_logical_addr(vcpu->arch.apic, dest)) mask |= 1 << vcpu->vcpu_id; } ioapic_debug("mask %x\n", mask); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 1a03b67eb921..ae2a1bf640bc 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -670,7 +670,7 @@ static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf) if (vmf->pgoff == 0) page = virt_to_page(vcpu->run); else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) - page = virt_to_page(vcpu->pio_data); + page = virt_to_page(vcpu->arch.pio_data); else return VM_FAULT_SIGBUS; get_page(page); diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index 466c37f02e85..5c9f46784c26 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -58,6 +58,7 @@ #define VEC_POS(v) ((v) & (32 - 1)) #define REG_POS(v) (((v) >> 5) << 4) + static inline u32 apic_get_reg(struct kvm_lapic *apic, int reg_off) { return *((u32 *) (apic->regs + reg_off)); @@ -90,7 +91,7 @@ static inline void apic_clear_vector(int vec, void *bitmap) static inline int apic_hw_enabled(struct kvm_lapic *apic) { - return (apic)->vcpu->apic_base & MSR_IA32_APICBASE_ENABLE; + return (apic)->vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE; } static inline int apic_sw_enabled(struct kvm_lapic *apic) @@ -174,7 +175,7 @@ static inline int apic_find_highest_irr(struct kvm_lapic *apic) int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; int highest_irr; if (!apic) @@ -187,7 +188,7 @@ EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr); int kvm_apic_set_irq(struct kvm_vcpu *vcpu, u8 vec, u8 trig) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; if (!apic_test_and_set_irr(vec, apic)) { /* a new pending irq is set in IRR */ @@ -272,7 +273,7 @@ static int apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, int short_hand, int dest, int dest_mode) { int result = 0; - struct kvm_lapic *target = vcpu->apic; + struct kvm_lapic *target = vcpu->arch.apic; apic_debug("target %p, source %p, dest 0x%x, " "dest_mode 0x%x, short_hand 0x%x", @@ -339,10 +340,10 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, } else apic_clear_vector(vector, apic->regs + APIC_TMR); - if (vcpu->mp_state == VCPU_MP_STATE_RUNNABLE) + if (vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE) kvm_vcpu_kick(vcpu); - else if (vcpu->mp_state == VCPU_MP_STATE_HALTED) { - vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; + else if (vcpu->arch.mp_state == VCPU_MP_STATE_HALTED) { + vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE; if (waitqueue_active(&vcpu->wq)) wake_up_interruptible(&vcpu->wq); } @@ -363,11 +364,11 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, case APIC_DM_INIT: if (level) { - if (vcpu->mp_state == VCPU_MP_STATE_RUNNABLE) + if (vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE) printk(KERN_DEBUG "INIT on a runnable vcpu %d\n", vcpu->vcpu_id); - vcpu->mp_state = VCPU_MP_STATE_INIT_RECEIVED; + vcpu->arch.mp_state = VCPU_MP_STATE_INIT_RECEIVED; kvm_vcpu_kick(vcpu); } else { printk(KERN_DEBUG @@ -380,9 +381,9 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, case APIC_DM_STARTUP: printk(KERN_DEBUG "SIPI to vcpu %d vector 0x%02x\n", vcpu->vcpu_id, vector); - if (vcpu->mp_state == VCPU_MP_STATE_INIT_RECEIVED) { - vcpu->sipi_vector = vector; - vcpu->mp_state = VCPU_MP_STATE_SIPI_RECEIVED; + if (vcpu->arch.mp_state == VCPU_MP_STATE_INIT_RECEIVED) { + vcpu->arch.sipi_vector = vector; + vcpu->arch.mp_state = VCPU_MP_STATE_SIPI_RECEIVED; if (waitqueue_active(&vcpu->wq)) wake_up_interruptible(&vcpu->wq); } @@ -411,7 +412,7 @@ static struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, next = 0; if (kvm->vcpus[next] == NULL || !test_bit(next, &bitmap)) continue; - apic = kvm->vcpus[next]->apic; + apic = kvm->vcpus[next]->arch.apic; if (apic && apic_enabled(apic)) break; apic = NULL; @@ -482,12 +483,12 @@ static void apic_send_ipi(struct kvm_lapic *apic) if (!vcpu) continue; - if (vcpu->apic && + if (vcpu->arch.apic && apic_match_dest(vcpu, apic, short_hand, dest, dest_mode)) { if (delivery_mode == APIC_DM_LOWEST) set_bit(vcpu->vcpu_id, &lpr_map); else - __apic_accept_irq(vcpu->apic, delivery_mode, + __apic_accept_irq(vcpu->arch.apic, delivery_mode, vector, level, trig_mode); } } @@ -495,7 +496,7 @@ static void apic_send_ipi(struct kvm_lapic *apic) if (delivery_mode == APIC_DM_LOWEST) { target = kvm_get_lowest_prio_vcpu(vcpu->kvm, vector, lpr_map); if (target != NULL) - __apic_accept_irq(target->apic, delivery_mode, + __apic_accept_irq(target->arch.apic, delivery_mode, vector, level, trig_mode); } } @@ -772,15 +773,15 @@ static int apic_mmio_range(struct kvm_io_device *this, gpa_t addr) void kvm_free_lapic(struct kvm_vcpu *vcpu) { - if (!vcpu->apic) + if (!vcpu->arch.apic) return; - hrtimer_cancel(&vcpu->apic->timer.dev); + hrtimer_cancel(&vcpu->arch.apic->timer.dev); - if (vcpu->apic->regs_page) - __free_page(vcpu->apic->regs_page); + if (vcpu->arch.apic->regs_page) + __free_page(vcpu->arch.apic->regs_page); - kfree(vcpu->apic); + kfree(vcpu->arch.apic); } /* @@ -791,7 +792,7 @@ void kvm_free_lapic(struct kvm_vcpu *vcpu) void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; if (!apic) return; @@ -800,7 +801,7 @@ void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; u64 tpr; if (!apic) @@ -813,29 +814,29 @@ EXPORT_SYMBOL_GPL(kvm_lapic_get_cr8); void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; if (!apic) { value |= MSR_IA32_APICBASE_BSP; - vcpu->apic_base = value; + vcpu->arch.apic_base = value; return; } if (apic->vcpu->vcpu_id) value &= ~MSR_IA32_APICBASE_BSP; - vcpu->apic_base = value; - apic->base_address = apic->vcpu->apic_base & + vcpu->arch.apic_base = value; + apic->base_address = apic->vcpu->arch.apic_base & MSR_IA32_APICBASE_BASE; /* with FSB delivery interrupt, we can restart APIC functionality */ apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is " - "0x%lx.\n", apic->vcpu->apic_base, apic->base_address); + "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address); } u64 kvm_lapic_get_base(struct kvm_vcpu *vcpu) { - return vcpu->apic_base; + return vcpu->arch.apic_base; } EXPORT_SYMBOL_GPL(kvm_lapic_get_base); @@ -847,7 +848,7 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu) apic_debug("%s\n", __FUNCTION__); ASSERT(vcpu); - apic = vcpu->apic; + apic = vcpu->arch.apic; ASSERT(apic != NULL); /* Stop the timer in case it's a reset to an active apic */ @@ -878,19 +879,19 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu) update_divide_count(apic); atomic_set(&apic->timer.pending, 0); if (vcpu->vcpu_id == 0) - vcpu->apic_base |= MSR_IA32_APICBASE_BSP; + vcpu->arch.apic_base |= MSR_IA32_APICBASE_BSP; apic_update_ppr(apic); apic_debug(KERN_INFO "%s: vcpu=%p, id=%d, base_msr=" "0x%016" PRIx64 ", base_address=0x%0lx.\n", __FUNCTION__, vcpu, kvm_apic_id(apic), - vcpu->apic_base, apic->base_address); + vcpu->arch.apic_base, apic->base_address); } EXPORT_SYMBOL_GPL(kvm_lapic_reset); int kvm_lapic_enabled(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; int ret = 0; if (!apic) @@ -915,7 +916,7 @@ static int __apic_timer_fn(struct kvm_lapic *apic) atomic_inc(&apic->timer.pending); if (waitqueue_active(q)) { - apic->vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; + apic->vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE; wake_up_interruptible(q); } if (apic_lvtt_period(apic)) { @@ -961,7 +962,7 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu) if (!apic) goto nomem; - vcpu->apic = apic; + vcpu->arch.apic = apic; apic->regs_page = alloc_page(GFP_KERNEL); if (apic->regs_page == NULL) { @@ -976,7 +977,7 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu) hrtimer_init(&apic->timer.dev, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); apic->timer.dev.function = apic_timer_fn; apic->base_address = APIC_DEFAULT_PHYS_BASE; - vcpu->apic_base = APIC_DEFAULT_PHYS_BASE; + vcpu->arch.apic_base = APIC_DEFAULT_PHYS_BASE; kvm_lapic_reset(vcpu); apic->dev.read = apic_mmio_read; @@ -994,7 +995,7 @@ EXPORT_SYMBOL_GPL(kvm_create_lapic); int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; int highest_irr; if (!apic || !apic_enabled(apic)) @@ -1010,11 +1011,11 @@ int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) { - u32 lvt0 = apic_get_reg(vcpu->apic, APIC_LVT0); + u32 lvt0 = apic_get_reg(vcpu->arch.apic, APIC_LVT0); int r = 0; if (vcpu->vcpu_id == 0) { - if (!apic_hw_enabled(vcpu->apic)) + if (!apic_hw_enabled(vcpu->arch.apic)) r = 1; if ((lvt0 & APIC_LVT_MASKED) == 0 && GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT) @@ -1025,7 +1026,7 @@ int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; if (apic && apic_lvt_enabled(apic, APIC_LVTT) && atomic_read(&apic->timer.pending) > 0) { @@ -1036,7 +1037,7 @@ void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) void kvm_apic_timer_intr_post(struct kvm_vcpu *vcpu, int vec) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; if (apic && apic_lvt_vector(apic, APIC_LVTT) == vec) apic->timer.last_update = ktime_add_ns( @@ -1047,7 +1048,7 @@ void kvm_apic_timer_intr_post(struct kvm_vcpu *vcpu, int vec) int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) { int vector = kvm_apic_has_interrupt(vcpu); - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; if (vector == -1) return -1; @@ -1060,9 +1061,9 @@ int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; - apic->base_address = vcpu->apic_base & + apic->base_address = vcpu->arch.apic_base & MSR_IA32_APICBASE_BASE; apic_set_reg(apic, APIC_LVR, APIC_VERSION); apic_update_ppr(apic); @@ -1073,7 +1074,7 @@ void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu) void kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; struct hrtimer *timer; if (!apic) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 92ac0d1106b4..da1dedb497b8 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -180,7 +180,7 @@ EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes); static int is_write_protection(struct kvm_vcpu *vcpu) { - return vcpu->cr0 & X86_CR0_WP; + return vcpu->arch.cr0 & X86_CR0_WP; } static int is_cpuid_PSE36(void) @@ -190,7 +190,7 @@ static int is_cpuid_PSE36(void) static int is_nx(struct kvm_vcpu *vcpu) { - return vcpu->shadow_efer & EFER_NX; + return vcpu->arch.shadow_efer & EFER_NX; } static int is_present_pte(unsigned long pte) @@ -292,18 +292,18 @@ static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu) int r; kvm_mmu_free_some_pages(vcpu); - r = mmu_topup_memory_cache(&vcpu->mmu_pte_chain_cache, + r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache, pte_chain_cache, 4); if (r) goto out; - r = mmu_topup_memory_cache(&vcpu->mmu_rmap_desc_cache, + r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache, rmap_desc_cache, 1); if (r) goto out; - r = mmu_topup_memory_cache_page(&vcpu->mmu_page_cache, 8); + r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8); if (r) goto out; - r = mmu_topup_memory_cache(&vcpu->mmu_page_header_cache, + r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache, mmu_page_header_cache, 4); out: return r; @@ -311,10 +311,10 @@ out: static void mmu_free_memory_caches(struct kvm_vcpu *vcpu) { - mmu_free_memory_cache(&vcpu->mmu_pte_chain_cache); - mmu_free_memory_cache(&vcpu->mmu_rmap_desc_cache); - mmu_free_memory_cache_page(&vcpu->mmu_page_cache); - mmu_free_memory_cache(&vcpu->mmu_page_header_cache); + mmu_free_memory_cache(&vcpu->arch.mmu_pte_chain_cache); + mmu_free_memory_cache(&vcpu->arch.mmu_rmap_desc_cache); + mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache); + mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache); } static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc, @@ -330,7 +330,7 @@ static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc, static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu) { - return mmu_memory_cache_alloc(&vcpu->mmu_pte_chain_cache, + return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_chain_cache, sizeof(struct kvm_pte_chain)); } @@ -341,7 +341,7 @@ static void mmu_free_pte_chain(struct kvm_pte_chain *pc) static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu) { - return mmu_memory_cache_alloc(&vcpu->mmu_rmap_desc_cache, + return mmu_memory_cache_alloc(&vcpu->arch.mmu_rmap_desc_cache, sizeof(struct kvm_rmap_desc)); } @@ -568,9 +568,9 @@ static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, if (!vcpu->kvm->n_free_mmu_pages) return NULL; - sp = mmu_memory_cache_alloc(&vcpu->mmu_page_header_cache, sizeof *sp); - sp->spt = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE); - sp->gfns = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE); + sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp); + sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE); + sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE); set_page_private(virt_to_page(sp->spt), (unsigned long)sp); list_add(&sp->link, &vcpu->kvm->active_mmu_pages); ASSERT(is_empty_shadow_page(sp->spt)); @@ -692,11 +692,11 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, struct hlist_node *node; role.word = 0; - role.glevels = vcpu->mmu.root_level; + role.glevels = vcpu->arch.mmu.root_level; role.level = level; role.metaphysical = metaphysical; role.access = access; - if (vcpu->mmu.root_level <= PT32_ROOT_LEVEL) { + if (vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) { quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level)); quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1; role.quadrant = quadrant; @@ -718,7 +718,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, sp->gfn = gfn; sp->role = role; hlist_add_head(&sp->hash_link, bucket); - vcpu->mmu.prefetch_page(vcpu, sp); + vcpu->arch.mmu.prefetch_page(vcpu, sp); if (!metaphysical) rmap_write_protect(vcpu->kvm, gfn); if (new_page) @@ -768,7 +768,7 @@ static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm) for (i = 0; i < KVM_MAX_VCPUS; ++i) if (kvm->vcpus[i]) - kvm->vcpus[i]->last_pte_updated = NULL; + kvm->vcpus[i]->arch.last_pte_updated = NULL; } static void kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp) @@ -875,7 +875,7 @@ static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn) struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva) { - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva); + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva); if (gpa == UNMAPPED_GVA) return NULL; @@ -962,7 +962,7 @@ unshadowed: else kvm_release_page_clean(page); if (!ptwrite || !*ptwrite) - vcpu->last_pte_updated = shadow_pte; + vcpu->arch.last_pte_updated = shadow_pte; } static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) @@ -972,7 +972,7 @@ static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) { int level = PT32E_ROOT_LEVEL; - hpa_t table_addr = vcpu->mmu.root_hpa; + hpa_t table_addr = vcpu->arch.mmu.root_hpa; int pt_write = 0; for (; ; level--) { @@ -1024,29 +1024,29 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu) int i; struct kvm_mmu_page *sp; - if (!VALID_PAGE(vcpu->mmu.root_hpa)) + if (!VALID_PAGE(vcpu->arch.mmu.root_hpa)) return; #ifdef CONFIG_X86_64 - if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) { - hpa_t root = vcpu->mmu.root_hpa; + if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) { + hpa_t root = vcpu->arch.mmu.root_hpa; sp = page_header(root); --sp->root_count; - vcpu->mmu.root_hpa = INVALID_PAGE; + vcpu->arch.mmu.root_hpa = INVALID_PAGE; return; } #endif for (i = 0; i < 4; ++i) { - hpa_t root = vcpu->mmu.pae_root[i]; + hpa_t root = vcpu->arch.mmu.pae_root[i]; if (root) { root &= PT64_BASE_ADDR_MASK; sp = page_header(root); --sp->root_count; } - vcpu->mmu.pae_root[i] = INVALID_PAGE; + vcpu->arch.mmu.pae_root[i] = INVALID_PAGE; } - vcpu->mmu.root_hpa = INVALID_PAGE; + vcpu->arch.mmu.root_hpa = INVALID_PAGE; } static void mmu_alloc_roots(struct kvm_vcpu *vcpu) @@ -1055,41 +1055,41 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) gfn_t root_gfn; struct kvm_mmu_page *sp; - root_gfn = vcpu->cr3 >> PAGE_SHIFT; + root_gfn = vcpu->arch.cr3 >> PAGE_SHIFT; #ifdef CONFIG_X86_64 - if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) { - hpa_t root = vcpu->mmu.root_hpa; + if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) { + hpa_t root = vcpu->arch.mmu.root_hpa; ASSERT(!VALID_PAGE(root)); sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL, 0, ACC_ALL, NULL, NULL); root = __pa(sp->spt); ++sp->root_count; - vcpu->mmu.root_hpa = root; + vcpu->arch.mmu.root_hpa = root; return; } #endif for (i = 0; i < 4; ++i) { - hpa_t root = vcpu->mmu.pae_root[i]; + hpa_t root = vcpu->arch.mmu.pae_root[i]; ASSERT(!VALID_PAGE(root)); - if (vcpu->mmu.root_level == PT32E_ROOT_LEVEL) { - if (!is_present_pte(vcpu->pdptrs[i])) { - vcpu->mmu.pae_root[i] = 0; + if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) { + if (!is_present_pte(vcpu->arch.pdptrs[i])) { + vcpu->arch.mmu.pae_root[i] = 0; continue; } - root_gfn = vcpu->pdptrs[i] >> PAGE_SHIFT; - } else if (vcpu->mmu.root_level == 0) + root_gfn = vcpu->arch.pdptrs[i] >> PAGE_SHIFT; + } else if (vcpu->arch.mmu.root_level == 0) root_gfn = 0; sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, PT32_ROOT_LEVEL, !is_paging(vcpu), ACC_ALL, NULL, NULL); root = __pa(sp->spt); ++sp->root_count; - vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK; + vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK; } - vcpu->mmu.root_hpa = __pa(vcpu->mmu.pae_root); + vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root); } static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr) @@ -1109,7 +1109,7 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, return r; ASSERT(vcpu); - ASSERT(VALID_PAGE(vcpu->mmu.root_hpa)); + ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa)); gfn = gva >> PAGE_SHIFT; @@ -1124,7 +1124,7 @@ static void nonpaging_free(struct kvm_vcpu *vcpu) static int nonpaging_init_context(struct kvm_vcpu *vcpu) { - struct kvm_mmu *context = &vcpu->mmu; + struct kvm_mmu *context = &vcpu->arch.mmu; context->new_cr3 = nonpaging_new_cr3; context->page_fault = nonpaging_page_fault; @@ -1171,7 +1171,7 @@ static void paging_free(struct kvm_vcpu *vcpu) static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level) { - struct kvm_mmu *context = &vcpu->mmu; + struct kvm_mmu *context = &vcpu->arch.mmu; ASSERT(is_pae(vcpu)); context->new_cr3 = paging_new_cr3; @@ -1192,7 +1192,7 @@ static int paging64_init_context(struct kvm_vcpu *vcpu) static int paging32_init_context(struct kvm_vcpu *vcpu) { - struct kvm_mmu *context = &vcpu->mmu; + struct kvm_mmu *context = &vcpu->arch.mmu; context->new_cr3 = paging_new_cr3; context->page_fault = paging32_page_fault; @@ -1213,7 +1213,7 @@ static int paging32E_init_context(struct kvm_vcpu *vcpu) static int init_kvm_mmu(struct kvm_vcpu *vcpu) { ASSERT(vcpu); - ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa)); + ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa)); if (!is_paging(vcpu)) return nonpaging_init_context(vcpu); @@ -1228,9 +1228,9 @@ static int init_kvm_mmu(struct kvm_vcpu *vcpu) static void destroy_kvm_mmu(struct kvm_vcpu *vcpu) { ASSERT(vcpu); - if (VALID_PAGE(vcpu->mmu.root_hpa)) { - vcpu->mmu.free(vcpu); - vcpu->mmu.root_hpa = INVALID_PAGE; + if (VALID_PAGE(vcpu->arch.mmu.root_hpa)) { + vcpu->arch.mmu.free(vcpu); + vcpu->arch.mmu.root_hpa = INVALID_PAGE; } } @@ -1250,7 +1250,7 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu) if (r) goto out; mmu_alloc_roots(vcpu); - kvm_x86_ops->set_cr3(vcpu, vcpu->mmu.root_hpa); + kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa); kvm_mmu_flush_tlb(vcpu); out: mutex_unlock(&vcpu->kvm->lock); @@ -1323,7 +1323,7 @@ static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, u64 old, u64 new) static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu) { - u64 *spte = vcpu->last_pte_updated; + u64 *spte = vcpu->arch.last_pte_updated; return !!(spte && (*spte & PT_ACCESSED_MASK)); } @@ -1350,15 +1350,15 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes); ++vcpu->kvm->stat.mmu_pte_write; kvm_mmu_audit(vcpu, "pre pte write"); - if (gfn == vcpu->last_pt_write_gfn + if (gfn == vcpu->arch.last_pt_write_gfn && !last_updated_pte_accessed(vcpu)) { - ++vcpu->last_pt_write_count; - if (vcpu->last_pt_write_count >= 3) + ++vcpu->arch.last_pt_write_count; + if (vcpu->arch.last_pt_write_count >= 3) flooded = 1; } else { - vcpu->last_pt_write_gfn = gfn; - vcpu->last_pt_write_count = 1; - vcpu->last_pte_updated = NULL; + vcpu->arch.last_pt_write_gfn = gfn; + vcpu->arch.last_pt_write_count = 1; + vcpu->arch.last_pte_updated = NULL; } index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; bucket = &vcpu->kvm->mmu_page_hash[index]; @@ -1420,7 +1420,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) { - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva); + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva); return kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT); } @@ -1443,7 +1443,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) enum emulation_result er; mutex_lock(&vcpu->kvm->lock); - r = vcpu->mmu.page_fault(vcpu, cr2, error_code); + r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code); if (r < 0) goto out; @@ -1486,7 +1486,7 @@ static void free_mmu_pages(struct kvm_vcpu *vcpu) struct kvm_mmu_page, link); kvm_mmu_zap_page(vcpu->kvm, sp); } - free_page((unsigned long)vcpu->mmu.pae_root); + free_page((unsigned long)vcpu->arch.mmu.pae_root); } static int alloc_mmu_pages(struct kvm_vcpu *vcpu) @@ -1508,9 +1508,9 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu) page = alloc_page(GFP_KERNEL | __GFP_DMA32); if (!page) goto error_1; - vcpu->mmu.pae_root = page_address(page); + vcpu->arch.mmu.pae_root = page_address(page); for (i = 0; i < 4; ++i) - vcpu->mmu.pae_root[i] = INVALID_PAGE; + vcpu->arch.mmu.pae_root[i] = INVALID_PAGE; return 0; @@ -1522,7 +1522,7 @@ error_1: int kvm_mmu_create(struct kvm_vcpu *vcpu) { ASSERT(vcpu); - ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa)); + ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa)); return alloc_mmu_pages(vcpu); } @@ -1530,7 +1530,7 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu) int kvm_mmu_setup(struct kvm_vcpu *vcpu) { ASSERT(vcpu); - ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa)); + ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa)); return init_kvm_mmu(vcpu); } @@ -1659,11 +1659,11 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, printk(KERN_ERR "audit: (%s) nontrapping pte" " in nonleaf level: levels %d gva %lx" " level %d pte %llx\n", audit_msg, - vcpu->mmu.root_level, va, level, ent); + vcpu->arch.mmu.root_level, va, level, ent); audit_mappings_page(vcpu, ent, va, level - 1); } else { - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va); + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, va); struct page *page = gpa_to_page(vcpu, gpa); hpa_t hpa = page_to_phys(page); @@ -1671,7 +1671,7 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, && (ent & PT64_BASE_ADDR_MASK) != hpa) printk(KERN_ERR "xx audit error: (%s) levels %d" " gva %lx gpa %llx hpa %llx ent %llx %d\n", - audit_msg, vcpu->mmu.root_level, + audit_msg, vcpu->arch.mmu.root_level, va, gpa, hpa, ent, is_shadow_present_pte(ent)); else if (ent == shadow_notrap_nonpresent_pte @@ -1688,13 +1688,13 @@ static void audit_mappings(struct kvm_vcpu *vcpu) { unsigned i; - if (vcpu->mmu.root_level == 4) - audit_mappings_page(vcpu, vcpu->mmu.root_hpa, 0, 4); + if (vcpu->arch.mmu.root_level == 4) + audit_mappings_page(vcpu, vcpu->arch.mmu.root_hpa, 0, 4); else for (i = 0; i < 4; ++i) - if (vcpu->mmu.pae_root[i] & PT_PRESENT_MASK) + if (vcpu->arch.mmu.pae_root[i] & PT_PRESENT_MASK) audit_mappings_page(vcpu, - vcpu->mmu.pae_root[i], + vcpu->arch.mmu.pae_root[i], i << 30, 2); } diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index fb19596c9589..56b88f7e83ef 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -129,11 +129,11 @@ static int FNAME(walk_addr)(struct guest_walker *walker, pgprintk("%s: addr %lx\n", __FUNCTION__, addr); walk: - walker->level = vcpu->mmu.root_level; - pte = vcpu->cr3; + walker->level = vcpu->arch.mmu.root_level; + pte = vcpu->arch.cr3; #if PTTYPE == 64 if (!is_long_mode(vcpu)) { - pte = vcpu->pdptrs[(addr >> 30) & 3]; + pte = vcpu->arch.pdptrs[(addr >> 30) & 3]; if (!is_present_pte(pte)) goto not_present; --walker->level; @@ -275,10 +275,10 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, if (!is_present_pte(walker->ptes[walker->level - 1])) return NULL; - shadow_addr = vcpu->mmu.root_hpa; - level = vcpu->mmu.shadow_root_level; + shadow_addr = vcpu->arch.mmu.root_hpa; + level = vcpu->arch.mmu.shadow_root_level; if (level == PT32E_ROOT_LEVEL) { - shadow_addr = vcpu->mmu.pae_root[(addr >> 30) & 3]; + shadow_addr = vcpu->arch.mmu.pae_root[(addr >> 30) & 3]; shadow_addr &= PT64_BASE_ADDR_MASK; --level; } @@ -380,7 +380,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, if (!r) { pgprintk("%s: guest page fault\n", __FUNCTION__); inject_page_fault(vcpu, addr, walker.error_code); - vcpu->last_pt_write_count = 0; /* reset fork detector */ + vcpu->arch.last_pt_write_count = 0; /* reset fork detector */ return 0; } @@ -390,7 +390,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, shadow_pte, *shadow_pte, write_pt); if (!write_pt) - vcpu->last_pt_write_count = 0; /* reset fork detector */ + vcpu->arch.last_pt_write_count = 0; /* reset fork detector */ /* * mmio: emulate if accessible, otherwise its a guest fault. diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index ef21804a5c5c..7888638c02e8 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -99,20 +99,20 @@ static inline u32 svm_has(u32 feat) static inline u8 pop_irq(struct kvm_vcpu *vcpu) { - int word_index = __ffs(vcpu->irq_summary); - int bit_index = __ffs(vcpu->irq_pending[word_index]); + int word_index = __ffs(vcpu->arch.irq_summary); + int bit_index = __ffs(vcpu->arch.irq_pending[word_index]); int irq = word_index * BITS_PER_LONG + bit_index; - clear_bit(bit_index, &vcpu->irq_pending[word_index]); - if (!vcpu->irq_pending[word_index]) - clear_bit(word_index, &vcpu->irq_summary); + clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]); + if (!vcpu->arch.irq_pending[word_index]) + clear_bit(word_index, &vcpu->arch.irq_summary); return irq; } static inline void push_irq(struct kvm_vcpu *vcpu, u8 irq) { - set_bit(irq, vcpu->irq_pending); - set_bit(irq / BITS_PER_LONG, &vcpu->irq_summary); + set_bit(irq, vcpu->arch.irq_pending); + set_bit(irq / BITS_PER_LONG, &vcpu->arch.irq_summary); } static inline void clgi(void) @@ -185,7 +185,7 @@ static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) efer &= ~EFER_LME; to_svm(vcpu)->vmcb->save.efer = efer | MSR_EFER_SVME_MASK; - vcpu->shadow_efer = efer; + vcpu->arch.shadow_efer = efer; } static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, @@ -227,10 +227,10 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) svm->vmcb->save.rip, svm->next_rip); - vcpu->rip = svm->vmcb->save.rip = svm->next_rip; + vcpu->arch.rip = svm->vmcb->save.rip = svm->next_rip; svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK; - vcpu->interrupt_window_open = 1; + vcpu->arch.interrupt_window_open = 1; } static int has_svm(void) @@ -559,8 +559,8 @@ static int svm_vcpu_reset(struct kvm_vcpu *vcpu) if (vcpu->vcpu_id != 0) { svm->vmcb->save.rip = 0; - svm->vmcb->save.cs.base = svm->vcpu.sipi_vector << 12; - svm->vmcb->save.cs.selector = svm->vcpu.sipi_vector << 8; + svm->vmcb->save.cs.base = svm->vcpu.arch.sipi_vector << 12; + svm->vmcb->save.cs.selector = svm->vcpu.arch.sipi_vector << 8; } return 0; @@ -597,9 +597,9 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id) fx_init(&svm->vcpu); svm->vcpu.fpu_active = 1; - svm->vcpu.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE; + svm->vcpu.arch.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE; if (svm->vcpu.vcpu_id == 0) - svm->vcpu.apic_base |= MSR_IA32_APICBASE_BSP; + svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP; return &svm->vcpu; @@ -633,7 +633,7 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) * increasing TSC. */ rdtscll(tsc_this); - delta = vcpu->host_tsc - tsc_this; + delta = vcpu->arch.host_tsc - tsc_this; svm->vmcb->control.tsc_offset += delta; vcpu->cpu = cpu; kvm_migrate_apic_timer(vcpu); @@ -652,7 +652,7 @@ static void svm_vcpu_put(struct kvm_vcpu *vcpu) for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); - rdtscll(vcpu->host_tsc); + rdtscll(vcpu->arch.host_tsc); } static void svm_vcpu_decache(struct kvm_vcpu *vcpu) @@ -663,17 +663,17 @@ static void svm_cache_regs(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - vcpu->regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; - vcpu->regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; - vcpu->rip = svm->vmcb->save.rip; + vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; + vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; + vcpu->arch.rip = svm->vmcb->save.rip; } static void svm_decache_regs(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - svm->vmcb->save.rax = vcpu->regs[VCPU_REGS_RAX]; - svm->vmcb->save.rsp = vcpu->regs[VCPU_REGS_RSP]; - svm->vmcb->save.rip = vcpu->rip; + svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; + svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; + svm->vmcb->save.rip = vcpu->arch.rip; } static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu) @@ -771,24 +771,24 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) struct vcpu_svm *svm = to_svm(vcpu); #ifdef CONFIG_X86_64 - if (vcpu->shadow_efer & EFER_LME) { + if (vcpu->arch.shadow_efer & EFER_LME) { if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { - vcpu->shadow_efer |= EFER_LMA; + vcpu->arch.shadow_efer |= EFER_LMA; svm->vmcb->save.efer |= EFER_LMA | EFER_LME; } if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) { - vcpu->shadow_efer &= ~EFER_LMA; + vcpu->arch.shadow_efer &= ~EFER_LMA; svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME); } } #endif - if ((vcpu->cr0 & X86_CR0_TS) && !(cr0 & X86_CR0_TS)) { + if ((vcpu->arch.cr0 & X86_CR0_TS) && !(cr0 & X86_CR0_TS)) { svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR); vcpu->fpu_active = 1; } - vcpu->cr0 = cr0; + vcpu->arch.cr0 = cr0; cr0 |= X86_CR0_PG | X86_CR0_WP; cr0 &= ~(X86_CR0_CD | X86_CR0_NW); svm->vmcb->save.cr0 = cr0; @@ -796,7 +796,7 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) { - vcpu->cr4 = cr4; + vcpu->arch.cr4 = cr4; to_svm(vcpu)->vmcb->save.cr4 = cr4 | X86_CR4_PAE; } @@ -901,7 +901,7 @@ static void svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value, svm->db_regs[dr] = value; return; case 4 ... 5: - if (vcpu->cr4 & X86_CR4_DE) { + if (vcpu->arch.cr4 & X86_CR4_DE) { *exception = UD_VECTOR; return; } @@ -950,7 +950,7 @@ static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) static int nm_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR); - if (!(svm->vcpu.cr0 & X86_CR0_TS)) + if (!(svm->vcpu.arch.cr0 & X86_CR0_TS)) svm->vmcb->save.cr0 &= ~X86_CR0_TS; svm->vcpu.fpu_active = 1; @@ -1103,14 +1103,14 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data) static int rdmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { - u32 ecx = svm->vcpu.regs[VCPU_REGS_RCX]; + u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; u64 data; if (svm_get_msr(&svm->vcpu, ecx, &data)) kvm_inject_gp(&svm->vcpu, 0); else { svm->vmcb->save.rax = data & 0xffffffff; - svm->vcpu.regs[VCPU_REGS_RDX] = data >> 32; + svm->vcpu.arch.regs[VCPU_REGS_RDX] = data >> 32; svm->next_rip = svm->vmcb->save.rip + 2; skip_emulated_instruction(&svm->vcpu); } @@ -1176,9 +1176,9 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data) static int wrmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { - u32 ecx = svm->vcpu.regs[VCPU_REGS_RCX]; + u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; u64 data = (svm->vmcb->save.rax & -1u) - | ((u64)(svm->vcpu.regs[VCPU_REGS_RDX] & -1u) << 32); + | ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32); svm->next_rip = svm->vmcb->save.rip + 2; if (svm_set_msr(&svm->vcpu, ecx, data)) kvm_inject_gp(&svm->vcpu, 0); @@ -1205,7 +1205,7 @@ static int interrupt_window_interception(struct vcpu_svm *svm, * possible */ if (kvm_run->request_interrupt_window && - !svm->vcpu.irq_summary) { + !svm->vcpu.arch.irq_summary) { ++svm->vcpu.stat.irq_window_exits; kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; return 0; @@ -1382,20 +1382,20 @@ static void kvm_reput_irq(struct vcpu_svm *svm) push_irq(&svm->vcpu, control->int_vector); } - svm->vcpu.interrupt_window_open = + svm->vcpu.arch.interrupt_window_open = !(control->int_state & SVM_INTERRUPT_SHADOW_MASK); } static void svm_do_inject_vector(struct vcpu_svm *svm) { struct kvm_vcpu *vcpu = &svm->vcpu; - int word_index = __ffs(vcpu->irq_summary); - int bit_index = __ffs(vcpu->irq_pending[word_index]); + int word_index = __ffs(vcpu->arch.irq_summary); + int bit_index = __ffs(vcpu->arch.irq_pending[word_index]); int irq = word_index * BITS_PER_LONG + bit_index; - clear_bit(bit_index, &vcpu->irq_pending[word_index]); - if (!vcpu->irq_pending[word_index]) - clear_bit(word_index, &vcpu->irq_summary); + clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]); + if (!vcpu->arch.irq_pending[word_index]) + clear_bit(word_index, &vcpu->arch.irq_summary); svm_inject_irq(svm, irq); } @@ -1405,11 +1405,11 @@ static void do_interrupt_requests(struct kvm_vcpu *vcpu, struct vcpu_svm *svm = to_svm(vcpu); struct vmcb_control_area *control = &svm->vmcb->control; - svm->vcpu.interrupt_window_open = + svm->vcpu.arch.interrupt_window_open = (!(control->int_state & SVM_INTERRUPT_SHADOW_MASK) && (svm->vmcb->save.rflags & X86_EFLAGS_IF)); - if (svm->vcpu.interrupt_window_open && svm->vcpu.irq_summary) + if (svm->vcpu.arch.interrupt_window_open && svm->vcpu.arch.irq_summary) /* * If interrupts enabled, and not blocked by sti or mov ss. Good. */ @@ -1418,8 +1418,8 @@ static void do_interrupt_requests(struct kvm_vcpu *vcpu, /* * Interrupts blocked. Wait for unblock. */ - if (!svm->vcpu.interrupt_window_open && - (svm->vcpu.irq_summary || kvm_run->request_interrupt_window)) + if (!svm->vcpu.arch.interrupt_window_open && + (svm->vcpu.arch.irq_summary || kvm_run->request_interrupt_window)) control->intercept |= 1ULL << INTERCEPT_VINTR; else control->intercept &= ~(1ULL << INTERCEPT_VINTR); @@ -1471,7 +1471,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) svm->host_cr2 = kvm_read_cr2(); svm->host_dr6 = read_dr6(); svm->host_dr7 = read_dr7(); - svm->vmcb->save.cr2 = vcpu->cr2; + svm->vmcb->save.cr2 = vcpu->arch.cr2; if (svm->vmcb->save.dr7 & 0xff) { write_dr7(0); @@ -1563,21 +1563,21 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) : : [svm]"a"(svm), [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)), - [rbx]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RBX])), - [rcx]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RCX])), - [rdx]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RDX])), - [rsi]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RSI])), - [rdi]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RDI])), - [rbp]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RBP])) + [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])), + [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])), + [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])), + [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])), + [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])), + [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP])) #ifdef CONFIG_X86_64 - , [r8]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R8])), - [r9]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R9])), - [r10]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R10])), - [r11]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R11])), - [r12]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R12])), - [r13]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R13])), - [r14]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R14])), - [r15]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R15])) + , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])), + [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])), + [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])), + [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])), + [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])), + [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])), + [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])), + [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15])) #endif : "cc", "memory" #ifdef CONFIG_X86_64 @@ -1591,7 +1591,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) if ((svm->vmcb->save.dr7 & 0xff)) load_db_regs(svm->host_db_regs); - vcpu->cr2 = svm->vmcb->save.cr2; + vcpu->arch.cr2 = svm->vmcb->save.cr2; write_dr6(svm->host_dr6); write_dr7(svm->host_dr7); diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 83084348581a..cf78ebb2f36e 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -247,7 +247,7 @@ static void __vcpu_clear(void *arg) vmcs_clear(vmx->vmcs); if (per_cpu(current_vmcs, cpu) == vmx->vmcs) per_cpu(current_vmcs, cpu) = NULL; - rdtscll(vmx->vcpu.host_tsc); + rdtscll(vmx->vcpu.arch.host_tsc); } static void vcpu_clear(struct vcpu_vmx *vmx) @@ -343,7 +343,7 @@ static void update_exception_bitmap(struct kvm_vcpu *vcpu) eb |= 1u << NM_VECTOR; if (vcpu->guest_debug.enabled) eb |= 1u << 1; - if (vcpu->rmode.active) + if (vcpu->arch.rmode.active) eb = ~0; vmcs_write32(EXCEPTION_BITMAP, eb); } @@ -528,7 +528,7 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) * Make sure the time stamp counter is monotonous. */ rdtscll(tsc_this); - delta = vcpu->host_tsc - tsc_this; + delta = vcpu->arch.host_tsc - tsc_this; vmcs_write64(TSC_OFFSET, vmcs_read64(TSC_OFFSET) + delta); } } @@ -544,7 +544,7 @@ static void vmx_fpu_activate(struct kvm_vcpu *vcpu) return; vcpu->fpu_active = 1; vmcs_clear_bits(GUEST_CR0, X86_CR0_TS); - if (vcpu->cr0 & X86_CR0_TS) + if (vcpu->arch.cr0 & X86_CR0_TS) vmcs_set_bits(GUEST_CR0, X86_CR0_TS); update_exception_bitmap(vcpu); } @@ -570,7 +570,7 @@ static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu) static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) { - if (vcpu->rmode.active) + if (vcpu->arch.rmode.active) rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; vmcs_writel(GUEST_RFLAGS, rflags); } @@ -592,7 +592,7 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) if (interruptibility & 3) vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility & ~3); - vcpu->interrupt_window_open = 1; + vcpu->arch.interrupt_window_open = 1; } static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, @@ -661,7 +661,7 @@ static void setup_msrs(struct vcpu_vmx *vmx) * if efer.sce is enabled. */ index = __find_msr_index(vmx, MSR_K6_STAR); - if ((index >= 0) && (vmx->vcpu.shadow_efer & EFER_SCE)) + if ((index >= 0) && (vmx->vcpu.arch.shadow_efer & EFER_SCE)) move_msr_up(vmx, index, save_nmsrs++); } #endif @@ -805,12 +805,12 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) /* * Sync the rsp and rip registers into the vcpu structure. This allows - * registers to be accessed by indexing vcpu->regs. + * registers to be accessed by indexing vcpu->arch.regs. */ static void vcpu_load_rsp_rip(struct kvm_vcpu *vcpu) { - vcpu->regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP); - vcpu->rip = vmcs_readl(GUEST_RIP); + vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP); + vcpu->arch.rip = vmcs_readl(GUEST_RIP); } /* @@ -819,8 +819,8 @@ static void vcpu_load_rsp_rip(struct kvm_vcpu *vcpu) */ static void vcpu_put_rsp_rip(struct kvm_vcpu *vcpu) { - vmcs_writel(GUEST_RSP, vcpu->regs[VCPU_REGS_RSP]); - vmcs_writel(GUEST_RIP, vcpu->rip); + vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]); + vmcs_writel(GUEST_RIP, vcpu->arch.rip); } static int set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg) @@ -1111,15 +1111,15 @@ static void enter_pmode(struct kvm_vcpu *vcpu) { unsigned long flags; - vcpu->rmode.active = 0; + vcpu->arch.rmode.active = 0; - vmcs_writel(GUEST_TR_BASE, vcpu->rmode.tr.base); - vmcs_write32(GUEST_TR_LIMIT, vcpu->rmode.tr.limit); - vmcs_write32(GUEST_TR_AR_BYTES, vcpu->rmode.tr.ar); + vmcs_writel(GUEST_TR_BASE, vcpu->arch.rmode.tr.base); + vmcs_write32(GUEST_TR_LIMIT, vcpu->arch.rmode.tr.limit); + vmcs_write32(GUEST_TR_AR_BYTES, vcpu->arch.rmode.tr.ar); flags = vmcs_readl(GUEST_RFLAGS); flags &= ~(X86_EFLAGS_IOPL | X86_EFLAGS_VM); - flags |= (vcpu->rmode.save_iopl << IOPL_SHIFT); + flags |= (vcpu->arch.rmode.save_iopl << IOPL_SHIFT); vmcs_writel(GUEST_RFLAGS, flags); vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) | @@ -1127,10 +1127,10 @@ static void enter_pmode(struct kvm_vcpu *vcpu) update_exception_bitmap(vcpu); - fix_pmode_dataseg(VCPU_SREG_ES, &vcpu->rmode.es); - fix_pmode_dataseg(VCPU_SREG_DS, &vcpu->rmode.ds); - fix_pmode_dataseg(VCPU_SREG_GS, &vcpu->rmode.gs); - fix_pmode_dataseg(VCPU_SREG_FS, &vcpu->rmode.fs); + fix_pmode_dataseg(VCPU_SREG_ES, &vcpu->arch.rmode.es); + fix_pmode_dataseg(VCPU_SREG_DS, &vcpu->arch.rmode.ds); + fix_pmode_dataseg(VCPU_SREG_GS, &vcpu->arch.rmode.gs); + fix_pmode_dataseg(VCPU_SREG_FS, &vcpu->arch.rmode.fs); vmcs_write16(GUEST_SS_SELECTOR, 0); vmcs_write32(GUEST_SS_AR_BYTES, 0x93); @@ -1168,19 +1168,20 @@ static void enter_rmode(struct kvm_vcpu *vcpu) { unsigned long flags; - vcpu->rmode.active = 1; + vcpu->arch.rmode.active = 1; - vcpu->rmode.tr.base = vmcs_readl(GUEST_TR_BASE); + vcpu->arch.rmode.tr.base = vmcs_readl(GUEST_TR_BASE); vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm)); - vcpu->rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT); + vcpu->arch.rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT); vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1); - vcpu->rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES); + vcpu->arch.rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES); vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); flags = vmcs_readl(GUEST_RFLAGS); - vcpu->rmode.save_iopl = (flags & X86_EFLAGS_IOPL) >> IOPL_SHIFT; + vcpu->arch.rmode.save_iopl + = (flags & X86_EFLAGS_IOPL) >> IOPL_SHIFT; flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; @@ -1198,10 +1199,10 @@ static void enter_rmode(struct kvm_vcpu *vcpu) vmcs_writel(GUEST_CS_BASE, 0xf0000); vmcs_write16(GUEST_CS_SELECTOR, vmcs_readl(GUEST_CS_BASE) >> 4); - fix_rmode_seg(VCPU_SREG_ES, &vcpu->rmode.es); - fix_rmode_seg(VCPU_SREG_DS, &vcpu->rmode.ds); - fix_rmode_seg(VCPU_SREG_GS, &vcpu->rmode.gs); - fix_rmode_seg(VCPU_SREG_FS, &vcpu->rmode.fs); + fix_rmode_seg(VCPU_SREG_ES, &vcpu->arch.rmode.es); + fix_rmode_seg(VCPU_SREG_DS, &vcpu->arch.rmode.ds); + fix_rmode_seg(VCPU_SREG_GS, &vcpu->arch.rmode.gs); + fix_rmode_seg(VCPU_SREG_FS, &vcpu->arch.rmode.fs); kvm_mmu_reset_context(vcpu); init_rmode_tss(vcpu->kvm); @@ -1222,7 +1223,7 @@ static void enter_lmode(struct kvm_vcpu *vcpu) | AR_TYPE_BUSY_64_TSS); } - vcpu->shadow_efer |= EFER_LMA; + vcpu->arch.shadow_efer |= EFER_LMA; find_msr_entry(to_vmx(vcpu), MSR_EFER)->data |= EFER_LMA | EFER_LME; vmcs_write32(VM_ENTRY_CONTROLS, @@ -1232,7 +1233,7 @@ static void enter_lmode(struct kvm_vcpu *vcpu) static void exit_lmode(struct kvm_vcpu *vcpu) { - vcpu->shadow_efer &= ~EFER_LMA; + vcpu->arch.shadow_efer &= ~EFER_LMA; vmcs_write32(VM_ENTRY_CONTROLS, vmcs_read32(VM_ENTRY_CONTROLS) @@ -1243,22 +1244,22 @@ static void exit_lmode(struct kvm_vcpu *vcpu) static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu) { - vcpu->cr4 &= KVM_GUEST_CR4_MASK; - vcpu->cr4 |= vmcs_readl(GUEST_CR4) & ~KVM_GUEST_CR4_MASK; + vcpu->arch.cr4 &= KVM_GUEST_CR4_MASK; + vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & ~KVM_GUEST_CR4_MASK; } static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) { vmx_fpu_deactivate(vcpu); - if (vcpu->rmode.active && (cr0 & X86_CR0_PE)) + if (vcpu->arch.rmode.active && (cr0 & X86_CR0_PE)) enter_pmode(vcpu); - if (!vcpu->rmode.active && !(cr0 & X86_CR0_PE)) + if (!vcpu->arch.rmode.active && !(cr0 & X86_CR0_PE)) enter_rmode(vcpu); #ifdef CONFIG_X86_64 - if (vcpu->shadow_efer & EFER_LME) { + if (vcpu->arch.shadow_efer & EFER_LME) { if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) enter_lmode(vcpu); if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) @@ -1269,7 +1270,7 @@ static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) vmcs_writel(CR0_READ_SHADOW, cr0); vmcs_writel(GUEST_CR0, (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON); - vcpu->cr0 = cr0; + vcpu->arch.cr0 = cr0; if (!(cr0 & X86_CR0_TS) || !(cr0 & X86_CR0_PE)) vmx_fpu_activate(vcpu); @@ -1278,16 +1279,16 @@ static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) { vmcs_writel(GUEST_CR3, cr3); - if (vcpu->cr0 & X86_CR0_PE) + if (vcpu->arch.cr0 & X86_CR0_PE) vmx_fpu_deactivate(vcpu); } static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) { vmcs_writel(CR4_READ_SHADOW, cr4); - vmcs_writel(GUEST_CR4, cr4 | (vcpu->rmode.active ? + vmcs_writel(GUEST_CR4, cr4 | (vcpu->arch.rmode.active ? KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON)); - vcpu->cr4 = cr4; + vcpu->arch.cr4 = cr4; } #ifdef CONFIG_X86_64 @@ -1297,7 +1298,7 @@ static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer) struct vcpu_vmx *vmx = to_vmx(vcpu); struct kvm_msr_entry *msr = find_msr_entry(vmx, MSR_EFER); - vcpu->shadow_efer = efer; + vcpu->arch.shadow_efer = efer; if (efer & EFER_LMA) { vmcs_write32(VM_ENTRY_CONTROLS, vmcs_read32(VM_ENTRY_CONTROLS) | @@ -1374,17 +1375,17 @@ static void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; u32 ar; - if (vcpu->rmode.active && seg == VCPU_SREG_TR) { - vcpu->rmode.tr.selector = var->selector; - vcpu->rmode.tr.base = var->base; - vcpu->rmode.tr.limit = var->limit; - vcpu->rmode.tr.ar = vmx_segment_access_rights(var); + if (vcpu->arch.rmode.active && seg == VCPU_SREG_TR) { + vcpu->arch.rmode.tr.selector = var->selector; + vcpu->arch.rmode.tr.base = var->base; + vcpu->arch.rmode.tr.limit = var->limit; + vcpu->arch.rmode.tr.ar = vmx_segment_access_rights(var); return; } vmcs_writel(sf->base, var->base); vmcs_write32(sf->limit, var->limit); vmcs_write16(sf->selector, var->selector); - if (vcpu->rmode.active && var->s) { + if (vcpu->arch.rmode.active && var->s) { /* * Hack real-mode segments into vm86 compatibility. */ @@ -1613,9 +1614,9 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) goto out; } - vmx->vcpu.rmode.active = 0; + vmx->vcpu.arch.rmode.active = 0; - vmx->vcpu.regs[VCPU_REGS_RDX] = get_rdx_init_val(); + vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val(); set_cr8(&vmx->vcpu, 0); msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE; if (vmx->vcpu.vcpu_id == 0) @@ -1632,8 +1633,8 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) vmcs_write16(GUEST_CS_SELECTOR, 0xf000); vmcs_writel(GUEST_CS_BASE, 0x000f0000); } else { - vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.sipi_vector << 8); - vmcs_writel(GUEST_CS_BASE, vmx->vcpu.sipi_vector << 12); + vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.arch.sipi_vector << 8); + vmcs_writel(GUEST_CS_BASE, vmx->vcpu.arch.sipi_vector << 12); } vmcs_write32(GUEST_CS_LIMIT, 0xffff); vmcs_write32(GUEST_CS_AR_BYTES, 0x9b); @@ -1691,7 +1692,7 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); if (vm_need_tpr_shadow(vmx->vcpu.kvm)) vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, - page_to_phys(vmx->vcpu.apic->regs_page)); + page_to_phys(vmx->vcpu.arch.apic->regs_page)); vmcs_write32(TPR_THRESHOLD, 0); } @@ -1699,8 +1700,8 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) vmcs_write64(APIC_ACCESS_ADDR, page_to_phys(vmx->vcpu.kvm->apic_access_page)); - vmx->vcpu.cr0 = 0x60000010; - vmx_set_cr0(&vmx->vcpu, vmx->vcpu.cr0); /* enter rmode */ + vmx->vcpu.arch.cr0 = 0x60000010; + vmx_set_cr0(&vmx->vcpu, vmx->vcpu.arch.cr0); /* enter rmode */ vmx_set_cr4(&vmx->vcpu, 0); #ifdef CONFIG_X86_64 vmx_set_efer(&vmx->vcpu, 0); @@ -1718,7 +1719,7 @@ static void vmx_inject_irq(struct kvm_vcpu *vcpu, int irq) { struct vcpu_vmx *vmx = to_vmx(vcpu); - if (vcpu->rmode.active) { + if (vcpu->arch.rmode.active) { vmx->rmode.irq.pending = true; vmx->rmode.irq.vector = irq; vmx->rmode.irq.rip = vmcs_readl(GUEST_RIP); @@ -1734,13 +1735,13 @@ static void vmx_inject_irq(struct kvm_vcpu *vcpu, int irq) static void kvm_do_inject_irq(struct kvm_vcpu *vcpu) { - int word_index = __ffs(vcpu->irq_summary); - int bit_index = __ffs(vcpu->irq_pending[word_index]); + int word_index = __ffs(vcpu->arch.irq_summary); + int bit_index = __ffs(vcpu->arch.irq_pending[word_index]); int irq = word_index * BITS_PER_LONG + bit_index; - clear_bit(bit_index, &vcpu->irq_pending[word_index]); - if (!vcpu->irq_pending[word_index]) - clear_bit(word_index, &vcpu->irq_summary); + clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]); + if (!vcpu->arch.irq_pending[word_index]) + clear_bit(word_index, &vcpu->arch.irq_summary); vmx_inject_irq(vcpu, irq); } @@ -1750,12 +1751,12 @@ static void do_interrupt_requests(struct kvm_vcpu *vcpu, { u32 cpu_based_vm_exec_control; - vcpu->interrupt_window_open = + vcpu->arch.interrupt_window_open = ((vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) && (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0); - if (vcpu->interrupt_window_open && - vcpu->irq_summary && + if (vcpu->arch.interrupt_window_open && + vcpu->arch.irq_summary && !(vmcs_read32(VM_ENTRY_INTR_INFO_FIELD) & INTR_INFO_VALID_MASK)) /* * If interrupts enabled, and not blocked by sti or mov ss. Good. @@ -1763,8 +1764,8 @@ static void do_interrupt_requests(struct kvm_vcpu *vcpu, kvm_do_inject_irq(vcpu); cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); - if (!vcpu->interrupt_window_open && - (vcpu->irq_summary || kvm_run->request_interrupt_window)) + if (!vcpu->arch.interrupt_window_open && + (vcpu->arch.irq_summary || kvm_run->request_interrupt_window)) /* * Interrupts blocked. Wait for unblock. */ @@ -1812,7 +1813,7 @@ static void kvm_guest_debug_pre(struct kvm_vcpu *vcpu) static int handle_rmode_exception(struct kvm_vcpu *vcpu, int vec, u32 err_code) { - if (!vcpu->rmode.active) + if (!vcpu->arch.rmode.active) return 0; /* @@ -1843,8 +1844,8 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) if (!irqchip_in_kernel(vcpu->kvm) && is_external_interrupt(vect_info)) { int irq = vect_info & VECTORING_INFO_VECTOR_MASK; - set_bit(irq, vcpu->irq_pending); - set_bit(irq / BITS_PER_LONG, &vcpu->irq_summary); + set_bit(irq, vcpu->arch.irq_pending); + set_bit(irq / BITS_PER_LONG, &vcpu->arch.irq_summary); } if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == 0x200) /* nmi */ @@ -1871,11 +1872,11 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) return kvm_mmu_page_fault(vcpu, cr2, error_code); } - if (vcpu->rmode.active && + if (vcpu->arch.rmode.active && handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK, error_code)) { - if (vcpu->halt_request) { - vcpu->halt_request = 0; + if (vcpu->arch.halt_request) { + vcpu->arch.halt_request = 0; return kvm_emulate_halt(vcpu); } return 1; @@ -1956,22 +1957,22 @@ static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) switch (cr) { case 0: vcpu_load_rsp_rip(vcpu); - set_cr0(vcpu, vcpu->regs[reg]); + set_cr0(vcpu, vcpu->arch.regs[reg]); skip_emulated_instruction(vcpu); return 1; case 3: vcpu_load_rsp_rip(vcpu); - set_cr3(vcpu, vcpu->regs[reg]); + set_cr3(vcpu, vcpu->arch.regs[reg]); skip_emulated_instruction(vcpu); return 1; case 4: vcpu_load_rsp_rip(vcpu); - set_cr4(vcpu, vcpu->regs[reg]); + set_cr4(vcpu, vcpu->arch.regs[reg]); skip_emulated_instruction(vcpu); return 1; case 8: vcpu_load_rsp_rip(vcpu); - set_cr8(vcpu, vcpu->regs[reg]); + set_cr8(vcpu, vcpu->arch.regs[reg]); skip_emulated_instruction(vcpu); if (irqchip_in_kernel(vcpu->kvm)) return 1; @@ -1982,8 +1983,8 @@ static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) case 2: /* clts */ vcpu_load_rsp_rip(vcpu); vmx_fpu_deactivate(vcpu); - vcpu->cr0 &= ~X86_CR0_TS; - vmcs_writel(CR0_READ_SHADOW, vcpu->cr0); + vcpu->arch.cr0 &= ~X86_CR0_TS; + vmcs_writel(CR0_READ_SHADOW, vcpu->arch.cr0); vmx_fpu_activate(vcpu); skip_emulated_instruction(vcpu); return 1; @@ -1991,13 +1992,13 @@ static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) switch (cr) { case 3: vcpu_load_rsp_rip(vcpu); - vcpu->regs[reg] = vcpu->cr3; + vcpu->arch.regs[reg] = vcpu->arch.cr3; vcpu_put_rsp_rip(vcpu); skip_emulated_instruction(vcpu); return 1; case 8: vcpu_load_rsp_rip(vcpu); - vcpu->regs[reg] = get_cr8(vcpu); + vcpu->arch.regs[reg] = get_cr8(vcpu); vcpu_put_rsp_rip(vcpu); skip_emulated_instruction(vcpu); return 1; @@ -2043,7 +2044,7 @@ static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) default: val = 0; } - vcpu->regs[reg] = val; + vcpu->arch.regs[reg] = val; } else { /* mov to dr */ } @@ -2060,7 +2061,7 @@ static int handle_cpuid(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) static int handle_rdmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { - u32 ecx = vcpu->regs[VCPU_REGS_RCX]; + u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX]; u64 data; if (vmx_get_msr(vcpu, ecx, &data)) { @@ -2069,17 +2070,17 @@ static int handle_rdmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) } /* FIXME: handling of bits 32:63 of rax, rdx */ - vcpu->regs[VCPU_REGS_RAX] = data & -1u; - vcpu->regs[VCPU_REGS_RDX] = (data >> 32) & -1u; + vcpu->arch.regs[VCPU_REGS_RAX] = data & -1u; + vcpu->arch.regs[VCPU_REGS_RDX] = (data >> 32) & -1u; skip_emulated_instruction(vcpu); return 1; } static int handle_wrmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { - u32 ecx = vcpu->regs[VCPU_REGS_RCX]; - u64 data = (vcpu->regs[VCPU_REGS_RAX] & -1u) - | ((u64)(vcpu->regs[VCPU_REGS_RDX] & -1u) << 32); + u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX]; + u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u) + | ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32); if (vmx_set_msr(vcpu, ecx, data) != 0) { kvm_inject_gp(vcpu, 0); @@ -2110,7 +2111,7 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu, * possible */ if (kvm_run->request_interrupt_window && - !vcpu->irq_summary) { + !vcpu->arch.irq_summary) { kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; ++vcpu->stat.irq_window_exits; return 0; @@ -2270,7 +2271,7 @@ static void vmx_intr_assist(struct kvm_vcpu *vcpu) if (unlikely(idtv_info_field & INTR_INFO_VALID_MASK)) { if ((idtv_info_field & VECTORING_INFO_TYPE_MASK) == INTR_TYPE_EXT_INTR - && vcpu->rmode.active) { + && vcpu->arch.rmode.active) { u8 vect = idtv_info_field & VECTORING_INFO_VECTOR_MASK; vmx_inject_irq(vcpu, vect); @@ -2424,24 +2425,24 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) : : "c"(vmx), "d"((unsigned long)HOST_RSP), [launched]"i"(offsetof(struct vcpu_vmx, launched)), [fail]"i"(offsetof(struct vcpu_vmx, fail)), - [rax]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RAX])), - [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RBX])), - [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RCX])), - [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RDX])), - [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RSI])), - [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RDI])), - [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RBP])), + [rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])), + [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])), + [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])), + [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])), + [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])), + [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])), + [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])), #ifdef CONFIG_X86_64 - [r8]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R8])), - [r9]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R9])), - [r10]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R10])), - [r11]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R11])), - [r12]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R12])), - [r13]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R13])), - [r14]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R14])), - [r15]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R15])), + [r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])), + [r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])), + [r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])), + [r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])), + [r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])), + [r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])), + [r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])), + [r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])), #endif - [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.cr2)) + [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)) : "cc", "memory" #ifdef CONFIG_X86_64 , "rbx", "rdi", "rsi" @@ -2455,7 +2456,7 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) if (vmx->rmode.irq.pending) fixup_rmode_irq(vmx); - vcpu->interrupt_window_open = + vcpu->arch.interrupt_window_open = (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0; asm("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS)); diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 3b79684a3c0c..5a2f33a84e4f 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -113,9 +113,9 @@ EXPORT_SYMBOL_GPL(segment_base); u64 kvm_get_apic_base(struct kvm_vcpu *vcpu) { if (irqchip_in_kernel(vcpu->kvm)) - return vcpu->apic_base; + return vcpu->arch.apic_base; else - return vcpu->apic_base; + return vcpu->arch.apic_base; } EXPORT_SYMBOL_GPL(kvm_get_apic_base); @@ -125,16 +125,16 @@ void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data) if (irqchip_in_kernel(vcpu->kvm)) kvm_lapic_set_base(vcpu, data); else - vcpu->apic_base = data; + vcpu->arch.apic_base = data; } EXPORT_SYMBOL_GPL(kvm_set_apic_base); void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) { - WARN_ON(vcpu->exception.pending); - vcpu->exception.pending = true; - vcpu->exception.has_error_code = false; - vcpu->exception.nr = nr; + WARN_ON(vcpu->arch.exception.pending); + vcpu->arch.exception.pending = true; + vcpu->arch.exception.has_error_code = false; + vcpu->arch.exception.nr = nr; } EXPORT_SYMBOL_GPL(kvm_queue_exception); @@ -142,32 +142,32 @@ void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr, u32 error_code) { ++vcpu->stat.pf_guest; - if (vcpu->exception.pending && vcpu->exception.nr == PF_VECTOR) { + if (vcpu->arch.exception.pending && vcpu->arch.exception.nr == PF_VECTOR) { printk(KERN_DEBUG "kvm: inject_page_fault:" " double fault 0x%lx\n", addr); - vcpu->exception.nr = DF_VECTOR; - vcpu->exception.error_code = 0; + vcpu->arch.exception.nr = DF_VECTOR; + vcpu->arch.exception.error_code = 0; return; } - vcpu->cr2 = addr; + vcpu->arch.cr2 = addr; kvm_queue_exception_e(vcpu, PF_VECTOR, error_code); } void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) { - WARN_ON(vcpu->exception.pending); - vcpu->exception.pending = true; - vcpu->exception.has_error_code = true; - vcpu->exception.nr = nr; - vcpu->exception.error_code = error_code; + WARN_ON(vcpu->arch.exception.pending); + vcpu->arch.exception.pending = true; + vcpu->arch.exception.has_error_code = true; + vcpu->arch.exception.nr = nr; + vcpu->arch.exception.error_code = error_code; } EXPORT_SYMBOL_GPL(kvm_queue_exception_e); static void __queue_exception(struct kvm_vcpu *vcpu) { - kvm_x86_ops->queue_exception(vcpu, vcpu->exception.nr, - vcpu->exception.has_error_code, - vcpu->exception.error_code); + kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr, + vcpu->arch.exception.has_error_code, + vcpu->arch.exception.error_code); } /* @@ -179,7 +179,7 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2; int i; int ret; - u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)]; + u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)]; mutex_lock(&vcpu->kvm->lock); ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte, @@ -196,7 +196,7 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) } ret = 1; - memcpy(vcpu->pdptrs, pdpte, sizeof(vcpu->pdptrs)); + memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs)); out: mutex_unlock(&vcpu->kvm->lock); @@ -205,7 +205,7 @@ out: static bool pdptrs_changed(struct kvm_vcpu *vcpu) { - u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)]; + u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)]; bool changed = true; int r; @@ -213,10 +213,10 @@ static bool pdptrs_changed(struct kvm_vcpu *vcpu) return false; mutex_lock(&vcpu->kvm->lock); - r = kvm_read_guest(vcpu->kvm, vcpu->cr3 & ~31u, pdpte, sizeof(pdpte)); + r = kvm_read_guest(vcpu->kvm, vcpu->arch.cr3 & ~31u, pdpte, sizeof(pdpte)); if (r < 0) goto out; - changed = memcmp(pdpte, vcpu->pdptrs, sizeof(pdpte)) != 0; + changed = memcmp(pdpte, vcpu->arch.pdptrs, sizeof(pdpte)) != 0; out: mutex_unlock(&vcpu->kvm->lock); @@ -227,7 +227,7 @@ void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) { if (cr0 & CR0_RESERVED_BITS) { printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n", - cr0, vcpu->cr0); + cr0, vcpu->arch.cr0); kvm_inject_gp(vcpu, 0); return; } @@ -247,7 +247,7 @@ void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { #ifdef CONFIG_X86_64 - if ((vcpu->shadow_efer & EFER_LME)) { + if ((vcpu->arch.shadow_efer & EFER_LME)) { int cs_db, cs_l; if (!is_pae(vcpu)) { @@ -266,7 +266,7 @@ void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) } } else #endif - if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) { + if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.cr3)) { printk(KERN_DEBUG "set_cr0: #GP, pdptrs " "reserved bits\n"); kvm_inject_gp(vcpu, 0); @@ -276,7 +276,7 @@ void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) } kvm_x86_ops->set_cr0(vcpu, cr0); - vcpu->cr0 = cr0; + vcpu->arch.cr0 = cr0; mutex_lock(&vcpu->kvm->lock); kvm_mmu_reset_context(vcpu); @@ -287,7 +287,7 @@ EXPORT_SYMBOL_GPL(set_cr0); void lmsw(struct kvm_vcpu *vcpu, unsigned long msw) { - set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f)); + set_cr0(vcpu, (vcpu->arch.cr0 & ~0x0ful) | (msw & 0x0f)); } EXPORT_SYMBOL_GPL(lmsw); @@ -307,7 +307,7 @@ void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) return; } } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE) - && !load_pdptrs(vcpu, vcpu->cr3)) { + && !load_pdptrs(vcpu, vcpu->arch.cr3)) { printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n"); kvm_inject_gp(vcpu, 0); return; @@ -319,7 +319,7 @@ void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) return; } kvm_x86_ops->set_cr4(vcpu, cr4); - vcpu->cr4 = cr4; + vcpu->arch.cr4 = cr4; mutex_lock(&vcpu->kvm->lock); kvm_mmu_reset_context(vcpu); mutex_unlock(&vcpu->kvm->lock); @@ -328,7 +328,7 @@ EXPORT_SYMBOL_GPL(set_cr4); void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) { - if (cr3 == vcpu->cr3 && !pdptrs_changed(vcpu)) { + if (cr3 == vcpu->arch.cr3 && !pdptrs_changed(vcpu)) { kvm_mmu_flush_tlb(vcpu); return; } @@ -373,8 +373,8 @@ void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT))) kvm_inject_gp(vcpu, 0); else { - vcpu->cr3 = cr3; - vcpu->mmu.new_cr3(vcpu); + vcpu->arch.cr3 = cr3; + vcpu->arch.mmu.new_cr3(vcpu); } mutex_unlock(&vcpu->kvm->lock); } @@ -390,7 +390,7 @@ void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) if (irqchip_in_kernel(vcpu->kvm)) kvm_lapic_set_tpr(vcpu, cr8); else - vcpu->cr8 = cr8; + vcpu->arch.cr8 = cr8; } EXPORT_SYMBOL_GPL(set_cr8); @@ -399,7 +399,7 @@ unsigned long get_cr8(struct kvm_vcpu *vcpu) if (irqchip_in_kernel(vcpu->kvm)) return kvm_lapic_get_cr8(vcpu); else - return vcpu->cr8; + return vcpu->arch.cr8; } EXPORT_SYMBOL_GPL(get_cr8); @@ -437,7 +437,7 @@ static void set_efer(struct kvm_vcpu *vcpu, u64 efer) } if (is_paging(vcpu) - && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) { + && (vcpu->arch.shadow_efer & EFER_LME) != (efer & EFER_LME)) { printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n"); kvm_inject_gp(vcpu, 0); return; @@ -446,9 +446,9 @@ static void set_efer(struct kvm_vcpu *vcpu, u64 efer) kvm_x86_ops->set_efer(vcpu, efer); efer &= ~EFER_LMA; - efer |= vcpu->shadow_efer & EFER_LMA; + efer |= vcpu->arch.shadow_efer & EFER_LMA; - vcpu->shadow_efer = efer; + vcpu->arch.shadow_efer = efer; } #endif @@ -496,7 +496,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) kvm_set_apic_base(vcpu, data); break; case MSR_IA32_MISC_ENABLE: - vcpu->ia32_misc_enable_msr = data; + vcpu->arch.ia32_misc_enable_msr = data; break; default: pr_unimpl(vcpu, "unhandled wrmsr: 0x%x\n", msr); @@ -550,11 +550,11 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) data = kvm_get_apic_base(vcpu); break; case MSR_IA32_MISC_ENABLE: - data = vcpu->ia32_misc_enable_msr; + data = vcpu->arch.ia32_misc_enable_msr; break; #ifdef CONFIG_X86_64 case MSR_EFER: - data = vcpu->shadow_efer; + data = vcpu->arch.shadow_efer; break; #endif default: @@ -760,8 +760,8 @@ static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) struct kvm_cpuid_entry2 *e, *entry; entry = NULL; - for (i = 0; i < vcpu->cpuid_nent; ++i) { - e = &vcpu->cpuid_entries[i]; + for (i = 0; i < vcpu->arch.cpuid_nent; ++i) { + e = &vcpu->arch.cpuid_entries[i]; if (e->function == 0x80000001) { entry = e; break; @@ -793,18 +793,18 @@ static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, cpuid->nent * sizeof(struct kvm_cpuid_entry))) goto out_free; for (i = 0; i < cpuid->nent; i++) { - vcpu->cpuid_entries[i].function = cpuid_entries[i].function; - vcpu->cpuid_entries[i].eax = cpuid_entries[i].eax; - vcpu->cpuid_entries[i].ebx = cpuid_entries[i].ebx; - vcpu->cpuid_entries[i].ecx = cpuid_entries[i].ecx; - vcpu->cpuid_entries[i].edx = cpuid_entries[i].edx; - vcpu->cpuid_entries[i].index = 0; - vcpu->cpuid_entries[i].flags = 0; - vcpu->cpuid_entries[i].padding[0] = 0; - vcpu->cpuid_entries[i].padding[1] = 0; - vcpu->cpuid_entries[i].padding[2] = 0; + vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function; + vcpu->arch.cpuid_entries[i].eax = cpuid_entries[i].eax; + vcpu->arch.cpuid_entries[i].ebx = cpuid_entries[i].ebx; + vcpu->arch.cpuid_entries[i].ecx = cpuid_entries[i].ecx; + vcpu->arch.cpuid_entries[i].edx = cpuid_entries[i].edx; + vcpu->arch.cpuid_entries[i].index = 0; + vcpu->arch.cpuid_entries[i].flags = 0; + vcpu->arch.cpuid_entries[i].padding[0] = 0; + vcpu->arch.cpuid_entries[i].padding[1] = 0; + vcpu->arch.cpuid_entries[i].padding[2] = 0; } - vcpu->cpuid_nent = cpuid->nent; + vcpu->arch.cpuid_nent = cpuid->nent; cpuid_fix_nx_cap(vcpu); r = 0; @@ -824,10 +824,10 @@ static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu, if (cpuid->nent > KVM_MAX_CPUID_ENTRIES) goto out; r = -EFAULT; - if (copy_from_user(&vcpu->cpuid_entries, entries, + if (copy_from_user(&vcpu->arch.cpuid_entries, entries, cpuid->nent * sizeof(struct kvm_cpuid_entry2))) goto out; - vcpu->cpuid_nent = cpuid->nent; + vcpu->arch.cpuid_nent = cpuid->nent; return 0; out: @@ -841,16 +841,16 @@ static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu, int r; r = -E2BIG; - if (cpuid->nent < vcpu->cpuid_nent) + if (cpuid->nent < vcpu->arch.cpuid_nent) goto out; r = -EFAULT; - if (copy_to_user(entries, &vcpu->cpuid_entries, - vcpu->cpuid_nent * sizeof(struct kvm_cpuid_entry2))) + if (copy_to_user(entries, &vcpu->arch.cpuid_entries, + vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2))) goto out; return 0; out: - cpuid->nent = vcpu->cpuid_nent; + cpuid->nent = vcpu->arch.cpuid_nent; return r; } @@ -1021,7 +1021,7 @@ static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) { vcpu_load(vcpu); - memcpy(s->regs, vcpu->apic->regs, sizeof *s); + memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s); vcpu_put(vcpu); return 0; @@ -1031,7 +1031,7 @@ static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) { vcpu_load(vcpu); - memcpy(vcpu->apic->regs, s->regs, sizeof *s); + memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s); kvm_apic_post_state_restore(vcpu); vcpu_put(vcpu); @@ -1047,8 +1047,8 @@ static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, return -ENXIO; vcpu_load(vcpu); - set_bit(irq->irq, vcpu->irq_pending); - set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary); + set_bit(irq->irq, vcpu->arch.irq_pending); + set_bit(irq->irq / BITS_PER_LONG, &vcpu->arch.irq_summary); vcpu_put(vcpu); @@ -1499,8 +1499,8 @@ static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu, { struct kvm_io_device *dev; - if (vcpu->apic) { - dev = &vcpu->apic->dev; + if (vcpu->arch.apic) { + dev = &vcpu->arch.apic->dev; if (dev->in_range(dev, addr)) return dev; } @@ -1527,7 +1527,7 @@ int emulator_read_std(unsigned long addr, void *data = val; while (bytes) { - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); unsigned offset = addr & (PAGE_SIZE-1); unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset); int ret; @@ -1561,7 +1561,7 @@ static int emulator_read_emulated(unsigned long addr, return X86EMUL_CONTINUE; } - gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); /* For APIC access vmexit */ if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) @@ -1609,7 +1609,7 @@ static int emulator_write_emulated_onepage(unsigned long addr, struct kvm_vcpu *vcpu) { struct kvm_io_device *mmio_dev; - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); if (gpa == UNMAPPED_GVA) { kvm_inject_page_fault(vcpu, addr, 2); @@ -1678,7 +1678,7 @@ static int emulator_cmpxchg_emulated(unsigned long addr, #ifndef CONFIG_X86_64 /* guests cmpxchg8b have to be emulated atomically */ if (bytes == 8) { - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); struct page *page; char *addr; u64 val; @@ -1715,7 +1715,7 @@ int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address) int emulate_clts(struct kvm_vcpu *vcpu) { - kvm_x86_ops->set_cr0(vcpu, vcpu->cr0 & ~X86_CR0_TS); + kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 & ~X86_CR0_TS); return X86EMUL_CONTINUE; } @@ -1750,7 +1750,7 @@ void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context) { static int reported; u8 opcodes[4]; - unsigned long rip = vcpu->rip; + unsigned long rip = vcpu->arch.rip; unsigned long rip_linear; rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS); @@ -1781,46 +1781,46 @@ int emulate_instruction(struct kvm_vcpu *vcpu, { int r; - vcpu->mmio_fault_cr2 = cr2; + vcpu->arch.mmio_fault_cr2 = cr2; kvm_x86_ops->cache_regs(vcpu); vcpu->mmio_is_write = 0; - vcpu->pio.string = 0; + vcpu->arch.pio.string = 0; if (!no_decode) { int cs_db, cs_l; kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); - vcpu->emulate_ctxt.vcpu = vcpu; - vcpu->emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu); - vcpu->emulate_ctxt.mode = - (vcpu->emulate_ctxt.eflags & X86_EFLAGS_VM) + vcpu->arch.emulate_ctxt.vcpu = vcpu; + vcpu->arch.emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu); + vcpu->arch.emulate_ctxt.mode = + (vcpu->arch.emulate_ctxt.eflags & X86_EFLAGS_VM) ? X86EMUL_MODE_REAL : cs_l ? X86EMUL_MODE_PROT64 : cs_db ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; - if (vcpu->emulate_ctxt.mode == X86EMUL_MODE_PROT64) { - vcpu->emulate_ctxt.cs_base = 0; - vcpu->emulate_ctxt.ds_base = 0; - vcpu->emulate_ctxt.es_base = 0; - vcpu->emulate_ctxt.ss_base = 0; + if (vcpu->arch.emulate_ctxt.mode == X86EMUL_MODE_PROT64) { + vcpu->arch.emulate_ctxt.cs_base = 0; + vcpu->arch.emulate_ctxt.ds_base = 0; + vcpu->arch.emulate_ctxt.es_base = 0; + vcpu->arch.emulate_ctxt.ss_base = 0; } else { - vcpu->emulate_ctxt.cs_base = + vcpu->arch.emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS); - vcpu->emulate_ctxt.ds_base = + vcpu->arch.emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS); - vcpu->emulate_ctxt.es_base = + vcpu->arch.emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES); - vcpu->emulate_ctxt.ss_base = + vcpu->arch.emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS); } - vcpu->emulate_ctxt.gs_base = + vcpu->arch.emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS); - vcpu->emulate_ctxt.fs_base = + vcpu->arch.emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS); - r = x86_decode_insn(&vcpu->emulate_ctxt, &emulate_ops); + r = x86_decode_insn(&vcpu->arch.emulate_ctxt, &emulate_ops); ++vcpu->stat.insn_emulation; if (r) { ++vcpu->stat.insn_emulation_fail; @@ -1830,9 +1830,9 @@ int emulate_instruction(struct kvm_vcpu *vcpu, } } - r = x86_emulate_insn(&vcpu->emulate_ctxt, &emulate_ops); + r = x86_emulate_insn(&vcpu->arch.emulate_ctxt, &emulate_ops); - if (vcpu->pio.string) + if (vcpu->arch.pio.string) return EMULATE_DO_MMIO; if ((r || vcpu->mmio_is_write) && run) { @@ -1854,7 +1854,7 @@ int emulate_instruction(struct kvm_vcpu *vcpu, } kvm_x86_ops->decache_regs(vcpu); - kvm_x86_ops->set_rflags(vcpu, vcpu->emulate_ctxt.eflags); + kvm_x86_ops->set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags); if (vcpu->mmio_is_write) { vcpu->mmio_needed = 0; @@ -1869,33 +1869,33 @@ static void free_pio_guest_pages(struct kvm_vcpu *vcpu) { int i; - for (i = 0; i < ARRAY_SIZE(vcpu->pio.guest_pages); ++i) - if (vcpu->pio.guest_pages[i]) { - kvm_release_page_dirty(vcpu->pio.guest_pages[i]); - vcpu->pio.guest_pages[i] = NULL; + for (i = 0; i < ARRAY_SIZE(vcpu->arch.pio.guest_pages); ++i) + if (vcpu->arch.pio.guest_pages[i]) { + kvm_release_page_dirty(vcpu->arch.pio.guest_pages[i]); + vcpu->arch.pio.guest_pages[i] = NULL; } } static int pio_copy_data(struct kvm_vcpu *vcpu) { - void *p = vcpu->pio_data; + void *p = vcpu->arch.pio_data; void *q; unsigned bytes; - int nr_pages = vcpu->pio.guest_pages[1] ? 2 : 1; + int nr_pages = vcpu->arch.pio.guest_pages[1] ? 2 : 1; - q = vmap(vcpu->pio.guest_pages, nr_pages, VM_READ|VM_WRITE, + q = vmap(vcpu->arch.pio.guest_pages, nr_pages, VM_READ|VM_WRITE, PAGE_KERNEL); if (!q) { free_pio_guest_pages(vcpu); return -ENOMEM; } - q += vcpu->pio.guest_page_offset; - bytes = vcpu->pio.size * vcpu->pio.cur_count; - if (vcpu->pio.in) + q += vcpu->arch.pio.guest_page_offset; + bytes = vcpu->arch.pio.size * vcpu->arch.pio.cur_count; + if (vcpu->arch.pio.in) memcpy(q, p, bytes); else memcpy(p, q, bytes); - q -= vcpu->pio.guest_page_offset; + q -= vcpu->arch.pio.guest_page_offset; vunmap(q); free_pio_guest_pages(vcpu); return 0; @@ -1903,7 +1903,7 @@ static int pio_copy_data(struct kvm_vcpu *vcpu) int complete_pio(struct kvm_vcpu *vcpu) { - struct kvm_pio_request *io = &vcpu->pio; + struct kvm_pio_request *io = &vcpu->arch.pio; long delta; int r; @@ -1911,7 +1911,7 @@ int complete_pio(struct kvm_vcpu *vcpu) if (!io->string) { if (io->in) - memcpy(&vcpu->regs[VCPU_REGS_RAX], vcpu->pio_data, + memcpy(&vcpu->arch.regs[VCPU_REGS_RAX], vcpu->arch.pio_data, io->size); } else { if (io->in) { @@ -1929,15 +1929,15 @@ int complete_pio(struct kvm_vcpu *vcpu) * The size of the register should really depend on * current address size. */ - vcpu->regs[VCPU_REGS_RCX] -= delta; + vcpu->arch.regs[VCPU_REGS_RCX] -= delta; } if (io->down) delta = -delta; delta *= io->size; if (io->in) - vcpu->regs[VCPU_REGS_RDI] += delta; + vcpu->arch.regs[VCPU_REGS_RDI] += delta; else - vcpu->regs[VCPU_REGS_RSI] += delta; + vcpu->arch.regs[VCPU_REGS_RSI] += delta; } kvm_x86_ops->decache_regs(vcpu); @@ -1955,13 +1955,13 @@ static void kernel_pio(struct kvm_io_device *pio_dev, /* TODO: String I/O for in kernel device */ mutex_lock(&vcpu->kvm->lock); - if (vcpu->pio.in) - kvm_iodevice_read(pio_dev, vcpu->pio.port, - vcpu->pio.size, + if (vcpu->arch.pio.in) + kvm_iodevice_read(pio_dev, vcpu->arch.pio.port, + vcpu->arch.pio.size, pd); else - kvm_iodevice_write(pio_dev, vcpu->pio.port, - vcpu->pio.size, + kvm_iodevice_write(pio_dev, vcpu->arch.pio.port, + vcpu->arch.pio.size, pd); mutex_unlock(&vcpu->kvm->lock); } @@ -1969,8 +1969,8 @@ static void kernel_pio(struct kvm_io_device *pio_dev, static void pio_string_write(struct kvm_io_device *pio_dev, struct kvm_vcpu *vcpu) { - struct kvm_pio_request *io = &vcpu->pio; - void *pd = vcpu->pio_data; + struct kvm_pio_request *io = &vcpu->arch.pio; + void *pd = vcpu->arch.pio_data; int i; mutex_lock(&vcpu->kvm->lock); @@ -1996,25 +1996,25 @@ int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, vcpu->run->exit_reason = KVM_EXIT_IO; vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; - vcpu->run->io.size = vcpu->pio.size = size; + vcpu->run->io.size = vcpu->arch.pio.size = size; vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; - vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = 1; - vcpu->run->io.port = vcpu->pio.port = port; - vcpu->pio.in = in; - vcpu->pio.string = 0; - vcpu->pio.down = 0; - vcpu->pio.guest_page_offset = 0; - vcpu->pio.rep = 0; + vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = 1; + vcpu->run->io.port = vcpu->arch.pio.port = port; + vcpu->arch.pio.in = in; + vcpu->arch.pio.string = 0; + vcpu->arch.pio.down = 0; + vcpu->arch.pio.guest_page_offset = 0; + vcpu->arch.pio.rep = 0; kvm_x86_ops->cache_regs(vcpu); - memcpy(vcpu->pio_data, &vcpu->regs[VCPU_REGS_RAX], 4); + memcpy(vcpu->arch.pio_data, &vcpu->arch.regs[VCPU_REGS_RAX], 4); kvm_x86_ops->decache_regs(vcpu); kvm_x86_ops->skip_emulated_instruction(vcpu); pio_dev = vcpu_find_pio_dev(vcpu, port); if (pio_dev) { - kernel_pio(pio_dev, vcpu, vcpu->pio_data); + kernel_pio(pio_dev, vcpu, vcpu->arch.pio_data); complete_pio(vcpu); return 1; } @@ -2034,15 +2034,15 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, vcpu->run->exit_reason = KVM_EXIT_IO; vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; - vcpu->run->io.size = vcpu->pio.size = size; + vcpu->run->io.size = vcpu->arch.pio.size = size; vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; - vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = count; - vcpu->run->io.port = vcpu->pio.port = port; - vcpu->pio.in = in; - vcpu->pio.string = 1; - vcpu->pio.down = down; - vcpu->pio.guest_page_offset = offset_in_page(address); - vcpu->pio.rep = rep; + vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = count; + vcpu->run->io.port = vcpu->arch.pio.port = port; + vcpu->arch.pio.in = in; + vcpu->arch.pio.string = 1; + vcpu->arch.pio.down = down; + vcpu->arch.pio.guest_page_offset = offset_in_page(address); + vcpu->arch.pio.rep = rep; if (!count) { kvm_x86_ops->skip_emulated_instruction(vcpu); @@ -2072,15 +2072,15 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, return 1; } vcpu->run->io.count = now; - vcpu->pio.cur_count = now; + vcpu->arch.pio.cur_count = now; - if (vcpu->pio.cur_count == vcpu->pio.count) + if (vcpu->arch.pio.cur_count == vcpu->arch.pio.count) kvm_x86_ops->skip_emulated_instruction(vcpu); for (i = 0; i < nr_pages; ++i) { mutex_lock(&vcpu->kvm->lock); page = gva_to_page(vcpu, address + i * PAGE_SIZE); - vcpu->pio.guest_pages[i] = page; + vcpu->arch.pio.guest_pages[i] = page; mutex_unlock(&vcpu->kvm->lock); if (!page) { kvm_inject_gp(vcpu, 0); @@ -2090,13 +2090,13 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, } pio_dev = vcpu_find_pio_dev(vcpu, port); - if (!vcpu->pio.in) { + if (!vcpu->arch.pio.in) { /* string PIO write */ ret = pio_copy_data(vcpu); if (ret >= 0 && pio_dev) { pio_string_write(pio_dev, vcpu); complete_pio(vcpu); - if (vcpu->pio.count == 0) + if (vcpu->arch.pio.count == 0) ret = 1; } } else if (pio_dev) @@ -2156,9 +2156,9 @@ int kvm_emulate_halt(struct kvm_vcpu *vcpu) { ++vcpu->stat.halt_exits; if (irqchip_in_kernel(vcpu->kvm)) { - vcpu->mp_state = VCPU_MP_STATE_HALTED; + vcpu->arch.mp_state = VCPU_MP_STATE_HALTED; kvm_vcpu_block(vcpu); - if (vcpu->mp_state != VCPU_MP_STATE_RUNNABLE) + if (vcpu->arch.mp_state != VCPU_MP_STATE_RUNNABLE) return -EINTR; return 1; } else { @@ -2174,11 +2174,11 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) kvm_x86_ops->cache_regs(vcpu); - nr = vcpu->regs[VCPU_REGS_RAX]; - a0 = vcpu->regs[VCPU_REGS_RBX]; - a1 = vcpu->regs[VCPU_REGS_RCX]; - a2 = vcpu->regs[VCPU_REGS_RDX]; - a3 = vcpu->regs[VCPU_REGS_RSI]; + nr = vcpu->arch.regs[VCPU_REGS_RAX]; + a0 = vcpu->arch.regs[VCPU_REGS_RBX]; + a1 = vcpu->arch.regs[VCPU_REGS_RCX]; + a2 = vcpu->arch.regs[VCPU_REGS_RDX]; + a3 = vcpu->arch.regs[VCPU_REGS_RSI]; if (!is_long_mode(vcpu)) { nr &= 0xFFFFFFFF; @@ -2193,7 +2193,7 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) ret = -KVM_ENOSYS; break; } - vcpu->regs[VCPU_REGS_RAX] = ret; + vcpu->arch.regs[VCPU_REGS_RAX] = ret; kvm_x86_ops->decache_regs(vcpu); return 0; } @@ -2215,7 +2215,7 @@ int kvm_fix_hypercall(struct kvm_vcpu *vcpu) kvm_x86_ops->cache_regs(vcpu); kvm_x86_ops->patch_hypercall(vcpu, instruction); - if (emulator_write_emulated(vcpu->rip, instruction, 3, vcpu) + if (emulator_write_emulated(vcpu->arch.rip, instruction, 3, vcpu) != X86EMUL_CONTINUE) ret = -EFAULT; @@ -2255,13 +2255,13 @@ unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr) kvm_x86_ops->decache_cr4_guest_bits(vcpu); switch (cr) { case 0: - return vcpu->cr0; + return vcpu->arch.cr0; case 2: - return vcpu->cr2; + return vcpu->arch.cr2; case 3: - return vcpu->cr3; + return vcpu->arch.cr3; case 4: - return vcpu->cr4; + return vcpu->arch.cr4; case 8: return get_cr8(vcpu); default: @@ -2275,17 +2275,17 @@ void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, { switch (cr) { case 0: - set_cr0(vcpu, mk_cr_64(vcpu->cr0, val)); + set_cr0(vcpu, mk_cr_64(vcpu->arch.cr0, val)); *rflags = kvm_x86_ops->get_rflags(vcpu); break; case 2: - vcpu->cr2 = val; + vcpu->arch.cr2 = val; break; case 3: set_cr3(vcpu, val); break; case 4: - set_cr4(vcpu, mk_cr_64(vcpu->cr4, val)); + set_cr4(vcpu, mk_cr_64(vcpu->arch.cr4, val)); break; case 8: set_cr8(vcpu, val & 0xfUL); @@ -2297,13 +2297,13 @@ void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i) { - struct kvm_cpuid_entry2 *e = &vcpu->cpuid_entries[i]; - int j, nent = vcpu->cpuid_nent; + struct kvm_cpuid_entry2 *e = &vcpu->arch.cpuid_entries[i]; + int j, nent = vcpu->arch.cpuid_nent; e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT; /* when no next entry is found, the current entry[i] is reselected */ for (j = i + 1; j == i; j = (j + 1) % nent) { - struct kvm_cpuid_entry2 *ej = &vcpu->cpuid_entries[j]; + struct kvm_cpuid_entry2 *ej = &vcpu->arch.cpuid_entries[j]; if (ej->function == e->function) { ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT; return j; @@ -2334,15 +2334,15 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) struct kvm_cpuid_entry2 *e, *best; kvm_x86_ops->cache_regs(vcpu); - function = vcpu->regs[VCPU_REGS_RAX]; - index = vcpu->regs[VCPU_REGS_RCX]; - vcpu->regs[VCPU_REGS_RAX] = 0; - vcpu->regs[VCPU_REGS_RBX] = 0; - vcpu->regs[VCPU_REGS_RCX] = 0; - vcpu->regs[VCPU_REGS_RDX] = 0; + function = vcpu->arch.regs[VCPU_REGS_RAX]; + index = vcpu->arch.regs[VCPU_REGS_RCX]; + vcpu->arch.regs[VCPU_REGS_RAX] = 0; + vcpu->arch.regs[VCPU_REGS_RBX] = 0; + vcpu->arch.regs[VCPU_REGS_RCX] = 0; + vcpu->arch.regs[VCPU_REGS_RDX] = 0; best = NULL; - for (i = 0; i < vcpu->cpuid_nent; ++i) { - e = &vcpu->cpuid_entries[i]; + for (i = 0; i < vcpu->arch.cpuid_nent; ++i) { + e = &vcpu->arch.cpuid_entries[i]; if (is_matching_cpuid_entry(e, function, index)) { if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) move_to_next_stateful_cpuid_entry(vcpu, i); @@ -2357,10 +2357,10 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) best = e; } if (best) { - vcpu->regs[VCPU_REGS_RAX] = best->eax; - vcpu->regs[VCPU_REGS_RBX] = best->ebx; - vcpu->regs[VCPU_REGS_RCX] = best->ecx; - vcpu->regs[VCPU_REGS_RDX] = best->edx; + vcpu->arch.regs[VCPU_REGS_RAX] = best->eax; + vcpu->arch.regs[VCPU_REGS_RBX] = best->ebx; + vcpu->arch.regs[VCPU_REGS_RCX] = best->ecx; + vcpu->arch.regs[VCPU_REGS_RDX] = best->edx; } kvm_x86_ops->decache_regs(vcpu); kvm_x86_ops->skip_emulated_instruction(vcpu); @@ -2376,9 +2376,9 @@ EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { - return (!vcpu->irq_summary && + return (!vcpu->arch.irq_summary && kvm_run->request_interrupt_window && - vcpu->interrupt_window_open && + vcpu->arch.interrupt_window_open && (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF)); } @@ -2392,22 +2392,22 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu, kvm_run->ready_for_interrupt_injection = 1; else kvm_run->ready_for_interrupt_injection = - (vcpu->interrupt_window_open && - vcpu->irq_summary == 0); + (vcpu->arch.interrupt_window_open && + vcpu->arch.irq_summary == 0); } static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { int r; - if (unlikely(vcpu->mp_state == VCPU_MP_STATE_SIPI_RECEIVED)) { + if (unlikely(vcpu->arch.mp_state == VCPU_MP_STATE_SIPI_RECEIVED)) { pr_debug("vcpu %d received sipi with vector # %x\n", - vcpu->vcpu_id, vcpu->sipi_vector); + vcpu->vcpu_id, vcpu->arch.sipi_vector); kvm_lapic_reset(vcpu); r = kvm_x86_ops->vcpu_reset(vcpu); if (r) return r; - vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; + vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE; } preempted: @@ -2437,7 +2437,7 @@ again: goto out; } - if (vcpu->exception.pending) + if (vcpu->arch.exception.pending) __queue_exception(vcpu); else if (irqchip_in_kernel(vcpu->kvm)) kvm_x86_ops->inject_pending_irq(vcpu); @@ -2475,11 +2475,11 @@ again: */ if (unlikely(prof_on == KVM_PROFILING)) { kvm_x86_ops->cache_regs(vcpu); - profile_hit(KVM_PROFILING, (void *)vcpu->rip); + profile_hit(KVM_PROFILING, (void *)vcpu->arch.rip); } - if (vcpu->exception.pending && kvm_x86_ops->exception_injected(vcpu)) - vcpu->exception.pending = false; + if (vcpu->arch.exception.pending && kvm_x86_ops->exception_injected(vcpu)) + vcpu->arch.exception.pending = false; r = kvm_x86_ops->handle_exit(kvm_run, vcpu); @@ -2512,7 +2512,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) vcpu_load(vcpu); - if (unlikely(vcpu->mp_state == VCPU_MP_STATE_UNINITIALIZED)) { + if (unlikely(vcpu->arch.mp_state == VCPU_MP_STATE_UNINITIALIZED)) { kvm_vcpu_block(vcpu); vcpu_put(vcpu); return -EAGAIN; @@ -2525,7 +2525,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) if (!irqchip_in_kernel(vcpu->kvm)) set_cr8(vcpu, kvm_run->cr8); - if (vcpu->pio.cur_count) { + if (vcpu->arch.pio.cur_count) { r = complete_pio(vcpu); if (r) goto out; @@ -2536,7 +2536,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) vcpu->mmio_read_completed = 1; vcpu->mmio_needed = 0; r = emulate_instruction(vcpu, kvm_run, - vcpu->mmio_fault_cr2, 0, 1); + vcpu->arch.mmio_fault_cr2, 0, 1); if (r == EMULATE_DO_MMIO) { /* * Read-modify-write. Back to userspace. @@ -2548,7 +2548,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) #endif if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) { kvm_x86_ops->cache_regs(vcpu); - vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret; + vcpu->arch.regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret; kvm_x86_ops->decache_regs(vcpu); } @@ -2568,26 +2568,26 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) kvm_x86_ops->cache_regs(vcpu); - regs->rax = vcpu->regs[VCPU_REGS_RAX]; - regs->rbx = vcpu->regs[VCPU_REGS_RBX]; - regs->rcx = vcpu->regs[VCPU_REGS_RCX]; - regs->rdx = vcpu->regs[VCPU_REGS_RDX]; - regs->rsi = vcpu->regs[VCPU_REGS_RSI]; - regs->rdi = vcpu->regs[VCPU_REGS_RDI]; - regs->rsp = vcpu->regs[VCPU_REGS_RSP]; - regs->rbp = vcpu->regs[VCPU_REGS_RBP]; + regs->rax = vcpu->arch.regs[VCPU_REGS_RAX]; + regs->rbx = vcpu->arch.regs[VCPU_REGS_RBX]; + regs->rcx = vcpu->arch.regs[VCPU_REGS_RCX]; + regs->rdx = vcpu->arch.regs[VCPU_REGS_RDX]; + regs->rsi = vcpu->arch.regs[VCPU_REGS_RSI]; + regs->rdi = vcpu->arch.regs[VCPU_REGS_RDI]; + regs->rsp = vcpu->arch.regs[VCPU_REGS_RSP]; + regs->rbp = vcpu->arch.regs[VCPU_REGS_RBP]; #ifdef CONFIG_X86_64 - regs->r8 = vcpu->regs[VCPU_REGS_R8]; - regs->r9 = vcpu->regs[VCPU_REGS_R9]; - regs->r10 = vcpu->regs[VCPU_REGS_R10]; - regs->r11 = vcpu->regs[VCPU_REGS_R11]; - regs->r12 = vcpu->regs[VCPU_REGS_R12]; - regs->r13 = vcpu->regs[VCPU_REGS_R13]; - regs->r14 = vcpu->regs[VCPU_REGS_R14]; - regs->r15 = vcpu->regs[VCPU_REGS_R15]; + regs->r8 = vcpu->arch.regs[VCPU_REGS_R8]; + regs->r9 = vcpu->arch.regs[VCPU_REGS_R9]; + regs->r10 = vcpu->arch.regs[VCPU_REGS_R10]; + regs->r11 = vcpu->arch.regs[VCPU_REGS_R11]; + regs->r12 = vcpu->arch.regs[VCPU_REGS_R12]; + regs->r13 = vcpu->arch.regs[VCPU_REGS_R13]; + regs->r14 = vcpu->arch.regs[VCPU_REGS_R14]; + regs->r15 = vcpu->arch.regs[VCPU_REGS_R15]; #endif - regs->rip = vcpu->rip; + regs->rip = vcpu->arch.rip; regs->rflags = kvm_x86_ops->get_rflags(vcpu); /* @@ -2605,26 +2605,26 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) { vcpu_load(vcpu); - vcpu->regs[VCPU_REGS_RAX] = regs->rax; - vcpu->regs[VCPU_REGS_RBX] = regs->rbx; - vcpu->regs[VCPU_REGS_RCX] = regs->rcx; - vcpu->regs[VCPU_REGS_RDX] = regs->rdx; - vcpu->regs[VCPU_REGS_RSI] = regs->rsi; - vcpu->regs[VCPU_REGS_RDI] = regs->rdi; - vcpu->regs[VCPU_REGS_RSP] = regs->rsp; - vcpu->regs[VCPU_REGS_RBP] = regs->rbp; + vcpu->arch.regs[VCPU_REGS_RAX] = regs->rax; + vcpu->arch.regs[VCPU_REGS_RBX] = regs->rbx; + vcpu->arch.regs[VCPU_REGS_RCX] = regs->rcx; + vcpu->arch.regs[VCPU_REGS_RDX] = regs->rdx; + vcpu->arch.regs[VCPU_REGS_RSI] = regs->rsi; + vcpu->arch.regs[VCPU_REGS_RDI] = regs->rdi; + vcpu->arch.regs[VCPU_REGS_RSP] = regs->rsp; + vcpu->arch.regs[VCPU_REGS_RBP] = regs->rbp; #ifdef CONFIG_X86_64 - vcpu->regs[VCPU_REGS_R8] = regs->r8; - vcpu->regs[VCPU_REGS_R9] = regs->r9; - vcpu->regs[VCPU_REGS_R10] = regs->r10; - vcpu->regs[VCPU_REGS_R11] = regs->r11; - vcpu->regs[VCPU_REGS_R12] = regs->r12; - vcpu->regs[VCPU_REGS_R13] = regs->r13; - vcpu->regs[VCPU_REGS_R14] = regs->r14; - vcpu->regs[VCPU_REGS_R15] = regs->r15; + vcpu->arch.regs[VCPU_REGS_R8] = regs->r8; + vcpu->arch.regs[VCPU_REGS_R9] = regs->r9; + vcpu->arch.regs[VCPU_REGS_R10] = regs->r10; + vcpu->arch.regs[VCPU_REGS_R11] = regs->r11; + vcpu->arch.regs[VCPU_REGS_R12] = regs->r12; + vcpu->arch.regs[VCPU_REGS_R13] = regs->r13; + vcpu->arch.regs[VCPU_REGS_R14] = regs->r14; + vcpu->arch.regs[VCPU_REGS_R15] = regs->r15; #endif - vcpu->rip = regs->rip; + vcpu->arch.rip = regs->rip; kvm_x86_ops->set_rflags(vcpu, regs->rflags); kvm_x86_ops->decache_regs(vcpu); @@ -2676,12 +2676,12 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, sregs->gdt.base = dt.base; kvm_x86_ops->decache_cr4_guest_bits(vcpu); - sregs->cr0 = vcpu->cr0; - sregs->cr2 = vcpu->cr2; - sregs->cr3 = vcpu->cr3; - sregs->cr4 = vcpu->cr4; + sregs->cr0 = vcpu->arch.cr0; + sregs->cr2 = vcpu->arch.cr2; + sregs->cr3 = vcpu->arch.cr3; + sregs->cr4 = vcpu->arch.cr4; sregs->cr8 = get_cr8(vcpu); - sregs->efer = vcpu->shadow_efer; + sregs->efer = vcpu->arch.shadow_efer; sregs->apic_base = kvm_get_apic_base(vcpu); if (irqchip_in_kernel(vcpu->kvm)) { @@ -2692,7 +2692,7 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, set_bit(pending_vec, (unsigned long *)sregs->interrupt_bitmap); } else - memcpy(sregs->interrupt_bitmap, vcpu->irq_pending, + memcpy(sregs->interrupt_bitmap, vcpu->arch.irq_pending, sizeof sregs->interrupt_bitmap); vcpu_put(vcpu); @@ -2722,13 +2722,13 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, dt.base = sregs->gdt.base; kvm_x86_ops->set_gdt(vcpu, &dt); - vcpu->cr2 = sregs->cr2; - mmu_reset_needed |= vcpu->cr3 != sregs->cr3; - vcpu->cr3 = sregs->cr3; + vcpu->arch.cr2 = sregs->cr2; + mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3; + vcpu->arch.cr3 = sregs->cr3; set_cr8(vcpu, sregs->cr8); - mmu_reset_needed |= vcpu->shadow_efer != sregs->efer; + mmu_reset_needed |= vcpu->arch.shadow_efer != sregs->efer; #ifdef CONFIG_X86_64 kvm_x86_ops->set_efer(vcpu, sregs->efer); #endif @@ -2736,25 +2736,25 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, kvm_x86_ops->decache_cr4_guest_bits(vcpu); - mmu_reset_needed |= vcpu->cr0 != sregs->cr0; - vcpu->cr0 = sregs->cr0; + mmu_reset_needed |= vcpu->arch.cr0 != sregs->cr0; + vcpu->arch.cr0 = sregs->cr0; kvm_x86_ops->set_cr0(vcpu, sregs->cr0); - mmu_reset_needed |= vcpu->cr4 != sregs->cr4; + mmu_reset_needed |= vcpu->arch.cr4 != sregs->cr4; kvm_x86_ops->set_cr4(vcpu, sregs->cr4); if (!is_long_mode(vcpu) && is_pae(vcpu)) - load_pdptrs(vcpu, vcpu->cr3); + load_pdptrs(vcpu, vcpu->arch.cr3); if (mmu_reset_needed) kvm_mmu_reset_context(vcpu); if (!irqchip_in_kernel(vcpu->kvm)) { - memcpy(vcpu->irq_pending, sregs->interrupt_bitmap, - sizeof vcpu->irq_pending); - vcpu->irq_summary = 0; - for (i = 0; i < ARRAY_SIZE(vcpu->irq_pending); ++i) - if (vcpu->irq_pending[i]) - __set_bit(i, &vcpu->irq_summary); + memcpy(vcpu->arch.irq_pending, sregs->interrupt_bitmap, + sizeof vcpu->arch.irq_pending); + vcpu->arch.irq_summary = 0; + for (i = 0; i < ARRAY_SIZE(vcpu->arch.irq_pending); ++i) + if (vcpu->arch.irq_pending[i]) + __set_bit(i, &vcpu->arch.irq_summary); } else { max_bits = (sizeof sregs->interrupt_bitmap) << 3; pending_vec = find_first_bit( @@ -2829,7 +2829,7 @@ int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, vcpu_load(vcpu); mutex_lock(&vcpu->kvm->lock); - gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr); + gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, vaddr); tr->physical_address = gpa; tr->valid = gpa != UNMAPPED_GVA; tr->writeable = 1; @@ -2842,7 +2842,7 @@ int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) { - struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image; + struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image; vcpu_load(vcpu); @@ -2862,7 +2862,7 @@ int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) { - struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image; + struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image; vcpu_load(vcpu); @@ -2886,16 +2886,16 @@ void fx_init(struct kvm_vcpu *vcpu) /* Initialize guest FPU by resetting ours and saving into guest's */ preempt_disable(); - fx_save(&vcpu->host_fx_image); + fx_save(&vcpu->arch.host_fx_image); fpu_init(); - fx_save(&vcpu->guest_fx_image); - fx_restore(&vcpu->host_fx_image); + fx_save(&vcpu->arch.guest_fx_image); + fx_restore(&vcpu->arch.host_fx_image); preempt_enable(); - vcpu->cr0 |= X86_CR0_ET; + vcpu->arch.cr0 |= X86_CR0_ET; after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space); - vcpu->guest_fx_image.mxcsr = 0x1f80; - memset((void *)&vcpu->guest_fx_image + after_mxcsr_mask, + vcpu->arch.guest_fx_image.mxcsr = 0x1f80; + memset((void *)&vcpu->arch.guest_fx_image + after_mxcsr_mask, 0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask); } EXPORT_SYMBOL_GPL(fx_init); @@ -2906,8 +2906,8 @@ void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) return; vcpu->guest_fpu_loaded = 1; - fx_save(&vcpu->host_fx_image); - fx_restore(&vcpu->guest_fx_image); + fx_save(&vcpu->arch.host_fx_image); + fx_restore(&vcpu->arch.guest_fx_image); } EXPORT_SYMBOL_GPL(kvm_load_guest_fpu); @@ -2917,8 +2917,8 @@ void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) return; vcpu->guest_fpu_loaded = 0; - fx_save(&vcpu->guest_fx_image); - fx_restore(&vcpu->host_fx_image); + fx_save(&vcpu->arch.guest_fx_image); + fx_restore(&vcpu->arch.host_fx_image); ++vcpu->stat.fpu_reload; } EXPORT_SYMBOL_GPL(kvm_put_guest_fpu); @@ -2939,7 +2939,7 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) int r; /* We do fxsave: this must be aligned. */ - BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF); + BUG_ON((unsigned long)&vcpu->arch.host_fx_image & 0xF); vcpu_load(vcpu); r = kvm_arch_vcpu_reset(vcpu); @@ -3003,18 +3003,18 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) BUG_ON(vcpu->kvm == NULL); kvm = vcpu->kvm; - vcpu->mmu.root_hpa = INVALID_PAGE; + vcpu->arch.mmu.root_hpa = INVALID_PAGE; if (!irqchip_in_kernel(kvm) || vcpu->vcpu_id == 0) - vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; + vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE; else - vcpu->mp_state = VCPU_MP_STATE_UNINITIALIZED; + vcpu->arch.mp_state = VCPU_MP_STATE_UNINITIALIZED; page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!page) { r = -ENOMEM; goto fail; } - vcpu->pio_data = page_address(page); + vcpu->arch.pio_data = page_address(page); r = kvm_mmu_create(vcpu); if (r < 0) @@ -3031,7 +3031,7 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) fail_mmu_destroy: kvm_mmu_destroy(vcpu); fail_free_pio_data: - free_page((unsigned long)vcpu->pio_data); + free_page((unsigned long)vcpu->arch.pio_data); fail: return r; } @@ -3040,7 +3040,7 @@ void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) { kvm_free_lapic(vcpu); kvm_mmu_destroy(vcpu); - free_page((unsigned long)vcpu->pio_data); + free_page((unsigned long)vcpu->arch.pio_data); } struct kvm *kvm_arch_create_vm(void) diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 0fc7020aa1a5..0e01ac75268c 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -92,8 +92,7 @@ enum { #include "x86_emulate.h" -struct kvm_vcpu { - KVM_VCPU_COMM; +struct kvm_vcpu_arch { u64 host_tsc; int interrupt_window_open; unsigned long irq_summary; /* bit vector: 1 per word in irq_pending */ @@ -130,7 +129,6 @@ struct kvm_vcpu { int last_pt_write_count; u64 *last_pte_updated; - struct i387_fxsave_struct host_fx_image; struct i387_fxsave_struct guest_fx_image; @@ -159,12 +157,17 @@ struct kvm_vcpu { int cpuid_nent; struct kvm_cpuid_entry2 cpuid_entries[KVM_MAX_CPUID_ENTRIES]; - /* emulate context */ struct x86_emulate_ctxt emulate_ctxt; }; +struct kvm_vcpu { + KVM_VCPU_COMM; + + struct kvm_vcpu_arch arch; +}; + struct descriptor_table { u16 limit; unsigned long base; @@ -339,7 +342,7 @@ static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) { - if (likely(vcpu->mmu.root_hpa != INVALID_PAGE)) + if (likely(vcpu->arch.mmu.root_hpa != INVALID_PAGE)) return 0; return kvm_mmu_load(vcpu); @@ -348,7 +351,7 @@ static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) static inline int is_long_mode(struct kvm_vcpu *vcpu) { #ifdef CONFIG_X86_64 - return vcpu->shadow_efer & EFER_LME; + return vcpu->arch.shadow_efer & EFER_LME; #else return 0; #endif @@ -356,17 +359,17 @@ static inline int is_long_mode(struct kvm_vcpu *vcpu) static inline int is_pae(struct kvm_vcpu *vcpu) { - return vcpu->cr4 & X86_CR4_PAE; + return vcpu->arch.cr4 & X86_CR4_PAE; } static inline int is_pse(struct kvm_vcpu *vcpu) { - return vcpu->cr4 & X86_CR4_PSE; + return vcpu->arch.cr4 & X86_CR4_PSE; } static inline int is_paging(struct kvm_vcpu *vcpu) { - return vcpu->cr0 & X86_CR0_PG; + return vcpu->arch.cr0 & X86_CR0_PG; } int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); @@ -489,8 +492,8 @@ static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code) static inline int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) { - return vcpu->mp_state == VCPU_MP_STATE_RUNNABLE - || vcpu->mp_state == VCPU_MP_STATE_SIPI_RECEIVED; + return vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE + || vcpu->arch.mp_state == VCPU_MP_STATE_SIPI_RECEIVED; } #endif diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 0a6ab06fde01..50b133f68743 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -769,8 +769,8 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) /* Shadow copy of register state. Committed on successful emulation. */ memset(c, 0, sizeof(struct decode_cache)); - c->eip = ctxt->vcpu->rip; - memcpy(c->regs, ctxt->vcpu->regs, sizeof c->regs); + c->eip = ctxt->vcpu->arch.rip; + memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs); switch (mode) { case X86EMUL_MODE_REAL: @@ -1226,7 +1226,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) * modify them. */ - memcpy(c->regs, ctxt->vcpu->regs, sizeof c->regs); + memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs); saved_eip = c->eip; if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs)) @@ -1235,7 +1235,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) if (c->rep_prefix && (c->d & String)) { /* All REP prefixes have the same first termination condition */ if (c->regs[VCPU_REGS_RCX] == 0) { - ctxt->vcpu->rip = c->eip; + ctxt->vcpu->arch.rip = c->eip; goto done; } /* The second termination condition only applies for REPE @@ -1249,17 +1249,17 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) (c->b == 0xae) || (c->b == 0xaf)) { if ((c->rep_prefix == REPE_PREFIX) && ((ctxt->eflags & EFLG_ZF) == 0)) { - ctxt->vcpu->rip = c->eip; + ctxt->vcpu->arch.rip = c->eip; goto done; } if ((c->rep_prefix == REPNE_PREFIX) && ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) { - ctxt->vcpu->rip = c->eip; + ctxt->vcpu->arch.rip = c->eip; goto done; } } c->regs[VCPU_REGS_RCX]--; - c->eip = ctxt->vcpu->rip; + c->eip = ctxt->vcpu->arch.rip; } if (c->src.type == OP_MEM) { @@ -1628,7 +1628,7 @@ special_insn: c->dst.type = OP_NONE; /* Disable writeback. */ break; case 0xf4: /* hlt */ - ctxt->vcpu->halt_request = 1; + ctxt->vcpu->arch.halt_request = 1; goto done; case 0xf5: /* cmc */ /* complement carry flag from eflags reg */ @@ -1665,8 +1665,8 @@ writeback: goto done; /* Commit shadow register state. */ - memcpy(ctxt->vcpu->regs, c->regs, sizeof c->regs); - ctxt->vcpu->rip = c->eip; + memcpy(ctxt->vcpu->arch.regs, c->regs, sizeof c->regs); + ctxt->vcpu->arch.rip = c->eip; done: if (rc == X86EMUL_UNHANDLEABLE) { @@ -1783,7 +1783,7 @@ twobyte_insn: rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data); if (rc) { kvm_inject_gp(ctxt->vcpu, 0); - c->eip = ctxt->vcpu->rip; + c->eip = ctxt->vcpu->arch.rip; } rc = X86EMUL_CONTINUE; c->dst.type = OP_NONE; @@ -1793,7 +1793,7 @@ twobyte_insn: rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data); if (rc) { kvm_inject_gp(ctxt->vcpu, 0); - c->eip = ctxt->vcpu->rip; + c->eip = ctxt->vcpu->arch.rip; } else { c->regs[VCPU_REGS_RAX] = (u32)msr_data; c->regs[VCPU_REGS_RDX] = msr_data >> 32; From 1d737c8a68736db36e0aa502ace9da240704c5ae Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 09:35:10 +0800 Subject: [PATCH 215/249] KVM: Portability: Split mmu-related static inline functions to mmu.h Since these functions need to know the details of kvm or kvm_vcpu structure, it can't be put in x86.h. Create mmu.h to hold them. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/mmu.c | 1 + drivers/kvm/mmu.h | 44 ++++++++++++++++++++++++++++++++++++++++++++ drivers/kvm/svm.c | 1 + drivers/kvm/vmx.c | 1 + drivers/kvm/x86.c | 7 +++++++ drivers/kvm/x86.h | 44 -------------------------------------------- 7 files changed, 55 insertions(+), 44 deletions(-) create mode 100644 drivers/kvm/mmu.h diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 668a8300365d..c040bb3ac819 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -396,6 +396,7 @@ void kvm_arch_hardware_disable(void *garbage); int kvm_arch_hardware_setup(void); void kvm_arch_hardware_unsetup(void); void kvm_arch_check_processor_compat(void *rtn); +int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu); void kvm_free_physmem(struct kvm *kvm); diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index da1dedb497b8..1dc0e8c02c70 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -20,6 +20,7 @@ #include "vmx.h" #include "kvm.h" #include "x86.h" +#include "mmu.h" #include #include diff --git a/drivers/kvm/mmu.h b/drivers/kvm/mmu.h new file mode 100644 index 000000000000..9ebfd1cafe62 --- /dev/null +++ b/drivers/kvm/mmu.h @@ -0,0 +1,44 @@ +#ifndef __KVM_X86_MMU_H +#define __KVM_X86_MMU_H + +#include "kvm.h" + +static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) +{ + if (unlikely(vcpu->kvm->n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES)) + __kvm_mmu_free_some_pages(vcpu); +} + +static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) +{ + if (likely(vcpu->arch.mmu.root_hpa != INVALID_PAGE)) + return 0; + + return kvm_mmu_load(vcpu); +} + +static inline int is_long_mode(struct kvm_vcpu *vcpu) +{ +#ifdef CONFIG_X86_64 + return vcpu->arch.shadow_efer & EFER_LME; +#else + return 0; +#endif +} + +static inline int is_pae(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.cr4 & X86_CR4_PAE; +} + +static inline int is_pse(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.cr4 & X86_CR4_PSE; +} + +static inline int is_paging(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.cr0 & X86_CR0_PG; +} + +#endif diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 7888638c02e8..e606f6d18669 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -17,6 +17,7 @@ #include "kvm_svm.h" #include "x86_emulate.h" #include "irq.h" +#include "mmu.h" #include #include diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index cf78ebb2f36e..d0f431d4fe4d 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -21,6 +21,7 @@ #include "irq.h" #include "vmx.h" #include "segment_descriptor.h" +#include "mmu.h" #include #include diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 5a2f33a84e4f..60f9722a06ba 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -19,6 +19,7 @@ #include "x86_emulate.h" #include "segment_descriptor.h" #include "irq.h" +#include "mmu.h" #include #include @@ -3139,3 +3140,9 @@ int kvm_arch_set_memory_region(struct kvm *kvm, return 0; } + +int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE + || vcpu->arch.mp_state == VCPU_MP_STATE_SIPI_RECEIVED; +} diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 0e01ac75268c..0da4be9b0bad 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -334,44 +334,6 @@ int kvm_fix_hypercall(struct kvm_vcpu *vcpu); int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code); -static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) -{ - if (unlikely(vcpu->kvm->n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES)) - __kvm_mmu_free_some_pages(vcpu); -} - -static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) -{ - if (likely(vcpu->arch.mmu.root_hpa != INVALID_PAGE)) - return 0; - - return kvm_mmu_load(vcpu); -} - -static inline int is_long_mode(struct kvm_vcpu *vcpu) -{ -#ifdef CONFIG_X86_64 - return vcpu->arch.shadow_efer & EFER_LME; -#else - return 0; -#endif -} - -static inline int is_pae(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.cr4 & X86_CR4_PAE; -} - -static inline int is_pse(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.cr4 & X86_CR4_PSE; -} - -static inline int is_paging(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.cr0 & X86_CR0_PG; -} - int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); int complete_pio(struct kvm_vcpu *vcpu); @@ -490,10 +452,4 @@ static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code) #define TSS_REDIRECTION_SIZE (256 / 8) #define RMODE_TSS_SIZE (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) -static inline int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE - || vcpu->arch.mp_state == VCPU_MP_STATE_SIPI_RECEIVED; -} - #endif From d657a98e3c20537d8b4d44aef51cf4311d96f2b0 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 09:41:22 +0800 Subject: [PATCH 216/249] KVM: Portability: Move kvm_vcpu definition back to kvm.h This patches moves kvm_vcpu definition to kvm.h, and finally kvm.h includes x86.h. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/irq.h | 2 +- drivers/kvm/kvm.h | 113 ++++---------------------------------------- drivers/kvm/types.h | 13 +++++ drivers/kvm/x86.h | 102 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 117 insertions(+), 113 deletions(-) diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index e88d93957d24..a688c51d0a39 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -26,7 +26,7 @@ #include #include #include "iodev.h" -#include "x86.h" +#include "kvm.h" struct kvm; struct kvm_vcpu; diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index c040bb3ac819..53717be80bbe 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -22,17 +22,13 @@ #include "types.h" +#include "x86.h" + #define KVM_MAX_VCPUS 4 #define KVM_ALIAS_SLOTS 4 #define KVM_MEMORY_SLOTS 8 /* memory slots that does not exposed to userspace */ #define KVM_PRIVATE_MEM_SLOTS 4 -#define KVM_PERMILLE_MMU_PAGES 20 -#define KVM_MIN_ALLOC_MMU_PAGES 64 -#define KVM_NUM_MMU_PAGES 1024 -#define KVM_MIN_FREE_MMU_PAGES 5 -#define KVM_REFILL_PAGES 25 -#define KVM_MAX_CPUID_ENTRIES 40 #define KVM_PIO_PAGE_OFFSET 1 @@ -41,111 +37,16 @@ */ #define KVM_REQ_TLB_FLUSH 0 -#define NR_PTE_CHAIN_ENTRIES 5 - -struct kvm_pte_chain { - u64 *parent_ptes[NR_PTE_CHAIN_ENTRIES]; - struct hlist_node link; -}; - -/* - * kvm_mmu_page_role, below, is defined as: - * - * bits 0:3 - total guest paging levels (2-4, or zero for real mode) - * bits 4:7 - page table level for this shadow (1-4) - * bits 8:9 - page table quadrant for 2-level guests - * bit 16 - "metaphysical" - gfn is not a real page (huge page/real mode) - * bits 17:19 - common access permissions for all ptes in this shadow page - */ -union kvm_mmu_page_role { - unsigned word; - struct { - unsigned glevels : 4; - unsigned level : 4; - unsigned quadrant : 2; - unsigned pad_for_nice_hex_output : 6; - unsigned metaphysical : 1; - unsigned access : 3; - }; -}; - -struct kvm_mmu_page { - struct list_head link; - struct hlist_node hash_link; - - /* - * The following two entries are used to key the shadow page in the - * hash table. - */ - gfn_t gfn; - union kvm_mmu_page_role role; - - u64 *spt; - /* hold the gfn of each spte inside spt */ - gfn_t *gfns; - unsigned long slot_bitmap; /* One bit set per slot which has memory - * in this shadow page. - */ - int multimapped; /* More than one parent_pte? */ - int root_count; /* Currently serving as active root */ - union { - u64 *parent_pte; /* !multimapped */ - struct hlist_head parent_ptes; /* multimapped, kvm_pte_chain */ - }; -}; struct kvm_vcpu; extern struct kmem_cache *kvm_vcpu_cache; -/* - * x86 supports 3 paging modes (4-level 64-bit, 3-level 64-bit, and 2-level - * 32-bit). The kvm_mmu structure abstracts the details of the current mmu - * mode. - */ -struct kvm_mmu { - void (*new_cr3)(struct kvm_vcpu *vcpu); - int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err); - void (*free)(struct kvm_vcpu *vcpu); - gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva); - void (*prefetch_page)(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *page); - hpa_t root_hpa; - int root_level; - int shadow_root_level; - - u64 *pae_root; -}; - -#define KVM_NR_MEM_OBJS 40 - -/* - * We don't want allocation failures within the mmu code, so we preallocate - * enough memory for a single page fault in a cache. - */ -struct kvm_mmu_memory_cache { - int nobjs; - void *objects[KVM_NR_MEM_OBJS]; -}; - struct kvm_guest_debug { int enabled; unsigned long bp[4]; int singlestep; }; -struct kvm_pio_request { - unsigned long count; - int cur_count; - struct page *guest_pages[2]; - unsigned guest_page_offset; - int in; - int port; - int size; - int string; - int down; - int rep; -}; - struct kvm_vcpu_stat { u32 pf_fixed; u32 pf_guest; @@ -218,6 +119,12 @@ void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_vcpu_stat stat; \ KVM_VCPU_MMIO +struct kvm_vcpu { + KVM_VCPU_COMM; + + struct kvm_vcpu_arch arch; +}; + struct kvm_mem_alias { gfn_t base_gfn; unsigned long npages; @@ -441,8 +348,4 @@ struct kvm_stats_debugfs_item { }; extern struct kvm_stats_debugfs_item debugfs_entries[]; -#if defined(CONFIG_X86) -#include "x86.h" -#endif - #endif diff --git a/drivers/kvm/types.h b/drivers/kvm/types.h index 6ad76231b8ff..1c4e46decb22 100644 --- a/drivers/kvm/types.h +++ b/drivers/kvm/types.h @@ -38,4 +38,17 @@ typedef unsigned long hva_t; typedef u64 hpa_t; typedef unsigned long hfn_t; +struct kvm_pio_request { + unsigned long count; + int cur_count; + struct page *guest_pages[2]; + unsigned guest_page_offset; + int in; + int port; + int size; + int string; + int down; + int rep; +}; + #endif /* __KVM_TYPES_H__ */ diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 0da4be9b0bad..a0dd4473b8f8 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -11,8 +11,6 @@ #ifndef KVM_X86_H #define KVM_X86_H -#include "kvm.h" - #include #include @@ -21,6 +19,8 @@ #include +#include "types.h" + #define CR3_PAE_RESERVED_BITS ((X86_CR3_PWT | X86_CR3_PCD) - 1) #define CR3_NONPAE_RESERVED_BITS ((PAGE_SIZE-1) & ~(X86_CR3_PWT | X86_CR3_PCD)) #define CR3_L_MODE_RESERVED_BITS (CR3_NONPAE_RESERVED_BITS|0xFFFFFF0000000000ULL) @@ -54,9 +54,19 @@ #define IOPL_SHIFT 12 +#define KVM_PERMILLE_MMU_PAGES 20 +#define KVM_MIN_ALLOC_MMU_PAGES 64 +#define KVM_NUM_MMU_PAGES 1024 +#define KVM_MIN_FREE_MMU_PAGES 5 +#define KVM_REFILL_PAGES 25 +#define KVM_MAX_CPUID_ENTRIES 40 + extern spinlock_t kvm_lock; extern struct list_head vm_list; +struct kvm_vcpu; +struct kvm; + enum { VCPU_REGS_RAX = 0, VCPU_REGS_RCX = 1, @@ -92,6 +102,89 @@ enum { #include "x86_emulate.h" +#define KVM_NR_MEM_OBJS 40 + +/* + * We don't want allocation failures within the mmu code, so we preallocate + * enough memory for a single page fault in a cache. + */ +struct kvm_mmu_memory_cache { + int nobjs; + void *objects[KVM_NR_MEM_OBJS]; +}; + +#define NR_PTE_CHAIN_ENTRIES 5 + +struct kvm_pte_chain { + u64 *parent_ptes[NR_PTE_CHAIN_ENTRIES]; + struct hlist_node link; +}; + +/* + * kvm_mmu_page_role, below, is defined as: + * + * bits 0:3 - total guest paging levels (2-4, or zero for real mode) + * bits 4:7 - page table level for this shadow (1-4) + * bits 8:9 - page table quadrant for 2-level guests + * bit 16 - "metaphysical" - gfn is not a real page (huge page/real mode) + * bits 17:19 - common access permissions for all ptes in this shadow page + */ +union kvm_mmu_page_role { + unsigned word; + struct { + unsigned glevels : 4; + unsigned level : 4; + unsigned quadrant : 2; + unsigned pad_for_nice_hex_output : 6; + unsigned metaphysical : 1; + unsigned access : 3; + }; +}; + +struct kvm_mmu_page { + struct list_head link; + struct hlist_node hash_link; + + /* + * The following two entries are used to key the shadow page in the + * hash table. + */ + gfn_t gfn; + union kvm_mmu_page_role role; + + u64 *spt; + /* hold the gfn of each spte inside spt */ + gfn_t *gfns; + unsigned long slot_bitmap; /* One bit set per slot which has memory + * in this shadow page. + */ + int multimapped; /* More than one parent_pte? */ + int root_count; /* Currently serving as active root */ + union { + u64 *parent_pte; /* !multimapped */ + struct hlist_head parent_ptes; /* multimapped, kvm_pte_chain */ + }; +}; + +/* + * x86 supports 3 paging modes (4-level 64-bit, 3-level 64-bit, and 2-level + * 32-bit). The kvm_mmu structure abstracts the details of the current mmu + * mode. + */ +struct kvm_mmu { + void (*new_cr3)(struct kvm_vcpu *vcpu); + int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err); + void (*free)(struct kvm_vcpu *vcpu); + gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva); + void (*prefetch_page)(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *page); + hpa_t root_hpa; + int root_level; + int shadow_root_level; + + u64 *pae_root; +}; + struct kvm_vcpu_arch { u64 host_tsc; int interrupt_window_open; @@ -162,11 +255,6 @@ struct kvm_vcpu_arch { struct x86_emulate_ctxt emulate_ctxt; }; -struct kvm_vcpu { - KVM_VCPU_COMM; - - struct kvm_vcpu_arch arch; -}; struct descriptor_table { u16 limit; From d17fbbf7384d3b639f3c3299b5248ec4c4404556 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 09:45:31 +0800 Subject: [PATCH 217/249] KVM: Portability: Expand the KVM_VCPU_COMM in kvm_vcpu structure. This patches removes KVM_COMM macro, original it is hold kvm_vcpu common fields. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 57 ++++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 53717be80bbe..039faa766c57 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -87,40 +87,31 @@ struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr); void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev); -#ifdef CONFIG_HAS_IOMEM -#define KVM_VCPU_MMIO \ - int mmio_needed; \ - int mmio_read_completed; \ - int mmio_is_write; \ - int mmio_size; \ - unsigned char mmio_data[8]; \ - gpa_t mmio_phys_addr; - -#else -#define KVM_VCPU_MMIO - -#endif - -#define KVM_VCPU_COMM \ - struct kvm *kvm; \ - struct preempt_notifier preempt_notifier; \ - int vcpu_id; \ - struct mutex mutex; \ - int cpu; \ - struct kvm_run *run; \ - int guest_mode; \ - unsigned long requests; \ - struct kvm_guest_debug guest_debug; \ - int fpu_active; \ - int guest_fpu_loaded; \ - wait_queue_head_t wq; \ - int sigset_active; \ - sigset_t sigset; \ - struct kvm_vcpu_stat stat; \ - KVM_VCPU_MMIO - struct kvm_vcpu { - KVM_VCPU_COMM; + struct kvm *kvm; + struct preempt_notifier preempt_notifier; + int vcpu_id; + struct mutex mutex; + int cpu; + struct kvm_run *run; + int guest_mode; + unsigned long requests; + struct kvm_guest_debug guest_debug; + int fpu_active; + int guest_fpu_loaded; + wait_queue_head_t wq; + int sigset_active; + sigset_t sigset; + struct kvm_vcpu_stat stat; + +#ifdef CONFIG_HAS_IOMEM + int mmio_needed; + int mmio_read_completed; + int mmio_is_write; + int mmio_size; + unsigned char mmio_data[8]; + gpa_t mmio_phys_addr; +#endif struct kvm_vcpu_arch arch; }; From 77b4c255af34e73ea1efd1c3384bbe91361c81e6 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 09:49:26 +0800 Subject: [PATCH 218/249] KVM: Portability: Move kvm_vcpu_stat to x86.h This patches moves kvm_vcpu_stat to x86.h, so every arch can define its own kvm_vcpu_stat structure. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 22 ---------------------- drivers/kvm/x86.h | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 039faa766c57..9ea7149e741d 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -47,28 +47,6 @@ struct kvm_guest_debug { int singlestep; }; -struct kvm_vcpu_stat { - u32 pf_fixed; - u32 pf_guest; - u32 tlb_flush; - u32 invlpg; - - u32 exits; - u32 io_exits; - u32 mmio_exits; - u32 signal_exits; - u32 irq_window_exits; - u32 halt_exits; - u32 halt_wakeup; - u32 request_irq_exits; - u32 irq_exits; - u32 host_state_reload; - u32 efer_reload; - u32 fpu_reload; - u32 insn_emulation; - u32 insn_emulation_fail; -}; - /* * It would be nice to use something smarter than a linear search, TBD... * Thankfully we dont expect many devices to register (famous last words :), diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index a0dd4473b8f8..4ca848536365 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -255,6 +255,27 @@ struct kvm_vcpu_arch { struct x86_emulate_ctxt emulate_ctxt; }; +struct kvm_vcpu_stat { + u32 pf_fixed; + u32 pf_guest; + u32 tlb_flush; + u32 invlpg; + + u32 exits; + u32 io_exits; + u32 mmio_exits; + u32 signal_exits; + u32 irq_window_exits; + u32 halt_exits; + u32 halt_wakeup; + u32 request_irq_exits; + u32 irq_exits; + u32 host_state_reload; + u32 efer_reload; + u32 fpu_reload; + u32 insn_emulation; + u32 insn_emulation_fail; +}; struct descriptor_table { u16 limit; From d69fb81f0554fb980e4b1d3db4e44351c2c4a4a2 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 09:54:20 +0800 Subject: [PATCH 219/249] KVM: Portability: Move memslot aliases to new struct kvm_arch This patches create kvm_arch to hold arch-specific kvm fileds and moves fields naliases and aliases to kvm_arch. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 10 +--------- drivers/kvm/x86.c | 10 +++++----- drivers/kvm/x86.h | 13 +++++++++++++ 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 9ea7149e741d..bf5b85c1f094 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -25,7 +25,6 @@ #include "x86.h" #define KVM_MAX_VCPUS 4 -#define KVM_ALIAS_SLOTS 4 #define KVM_MEMORY_SLOTS 8 /* memory slots that does not exposed to userspace */ #define KVM_PRIVATE_MEM_SLOTS 4 @@ -94,12 +93,6 @@ struct kvm_vcpu { struct kvm_vcpu_arch arch; }; -struct kvm_mem_alias { - gfn_t base_gfn; - unsigned long npages; - gfn_t target_gfn; -}; - struct kvm_memory_slot { gfn_t base_gfn; unsigned long npages; @@ -123,8 +116,6 @@ struct kvm_vm_stat { struct kvm { struct mutex lock; /* protects everything except vcpus */ struct mm_struct *mm; /* userspace tied to this vm */ - int naliases; - struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS]; int nmemslots; struct kvm_memory_slot memslots[KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS]; @@ -147,6 +138,7 @@ struct kvm { unsigned int tss_addr; struct page *apic_access_page; struct kvm_vm_stat stat; + struct kvm_arch arch; }; /* The guest did something we don't support. */ diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 60f9722a06ba..7e1bd526bd5c 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1191,8 +1191,8 @@ gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) int i; struct kvm_mem_alias *alias; - for (i = 0; i < kvm->naliases; ++i) { - alias = &kvm->aliases[i]; + for (i = 0; i < kvm->arch.naliases; ++i) { + alias = &kvm->arch.aliases[i]; if (gfn >= alias->base_gfn && gfn < alias->base_gfn + alias->npages) return alias->target_gfn + gfn - alias->base_gfn; @@ -1228,15 +1228,15 @@ static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm, mutex_lock(&kvm->lock); - p = &kvm->aliases[alias->slot]; + p = &kvm->arch.aliases[alias->slot]; p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT; p->npages = alias->memory_size >> PAGE_SHIFT; p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT; for (n = KVM_ALIAS_SLOTS; n > 0; --n) - if (kvm->aliases[n - 1].npages) + if (kvm->arch.aliases[n - 1].npages) break; - kvm->naliases = n; + kvm->arch.naliases = n; kvm_mmu_zap_all(kvm); diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 4ca848536365..be84f2b89095 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -54,6 +54,8 @@ #define IOPL_SHIFT 12 +#define KVM_ALIAS_SLOTS 4 + #define KVM_PERMILLE_MMU_PAGES 20 #define KVM_MIN_ALLOC_MMU_PAGES 64 #define KVM_NUM_MMU_PAGES 1024 @@ -255,6 +257,17 @@ struct kvm_vcpu_arch { struct x86_emulate_ctxt emulate_ctxt; }; +struct kvm_mem_alias { + gfn_t base_gfn; + unsigned long npages; + gfn_t target_gfn; +}; + +struct kvm_arch{ + int naliases; + struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS]; +}; + struct kvm_vcpu_stat { u32 pf_fixed; u32 pf_guest; From f05e70ac03a6614af12194a014b338ec5594cb5c Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 10:01:48 +0800 Subject: [PATCH 220/249] KVM: Portability: Move mmu-related fields to kvm_arch This patches moves mmu-related fields to kvm_arch. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 8 ------- drivers/kvm/mmu.c | 58 ++++++++++++++++++++++++----------------------- drivers/kvm/mmu.h | 2 +- drivers/kvm/x86.c | 8 +++---- drivers/kvm/x86.h | 9 ++++++++ 5 files changed, 44 insertions(+), 41 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index bf5b85c1f094..65de5e4225f7 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -119,14 +119,6 @@ struct kvm { int nmemslots; struct kvm_memory_slot memslots[KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS]; - /* - * Hash table of struct kvm_mmu_page. - */ - struct list_head active_mmu_pages; - unsigned int n_free_mmu_pages; - unsigned int n_requested_mmu_pages; - unsigned int n_alloc_mmu_pages; - struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES]; struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; struct list_head vm_list; struct file *filp; diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 1dc0e8c02c70..c26d83f86a3a 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -553,7 +553,7 @@ static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp) __free_page(virt_to_page(sp->spt)); __free_page(virt_to_page(sp->gfns)); kfree(sp); - ++kvm->n_free_mmu_pages; + ++kvm->arch.n_free_mmu_pages; } static unsigned kvm_page_table_hashfn(gfn_t gfn) @@ -566,19 +566,19 @@ static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, { struct kvm_mmu_page *sp; - if (!vcpu->kvm->n_free_mmu_pages) + if (!vcpu->kvm->arch.n_free_mmu_pages) return NULL; sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp); sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE); sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE); set_page_private(virt_to_page(sp->spt), (unsigned long)sp); - list_add(&sp->link, &vcpu->kvm->active_mmu_pages); + list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages); ASSERT(is_empty_shadow_page(sp->spt)); sp->slot_bitmap = 0; sp->multimapped = 0; sp->parent_pte = parent_pte; - --vcpu->kvm->n_free_mmu_pages; + --vcpu->kvm->arch.n_free_mmu_pages; return sp; } @@ -666,7 +666,7 @@ static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn) pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn); index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; - bucket = &kvm->mmu_page_hash[index]; + bucket = &kvm->arch.mmu_page_hash[index]; hlist_for_each_entry(sp, node, bucket, hash_link) if (sp->gfn == gfn && !sp->role.metaphysical) { pgprintk("%s: found role %x\n", @@ -705,7 +705,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, pgprintk("%s: looking gfn %lx role %x\n", __FUNCTION__, gfn, role.word); index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; - bucket = &vcpu->kvm->mmu_page_hash[index]; + bucket = &vcpu->kvm->arch.mmu_page_hash[index]; hlist_for_each_entry(sp, node, bucket, hash_link) if (sp->gfn == gfn && sp->role.word == role.word) { mmu_page_add_parent_pte(vcpu, sp, parent_pte); @@ -796,7 +796,7 @@ static void kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp) hlist_del(&sp->hash_link); kvm_mmu_free_page(kvm, sp); } else - list_move(&sp->link, &kvm->active_mmu_pages); + list_move(&sp->link, &kvm->arch.active_mmu_pages); kvm_mmu_reset_last_pte_updated(kvm); } @@ -812,26 +812,26 @@ void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages) * change the value */ - if ((kvm->n_alloc_mmu_pages - kvm->n_free_mmu_pages) > + if ((kvm->arch.n_alloc_mmu_pages - kvm->arch.n_free_mmu_pages) > kvm_nr_mmu_pages) { - int n_used_mmu_pages = kvm->n_alloc_mmu_pages - - kvm->n_free_mmu_pages; + int n_used_mmu_pages = kvm->arch.n_alloc_mmu_pages + - kvm->arch.n_free_mmu_pages; while (n_used_mmu_pages > kvm_nr_mmu_pages) { struct kvm_mmu_page *page; - page = container_of(kvm->active_mmu_pages.prev, + page = container_of(kvm->arch.active_mmu_pages.prev, struct kvm_mmu_page, link); kvm_mmu_zap_page(kvm, page); n_used_mmu_pages--; } - kvm->n_free_mmu_pages = 0; + kvm->arch.n_free_mmu_pages = 0; } else - kvm->n_free_mmu_pages += kvm_nr_mmu_pages - - kvm->n_alloc_mmu_pages; + kvm->arch.n_free_mmu_pages += kvm_nr_mmu_pages + - kvm->arch.n_alloc_mmu_pages; - kvm->n_alloc_mmu_pages = kvm_nr_mmu_pages; + kvm->arch.n_alloc_mmu_pages = kvm_nr_mmu_pages; } static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) @@ -845,7 +845,7 @@ static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn); r = 0; index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; - bucket = &kvm->mmu_page_hash[index]; + bucket = &kvm->arch.mmu_page_hash[index]; hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) if (sp->gfn == gfn && !sp->role.metaphysical) { pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn, @@ -1362,7 +1362,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, vcpu->arch.last_pte_updated = NULL; } index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; - bucket = &vcpu->kvm->mmu_page_hash[index]; + bucket = &vcpu->kvm->arch.mmu_page_hash[index]; hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) { if (sp->gfn != gfn || sp->role.metaphysical) continue; @@ -1428,10 +1428,10 @@ int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) { - while (vcpu->kvm->n_free_mmu_pages < KVM_REFILL_PAGES) { + while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) { struct kvm_mmu_page *sp; - sp = container_of(vcpu->kvm->active_mmu_pages.prev, + sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev, struct kvm_mmu_page, link); kvm_mmu_zap_page(vcpu->kvm, sp); ++vcpu->kvm->stat.mmu_recycled; @@ -1482,8 +1482,8 @@ static void free_mmu_pages(struct kvm_vcpu *vcpu) { struct kvm_mmu_page *sp; - while (!list_empty(&vcpu->kvm->active_mmu_pages)) { - sp = container_of(vcpu->kvm->active_mmu_pages.next, + while (!list_empty(&vcpu->kvm->arch.active_mmu_pages)) { + sp = container_of(vcpu->kvm->arch.active_mmu_pages.next, struct kvm_mmu_page, link); kvm_mmu_zap_page(vcpu->kvm, sp); } @@ -1497,10 +1497,12 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu) ASSERT(vcpu); - if (vcpu->kvm->n_requested_mmu_pages) - vcpu->kvm->n_free_mmu_pages = vcpu->kvm->n_requested_mmu_pages; + if (vcpu->kvm->arch.n_requested_mmu_pages) + vcpu->kvm->arch.n_free_mmu_pages = + vcpu->kvm->arch.n_requested_mmu_pages; else - vcpu->kvm->n_free_mmu_pages = vcpu->kvm->n_alloc_mmu_pages; + vcpu->kvm->arch.n_free_mmu_pages = + vcpu->kvm->arch.n_alloc_mmu_pages; /* * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64. * Therefore we need to allocate shadow page tables in the first @@ -1549,7 +1551,7 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot) { struct kvm_mmu_page *sp; - list_for_each_entry(sp, &kvm->active_mmu_pages, link) { + list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) { int i; u64 *pt; @@ -1568,7 +1570,7 @@ void kvm_mmu_zap_all(struct kvm *kvm) { struct kvm_mmu_page *sp, *node; - list_for_each_entry_safe(sp, node, &kvm->active_mmu_pages, link) + list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) kvm_mmu_zap_page(kvm, sp); kvm_flush_remote_tlbs(kvm); @@ -1738,7 +1740,7 @@ static int count_writable_mappings(struct kvm_vcpu *vcpu) struct kvm_mmu_page *sp; int i; - list_for_each_entry(sp, &vcpu->kvm->active_mmu_pages, link) { + list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) { u64 *pt = sp->spt; if (sp->role.level != PT_PAGE_TABLE_LEVEL) @@ -1774,7 +1776,7 @@ static void audit_write_protection(struct kvm_vcpu *vcpu) unsigned long *rmapp; gfn_t gfn; - list_for_each_entry(sp, &vcpu->kvm->active_mmu_pages, link) { + list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) { if (sp->role.metaphysical) continue; diff --git a/drivers/kvm/mmu.h b/drivers/kvm/mmu.h index 9ebfd1cafe62..cbfc272262df 100644 --- a/drivers/kvm/mmu.h +++ b/drivers/kvm/mmu.h @@ -5,7 +5,7 @@ static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) { - if (unlikely(vcpu->kvm->n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES)) + if (unlikely(vcpu->kvm->arch.n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES)) __kvm_mmu_free_some_pages(vcpu); } diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 7e1bd526bd5c..c0e95fb9f46c 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1175,7 +1175,7 @@ static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, mutex_lock(&kvm->lock); kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages); - kvm->n_requested_mmu_pages = kvm_nr_mmu_pages; + kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages; mutex_unlock(&kvm->lock); return 0; @@ -1183,7 +1183,7 @@ static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm) { - return kvm->n_alloc_mmu_pages; + return kvm->arch.n_alloc_mmu_pages; } gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) @@ -3051,7 +3051,7 @@ struct kvm *kvm_arch_create_vm(void) if (!kvm) return ERR_PTR(-ENOMEM); - INIT_LIST_HEAD(&kvm->active_mmu_pages); + INIT_LIST_HEAD(&kvm->arch.active_mmu_pages); return kvm; } @@ -3130,7 +3130,7 @@ int kvm_arch_set_memory_region(struct kvm *kvm, } } - if (!kvm->n_requested_mmu_pages) { + if (!kvm->arch.n_requested_mmu_pages) { unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm); kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); } diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index be84f2b89095..5cdc3666e212 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -266,6 +266,15 @@ struct kvm_mem_alias { struct kvm_arch{ int naliases; struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS]; + + unsigned int n_free_mmu_pages; + unsigned int n_requested_mmu_pages; + unsigned int n_alloc_mmu_pages; + struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES]; + /* + * Hash table of struct kvm_mmu_page. + */ + struct list_head active_mmu_pages; }; struct kvm_vcpu_stat { From d7deeeb02cf73fd98cb71a6a0a5dffab9ef79556 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 10:17:34 +0800 Subject: [PATCH 221/249] KVM: Portability: move vpic and vioapic to kvm_arch This patches moves two fields vpid and vioapic to kvm_arch Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/ioapic.c | 4 ++-- drivers/kvm/irq.h | 4 ++-- drivers/kvm/kvm.h | 2 -- drivers/kvm/x86.c | 14 +++++++------- drivers/kvm/x86.h | 2 ++ 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/drivers/kvm/ioapic.c b/drivers/kvm/ioapic.c index 04910368c251..f8236774c1b4 100644 --- a/drivers/kvm/ioapic.c +++ b/drivers/kvm/ioapic.c @@ -271,7 +271,7 @@ static int get_eoi_gsi(struct kvm_ioapic *ioapic, int vector) void kvm_ioapic_update_eoi(struct kvm *kvm, int vector) { - struct kvm_ioapic *ioapic = kvm->vioapic; + struct kvm_ioapic *ioapic = kvm->arch.vioapic; union ioapic_redir_entry *ent; int gsi; @@ -390,7 +390,7 @@ int kvm_ioapic_init(struct kvm *kvm) ioapic = kzalloc(sizeof(struct kvm_ioapic), GFP_KERNEL); if (!ioapic) return -ENOMEM; - kvm->vioapic = ioapic; + kvm->arch.vioapic = ioapic; kvm_ioapic_reset(ioapic); ioapic->dev.read = ioapic_mmio_read; ioapic->dev.write = ioapic_mmio_write; diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index a688c51d0a39..6e023dc3f848 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -148,12 +148,12 @@ do { \ static inline struct kvm_pic *pic_irqchip(struct kvm *kvm) { - return kvm->vpic; + return kvm->arch.vpic; } static inline struct kvm_ioapic *ioapic_irqchip(struct kvm *kvm) { - return kvm->vioapic; + return kvm->arch.vioapic; } static inline int irqchip_in_kernel(struct kvm *kvm) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 65de5e4225f7..d9ce916e98ab 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -124,8 +124,6 @@ struct kvm { struct file *filp; struct kvm_io_bus mmio_bus; struct kvm_io_bus pio_bus; - struct kvm_pic *vpic; - struct kvm_ioapic *vioapic; int round_robin_prev_vcpu; unsigned int tss_addr; struct page *apic_access_page; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index c0e95fb9f46c..b37c0093d728 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1386,12 +1386,12 @@ long kvm_arch_vm_ioctl(struct file *filp, } case KVM_CREATE_IRQCHIP: r = -ENOMEM; - kvm->vpic = kvm_create_pic(kvm); - if (kvm->vpic) { + kvm->arch.vpic = kvm_create_pic(kvm); + if (kvm->arch.vpic) { r = kvm_ioapic_init(kvm); if (r) { - kfree(kvm->vpic); - kvm->vpic = NULL; + kfree(kvm->arch.vpic); + kvm->arch.vpic = NULL; goto out; } } else @@ -1409,7 +1409,7 @@ long kvm_arch_vm_ioctl(struct file *filp, kvm_pic_set_irq(pic_irqchip(kvm), irq_event.irq, irq_event.level); - kvm_ioapic_set_irq(kvm->vioapic, + kvm_ioapic_set_irq(kvm->arch.vioapic, irq_event.irq, irq_event.level); mutex_unlock(&kvm->lock); @@ -3084,8 +3084,8 @@ static void kvm_free_vcpus(struct kvm *kvm) void kvm_arch_destroy_vm(struct kvm *kvm) { - kfree(kvm->vpic); - kfree(kvm->vioapic); + kfree(kvm->arch.vpic); + kfree(kvm->arch.vioapic); kvm_free_vcpus(kvm); kvm_free_physmem(kvm); kfree(kvm); diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 5cdc3666e212..ef23a30bf048 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -275,6 +275,8 @@ struct kvm_arch{ * Hash table of struct kvm_mmu_page. */ struct list_head active_mmu_pages; + struct kvm_pic *vpic; + struct kvm_ioapic *vioapic; }; struct kvm_vcpu_stat { From bfc6d222bdb1123d12c1193bcd7c755e6617b405 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 10:20:16 +0800 Subject: [PATCH 222/249] KVM: Portability: Move round_robin_prev_vcpu and tss_addr to kvm_arch This patches moves two fields round_robin_prev_vcpu and tss to kvm_arch. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 --- drivers/kvm/lapic.c | 4 ++-- drivers/kvm/vmx.c | 12 ++++++------ drivers/kvm/x86.h | 4 ++++ 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index d9ce916e98ab..ab9657a6f0ce 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -124,9 +124,6 @@ struct kvm { struct file *filp; struct kvm_io_bus mmio_bus; struct kvm_io_bus pio_bus; - int round_robin_prev_vcpu; - unsigned int tss_addr; - struct page *apic_access_page; struct kvm_vm_stat stat; struct kvm_arch arch; }; diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index 5c9f46784c26..8c74bf184a07 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -404,7 +404,7 @@ static struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, int next; struct kvm_lapic *apic = NULL; - last = kvm->round_robin_prev_vcpu; + last = kvm->arch.round_robin_prev_vcpu; next = last; do { @@ -417,7 +417,7 @@ static struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, break; apic = NULL; } while (next != last); - kvm->round_robin_prev_vcpu = next; + kvm->arch.round_robin_prev_vcpu = next; if (!apic) printk(KERN_DEBUG "vcpu not ready for apic_round_robin\n"); diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index d0f431d4fe4d..7e61a560aab5 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1143,12 +1143,12 @@ static void enter_pmode(struct kvm_vcpu *vcpu) static gva_t rmode_tss_base(struct kvm *kvm) { - if (!kvm->tss_addr) { + if (!kvm->arch.tss_addr) { gfn_t base_gfn = kvm->memslots[0].base_gfn + kvm->memslots[0].npages - 3; return base_gfn << PAGE_SHIFT; } - return kvm->tss_addr; + return kvm->arch.tss_addr; } static void fix_rmode_seg(int seg, struct kvm_save_segment *save) @@ -1473,7 +1473,7 @@ static int alloc_apic_access_page(struct kvm *kvm) int r = 0; mutex_lock(&kvm->lock); - if (kvm->apic_access_page) + if (kvm->arch.apic_access_page) goto out; kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT; kvm_userspace_mem.flags = 0; @@ -1482,7 +1482,7 @@ static int alloc_apic_access_page(struct kvm *kvm) r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0); if (r) goto out; - kvm->apic_access_page = gfn_to_page(kvm, 0xfee00); + kvm->arch.apic_access_page = gfn_to_page(kvm, 0xfee00); out: mutex_unlock(&kvm->lock); return r; @@ -1699,7 +1699,7 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm)) vmcs_write64(APIC_ACCESS_ADDR, - page_to_phys(vmx->vcpu.kvm->apic_access_page)); + page_to_phys(vmx->vcpu.kvm->arch.apic_access_page)); vmx->vcpu.arch.cr0 = 0x60000010; vmx_set_cr0(&vmx->vcpu, vmx->vcpu.arch.cr0); /* enter rmode */ @@ -1789,7 +1789,7 @@ static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr) ret = kvm_set_memory_region(kvm, &tss_mem, 0); if (ret) return ret; - kvm->tss_addr = addr; + kvm->arch.tss_addr = addr; return 0; } diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index ef23a30bf048..945583934853 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -277,6 +277,10 @@ struct kvm_arch{ struct list_head active_mmu_pages; struct kvm_pic *vpic; struct kvm_ioapic *vioapic; + + int round_robin_prev_vcpu; + unsigned int tss_addr; + struct page *apic_access_page; }; struct kvm_vcpu_stat { From 0711456c0d78335d24e4f92aa40d32c78fd44aa8 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 10:23:23 +0800 Subject: [PATCH 223/249] KVM: Portability: Move kvm_vm_stat to x86.h This patch moves kvm_vm_stat to x86.h, and every arch can define its own kvm_vm_stat in $arch.h Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 10 ---------- drivers/kvm/x86.h | 10 ++++++++++ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index ab9657a6f0ce..bf6a3b330a3d 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -103,16 +103,6 @@ struct kvm_memory_slot { int user_alloc; }; -struct kvm_vm_stat { - u32 mmu_shadow_zapped; - u32 mmu_pte_write; - u32 mmu_pte_updated; - u32 mmu_pde_zapped; - u32 mmu_flooded; - u32 mmu_recycled; - u32 remote_tlb_flush; -}; - struct kvm { struct mutex lock; /* protects everything except vcpus */ struct mm_struct *mm; /* userspace tied to this vm */ diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 945583934853..dfb8091971a9 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -283,6 +283,16 @@ struct kvm_arch{ struct page *apic_access_page; }; +struct kvm_vm_stat { + u32 mmu_shadow_zapped; + u32 mmu_pte_write; + u32 mmu_pte_updated; + u32 mmu_pde_zapped; + u32 mmu_flooded; + u32 mmu_recycled; + u32 remote_tlb_flush; +}; + struct kvm_vcpu_stat { u32 pf_fixed; u32 pf_guest; From 9584bf2c93f56656dba0de8f6c75b54ca7995143 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 13 Dec 2007 10:21:10 -0600 Subject: [PATCH 224/249] KVM: VMX: Add printk_ratelimit in vmx_intr_assist Add printk_ratelimit check in front of printk. This prevents spamming of the message during 32-bit ubuntu 6.06server install. Previously, it would hang during the partition formatting stage. Signed-off-by: Ryan Harper Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 7e61a560aab5..11ca2340d38f 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -2263,7 +2263,8 @@ static void vmx_intr_assist(struct kvm_vcpu *vcpu) if (intr_info_field & INTR_INFO_VALID_MASK) { if (idtv_info_field & INTR_INFO_VALID_MASK) { /* TODO: fault when IDT_Vectoring */ - printk(KERN_ERR "Fault when IDT_Vectoring\n"); + if (printk_ratelimit()) + printk(KERN_ERR "Fault when IDT_Vectoring\n"); } if (has_ext_irq) enable_irq_window(vcpu); From edf884172e9828c6234b254208af04655855038d Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 16 Dec 2007 11:02:48 +0200 Subject: [PATCH 225/249] KVM: Move arch dependent files to new directory arch/x86/kvm/ This paves the way for multiple architecture support. Note that while ioapic.c could potentially be shared with ia64, it is also moved. Signed-off-by: Avi Kivity --- arch/x86/Kconfig | 2 ++ arch/x86/Makefile | 2 ++ {drivers => arch/x86}/kvm/Kconfig | 0 {drivers => arch/x86}/kvm/Makefile | 7 ++++++- {drivers => arch/x86}/kvm/i8259.c | 3 ++- {drivers => arch/x86}/kvm/ioapic.c | 4 +--- {drivers => arch/x86}/kvm/irq.c | 3 +-- {drivers => arch/x86}/kvm/irq.h | 3 +-- {drivers => arch/x86}/kvm/kvm_svm.h | 2 +- {drivers => arch/x86}/kvm/lapic.c | 4 +--- {drivers => arch/x86}/kvm/mmu.c | 3 +-- {drivers => arch/x86}/kvm/mmu.h | 2 +- {drivers => arch/x86}/kvm/paging_tmpl.h | 0 {drivers => arch/x86}/kvm/segment_descriptor.h | 0 {drivers => arch/x86}/kvm/svm.c | 4 ++-- {drivers => arch/x86}/kvm/svm.h | 0 {drivers => arch/x86}/kvm/vmx.c | 4 +--- {drivers => arch/x86}/kvm/vmx.h | 0 {drivers => arch/x86}/kvm/x86.c | 4 +--- {drivers => arch/x86}/kvm/x86_emulate.c | 5 ++--- drivers/Kconfig | 2 -- drivers/Makefile | 1 - drivers/kvm/iodev.h | 2 +- drivers/kvm/kvm_main.c | 2 +- drivers/kvm/x86.h => include/asm-x86/kvm_host.h | 9 ++++----- .../x86_emulate.h => include/asm-x86/kvm_x86_emulate.h | 0 drivers/kvm/kvm.h => include/linux/kvm_host.h | 8 ++++---- drivers/kvm/types.h => include/linux/kvm_types.h | 0 28 files changed, 35 insertions(+), 41 deletions(-) rename {drivers => arch/x86}/kvm/Kconfig (100%) rename {drivers => arch/x86}/kvm/Makefile (52%) rename {drivers => arch/x86}/kvm/i8259.c (99%) rename {drivers => arch/x86}/kvm/ioapic.c (99%) rename {drivers => arch/x86}/kvm/irq.c (98%) rename {drivers => arch/x86}/kvm/irq.h (99%) rename {drivers => arch/x86}/kvm/kvm_svm.h (96%) rename {drivers => arch/x86}/kvm/lapic.c (99%) rename {drivers => arch/x86}/kvm/mmu.c (99%) rename {drivers => arch/x86}/kvm/mmu.h (96%) rename {drivers => arch/x86}/kvm/paging_tmpl.h (100%) rename {drivers => arch/x86}/kvm/segment_descriptor.h (100%) rename {drivers => arch/x86}/kvm/svm.c (99%) rename {drivers => arch/x86}/kvm/svm.h (100%) rename {drivers => arch/x86}/kvm/vmx.c (99%) rename {drivers => arch/x86}/kvm/vmx.h (100%) rename {drivers => arch/x86}/kvm/x86.c (99%) rename {drivers => arch/x86}/kvm/x86_emulate.c (99%) rename drivers/kvm/x86.h => include/asm-x86/kvm_host.h (99%) rename drivers/kvm/x86_emulate.h => include/asm-x86/kvm_x86_emulate.h (100%) rename drivers/kvm/kvm.h => include/linux/kvm_host.h (98%) rename drivers/kvm/types.h => include/linux/kvm_types.h (100%) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index d289cfcf92c4..65b449134cf7 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1599,4 +1599,6 @@ source "security/Kconfig" source "crypto/Kconfig" +source "arch/x86/kvm/Kconfig" + source "lib/Kconfig" diff --git a/arch/x86/Makefile b/arch/x86/Makefile index b08f18261df6..da8f4129780b 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -7,6 +7,8 @@ else KBUILD_DEFCONFIG := $(ARCH)_defconfig endif +core-$(CONFIG_KVM) += arch/x86/kvm/ + # BITS is used as extension for files which are available in a 32 bit # and a 64 bit version to simplify shared Makefiles. # e.g.: obj-y += foo_$(BITS).o diff --git a/drivers/kvm/Kconfig b/arch/x86/kvm/Kconfig similarity index 100% rename from drivers/kvm/Kconfig rename to arch/x86/kvm/Kconfig diff --git a/drivers/kvm/Makefile b/arch/x86/kvm/Makefile similarity index 52% rename from drivers/kvm/Makefile rename to arch/x86/kvm/Makefile index cf18ad46e987..880ffe403b35 100644 --- a/drivers/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -2,7 +2,12 @@ # Makefile for Kernel-based Virtual Machine module # -kvm-objs := kvm_main.o x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o ioapic.o +common-objs = $(addprefix ../../../drivers/kvm/, kvm_main.o) + +EXTRA_CFLAGS += -I drivers/kvm + +kvm-objs := $(common-objs) x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o \ + ioapic.o obj-$(CONFIG_KVM) += kvm.o kvm-intel-objs = vmx.o obj-$(CONFIG_KVM_INTEL) += kvm-intel.o diff --git a/drivers/kvm/i8259.c b/arch/x86/kvm/i8259.c similarity index 99% rename from drivers/kvm/i8259.c rename to arch/x86/kvm/i8259.c index b3cad632f3d5..ab29cf2def47 100644 --- a/drivers/kvm/i8259.c +++ b/arch/x86/kvm/i8259.c @@ -27,7 +27,8 @@ */ #include #include "irq.h" -#include "kvm.h" + +#include /* * set irq level. If an edge is detected, then the IRR is set to 1 diff --git a/drivers/kvm/ioapic.c b/arch/x86/kvm/ioapic.c similarity index 99% rename from drivers/kvm/ioapic.c rename to arch/x86/kvm/ioapic.c index f8236774c1b4..72f12f75495d 100644 --- a/drivers/kvm/ioapic.c +++ b/arch/x86/kvm/ioapic.c @@ -26,9 +26,7 @@ * Based on Xen 3.1 code. */ -#include "kvm.h" -#include "x86.h" - +#include #include #include #include diff --git a/drivers/kvm/irq.c b/arch/x86/kvm/irq.c similarity index 98% rename from drivers/kvm/irq.c rename to arch/x86/kvm/irq.c index 59b47c55fc76..07a09aad4fd6 100644 --- a/drivers/kvm/irq.c +++ b/arch/x86/kvm/irq.c @@ -20,9 +20,8 @@ */ #include +#include -#include "kvm.h" -#include "x86.h" #include "irq.h" /* diff --git a/drivers/kvm/irq.h b/arch/x86/kvm/irq.h similarity index 99% rename from drivers/kvm/irq.h rename to arch/x86/kvm/irq.h index 6e023dc3f848..6316638eec9f 100644 --- a/drivers/kvm/irq.h +++ b/arch/x86/kvm/irq.h @@ -24,9 +24,8 @@ #include #include -#include +#include #include "iodev.h" -#include "kvm.h" struct kvm; struct kvm_vcpu; diff --git a/drivers/kvm/kvm_svm.h b/arch/x86/kvm/kvm_svm.h similarity index 96% rename from drivers/kvm/kvm_svm.h rename to arch/x86/kvm/kvm_svm.h index a0e415daef5b..ecdfe97e4635 100644 --- a/drivers/kvm/kvm_svm.h +++ b/arch/x86/kvm/kvm_svm.h @@ -4,10 +4,10 @@ #include #include #include +#include #include #include "svm.h" -#include "kvm.h" static const u32 host_save_user_msrs[] = { #ifdef CONFIG_X86_64 diff --git a/drivers/kvm/lapic.c b/arch/x86/kvm/lapic.c similarity index 99% rename from drivers/kvm/lapic.c rename to arch/x86/kvm/lapic.c index 8c74bf184a07..4076331b01ee 100644 --- a/drivers/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -17,9 +17,7 @@ * the COPYING file in the top-level directory. */ -#include "kvm.h" -#include "x86.h" - +#include #include #include #include diff --git a/drivers/kvm/mmu.c b/arch/x86/kvm/mmu.c similarity index 99% rename from drivers/kvm/mmu.c rename to arch/x86/kvm/mmu.c index c26d83f86a3a..401eb7ce3207 100644 --- a/drivers/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c @@ -18,10 +18,9 @@ */ #include "vmx.h" -#include "kvm.h" -#include "x86.h" #include "mmu.h" +#include #include #include #include diff --git a/drivers/kvm/mmu.h b/arch/x86/kvm/mmu.h similarity index 96% rename from drivers/kvm/mmu.h rename to arch/x86/kvm/mmu.h index cbfc272262df..1fce19ec7a23 100644 --- a/drivers/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -1,7 +1,7 @@ #ifndef __KVM_X86_MMU_H #define __KVM_X86_MMU_H -#include "kvm.h" +#include static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) { diff --git a/drivers/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h similarity index 100% rename from drivers/kvm/paging_tmpl.h rename to arch/x86/kvm/paging_tmpl.h diff --git a/drivers/kvm/segment_descriptor.h b/arch/x86/kvm/segment_descriptor.h similarity index 100% rename from drivers/kvm/segment_descriptor.h rename to arch/x86/kvm/segment_descriptor.h diff --git a/drivers/kvm/svm.c b/arch/x86/kvm/svm.c similarity index 99% rename from drivers/kvm/svm.c rename to arch/x86/kvm/svm.c index e606f6d18669..3d4b71a94440 100644 --- a/drivers/kvm/svm.c +++ b/arch/x86/kvm/svm.c @@ -13,9 +13,9 @@ * the COPYING file in the top-level directory. * */ -#include "x86.h" +#include + #include "kvm_svm.h" -#include "x86_emulate.h" #include "irq.h" #include "mmu.h" diff --git a/drivers/kvm/svm.h b/arch/x86/kvm/svm.h similarity index 100% rename from drivers/kvm/svm.h rename to arch/x86/kvm/svm.h diff --git a/drivers/kvm/vmx.c b/arch/x86/kvm/vmx.c similarity index 99% rename from drivers/kvm/vmx.c rename to arch/x86/kvm/vmx.c index 11ca2340d38f..fc494aff5d8b 100644 --- a/drivers/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -15,14 +15,12 @@ * */ -#include "kvm.h" -#include "x86.h" -#include "x86_emulate.h" #include "irq.h" #include "vmx.h" #include "segment_descriptor.h" #include "mmu.h" +#include #include #include #include diff --git a/drivers/kvm/vmx.h b/arch/x86/kvm/vmx.h similarity index 100% rename from drivers/kvm/vmx.h rename to arch/x86/kvm/vmx.h diff --git a/drivers/kvm/x86.c b/arch/x86/kvm/x86.c similarity index 99% rename from drivers/kvm/x86.c rename to arch/x86/kvm/x86.c index b37c0093d728..5902c5cbc1bb 100644 --- a/drivers/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -14,9 +14,7 @@ * */ -#include "kvm.h" -#include "x86.h" -#include "x86_emulate.h" +#include #include "segment_descriptor.h" #include "irq.h" #include "mmu.h" diff --git a/drivers/kvm/x86_emulate.c b/arch/x86/kvm/x86_emulate.c similarity index 99% rename from drivers/kvm/x86_emulate.c rename to arch/x86/kvm/x86_emulate.c index 50b133f68743..79586003397a 100644 --- a/drivers/kvm/x86_emulate.c +++ b/arch/x86/kvm/x86_emulate.c @@ -25,12 +25,11 @@ #include #define DPRINTF(_f, _a ...) printf(_f , ## _a) #else -#include "kvm.h" -#include "x86.h" +#include #define DPRINTF(x...) do {} while (0) #endif -#include "x86_emulate.h" #include +#include /* * Opcode effective-address decode tables. diff --git a/drivers/Kconfig b/drivers/Kconfig index f4076d9e9902..08d4ae201597 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -90,8 +90,6 @@ source "drivers/dca/Kconfig" source "drivers/auxdisplay/Kconfig" -source "drivers/kvm/Kconfig" - source "drivers/uio/Kconfig" source "drivers/virtio/Kconfig" diff --git a/drivers/Makefile b/drivers/Makefile index d92d4d82d001..9e1f808e43cf 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -47,7 +47,6 @@ obj-$(CONFIG_SPI) += spi/ obj-$(CONFIG_PCCARD) += pcmcia/ obj-$(CONFIG_DIO) += dio/ obj-$(CONFIG_SBUS) += sbus/ -obj-$(CONFIG_KVM) += kvm/ obj-$(CONFIG_ZORRO) += zorro/ obj-$(CONFIG_MAC) += macintosh/ obj-$(CONFIG_ATA_OVER_ETH) += block/aoe/ diff --git a/drivers/kvm/iodev.h b/drivers/kvm/iodev.h index eb9e8a71843a..c14e642027b2 100644 --- a/drivers/kvm/iodev.h +++ b/drivers/kvm/iodev.h @@ -16,7 +16,7 @@ #ifndef __KVM_IODEV_H__ #define __KVM_IODEV_H__ -#include "types.h" +#include struct kvm_io_device { void (*read)(struct kvm_io_device *this, diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index ae2a1bf640bc..4026d7d64296 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -15,9 +15,9 @@ * */ -#include "kvm.h" #include "iodev.h" +#include #include #include #include diff --git a/drivers/kvm/x86.h b/include/asm-x86/kvm_host.h similarity index 99% rename from drivers/kvm/x86.h rename to include/asm-x86/kvm_host.h index dfb8091971a9..28940e1a9713 100644 --- a/drivers/kvm/x86.h +++ b/include/asm-x86/kvm_host.h @@ -8,19 +8,18 @@ * */ -#ifndef KVM_X86_H -#define KVM_X86_H +#ifndef ASM_KVM_HOST_H +#define ASM_KVM_HOST_H #include #include #include #include +#include #include -#include "types.h" - #define CR3_PAE_RESERVED_BITS ((X86_CR3_PWT | X86_CR3_PCD) - 1) #define CR3_NONPAE_RESERVED_BITS ((PAGE_SIZE-1) & ~(X86_CR3_PWT | X86_CR3_PCD)) #define CR3_L_MODE_RESERVED_BITS (CR3_NONPAE_RESERVED_BITS|0xFFFFFF0000000000ULL) @@ -102,7 +101,7 @@ enum { VCPU_SREG_LDTR, }; -#include "x86_emulate.h" +#include #define KVM_NR_MEM_OBJS 40 diff --git a/drivers/kvm/x86_emulate.h b/include/asm-x86/kvm_x86_emulate.h similarity index 100% rename from drivers/kvm/x86_emulate.h rename to include/asm-x86/kvm_x86_emulate.h diff --git a/drivers/kvm/kvm.h b/include/linux/kvm_host.h similarity index 98% rename from drivers/kvm/kvm.h rename to include/linux/kvm_host.h index bf6a3b330a3d..a85d5b6943de 100644 --- a/drivers/kvm/kvm.h +++ b/include/linux/kvm_host.h @@ -1,5 +1,5 @@ -#ifndef __KVM_H -#define __KVM_H +#ifndef __KVM_HOST_H +#define __KVM_HOST_H /* * This work is licensed under the terms of the GNU GPL, version 2. See @@ -20,9 +20,9 @@ #include #include -#include "types.h" +#include -#include "x86.h" +#include #define KVM_MAX_VCPUS 4 #define KVM_MEMORY_SLOTS 8 diff --git a/drivers/kvm/types.h b/include/linux/kvm_types.h similarity index 100% rename from drivers/kvm/types.h rename to include/linux/kvm_types.h From 0fce5623ba248d3af0d7fb719d5ac367cc9d5192 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 16 Dec 2007 11:13:16 +0200 Subject: [PATCH 226/249] KVM: Move drivers/kvm/* to virt/kvm/ Signed-off-by: Avi Kivity --- arch/x86/kvm/Makefile | 4 ++-- {drivers => virt}/kvm/iodev.h | 0 {drivers => virt}/kvm/kvm_main.c | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename {drivers => virt}/kvm/iodev.h (100%) rename {drivers => virt}/kvm/kvm_main.c (100%) diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 880ffe403b35..af41534af496 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -2,9 +2,9 @@ # Makefile for Kernel-based Virtual Machine module # -common-objs = $(addprefix ../../../drivers/kvm/, kvm_main.o) +common-objs = $(addprefix ../../../virt/kvm/, kvm_main.o) -EXTRA_CFLAGS += -I drivers/kvm +EXTRA_CFLAGS += -I virt/kvm kvm-objs := $(common-objs) x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o \ ioapic.o diff --git a/drivers/kvm/iodev.h b/virt/kvm/iodev.h similarity index 100% rename from drivers/kvm/iodev.h rename to virt/kvm/iodev.h diff --git a/drivers/kvm/kvm_main.c b/virt/kvm/kvm_main.c similarity index 100% rename from drivers/kvm/kvm_main.c rename to virt/kvm/kvm_main.c From 82470196fa346f50ab4071935597017de33c9ee3 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Mon, 17 Dec 2007 13:59:56 +0800 Subject: [PATCH 227/249] KVM: Move irqchip declarations into new ioapic.h and lapic.h This allows reuse of ioapic in ia64. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- arch/x86/kvm/ioapic.c | 5 +- arch/x86/kvm/ioapic.h | 95 ++++++++++++++++++++++++++++++++++ arch/x86/kvm/irq.h | 115 ++---------------------------------------- arch/x86/kvm/lapic.h | 44 ++++++++++++++++ 4 files changed, 148 insertions(+), 111 deletions(-) create mode 100644 arch/x86/kvm/ioapic.h create mode 100644 arch/x86/kvm/lapic.h diff --git a/arch/x86/kvm/ioapic.c b/arch/x86/kvm/ioapic.c index 72f12f75495d..40cd53ed4abd 100644 --- a/arch/x86/kvm/ioapic.c +++ b/arch/x86/kvm/ioapic.c @@ -36,7 +36,10 @@ #include #include #include -#include "irq.h" + +#include "ioapic.h" +#include "lapic.h" + #if 0 #define ioapic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) #else diff --git a/arch/x86/kvm/ioapic.h b/arch/x86/kvm/ioapic.h new file mode 100644 index 000000000000..7f16675fe783 --- /dev/null +++ b/arch/x86/kvm/ioapic.h @@ -0,0 +1,95 @@ +#ifndef __KVM_IO_APIC_H +#define __KVM_IO_APIC_H + +#include + +#include "iodev.h" + +struct kvm; +struct kvm_vcpu; + +#define IOAPIC_NUM_PINS KVM_IOAPIC_NUM_PINS +#define IOAPIC_VERSION_ID 0x11 /* IOAPIC version */ +#define IOAPIC_EDGE_TRIG 0 +#define IOAPIC_LEVEL_TRIG 1 + +#define IOAPIC_DEFAULT_BASE_ADDRESS 0xfec00000 +#define IOAPIC_MEM_LENGTH 0x100 + +/* Direct registers. */ +#define IOAPIC_REG_SELECT 0x00 +#define IOAPIC_REG_WINDOW 0x10 +#define IOAPIC_REG_EOI 0x40 /* IA64 IOSAPIC only */ + +/* Indirect registers. */ +#define IOAPIC_REG_APIC_ID 0x00 /* x86 IOAPIC only */ +#define IOAPIC_REG_VERSION 0x01 +#define IOAPIC_REG_ARB_ID 0x02 /* x86 IOAPIC only */ + +/*ioapic delivery mode*/ +#define IOAPIC_FIXED 0x0 +#define IOAPIC_LOWEST_PRIORITY 0x1 +#define IOAPIC_PMI 0x2 +#define IOAPIC_NMI 0x4 +#define IOAPIC_INIT 0x5 +#define IOAPIC_EXTINT 0x7 + +struct kvm_ioapic { + u64 base_address; + u32 ioregsel; + u32 id; + u32 irr; + u32 pad; + union ioapic_redir_entry { + u64 bits; + struct { + u8 vector; + u8 delivery_mode:3; + u8 dest_mode:1; + u8 delivery_status:1; + u8 polarity:1; + u8 remote_irr:1; + u8 trig_mode:1; + u8 mask:1; + u8 reserve:7; + u8 reserved[4]; + u8 dest_id; + } fields; + } redirtbl[IOAPIC_NUM_PINS]; + struct kvm_io_device dev; + struct kvm *kvm; +}; + +#ifdef DEBUG +#define ASSERT(x) \ +do { \ + if (!(x)) { \ + printk(KERN_EMERG "assertion failed %s: %d: %s\n", \ + __FILE__, __LINE__, #x); \ + BUG(); \ + } \ +} while (0) +#else +#define ASSERT(x) do { } while (0) +#endif + +static inline struct kvm_ioapic *ioapic_irqchip(struct kvm *kvm) +{ + return kvm->arch.vioapic; +} + +#ifdef CONFIG_IA64 +static inline int irqchip_in_kernel(struct kvm *kvm) +{ + return 1; +} +#endif + +struct kvm_vcpu *kvm_get_lowest_prio_vcpu(struct kvm *kvm, u8 vector, + unsigned long bitmap); +void kvm_ioapic_update_eoi(struct kvm *kvm, int vector); +int kvm_ioapic_init(struct kvm *kvm); +void kvm_ioapic_set_irq(struct kvm_ioapic *ioapic, int irq, int level); +void kvm_ioapic_reset(struct kvm_ioapic *ioapic); + +#endif diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h index 6316638eec9f..c376b5b41bba 100644 --- a/arch/x86/kvm/irq.h +++ b/arch/x86/kvm/irq.h @@ -25,7 +25,10 @@ #include #include #include + #include "iodev.h" +#include "ioapic.h" +#include "lapic.h" struct kvm; struct kvm_vcpu; @@ -65,131 +68,23 @@ void kvm_pic_set_irq(void *opaque, int irq, int level); int kvm_pic_read_irq(struct kvm_pic *s); void kvm_pic_update_irq(struct kvm_pic *s); -#define IOAPIC_NUM_PINS KVM_IOAPIC_NUM_PINS -#define IOAPIC_VERSION_ID 0x11 /* IOAPIC version */ -#define IOAPIC_EDGE_TRIG 0 -#define IOAPIC_LEVEL_TRIG 1 - -#define IOAPIC_DEFAULT_BASE_ADDRESS 0xfec00000 -#define IOAPIC_MEM_LENGTH 0x100 - -/* Direct registers. */ -#define IOAPIC_REG_SELECT 0x00 -#define IOAPIC_REG_WINDOW 0x10 -#define IOAPIC_REG_EOI 0x40 /* IA64 IOSAPIC only */ - -/* Indirect registers. */ -#define IOAPIC_REG_APIC_ID 0x00 /* x86 IOAPIC only */ -#define IOAPIC_REG_VERSION 0x01 -#define IOAPIC_REG_ARB_ID 0x02 /* x86 IOAPIC only */ - -/*ioapic delivery mode*/ -#define IOAPIC_FIXED 0x0 -#define IOAPIC_LOWEST_PRIORITY 0x1 -#define IOAPIC_PMI 0x2 -#define IOAPIC_NMI 0x4 -#define IOAPIC_INIT 0x5 -#define IOAPIC_EXTINT 0x7 - -struct kvm_ioapic { - u64 base_address; - u32 ioregsel; - u32 id; - u32 irr; - u32 pad; - union ioapic_redir_entry { - u64 bits; - struct { - u8 vector; - u8 delivery_mode:3; - u8 dest_mode:1; - u8 delivery_status:1; - u8 polarity:1; - u8 remote_irr:1; - u8 trig_mode:1; - u8 mask:1; - u8 reserve:7; - u8 reserved[4]; - u8 dest_id; - } fields; - } redirtbl[IOAPIC_NUM_PINS]; - struct kvm_io_device dev; - struct kvm *kvm; -}; - -struct kvm_lapic { - unsigned long base_address; - struct kvm_io_device dev; - struct { - atomic_t pending; - s64 period; /* unit: ns */ - u32 divide_count; - ktime_t last_update; - struct hrtimer dev; - } timer; - struct kvm_vcpu *vcpu; - struct page *regs_page; - void *regs; -}; - -#ifdef DEBUG -#define ASSERT(x) \ -do { \ - if (!(x)) { \ - printk(KERN_EMERG "assertion failed %s: %d: %s\n", \ - __FILE__, __LINE__, #x); \ - BUG(); \ - } \ -} while (0) -#else -#define ASSERT(x) do { } while (0) -#endif - static inline struct kvm_pic *pic_irqchip(struct kvm *kvm) { return kvm->arch.vpic; } -static inline struct kvm_ioapic *ioapic_irqchip(struct kvm *kvm) -{ - return kvm->arch.vioapic; -} - static inline int irqchip_in_kernel(struct kvm *kvm) { return pic_irqchip(kvm) != NULL; } -void kvm_vcpu_kick(struct kvm_vcpu *vcpu); -int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu); -int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu); -int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu); -int kvm_create_lapic(struct kvm_vcpu *vcpu); -void kvm_lapic_reset(struct kvm_vcpu *vcpu); void kvm_pic_reset(struct kvm_kpic_state *s); -void kvm_ioapic_reset(struct kvm_ioapic *ioapic); -void kvm_free_lapic(struct kvm_vcpu *vcpu); -u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu); -void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8); -void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value); -struct kvm_vcpu *kvm_get_lowest_prio_vcpu(struct kvm *kvm, u8 vector, - unsigned long bitmap); -u64 kvm_get_apic_base(struct kvm_vcpu *vcpu); -void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data); -int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest); -void kvm_ioapic_update_eoi(struct kvm *kvm, int vector); -int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda); -int kvm_apic_set_irq(struct kvm_vcpu *vcpu, u8 vec, u8 trig); -void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu); -int kvm_ioapic_init(struct kvm *kvm); -void kvm_ioapic_set_irq(struct kvm_ioapic *ioapic, int irq, int level); -int kvm_lapic_enabled(struct kvm_vcpu *vcpu); -int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu); -void kvm_apic_timer_intr_post(struct kvm_vcpu *vcpu, int vec); void kvm_timer_intr_post(struct kvm_vcpu *vcpu, int vec); void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu); void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu); void kvm_migrate_apic_timer(struct kvm_vcpu *vcpu); +void kvm_vcpu_kick(struct kvm_vcpu *vcpu); + #endif diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h new file mode 100644 index 000000000000..447b654aefbb --- /dev/null +++ b/arch/x86/kvm/lapic.h @@ -0,0 +1,44 @@ +#ifndef __KVM_X86_LAPIC_H +#define __KVM_X86_LAPIC_H + +#include "iodev.h" + +#include + +struct kvm_lapic { + unsigned long base_address; + struct kvm_io_device dev; + struct { + atomic_t pending; + s64 period; /* unit: ns */ + u32 divide_count; + ktime_t last_update; + struct hrtimer dev; + } timer; + struct kvm_vcpu *vcpu; + struct page *regs_page; + void *regs; +}; +int kvm_create_lapic(struct kvm_vcpu *vcpu); +void kvm_free_lapic(struct kvm_vcpu *vcpu); + +int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu); +int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu); +int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu); +void kvm_lapic_reset(struct kvm_vcpu *vcpu); +u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu); +void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8); +void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value); + +int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest); +int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda); +int kvm_apic_set_irq(struct kvm_vcpu *vcpu, u8 vec, u8 trig); + +u64 kvm_get_apic_base(struct kvm_vcpu *vcpu); +void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data); +void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu); +int kvm_lapic_enabled(struct kvm_vcpu *vcpu); +int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu); +void kvm_apic_timer_intr_post(struct kvm_vcpu *vcpu, int vec); + +#endif From 0eb8f4984824b8a811d44963995133f47813330a Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Mon, 17 Dec 2007 14:16:14 +0800 Subject: [PATCH 228/249] KVM: Move ioapic code to common directory. Move ioapic code to common, since IA64 also needs it. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- arch/x86/kvm/Makefile | 7 +++---- {arch/x86 => virt}/kvm/ioapic.c | 2 +- {arch/x86 => virt}/kvm/ioapic.h | 0 3 files changed, 4 insertions(+), 5 deletions(-) rename {arch/x86 => virt}/kvm/ioapic.c (99%) rename {arch/x86 => virt}/kvm/ioapic.h (100%) diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index af41534af496..ffdd0b310784 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -2,12 +2,11 @@ # Makefile for Kernel-based Virtual Machine module # -common-objs = $(addprefix ../../../virt/kvm/, kvm_main.o) +common-objs = $(addprefix ../../../virt/kvm/, kvm_main.o ioapic.o) -EXTRA_CFLAGS += -I virt/kvm +EXTRA_CFLAGS += -Ivirt/kvm -Iarch/x86/kvm -kvm-objs := $(common-objs) x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o \ - ioapic.o +kvm-objs := $(common-objs) x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o obj-$(CONFIG_KVM) += kvm.o kvm-intel-objs = vmx.o obj-$(CONFIG_KVM_INTEL) += kvm-intel.o diff --git a/arch/x86/kvm/ioapic.c b/virt/kvm/ioapic.c similarity index 99% rename from arch/x86/kvm/ioapic.c rename to virt/kvm/ioapic.c index 40cd53ed4abd..317f8e211cd2 100644 --- a/arch/x86/kvm/ioapic.c +++ b/virt/kvm/ioapic.c @@ -363,7 +363,7 @@ static void ioapic_mmio_write(struct kvm_io_device *this, gpa_t addr, int len, break; #ifdef CONFIG_IA64 case IOAPIC_REG_EOI: - kvm_ioapic_update_eoi(ioapic, data); + kvm_ioapic_update_eoi(ioapic->kvm, data); break; #endif diff --git a/arch/x86/kvm/ioapic.h b/virt/kvm/ioapic.h similarity index 100% rename from arch/x86/kvm/ioapic.h rename to virt/kvm/ioapic.h From 5736199afba8a8bb60a1ea282ab72857d6b16400 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Mon, 17 Dec 2007 14:21:40 +0800 Subject: [PATCH 229/249] KVM: Move kvm_vcpu_kick() to x86.c Moving kvm_vcpu_kick() to x86.c. Since it should be common for all archs, put its declarations in Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- arch/x86/kvm/irq.c | 20 -------------------- arch/x86/kvm/irq.h | 2 -- arch/x86/kvm/x86.c | 20 ++++++++++++++++++++ include/linux/kvm_host.h | 1 + 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/arch/x86/kvm/irq.c b/arch/x86/kvm/irq.c index 07a09aad4fd6..e5714759e97f 100644 --- a/arch/x86/kvm/irq.c +++ b/arch/x86/kvm/irq.c @@ -63,26 +63,6 @@ int kvm_cpu_get_interrupt(struct kvm_vcpu *v) } EXPORT_SYMBOL_GPL(kvm_cpu_get_interrupt); -static void vcpu_kick_intr(void *info) -{ -#ifdef DEBUG - struct kvm_vcpu *vcpu = (struct kvm_vcpu *)info; - printk(KERN_DEBUG "vcpu_kick_intr %p \n", vcpu); -#endif -} - -void kvm_vcpu_kick(struct kvm_vcpu *vcpu) -{ - int ipi_pcpu = vcpu->cpu; - - if (waitqueue_active(&vcpu->wq)) { - wake_up_interruptible(&vcpu->wq); - ++vcpu->stat.halt_wakeup; - } - if (vcpu->guest_mode) - smp_call_function_single(ipi_pcpu, vcpu_kick_intr, vcpu, 0, 0); -} - void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu) { kvm_inject_apic_timer_irqs(vcpu); diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h index c376b5b41bba..53c7f48254be 100644 --- a/arch/x86/kvm/irq.h +++ b/arch/x86/kvm/irq.h @@ -85,6 +85,4 @@ void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu); void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu); void kvm_migrate_apic_timer(struct kvm_vcpu *vcpu); -void kvm_vcpu_kick(struct kvm_vcpu *vcpu); - #endif diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 5902c5cbc1bb..733bff65d9a9 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3144,3 +3144,23 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) return vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE || vcpu->arch.mp_state == VCPU_MP_STATE_SIPI_RECEIVED; } + +static void vcpu_kick_intr(void *info) +{ +#ifdef DEBUG + struct kvm_vcpu *vcpu = (struct kvm_vcpu *)info; + printk(KERN_DEBUG "vcpu_kick_intr %p \n", vcpu); +#endif +} + +void kvm_vcpu_kick(struct kvm_vcpu *vcpu) +{ + int ipi_pcpu = vcpu->cpu; + + if (waitqueue_active(&vcpu->wq)) { + wake_up_interruptible(&vcpu->wq); + ++vcpu->stat.halt_wakeup; + } + if (vcpu->guest_mode) + smp_call_function_single(ipi_pcpu, vcpu_kick_intr, vcpu, 0, 0); +} diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index a85d5b6943de..953b50aa0e61 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -250,6 +250,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm); int kvm_cpu_get_interrupt(struct kvm_vcpu *v); int kvm_cpu_has_interrupt(struct kvm_vcpu *v); +void kvm_vcpu_kick(struct kvm_vcpu *vcpu); static inline void kvm_guest_enter(void) { From ec10f4750db5e810cafef114a592428f8242b640 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Mon, 17 Dec 2007 20:27:27 +0800 Subject: [PATCH 230/249] KVM: Expose ioapic to ia64 save/restore APIs IA64 also needs to see ioapic structure in irqchip. Signed-off-by: xiantao.zhang@intel.com Signed-off-by: Avi Kivity --- include/linux/kvm.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/linux/kvm.h b/include/linux/kvm.h index f0bebd61ff77..de9f28d96ced 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -53,6 +53,8 @@ struct kvm_irqchip { char dummy[512]; /* reserving space */ #ifdef CONFIG_X86 struct kvm_pic_state pic; +#endif +#if defined(CONFIG_X86) || defined(CONFIG_IA64) struct kvm_ioapic_state ioapic; #endif } chip; From caa5b8a5ed27708521e0667649699fea202043b2 Mon Sep 17 00:00:00 2001 From: Eddie Dong Date: Tue, 18 Dec 2007 06:08:27 +0800 Subject: [PATCH 231/249] KVM: MMU: Coalesce remote tlb flushes Host side TLB flush can be merged together if multiple spte need to be write-protected. Signed-off-by: Yaozu (Eddie) Dong Signed-off-by: Avi Kivity --- arch/x86/kvm/mmu.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index 401eb7ce3207..9a57e1a01449 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c @@ -513,6 +513,7 @@ static void rmap_write_protect(struct kvm *kvm, u64 gfn) { unsigned long *rmapp; u64 *spte; + int write_protected = 0; gfn = unalias_gfn(kvm, gfn); rmapp = gfn_to_rmap(kvm, gfn); @@ -522,11 +523,14 @@ static void rmap_write_protect(struct kvm *kvm, u64 gfn) BUG_ON(!spte); BUG_ON(!(*spte & PT_PRESENT_MASK)); rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte); - if (is_writeble_pte(*spte)) + if (is_writeble_pte(*spte)) { set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK); - kvm_flush_remote_tlbs(kvm); + write_protected = 1; + } spte = rmap_next(kvm, rmapp, spte); } + if (write_protected) + kvm_flush_remote_tlbs(kvm); } #ifdef MMU_DEBUG From dfc5aa00cbe888d3a9ea97775bbac74cb1c1a1d8 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 18 Dec 2007 19:47:18 +0200 Subject: [PATCH 232/249] KVM: MMU: Add cache miss statistic Signed-off-by: Avi Kivity --- arch/x86/kvm/mmu.c | 1 + arch/x86/kvm/x86.c | 1 + include/asm-x86/kvm_host.h | 1 + 3 files changed, 3 insertions(+) diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index 9a57e1a01449..8f12ec52ad86 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c @@ -715,6 +715,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, pgprintk("%s: found\n", __FUNCTION__); return sp; } + ++vcpu->kvm->stat.mmu_cache_miss; sp = kvm_mmu_alloc_page(vcpu, parent_pte); if (!sp) return sp; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 733bff65d9a9..fa9e42c03741 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -73,6 +73,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) }, { "mmu_flooded", VM_STAT(mmu_flooded) }, { "mmu_recycled", VM_STAT(mmu_recycled) }, + { "mmu_cache_miss", VM_STAT(mmu_cache_miss) }, { "remote_tlb_flush", VM_STAT(remote_tlb_flush) }, { NULL } }; diff --git a/include/asm-x86/kvm_host.h b/include/asm-x86/kvm_host.h index 28940e1a9713..ced1bebabbc8 100644 --- a/include/asm-x86/kvm_host.h +++ b/include/asm-x86/kvm_host.h @@ -289,6 +289,7 @@ struct kvm_vm_stat { u32 mmu_pde_zapped; u32 mmu_flooded; u32 mmu_recycled; + u32 mmu_cache_miss; u32 remote_tlb_flush; }; From 565f1fbd9d2f766dcfed5db90b89ef80afe8b49a Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 19 Dec 2007 12:02:40 +0200 Subject: [PATCH 233/249] KVM: Print data for unimplemented wrmsr This can help diagnosing what the guest is trying to do. In many cases we can get away with partial emulation of msrs. Signed-off-by: Avi Kivity --- arch/x86/kvm/x86.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index fa9e42c03741..513258c797ca 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -499,7 +499,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) vcpu->arch.ia32_misc_enable_msr = data; break; default: - pr_unimpl(vcpu, "unhandled wrmsr: 0x%x\n", msr); + pr_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n", msr, data); return 1; } return 0; From b209749f528488c4c0d20a42c0fbcbf49e6933b3 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Mon, 22 Oct 2007 16:50:39 +0200 Subject: [PATCH 234/249] KVM: local APIC TPR access reporting facility Add a facility to report on accesses to the local apic tpr even if the local apic is emulated in the kernel. This is basically a hack that allows userspace to patch Windows which tends to bang on the tpr a lot. Signed-off-by: Avi Kivity --- arch/x86/kvm/lapic.c | 21 +++++++++++++++++++++ arch/x86/kvm/x86.c | 25 +++++++++++++++++++++++++ include/asm-x86/kvm_host.h | 1 + include/linux/kvm.h | 17 +++++++++++++++++ include/linux/kvm_host.h | 2 +- 5 files changed, 65 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 4076331b01ee..50c3f3a8dd3d 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -551,6 +551,23 @@ static u32 apic_get_tmcct(struct kvm_lapic *apic) return tmcct; } +static void __report_tpr_access(struct kvm_lapic *apic, bool write) +{ + struct kvm_vcpu *vcpu = apic->vcpu; + struct kvm_run *run = vcpu->run; + + set_bit(KVM_REQ_REPORT_TPR_ACCESS, &vcpu->requests); + kvm_x86_ops->cache_regs(vcpu); + run->tpr_access.rip = vcpu->arch.rip; + run->tpr_access.is_write = write; +} + +static inline void report_tpr_access(struct kvm_lapic *apic, bool write) +{ + if (apic->vcpu->arch.tpr_access_reporting) + __report_tpr_access(apic, write); +} + static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset) { u32 val = 0; @@ -568,6 +585,9 @@ static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset) val = apic_get_tmcct(apic); break; + case APIC_TASKPRI: + report_tpr_access(apic, false); + /* fall thru */ default: apic_update_ppr(apic); val = apic_get_reg(apic, offset); @@ -677,6 +697,7 @@ static void apic_mmio_write(struct kvm_io_device *this, break; case APIC_TASKPRI: + report_tpr_access(apic, true); apic_set_tpr(apic, val & 0xff); break; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 513258c797ca..c2b80884447e 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -684,6 +684,7 @@ int kvm_dev_ioctl_check_extension(long ext) case KVM_CAP_USER_MEMORY: case KVM_CAP_SET_TSS_ADDR: case KVM_CAP_EXT_CPUID: + case KVM_CAP_VAPIC: r = 1; break; default: @@ -1055,6 +1056,15 @@ static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, return 0; } +static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu, + struct kvm_tpr_access_ctl *tac) +{ + if (tac->flags) + return -EINVAL; + vcpu->arch.tpr_access_reporting = !!tac->enabled; + return 0; +} + long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -1148,6 +1158,21 @@ long kvm_arch_vcpu_ioctl(struct file *filp, case KVM_SET_MSRS: r = msr_io(vcpu, argp, do_set_msr, 0); break; + case KVM_TPR_ACCESS_REPORTING: { + struct kvm_tpr_access_ctl tac; + + r = -EFAULT; + if (copy_from_user(&tac, argp, sizeof tac)) + goto out; + r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac); + if (r) + goto out; + r = -EFAULT; + if (copy_to_user(argp, &tac, sizeof tac)) + goto out; + r = 0; + break; + }; default: r = -EINVAL; } diff --git a/include/asm-x86/kvm_host.h b/include/asm-x86/kvm_host.h index ced1bebabbc8..6e649af08d19 100644 --- a/include/asm-x86/kvm_host.h +++ b/include/asm-x86/kvm_host.h @@ -211,6 +211,7 @@ struct kvm_vcpu_arch { int mp_state; int sipi_vector; u64 ia32_misc_enable_msr; + bool tpr_access_reporting; struct kvm_mmu mmu; diff --git a/include/linux/kvm.h b/include/linux/kvm.h index de9f28d96ced..850f5ef76636 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -72,6 +72,7 @@ struct kvm_irqchip { #define KVM_EXIT_FAIL_ENTRY 9 #define KVM_EXIT_INTR 10 #define KVM_EXIT_SET_TPR 11 +#define KVM_EXIT_TPR_ACCESS 12 /* for KVM_RUN, returned by mmap(vcpu_fd, offset=0) */ struct kvm_run { @@ -130,6 +131,12 @@ struct kvm_run { __u32 longmode; __u32 pad; } hypercall; + /* KVM_EXIT_TPR_ACCESS */ + struct { + __u64 rip; + __u32 is_write; + __u32 pad; + } tpr_access; /* Fix the size of the union. */ char padding[256]; }; @@ -202,6 +209,13 @@ struct kvm_signal_mask { __u8 sigset[0]; }; +/* for KVM_TPR_ACCESS_REPORTING */ +struct kvm_tpr_access_ctl { + __u32 enabled; + __u32 flags; + __u32 reserved[8]; +}; + #define KVMIO 0xAE /* @@ -229,6 +243,7 @@ struct kvm_signal_mask { #define KVM_CAP_USER_MEMORY 3 #define KVM_CAP_SET_TSS_ADDR 4 #define KVM_CAP_EXT_CPUID 5 +#define KVM_CAP_VAPIC 6 /* * ioctls for VM fds @@ -274,5 +289,7 @@ struct kvm_signal_mask { #define KVM_SET_LAPIC _IOW(KVMIO, 0x8f, struct kvm_lapic_state) #define KVM_SET_CPUID2 _IOW(KVMIO, 0x90, struct kvm_cpuid2) #define KVM_GET_CPUID2 _IOWR(KVMIO, 0x91, struct kvm_cpuid2) +/* Available with KVM_CAP_VAPIC */ +#define KVM_TPR_ACCESS_REPORTING _IOWR(KVMIO, 0x92, struct kvm_tpr_access_ctl) #endif diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 953b50aa0e61..9ff5904c5072 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -35,7 +35,7 @@ * vcpu->requests bit members */ #define KVM_REQ_TLB_FLUSH 0 - +#define KVM_REQ_REPORT_TPR_ACCESS 2 struct kvm_vcpu; extern struct kmem_cache *kvm_vcpu_cache; From b93463aa59d67b21b4921e30bd5c5dcc7c35ffbd Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 25 Oct 2007 16:52:32 +0200 Subject: [PATCH 235/249] KVM: Accelerated apic support This adds a mechanism for exposing the virtual apic tpr to the guest, and a protocol for letting the guest update the tpr without causing a vmexit if conditions allow (e.g. there is no interrupt pending with a higher priority than the new tpr). Signed-off-by: Avi Kivity --- arch/x86/kvm/lapic.c | 51 ++++++++++++++++++++++++++++++++++++- arch/x86/kvm/lapic.h | 6 +++++ arch/x86/kvm/x86.c | 55 ++++++++++++++++++++++++++++++++++++++++ include/linux/kvm.h | 7 +++++ include/linux/kvm_para.h | 2 ++ 5 files changed, 120 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 50c3f3a8dd3d..e7513bb98af1 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -815,7 +815,8 @@ void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) if (!apic) return; - apic_set_tpr(apic, ((cr8 & 0x0f) << 4)); + apic_set_tpr(apic, ((cr8 & 0x0f) << 4) + | (apic_get_reg(apic, APIC_TASKPRI) & 4)); } u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) @@ -1104,3 +1105,51 @@ void kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) hrtimer_start(timer, timer->expires, HRTIMER_MODE_ABS); } EXPORT_SYMBOL_GPL(kvm_migrate_apic_timer); + +void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu) +{ + u32 data; + void *vapic; + + if (!irqchip_in_kernel(vcpu->kvm) || !vcpu->arch.apic->vapic_addr) + return; + + vapic = kmap_atomic(vcpu->arch.apic->vapic_page, KM_USER0); + data = *(u32 *)(vapic + offset_in_page(vcpu->arch.apic->vapic_addr)); + kunmap_atomic(vapic, KM_USER0); + + apic_set_tpr(vcpu->arch.apic, data & 0xff); +} + +void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu) +{ + u32 data, tpr; + int max_irr, max_isr; + struct kvm_lapic *apic; + void *vapic; + + if (!irqchip_in_kernel(vcpu->kvm) || !vcpu->arch.apic->vapic_addr) + return; + + apic = vcpu->arch.apic; + tpr = apic_get_reg(apic, APIC_TASKPRI) & 0xff; + max_irr = apic_find_highest_irr(apic); + if (max_irr < 0) + max_irr = 0; + max_isr = apic_find_highest_isr(apic); + if (max_isr < 0) + max_isr = 0; + data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24); + + vapic = kmap_atomic(vcpu->arch.apic->vapic_page, KM_USER0); + *(u32 *)(vapic + offset_in_page(vcpu->arch.apic->vapic_addr)) = data; + kunmap_atomic(vapic, KM_USER0); +} + +void kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr) +{ + if (!irqchip_in_kernel(vcpu->kvm)) + return; + + vcpu->arch.apic->vapic_addr = vapic_addr; +} diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 447b654aefbb..676c396c9cee 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -18,6 +18,8 @@ struct kvm_lapic { struct kvm_vcpu *vcpu; struct page *regs_page; void *regs; + gpa_t vapic_addr; + struct page *vapic_page; }; int kvm_create_lapic(struct kvm_vcpu *vcpu); void kvm_free_lapic(struct kvm_vcpu *vcpu); @@ -41,4 +43,8 @@ int kvm_lapic_enabled(struct kvm_vcpu *vcpu); int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu); void kvm_apic_timer_intr_post(struct kvm_vcpu *vcpu, int vec); +void kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr); +void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu); +void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu); + #endif diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index c2b80884447e..e7eac27adb7f 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1173,6 +1173,19 @@ long kvm_arch_vcpu_ioctl(struct file *filp, r = 0; break; }; + case KVM_SET_VAPIC_ADDR: { + struct kvm_vapic_addr va; + + r = -EINVAL; + if (!irqchip_in_kernel(vcpu->kvm)) + goto out; + r = -EFAULT; + if (copy_from_user(&va, argp, sizeof va)) + goto out; + r = 0; + kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr); + break; + } default: r = -EINVAL; } @@ -2214,6 +2227,9 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) } switch (nr) { + case KVM_HC_VAPIC_POLL_IRQ: + ret = 0; + break; default: ret = -KVM_ENOSYS; break; @@ -2421,6 +2437,29 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu, vcpu->arch.irq_summary == 0); } +static void vapic_enter(struct kvm_vcpu *vcpu) +{ + struct kvm_lapic *apic = vcpu->arch.apic; + struct page *page; + + if (!apic || !apic->vapic_addr) + return; + + page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT); + vcpu->arch.apic->vapic_page = page; +} + +static void vapic_exit(struct kvm_vcpu *vcpu) +{ + struct kvm_lapic *apic = vcpu->arch.apic; + + if (!apic || !apic->vapic_addr) + return; + + kvm_release_page_dirty(apic->vapic_page); + mark_page_dirty(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT); +} + static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { int r; @@ -2435,6 +2474,8 @@ static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE; } + vapic_enter(vcpu); + preempted: if (vcpu->guest_debug.enabled) kvm_x86_ops->guest_debug_pre(vcpu); @@ -2444,6 +2485,14 @@ again: if (unlikely(r)) goto out; + if (vcpu->requests) + if (test_and_clear_bit(KVM_REQ_REPORT_TPR_ACCESS, + &vcpu->requests)) { + kvm_run->exit_reason = KVM_EXIT_TPR_ACCESS; + r = 0; + goto out; + } + kvm_inject_pending_timer_irqs(vcpu); preempt_disable(); @@ -2469,6 +2518,8 @@ again: else kvm_x86_ops->inject_pending_vectors(vcpu, kvm_run); + kvm_lapic_sync_to_vapic(vcpu); + vcpu->guest_mode = 1; kvm_guest_enter(); @@ -2506,6 +2557,8 @@ again: if (vcpu->arch.exception.pending && kvm_x86_ops->exception_injected(vcpu)) vcpu->arch.exception.pending = false; + kvm_lapic_sync_from_vapic(vcpu); + r = kvm_x86_ops->handle_exit(kvm_run, vcpu); if (r > 0) { @@ -2527,6 +2580,8 @@ out: post_kvm_run_save(vcpu, kvm_run); + vapic_exit(vcpu); + return r; } diff --git a/include/linux/kvm.h b/include/linux/kvm.h index 850f5ef76636..b27a381db5e8 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -216,6 +216,11 @@ struct kvm_tpr_access_ctl { __u32 reserved[8]; }; +/* for KVM_SET_VAPIC_ADDR */ +struct kvm_vapic_addr { + __u64 vapic_addr; +}; + #define KVMIO 0xAE /* @@ -291,5 +296,7 @@ struct kvm_tpr_access_ctl { #define KVM_GET_CPUID2 _IOWR(KVMIO, 0x91, struct kvm_cpuid2) /* Available with KVM_CAP_VAPIC */ #define KVM_TPR_ACCESS_REPORTING _IOWR(KVMIO, 0x92, struct kvm_tpr_access_ctl) +/* Available with KVM_CAP_VAPIC */ +#define KVM_SET_VAPIC_ADDR _IOW(KVMIO, 0x93, struct kvm_vapic_addr) #endif diff --git a/include/linux/kvm_para.h b/include/linux/kvm_para.h index e4db25ffdb52..6af91a506fbf 100644 --- a/include/linux/kvm_para.h +++ b/include/linux/kvm_para.h @@ -12,6 +12,8 @@ /* Return values for hypercalls */ #define KVM_ENOSYS 1000 +#define KVM_HC_VAPIC_POLL_IRQ 1 + #ifdef __KERNEL__ /* * hypercalls use architecture specific From 774ead3ad9bcbc05ef6aaebb9bdf8b4c3126923b Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 26 Dec 2007 13:57:04 +0200 Subject: [PATCH 236/249] KVM: Disable vapic support on Intel machines with FlexPriority FlexPriority accelerates the tpr without any patching. Signed-off-by: Avi Kivity --- arch/x86/kvm/svm.c | 6 ++++++ arch/x86/kvm/vmx.c | 3 ++- arch/x86/kvm/x86.c | 4 +++- include/asm-x86/kvm_host.h | 1 + 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index 3d4b71a94440..33f77b044c52 100644 --- a/arch/x86/kvm/svm.c +++ b/arch/x86/kvm/svm.c @@ -1653,6 +1653,11 @@ static void svm_check_processor_compat(void *rtn) *(int *)rtn = 0; } +static bool svm_cpu_has_accelerated_tpr(void) +{ + return false; +} + static struct kvm_x86_ops svm_x86_ops = { .cpu_has_kvm_support = has_svm, .disabled_by_bios = is_disabled, @@ -1661,6 +1666,7 @@ static struct kvm_x86_ops svm_x86_ops = { .check_processor_compatibility = svm_check_processor_compat, .hardware_enable = svm_hardware_enable, .hardware_disable = svm_hardware_disable, + .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr, .vcpu_create = svm_create_vcpu, .vcpu_free = svm_free_vcpu, diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index fc494aff5d8b..d1167fc303d6 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -192,7 +192,7 @@ static inline int cpu_has_secondary_exec_ctrls(void) CPU_BASED_ACTIVATE_SECONDARY_CONTROLS); } -static inline int cpu_has_vmx_virtualize_apic_accesses(void) +static inline bool cpu_has_vmx_virtualize_apic_accesses(void) { return (vmcs_config.cpu_based_2nd_exec_ctrl & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES); @@ -2565,6 +2565,7 @@ static struct kvm_x86_ops vmx_x86_ops = { .check_processor_compatibility = vmx_check_processor_compat, .hardware_enable = hardware_enable, .hardware_disable = hardware_disable, + .cpu_has_accelerated_tpr = cpu_has_vmx_virtualize_apic_accesses, .vcpu_create = vmx_create_vcpu, .vcpu_free = vmx_free_vcpu, diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index e7eac27adb7f..1f48ec871035 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -684,9 +684,11 @@ int kvm_dev_ioctl_check_extension(long ext) case KVM_CAP_USER_MEMORY: case KVM_CAP_SET_TSS_ADDR: case KVM_CAP_EXT_CPUID: - case KVM_CAP_VAPIC: r = 1; break; + case KVM_CAP_VAPIC: + r = !kvm_x86_ops->cpu_has_accelerated_tpr(); + break; default: r = 0; break; diff --git a/include/asm-x86/kvm_host.h b/include/asm-x86/kvm_host.h index 6e649af08d19..44b89259f6c4 100644 --- a/include/asm-x86/kvm_host.h +++ b/include/asm-x86/kvm_host.h @@ -329,6 +329,7 @@ struct kvm_x86_ops { void (*check_processor_compatibility)(void *rtn); int (*hardware_setup)(void); /* __init */ void (*hardware_unsetup)(void); /* __exit */ + bool (*cpu_has_accelerated_tpr)(void); /* Create, but do not attach this VCPU */ struct kvm_vcpu *(*vcpu_create)(struct kvm *kvm, unsigned id); From 10589a4699bb978c781ce73bbae8ca942c5250c9 Mon Sep 17 00:00:00 2001 From: Marcelo Tosatti Date: Thu, 20 Dec 2007 19:18:22 -0500 Subject: [PATCH 237/249] KVM: MMU: Concurrent guest walkers Do not hold kvm->lock mutex across the entire pagefault code, only acquire it in places where it is necessary, such as mmu hash list, active list, rmap and parent pte handling. Allow concurrent guest walkers by switching walk_addr() to use mmap_sem in read-mode. And get rid of the lockless __gfn_to_page. [avi: move kvm_mmu_pte_write() locking inside the function] [avi: add locking for real mode] [avi: fix cmpxchg locking] Signed-off-by: Marcelo Tosatti Signed-off-by: Avi Kivity --- arch/x86/kvm/mmu.c | 41 +++++++++++++---- arch/x86/kvm/paging_tmpl.h | 8 +++- arch/x86/kvm/vmx.c | 25 +++++++---- arch/x86/kvm/x86.c | 92 +++++++++++++++++++++++--------------- virt/kvm/kvm_main.c | 22 +++------ 5 files changed, 117 insertions(+), 71 deletions(-) diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index 8f12ec52ad86..3b91227969a5 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c @@ -974,7 +974,7 @@ static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) { } -static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) +static int __nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) { int level = PT32E_ROOT_LEVEL; hpa_t table_addr = vcpu->arch.mmu.root_hpa; @@ -1015,6 +1015,17 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) } } +static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) +{ + int r; + + mutex_lock(&vcpu->kvm->lock); + r = __nonpaging_map(vcpu, v, write, gfn); + mutex_unlock(&vcpu->kvm->lock); + return r; +} + + static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) { @@ -1031,6 +1042,7 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu) if (!VALID_PAGE(vcpu->arch.mmu.root_hpa)) return; + mutex_lock(&vcpu->kvm->lock); #ifdef CONFIG_X86_64 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) { hpa_t root = vcpu->arch.mmu.root_hpa; @@ -1038,6 +1050,7 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu) sp = page_header(root); --sp->root_count; vcpu->arch.mmu.root_hpa = INVALID_PAGE; + mutex_unlock(&vcpu->kvm->lock); return; } #endif @@ -1051,6 +1064,7 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu) } vcpu->arch.mmu.pae_root[i] = INVALID_PAGE; } + mutex_unlock(&vcpu->kvm->lock); vcpu->arch.mmu.root_hpa = INVALID_PAGE; } @@ -1250,15 +1264,15 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu) { int r; - mutex_lock(&vcpu->kvm->lock); r = mmu_topup_memory_caches(vcpu); if (r) goto out; + mutex_lock(&vcpu->kvm->lock); mmu_alloc_roots(vcpu); + mutex_unlock(&vcpu->kvm->lock); kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa); kvm_mmu_flush_tlb(vcpu); out: - mutex_unlock(&vcpu->kvm->lock); return r; } EXPORT_SYMBOL_GPL(kvm_mmu_load); @@ -1353,6 +1367,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, int npte; pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes); + mutex_lock(&vcpu->kvm->lock); ++vcpu->kvm->stat.mmu_pte_write; kvm_mmu_audit(vcpu, "pre pte write"); if (gfn == vcpu->arch.last_pt_write_gfn @@ -1421,17 +1436,27 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, } } kvm_mmu_audit(vcpu, "post pte write"); + mutex_unlock(&vcpu->kvm->lock); } int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) { - gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva); + gpa_t gpa; + int r; - return kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT); + down_read(¤t->mm->mmap_sem); + gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva); + up_read(¤t->mm->mmap_sem); + + mutex_lock(&vcpu->kvm->lock); + r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT); + mutex_unlock(&vcpu->kvm->lock); + return r; } void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) { + mutex_lock(&vcpu->kvm->lock); while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) { struct kvm_mmu_page *sp; @@ -1440,6 +1465,7 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) kvm_mmu_zap_page(vcpu->kvm, sp); ++vcpu->kvm->stat.mmu_recycled; } + mutex_unlock(&vcpu->kvm->lock); } int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) @@ -1447,7 +1473,6 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) int r; enum emulation_result er; - mutex_lock(&vcpu->kvm->lock); r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code); if (r < 0) goto out; @@ -1462,7 +1487,6 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) goto out; er = emulate_instruction(vcpu, vcpu->run, cr2, error_code, 0); - mutex_unlock(&vcpu->kvm->lock); switch (er) { case EMULATE_DONE: @@ -1477,7 +1501,6 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) BUG(); } out: - mutex_unlock(&vcpu->kvm->lock); return r; } EXPORT_SYMBOL_GPL(kvm_mmu_page_fault); @@ -1574,8 +1597,10 @@ void kvm_mmu_zap_all(struct kvm *kvm) { struct kvm_mmu_page *sp, *node; + mutex_lock(&kvm->lock); list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) kvm_mmu_zap_page(kvm, sp); + mutex_unlock(&kvm->lock); kvm_flush_remote_tlbs(kvm); } diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h index 56b88f7e83ef..7f83f5557d5e 100644 --- a/arch/x86/kvm/paging_tmpl.h +++ b/arch/x86/kvm/paging_tmpl.h @@ -368,11 +368,13 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, if (r) return r; + down_read(¤t->mm->mmap_sem); /* * Look up the shadow pte for the faulting address. */ r = FNAME(walk_addr)(&walker, vcpu, addr, write_fault, user_fault, fetch_fault); + up_read(¤t->mm->mmap_sem); /* * The page is not mapped by the guest. Let the guest handle it. @@ -384,6 +386,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, return 0; } + mutex_lock(&vcpu->kvm->lock); shadow_pte = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault, &write_pt); pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __FUNCTION__, @@ -395,11 +398,14 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, /* * mmio: emulate if accessible, otherwise its a guest fault. */ - if (shadow_pte && is_io_pte(*shadow_pte)) + if (shadow_pte && is_io_pte(*shadow_pte)) { + mutex_unlock(&vcpu->kvm->lock); return 1; + } ++vcpu->stat.pf_fixed; kvm_mmu_audit(vcpu, "post page fault (fixed)"); + mutex_unlock(&vcpu->kvm->lock); return write_pt; } diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index d1167fc303d6..c39493feba46 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -1432,27 +1432,34 @@ static int init_rmode_tss(struct kvm *kvm) { gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT; u16 data = 0; + int ret = 0; int r; + down_read(¤t->mm->mmap_sem); r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE); if (r < 0) - return 0; + goto out; data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE; r = kvm_write_guest_page(kvm, fn++, &data, 0x66, sizeof(u16)); if (r < 0) - return 0; + goto out; r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE); if (r < 0) - return 0; + goto out; r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE); if (r < 0) - return 0; + goto out; data = ~0; - r = kvm_write_guest_page(kvm, fn, &data, RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1, - sizeof(u8)); + r = kvm_write_guest_page(kvm, fn, &data, + RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1, + sizeof(u8)); if (r < 0) - return 0; - return 1; + goto out; + + ret = 1; +out: + up_read(¤t->mm->mmap_sem); + return ret; } static void seg_setup(int seg) @@ -1471,6 +1478,7 @@ static int alloc_apic_access_page(struct kvm *kvm) int r = 0; mutex_lock(&kvm->lock); + down_write(¤t->mm->mmap_sem); if (kvm->arch.apic_access_page) goto out; kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT; @@ -1482,6 +1490,7 @@ static int alloc_apic_access_page(struct kvm *kvm) goto out; kvm->arch.apic_access_page = gfn_to_page(kvm, 0xfee00); out: + up_write(¤t->mm->mmap_sem); mutex_unlock(&kvm->lock); return r; } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 1f48ec871035..e3b3141db13c 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -181,7 +181,7 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) int ret; u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)]; - mutex_lock(&vcpu->kvm->lock); + down_read(¤t->mm->mmap_sem); ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte, offset * sizeof(u64), sizeof(pdpte)); if (ret < 0) { @@ -198,7 +198,7 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs)); out: - mutex_unlock(&vcpu->kvm->lock); + up_read(¤t->mm->mmap_sem); return ret; } @@ -212,13 +212,13 @@ static bool pdptrs_changed(struct kvm_vcpu *vcpu) if (is_long_mode(vcpu) || !is_pae(vcpu)) return false; - mutex_lock(&vcpu->kvm->lock); + down_read(¤t->mm->mmap_sem); r = kvm_read_guest(vcpu->kvm, vcpu->arch.cr3 & ~31u, pdpte, sizeof(pdpte)); if (r < 0) goto out; changed = memcmp(pdpte, vcpu->arch.pdptrs, sizeof(pdpte)) != 0; out: - mutex_unlock(&vcpu->kvm->lock); + up_read(¤t->mm->mmap_sem); return changed; } @@ -278,9 +278,7 @@ void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) kvm_x86_ops->set_cr0(vcpu, cr0); vcpu->arch.cr0 = cr0; - mutex_lock(&vcpu->kvm->lock); kvm_mmu_reset_context(vcpu); - mutex_unlock(&vcpu->kvm->lock); return; } EXPORT_SYMBOL_GPL(set_cr0); @@ -320,9 +318,7 @@ void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) } kvm_x86_ops->set_cr4(vcpu, cr4); vcpu->arch.cr4 = cr4; - mutex_lock(&vcpu->kvm->lock); kvm_mmu_reset_context(vcpu); - mutex_unlock(&vcpu->kvm->lock); } EXPORT_SYMBOL_GPL(set_cr4); @@ -360,7 +356,7 @@ void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) */ } - mutex_lock(&vcpu->kvm->lock); + down_read(¤t->mm->mmap_sem); /* * Does the new cr3 value map to physical memory? (Note, we * catch an invalid cr3 even in real-mode, because it would @@ -376,7 +372,7 @@ void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) vcpu->arch.cr3 = cr3; vcpu->arch.mmu.new_cr3(vcpu); } - mutex_unlock(&vcpu->kvm->lock); + up_read(¤t->mm->mmap_sem); } EXPORT_SYMBOL_GPL(set_cr3); @@ -1211,12 +1207,12 @@ static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES) return -EINVAL; - mutex_lock(&kvm->lock); + down_write(¤t->mm->mmap_sem); kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages); kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages; - mutex_unlock(&kvm->lock); + up_write(¤t->mm->mmap_sem); return 0; } @@ -1265,7 +1261,7 @@ static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm, < alias->target_phys_addr) goto out; - mutex_lock(&kvm->lock); + down_write(¤t->mm->mmap_sem); p = &kvm->arch.aliases[alias->slot]; p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT; @@ -1279,7 +1275,7 @@ static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm, kvm_mmu_zap_all(kvm); - mutex_unlock(&kvm->lock); + up_write(¤t->mm->mmap_sem); return 0; @@ -1355,7 +1351,7 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot; int is_dirty = 0; - mutex_lock(&kvm->lock); + down_write(¤t->mm->mmap_sem); r = kvm_get_dirty_log(kvm, log, &is_dirty); if (r) @@ -1371,7 +1367,7 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, } r = 0; out: - mutex_unlock(&kvm->lock); + up_write(¤t->mm->mmap_sem); return r; } @@ -1565,25 +1561,32 @@ int emulator_read_std(unsigned long addr, struct kvm_vcpu *vcpu) { void *data = val; + int r = X86EMUL_CONTINUE; + down_read(¤t->mm->mmap_sem); while (bytes) { gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); unsigned offset = addr & (PAGE_SIZE-1); unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset); int ret; - if (gpa == UNMAPPED_GVA) - return X86EMUL_PROPAGATE_FAULT; + if (gpa == UNMAPPED_GVA) { + r = X86EMUL_PROPAGATE_FAULT; + goto out; + } ret = kvm_read_guest(vcpu->kvm, gpa, data, tocopy); - if (ret < 0) - return X86EMUL_UNHANDLEABLE; + if (ret < 0) { + r = X86EMUL_UNHANDLEABLE; + goto out; + } bytes -= tocopy; data += tocopy; addr += tocopy; } - - return X86EMUL_CONTINUE; +out: + up_read(¤t->mm->mmap_sem); + return r; } EXPORT_SYMBOL_GPL(emulator_read_std); @@ -1601,7 +1604,9 @@ static int emulator_read_emulated(unsigned long addr, return X86EMUL_CONTINUE; } + down_read(¤t->mm->mmap_sem); gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); + up_read(¤t->mm->mmap_sem); /* For APIC access vmexit */ if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) @@ -1617,11 +1622,14 @@ mmio: /* * Is this MMIO handled locally? */ + mutex_lock(&vcpu->kvm->lock); mmio_dev = vcpu_find_mmio_dev(vcpu, gpa); if (mmio_dev) { kvm_iodevice_read(mmio_dev, gpa, bytes, val); + mutex_unlock(&vcpu->kvm->lock); return X86EMUL_CONTINUE; } + mutex_unlock(&vcpu->kvm->lock); vcpu->mmio_needed = 1; vcpu->mmio_phys_addr = gpa; @@ -1636,10 +1644,14 @@ static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, { int ret; + down_read(¤t->mm->mmap_sem); ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes); - if (ret < 0) + if (ret < 0) { + up_read(¤t->mm->mmap_sem); return 0; + } kvm_mmu_pte_write(vcpu, gpa, val, bytes); + up_read(¤t->mm->mmap_sem); return 1; } @@ -1649,7 +1661,11 @@ static int emulator_write_emulated_onepage(unsigned long addr, struct kvm_vcpu *vcpu) { struct kvm_io_device *mmio_dev; - gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); + gpa_t gpa; + + down_read(¤t->mm->mmap_sem); + gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); + up_read(¤t->mm->mmap_sem); if (gpa == UNMAPPED_GVA) { kvm_inject_page_fault(vcpu, addr, 2); @@ -1667,11 +1683,14 @@ mmio: /* * Is this MMIO handled locally? */ + mutex_lock(&vcpu->kvm->lock); mmio_dev = vcpu_find_mmio_dev(vcpu, gpa); if (mmio_dev) { kvm_iodevice_write(mmio_dev, gpa, bytes, val); + mutex_unlock(&vcpu->kvm->lock); return X86EMUL_CONTINUE; } + mutex_unlock(&vcpu->kvm->lock); vcpu->mmio_needed = 1; vcpu->mmio_phys_addr = gpa; @@ -1718,11 +1737,14 @@ static int emulator_cmpxchg_emulated(unsigned long addr, #ifndef CONFIG_X86_64 /* guests cmpxchg8b have to be emulated atomically */ if (bytes == 8) { - gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); + gpa_t gpa; struct page *page; char *addr; u64 val; + down_read(¤t->mm->mmap_sem); + gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); + if (gpa == UNMAPPED_GVA || (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) goto emul_write; @@ -1736,8 +1758,9 @@ static int emulator_cmpxchg_emulated(unsigned long addr, set_64bit((u64 *)(addr + offset_in_page(gpa)), val); kunmap_atomic(addr, KM_USER0); kvm_release_page_dirty(page); + emul_write: + up_read(¤t->mm->mmap_sem); } -emul_write: #endif return emulator_write_emulated(addr, new, bytes, vcpu); @@ -2118,10 +2141,10 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, kvm_x86_ops->skip_emulated_instruction(vcpu); for (i = 0; i < nr_pages; ++i) { - mutex_lock(&vcpu->kvm->lock); + down_read(¤t->mm->mmap_sem); page = gva_to_page(vcpu, address + i * PAGE_SIZE); vcpu->arch.pio.guest_pages[i] = page; - mutex_unlock(&vcpu->kvm->lock); + up_read(¤t->mm->mmap_sem); if (!page) { kvm_inject_gp(vcpu, 0); free_pio_guest_pages(vcpu); @@ -2247,7 +2270,6 @@ int kvm_fix_hypercall(struct kvm_vcpu *vcpu) char instruction[3]; int ret = 0; - mutex_lock(&vcpu->kvm->lock); /* * Blow out the MMU to ensure that no other VCPU has an active mapping @@ -2262,8 +2284,6 @@ int kvm_fix_hypercall(struct kvm_vcpu *vcpu) != X86EMUL_CONTINUE) ret = -EFAULT; - mutex_unlock(&vcpu->kvm->lock); - return ret; } @@ -2447,8 +2467,10 @@ static void vapic_enter(struct kvm_vcpu *vcpu) if (!apic || !apic->vapic_addr) return; + down_read(¤t->mm->mmap_sem); page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT); vcpu->arch.apic->vapic_page = page; + up_read(¤t->mm->mmap_sem); } static void vapic_exit(struct kvm_vcpu *vcpu) @@ -2910,13 +2932,13 @@ int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, gpa_t gpa; vcpu_load(vcpu); - mutex_lock(&vcpu->kvm->lock); + down_read(¤t->mm->mmap_sem); gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, vaddr); + up_read(¤t->mm->mmap_sem); tr->physical_address = gpa; tr->valid = gpa != UNMAPPED_GVA; tr->writeable = 1; tr->usermode = 0; - mutex_unlock(&vcpu->kvm->lock); vcpu_put(vcpu); return 0; @@ -3185,13 +3207,11 @@ int kvm_arch_set_memory_region(struct kvm *kvm, */ if (!user_alloc) { if (npages && !old.rmap) { - down_write(¤t->mm->mmap_sem); memslot->userspace_addr = do_mmap(NULL, 0, npages * PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, 0); - up_write(¤t->mm->mmap_sem); if (IS_ERR((void *)memslot->userspace_addr)) return PTR_ERR((void *)memslot->userspace_addr); @@ -3199,10 +3219,8 @@ int kvm_arch_set_memory_region(struct kvm *kvm, if (!old.user_alloc && old.rmap) { int ret; - down_write(¤t->mm->mmap_sem); ret = do_munmap(current->mm, old.userspace_addr, old.npages * PAGE_SIZE); - up_write(¤t->mm->mmap_sem); if (ret < 0) printk(KERN_WARNING "kvm_vm_ioctl_set_memory_region: " diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 4026d7d64296..678e80561b74 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -227,7 +227,7 @@ static int kvm_vm_release(struct inode *inode, struct file *filp) * * Discontiguous memory is allowed, mostly for framebuffers. * - * Must be called holding kvm->lock. + * Must be called holding mmap_sem for write. */ int __kvm_set_memory_region(struct kvm *kvm, struct kvm_userspace_memory_region *mem, @@ -338,9 +338,9 @@ int kvm_set_memory_region(struct kvm *kvm, { int r; - mutex_lock(&kvm->lock); + down_write(¤t->mm->mmap_sem); r = __kvm_set_memory_region(kvm, mem, user_alloc); - mutex_unlock(&kvm->lock); + up_write(¤t->mm->mmap_sem); return r; } EXPORT_SYMBOL_GPL(kvm_set_memory_region); @@ -456,7 +456,7 @@ static unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) /* * Requires current->mm->mmap_sem to be held */ -static struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn) +struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) { struct page *page[1]; unsigned long addr; @@ -481,17 +481,6 @@ static struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn) return page[0]; } -struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) -{ - struct page *page; - - down_read(¤t->mm->mmap_sem); - page = __gfn_to_page(kvm, gfn); - up_read(¤t->mm->mmap_sem); - - return page; -} - EXPORT_SYMBOL_GPL(gfn_to_page); void kvm_release_page_clean(struct page *page) @@ -977,8 +966,7 @@ static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) if (!kvm_is_visible_gfn(kvm, vmf->pgoff)) return VM_FAULT_SIGBUS; - /* current->mm->mmap_sem is already held so call lockless version */ - page = __gfn_to_page(kvm, vmf->pgoff); + page = gfn_to_page(kvm, vmf->pgoff); if (is_error_page(page)) { kvm_release_page_clean(page); return VM_FAULT_SIGBUS; From 7ec54588210df29ea637e6054489bc942c0ef371 Mon Sep 17 00:00:00 2001 From: Marcelo Tosatti Date: Thu, 20 Dec 2007 19:18:23 -0500 Subject: [PATCH 238/249] KVM: Add kvm_read_guest_atomic() In preparation for a mmu spinlock, add kvm_read_guest_atomic() and use it in fetch() and prefetch_page(). Signed-off-by: Marcelo Tosatti Signed-off-by: Avi Kivity --- arch/x86/kvm/paging_tmpl.h | 28 ++++++++++++++++------------ include/linux/kvm_host.h | 2 ++ virt/kvm/kvm_main.c | 20 ++++++++++++++++++++ 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h index 7f83f5557d5e..136a65d72b0a 100644 --- a/arch/x86/kvm/paging_tmpl.h +++ b/arch/x86/kvm/paging_tmpl.h @@ -316,10 +316,12 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, metaphysical, access, shadow_ent, &new_page); if (new_page && !metaphysical) { + int r; pt_element_t curr_pte; - kvm_read_guest(vcpu->kvm, walker->pte_gpa[level - 2], - &curr_pte, sizeof(curr_pte)); - if (curr_pte != walker->ptes[level - 2]) + r = kvm_read_guest_atomic(vcpu->kvm, + walker->pte_gpa[level - 2], + &curr_pte, sizeof(curr_pte)); + if (r || curr_pte != walker->ptes[level - 2]) return NULL; } shadow_addr = __pa(shadow_page->spt); @@ -429,9 +431,8 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr) static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) { - int i, offset = 0; - pt_element_t *gpt; - struct page *page; + int i, offset = 0, r = 0; + pt_element_t pt; if (sp->role.metaphysical || (PTTYPE == 32 && sp->role.level > PT_PAGE_TABLE_LEVEL)) { @@ -441,15 +442,18 @@ static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, if (PTTYPE == 32) offset = sp->role.quadrant << PT64_LEVEL_BITS; - page = gfn_to_page(vcpu->kvm, sp->gfn); - gpt = kmap_atomic(page, KM_USER0); - for (i = 0; i < PT64_ENT_PER_PAGE; ++i) - if (is_present_pte(gpt[offset + i])) + + for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { + gpa_t pte_gpa = gfn_to_gpa(sp->gfn); + pte_gpa += (i+offset) * sizeof(pt_element_t); + + r = kvm_read_guest_atomic(vcpu->kvm, pte_gpa, &pt, + sizeof(pt_element_t)); + if (r || is_present_pte(pt)) sp->spt[i] = shadow_trap_nonpresent_pte; else sp->spt[i] = shadow_notrap_nonpresent_pte; - kunmap_atomic(gpt, KM_USER0); - kvm_release_page_clean(page); + } } #undef pt_element_t diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 9ff5904c5072..a020fb280540 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -167,6 +167,8 @@ void kvm_release_page_clean(struct page *page); void kvm_release_page_dirty(struct page *page); int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, int len); +int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, + unsigned long len); int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len); int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, int offset, int len); diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 678e80561b74..8d0b7c16c2f7 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -541,6 +541,26 @@ int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) } EXPORT_SYMBOL_GPL(kvm_read_guest); +int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, + unsigned long len) +{ + int r; + unsigned long addr; + gfn_t gfn = gpa >> PAGE_SHIFT; + int offset = offset_in_page(gpa); + + addr = gfn_to_hva(kvm, gfn); + if (kvm_is_error_hva(addr)) + return -EFAULT; + pagefault_disable(); + r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len); + pagefault_enable(); + if (r) + return -EFAULT; + return 0; +} +EXPORT_SYMBOL(kvm_read_guest_atomic); + int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, int offset, int len) { From d7824fff896a1698a07a8046dc362f4500c302f7 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 30 Dec 2007 12:29:05 +0200 Subject: [PATCH 239/249] KVM: MMU: Avoid calling gfn_to_page() in mmu_set_spte() Since gfn_to_page() is a sleeping function, and we want to make the core mmu spinlocked, we need to pass the page from the walker context (which can sleep) to the shadow context (which cannot). [marcelo: avoid recursive locking of mmap_sem] Signed-off-by: Avi Kivity --- arch/x86/kvm/mmu.c | 55 ++++++++++++++++++++++++++++++++++---- arch/x86/kvm/paging_tmpl.h | 23 ++++++++++++---- include/asm-x86/kvm_host.h | 5 ++++ 3 files changed, 73 insertions(+), 10 deletions(-) diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index 3b91227969a5..c0b757be7b99 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c @@ -890,11 +890,10 @@ struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva) static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte, unsigned pt_access, unsigned pte_access, int user_fault, int write_fault, int dirty, - int *ptwrite, gfn_t gfn) + int *ptwrite, gfn_t gfn, struct page *page) { u64 spte; int was_rmapped = is_rmap_pte(*shadow_pte); - struct page *page; pgprintk("%s: spte %llx access %x write_fault %d" " user_fault %d gfn %lx\n", @@ -912,8 +911,6 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte, if (!(pte_access & ACC_EXEC_MASK)) spte |= PT64_NX_MASK; - page = gfn_to_page(vcpu->kvm, gfn); - spte |= PT_PRESENT_MASK; if (pte_access & ACC_USER_MASK) spte |= PT_USER_MASK; @@ -979,6 +976,11 @@ static int __nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) int level = PT32E_ROOT_LEVEL; hpa_t table_addr = vcpu->arch.mmu.root_hpa; int pt_write = 0; + struct page *page; + + down_read(¤t->mm->mmap_sem); + page = gfn_to_page(vcpu->kvm, gfn); + up_read(¤t->mm->mmap_sem); for (; ; level--) { u32 index = PT64_INDEX(v, level); @@ -989,7 +991,7 @@ static int __nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) if (level == 1) { mmu_set_spte(vcpu, &table[index], ACC_ALL, ACC_ALL, - 0, write, 1, &pt_write, gfn); + 0, write, 1, &pt_write, gfn, page); return pt_write || is_io_pte(table[index]); } @@ -1005,6 +1007,7 @@ static int __nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) NULL); if (!new_table) { pgprintk("nonpaging_map: ENOMEM\n"); + kvm_release_page_clean(page); return -ENOMEM; } @@ -1347,6 +1350,43 @@ static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu) return !!(spte && (*spte & PT_ACCESSED_MASK)); } +static void mmu_guess_page_from_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, + const u8 *new, int bytes) +{ + gfn_t gfn; + int r; + u64 gpte = 0; + + if (bytes != 4 && bytes != 8) + return; + + /* + * Assume that the pte write on a page table of the same type + * as the current vcpu paging mode. This is nearly always true + * (might be false while changing modes). Note it is verified later + * by update_pte(). + */ + if (is_pae(vcpu)) { + /* Handle a 32-bit guest writing two halves of a 64-bit gpte */ + if ((bytes == 4) && (gpa % 4 == 0)) { + r = kvm_read_guest(vcpu->kvm, gpa & ~(u64)7, &gpte, 8); + if (r) + return; + memcpy((void *)&gpte + (gpa % 8), new, 4); + } else if ((bytes == 8) && (gpa % 8 == 0)) { + memcpy((void *)&gpte, new, 8); + } + } else { + if ((bytes == 4) && (gpa % 4 == 0)) + memcpy((void *)&gpte, new, 4); + } + if (!is_present_pte(gpte)) + return; + gfn = (gpte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT; + vcpu->arch.update_pte.gfn = gfn; + vcpu->arch.update_pte.page = gfn_to_page(vcpu->kvm, gfn); +} + void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, int bytes) { @@ -1367,6 +1407,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, int npte; pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes); + mmu_guess_page_from_pte_write(vcpu, gpa, new, bytes); mutex_lock(&vcpu->kvm->lock); ++vcpu->kvm->stat.mmu_pte_write; kvm_mmu_audit(vcpu, "pre pte write"); @@ -1437,6 +1478,10 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, } kvm_mmu_audit(vcpu, "post pte write"); mutex_unlock(&vcpu->kvm->lock); + if (vcpu->arch.update_pte.page) { + kvm_release_page_clean(vcpu->arch.update_pte.page); + vcpu->arch.update_pte.page = NULL; + } } int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h index 136a65d72b0a..3d7846ba26e1 100644 --- a/arch/x86/kvm/paging_tmpl.h +++ b/arch/x86/kvm/paging_tmpl.h @@ -245,6 +245,7 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, { pt_element_t gpte; unsigned pte_access; + struct page *npage; gpte = *(const pt_element_t *)pte; if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) { @@ -256,8 +257,14 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, return; pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); pte_access = page->role.access & FNAME(gpte_access)(vcpu, gpte); + if (gpte_to_gfn(gpte) != vcpu->arch.update_pte.gfn) + return; + npage = vcpu->arch.update_pte.page; + if (!npage) + return; + get_page(npage); mmu_set_spte(vcpu, spte, page->role.access, pte_access, 0, 0, - gpte & PT_DIRTY_MASK, NULL, gpte_to_gfn(gpte)); + gpte & PT_DIRTY_MASK, NULL, gpte_to_gfn(gpte), npage); } /* @@ -265,7 +272,8 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, */ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, struct guest_walker *walker, - int user_fault, int write_fault, int *ptwrite) + int user_fault, int write_fault, int *ptwrite, + struct page *page) { hpa_t shadow_addr; int level; @@ -321,8 +329,10 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, r = kvm_read_guest_atomic(vcpu->kvm, walker->pte_gpa[level - 2], &curr_pte, sizeof(curr_pte)); - if (r || curr_pte != walker->ptes[level - 2]) + if (r || curr_pte != walker->ptes[level - 2]) { + kvm_release_page_clean(page); return NULL; + } } shadow_addr = __pa(shadow_page->spt); shadow_pte = shadow_addr | PT_PRESENT_MASK | PT_ACCESSED_MASK @@ -333,7 +343,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, mmu_set_spte(vcpu, shadow_ent, access, walker->pte_access & access, user_fault, write_fault, walker->ptes[walker->level-1] & PT_DIRTY_MASK, - ptwrite, walker->gfn); + ptwrite, walker->gfn, page); return shadow_ent; } @@ -362,6 +372,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, u64 *shadow_pte; int write_pt = 0; int r; + struct page *page; pgprintk("%s: addr %lx err %x\n", __FUNCTION__, addr, error_code); kvm_mmu_audit(vcpu, "pre page fault"); @@ -388,9 +399,11 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, return 0; } + page = gfn_to_page(vcpu->kvm, walker.gfn); + mutex_lock(&vcpu->kvm->lock); shadow_pte = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault, - &write_pt); + &write_pt, page); pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __FUNCTION__, shadow_pte, *shadow_pte, write_pt); diff --git a/include/asm-x86/kvm_host.h b/include/asm-x86/kvm_host.h index 44b89259f6c4..20597bc16744 100644 --- a/include/asm-x86/kvm_host.h +++ b/include/asm-x86/kvm_host.h @@ -224,6 +224,11 @@ struct kvm_vcpu_arch { int last_pt_write_count; u64 *last_pte_updated; + struct { + gfn_t gfn; /* presumed gfn during guest pte update */ + struct page *page; /* page corresponding to that gfn */ + } update_pte; + struct i387_fxsave_struct host_fx_image; struct i387_fxsave_struct guest_fx_image; From aaee2c94f7a1f7726e360a6cfb40173bd552bcff Mon Sep 17 00:00:00 2001 From: Marcelo Tosatti Date: Thu, 20 Dec 2007 19:18:26 -0500 Subject: [PATCH 240/249] KVM: MMU: Switch to mmu spinlock Convert the synchronization of the shadow handling to a separate mmu_lock spinlock. Also guard fetch() by mmap_sem in read-mode to protect against alias and memslot changes. Signed-off-by: Marcelo Tosatti Signed-off-by: Avi Kivity --- arch/x86/kvm/mmu.c | 48 +++++++++++++++++++++----------------- arch/x86/kvm/paging_tmpl.h | 10 ++++---- arch/x86/kvm/vmx.c | 2 -- include/linux/kvm_host.h | 3 ++- virt/kvm/kvm_main.c | 3 +-- 5 files changed, 35 insertions(+), 31 deletions(-) diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index c0b757be7b99..834698d24595 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c @@ -971,16 +971,12 @@ static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) { } -static int __nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) +static int __nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, + gfn_t gfn, struct page *page) { int level = PT32E_ROOT_LEVEL; hpa_t table_addr = vcpu->arch.mmu.root_hpa; int pt_write = 0; - struct page *page; - - down_read(¤t->mm->mmap_sem); - page = gfn_to_page(vcpu->kvm, gfn); - up_read(¤t->mm->mmap_sem); for (; ; level--) { u32 index = PT64_INDEX(v, level); @@ -1022,9 +1018,17 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) { int r; - mutex_lock(&vcpu->kvm->lock); - r = __nonpaging_map(vcpu, v, write, gfn); - mutex_unlock(&vcpu->kvm->lock); + struct page *page; + + down_read(¤t->mm->mmap_sem); + page = gfn_to_page(vcpu->kvm, gfn); + + spin_lock(&vcpu->kvm->mmu_lock); + r = __nonpaging_map(vcpu, v, write, gfn, page); + spin_unlock(&vcpu->kvm->mmu_lock); + + up_read(¤t->mm->mmap_sem); + return r; } @@ -1045,7 +1049,7 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu) if (!VALID_PAGE(vcpu->arch.mmu.root_hpa)) return; - mutex_lock(&vcpu->kvm->lock); + spin_lock(&vcpu->kvm->mmu_lock); #ifdef CONFIG_X86_64 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) { hpa_t root = vcpu->arch.mmu.root_hpa; @@ -1053,7 +1057,7 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu) sp = page_header(root); --sp->root_count; vcpu->arch.mmu.root_hpa = INVALID_PAGE; - mutex_unlock(&vcpu->kvm->lock); + spin_unlock(&vcpu->kvm->mmu_lock); return; } #endif @@ -1067,7 +1071,7 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu) } vcpu->arch.mmu.pae_root[i] = INVALID_PAGE; } - mutex_unlock(&vcpu->kvm->lock); + spin_unlock(&vcpu->kvm->mmu_lock); vcpu->arch.mmu.root_hpa = INVALID_PAGE; } @@ -1270,9 +1274,9 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu) r = mmu_topup_memory_caches(vcpu); if (r) goto out; - mutex_lock(&vcpu->kvm->lock); + spin_lock(&vcpu->kvm->mmu_lock); mmu_alloc_roots(vcpu); - mutex_unlock(&vcpu->kvm->lock); + spin_unlock(&vcpu->kvm->mmu_lock); kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa); kvm_mmu_flush_tlb(vcpu); out: @@ -1408,7 +1412,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes); mmu_guess_page_from_pte_write(vcpu, gpa, new, bytes); - mutex_lock(&vcpu->kvm->lock); + spin_lock(&vcpu->kvm->mmu_lock); ++vcpu->kvm->stat.mmu_pte_write; kvm_mmu_audit(vcpu, "pre pte write"); if (gfn == vcpu->arch.last_pt_write_gfn @@ -1477,7 +1481,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, } } kvm_mmu_audit(vcpu, "post pte write"); - mutex_unlock(&vcpu->kvm->lock); + spin_unlock(&vcpu->kvm->mmu_lock); if (vcpu->arch.update_pte.page) { kvm_release_page_clean(vcpu->arch.update_pte.page); vcpu->arch.update_pte.page = NULL; @@ -1493,15 +1497,15 @@ int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva); up_read(¤t->mm->mmap_sem); - mutex_lock(&vcpu->kvm->lock); + spin_lock(&vcpu->kvm->mmu_lock); r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT); - mutex_unlock(&vcpu->kvm->lock); + spin_unlock(&vcpu->kvm->mmu_lock); return r; } void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) { - mutex_lock(&vcpu->kvm->lock); + spin_lock(&vcpu->kvm->mmu_lock); while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) { struct kvm_mmu_page *sp; @@ -1510,7 +1514,7 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) kvm_mmu_zap_page(vcpu->kvm, sp); ++vcpu->kvm->stat.mmu_recycled; } - mutex_unlock(&vcpu->kvm->lock); + spin_unlock(&vcpu->kvm->mmu_lock); } int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) @@ -1642,10 +1646,10 @@ void kvm_mmu_zap_all(struct kvm *kvm) { struct kvm_mmu_page *sp, *node; - mutex_lock(&kvm->lock); + spin_lock(&kvm->mmu_lock); list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) kvm_mmu_zap_page(kvm, sp); - mutex_unlock(&kvm->lock); + spin_unlock(&kvm->mmu_lock); kvm_flush_remote_tlbs(kvm); } diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h index 3d7846ba26e1..a35b83a4fef2 100644 --- a/arch/x86/kvm/paging_tmpl.h +++ b/arch/x86/kvm/paging_tmpl.h @@ -387,7 +387,6 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, */ r = FNAME(walk_addr)(&walker, vcpu, addr, write_fault, user_fault, fetch_fault); - up_read(¤t->mm->mmap_sem); /* * The page is not mapped by the guest. Let the guest handle it. @@ -396,12 +395,13 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, pgprintk("%s: guest page fault\n", __FUNCTION__); inject_page_fault(vcpu, addr, walker.error_code); vcpu->arch.last_pt_write_count = 0; /* reset fork detector */ + up_read(¤t->mm->mmap_sem); return 0; } page = gfn_to_page(vcpu->kvm, walker.gfn); - mutex_lock(&vcpu->kvm->lock); + spin_lock(&vcpu->kvm->mmu_lock); shadow_pte = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault, &write_pt, page); pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __FUNCTION__, @@ -414,13 +414,15 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, * mmio: emulate if accessible, otherwise its a guest fault. */ if (shadow_pte && is_io_pte(*shadow_pte)) { - mutex_unlock(&vcpu->kvm->lock); + spin_unlock(&vcpu->kvm->mmu_lock); + up_read(¤t->mm->mmap_sem); return 1; } ++vcpu->stat.pf_fixed; kvm_mmu_audit(vcpu, "post page fault (fixed)"); - mutex_unlock(&vcpu->kvm->lock); + spin_unlock(&vcpu->kvm->mmu_lock); + up_read(¤t->mm->mmap_sem); return write_pt; } diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index c39493feba46..3d251f894a8d 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -1477,7 +1477,6 @@ static int alloc_apic_access_page(struct kvm *kvm) struct kvm_userspace_memory_region kvm_userspace_mem; int r = 0; - mutex_lock(&kvm->lock); down_write(¤t->mm->mmap_sem); if (kvm->arch.apic_access_page) goto out; @@ -1491,7 +1490,6 @@ static int alloc_apic_access_page(struct kvm *kvm) kvm->arch.apic_access_page = gfn_to_page(kvm, 0xfee00); out: up_write(¤t->mm->mmap_sem); - mutex_unlock(&kvm->lock); return r; } diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index a020fb280540..2714068ee8bc 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -104,7 +104,8 @@ struct kvm_memory_slot { }; struct kvm { - struct mutex lock; /* protects everything except vcpus */ + struct mutex lock; /* protects the vcpus array and APIC accesses */ + spinlock_t mmu_lock; struct mm_struct *mm; /* userspace tied to this vm */ int nmemslots; struct kvm_memory_slot memslots[KVM_MEMORY_SLOTS + diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 8d0b7c16c2f7..3c4fe26096fc 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -165,6 +165,7 @@ static struct kvm *kvm_create_vm(void) kvm->mm = current->mm; atomic_inc(&kvm->mm->mm_count); + spin_lock_init(&kvm->mmu_lock); kvm_io_bus_init(&kvm->pio_bus); mutex_init(&kvm->lock); kvm_io_bus_init(&kvm->mmio_bus); @@ -552,9 +553,7 @@ int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, addr = gfn_to_hva(kvm, gfn); if (kvm_is_error_hva(addr)) return -EFAULT; - pagefault_disable(); r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len); - pagefault_enable(); if (r) return -EFAULT; return 0; From eb787d10af8045dd00d4d4c9a8e90fa495f1b0c1 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Mon, 31 Dec 2007 15:27:49 +0200 Subject: [PATCH 241/249] KVM: MMU: Move kvm_free_some_pages() into critical section If some other cpu steals mmu pages between our check and an attempt to allocate, we can run out of mmu pages. Fix by moving the check into the same critical section as the allocation. Signed-off-by: Avi Kivity --- arch/x86/kvm/mmu.c | 9 +++------ arch/x86/kvm/paging_tmpl.h | 1 + 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index 834698d24595..c478ee25de66 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c @@ -291,7 +291,6 @@ static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu) { int r; - kvm_mmu_free_some_pages(vcpu); r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache, pte_chain_cache, 4); if (r) @@ -569,9 +568,6 @@ static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, { struct kvm_mmu_page *sp; - if (!vcpu->kvm->arch.n_free_mmu_pages) - return NULL; - sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp); sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE); sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE); @@ -1024,6 +1020,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) page = gfn_to_page(vcpu->kvm, gfn); spin_lock(&vcpu->kvm->mmu_lock); + kvm_mmu_free_some_pages(vcpu); r = __nonpaging_map(vcpu, v, write, gfn, page); spin_unlock(&vcpu->kvm->mmu_lock); @@ -1275,6 +1272,7 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu) if (r) goto out; spin_lock(&vcpu->kvm->mmu_lock); + kvm_mmu_free_some_pages(vcpu); mmu_alloc_roots(vcpu); spin_unlock(&vcpu->kvm->mmu_lock); kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa); @@ -1413,6 +1411,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes); mmu_guess_page_from_pte_write(vcpu, gpa, new, bytes); spin_lock(&vcpu->kvm->mmu_lock); + kvm_mmu_free_some_pages(vcpu); ++vcpu->kvm->stat.mmu_pte_write; kvm_mmu_audit(vcpu, "pre pte write"); if (gfn == vcpu->arch.last_pt_write_gfn @@ -1505,7 +1504,6 @@ int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) { - spin_lock(&vcpu->kvm->mmu_lock); while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) { struct kvm_mmu_page *sp; @@ -1514,7 +1512,6 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) kvm_mmu_zap_page(vcpu->kvm, sp); ++vcpu->kvm->stat.mmu_recycled; } - spin_unlock(&vcpu->kvm->mmu_lock); } int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h index a35b83a4fef2..349920556be3 100644 --- a/arch/x86/kvm/paging_tmpl.h +++ b/arch/x86/kvm/paging_tmpl.h @@ -402,6 +402,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, page = gfn_to_page(vcpu->kvm, walker.gfn); spin_lock(&vcpu->kvm->mmu_lock); + kvm_mmu_free_some_pages(vcpu); shadow_pte = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault, &write_pt, page); pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __FUNCTION__, From 5882842f9b86179e1fe2e17e35e0e46a268d04e4 Mon Sep 17 00:00:00 2001 From: "Dong, Eddie" Date: Wed, 2 Jan 2008 14:29:08 +0800 Subject: [PATCH 242/249] KVM: MMU: Merge shadow level check in FNAME(fetch) Remove the redundant level check when fetching shadow pte for present & non-present spte. Signed-off-by: Yaozu (Eddie) Dong Signed-off-by: Avi Kivity --- arch/x86/kvm/paging_tmpl.h | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h index 349920556be3..03ba8608fe0f 100644 --- a/arch/x86/kvm/paging_tmpl.h +++ b/arch/x86/kvm/paging_tmpl.h @@ -300,16 +300,13 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, bool new_page = 0; shadow_ent = ((u64 *)__va(shadow_addr)) + index; + if (level == PT_PAGE_TABLE_LEVEL) + break; if (is_shadow_present_pte(*shadow_ent)) { - if (level == PT_PAGE_TABLE_LEVEL) - break; shadow_addr = *shadow_ent & PT64_BASE_ADDR_MASK; continue; } - if (level == PT_PAGE_TABLE_LEVEL) - break; - if (level - 1 == PT_PAGE_TABLE_LEVEL && walker->level == PT_DIRECTORY_LEVEL) { metaphysical = 1; From 571008daccc17c03ccec810922c2bcaed86b15c1 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Wed, 2 Jan 2008 14:49:22 +0800 Subject: [PATCH 243/249] KVM: x86 emulator: Only allow VMCALL/VMMCALL trapped by #UD When executing a test program called "crashme", we found the KVM guest cannot survive more than ten seconds, then encounterd kernel panic. The basic concept of "crashme" is generating random assembly code and trying to execute it. After some fixes on emulator insn validity judgment, we found it's hard to get the current emulator handle the invalid instructions correctly, for the #UD trap for hypercall patching caused troubles. The problem is, if the opcode itself was OK, but combination of opcode and modrm_reg was invalid, and one operand of the opcode was memory (SrcMem or DstMem), the emulator will fetch the memory operand first rather than checking the validity, and may encounter an error there. For example, ".byte 0xfe, 0x34, 0xcd" has this problem. In the patch, we simply check that if the invalid opcode wasn't vmcall/vmmcall, then return from emulate_instruction() and inject a #UD to guest. With the patch, the guest had been running for more than 12 hours. Signed-off-by: Feng (Eric) Liu Signed-off-by: Sheng Yang Signed-off-by: Avi Kivity --- arch/x86/kvm/svm.c | 2 +- arch/x86/kvm/vmx.c | 2 +- arch/x86/kvm/x86.c | 18 +++++++++++++++--- include/asm-x86/kvm_host.h | 4 +++- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index 33f77b044c52..de755cb1431d 100644 --- a/arch/x86/kvm/svm.c +++ b/arch/x86/kvm/svm.c @@ -942,7 +942,7 @@ static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { int er; - er = emulate_instruction(&svm->vcpu, kvm_run, 0, 0, 0); + er = emulate_instruction(&svm->vcpu, kvm_run, 0, 0, EMULTYPE_TRAP_UD); if (er != EMULATE_DONE) kvm_queue_exception(&svm->vcpu, UD_VECTOR); return 1; diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index 3d251f894a8d..ad36447e696e 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -1863,7 +1863,7 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) } if (is_invalid_opcode(intr_info)) { - er = emulate_instruction(vcpu, kvm_run, 0, 0, 0); + er = emulate_instruction(vcpu, kvm_run, 0, 0, EMULTYPE_TRAP_UD); if (er != EMULATE_DONE) kvm_queue_exception(vcpu, UD_VECTOR); return 1; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index e3b3141db13c..8a90403272e2 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1840,9 +1840,10 @@ int emulate_instruction(struct kvm_vcpu *vcpu, struct kvm_run *run, unsigned long cr2, u16 error_code, - int no_decode) + int emulation_type) { int r; + struct decode_cache *c; vcpu->arch.mmio_fault_cr2 = cr2; kvm_x86_ops->cache_regs(vcpu); @@ -1850,7 +1851,7 @@ int emulate_instruction(struct kvm_vcpu *vcpu, vcpu->mmio_is_write = 0; vcpu->arch.pio.string = 0; - if (!no_decode) { + if (!(emulation_type & EMULTYPE_NO_DECODE)) { int cs_db, cs_l; kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); @@ -1884,6 +1885,16 @@ int emulate_instruction(struct kvm_vcpu *vcpu, get_segment_base(vcpu, VCPU_SREG_FS); r = x86_decode_insn(&vcpu->arch.emulate_ctxt, &emulate_ops); + + /* Reject the instructions other than VMCALL/VMMCALL when + * try to emulate invalid opcode */ + c = &vcpu->arch.emulate_ctxt.decode; + if ((emulation_type & EMULTYPE_TRAP_UD) && + (!(c->twobyte && c->b == 0x01 && + (c->modrm_reg == 0 || c->modrm_reg == 3) && + c->modrm_mod == 3 && c->modrm_rm == 1))) + return EMULATE_FAIL; + ++vcpu->stat.insn_emulation; if (r) { ++vcpu->stat.insn_emulation_fail; @@ -2640,7 +2651,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) vcpu->mmio_read_completed = 1; vcpu->mmio_needed = 0; r = emulate_instruction(vcpu, kvm_run, - vcpu->arch.mmio_fault_cr2, 0, 1); + vcpu->arch.mmio_fault_cr2, 0, + EMULTYPE_NO_DECODE); if (r == EMULATE_DO_MMIO) { /* * Read-modify-write. Back to userspace. diff --git a/include/asm-x86/kvm_host.h b/include/asm-x86/kvm_host.h index 20597bc16744..4702b04b979a 100644 --- a/include/asm-x86/kvm_host.h +++ b/include/asm-x86/kvm_host.h @@ -415,8 +415,10 @@ enum emulation_result { EMULATE_FAIL, /* can't emulate this instruction */ }; +#define EMULTYPE_NO_DECODE (1 << 0) +#define EMULTYPE_TRAP_UD (1 << 1) int emulate_instruction(struct kvm_vcpu *vcpu, struct kvm_run *run, - unsigned long cr2, u16 error_code, int no_decode); + unsigned long cr2, u16 error_code, int emulation_type); void kvm_report_emulation_failure(struct kvm_vcpu *cvpu, const char *context); void realmode_lgdt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); void realmode_lidt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); From 6f723c7911e7827091586ae63f4040874eeb75e5 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 8 Jan 2008 08:04:50 +0100 Subject: [PATCH 244/249] KVM: Portability: Move kvm_fpu to asm-x86/kvm.h This patch moves kvm_fpu asm-x86/kvm.h to allow every architecture to define an own representation used for KVM_GET_FPU/KVM_SET_FPU. Signed-off-by: Christian Ehrhardt Acked-by: Carsten Otte Acked-by: Zhang Xiantao Signed-off-by: Avi Kivity --- include/asm-x86/kvm.h | 15 +++++++++++++++ include/linux/kvm.h | 17 ----------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/include/asm-x86/kvm.h b/include/asm-x86/kvm.h index 17afa81d6d66..7a71120426a3 100644 --- a/include/asm-x86/kvm.h +++ b/include/asm-x86/kvm.h @@ -114,6 +114,21 @@ struct kvm_sregs { __u64 interrupt_bitmap[(KVM_NR_INTERRUPTS + 63) / 64]; }; +/* for KVM_GET_FPU and KVM_SET_FPU */ +struct kvm_fpu { + __u8 fpr[8][16]; + __u16 fcw; + __u16 fsw; + __u8 ftwx; /* in fxsave format */ + __u8 pad1; + __u16 last_opcode; + __u64 last_ip; + __u64 last_dp; + __u8 xmm[16][16]; + __u32 mxcsr; + __u32 pad2; +}; + struct kvm_msr_entry { __u32 index; __u32 reserved; diff --git a/include/linux/kvm.h b/include/linux/kvm.h index b27a381db5e8..4de4fd2d8607 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -142,23 +142,6 @@ struct kvm_run { }; }; - -/* for KVM_GET_FPU and KVM_SET_FPU */ -struct kvm_fpu { - __u8 fpr[8][16]; - __u16 fcw; - __u16 fsw; - __u8 ftwx; /* in fxsave format */ - __u8 pad1; - __u16 last_opcode; - __u64 last_ip; - __u64 last_dp; - __u8 xmm[16][16]; - __u32 mxcsr; - __u32 pad2; -}; - - /* for KVM_TRANSLATE */ struct kvm_translation { /* in */ From 75e68e607896c84310dee37c783c45220e56ce8c Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Sat, 12 Jan 2008 23:49:09 +0200 Subject: [PATCH 245/249] KVM: MMU: Fix dirty page setting for pages removed from rmap Right now rmap_remove won't set the page as dirty if the shadow pte pointed to this page had write access and then it became readonly. This patches fixes that, by setting the page as dirty for spte changes from write to readonly access. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- arch/x86/kvm/mmu.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index c478ee25de66..8efdcdbebb03 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c @@ -890,6 +890,7 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte, { u64 spte; int was_rmapped = is_rmap_pte(*shadow_pte); + int was_writeble = is_writeble_pte(*shadow_pte); pgprintk("%s: spte %llx access %x write_fault %d" " user_fault %d gfn %lx\n", @@ -956,9 +957,12 @@ unshadowed: rmap_add(vcpu, shadow_pte, gfn); if (!is_rmap_pte(*shadow_pte)) kvm_release_page_clean(page); + } else { + if (was_writeble) + kvm_release_page_dirty(page); + else + kvm_release_page_clean(page); } - else - kvm_release_page_clean(page); if (!ptwrite || !*ptwrite) vcpu->arch.last_pte_updated = shadow_pte; } From 97db56ce6cc2a0766b536cc227a1da20acc5d82f Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 13 Jan 2008 13:23:56 +0200 Subject: [PATCH 246/249] KVM: Initialize the mmu caches only after verifying cpu support Otherwise we re-initialize the mmu caches, which will fail since the caches are already registered, which will cause us to deinitialize said caches. Signed-off-by: Avi Kivity --- arch/x86/kvm/x86.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 8a90403272e2..77d57ff3d74e 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -2187,12 +2187,6 @@ int kvm_arch_init(void *opaque) int r; struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque; - r = kvm_mmu_module_init(); - if (r) - goto out_fail; - - kvm_init_msr_list(); - if (kvm_x86_ops) { printk(KERN_ERR "kvm: already loaded the other module\n"); r = -EEXIST; @@ -2210,13 +2204,17 @@ int kvm_arch_init(void *opaque) goto out; } + r = kvm_mmu_module_init(); + if (r) + goto out; + + kvm_init_msr_list(); + kvm_x86_ops = ops; kvm_mmu_set_nonpresent_ptes(0ull, 0ull); return 0; out: - kvm_mmu_module_exit(); -out_fail: return r; } From 6c14280125a374d4c279a68276f97245e03e3f68 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 15 Jan 2008 18:27:32 +0200 Subject: [PATCH 247/249] KVM: Fix unbounded preemption latency When preparing to enter the guest, if an interrupt comes in while preemption is disabled but interrupts are still enabled, we miss a preemption point. Fix by explicitly checking whether we need to reschedule. Signed-off-by: Thomas Gleixner Signed-off-by: Avi Kivity --- arch/x86/kvm/x86.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 77d57ff3d74e..f0493e7dcf0c 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -2535,6 +2535,13 @@ again: local_irq_disable(); + if (need_resched()) { + local_irq_enable(); + preempt_enable(); + r = 1; + goto out; + } + if (signal_pending(current)) { local_irq_enable(); preempt_enable(); From a03d7f4b544f699bbdd3cf14692bd8f476cb9d24 Mon Sep 17 00:00:00 2001 From: Glauber de Oliveira Costa Date: Tue, 15 Jan 2008 13:10:15 -0200 Subject: [PATCH 248/249] KVM: Put kvm_para.h include outside __KERNEL__ kvm_para.h potentially contains definitions that are to be used by userspace, so it should not be included inside the __KERNEL__ block. To protect its own data structures, kvm_para.h already includes its own __KERNEL__ block. Signed-off-by: Glauber de Oliveira Costa Acked-by: Amit Shah Signed-off-by: Avi Kivity --- include/linux/kvm_para.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/kvm_para.h b/include/linux/kvm_para.h index 6af91a506fbf..5497aac0d2f8 100644 --- a/include/linux/kvm_para.h +++ b/include/linux/kvm_para.h @@ -14,12 +14,12 @@ #define KVM_HC_VAPIC_POLL_IRQ 1 -#ifdef __KERNEL__ /* * hypercalls use architecture specific */ #include +#ifdef __KERNEL__ static inline int kvm_para_has_feature(unsigned int feature) { if (kvm_arch_para_features() & (1UL << feature)) From 2f52d58c92d971bf421f461ad06eb93fb4f34981 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 16 Jan 2008 12:49:30 +0200 Subject: [PATCH 249/249] KVM: Move apic timer migration away from critical section Migrating the apic timer in the critical section is not very nice, and is absolutely horrible with the real-time port. Move migration to the regular vcpu execution path, triggered by a new bitflag. Cc: Thomas Gleixner Signed-off-by: Avi Kivity --- arch/x86/kvm/irq.h | 2 +- arch/x86/kvm/lapic.c | 3 +-- arch/x86/kvm/x86.c | 5 ++++- include/linux/kvm_host.h | 6 ++++++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h index 53c7f48254be..fa5ed5d59b5d 100644 --- a/arch/x86/kvm/irq.h +++ b/arch/x86/kvm/irq.h @@ -83,6 +83,6 @@ void kvm_pic_reset(struct kvm_kpic_state *s); void kvm_timer_intr_post(struct kvm_vcpu *vcpu, int vec); void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu); void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu); -void kvm_migrate_apic_timer(struct kvm_vcpu *vcpu); +void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu); #endif diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index e7513bb98af1..2cbee9479ce4 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1092,7 +1092,7 @@ void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu) start_apic_timer(apic); } -void kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) +void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) { struct kvm_lapic *apic = vcpu->arch.apic; struct hrtimer *timer; @@ -1104,7 +1104,6 @@ void kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) if (hrtimer_cancel(timer)) hrtimer_start(timer, timer->expires, HRTIMER_MODE_ABS); } -EXPORT_SYMBOL_GPL(kvm_migrate_apic_timer); void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu) { diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index f0493e7dcf0c..8f94a0b89dff 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -2518,13 +2518,16 @@ again: if (unlikely(r)) goto out; - if (vcpu->requests) + if (vcpu->requests) { + if (test_and_clear_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests)) + __kvm_migrate_apic_timer(vcpu); if (test_and_clear_bit(KVM_REQ_REPORT_TPR_ACCESS, &vcpu->requests)) { kvm_run->exit_reason = KVM_EXIT_TPR_ACCESS; r = 0; goto out; } + } kvm_inject_pending_timer_irqs(vcpu); diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 2714068ee8bc..ea4764b0a2f4 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -35,6 +35,7 @@ * vcpu->requests bit members */ #define KVM_REQ_TLB_FLUSH 0 +#define KVM_REQ_MIGRATE_TIMER 1 #define KVM_REQ_REPORT_TPR_ACCESS 2 struct kvm_vcpu; @@ -277,6 +278,11 @@ static inline gpa_t gfn_to_gpa(gfn_t gfn) return (gpa_t)gfn << PAGE_SHIFT; } +static inline void kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) +{ + set_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests); +} + enum kvm_stat_kind { KVM_STAT_VM, KVM_STAT_VCPU,