arm64: KVM: HYP mode world switch implementation
The HYP mode world switch in all its glory. Implements save/restore of host/guest registers, EL2 trapping, IPA resolution, and additional services (tlb invalidation). Reviewed-by: Christopher Covington <cov@codeaurora.org> Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
This commit is contained in:
Marc Zyngier
Родитель
092bd143cb
Коммит
55c7401d92
|
|
@ -104,5 +104,38 @@ int main(void)
|
|||
BLANK();
|
||||
DEFINE(TZ_MINWEST, offsetof(struct timezone, tz_minuteswest));
|
||||
DEFINE(TZ_DSTTIME, offsetof(struct timezone, tz_dsttime));
|
||||
BLANK();
|
||||
#ifdef CONFIG_KVM_ARM_HOST
|
||||
DEFINE(VCPU_CONTEXT, offsetof(struct kvm_vcpu, arch.ctxt));
|
||||
DEFINE(CPU_GP_REGS, offsetof(struct kvm_cpu_context, gp_regs));
|
||||
DEFINE(CPU_USER_PT_REGS, offsetof(struct kvm_regs, regs));
|
||||
DEFINE(CPU_FP_REGS, offsetof(struct kvm_regs, fp_regs));
|
||||
DEFINE(CPU_SP_EL1, offsetof(struct kvm_regs, sp_el1));
|
||||
DEFINE(CPU_ELR_EL1, offsetof(struct kvm_regs, elr_el1));
|
||||
DEFINE(CPU_SPSR, offsetof(struct kvm_regs, spsr));
|
||||
DEFINE(CPU_SYSREGS, offsetof(struct kvm_cpu_context, sys_regs));
|
||||
DEFINE(VCPU_ESR_EL2, offsetof(struct kvm_vcpu, arch.fault.esr_el2));
|
||||
DEFINE(VCPU_FAR_EL2, offsetof(struct kvm_vcpu, arch.fault.far_el2));
|
||||
DEFINE(VCPU_HPFAR_EL2, offsetof(struct kvm_vcpu, arch.fault.hpfar_el2));
|
||||
DEFINE(VCPU_HCR_EL2, offsetof(struct kvm_vcpu, arch.hcr_el2));
|
||||
DEFINE(VCPU_IRQ_LINES, offsetof(struct kvm_vcpu, arch.irq_lines));
|
||||
DEFINE(VCPU_HOST_CONTEXT, offsetof(struct kvm_vcpu, arch.host_cpu_context));
|
||||
DEFINE(VCPU_TIMER_CNTV_CTL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_ctl));
|
||||
DEFINE(VCPU_TIMER_CNTV_CVAL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_cval));
|
||||
DEFINE(KVM_TIMER_CNTVOFF, offsetof(struct kvm, arch.timer.cntvoff));
|
||||
DEFINE(KVM_TIMER_ENABLED, offsetof(struct kvm, arch.timer.enabled));
|
||||
DEFINE(VCPU_KVM, offsetof(struct kvm_vcpu, kvm));
|
||||
DEFINE(VCPU_VGIC_CPU, offsetof(struct kvm_vcpu, arch.vgic_cpu));
|
||||
DEFINE(VGIC_CPU_HCR, offsetof(struct vgic_cpu, vgic_hcr));
|
||||
DEFINE(VGIC_CPU_VMCR, offsetof(struct vgic_cpu, vgic_vmcr));
|
||||
DEFINE(VGIC_CPU_MISR, offsetof(struct vgic_cpu, vgic_misr));
|
||||
DEFINE(VGIC_CPU_EISR, offsetof(struct vgic_cpu, vgic_eisr));
|
||||
DEFINE(VGIC_CPU_ELRSR, offsetof(struct vgic_cpu, vgic_elrsr));
|
||||
DEFINE(VGIC_CPU_APR, offsetof(struct vgic_cpu, vgic_apr));
|
||||
DEFINE(VGIC_CPU_LR, offsetof(struct vgic_cpu, vgic_lr));
|
||||
DEFINE(VGIC_CPU_NR_LR, offsetof(struct vgic_cpu, nr_lr));
|
||||
DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr));
|
||||
DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base));
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,617 @@
|
|||
/*
|
||||
* Copyright (C) 2012,2013 - ARM Ltd
|
||||
* Author: Marc Zyngier <marc.zyngier@arm.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License version 2 as
|
||||
* published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <linux/irqchip/arm-gic.h>
|
||||
|
||||
#include <asm/assembler.h>
|
||||
#include <asm/memory.h>
|
||||
#include <asm/asm-offsets.h>
|
||||
#include <asm/fpsimdmacros.h>
|
||||
#include <asm/kvm.h>
|
||||
#include <asm/kvm_asm.h>
|
||||
#include <asm/kvm_arm.h>
|
||||
#include <asm/kvm_mmu.h>
|
||||
|
||||
#define CPU_GP_REG_OFFSET(x) (CPU_GP_REGS + x)
|
||||
#define CPU_XREG_OFFSET(x) CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x)
|
||||
#define CPU_SPSR_OFFSET(x) CPU_GP_REG_OFFSET(CPU_SPSR + 8*x)
|
||||
#define CPU_SYSREG_OFFSET(x) (CPU_SYSREGS + 8*x)
|
||||
|
||||
.text
|
||||
.pushsection .hyp.text, "ax"
|
||||
.align PAGE_SHIFT
|
||||
|
||||
__kvm_hyp_code_start:
|
||||
.globl __kvm_hyp_code_start
|
||||
|
||||
.macro save_common_regs
|
||||
// x2: base address for cpu context
|
||||
// x3: tmp register
|
||||
|
||||
add x3, x2, #CPU_XREG_OFFSET(19)
|
||||
stp x19, x20, [x3]
|
||||
stp x21, x22, [x3, #16]
|
||||
stp x23, x24, [x3, #32]
|
||||
stp x25, x26, [x3, #48]
|
||||
stp x27, x28, [x3, #64]
|
||||
stp x29, lr, [x3, #80]
|
||||
|
||||
mrs x19, sp_el0
|
||||
mrs x20, elr_el2 // EL1 PC
|
||||
mrs x21, spsr_el2 // EL1 pstate
|
||||
|
||||
stp x19, x20, [x3, #96]
|
||||
str x21, [x3, #112]
|
||||
|
||||
mrs x22, sp_el1
|
||||
mrs x23, elr_el1
|
||||
mrs x24, spsr_el1
|
||||
|
||||
str x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]
|
||||
str x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)]
|
||||
str x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)]
|
||||
.endm
|
||||
|
||||
.macro restore_common_regs
|
||||
// x2: base address for cpu context
|
||||
// x3: tmp register
|
||||
|
||||
ldr x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]
|
||||
ldr x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)]
|
||||
ldr x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)]
|
||||
|
||||
msr sp_el1, x22
|
||||
msr elr_el1, x23
|
||||
msr spsr_el1, x24
|
||||
|
||||
add x3, x2, #CPU_XREG_OFFSET(31) // SP_EL0
|
||||
ldp x19, x20, [x3]
|
||||
ldr x21, [x3, #16]
|
||||
|
||||
msr sp_el0, x19
|
||||
msr elr_el2, x20 // EL1 PC
|
||||
msr spsr_el2, x21 // EL1 pstate
|
||||
|
||||
add x3, x2, #CPU_XREG_OFFSET(19)
|
||||
ldp x19, x20, [x3]
|
||||
ldp x21, x22, [x3, #16]
|
||||
ldp x23, x24, [x3, #32]
|
||||
ldp x25, x26, [x3, #48]
|
||||
ldp x27, x28, [x3, #64]
|
||||
ldp x29, lr, [x3, #80]
|
||||
.endm
|
||||
|
||||
.macro save_host_regs
|
||||
save_common_regs
|
||||
.endm
|
||||
|
||||
.macro restore_host_regs
|
||||
restore_common_regs
|
||||
.endm
|
||||
|
||||
.macro save_fpsimd
|
||||
// x2: cpu context address
|
||||
// x3, x4: tmp regs
|
||||
add x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
|
||||
fpsimd_save x3, 4
|
||||
.endm
|
||||
|
||||
.macro restore_fpsimd
|
||||
// x2: cpu context address
|
||||
// x3, x4: tmp regs
|
||||
add x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
|
||||
fpsimd_restore x3, 4
|
||||
.endm
|
||||
|
||||
.macro save_guest_regs
|
||||
// x0 is the vcpu address
|
||||
// x1 is the return code, do not corrupt!
|
||||
// x2 is the cpu context
|
||||
// x3 is a tmp register
|
||||
// Guest's x0-x3 are on the stack
|
||||
|
||||
// Compute base to save registers
|
||||
add x3, x2, #CPU_XREG_OFFSET(4)
|
||||
stp x4, x5, [x3]
|
||||
stp x6, x7, [x3, #16]
|
||||
stp x8, x9, [x3, #32]
|
||||
stp x10, x11, [x3, #48]
|
||||
stp x12, x13, [x3, #64]
|
||||
stp x14, x15, [x3, #80]
|
||||
stp x16, x17, [x3, #96]
|
||||
str x18, [x3, #112]
|
||||
|
||||
pop x6, x7 // x2, x3
|
||||
pop x4, x5 // x0, x1
|
||||
|
||||
add x3, x2, #CPU_XREG_OFFSET(0)
|
||||
stp x4, x5, [x3]
|
||||
stp x6, x7, [x3, #16]
|
||||
|
||||
save_common_regs
|
||||
.endm
|
||||
|
||||
.macro restore_guest_regs
|
||||
// x0 is the vcpu address.
|
||||
// x2 is the cpu context
|
||||
// x3 is a tmp register
|
||||
|
||||
// Prepare x0-x3 for later restore
|
||||
add x3, x2, #CPU_XREG_OFFSET(0)
|
||||
ldp x4, x5, [x3]
|
||||
ldp x6, x7, [x3, #16]
|
||||
push x4, x5 // Push x0-x3 on the stack
|
||||
push x6, x7
|
||||
|
||||
// x4-x18
|
||||
ldp x4, x5, [x3, #32]
|
||||
ldp x6, x7, [x3, #48]
|
||||
ldp x8, x9, [x3, #64]
|
||||
ldp x10, x11, [x3, #80]
|
||||
ldp x12, x13, [x3, #96]
|
||||
ldp x14, x15, [x3, #112]
|
||||
ldp x16, x17, [x3, #128]
|
||||
ldr x18, [x3, #144]
|
||||
|
||||
// x19-x29, lr, sp*, elr*, spsr*
|
||||
restore_common_regs
|
||||
|
||||
// Last bits of the 64bit state
|
||||
pop x2, x3
|
||||
pop x0, x1
|
||||
|
||||
// Do not touch any register after this!
|
||||
.endm
|
||||
|
||||
/*
|
||||
* Macros to perform system register save/restore.
|
||||
*
|
||||
* Ordering here is absolutely critical, and must be kept consistent
|
||||
* in {save,restore}_sysregs, {save,restore}_guest_32bit_state,
|
||||
* and in kvm_asm.h.
|
||||
*
|
||||
* In other words, don't touch any of these unless you know what
|
||||
* you are doing.
|
||||
*/
|
||||
.macro save_sysregs
|
||||
// x2: base address for cpu context
|
||||
// x3: tmp register
|
||||
|
||||
add x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1)
|
||||
|
||||
mrs x4, vmpidr_el2
|
||||
mrs x5, csselr_el1
|
||||
mrs x6, sctlr_el1
|
||||
mrs x7, actlr_el1
|
||||
mrs x8, cpacr_el1
|
||||
mrs x9, ttbr0_el1
|
||||
mrs x10, ttbr1_el1
|
||||
mrs x11, tcr_el1
|
||||
mrs x12, esr_el1
|
||||
mrs x13, afsr0_el1
|
||||
mrs x14, afsr1_el1
|
||||
mrs x15, far_el1
|
||||
mrs x16, mair_el1
|
||||
mrs x17, vbar_el1
|
||||
mrs x18, contextidr_el1
|
||||
mrs x19, tpidr_el0
|
||||
mrs x20, tpidrro_el0
|
||||
mrs x21, tpidr_el1
|
||||
mrs x22, amair_el1
|
||||
mrs x23, cntkctl_el1
|
||||
|
||||
stp x4, x5, [x3]
|
||||
stp x6, x7, [x3, #16]
|
||||
stp x8, x9, [x3, #32]
|
||||
stp x10, x11, [x3, #48]
|
||||
stp x12, x13, [x3, #64]
|
||||
stp x14, x15, [x3, #80]
|
||||
stp x16, x17, [x3, #96]
|
||||
stp x18, x19, [x3, #112]
|
||||
stp x20, x21, [x3, #128]
|
||||
stp x22, x23, [x3, #144]
|
||||
.endm
|
||||
|
||||
.macro restore_sysregs
|
||||
// x2: base address for cpu context
|
||||
// x3: tmp register
|
||||
|
||||
add x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1)
|
||||
|
||||
ldp x4, x5, [x3]
|
||||
ldp x6, x7, [x3, #16]
|
||||
ldp x8, x9, [x3, #32]
|
||||
ldp x10, x11, [x3, #48]
|
||||
ldp x12, x13, [x3, #64]
|
||||
ldp x14, x15, [x3, #80]
|
||||
ldp x16, x17, [x3, #96]
|
||||
ldp x18, x19, [x3, #112]
|
||||
ldp x20, x21, [x3, #128]
|
||||
ldp x22, x23, [x3, #144]
|
||||
|
||||
msr vmpidr_el2, x4
|
||||
msr csselr_el1, x5
|
||||
msr sctlr_el1, x6
|
||||
msr actlr_el1, x7
|
||||
msr cpacr_el1, x8
|
||||
msr ttbr0_el1, x9
|
||||
msr ttbr1_el1, x10
|
||||
msr tcr_el1, x11
|
||||
msr esr_el1, x12
|
||||
msr afsr0_el1, x13
|
||||
msr afsr1_el1, x14
|
||||
msr far_el1, x15
|
||||
msr mair_el1, x16
|
||||
msr vbar_el1, x17
|
||||
msr contextidr_el1, x18
|
||||
msr tpidr_el0, x19
|
||||
msr tpidrro_el0, x20
|
||||
msr tpidr_el1, x21
|
||||
msr amair_el1, x22
|
||||
msr cntkctl_el1, x23
|
||||
.endm
|
||||
|
||||
.macro activate_traps
|
||||
ldr x2, [x0, #VCPU_IRQ_LINES]
|
||||
ldr x1, [x0, #VCPU_HCR_EL2]
|
||||
orr x2, x2, x1
|
||||
msr hcr_el2, x2
|
||||
|
||||
ldr x2, =(CPTR_EL2_TTA)
|
||||
msr cptr_el2, x2
|
||||
|
||||
ldr x2, =(1 << 15) // Trap CP15 Cr=15
|
||||
msr hstr_el2, x2
|
||||
|
||||
mrs x2, mdcr_el2
|
||||
and x2, x2, #MDCR_EL2_HPMN_MASK
|
||||
orr x2, x2, #(MDCR_EL2_TPM | MDCR_EL2_TPMCR)
|
||||
msr mdcr_el2, x2
|
||||
.endm
|
||||
|
||||
.macro deactivate_traps
|
||||
mov x2, #HCR_RW
|
||||
msr hcr_el2, x2
|
||||
msr cptr_el2, xzr
|
||||
msr hstr_el2, xzr
|
||||
|
||||
mrs x2, mdcr_el2
|
||||
and x2, x2, #MDCR_EL2_HPMN_MASK
|
||||
msr mdcr_el2, x2
|
||||
.endm
|
||||
|
||||
.macro activate_vm
|
||||
ldr x1, [x0, #VCPU_KVM]
|
||||
kern_hyp_va x1
|
||||
ldr x2, [x1, #KVM_VTTBR]
|
||||
msr vttbr_el2, x2
|
||||
.endm
|
||||
|
||||
.macro deactivate_vm
|
||||
msr vttbr_el2, xzr
|
||||
.endm
|
||||
|
||||
__save_sysregs:
|
||||
save_sysregs
|
||||
ret
|
||||
|
||||
__restore_sysregs:
|
||||
restore_sysregs
|
||||
ret
|
||||
|
||||
__save_fpsimd:
|
||||
save_fpsimd
|
||||
ret
|
||||
|
||||
__restore_fpsimd:
|
||||
restore_fpsimd
|
||||
ret
|
||||
|
||||
/*
|
||||
* u64 __kvm_vcpu_run(struct kvm_vcpu *vcpu);
|
||||
*
|
||||
* This is the world switch. The first half of the function
|
||||
* deals with entering the guest, and anything from __kvm_vcpu_return
|
||||
* to the end of the function deals with reentering the host.
|
||||
* On the enter path, only x0 (vcpu pointer) must be preserved until
|
||||
* the last moment. On the exit path, x0 (vcpu pointer) and x1 (exception
|
||||
* code) must both be preserved until the epilogue.
|
||||
* In both cases, x2 points to the CPU context we're saving/restoring from/to.
|
||||
*/
|
||||
ENTRY(__kvm_vcpu_run)
|
||||
kern_hyp_va x0
|
||||
msr tpidr_el2, x0 // Save the vcpu register
|
||||
|
||||
// Host context
|
||||
ldr x2, [x0, #VCPU_HOST_CONTEXT]
|
||||
kern_hyp_va x2
|
||||
|
||||
save_host_regs
|
||||
bl __save_fpsimd
|
||||
bl __save_sysregs
|
||||
|
||||
activate_traps
|
||||
activate_vm
|
||||
|
||||
// Guest context
|
||||
add x2, x0, #VCPU_CONTEXT
|
||||
|
||||
bl __restore_sysregs
|
||||
bl __restore_fpsimd
|
||||
restore_guest_regs
|
||||
|
||||
// That's it, no more messing around.
|
||||
eret
|
||||
|
||||
__kvm_vcpu_return:
|
||||
// Assume x0 is the vcpu pointer, x1 the return code
|
||||
// Guest's x0-x3 are on the stack
|
||||
|
||||
// Guest context
|
||||
add x2, x0, #VCPU_CONTEXT
|
||||
|
||||
save_guest_regs
|
||||
bl __save_fpsimd
|
||||
bl __save_sysregs
|
||||
|
||||
deactivate_traps
|
||||
deactivate_vm
|
||||
|
||||
// Host context
|
||||
ldr x2, [x0, #VCPU_HOST_CONTEXT]
|
||||
kern_hyp_va x2
|
||||
|
||||
bl __restore_sysregs
|
||||
bl __restore_fpsimd
|
||||
restore_host_regs
|
||||
|
||||
mov x0, x1
|
||||
ret
|
||||
END(__kvm_vcpu_run)
|
||||
|
||||
// void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
|
||||
ENTRY(__kvm_tlb_flush_vmid_ipa)
|
||||
kern_hyp_va x0
|
||||
ldr x2, [x0, #KVM_VTTBR]
|
||||
msr vttbr_el2, x2
|
||||
isb
|
||||
|
||||
/*
|
||||
* We could do so much better if we had the VA as well.
|
||||
* Instead, we invalidate Stage-2 for this IPA, and the
|
||||
* whole of Stage-1. Weep...
|
||||
*/
|
||||
tlbi ipas2e1is, x1
|
||||
dsb sy
|
||||
tlbi vmalle1is
|
||||
dsb sy
|
||||
isb
|
||||
|
||||
msr vttbr_el2, xzr
|
||||
ret
|
||||
ENDPROC(__kvm_tlb_flush_vmid_ipa)
|
||||
|
||||
ENTRY(__kvm_flush_vm_context)
|
||||
tlbi alle1is
|
||||
ic ialluis
|
||||
dsb sy
|
||||
ret
|
||||
ENDPROC(__kvm_flush_vm_context)
|
||||
|
||||
__kvm_hyp_panic:
|
||||
// Guess the context by looking at VTTBR:
|
||||
// If zero, then we're already a host.
|
||||
// Otherwise restore a minimal host context before panicing.
|
||||
mrs x0, vttbr_el2
|
||||
cbz x0, 1f
|
||||
|
||||
mrs x0, tpidr_el2
|
||||
|
||||
deactivate_traps
|
||||
deactivate_vm
|
||||
|
||||
ldr x2, [x0, #VCPU_HOST_CONTEXT]
|
||||
kern_hyp_va x2
|
||||
|
||||
bl __restore_sysregs
|
||||
|
||||
1: adr x0, __hyp_panic_str
|
||||
adr x1, 2f
|
||||
ldp x2, x3, [x1]
|
||||
sub x0, x0, x2
|
||||
add x0, x0, x3
|
||||
mrs x1, spsr_el2
|
||||
mrs x2, elr_el2
|
||||
mrs x3, esr_el2
|
||||
mrs x4, far_el2
|
||||
mrs x5, hpfar_el2
|
||||
mrs x6, par_el1
|
||||
mrs x7, tpidr_el2
|
||||
|
||||
mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
|
||||
PSR_MODE_EL1h)
|
||||
msr spsr_el2, lr
|
||||
ldr lr, =panic
|
||||
msr elr_el2, lr
|
||||
eret
|
||||
|
||||
.align 3
|
||||
2: .quad HYP_PAGE_OFFSET
|
||||
.quad PAGE_OFFSET
|
||||
ENDPROC(__kvm_hyp_panic)
|
||||
|
||||
__hyp_panic_str:
|
||||
.ascii "HYP panic:\nPS:%08x PC:%p ESR:%p\nFAR:%p HPFAR:%p PAR:%p\nVCPU:%p\n\0"
|
||||
|
||||
.align 2
|
||||
|
||||
ENTRY(kvm_call_hyp)
|
||||
hvc #0
|
||||
ret
|
||||
ENDPROC(kvm_call_hyp)
|
||||
|
||||
.macro invalid_vector label, target
|
||||
.align 2
|
||||
\label:
|
||||
b \target
|
||||
ENDPROC(\label)
|
||||
.endm
|
||||
|
||||
/* None of these should ever happen */
|
||||
invalid_vector el2t_sync_invalid, __kvm_hyp_panic
|
||||
invalid_vector el2t_irq_invalid, __kvm_hyp_panic
|
||||
invalid_vector el2t_fiq_invalid, __kvm_hyp_panic
|
||||
invalid_vector el2t_error_invalid, __kvm_hyp_panic
|
||||
invalid_vector el2h_sync_invalid, __kvm_hyp_panic
|
||||
invalid_vector el2h_irq_invalid, __kvm_hyp_panic
|
||||
invalid_vector el2h_fiq_invalid, __kvm_hyp_panic
|
||||
invalid_vector el2h_error_invalid, __kvm_hyp_panic
|
||||
invalid_vector el1_sync_invalid, __kvm_hyp_panic
|
||||
invalid_vector el1_irq_invalid, __kvm_hyp_panic
|
||||
invalid_vector el1_fiq_invalid, __kvm_hyp_panic
|
||||
invalid_vector el1_error_invalid, __kvm_hyp_panic
|
||||
|
||||
el1_sync: // Guest trapped into EL2
|
||||
push x0, x1
|
||||
push x2, x3
|
||||
|
||||
mrs x1, esr_el2
|
||||
lsr x2, x1, #ESR_EL2_EC_SHIFT
|
||||
|
||||
cmp x2, #ESR_EL2_EC_HVC64
|
||||
b.ne el1_trap
|
||||
|
||||
mrs x3, vttbr_el2 // If vttbr is valid, the 64bit guest
|
||||
cbnz x3, el1_trap // called HVC
|
||||
|
||||
/* Here, we're pretty sure the host called HVC. */
|
||||
pop x2, x3
|
||||
pop x0, x1
|
||||
|
||||
push lr, xzr
|
||||
|
||||
/*
|
||||
* Compute the function address in EL2, and shuffle the parameters.
|
||||
*/
|
||||
kern_hyp_va x0
|
||||
mov lr, x0
|
||||
mov x0, x1
|
||||
mov x1, x2
|
||||
mov x2, x3
|
||||
blr lr
|
||||
|
||||
pop lr, xzr
|
||||
eret
|
||||
|
||||
el1_trap:
|
||||
/*
|
||||
* x1: ESR
|
||||
* x2: ESR_EC
|
||||
*/
|
||||
cmp x2, #ESR_EL2_EC_DABT
|
||||
mov x0, #ESR_EL2_EC_IABT
|
||||
ccmp x2, x0, #4, ne
|
||||
b.ne 1f // Not an abort we care about
|
||||
|
||||
/* This is an abort. Check for permission fault */
|
||||
and x2, x1, #ESR_EL2_FSC_TYPE
|
||||
cmp x2, #FSC_PERM
|
||||
b.ne 1f // Not a permission fault
|
||||
|
||||
/*
|
||||
* Check for Stage-1 page table walk, which is guaranteed
|
||||
* to give a valid HPFAR_EL2.
|
||||
*/
|
||||
tbnz x1, #7, 1f // S1PTW is set
|
||||
|
||||
/*
|
||||
* Permission fault, HPFAR_EL2 is invalid.
|
||||
* Resolve the IPA the hard way using the guest VA.
|
||||
* Stage-1 translation already validated the memory access rights.
|
||||
* As such, we can use the EL1 translation regime, and don't have
|
||||
* to distinguish between EL0 and EL1 access.
|
||||
*/
|
||||
mrs x2, far_el2
|
||||
at s1e1r, x2
|
||||
isb
|
||||
|
||||
/* Read result */
|
||||
mrs x3, par_el1
|
||||
tbnz x3, #0, 3f // Bail out if we failed the translation
|
||||
ubfx x3, x3, #12, #36 // Extract IPA
|
||||
lsl x3, x3, #4 // and present it like HPFAR
|
||||
b 2f
|
||||
|
||||
1: mrs x3, hpfar_el2
|
||||
mrs x2, far_el2
|
||||
|
||||
2: mrs x0, tpidr_el2
|
||||
str x1, [x0, #VCPU_ESR_EL2]
|
||||
str x2, [x0, #VCPU_FAR_EL2]
|
||||
str x3, [x0, #VCPU_HPFAR_EL2]
|
||||
|
||||
mov x1, #ARM_EXCEPTION_TRAP
|
||||
b __kvm_vcpu_return
|
||||
|
||||
/*
|
||||
* Translation failed. Just return to the guest and
|
||||
* let it fault again. Another CPU is probably playing
|
||||
* behind our back.
|
||||
*/
|
||||
3: pop x2, x3
|
||||
pop x0, x1
|
||||
|
||||
eret
|
||||
|
||||
el1_irq:
|
||||
push x0, x1
|
||||
push x2, x3
|
||||
mrs x0, tpidr_el2
|
||||
mov x1, #ARM_EXCEPTION_IRQ
|
||||
b __kvm_vcpu_return
|
||||
|
||||
.ltorg
|
||||
|
||||
.align 11
|
||||
|
||||
ENTRY(__kvm_hyp_vector)
|
||||
ventry el2t_sync_invalid // Synchronous EL2t
|
||||
ventry el2t_irq_invalid // IRQ EL2t
|
||||
ventry el2t_fiq_invalid // FIQ EL2t
|
||||
ventry el2t_error_invalid // Error EL2t
|
||||
|
||||
ventry el2h_sync_invalid // Synchronous EL2h
|
||||
ventry el2h_irq_invalid // IRQ EL2h
|
||||
ventry el2h_fiq_invalid // FIQ EL2h
|
||||
ventry el2h_error_invalid // Error EL2h
|
||||
|
||||
ventry el1_sync // Synchronous 64-bit EL1
|
||||
ventry el1_irq // IRQ 64-bit EL1
|
||||
ventry el1_fiq_invalid // FIQ 64-bit EL1
|
||||
ventry el1_error_invalid // Error 64-bit EL1
|
||||
|
||||
ventry el1_sync // Synchronous 32-bit EL1
|
||||
ventry el1_irq // IRQ 32-bit EL1
|
||||
ventry el1_fiq_invalid // FIQ 32-bit EL1
|
||||
ventry el1_error_invalid // Error 32-bit EL1
|
||||
ENDPROC(__kvm_hyp_vector)
|
||||
|
||||
__kvm_hyp_code_end:
|
||||
.globl __kvm_hyp_code_end
|
||||
|
||||
.popsection
|
||||
Загрузка…
Ссылка в новой задаче