| /* | |
| * Copyright (C) 2012,2013 - ARM Ltd | |
| * Author: Marc Zyngier <marc.zyngier@arm.com> | |
| * | |
| * This program is free software; you can redistribute it and/or modify | |
| * it under the terms of the GNU General Public License version 2 as | |
| * published by the Free Software Foundation. | |
| * | |
| * This program is distributed in the hope that it will be useful, | |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| * GNU General Public License for more details. | |
| * | |
| * You should have received a copy of the GNU General Public License | |
| * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| */ | |
| #include <linux/linkage.h> | |
| #include <linux/irqchip/arm-gic.h> | |
| #include <asm/assembler.h> | |
| #include <asm/memory.h> | |
| #include <asm/asm-offsets.h> | |
| #include <asm/fpsimdmacros.h> | |
| #include <asm/kvm.h> | |
| #include <asm/kvm_asm.h> | |
| #include <asm/kvm_arm.h> | |
| #include <asm/kvm_mmu.h> | |
| #define CPU_GP_REG_OFFSET(x) (CPU_GP_REGS + x) | |
| #define CPU_XREG_OFFSET(x) CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x) | |
| #define CPU_SPSR_OFFSET(x) CPU_GP_REG_OFFSET(CPU_SPSR + 8*x) | |
| #define CPU_SYSREG_OFFSET(x) (CPU_SYSREGS + 8*x) | |
| .text | |
| .pushsection .hyp.text, "ax" | |
| .align PAGE_SHIFT | |
| __kvm_hyp_code_start: | |
| .globl __kvm_hyp_code_start | |
| .macro save_common_regs | |
| // x2: base address for cpu context | |
| // x3: tmp register | |
| add x3, x2, #CPU_XREG_OFFSET(19) | |
| stp x19, x20, [x3] | |
| stp x21, x22, [x3, #16] | |
| stp x23, x24, [x3, #32] | |
| stp x25, x26, [x3, #48] | |
| stp x27, x28, [x3, #64] | |
| stp x29, lr, [x3, #80] | |
| mrs x19, sp_el0 | |
| mrs x20, elr_el2 // EL1 PC | |
| mrs x21, spsr_el2 // EL1 pstate | |
| stp x19, x20, [x3, #96] | |
| str x21, [x3, #112] | |
| mrs x22, sp_el1 | |
| mrs x23, elr_el1 | |
| mrs x24, spsr_el1 | |
| str x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)] | |
| str x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)] | |
| str x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)] | |
| .endm | |
| .macro restore_common_regs | |
| // x2: base address for cpu context | |
| // x3: tmp register | |
| ldr x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)] | |
| ldr x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)] | |
| ldr x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)] | |
| msr sp_el1, x22 | |
| msr elr_el1, x23 | |
| msr spsr_el1, x24 | |
| add x3, x2, #CPU_XREG_OFFSET(31) // SP_EL0 | |
| ldp x19, x20, [x3] | |
| ldr x21, [x3, #16] | |
| msr sp_el0, x19 | |
| msr elr_el2, x20 // EL1 PC | |
| msr spsr_el2, x21 // EL1 pstate | |
| add x3, x2, #CPU_XREG_OFFSET(19) | |
| ldp x19, x20, [x3] | |
| ldp x21, x22, [x3, #16] | |
| ldp x23, x24, [x3, #32] | |
| ldp x25, x26, [x3, #48] | |
| ldp x27, x28, [x3, #64] | |
| ldp x29, lr, [x3, #80] | |
| .endm | |
| .macro save_host_regs | |
| save_common_regs | |
| .endm | |
| .macro restore_host_regs | |
| restore_common_regs | |
| .endm | |
| .macro save_fpsimd | |
| // x2: cpu context address | |
| // x3, x4: tmp regs | |
| add x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS) | |
| fpsimd_save x3, 4 | |
| .endm | |
| .macro restore_fpsimd | |
| // x2: cpu context address | |
| // x3, x4: tmp regs | |
| add x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS) | |
| fpsimd_restore x3, 4 | |
| .endm | |
| .macro save_guest_regs | |
| // x0 is the vcpu address | |
| // x1 is the return code, do not corrupt! | |
| // x2 is the cpu context | |
| // x3 is a tmp register | |
| // Guest's x0-x3 are on the stack | |
| // Compute base to save registers | |
| add x3, x2, #CPU_XREG_OFFSET(4) | |
| stp x4, x5, [x3] | |
| stp x6, x7, [x3, #16] | |
| stp x8, x9, [x3, #32] | |
| stp x10, x11, [x3, #48] | |
| stp x12, x13, [x3, #64] | |
| stp x14, x15, [x3, #80] | |
| stp x16, x17, [x3, #96] | |
| str x18, [x3, #112] | |
| pop x6, x7 // x2, x3 | |
| pop x4, x5 // x0, x1 | |
| add x3, x2, #CPU_XREG_OFFSET(0) | |
| stp x4, x5, [x3] | |
| stp x6, x7, [x3, #16] | |
| save_common_regs | |
| .endm | |
| .macro restore_guest_regs | |
| // x0 is the vcpu address. | |
| // x2 is the cpu context | |
| // x3 is a tmp register | |
| // Prepare x0-x3 for later restore | |
| add x3, x2, #CPU_XREG_OFFSET(0) | |
| ldp x4, x5, [x3] | |
| ldp x6, x7, [x3, #16] | |
| push x4, x5 // Push x0-x3 on the stack | |
| push x6, x7 | |
| // x4-x18 | |
| ldp x4, x5, [x3, #32] | |
| ldp x6, x7, [x3, #48] | |
| ldp x8, x9, [x3, #64] | |
| ldp x10, x11, [x3, #80] | |
| ldp x12, x13, [x3, #96] | |
| ldp x14, x15, [x3, #112] | |
| ldp x16, x17, [x3, #128] | |
| ldr x18, [x3, #144] | |
| // x19-x29, lr, sp*, elr*, spsr* | |
| restore_common_regs | |
| // Last bits of the 64bit state | |
| pop x2, x3 | |
| pop x0, x1 | |
| // Do not touch any register after this! | |
| .endm | |
| /* | |
| * Macros to perform system register save/restore. | |
| * | |
| * Ordering here is absolutely critical, and must be kept consistent | |
| * in {save,restore}_sysregs, {save,restore}_guest_32bit_state, | |
| * and in kvm_asm.h. | |
| * | |
| * In other words, don't touch any of these unless you know what | |
| * you are doing. | |
| */ | |
| .macro save_sysregs | |
| // x2: base address for cpu context | |
| // x3: tmp register | |
| add x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1) | |
| mrs x4, vmpidr_el2 | |
| mrs x5, csselr_el1 | |
| mrs x6, sctlr_el1 | |
| mrs x7, actlr_el1 | |
| mrs x8, cpacr_el1 | |
| mrs x9, ttbr0_el1 | |
| mrs x10, ttbr1_el1 | |
| mrs x11, tcr_el1 | |
| mrs x12, esr_el1 | |
| mrs x13, afsr0_el1 | |
| mrs x14, afsr1_el1 | |
| mrs x15, far_el1 | |
| mrs x16, mair_el1 | |
| mrs x17, vbar_el1 | |
| mrs x18, contextidr_el1 | |
| mrs x19, tpidr_el0 | |
| mrs x20, tpidrro_el0 | |
| mrs x21, tpidr_el1 | |
| mrs x22, amair_el1 | |
| mrs x23, cntkctl_el1 | |
| mrs x24, par_el1 | |
| stp x4, x5, [x3] | |
| stp x6, x7, [x3, #16] | |
| stp x8, x9, [x3, #32] | |
| stp x10, x11, [x3, #48] | |
| stp x12, x13, [x3, #64] | |
| stp x14, x15, [x3, #80] | |
| stp x16, x17, [x3, #96] | |
| stp x18, x19, [x3, #112] | |
| stp x20, x21, [x3, #128] | |
| stp x22, x23, [x3, #144] | |
| str x24, [x3, #160] | |
| .endm | |
| .macro restore_sysregs | |
| // x2: base address for cpu context | |
| // x3: tmp register | |
| add x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1) | |
| ldp x4, x5, [x3] | |
| ldp x6, x7, [x3, #16] | |
| ldp x8, x9, [x3, #32] | |
| ldp x10, x11, [x3, #48] | |
| ldp x12, x13, [x3, #64] | |
| ldp x14, x15, [x3, #80] | |
| ldp x16, x17, [x3, #96] | |
| ldp x18, x19, [x3, #112] | |
| ldp x20, x21, [x3, #128] | |
| ldp x22, x23, [x3, #144] | |
| ldr x24, [x3, #160] | |
| msr vmpidr_el2, x4 | |
| msr csselr_el1, x5 | |
| msr sctlr_el1, x6 | |
| msr actlr_el1, x7 | |
| msr cpacr_el1, x8 | |
| msr ttbr0_el1, x9 | |
| msr ttbr1_el1, x10 | |
| msr tcr_el1, x11 | |
| msr esr_el1, x12 | |
| msr afsr0_el1, x13 | |
| msr afsr1_el1, x14 | |
| msr far_el1, x15 | |
| msr mair_el1, x16 | |
| msr vbar_el1, x17 | |
| msr contextidr_el1, x18 | |
| msr tpidr_el0, x19 | |
| msr tpidrro_el0, x20 | |
| msr tpidr_el1, x21 | |
| msr amair_el1, x22 | |
| msr cntkctl_el1, x23 | |
| msr par_el1, x24 | |
| .endm | |
| .macro skip_32bit_state tmp, target | |
| // Skip 32bit state if not needed | |
| mrs \tmp, hcr_el2 | |
| tbnz \tmp, #HCR_RW_SHIFT, \target | |
| .endm | |
| .macro skip_tee_state tmp, target | |
| // Skip ThumbEE state if not needed | |
| mrs \tmp, id_pfr0_el1 | |
| tbz \tmp, #12, \target | |
| .endm | |
| .macro save_guest_32bit_state | |
| skip_32bit_state x3, 1f | |
| add x3, x2, #CPU_SPSR_OFFSET(KVM_SPSR_ABT) | |
| mrs x4, spsr_abt | |
| mrs x5, spsr_und | |
| mrs x6, spsr_irq | |
| mrs x7, spsr_fiq | |
| stp x4, x5, [x3] | |
| stp x6, x7, [x3, #16] | |
| add x3, x2, #CPU_SYSREG_OFFSET(DACR32_EL2) | |
| mrs x4, dacr32_el2 | |
| mrs x5, ifsr32_el2 | |
| mrs x6, fpexc32_el2 | |
| mrs x7, dbgvcr32_el2 | |
| stp x4, x5, [x3] | |
| stp x6, x7, [x3, #16] | |
| skip_tee_state x8, 1f | |
| add x3, x2, #CPU_SYSREG_OFFSET(TEECR32_EL1) | |
| mrs x4, teecr32_el1 | |
| mrs x5, teehbr32_el1 | |
| stp x4, x5, [x3] | |
| 1: | |
| .endm | |
| .macro restore_guest_32bit_state | |
| skip_32bit_state x3, 1f | |
| add x3, x2, #CPU_SPSR_OFFSET(KVM_SPSR_ABT) | |
| ldp x4, x5, [x3] | |
| ldp x6, x7, [x3, #16] | |
| msr spsr_abt, x4 | |
| msr spsr_und, x5 | |
| msr spsr_irq, x6 | |
| msr spsr_fiq, x7 | |
| add x3, x2, #CPU_SYSREG_OFFSET(DACR32_EL2) | |
| ldp x4, x5, [x3] | |
| ldp x6, x7, [x3, #16] | |
| msr dacr32_el2, x4 | |
| msr ifsr32_el2, x5 | |
| msr fpexc32_el2, x6 | |
| msr dbgvcr32_el2, x7 | |
| skip_tee_state x8, 1f | |
| add x3, x2, #CPU_SYSREG_OFFSET(TEECR32_EL1) | |
| ldp x4, x5, [x3] | |
| msr teecr32_el1, x4 | |
| msr teehbr32_el1, x5 | |
| 1: | |
| .endm | |
| .macro activate_traps | |
| ldr x2, [x0, #VCPU_IRQ_LINES] | |
| ldr x1, [x0, #VCPU_HCR_EL2] | |
| orr x2, x2, x1 | |
| msr hcr_el2, x2 | |
| ldr x2, =(CPTR_EL2_TTA) | |
| msr cptr_el2, x2 | |
| ldr x2, =(1 << 15) // Trap CP15 Cr=15 | |
| msr hstr_el2, x2 | |
| mrs x2, mdcr_el2 | |
| and x2, x2, #MDCR_EL2_HPMN_MASK | |
| orr x2, x2, #(MDCR_EL2_TPM | MDCR_EL2_TPMCR) | |
| msr mdcr_el2, x2 | |
| .endm | |
| .macro deactivate_traps | |
| mov x2, #HCR_RW | |
| msr hcr_el2, x2 | |
| msr cptr_el2, xzr | |
| msr hstr_el2, xzr | |
| mrs x2, mdcr_el2 | |
| and x2, x2, #MDCR_EL2_HPMN_MASK | |
| msr mdcr_el2, x2 | |
| .endm | |
| .macro activate_vm | |
| ldr x1, [x0, #VCPU_KVM] | |
| kern_hyp_va x1 | |
| ldr x2, [x1, #KVM_VTTBR] | |
| msr vttbr_el2, x2 | |
| .endm | |
| .macro deactivate_vm | |
| msr vttbr_el2, xzr | |
| .endm | |
| /* | |
| * Save the VGIC CPU state into memory | |
| * x0: Register pointing to VCPU struct | |
| * Do not corrupt x1!!! | |
| */ | |
| .macro save_vgic_state | |
| /* Get VGIC VCTRL base into x2 */ | |
| ldr x2, [x0, #VCPU_KVM] | |
| kern_hyp_va x2 | |
| ldr x2, [x2, #KVM_VGIC_VCTRL] | |
| kern_hyp_va x2 | |
| cbz x2, 2f // disabled | |
| /* Compute the address of struct vgic_cpu */ | |
| add x3, x0, #VCPU_VGIC_CPU | |
| /* Save all interesting registers */ | |
| ldr w4, [x2, #GICH_HCR] | |
| ldr w5, [x2, #GICH_VMCR] | |
| ldr w6, [x2, #GICH_MISR] | |
| ldr w7, [x2, #GICH_EISR0] | |
| ldr w8, [x2, #GICH_EISR1] | |
| ldr w9, [x2, #GICH_ELRSR0] | |
| ldr w10, [x2, #GICH_ELRSR1] | |
| ldr w11, [x2, #GICH_APR] | |
| CPU_BE( rev w4, w4 ) | |
| CPU_BE( rev w5, w5 ) | |
| CPU_BE( rev w6, w6 ) | |
| CPU_BE( rev w7, w7 ) | |
| CPU_BE( rev w8, w8 ) | |
| CPU_BE( rev w9, w9 ) | |
| CPU_BE( rev w10, w10 ) | |
| CPU_BE( rev w11, w11 ) | |
| str w4, [x3, #VGIC_CPU_HCR] | |
| str w5, [x3, #VGIC_CPU_VMCR] | |
| str w6, [x3, #VGIC_CPU_MISR] | |
| str w7, [x3, #VGIC_CPU_EISR] | |
| str w8, [x3, #(VGIC_CPU_EISR + 4)] | |
| str w9, [x3, #VGIC_CPU_ELRSR] | |
| str w10, [x3, #(VGIC_CPU_ELRSR + 4)] | |
| str w11, [x3, #VGIC_CPU_APR] | |
| /* Clear GICH_HCR */ | |
| str wzr, [x2, #GICH_HCR] | |
| /* Save list registers */ | |
| add x2, x2, #GICH_LR0 | |
| ldr w4, [x3, #VGIC_CPU_NR_LR] | |
| add x3, x3, #VGIC_CPU_LR | |
| 1: ldr w5, [x2], #4 | |
| CPU_BE( rev w5, w5 ) | |
| str w5, [x3], #4 | |
| sub w4, w4, #1 | |
| cbnz w4, 1b | |
| 2: | |
| .endm | |
| /* | |
| * Restore the VGIC CPU state from memory | |
| * x0: Register pointing to VCPU struct | |
| */ | |
| .macro restore_vgic_state | |
| /* Get VGIC VCTRL base into x2 */ | |
| ldr x2, [x0, #VCPU_KVM] | |
| kern_hyp_va x2 | |
| ldr x2, [x2, #KVM_VGIC_VCTRL] | |
| kern_hyp_va x2 | |
| cbz x2, 2f // disabled | |
| /* Compute the address of struct vgic_cpu */ | |
| add x3, x0, #VCPU_VGIC_CPU | |
| /* We only restore a minimal set of registers */ | |
| ldr w4, [x3, #VGIC_CPU_HCR] | |
| ldr w5, [x3, #VGIC_CPU_VMCR] | |
| ldr w6, [x3, #VGIC_CPU_APR] | |
| CPU_BE( rev w4, w4 ) | |
| CPU_BE( rev w5, w5 ) | |
| CPU_BE( rev w6, w6 ) | |
| str w4, [x2, #GICH_HCR] | |
| str w5, [x2, #GICH_VMCR] | |
| str w6, [x2, #GICH_APR] | |
| /* Restore list registers */ | |
| add x2, x2, #GICH_LR0 | |
| ldr w4, [x3, #VGIC_CPU_NR_LR] | |
| add x3, x3, #VGIC_CPU_LR | |
| 1: ldr w5, [x3], #4 | |
| CPU_BE( rev w5, w5 ) | |
| str w5, [x2], #4 | |
| sub w4, w4, #1 | |
| cbnz w4, 1b | |
| 2: | |
| .endm | |
| .macro save_timer_state | |
| // x0: vcpu pointer | |
| ldr x2, [x0, #VCPU_KVM] | |
| kern_hyp_va x2 | |
| ldr w3, [x2, #KVM_TIMER_ENABLED] | |
| cbz w3, 1f | |
| mrs x3, cntv_ctl_el0 | |
| and x3, x3, #3 | |
| str w3, [x0, #VCPU_TIMER_CNTV_CTL] | |
| bic x3, x3, #1 // Clear Enable | |
| msr cntv_ctl_el0, x3 | |
| isb | |
| mrs x3, cntv_cval_el0 | |
| str x3, [x0, #VCPU_TIMER_CNTV_CVAL] | |
| 1: | |
| // Allow physical timer/counter access for the host | |
| mrs x2, cnthctl_el2 | |
| orr x2, x2, #3 | |
| msr cnthctl_el2, x2 | |
| // Clear cntvoff for the host | |
| msr cntvoff_el2, xzr | |
| .endm | |
| .macro restore_timer_state | |
| // x0: vcpu pointer | |
| // Disallow physical timer access for the guest | |
| // Physical counter access is allowed | |
| mrs x2, cnthctl_el2 | |
| orr x2, x2, #1 | |
| bic x2, x2, #2 | |
| msr cnthctl_el2, x2 | |
| ldr x2, [x0, #VCPU_KVM] | |
| kern_hyp_va x2 | |
| ldr w3, [x2, #KVM_TIMER_ENABLED] | |
| cbz w3, 1f | |
| ldr x3, [x2, #KVM_TIMER_CNTVOFF] | |
| msr cntvoff_el2, x3 | |
| ldr x2, [x0, #VCPU_TIMER_CNTV_CVAL] | |
| msr cntv_cval_el0, x2 | |
| isb | |
| ldr w2, [x0, #VCPU_TIMER_CNTV_CTL] | |
| and x2, x2, #3 | |
| msr cntv_ctl_el0, x2 | |
| 1: | |
| .endm | |
| __save_sysregs: | |
| save_sysregs | |
| ret | |
| __restore_sysregs: | |
| restore_sysregs | |
| ret | |
| __save_fpsimd: | |
| save_fpsimd | |
| ret | |
| __restore_fpsimd: | |
| restore_fpsimd | |
| ret | |
| /* | |
| * u64 __kvm_vcpu_run(struct kvm_vcpu *vcpu); | |
| * | |
| * This is the world switch. The first half of the function | |
| * deals with entering the guest, and anything from __kvm_vcpu_return | |
| * to the end of the function deals with reentering the host. | |
| * On the enter path, only x0 (vcpu pointer) must be preserved until | |
| * the last moment. On the exit path, x0 (vcpu pointer) and x1 (exception | |
| * code) must both be preserved until the epilogue. | |
| * In both cases, x2 points to the CPU context we're saving/restoring from/to. | |
| */ | |
| ENTRY(__kvm_vcpu_run) | |
| kern_hyp_va x0 | |
| msr tpidr_el2, x0 // Save the vcpu register | |
| // Host context | |
| ldr x2, [x0, #VCPU_HOST_CONTEXT] | |
| kern_hyp_va x2 | |
| save_host_regs | |
| bl __save_fpsimd | |
| bl __save_sysregs | |
| activate_traps | |
| activate_vm | |
| restore_vgic_state | |
| restore_timer_state | |
| // Guest context | |
| add x2, x0, #VCPU_CONTEXT | |
| bl __restore_sysregs | |
| bl __restore_fpsimd | |
| restore_guest_32bit_state | |
| restore_guest_regs | |
| // That's it, no more messing around. | |
| eret | |
| __kvm_vcpu_return: | |
| // Assume x0 is the vcpu pointer, x1 the return code | |
| // Guest's x0-x3 are on the stack | |
| // Guest context | |
| add x2, x0, #VCPU_CONTEXT | |
| save_guest_regs | |
| bl __save_fpsimd | |
| bl __save_sysregs | |
| save_guest_32bit_state | |
| save_timer_state | |
| save_vgic_state | |
| deactivate_traps | |
| deactivate_vm | |
| // Host context | |
| ldr x2, [x0, #VCPU_HOST_CONTEXT] | |
| kern_hyp_va x2 | |
| bl __restore_sysregs | |
| bl __restore_fpsimd | |
| restore_host_regs | |
| mov x0, x1 | |
| ret | |
| END(__kvm_vcpu_run) | |
| // void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa); | |
| ENTRY(__kvm_tlb_flush_vmid_ipa) | |
| dsb ishst | |
| kern_hyp_va x0 | |
| ldr x2, [x0, #KVM_VTTBR] | |
| msr vttbr_el2, x2 | |
| isb | |
| /* | |
| * We could do so much better if we had the VA as well. | |
| * Instead, we invalidate Stage-2 for this IPA, and the | |
| * whole of Stage-1. Weep... | |
| */ | |
| tlbi ipas2e1is, x1 | |
| dsb sy | |
| tlbi vmalle1is | |
| dsb sy | |
| isb | |
| msr vttbr_el2, xzr | |
| ret | |
| ENDPROC(__kvm_tlb_flush_vmid_ipa) | |
| ENTRY(__kvm_flush_vm_context) | |
| dsb ishst | |
| tlbi alle1is | |
| ic ialluis | |
| dsb sy | |
| ret | |
| ENDPROC(__kvm_flush_vm_context) | |
| __kvm_hyp_panic: | |
| // Guess the context by looking at VTTBR: | |
| // If zero, then we're already a host. | |
| // Otherwise restore a minimal host context before panicing. | |
| mrs x0, vttbr_el2 | |
| cbz x0, 1f | |
| mrs x0, tpidr_el2 | |
| deactivate_traps | |
| deactivate_vm | |
| ldr x2, [x0, #VCPU_HOST_CONTEXT] | |
| kern_hyp_va x2 | |
| bl __restore_sysregs | |
| 1: adr x0, __hyp_panic_str | |
| adr x1, 2f | |
| ldp x2, x3, [x1] | |
| sub x0, x0, x2 | |
| add x0, x0, x3 | |
| mrs x1, spsr_el2 | |
| mrs x2, elr_el2 | |
| mrs x3, esr_el2 | |
| mrs x4, far_el2 | |
| mrs x5, hpfar_el2 | |
| mrs x6, par_el1 | |
| mrs x7, tpidr_el2 | |
| mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ | |
| PSR_MODE_EL1h) | |
| msr spsr_el2, lr | |
| ldr lr, =panic | |
| msr elr_el2, lr | |
| eret | |
| .align 3 | |
| 2: .quad HYP_PAGE_OFFSET | |
| .quad PAGE_OFFSET | |
| ENDPROC(__kvm_hyp_panic) | |
| __hyp_panic_str: | |
| .ascii "HYP panic:\nPS:%08x PC:%p ESR:%p\nFAR:%p HPFAR:%p PAR:%p\nVCPU:%p\n\0" | |
| .align 2 | |
| ENTRY(kvm_call_hyp) | |
| hvc #0 | |
| ret | |
| ENDPROC(kvm_call_hyp) | |
| .macro invalid_vector label, target | |
| .align 2 | |
| \label: | |
| b \target | |
| ENDPROC(\label) | |
| .endm | |
| /* None of these should ever happen */ | |
| invalid_vector el2t_sync_invalid, __kvm_hyp_panic | |
| invalid_vector el2t_irq_invalid, __kvm_hyp_panic | |
| invalid_vector el2t_fiq_invalid, __kvm_hyp_panic | |
| invalid_vector el2t_error_invalid, __kvm_hyp_panic | |
| invalid_vector el2h_sync_invalid, __kvm_hyp_panic | |
| invalid_vector el2h_irq_invalid, __kvm_hyp_panic | |
| invalid_vector el2h_fiq_invalid, __kvm_hyp_panic | |
| invalid_vector el2h_error_invalid, __kvm_hyp_panic | |
| invalid_vector el1_sync_invalid, __kvm_hyp_panic | |
| invalid_vector el1_irq_invalid, __kvm_hyp_panic | |
| invalid_vector el1_fiq_invalid, __kvm_hyp_panic | |
| invalid_vector el1_error_invalid, __kvm_hyp_panic | |
| el1_sync: // Guest trapped into EL2 | |
| push x0, x1 | |
| push x2, x3 | |
| mrs x1, esr_el2 | |
| lsr x2, x1, #ESR_EL2_EC_SHIFT | |
| cmp x2, #ESR_EL2_EC_HVC64 | |
| b.ne el1_trap | |
| mrs x3, vttbr_el2 // If vttbr is valid, the 64bit guest | |
| cbnz x3, el1_trap // called HVC | |
| /* Here, we're pretty sure the host called HVC. */ | |
| pop x2, x3 | |
| pop x0, x1 | |
| push lr, xzr | |
| /* | |
| * Compute the function address in EL2, and shuffle the parameters. | |
| */ | |
| kern_hyp_va x0 | |
| mov lr, x0 | |
| mov x0, x1 | |
| mov x1, x2 | |
| mov x2, x3 | |
| blr lr | |
| pop lr, xzr | |
| eret | |
| el1_trap: | |
| /* | |
| * x1: ESR | |
| * x2: ESR_EC | |
| */ | |
| cmp x2, #ESR_EL2_EC_DABT | |
| mov x0, #ESR_EL2_EC_IABT | |
| ccmp x2, x0, #4, ne | |
| b.ne 1f // Not an abort we care about | |
| /* This is an abort. Check for permission fault */ | |
| and x2, x1, #ESR_EL2_FSC_TYPE | |
| cmp x2, #FSC_PERM | |
| b.ne 1f // Not a permission fault | |
| /* | |
| * Check for Stage-1 page table walk, which is guaranteed | |
| * to give a valid HPFAR_EL2. | |
| */ | |
| tbnz x1, #7, 1f // S1PTW is set | |
| /* Preserve PAR_EL1 */ | |
| mrs x3, par_el1 | |
| push x3, xzr | |
| /* | |
| * Permission fault, HPFAR_EL2 is invalid. | |
| * Resolve the IPA the hard way using the guest VA. | |
| * Stage-1 translation already validated the memory access rights. | |
| * As such, we can use the EL1 translation regime, and don't have | |
| * to distinguish between EL0 and EL1 access. | |
| */ | |
| mrs x2, far_el2 | |
| at s1e1r, x2 | |
| isb | |
| /* Read result */ | |
| mrs x3, par_el1 | |
| pop x0, xzr // Restore PAR_EL1 from the stack | |
| msr par_el1, x0 | |
| tbnz x3, #0, 3f // Bail out if we failed the translation | |
| ubfx x3, x3, #12, #36 // Extract IPA | |
| lsl x3, x3, #4 // and present it like HPFAR | |
| b 2f | |
| 1: mrs x3, hpfar_el2 | |
| mrs x2, far_el2 | |
| 2: mrs x0, tpidr_el2 | |
| str x1, [x0, #VCPU_ESR_EL2] | |
| str x2, [x0, #VCPU_FAR_EL2] | |
| str x3, [x0, #VCPU_HPFAR_EL2] | |
| mov x1, #ARM_EXCEPTION_TRAP | |
| b __kvm_vcpu_return | |
| /* | |
| * Translation failed. Just return to the guest and | |
| * let it fault again. Another CPU is probably playing | |
| * behind our back. | |
| */ | |
| 3: pop x2, x3 | |
| pop x0, x1 | |
| eret | |
| el1_irq: | |
| push x0, x1 | |
| push x2, x3 | |
| mrs x0, tpidr_el2 | |
| mov x1, #ARM_EXCEPTION_IRQ | |
| b __kvm_vcpu_return | |
| .ltorg | |
| .align 11 | |
| ENTRY(__kvm_hyp_vector) | |
| ventry el2t_sync_invalid // Synchronous EL2t | |
| ventry el2t_irq_invalid // IRQ EL2t | |
| ventry el2t_fiq_invalid // FIQ EL2t | |
| ventry el2t_error_invalid // Error EL2t | |
| ventry el2h_sync_invalid // Synchronous EL2h | |
| ventry el2h_irq_invalid // IRQ EL2h | |
| ventry el2h_fiq_invalid // FIQ EL2h | |
| ventry el2h_error_invalid // Error EL2h | |
| ventry el1_sync // Synchronous 64-bit EL1 | |
| ventry el1_irq // IRQ 64-bit EL1 | |
| ventry el1_fiq_invalid // FIQ 64-bit EL1 | |
| ventry el1_error_invalid // Error 64-bit EL1 | |
| ventry el1_sync // Synchronous 32-bit EL1 | |
| ventry el1_irq // IRQ 32-bit EL1 | |
| ventry el1_fiq_invalid // FIQ 32-bit EL1 | |
| ventry el1_error_invalid // Error 32-bit EL1 | |
| ENDPROC(__kvm_hyp_vector) | |
| __kvm_hyp_code_end: | |
| .globl __kvm_hyp_code_end | |
| .popsection |