| #include <asm/asm-offsets.h> |
| #include <asm/bug.h> |
| #ifdef CONFIG_PPC_BOOK3S |
| #include <asm/exception-64s.h> |
| #else |
| #include <asm/exception-64e.h> |
| #endif |
| #include <asm/feature-fixups.h> |
| #include <asm/head-64.h> |
| #include <asm/hw_irq.h> |
| #include <asm/kup.h> |
| #include <asm/mmu.h> |
| #include <asm/ppc_asm.h> |
| #include <asm/ptrace.h> |
| |
| .align 7 |
| |
| .macro DEBUG_SRR_VALID srr |
| #ifdef CONFIG_PPC_RFI_SRR_DEBUG |
| .ifc \srr,srr |
| mfspr r11,SPRN_SRR0 |
| ld r12,_NIP(r1) |
| clrrdi r11,r11,2 |
| clrrdi r12,r12,2 |
| 100: tdne r11,r12 |
| EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE) |
| mfspr r11,SPRN_SRR1 |
| ld r12,_MSR(r1) |
| 100: tdne r11,r12 |
| EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE) |
| .else |
| mfspr r11,SPRN_HSRR0 |
| ld r12,_NIP(r1) |
| clrrdi r11,r11,2 |
| clrrdi r12,r12,2 |
| 100: tdne r11,r12 |
| EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE) |
| mfspr r11,SPRN_HSRR1 |
| ld r12,_MSR(r1) |
| 100: tdne r11,r12 |
| EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE) |
| .endif |
| #endif |
| .endm |
| |
| #ifdef CONFIG_PPC_BOOK3S |
| .macro system_call_vectored name trapnr |
| .globl system_call_vectored_\name |
| system_call_vectored_\name: |
| _ASM_NOKPROBE_SYMBOL(system_call_vectored_\name) |
| SCV_INTERRUPT_TO_KERNEL |
| mr r10,r1 |
| ld r1,PACAKSAVE(r13) |
| std r10,0(r1) |
| std r11,_LINK(r1) |
| std r11,_NIP(r1) /* Saved LR is also the next instruction */ |
| std r12,_MSR(r1) |
| std r0,GPR0(r1) |
| std r10,GPR1(r1) |
| std r2,GPR2(r1) |
| LOAD_PACA_TOC() |
| mfcr r12 |
| li r11,0 |
| /* Save syscall parameters in r3-r8 */ |
| SAVE_GPRS(3, 8, r1) |
| /* Zero r9-r12, this should only be required when restoring all GPRs */ |
| std r11,GPR9(r1) |
| std r11,GPR10(r1) |
| std r11,GPR11(r1) |
| std r11,GPR12(r1) |
| std r9,GPR13(r1) |
| SAVE_NVGPRS(r1) |
| std r11,_XER(r1) |
| std r11,_CTR(r1) |
| |
| li r11,\trapnr |
| std r11,_TRAP(r1) |
| std r12,_CCR(r1) |
| std r3,ORIG_GPR3(r1) |
| LOAD_REG_IMMEDIATE(r11, STACK_FRAME_REGS_MARKER) |
| std r11,STACK_INT_FRAME_MARKER(r1) /* "regs" marker */ |
| /* Calling convention has r3 = regs, r4 = orig r0 */ |
| addi r3,r1,STACK_INT_FRAME_REGS |
| mr r4,r0 |
| |
| BEGIN_FTR_SECTION |
| HMT_MEDIUM |
| END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) |
| |
| /* |
| * scv enters with MSR[EE]=1 and is immediately considered soft-masked. |
| * The entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED, |
| * and interrupts may be masked and pending already. |
| * system_call_exception() will call trace_hardirqs_off() which means |
| * interrupts could already have been blocked before trace_hardirqs_off, |
| * but this is the best we can do. |
| */ |
| |
| /* |
| * Zero user registers to prevent influencing speculative execution |
| * state of kernel code. |
| */ |
| SANITIZE_SYSCALL_GPRS() |
| bl CFUNC(system_call_exception) |
| |
| .Lsyscall_vectored_\name\()_exit: |
| addi r4,r1,STACK_INT_FRAME_REGS |
| li r5,1 /* scv */ |
| bl CFUNC(syscall_exit_prepare) |
| std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */ |
| .Lsyscall_vectored_\name\()_rst_start: |
| lbz r11,PACAIRQHAPPENED(r13) |
| andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l |
| bne- syscall_vectored_\name\()_restart |
| li r11,IRQS_ENABLED |
| stb r11,PACAIRQSOFTMASK(r13) |
| li r11,0 |
| stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS |
| |
| ld r2,_CCR(r1) |
| ld r4,_NIP(r1) |
| ld r5,_MSR(r1) |
| |
| BEGIN_FTR_SECTION |
| stdcx. r0,0,r1 /* to clear the reservation */ |
| END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) |
| |
| BEGIN_FTR_SECTION |
| HMT_MEDIUM_LOW |
| END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) |
| |
| SANITIZE_RESTORE_NVGPRS() |
| cmpdi r3,0 |
| bne .Lsyscall_vectored_\name\()_restore_regs |
| |
| /* rfscv returns with LR->NIA and CTR->MSR */ |
| mtlr r4 |
| mtctr r5 |
| |
| /* Could zero these as per ABI, but we may consider a stricter ABI |
| * which preserves these if libc implementations can benefit, so |
| * restore them for now until further measurement is done. */ |
| REST_GPR(0, r1) |
| REST_GPRS(4, 8, r1) |
| /* Zero volatile regs that may contain sensitive kernel data */ |
| ZEROIZE_GPRS(9, 12) |
| mtspr SPRN_XER,r0 |
| |
| /* |
| * We don't need to restore AMR on the way back to userspace for KUAP. |
| * The value of AMR only matters while we're in the kernel. |
| */ |
| mtcr r2 |
| REST_GPRS(2, 3, r1) |
| REST_GPR(13, r1) |
| REST_GPR(1, r1) |
| RFSCV_TO_USER |
| b . /* prevent speculative execution */ |
| |
| .Lsyscall_vectored_\name\()_restore_regs: |
| mtspr SPRN_SRR0,r4 |
| mtspr SPRN_SRR1,r5 |
| |
| ld r3,_CTR(r1) |
| ld r4,_LINK(r1) |
| ld r5,_XER(r1) |
| |
| HANDLER_RESTORE_NVGPRS() |
| REST_GPR(0, r1) |
| mtcr r2 |
| mtctr r3 |
| mtlr r4 |
| mtspr SPRN_XER,r5 |
| REST_GPRS(2, 13, r1) |
| REST_GPR(1, r1) |
| RFI_TO_USER |
| .Lsyscall_vectored_\name\()_rst_end: |
| |
| syscall_vectored_\name\()_restart: |
| _ASM_NOKPROBE_SYMBOL(syscall_vectored_\name\()_restart) |
| GET_PACA(r13) |
| ld r1,PACA_EXIT_SAVE_R1(r13) |
| LOAD_PACA_TOC() |
| ld r3,RESULT(r1) |
| addi r4,r1,STACK_INT_FRAME_REGS |
| li r11,IRQS_ALL_DISABLED |
| stb r11,PACAIRQSOFTMASK(r13) |
| bl CFUNC(syscall_exit_restart) |
| std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */ |
| b .Lsyscall_vectored_\name\()_rst_start |
| 1: |
| |
| SOFT_MASK_TABLE(.Lsyscall_vectored_\name\()_rst_start, 1b) |
| RESTART_TABLE(.Lsyscall_vectored_\name\()_rst_start, .Lsyscall_vectored_\name\()_rst_end, syscall_vectored_\name\()_restart) |
| |
| .endm |
| |
| system_call_vectored common 0x3000 |
| |
| /* |
| * We instantiate another entry copy for the SIGILL variant, with TRAP=0x7ff0 |
| * which is tested by system_call_exception when r0 is -1 (as set by vector |
| * entry code). |
| */ |
| system_call_vectored sigill 0x7ff0 |
| |
| #endif /* CONFIG_PPC_BOOK3S */ |
| |
| .balign IFETCH_ALIGN_BYTES |
| .globl system_call_common_real |
| system_call_common_real: |
| _ASM_NOKPROBE_SYMBOL(system_call_common_real) |
| ld r10,PACAKMSR(r13) /* get MSR value for kernel */ |
| mtmsrd r10 |
| |
| .balign IFETCH_ALIGN_BYTES |
| .globl system_call_common |
| system_call_common: |
| _ASM_NOKPROBE_SYMBOL(system_call_common) |
| mr r10,r1 |
| ld r1,PACAKSAVE(r13) |
| std r10,0(r1) |
| std r11,_NIP(r1) |
| std r12,_MSR(r1) |
| std r0,GPR0(r1) |
| std r10,GPR1(r1) |
| std r2,GPR2(r1) |
| #ifdef CONFIG_PPC_E500 |
| START_BTB_FLUSH_SECTION |
| BTB_FLUSH(r10) |
| END_BTB_FLUSH_SECTION |
| #endif |
| LOAD_PACA_TOC() |
| mfcr r12 |
| li r11,0 |
| /* Save syscall parameters in r3-r8 */ |
| SAVE_GPRS(3, 8, r1) |
| /* Zero r9-r12, this should only be required when restoring all GPRs */ |
| std r11,GPR9(r1) |
| std r11,GPR10(r1) |
| std r11,GPR11(r1) |
| std r11,GPR12(r1) |
| std r9,GPR13(r1) |
| SAVE_NVGPRS(r1) |
| std r11,_XER(r1) |
| std r11,_CTR(r1) |
| mflr r10 |
| |
| /* |
| * This clears CR0.SO (bit 28), which is the error indication on |
| * return from this system call. |
| */ |
| rldimi r12,r11,28,(63-28) |
| li r11,0xc00 |
| std r10,_LINK(r1) |
| std r11,_TRAP(r1) |
| std r12,_CCR(r1) |
| std r3,ORIG_GPR3(r1) |
| LOAD_REG_IMMEDIATE(r11, STACK_FRAME_REGS_MARKER) |
| std r11,STACK_INT_FRAME_MARKER(r1) /* "regs" marker */ |
| /* Calling convention has r3 = regs, r4 = orig r0 */ |
| addi r3,r1,STACK_INT_FRAME_REGS |
| mr r4,r0 |
| |
| #ifdef CONFIG_PPC_BOOK3S |
| li r11,1 |
| stb r11,PACASRR_VALID(r13) |
| #endif |
| |
| /* |
| * We always enter kernel from userspace with irq soft-mask enabled and |
| * nothing pending. system_call_exception() will call |
| * trace_hardirqs_off(). |
| */ |
| li r11,IRQS_ALL_DISABLED |
| stb r11,PACAIRQSOFTMASK(r13) |
| #ifdef CONFIG_PPC_BOOK3S |
| li r12,-1 /* Set MSR_EE and MSR_RI */ |
| mtmsrd r12,1 |
| #else |
| wrteei 1 |
| #endif |
| |
| /* |
| * Zero user registers to prevent influencing speculative execution |
| * state of kernel code. |
| */ |
| SANITIZE_SYSCALL_GPRS() |
| bl CFUNC(system_call_exception) |
| |
| .Lsyscall_exit: |
| addi r4,r1,STACK_INT_FRAME_REGS |
| li r5,0 /* !scv */ |
| bl CFUNC(syscall_exit_prepare) |
| std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */ |
| #ifdef CONFIG_PPC_BOOK3S |
| .Lsyscall_rst_start: |
| lbz r11,PACAIRQHAPPENED(r13) |
| andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l |
| bne- syscall_restart |
| #endif |
| li r11,IRQS_ENABLED |
| stb r11,PACAIRQSOFTMASK(r13) |
| li r11,0 |
| stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS |
| |
| ld r2,_CCR(r1) |
| ld r6,_LINK(r1) |
| mtlr r6 |
| |
| #ifdef CONFIG_PPC_BOOK3S |
| lbz r4,PACASRR_VALID(r13) |
| cmpdi r4,0 |
| bne 1f |
| li r4,0 |
| stb r4,PACASRR_VALID(r13) |
| #endif |
| ld r4,_NIP(r1) |
| ld r5,_MSR(r1) |
| mtspr SPRN_SRR0,r4 |
| mtspr SPRN_SRR1,r5 |
| 1: |
| DEBUG_SRR_VALID srr |
| |
| BEGIN_FTR_SECTION |
| stdcx. r0,0,r1 /* to clear the reservation */ |
| END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) |
| |
| SANITIZE_RESTORE_NVGPRS() |
| cmpdi r3,0 |
| bne .Lsyscall_restore_regs |
| /* Zero volatile regs that may contain sensitive kernel data */ |
| ZEROIZE_GPR(0) |
| ZEROIZE_GPRS(4, 12) |
| mtctr r0 |
| mtspr SPRN_XER,r0 |
| .Lsyscall_restore_regs_cont: |
| |
| BEGIN_FTR_SECTION |
| HMT_MEDIUM_LOW |
| END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) |
| |
| /* |
| * We don't need to restore AMR on the way back to userspace for KUAP. |
| * The value of AMR only matters while we're in the kernel. |
| */ |
| mtcr r2 |
| REST_GPRS(2, 3, r1) |
| REST_GPR(13, r1) |
| REST_GPR(1, r1) |
| RFI_TO_USER |
| b . /* prevent speculative execution */ |
| |
| .Lsyscall_restore_regs: |
| ld r3,_CTR(r1) |
| ld r4,_XER(r1) |
| HANDLER_RESTORE_NVGPRS() |
| mtctr r3 |
| mtspr SPRN_XER,r4 |
| REST_GPR(0, r1) |
| REST_GPRS(4, 12, r1) |
| b .Lsyscall_restore_regs_cont |
| .Lsyscall_rst_end: |
| |
| #ifdef CONFIG_PPC_BOOK3S |
| syscall_restart: |
| _ASM_NOKPROBE_SYMBOL(syscall_restart) |
| GET_PACA(r13) |
| ld r1,PACA_EXIT_SAVE_R1(r13) |
| LOAD_PACA_TOC() |
| ld r3,RESULT(r1) |
| addi r4,r1,STACK_INT_FRAME_REGS |
| li r11,IRQS_ALL_DISABLED |
| stb r11,PACAIRQSOFTMASK(r13) |
| bl CFUNC(syscall_exit_restart) |
| std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */ |
| b .Lsyscall_rst_start |
| 1: |
| |
| SOFT_MASK_TABLE(.Lsyscall_rst_start, 1b) |
| RESTART_TABLE(.Lsyscall_rst_start, .Lsyscall_rst_end, syscall_restart) |
| #endif |
| |
| /* |
| * If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not |
| * touched, no exit work created, then this can be used. |
| */ |
| .balign IFETCH_ALIGN_BYTES |
| .globl fast_interrupt_return_srr |
| fast_interrupt_return_srr: |
| _ASM_NOKPROBE_SYMBOL(fast_interrupt_return_srr) |
| kuap_check_amr r3, r4 |
| ld r5,_MSR(r1) |
| andi. r0,r5,MSR_PR |
| #ifdef CONFIG_PPC_BOOK3S |
| beq 1f |
| kuap_user_restore r3, r4 |
| b .Lfast_user_interrupt_return_srr |
| 1: kuap_kernel_restore r3, r4 |
| andi. r0,r5,MSR_RI |
| li r3,0 /* 0 return value, no EMULATE_STACK_STORE */ |
| bne+ .Lfast_kernel_interrupt_return_srr |
| addi r3,r1,STACK_INT_FRAME_REGS |
| bl CFUNC(unrecoverable_exception) |
| b . /* should not get here */ |
| #else |
| bne .Lfast_user_interrupt_return_srr |
| b .Lfast_kernel_interrupt_return_srr |
| #endif |
| |
| .macro interrupt_return_macro srr |
| .balign IFETCH_ALIGN_BYTES |
| .globl interrupt_return_\srr |
| interrupt_return_\srr\(): |
| _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()) |
| ld r4,_MSR(r1) |
| andi. r0,r4,MSR_PR |
| beq interrupt_return_\srr\()_kernel |
| interrupt_return_\srr\()_user: /* make backtraces match the _kernel variant */ |
| _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user) |
| addi r3,r1,STACK_INT_FRAME_REGS |
| bl CFUNC(interrupt_exit_user_prepare) |
| #ifndef CONFIG_INTERRUPT_SANITIZE_REGISTERS |
| cmpdi r3,0 |
| bne- .Lrestore_nvgprs_\srr |
| .Lrestore_nvgprs_\srr\()_cont: |
| #endif |
| std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */ |
| #ifdef CONFIG_PPC_BOOK3S |
| .Linterrupt_return_\srr\()_user_rst_start: |
| lbz r11,PACAIRQHAPPENED(r13) |
| andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l |
| bne- interrupt_return_\srr\()_user_restart |
| #endif |
| li r11,IRQS_ENABLED |
| stb r11,PACAIRQSOFTMASK(r13) |
| li r11,0 |
| stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS |
| |
| .Lfast_user_interrupt_return_\srr\(): |
| SANITIZE_RESTORE_NVGPRS() |
| #ifdef CONFIG_PPC_BOOK3S |
| .ifc \srr,srr |
| lbz r4,PACASRR_VALID(r13) |
| .else |
| lbz r4,PACAHSRR_VALID(r13) |
| .endif |
| cmpdi r4,0 |
| li r4,0 |
| bne 1f |
| #endif |
| ld r11,_NIP(r1) |
| ld r12,_MSR(r1) |
| .ifc \srr,srr |
| mtspr SPRN_SRR0,r11 |
| mtspr SPRN_SRR1,r12 |
| 1: |
| #ifdef CONFIG_PPC_BOOK3S |
| stb r4,PACASRR_VALID(r13) |
| #endif |
| .else |
| mtspr SPRN_HSRR0,r11 |
| mtspr SPRN_HSRR1,r12 |
| 1: |
| #ifdef CONFIG_PPC_BOOK3S |
| stb r4,PACAHSRR_VALID(r13) |
| #endif |
| .endif |
| DEBUG_SRR_VALID \srr |
| |
| #ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG |
| lbz r4,PACAIRQSOFTMASK(r13) |
| tdnei r4,IRQS_ENABLED |
| #endif |
| |
| BEGIN_FTR_SECTION |
| ld r10,_PPR(r1) |
| mtspr SPRN_PPR,r10 |
| END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) |
| |
| BEGIN_FTR_SECTION |
| stdcx. r0,0,r1 /* to clear the reservation */ |
| FTR_SECTION_ELSE |
| ldarx r0,0,r1 |
| ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) |
| |
| ld r3,_CCR(r1) |
| ld r4,_LINK(r1) |
| ld r5,_CTR(r1) |
| ld r6,_XER(r1) |
| li r0,0 |
| |
| REST_GPRS(7, 13, r1) |
| |
| mtcr r3 |
| mtlr r4 |
| mtctr r5 |
| mtspr SPRN_XER,r6 |
| |
| REST_GPRS(2, 6, r1) |
| REST_GPR(0, r1) |
| REST_GPR(1, r1) |
| .ifc \srr,srr |
| RFI_TO_USER |
| .else |
| HRFI_TO_USER |
| .endif |
| b . /* prevent speculative execution */ |
| .Linterrupt_return_\srr\()_user_rst_end: |
| |
| #ifndef CONFIG_INTERRUPT_SANITIZE_REGISTERS |
| .Lrestore_nvgprs_\srr\(): |
| REST_NVGPRS(r1) |
| b .Lrestore_nvgprs_\srr\()_cont |
| #endif |
| |
| #ifdef CONFIG_PPC_BOOK3S |
| interrupt_return_\srr\()_user_restart: |
| _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user_restart) |
| GET_PACA(r13) |
| ld r1,PACA_EXIT_SAVE_R1(r13) |
| LOAD_PACA_TOC() |
| addi r3,r1,STACK_INT_FRAME_REGS |
| li r11,IRQS_ALL_DISABLED |
| stb r11,PACAIRQSOFTMASK(r13) |
| bl CFUNC(interrupt_exit_user_restart) |
| std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */ |
| b .Linterrupt_return_\srr\()_user_rst_start |
| 1: |
| |
| SOFT_MASK_TABLE(.Linterrupt_return_\srr\()_user_rst_start, 1b) |
| RESTART_TABLE(.Linterrupt_return_\srr\()_user_rst_start, .Linterrupt_return_\srr\()_user_rst_end, interrupt_return_\srr\()_user_restart) |
| #endif |
| |
| .balign IFETCH_ALIGN_BYTES |
| interrupt_return_\srr\()_kernel: |
| _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel) |
| addi r3,r1,STACK_INT_FRAME_REGS |
| bl CFUNC(interrupt_exit_kernel_prepare) |
| |
| std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */ |
| .Linterrupt_return_\srr\()_kernel_rst_start: |
| ld r11,SOFTE(r1) |
| cmpwi r11,IRQS_ENABLED |
| stb r11,PACAIRQSOFTMASK(r13) |
| beq .Linterrupt_return_\srr\()_soft_enabled |
| |
| /* |
| * Returning to soft-disabled context. |
| * Check if a MUST_HARD_MASK interrupt has become pending, in which |
| * case we need to disable MSR[EE] in the return context. |
| * |
| * The MSR[EE] check catches among other things the short incoherency |
| * in hard_irq_disable() between clearing MSR[EE] and setting |
| * PACA_IRQ_HARD_DIS. |
| */ |
| ld r12,_MSR(r1) |
| andi. r10,r12,MSR_EE |
| beq .Lfast_kernel_interrupt_return_\srr\() // EE already disabled |
| lbz r11,PACAIRQHAPPENED(r13) |
| andi. r10,r11,PACA_IRQ_MUST_HARD_MASK |
| bne 1f // HARD_MASK is pending |
| // No HARD_MASK pending, clear possible HARD_DIS set by interrupt |
| andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l |
| stb r11,PACAIRQHAPPENED(r13) |
| b .Lfast_kernel_interrupt_return_\srr\() |
| |
| |
| 1: /* Must clear MSR_EE from _MSR */ |
| #ifdef CONFIG_PPC_BOOK3S |
| li r10,0 |
| /* Clear valid before changing _MSR */ |
| .ifc \srr,srr |
| stb r10,PACASRR_VALID(r13) |
| .else |
| stb r10,PACAHSRR_VALID(r13) |
| .endif |
| #endif |
| xori r12,r12,MSR_EE |
| std r12,_MSR(r1) |
| b .Lfast_kernel_interrupt_return_\srr\() |
| |
| .Linterrupt_return_\srr\()_soft_enabled: |
| /* |
| * In the soft-enabled case, need to double-check that we have no |
| * pending interrupts that might have come in before we reached the |
| * restart section of code, and restart the exit so those can be |
| * handled. |
| * |
| * If there are none, it is be possible that the interrupt still |
| * has PACA_IRQ_HARD_DIS set, which needs to be cleared for the |
| * interrupted context. This clear will not clobber a new pending |
| * interrupt coming in, because we're in the restart section, so |
| * such would return to the restart location. |
| */ |
| #ifdef CONFIG_PPC_BOOK3S |
| lbz r11,PACAIRQHAPPENED(r13) |
| andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l |
| bne- interrupt_return_\srr\()_kernel_restart |
| #endif |
| li r11,0 |
| stb r11,PACAIRQHAPPENED(r13) // clear the possible HARD_DIS |
| |
| .Lfast_kernel_interrupt_return_\srr\(): |
| SANITIZE_RESTORE_NVGPRS() |
| cmpdi cr1,r3,0 |
| #ifdef CONFIG_PPC_BOOK3S |
| .ifc \srr,srr |
| lbz r4,PACASRR_VALID(r13) |
| .else |
| lbz r4,PACAHSRR_VALID(r13) |
| .endif |
| cmpdi r4,0 |
| li r4,0 |
| bne 1f |
| #endif |
| ld r11,_NIP(r1) |
| ld r12,_MSR(r1) |
| .ifc \srr,srr |
| mtspr SPRN_SRR0,r11 |
| mtspr SPRN_SRR1,r12 |
| 1: |
| #ifdef CONFIG_PPC_BOOK3S |
| stb r4,PACASRR_VALID(r13) |
| #endif |
| .else |
| mtspr SPRN_HSRR0,r11 |
| mtspr SPRN_HSRR1,r12 |
| 1: |
| #ifdef CONFIG_PPC_BOOK3S |
| stb r4,PACAHSRR_VALID(r13) |
| #endif |
| .endif |
| DEBUG_SRR_VALID \srr |
| |
| BEGIN_FTR_SECTION |
| stdcx. r0,0,r1 /* to clear the reservation */ |
| FTR_SECTION_ELSE |
| ldarx r0,0,r1 |
| ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) |
| |
| ld r3,_LINK(r1) |
| ld r4,_CTR(r1) |
| ld r5,_XER(r1) |
| ld r6,_CCR(r1) |
| li r0,0 |
| |
| REST_GPRS(7, 12, r1) |
| |
| mtlr r3 |
| mtctr r4 |
| mtspr SPRN_XER,r5 |
| |
| /* |
| * Leaving a stale STACK_FRAME_REGS_MARKER on the stack can confuse |
| * the reliable stack unwinder later on. Clear it. |
| */ |
| std r0,STACK_INT_FRAME_MARKER(r1) |
| |
| REST_GPRS(2, 5, r1) |
| |
| bne- cr1,1f /* emulate stack store */ |
| mtcr r6 |
| REST_GPR(6, r1) |
| REST_GPR(0, r1) |
| REST_GPR(1, r1) |
| .ifc \srr,srr |
| RFI_TO_KERNEL |
| .else |
| HRFI_TO_KERNEL |
| .endif |
| b . /* prevent speculative execution */ |
| |
| 1: /* |
| * Emulate stack store with update. New r1 value was already calculated |
| * and updated in our interrupt regs by emulate_loadstore, but we can't |
| * store the previous value of r1 to the stack before re-loading our |
| * registers from it, otherwise they could be clobbered. Use |
| * PACA_EXGEN as temporary storage to hold the store data, as |
| * interrupts are disabled here so it won't be clobbered. |
| */ |
| mtcr r6 |
| std r9,PACA_EXGEN+0(r13) |
| addi r9,r1,INT_FRAME_SIZE /* get original r1 */ |
| REST_GPR(6, r1) |
| REST_GPR(0, r1) |
| REST_GPR(1, r1) |
| std r9,0(r1) /* perform store component of stdu */ |
| ld r9,PACA_EXGEN+0(r13) |
| |
| .ifc \srr,srr |
| RFI_TO_KERNEL |
| .else |
| HRFI_TO_KERNEL |
| .endif |
| b . /* prevent speculative execution */ |
| .Linterrupt_return_\srr\()_kernel_rst_end: |
| |
| #ifdef CONFIG_PPC_BOOK3S |
| interrupt_return_\srr\()_kernel_restart: |
| _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel_restart) |
| GET_PACA(r13) |
| ld r1,PACA_EXIT_SAVE_R1(r13) |
| LOAD_PACA_TOC() |
| addi r3,r1,STACK_INT_FRAME_REGS |
| li r11,IRQS_ALL_DISABLED |
| stb r11,PACAIRQSOFTMASK(r13) |
| bl CFUNC(interrupt_exit_kernel_restart) |
| std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */ |
| b .Linterrupt_return_\srr\()_kernel_rst_start |
| 1: |
| |
| SOFT_MASK_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, 1b) |
| RESTART_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, .Linterrupt_return_\srr\()_kernel_rst_end, interrupt_return_\srr\()_kernel_restart) |
| #endif |
| |
| .endm |
| |
| interrupt_return_macro srr |
| #ifdef CONFIG_PPC_BOOK3S |
| interrupt_return_macro hsrr |
| |
| .globl __end_soft_masked |
| __end_soft_masked: |
| DEFINE_FIXED_SYMBOL(__end_soft_masked, text) |
| #endif /* CONFIG_PPC_BOOK3S */ |
| |
| #ifdef CONFIG_PPC_BOOK3S |
| _GLOBAL(ret_from_fork_scv) |
| bl CFUNC(schedule_tail) |
| HANDLER_RESTORE_NVGPRS() |
| li r3,0 /* fork() return value */ |
| b .Lsyscall_vectored_common_exit |
| #endif |
| |
| _GLOBAL(ret_from_fork) |
| bl CFUNC(schedule_tail) |
| HANDLER_RESTORE_NVGPRS() |
| li r3,0 /* fork() return value */ |
| b .Lsyscall_exit |
| |
| _GLOBAL(ret_from_kernel_user_thread) |
| bl CFUNC(schedule_tail) |
| mtctr r14 |
| mr r3,r15 |
| #ifdef CONFIG_PPC64_ELF_ABI_V2 |
| mr r12,r14 |
| #endif |
| bctrl |
| li r3,0 |
| /* |
| * It does not matter whether this returns via the scv or sc path |
| * because it returns as execve() and therefore has no calling ABI |
| * (i.e., it sets registers according to the exec()ed entry point). |
| */ |
| b .Lsyscall_exit |
| |
| _GLOBAL(start_kernel_thread) |
| bl CFUNC(schedule_tail) |
| mtctr r14 |
| mr r3,r15 |
| #ifdef CONFIG_PPC64_ELF_ABI_V2 |
| mr r12,r14 |
| #endif |
| bctrl |
| /* |
| * This must not return. We actually want to BUG here, not WARN, |
| * because BUG will exit the process which is what the kernel thread |
| * should have done, which may give some hope of continuing. |
| */ |
| 100: trap |
| EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0 |
