1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2012,2013 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7 #include <linux/arm-smccc.h>
8 #include <linux/cfi_types.h>
9 #include <linux/linkage.h>
10
11 #include <asm/alternative.h>
12 #include <asm/assembler.h>
13 #include <asm/el2_setup.h>
14 #include <asm/kvm_arm.h>
15 #include <asm/kvm_asm.h>
16 #include <asm/kvm_mmu.h>
17 #include <asm/pgtable-hwdef.h>
18 #include <asm/sysreg.h>
19 #include <asm/virt.h>
20
21 .text
22 .pushsection .idmap.text, "ax"
23
24 .align 11
25
26 SYM_CODE_START(__kvm_hyp_init)
27 ventry . // Synchronous EL2t
28 ventry . // IRQ EL2t
29 ventry . // FIQ EL2t
30 ventry . // Error EL2t
31
32 ventry . // Synchronous EL2h
33 ventry . // IRQ EL2h
34 ventry . // FIQ EL2h
35 ventry . // Error EL2h
36
37 ventry __do_hyp_init // Synchronous 64-bit EL1
38 ventry . // IRQ 64-bit EL1
39 ventry . // FIQ 64-bit EL1
40 ventry . // Error 64-bit EL1
41
42 ventry . // Synchronous 32-bit EL1
43 ventry . // IRQ 32-bit EL1
44 ventry . // FIQ 32-bit EL1
45 ventry . // Error 32-bit EL1
46
47 /*
48 * Only uses x0..x3 so as to not clobber callee-saved SMCCC registers.
49 *
50 * x0: SMCCC function ID
51 * x1: struct kvm_nvhe_init_params PA
52 */
53 __do_hyp_init:
54 /* Check for a stub HVC call */
55 cmp x0, #HVC_STUB_HCALL_NR
56 b.lo __kvm_handle_stub_hvc
57
58 bic x0, x0, #ARM_SMCCC_CALL_HINTS
59 mov x3, #KVM_HOST_SMCCC_FUNC(__kvm_hyp_init)
60 cmp x0, x3
61 b.eq 1f
62
63 mov x0, #SMCCC_RET_NOT_SUPPORTED
64 eret
65
66 1: mov x0, x1
67 mov x3, lr
68 bl ___kvm_hyp_init // Clobbers x0..x2
69 mov lr, x3
70
71 /* Hello, World! */
72 mov x0, #SMCCC_RET_SUCCESS
73 eret
74 SYM_CODE_END(__kvm_hyp_init)
75
76 SYM_CODE_START_LOCAL(__kvm_init_el2_state)
77 /* Initialize EL2 CPU state to sane values. */
78 init_el2_state // Clobbers x0..x2
79 finalise_el2_state
80 ret
81 SYM_CODE_END(__kvm_init_el2_state)
82
83 /*
84 * Initialize the hypervisor in EL2.
85 *
86 * Only uses x0..x2 so as to not clobber callee-saved SMCCC registers
87 * and leave x3 for the caller.
88 *
89 * x0: struct kvm_nvhe_init_params PA
90 */
91 SYM_CODE_START_LOCAL(___kvm_hyp_init)
92 ldr x1, [x0, #NVHE_INIT_STACK_HYP_VA]
93 mov sp, x1
94
95 ldr x1, [x0, #NVHE_INIT_MAIR_EL2]
96 msr mair_el2, x1
97
98 ldr x1, [x0, #NVHE_INIT_HCR_EL2]
99 msr hcr_el2, x1
100
101 mov x2, #HCR_E2H
102 and x2, x1, x2
103 cbz x2, 1f
104
105 // hVHE: Replay the EL2 setup to account for the E2H bit
106 // TPIDR_EL2 is used to preserve x0 across the macro maze...
107 isb
108 msr tpidr_el2, x0
109 str lr, [x0, #NVHE_INIT_TMP]
110
111 bl __kvm_init_el2_state
112
113 mrs x0, tpidr_el2
114 ldr lr, [x0, #NVHE_INIT_TMP]
115
116 1:
117 ldr x1, [x0, #NVHE_INIT_TPIDR_EL2]
118 msr tpidr_el2, x1
119
120 ldr x1, [x0, #NVHE_INIT_VTTBR]
121 msr vttbr_el2, x1
122
123 ldr x1, [x0, #NVHE_INIT_VTCR]
124 msr vtcr_el2, x1
125
126 ldr x1, [x0, #NVHE_INIT_PGD_PA]
127 phys_to_ttbr x2, x1
128 alternative_if ARM64_HAS_CNP
129 orr x2, x2, #TTBR_CNP_BIT
130 alternative_else_nop_endif
131 msr ttbr0_el2, x2
132
133 ldr x0, [x0, #NVHE_INIT_TCR_EL2]
134 msr tcr_el2, x0
135
136 isb
137
138 /* Invalidate the stale TLBs from Bootloader */
139 tlbi alle2
140 tlbi alle1
141 dsb sy
142
143 mov_q x0, INIT_SCTLR_EL2_MMU_ON
144 alternative_if ARM64_HAS_ADDRESS_AUTH
145 mov_q x1, (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | \
146 SCTLR_ELx_ENDA | SCTLR_ELx_ENDB)
147 orr x0, x0, x1
148 alternative_else_nop_endif
149
150 #ifdef CONFIG_ARM64_BTI_KERNEL
151 alternative_if ARM64_BTI
152 orr x0, x0, #SCTLR_EL2_BT
153 alternative_else_nop_endif
154 #endif /* CONFIG_ARM64_BTI_KERNEL */
155
156 msr sctlr_el2, x0
157 isb
158
159 /* Set the host vector */
160 ldr x0, =__kvm_hyp_host_vector
161 msr vbar_el2, x0
162
163 ret
164 SYM_CODE_END(___kvm_hyp_init)
165
166 /*
167 * PSCI CPU_ON entry point
168 *
169 * x0: struct kvm_nvhe_init_params PA
170 */
171 SYM_CODE_START(kvm_hyp_cpu_entry)
172 mov x1, #1 // is_cpu_on = true
173 b __kvm_hyp_init_cpu
174 SYM_CODE_END(kvm_hyp_cpu_entry)
175
176 /*
177 * PSCI CPU_SUSPEND / SYSTEM_SUSPEND entry point
178 *
179 * x0: struct kvm_nvhe_init_params PA
180 */
181 SYM_CODE_START(kvm_hyp_cpu_resume)
182 mov x1, #0 // is_cpu_on = false
183 b __kvm_hyp_init_cpu
184 SYM_CODE_END(kvm_hyp_cpu_resume)
185
186 /*
187 * Common code for CPU entry points. Initializes EL2 state and
188 * installs the hypervisor before handing over to a C handler.
189 *
190 * x0: struct kvm_nvhe_init_params PA
191 * x1: bool is_cpu_on
192 */
193 SYM_CODE_START_LOCAL(__kvm_hyp_init_cpu)
194 mov x28, x0 // Stash arguments
195 mov x29, x1
196
197 /* Check that the core was booted in EL2. */
198 mrs x0, CurrentEL
199 cmp x0, #CurrentEL_EL2
200 b.eq 2f
201
202 /* The core booted in EL1. KVM cannot be initialized on it. */
203 1: wfe
204 wfi
205 b 1b
206
207 2: msr SPsel, #1 // We want to use SP_EL{1,2}
208
209 bl __kvm_init_el2_state
210
211 __init_el2_nvhe_prepare_eret
212
213 /* Enable MMU, set vectors and stack. */
214 mov x0, x28
215 bl ___kvm_hyp_init // Clobbers x0..x2
216
217 /* Leave idmap. */
218 mov x0, x29
219 ldr x1, =kvm_host_psci_cpu_entry
220 br x1
221 SYM_CODE_END(__kvm_hyp_init_cpu)
222
223 SYM_CODE_START(__kvm_handle_stub_hvc)
224 /*
225 * __kvm_handle_stub_hvc called from __host_hvc through branch instruction(br) so
226 * we need bti j at beginning.
227 */
228 bti j
229 cmp x0, #HVC_SOFT_RESTART
230 b.ne 1f
231
232 /* This is where we're about to jump, staying at EL2 */
233 msr elr_el2, x1
234 mov x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT | PSR_MODE_EL2h)
235 msr spsr_el2, x0
236
237 /* Shuffle the arguments, and don't come back */
238 mov x0, x2
239 mov x1, x3
240 mov x2, x4
241 b reset
242
243 1: cmp x0, #HVC_RESET_VECTORS
244 b.ne 1f
245
246 /*
247 * Set the HVC_RESET_VECTORS return code before entering the common
248 * path so that we do not clobber x0-x2 in case we are coming via
249 * HVC_SOFT_RESTART.
250 */
251 mov x0, xzr
252 reset:
253 /* Reset kvm back to the hyp stub. */
254 mov_q x5, INIT_SCTLR_EL2_MMU_OFF
255 pre_disable_mmu_workaround
256 msr sctlr_el2, x5
257 isb
258
259 alternative_if ARM64_KVM_PROTECTED_MODE
260 mov_q x5, HCR_HOST_NVHE_FLAGS
261 msr hcr_el2, x5
262 alternative_else_nop_endif
263
264 /* Install stub vectors */
265 adr_l x5, __hyp_stub_vectors
266 msr vbar_el2, x5
267 eret
268
269 1: /* Bad stub call */
270 mov_q x0, HVC_STUB_ERR
271 eret
272
273 SYM_CODE_END(__kvm_handle_stub_hvc)
274
275 /*
276 * void __pkvm_init_switch_pgd(phys_addr_t pgd, unsigned long sp,
277 * void (*fn)(void));
278 *
279 * SYM_TYPED_FUNC_START() allows C to call this ID-mapped function indirectly
280 * using a physical pointer without triggering a kCFI failure.
281 */
282 SYM_TYPED_FUNC_START(__pkvm_init_switch_pgd)
283 /* Turn the MMU off */
284 pre_disable_mmu_workaround
285 mrs x3, sctlr_el2
286 bic x4, x3, #SCTLR_ELx_M
287 msr sctlr_el2, x4
288 isb
289
290 tlbi alle2
291
292 /* Install the new pgtables */
293 phys_to_ttbr x5, x0
294 alternative_if ARM64_HAS_CNP
295 orr x5, x5, #TTBR_CNP_BIT
296 alternative_else_nop_endif
297 msr ttbr0_el2, x5
298
299 /* Set the new stack pointer */
300 mov sp, x1
301
302 /* And turn the MMU back on! */
303 dsb nsh
304 isb
305 set_sctlr_el2 x3
306 ret x2
307 SYM_FUNC_END(__pkvm_init_switch_pgd)
308
309 .popsection
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.