1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020 - Google Inc
4 * Author: Andrew Scull <ascull@google.com>
5 */
6
7 #include <hyp/adjust_pc.h>
8
9 #include <asm/pgtable-types.h>
10 #include <asm/kvm_asm.h>
11 #include <asm/kvm_emulate.h>
12 #include <asm/kvm_host.h>
13 #include <asm/kvm_hyp.h>
14 #include <asm/kvm_mmu.h>
15
16 #include <nvhe/ffa.h>
17 #include <nvhe/mem_protect.h>
18 #include <nvhe/mm.h>
19 #include <nvhe/pkvm.h>
20 #include <nvhe/trap_handler.h>
21
22 DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
23
24 void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt);
25
26 static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu)
27 {
28 __vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_el1(SYS_ZCR);
29 /*
30 * On saving/restoring guest sve state, always use the maximum VL for
31 * the guest. The layout of the data when saving the sve state depends
32 * on the VL, so use a consistent (i.e., the maximum) guest VL.
33 */
34 sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
35 __sve_save_state(vcpu_sve_pffr(vcpu), &vcpu->arch.ctxt.fp_regs.fpsr, true);
36 write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL2);
37 }
38
39 static void __hyp_sve_restore_host(void)
40 {
41 struct cpu_sve_state *sve_state = *host_data_ptr(sve_state);
42
43 /*
44 * On saving/restoring host sve state, always use the maximum VL for
45 * the host. The layout of the data when saving the sve state depends
46 * on the VL, so use a consistent (i.e., the maximum) host VL.
47 *
48 * Setting ZCR_EL2 to ZCR_ELx_LEN_MASK sets the effective length
49 * supported by the system (or limited at EL3).
50 */
51 write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL2);
52 __sve_restore_state(sve_state->sve_regs + sve_ffr_offset(kvm_host_sve_max_vl),
53 &sve_state->fpsr,
54 true);
55 write_sysreg_el1(sve_state->zcr_el1, SYS_ZCR);
56 }
57
58 static void fpsimd_sve_flush(void)
59 {
60 *host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED;
61 }
62
63 static void fpsimd_sve_sync(struct kvm_vcpu *vcpu)
64 {
65 if (!guest_owns_fp_regs())
66 return;
67
68 cpacr_clear_set(0, CPACR_ELx_FPEN | CPACR_ELx_ZEN);
69 isb();
70
71 if (vcpu_has_sve(vcpu))
72 __hyp_sve_save_guest(vcpu);
73 else
74 __fpsimd_save_state(&vcpu->arch.ctxt.fp_regs);
75
76 if (system_supports_sve())
77 __hyp_sve_restore_host();
78 else
79 __fpsimd_restore_state(*host_data_ptr(fpsimd_state));
80
81 *host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED;
82 }
83
84 static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
85 {
86 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu;
87
88 fpsimd_sve_flush();
89
90 hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt;
91
92 hyp_vcpu->vcpu.arch.sve_state = kern_hyp_va(host_vcpu->arch.sve_state);
93 /* Limit guest vector length to the maximum supported by the host. */
94 hyp_vcpu->vcpu.arch.sve_max_vl = min(host_vcpu->arch.sve_max_vl, kvm_host_sve_max_vl);
95
96 hyp_vcpu->vcpu.arch.hw_mmu = host_vcpu->arch.hw_mmu;
97
98 hyp_vcpu->vcpu.arch.hcr_el2 = host_vcpu->arch.hcr_el2;
99 hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2;
100
101 hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags;
102
103 hyp_vcpu->vcpu.arch.debug_ptr = kern_hyp_va(host_vcpu->arch.debug_ptr);
104
105 hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2;
106
107 hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3 = host_vcpu->arch.vgic_cpu.vgic_v3;
108 }
109
110 static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
111 {
112 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu;
113 struct vgic_v3_cpu_if *hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3;
114 struct vgic_v3_cpu_if *host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3;
115 unsigned int i;
116
117 fpsimd_sve_sync(&hyp_vcpu->vcpu);
118
119 host_vcpu->arch.ctxt = hyp_vcpu->vcpu.arch.ctxt;
120
121 host_vcpu->arch.hcr_el2 = hyp_vcpu->vcpu.arch.hcr_el2;
122
123 host_vcpu->arch.fault = hyp_vcpu->vcpu.arch.fault;
124
125 host_vcpu->arch.iflags = hyp_vcpu->vcpu.arch.iflags;
126
127 host_cpu_if->vgic_hcr = hyp_cpu_if->vgic_hcr;
128 for (i = 0; i < hyp_cpu_if->used_lrs; ++i)
129 host_cpu_if->vgic_lr[i] = hyp_cpu_if->vgic_lr[i];
130 }
131
132 static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt)
133 {
134 DECLARE_REG(struct kvm_vcpu *, host_vcpu, host_ctxt, 1);
135 int ret;
136
137 host_vcpu = kern_hyp_va(host_vcpu);
138
139 if (unlikely(is_protected_kvm_enabled())) {
140 struct pkvm_hyp_vcpu *hyp_vcpu;
141 struct kvm *host_kvm;
142
143 /*
144 * KVM (and pKVM) doesn't support SME guests for now, and
145 * ensures that SME features aren't enabled in pstate when
146 * loading a vcpu. Therefore, if SME features enabled the host
147 * is misbehaving.
148 */
149 if (unlikely(system_supports_sme() && read_sysreg_s(SYS_SVCR))) {
150 ret = -EINVAL;
151 goto out;
152 }
153
154 host_kvm = kern_hyp_va(host_vcpu->kvm);
155 hyp_vcpu = pkvm_load_hyp_vcpu(host_kvm->arch.pkvm.handle,
156 host_vcpu->vcpu_idx);
157 if (!hyp_vcpu) {
158 ret = -EINVAL;
159 goto out;
160 }
161
162 flush_hyp_vcpu(hyp_vcpu);
163
164 ret = __kvm_vcpu_run(&hyp_vcpu->vcpu);
165
166 sync_hyp_vcpu(hyp_vcpu);
167 pkvm_put_hyp_vcpu(hyp_vcpu);
168 } else {
169 /* The host is fully trusted, run its vCPU directly. */
170 ret = __kvm_vcpu_run(host_vcpu);
171 }
172
173 out:
174 cpu_reg(host_ctxt, 1) = ret;
175 }
176
177 static void handle___kvm_adjust_pc(struct kvm_cpu_context *host_ctxt)
178 {
179 DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1);
180
181 __kvm_adjust_pc(kern_hyp_va(vcpu));
182 }
183
184 static void handle___kvm_flush_vm_context(struct kvm_cpu_context *host_ctxt)
185 {
186 __kvm_flush_vm_context();
187 }
188
189 static void handle___kvm_tlb_flush_vmid_ipa(struct kvm_cpu_context *host_ctxt)
190 {
191 DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1);
192 DECLARE_REG(phys_addr_t, ipa, host_ctxt, 2);
193 DECLARE_REG(int, level, host_ctxt, 3);
194
195 __kvm_tlb_flush_vmid_ipa(kern_hyp_va(mmu), ipa, level);
196 }
197
198 static void handle___kvm_tlb_flush_vmid_ipa_nsh(struct kvm_cpu_context *host_ctxt)
199 {
200 DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1);
201 DECLARE_REG(phys_addr_t, ipa, host_ctxt, 2);
202 DECLARE_REG(int, level, host_ctxt, 3);
203
204 __kvm_tlb_flush_vmid_ipa_nsh(kern_hyp_va(mmu), ipa, level);
205 }
206
207 static void
208 handle___kvm_tlb_flush_vmid_range(struct kvm_cpu_context *host_ctxt)
209 {
210 DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1);
211 DECLARE_REG(phys_addr_t, start, host_ctxt, 2);
212 DECLARE_REG(unsigned long, pages, host_ctxt, 3);
213
214 __kvm_tlb_flush_vmid_range(kern_hyp_va(mmu), start, pages);
215 }
216
217 static void handle___kvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt)
218 {
219 DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1);
220
221 __kvm_tlb_flush_vmid(kern_hyp_va(mmu));
222 }
223
224 static void handle___kvm_flush_cpu_context(struct kvm_cpu_context *host_ctxt)
225 {
226 DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1);
227
228 __kvm_flush_cpu_context(kern_hyp_va(mmu));
229 }
230
231 static void handle___kvm_timer_set_cntvoff(struct kvm_cpu_context *host_ctxt)
232 {
233 __kvm_timer_set_cntvoff(cpu_reg(host_ctxt, 1));
234 }
235
236 static void handle___kvm_enable_ssbs(struct kvm_cpu_context *host_ctxt)
237 {
238 u64 tmp;
239
240 tmp = read_sysreg_el2(SYS_SCTLR);
241 tmp |= SCTLR_ELx_DSSBS;
242 write_sysreg_el2(tmp, SYS_SCTLR);
243 }
244
245 static void handle___vgic_v3_get_gic_config(struct kvm_cpu_context *host_ctxt)
246 {
247 cpu_reg(host_ctxt, 1) = __vgic_v3_get_gic_config();
248 }
249
250 static void handle___vgic_v3_init_lrs(struct kvm_cpu_context *host_ctxt)
251 {
252 __vgic_v3_init_lrs();
253 }
254
255 static void handle___kvm_get_mdcr_el2(struct kvm_cpu_context *host_ctxt)
256 {
257 cpu_reg(host_ctxt, 1) = __kvm_get_mdcr_el2();
258 }
259
260 static void handle___vgic_v3_save_vmcr_aprs(struct kvm_cpu_context *host_ctxt)
261 {
262 DECLARE_REG(struct vgic_v3_cpu_if *, cpu_if, host_ctxt, 1);
263
264 __vgic_v3_save_vmcr_aprs(kern_hyp_va(cpu_if));
265 }
266
267 static void handle___vgic_v3_restore_vmcr_aprs(struct kvm_cpu_context *host_ctxt)
268 {
269 DECLARE_REG(struct vgic_v3_cpu_if *, cpu_if, host_ctxt, 1);
270
271 __vgic_v3_restore_vmcr_aprs(kern_hyp_va(cpu_if));
272 }
273
274 static void handle___pkvm_init(struct kvm_cpu_context *host_ctxt)
275 {
276 DECLARE_REG(phys_addr_t, phys, host_ctxt, 1);
277 DECLARE_REG(unsigned long, size, host_ctxt, 2);
278 DECLARE_REG(unsigned long, nr_cpus, host_ctxt, 3);
279 DECLARE_REG(unsigned long *, per_cpu_base, host_ctxt, 4);
280 DECLARE_REG(u32, hyp_va_bits, host_ctxt, 5);
281
282 /*
283 * __pkvm_init() will return only if an error occurred, otherwise it
284 * will tail-call in __pkvm_init_finalise() which will have to deal
285 * with the host context directly.
286 */
287 cpu_reg(host_ctxt, 1) = __pkvm_init(phys, size, nr_cpus, per_cpu_base,
288 hyp_va_bits);
289 }
290
291 static void handle___pkvm_cpu_set_vector(struct kvm_cpu_context *host_ctxt)
292 {
293 DECLARE_REG(enum arm64_hyp_spectre_vector, slot, host_ctxt, 1);
294
295 cpu_reg(host_ctxt, 1) = pkvm_cpu_set_vector(slot);
296 }
297
298 static void handle___pkvm_host_share_hyp(struct kvm_cpu_context *host_ctxt)
299 {
300 DECLARE_REG(u64, pfn, host_ctxt, 1);
301
302 cpu_reg(host_ctxt, 1) = __pkvm_host_share_hyp(pfn);
303 }
304
305 static void handle___pkvm_host_unshare_hyp(struct kvm_cpu_context *host_ctxt)
306 {
307 DECLARE_REG(u64, pfn, host_ctxt, 1);
308
309 cpu_reg(host_ctxt, 1) = __pkvm_host_unshare_hyp(pfn);
310 }
311
312 static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ctxt)
313 {
314 DECLARE_REG(phys_addr_t, phys, host_ctxt, 1);
315 DECLARE_REG(size_t, size, host_ctxt, 2);
316 DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 3);
317
318 /*
319 * __pkvm_create_private_mapping() populates a pointer with the
320 * hypervisor start address of the allocation.
321 *
322 * However, handle___pkvm_create_private_mapping() hypercall crosses the
323 * EL1/EL2 boundary so the pointer would not be valid in this context.
324 *
325 * Instead pass the allocation address as the return value (or return
326 * ERR_PTR() on failure).
327 */
328 unsigned long haddr;
329 int err = __pkvm_create_private_mapping(phys, size, prot, &haddr);
330
331 if (err)
332 haddr = (unsigned long)ERR_PTR(err);
333
334 cpu_reg(host_ctxt, 1) = haddr;
335 }
336
337 static void handle___pkvm_prot_finalize(struct kvm_cpu_context *host_ctxt)
338 {
339 cpu_reg(host_ctxt, 1) = __pkvm_prot_finalize();
340 }
341
342 static void handle___pkvm_vcpu_init_traps(struct kvm_cpu_context *host_ctxt)
343 {
344 DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1);
345
346 __pkvm_vcpu_init_traps(kern_hyp_va(vcpu));
347 }
348
349 static void handle___pkvm_init_vm(struct kvm_cpu_context *host_ctxt)
350 {
351 DECLARE_REG(struct kvm *, host_kvm, host_ctxt, 1);
352 DECLARE_REG(unsigned long, vm_hva, host_ctxt, 2);
353 DECLARE_REG(unsigned long, pgd_hva, host_ctxt, 3);
354
355 host_kvm = kern_hyp_va(host_kvm);
356 cpu_reg(host_ctxt, 1) = __pkvm_init_vm(host_kvm, vm_hva, pgd_hva);
357 }
358
359 static void handle___pkvm_init_vcpu(struct kvm_cpu_context *host_ctxt)
360 {
361 DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1);
362 DECLARE_REG(struct kvm_vcpu *, host_vcpu, host_ctxt, 2);
363 DECLARE_REG(unsigned long, vcpu_hva, host_ctxt, 3);
364
365 host_vcpu = kern_hyp_va(host_vcpu);
366 cpu_reg(host_ctxt, 1) = __pkvm_init_vcpu(handle, host_vcpu, vcpu_hva);
367 }
368
369 static void handle___pkvm_teardown_vm(struct kvm_cpu_context *host_ctxt)
370 {
371 DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1);
372
373 cpu_reg(host_ctxt, 1) = __pkvm_teardown_vm(handle);
374 }
375
376 typedef void (*hcall_t)(struct kvm_cpu_context *);
377
378 #define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x
379
380 static const hcall_t host_hcall[] = {
381 /* ___kvm_hyp_init */
382 HANDLE_FUNC(__kvm_get_mdcr_el2),
383 HANDLE_FUNC(__pkvm_init),
384 HANDLE_FUNC(__pkvm_create_private_mapping),
385 HANDLE_FUNC(__pkvm_cpu_set_vector),
386 HANDLE_FUNC(__kvm_enable_ssbs),
387 HANDLE_FUNC(__vgic_v3_init_lrs),
388 HANDLE_FUNC(__vgic_v3_get_gic_config),
389 HANDLE_FUNC(__pkvm_prot_finalize),
390
391 HANDLE_FUNC(__pkvm_host_share_hyp),
392 HANDLE_FUNC(__pkvm_host_unshare_hyp),
393 HANDLE_FUNC(__kvm_adjust_pc),
394 HANDLE_FUNC(__kvm_vcpu_run),
395 HANDLE_FUNC(__kvm_flush_vm_context),
396 HANDLE_FUNC(__kvm_tlb_flush_vmid_ipa),
397 HANDLE_FUNC(__kvm_tlb_flush_vmid_ipa_nsh),
398 HANDLE_FUNC(__kvm_tlb_flush_vmid),
399 HANDLE_FUNC(__kvm_tlb_flush_vmid_range),
400 HANDLE_FUNC(__kvm_flush_cpu_context),
401 HANDLE_FUNC(__kvm_timer_set_cntvoff),
402 HANDLE_FUNC(__vgic_v3_save_vmcr_aprs),
403 HANDLE_FUNC(__vgic_v3_restore_vmcr_aprs),
404 HANDLE_FUNC(__pkvm_vcpu_init_traps),
405 HANDLE_FUNC(__pkvm_init_vm),
406 HANDLE_FUNC(__pkvm_init_vcpu),
407 HANDLE_FUNC(__pkvm_teardown_vm),
408 };
409
410 static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)
411 {
412 DECLARE_REG(unsigned long, id, host_ctxt, 0);
413 unsigned long hcall_min = 0;
414 hcall_t hfn;
415
416 /*
417 * If pKVM has been initialised then reject any calls to the
418 * early "privileged" hypercalls. Note that we cannot reject
419 * calls to __pkvm_prot_finalize for two reasons: (1) The static
420 * key used to determine initialisation must be toggled prior to
421 * finalisation and (2) finalisation is performed on a per-CPU
422 * basis. This is all fine, however, since __pkvm_prot_finalize
423 * returns -EPERM after the first call for a given CPU.
424 */
425 if (static_branch_unlikely(&kvm_protected_mode_initialized))
426 hcall_min = __KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize;
427
428 id &= ~ARM_SMCCC_CALL_HINTS;
429 id -= KVM_HOST_SMCCC_ID(0);
430
431 if (unlikely(id < hcall_min || id >= ARRAY_SIZE(host_hcall)))
432 goto inval;
433
434 hfn = host_hcall[id];
435 if (unlikely(!hfn))
436 goto inval;
437
438 cpu_reg(host_ctxt, 0) = SMCCC_RET_SUCCESS;
439 hfn(host_ctxt);
440
441 return;
442 inval:
443 cpu_reg(host_ctxt, 0) = SMCCC_RET_NOT_SUPPORTED;
444 }
445
446 static void default_host_smc_handler(struct kvm_cpu_context *host_ctxt)
447 {
448 __kvm_hyp_host_forward_smc(host_ctxt);
449 }
450
451 static void handle_host_smc(struct kvm_cpu_context *host_ctxt)
452 {
453 DECLARE_REG(u64, func_id, host_ctxt, 0);
454 bool handled;
455
456 func_id &= ~ARM_SMCCC_CALL_HINTS;
457
458 handled = kvm_host_psci_handler(host_ctxt, func_id);
459 if (!handled)
460 handled = kvm_host_ffa_handler(host_ctxt, func_id);
461 if (!handled)
462 default_host_smc_handler(host_ctxt);
463
464 /* SMC was trapped, move ELR past the current PC. */
465 kvm_skip_host_instr();
466 }
467
468 void handle_trap(struct kvm_cpu_context *host_ctxt)
469 {
470 u64 esr = read_sysreg_el2(SYS_ESR);
471
472 switch (ESR_ELx_EC(esr)) {
473 case ESR_ELx_EC_HVC64:
474 handle_host_hcall(host_ctxt);
475 break;
476 case ESR_ELx_EC_SMC64:
477 handle_host_smc(host_ctxt);
478 break;
479 case ESR_ELx_EC_SVE:
480 cpacr_clear_set(0, CPACR_ELx_ZEN);
481 isb();
482 sve_cond_update_zcr_vq(ZCR_ELx_LEN_MASK, SYS_ZCR_EL2);
483 break;
484 case ESR_ELx_EC_IABT_LOW:
485 case ESR_ELx_EC_DABT_LOW:
486 handle_host_mem_abort(host_ctxt);
487 break;
488 default:
489 BUG();
490 }
491 }
492
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.