1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
4 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
5 */
6
7 #include <linux/bug.h>
8 #include <linux/cpu_pm.h>
9 #include <linux/entry-kvm.h>
10 #include <linux/errno.h>
11 #include <linux/err.h>
12 #include <linux/kvm_host.h>
13 #include <linux/list.h>
14 #include <linux/module.h>
15 #include <linux/vmalloc.h>
16 #include <linux/fs.h>
17 #include <linux/mman.h>
18 #include <linux/sched.h>
19 #include <linux/kvm.h>
20 #include <linux/kvm_irqfd.h>
21 #include <linux/irqbypass.h>
22 #include <linux/sched/stat.h>
23 #include <linux/psci.h>
24 #include <trace/events/kvm.h>
25
26 #define CREATE_TRACE_POINTS
27 #include "trace_arm.h"
28
29 #include <linux/uaccess.h>
30 #include <asm/ptrace.h>
31 #include <asm/mman.h>
32 #include <asm/tlbflush.h>
33 #include <asm/cacheflush.h>
34 #include <asm/cpufeature.h>
35 #include <asm/virt.h>
36 #include <asm/kvm_arm.h>
37 #include <asm/kvm_asm.h>
38 #include <asm/kvm_emulate.h>
39 #include <asm/kvm_mmu.h>
40 #include <asm/kvm_nested.h>
41 #include <asm/kvm_pkvm.h>
42 #include <asm/kvm_ptrauth.h>
43 #include <asm/sections.h>
44
45 #include <kvm/arm_hypercalls.h>
46 #include <kvm/arm_pmu.h>
47 #include <kvm/arm_psci.h>
48
49 static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT;
50
51 enum kvm_wfx_trap_policy {
52 KVM_WFX_NOTRAP_SINGLE_TASK, /* Default option */
53 KVM_WFX_NOTRAP,
54 KVM_WFX_TRAP,
55 };
56
57 static enum kvm_wfx_trap_policy kvm_wfi_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK;
58 static enum kvm_wfx_trap_policy kvm_wfe_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK;
59
60 DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
61
62 DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
63 DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
64
65 DECLARE_KVM_NVHE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
66
67 static bool vgic_present, kvm_arm_initialised;
68
69 static DEFINE_PER_CPU(unsigned char, kvm_hyp_initialized);
70 DEFINE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
71
72 bool is_kvm_arm_initialised(void)
73 {
74 return kvm_arm_initialised;
75 }
76
77 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
78 {
79 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
80 }
81
82 /*
83 * This functions as an allow-list of protected VM capabilities.
84 * Features not explicitly allowed by this function are denied.
85 */
86 static bool pkvm_ext_allowed(struct kvm *kvm, long ext)
87 {
88 switch (ext) {
89 case KVM_CAP_IRQCHIP:
90 case KVM_CAP_ARM_PSCI:
91 case KVM_CAP_ARM_PSCI_0_2:
92 case KVM_CAP_NR_VCPUS:
93 case KVM_CAP_MAX_VCPUS:
94 case KVM_CAP_MAX_VCPU_ID:
95 case KVM_CAP_MSI_DEVID:
96 case KVM_CAP_ARM_VM_IPA_SIZE:
97 case KVM_CAP_ARM_PMU_V3:
98 case KVM_CAP_ARM_SVE:
99 case KVM_CAP_ARM_PTRAUTH_ADDRESS:
100 case KVM_CAP_ARM_PTRAUTH_GENERIC:
101 return true;
102 default:
103 return false;
104 }
105 }
106
107 int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
108 struct kvm_enable_cap *cap)
109 {
110 int r = -EINVAL;
111
112 if (cap->flags)
113 return -EINVAL;
114
115 if (kvm_vm_is_protected(kvm) && !pkvm_ext_allowed(kvm, cap->cap))
116 return -EINVAL;
117
118 switch (cap->cap) {
119 case KVM_CAP_ARM_NISV_TO_USER:
120 r = 0;
121 set_bit(KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER,
122 &kvm->arch.flags);
123 break;
124 case KVM_CAP_ARM_MTE:
125 mutex_lock(&kvm->lock);
126 if (system_supports_mte() && !kvm->created_vcpus) {
127 r = 0;
128 set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags);
129 }
130 mutex_unlock(&kvm->lock);
131 break;
132 case KVM_CAP_ARM_SYSTEM_SUSPEND:
133 r = 0;
134 set_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags);
135 break;
136 case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE:
137 mutex_lock(&kvm->slots_lock);
138 /*
139 * To keep things simple, allow changing the chunk
140 * size only when no memory slots have been created.
141 */
142 if (kvm_are_all_memslots_empty(kvm)) {
143 u64 new_cap = cap->args[0];
144
145 if (!new_cap || kvm_is_block_size_supported(new_cap)) {
146 r = 0;
147 kvm->arch.mmu.split_page_chunk_size = new_cap;
148 }
149 }
150 mutex_unlock(&kvm->slots_lock);
151 break;
152 default:
153 break;
154 }
155
156 return r;
157 }
158
159 static int kvm_arm_default_max_vcpus(void)
160 {
161 return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS;
162 }
163
164 /**
165 * kvm_arch_init_vm - initializes a VM data structure
166 * @kvm: pointer to the KVM struct
167 * @type: kvm device type
168 */
169 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
170 {
171 int ret;
172
173 mutex_init(&kvm->arch.config_lock);
174
175 #ifdef CONFIG_LOCKDEP
176 /* Clue in lockdep that the config_lock must be taken inside kvm->lock */
177 mutex_lock(&kvm->lock);
178 mutex_lock(&kvm->arch.config_lock);
179 mutex_unlock(&kvm->arch.config_lock);
180 mutex_unlock(&kvm->lock);
181 #endif
182
183 kvm_init_nested(kvm);
184
185 ret = kvm_share_hyp(kvm, kvm + 1);
186 if (ret)
187 return ret;
188
189 ret = pkvm_init_host_vm(kvm);
190 if (ret)
191 goto err_unshare_kvm;
192
193 if (!zalloc_cpumask_var(&kvm->arch.supported_cpus, GFP_KERNEL_ACCOUNT)) {
194 ret = -ENOMEM;
195 goto err_unshare_kvm;
196 }
197 cpumask_copy(kvm->arch.supported_cpus, cpu_possible_mask);
198
199 ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu, type);
200 if (ret)
201 goto err_free_cpumask;
202
203 kvm_vgic_early_init(kvm);
204
205 kvm_timer_init_vm(kvm);
206
207 /* The maximum number of VCPUs is limited by the host's GIC model */
208 kvm->max_vcpus = kvm_arm_default_max_vcpus();
209
210 kvm_arm_init_hypercalls(kvm);
211
212 bitmap_zero(kvm->arch.vcpu_features, KVM_VCPU_MAX_FEATURES);
213
214 return 0;
215
216 err_free_cpumask:
217 free_cpumask_var(kvm->arch.supported_cpus);
218 err_unshare_kvm:
219 kvm_unshare_hyp(kvm, kvm + 1);
220 return ret;
221 }
222
223 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
224 {
225 return VM_FAULT_SIGBUS;
226 }
227
228 void kvm_arch_create_vm_debugfs(struct kvm *kvm)
229 {
230 kvm_sys_regs_create_debugfs(kvm);
231 }
232
233 static void kvm_destroy_mpidr_data(struct kvm *kvm)
234 {
235 struct kvm_mpidr_data *data;
236
237 mutex_lock(&kvm->arch.config_lock);
238
239 data = rcu_dereference_protected(kvm->arch.mpidr_data,
240 lockdep_is_held(&kvm->arch.config_lock));
241 if (data) {
242 rcu_assign_pointer(kvm->arch.mpidr_data, NULL);
243 synchronize_rcu();
244 kfree(data);
245 }
246
247 mutex_unlock(&kvm->arch.config_lock);
248 }
249
250 /**
251 * kvm_arch_destroy_vm - destroy the VM data structure
252 * @kvm: pointer to the KVM struct
253 */
254 void kvm_arch_destroy_vm(struct kvm *kvm)
255 {
256 bitmap_free(kvm->arch.pmu_filter);
257 free_cpumask_var(kvm->arch.supported_cpus);
258
259 kvm_vgic_destroy(kvm);
260
261 if (is_protected_kvm_enabled())
262 pkvm_destroy_hyp_vm(kvm);
263
264 kvm_destroy_mpidr_data(kvm);
265
266 kfree(kvm->arch.sysreg_masks);
267 kvm_destroy_vcpus(kvm);
268
269 kvm_unshare_hyp(kvm, kvm + 1);
270
271 kvm_arm_teardown_hypercalls(kvm);
272 }
273
274 static bool kvm_has_full_ptr_auth(void)
275 {
276 bool apa, gpa, api, gpi, apa3, gpa3;
277 u64 isar1, isar2, val;
278
279 /*
280 * Check that:
281 *
282 * - both Address and Generic auth are implemented for a given
283 * algorithm (Q5, IMPDEF or Q3)
284 * - only a single algorithm is implemented.
285 */
286 if (!system_has_full_ptr_auth())
287 return false;
288
289 isar1 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1);
290 isar2 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
291
292 apa = !!FIELD_GET(ID_AA64ISAR1_EL1_APA_MASK, isar1);
293 val = FIELD_GET(ID_AA64ISAR1_EL1_GPA_MASK, isar1);
294 gpa = (val == ID_AA64ISAR1_EL1_GPA_IMP);
295
296 api = !!FIELD_GET(ID_AA64ISAR1_EL1_API_MASK, isar1);
297 val = FIELD_GET(ID_AA64ISAR1_EL1_GPI_MASK, isar1);
298 gpi = (val == ID_AA64ISAR1_EL1_GPI_IMP);
299
300 apa3 = !!FIELD_GET(ID_AA64ISAR2_EL1_APA3_MASK, isar2);
301 val = FIELD_GET(ID_AA64ISAR2_EL1_GPA3_MASK, isar2);
302 gpa3 = (val == ID_AA64ISAR2_EL1_GPA3_IMP);
303
304 return (apa == gpa && api == gpi && apa3 == gpa3 &&
305 (apa + api + apa3) == 1);
306 }
307
308 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
309 {
310 int r;
311
312 if (kvm && kvm_vm_is_protected(kvm) && !pkvm_ext_allowed(kvm, ext))
313 return 0;
314
315 switch (ext) {
316 case KVM_CAP_IRQCHIP:
317 r = vgic_present;
318 break;
319 case KVM_CAP_IOEVENTFD:
320 case KVM_CAP_USER_MEMORY:
321 case KVM_CAP_SYNC_MMU:
322 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
323 case KVM_CAP_ONE_REG:
324 case KVM_CAP_ARM_PSCI:
325 case KVM_CAP_ARM_PSCI_0_2:
326 case KVM_CAP_READONLY_MEM:
327 case KVM_CAP_MP_STATE:
328 case KVM_CAP_IMMEDIATE_EXIT:
329 case KVM_CAP_VCPU_EVENTS:
330 case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2:
331 case KVM_CAP_ARM_NISV_TO_USER:
332 case KVM_CAP_ARM_INJECT_EXT_DABT:
333 case KVM_CAP_SET_GUEST_DEBUG:
334 case KVM_CAP_VCPU_ATTRIBUTES:
335 case KVM_CAP_PTP_KVM:
336 case KVM_CAP_ARM_SYSTEM_SUSPEND:
337 case KVM_CAP_IRQFD_RESAMPLE:
338 case KVM_CAP_COUNTER_OFFSET:
339 r = 1;
340 break;
341 case KVM_CAP_SET_GUEST_DEBUG2:
342 return KVM_GUESTDBG_VALID_MASK;
343 case KVM_CAP_ARM_SET_DEVICE_ADDR:
344 r = 1;
345 break;
346 case KVM_CAP_NR_VCPUS:
347 /*
348 * ARM64 treats KVM_CAP_NR_CPUS differently from all other
349 * architectures, as it does not always bound it to
350 * KVM_CAP_MAX_VCPUS. It should not matter much because
351 * this is just an advisory value.
352 */
353 r = min_t(unsigned int, num_online_cpus(),
354 kvm_arm_default_max_vcpus());
355 break;
356 case KVM_CAP_MAX_VCPUS:
357 case KVM_CAP_MAX_VCPU_ID:
358 if (kvm)
359 r = kvm->max_vcpus;
360 else
361 r = kvm_arm_default_max_vcpus();
362 break;
363 case KVM_CAP_MSI_DEVID:
364 if (!kvm)
365 r = -EINVAL;
366 else
367 r = kvm->arch.vgic.msis_require_devid;
368 break;
369 case KVM_CAP_ARM_USER_IRQ:
370 /*
371 * 1: EL1_VTIMER, EL1_PTIMER, and PMU.
372 * (bump this number if adding more devices)
373 */
374 r = 1;
375 break;
376 case KVM_CAP_ARM_MTE:
377 r = system_supports_mte();
378 break;
379 case KVM_CAP_STEAL_TIME:
380 r = kvm_arm_pvtime_supported();
381 break;
382 case KVM_CAP_ARM_EL1_32BIT:
383 r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1);
384 break;
385 case KVM_CAP_GUEST_DEBUG_HW_BPS:
386 r = get_num_brps();
387 break;
388 case KVM_CAP_GUEST_DEBUG_HW_WPS:
389 r = get_num_wrps();
390 break;
391 case KVM_CAP_ARM_PMU_V3:
392 r = kvm_arm_support_pmu_v3();
393 break;
394 case KVM_CAP_ARM_INJECT_SERROR_ESR:
395 r = cpus_have_final_cap(ARM64_HAS_RAS_EXTN);
396 break;
397 case KVM_CAP_ARM_VM_IPA_SIZE:
398 r = get_kvm_ipa_limit();
399 break;
400 case KVM_CAP_ARM_SVE:
401 r = system_supports_sve();
402 break;
403 case KVM_CAP_ARM_PTRAUTH_ADDRESS:
404 case KVM_CAP_ARM_PTRAUTH_GENERIC:
405 r = kvm_has_full_ptr_auth();
406 break;
407 case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE:
408 if (kvm)
409 r = kvm->arch.mmu.split_page_chunk_size;
410 else
411 r = KVM_ARM_EAGER_SPLIT_CHUNK_SIZE_DEFAULT;
412 break;
413 case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES:
414 r = kvm_supported_block_sizes();
415 break;
416 case KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES:
417 r = BIT(0);
418 break;
419 default:
420 r = 0;
421 }
422
423 return r;
424 }
425
426 long kvm_arch_dev_ioctl(struct file *filp,
427 unsigned int ioctl, unsigned long arg)
428 {
429 return -EINVAL;
430 }
431
432 struct kvm *kvm_arch_alloc_vm(void)
433 {
434 size_t sz = sizeof(struct kvm);
435
436 if (!has_vhe())
437 return kzalloc(sz, GFP_KERNEL_ACCOUNT);
438
439 return __vmalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_HIGHMEM | __GFP_ZERO);
440 }
441
442 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
443 {
444 if (irqchip_in_kernel(kvm) && vgic_initialized(kvm))
445 return -EBUSY;
446
447 if (id >= kvm->max_vcpus)
448 return -EINVAL;
449
450 return 0;
451 }
452
453 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
454 {
455 int err;
456
457 spin_lock_init(&vcpu->arch.mp_state_lock);
458
459 #ifdef CONFIG_LOCKDEP
460 /* Inform lockdep that the config_lock is acquired after vcpu->mutex */
461 mutex_lock(&vcpu->mutex);
462 mutex_lock(&vcpu->kvm->arch.config_lock);
463 mutex_unlock(&vcpu->kvm->arch.config_lock);
464 mutex_unlock(&vcpu->mutex);
465 #endif
466
467 /* Force users to call KVM_ARM_VCPU_INIT */
468 vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
469
470 vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;
471
472 /* Set up the timer */
473 kvm_timer_vcpu_init(vcpu);
474
475 kvm_pmu_vcpu_init(vcpu);
476
477 kvm_arm_reset_debug_ptr(vcpu);
478
479 kvm_arm_pvtime_vcpu_init(&vcpu->arch);
480
481 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
482
483 /*
484 * This vCPU may have been created after mpidr_data was initialized.
485 * Throw out the pre-computed mappings if that is the case which forces
486 * KVM to fall back to iteratively searching the vCPUs.
487 */
488 kvm_destroy_mpidr_data(vcpu->kvm);
489
490 err = kvm_vgic_vcpu_init(vcpu);
491 if (err)
492 return err;
493
494 return kvm_share_hyp(vcpu, vcpu + 1);
495 }
496
497 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
498 {
499 }
500
501 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
502 {
503 if (vcpu_has_run_once(vcpu) && unlikely(!irqchip_in_kernel(vcpu->kvm)))
504 static_branch_dec(&userspace_irqchip_in_use);
505
506 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
507 kvm_timer_vcpu_terminate(vcpu);
508 kvm_pmu_vcpu_destroy(vcpu);
509 kvm_vgic_vcpu_destroy(vcpu);
510 kvm_arm_vcpu_destroy(vcpu);
511 }
512
513 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
514 {
515
516 }
517
518 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
519 {
520
521 }
522
523 static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu)
524 {
525 if (vcpu_has_ptrauth(vcpu) && !is_protected_kvm_enabled()) {
526 /*
527 * Either we're running an L2 guest, and the API/APK bits come
528 * from L1's HCR_EL2, or API/APK are both set.
529 */
530 if (unlikely(vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu))) {
531 u64 val;
532
533 val = __vcpu_sys_reg(vcpu, HCR_EL2);
534 val &= (HCR_API | HCR_APK);
535 vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
536 vcpu->arch.hcr_el2 |= val;
537 } else {
538 vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK);
539 }
540
541 /*
542 * Save the host keys if there is any chance for the guest
543 * to use pauth, as the entry code will reload the guest
544 * keys in that case.
545 */
546 if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) {
547 struct kvm_cpu_context *ctxt;
548
549 ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt);
550 ptrauth_save_keys(ctxt);
551 }
552 }
553 }
554
555 static bool kvm_vcpu_should_clear_twi(struct kvm_vcpu *vcpu)
556 {
557 if (unlikely(kvm_wfi_trap_policy != KVM_WFX_NOTRAP_SINGLE_TASK))
558 return kvm_wfi_trap_policy == KVM_WFX_NOTRAP;
559
560 return single_task_running() &&
561 (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count) ||
562 vcpu->kvm->arch.vgic.nassgireq);
563 }
564
565 static bool kvm_vcpu_should_clear_twe(struct kvm_vcpu *vcpu)
566 {
567 if (unlikely(kvm_wfe_trap_policy != KVM_WFX_NOTRAP_SINGLE_TASK))
568 return kvm_wfe_trap_policy == KVM_WFX_NOTRAP;
569
570 return single_task_running();
571 }
572
573 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
574 {
575 struct kvm_s2_mmu *mmu;
576 int *last_ran;
577
578 if (vcpu_has_nv(vcpu))
579 kvm_vcpu_load_hw_mmu(vcpu);
580
581 mmu = vcpu->arch.hw_mmu;
582 last_ran = this_cpu_ptr(mmu->last_vcpu_ran);
583
584 /*
585 * We guarantee that both TLBs and I-cache are private to each
586 * vcpu. If detecting that a vcpu from the same VM has
587 * previously run on the same physical CPU, call into the
588 * hypervisor code to nuke the relevant contexts.
589 *
590 * We might get preempted before the vCPU actually runs, but
591 * over-invalidation doesn't affect correctness.
592 */
593 if (*last_ran != vcpu->vcpu_idx) {
594 kvm_call_hyp(__kvm_flush_cpu_context, mmu);
595 *last_ran = vcpu->vcpu_idx;
596 }
597
598 vcpu->cpu = cpu;
599
600 kvm_vgic_load(vcpu);
601 kvm_timer_vcpu_load(vcpu);
602 if (has_vhe())
603 kvm_vcpu_load_vhe(vcpu);
604 kvm_arch_vcpu_load_fp(vcpu);
605 kvm_vcpu_pmu_restore_guest(vcpu);
606 if (kvm_arm_is_pvtime_enabled(&vcpu->arch))
607 kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu);
608
609 if (kvm_vcpu_should_clear_twe(vcpu))
610 vcpu->arch.hcr_el2 &= ~HCR_TWE;
611 else
612 vcpu->arch.hcr_el2 |= HCR_TWE;
613
614 if (kvm_vcpu_should_clear_twi(vcpu))
615 vcpu->arch.hcr_el2 &= ~HCR_TWI;
616 else
617 vcpu->arch.hcr_el2 |= HCR_TWI;
618
619 vcpu_set_pauth_traps(vcpu);
620
621 kvm_arch_vcpu_load_debug_state_flags(vcpu);
622
623 if (!cpumask_test_cpu(cpu, vcpu->kvm->arch.supported_cpus))
624 vcpu_set_on_unsupported_cpu(vcpu);
625 }
626
627 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
628 {
629 kvm_arch_vcpu_put_debug_state_flags(vcpu);
630 kvm_arch_vcpu_put_fp(vcpu);
631 if (has_vhe())
632 kvm_vcpu_put_vhe(vcpu);
633 kvm_timer_vcpu_put(vcpu);
634 kvm_vgic_put(vcpu);
635 kvm_vcpu_pmu_restore_host(vcpu);
636 if (vcpu_has_nv(vcpu))
637 kvm_vcpu_put_hw_mmu(vcpu);
638 kvm_arm_vmid_clear_active();
639
640 vcpu_clear_on_unsupported_cpu(vcpu);
641 vcpu->cpu = -1;
642 }
643
644 static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
645 {
646 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);
647 kvm_make_request(KVM_REQ_SLEEP, vcpu);
648 kvm_vcpu_kick(vcpu);
649 }
650
651 void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
652 {
653 spin_lock(&vcpu->arch.mp_state_lock);
654 __kvm_arm_vcpu_power_off(vcpu);
655 spin_unlock(&vcpu->arch.mp_state_lock);
656 }
657
658 bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu)
659 {
660 return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED;
661 }
662
663 static void kvm_arm_vcpu_suspend(struct kvm_vcpu *vcpu)
664 {
665 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_SUSPENDED);
666 kvm_make_request(KVM_REQ_SUSPEND, vcpu);
667 kvm_vcpu_kick(vcpu);
668 }
669
670 static bool kvm_arm_vcpu_suspended(struct kvm_vcpu *vcpu)
671 {
672 return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_SUSPENDED;
673 }
674
675 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
676 struct kvm_mp_state *mp_state)
677 {
678 *mp_state = READ_ONCE(vcpu->arch.mp_state);
679
680 return 0;
681 }
682
683 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
684 struct kvm_mp_state *mp_state)
685 {
686 int ret = 0;
687
688 spin_lock(&vcpu->arch.mp_state_lock);
689
690 switch (mp_state->mp_state) {
691 case KVM_MP_STATE_RUNNABLE:
692 WRITE_ONCE(vcpu->arch.mp_state, *mp_state);
693 break;
694 case KVM_MP_STATE_STOPPED:
695 __kvm_arm_vcpu_power_off(vcpu);
696 break;
697 case KVM_MP_STATE_SUSPENDED:
698 kvm_arm_vcpu_suspend(vcpu);
699 break;
700 default:
701 ret = -EINVAL;
702 }
703
704 spin_unlock(&vcpu->arch.mp_state_lock);
705
706 return ret;
707 }
708
709 /**
710 * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
711 * @v: The VCPU pointer
712 *
713 * If the guest CPU is not waiting for interrupts or an interrupt line is
714 * asserted, the CPU is by definition runnable.
715 */
716 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
717 {
718 bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF);
719 return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
720 && !kvm_arm_vcpu_stopped(v) && !v->arch.pause);
721 }
722
723 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
724 {
725 return vcpu_mode_priv(vcpu);
726 }
727
728 #ifdef CONFIG_GUEST_PERF_EVENTS
729 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)
730 {
731 return *vcpu_pc(vcpu);
732 }
733 #endif
734
735 static void kvm_init_mpidr_data(struct kvm *kvm)
736 {
737 struct kvm_mpidr_data *data = NULL;
738 unsigned long c, mask, nr_entries;
739 u64 aff_set = 0, aff_clr = ~0UL;
740 struct kvm_vcpu *vcpu;
741
742 mutex_lock(&kvm->arch.config_lock);
743
744 if (rcu_access_pointer(kvm->arch.mpidr_data) ||
745 atomic_read(&kvm->online_vcpus) == 1)
746 goto out;
747
748 kvm_for_each_vcpu(c, vcpu, kvm) {
749 u64 aff = kvm_vcpu_get_mpidr_aff(vcpu);
750 aff_set |= aff;
751 aff_clr &= aff;
752 }
753
754 /*
755 * A significant bit can be either 0 or 1, and will only appear in
756 * aff_set. Use aff_clr to weed out the useless stuff.
757 */
758 mask = aff_set ^ aff_clr;
759 nr_entries = BIT_ULL(hweight_long(mask));
760
761 /*
762 * Don't let userspace fool us. If we need more than a single page
763 * to describe the compressed MPIDR array, just fall back to the
764 * iterative method. Single vcpu VMs do not need this either.
765 */
766 if (struct_size(data, cmpidr_to_idx, nr_entries) <= PAGE_SIZE)
767 data = kzalloc(struct_size(data, cmpidr_to_idx, nr_entries),
768 GFP_KERNEL_ACCOUNT);
769
770 if (!data)
771 goto out;
772
773 data->mpidr_mask = mask;
774
775 kvm_for_each_vcpu(c, vcpu, kvm) {
776 u64 aff = kvm_vcpu_get_mpidr_aff(vcpu);
777 u16 index = kvm_mpidr_index(data, aff);
778
779 data->cmpidr_to_idx[index] = c;
780 }
781
782 rcu_assign_pointer(kvm->arch.mpidr_data, data);
783 out:
784 mutex_unlock(&kvm->arch.config_lock);
785 }
786
787 /*
788 * Handle both the initialisation that is being done when the vcpu is
789 * run for the first time, as well as the updates that must be
790 * performed each time we get a new thread dealing with this vcpu.
791 */
792 int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
793 {
794 struct kvm *kvm = vcpu->kvm;
795 int ret;
796
797 if (!kvm_vcpu_initialized(vcpu))
798 return -ENOEXEC;
799
800 if (!kvm_arm_vcpu_is_finalized(vcpu))
801 return -EPERM;
802
803 ret = kvm_arch_vcpu_run_map_fp(vcpu);
804 if (ret)
805 return ret;
806
807 if (likely(vcpu_has_run_once(vcpu)))
808 return 0;
809
810 kvm_init_mpidr_data(kvm);
811
812 kvm_arm_vcpu_init_debug(vcpu);
813
814 if (likely(irqchip_in_kernel(kvm))) {
815 /*
816 * Map the VGIC hardware resources before running a vcpu the
817 * first time on this VM.
818 */
819 ret = kvm_vgic_map_resources(kvm);
820 if (ret)
821 return ret;
822 }
823
824 if (vcpu_has_nv(vcpu)) {
825 ret = kvm_init_nv_sysregs(vcpu->kvm);
826 if (ret)
827 return ret;
828 }
829
830 /*
831 * This needs to happen after NV has imposed its own restrictions on
832 * the feature set
833 */
834 kvm_calculate_traps(vcpu);
835
836 ret = kvm_timer_enable(vcpu);
837 if (ret)
838 return ret;
839
840 ret = kvm_arm_pmu_v3_enable(vcpu);
841 if (ret)
842 return ret;
843
844 if (is_protected_kvm_enabled()) {
845 ret = pkvm_create_hyp_vm(kvm);
846 if (ret)
847 return ret;
848 }
849
850 if (!irqchip_in_kernel(kvm)) {
851 /*
852 * Tell the rest of the code that there are userspace irqchip
853 * VMs in the wild.
854 */
855 static_branch_inc(&userspace_irqchip_in_use);
856 }
857
858 /*
859 * Initialize traps for protected VMs.
860 * NOTE: Move to run in EL2 directly, rather than via a hypercall, once
861 * the code is in place for first run initialization at EL2.
862 */
863 if (kvm_vm_is_protected(kvm))
864 kvm_call_hyp_nvhe(__pkvm_vcpu_init_traps, vcpu);
865
866 mutex_lock(&kvm->arch.config_lock);
867 set_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags);
868 mutex_unlock(&kvm->arch.config_lock);
869
870 return ret;
871 }
872
873 bool kvm_arch_intc_initialized(struct kvm *kvm)
874 {
875 return vgic_initialized(kvm);
876 }
877
878 void kvm_arm_halt_guest(struct kvm *kvm)
879 {
880 unsigned long i;
881 struct kvm_vcpu *vcpu;
882
883 kvm_for_each_vcpu(i, vcpu, kvm)
884 vcpu->arch.pause = true;
885 kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP);
886 }
887
888 void kvm_arm_resume_guest(struct kvm *kvm)
889 {
890 unsigned long i;
891 struct kvm_vcpu *vcpu;
892
893 kvm_for_each_vcpu(i, vcpu, kvm) {
894 vcpu->arch.pause = false;
895 __kvm_vcpu_wake_up(vcpu);
896 }
897 }
898
899 static void kvm_vcpu_sleep(struct kvm_vcpu *vcpu)
900 {
901 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
902
903 rcuwait_wait_event(wait,
904 (!kvm_arm_vcpu_stopped(vcpu)) && (!vcpu->arch.pause),
905 TASK_INTERRUPTIBLE);
906
907 if (kvm_arm_vcpu_stopped(vcpu) || vcpu->arch.pause) {
908 /* Awaken to handle a signal, request we sleep again later. */
909 kvm_make_request(KVM_REQ_SLEEP, vcpu);
910 }
911
912 /*
913 * Make sure we will observe a potential reset request if we've
914 * observed a change to the power state. Pairs with the smp_wmb() in
915 * kvm_psci_vcpu_on().
916 */
917 smp_rmb();
918 }
919
920 /**
921 * kvm_vcpu_wfi - emulate Wait-For-Interrupt behavior
922 * @vcpu: The VCPU pointer
923 *
924 * Suspend execution of a vCPU until a valid wake event is detected, i.e. until
925 * the vCPU is runnable. The vCPU may or may not be scheduled out, depending
926 * on when a wake event arrives, e.g. there may already be a pending wake event.
927 */
928 void kvm_vcpu_wfi(struct kvm_vcpu *vcpu)
929 {
930 /*
931 * Sync back the state of the GIC CPU interface so that we have
932 * the latest PMR and group enables. This ensures that
933 * kvm_arch_vcpu_runnable has up-to-date data to decide whether
934 * we have pending interrupts, e.g. when determining if the
935 * vCPU should block.
936 *
937 * For the same reason, we want to tell GICv4 that we need
938 * doorbells to be signalled, should an interrupt become pending.
939 */
940 preempt_disable();
941 vcpu_set_flag(vcpu, IN_WFI);
942 kvm_vgic_put(vcpu);
943 preempt_enable();
944
945 kvm_vcpu_halt(vcpu);
946 vcpu_clear_flag(vcpu, IN_WFIT);
947
948 preempt_disable();
949 vcpu_clear_flag(vcpu, IN_WFI);
950 kvm_vgic_load(vcpu);
951 preempt_enable();
952 }
953
954 static int kvm_vcpu_suspend(struct kvm_vcpu *vcpu)
955 {
956 if (!kvm_arm_vcpu_suspended(vcpu))
957 return 1;
958
959 kvm_vcpu_wfi(vcpu);
960
961 /*
962 * The suspend state is sticky; we do not leave it until userspace
963 * explicitly marks the vCPU as runnable. Request that we suspend again
964 * later.
965 */
966 kvm_make_request(KVM_REQ_SUSPEND, vcpu);
967
968 /*
969 * Check to make sure the vCPU is actually runnable. If so, exit to
970 * userspace informing it of the wakeup condition.
971 */
972 if (kvm_arch_vcpu_runnable(vcpu)) {
973 memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event));
974 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_WAKEUP;
975 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
976 return 0;
977 }
978
979 /*
980 * Otherwise, we were unblocked to process a different event, such as a
981 * pending signal. Return 1 and allow kvm_arch_vcpu_ioctl_run() to
982 * process the event.
983 */
984 return 1;
985 }
986
987 /**
988 * check_vcpu_requests - check and handle pending vCPU requests
989 * @vcpu: the VCPU pointer
990 *
991 * Return: 1 if we should enter the guest
992 * 0 if we should exit to userspace
993 * < 0 if we should exit to userspace, where the return value indicates
994 * an error
995 */
996 static int check_vcpu_requests(struct kvm_vcpu *vcpu)
997 {
998 if (kvm_request_pending(vcpu)) {
999 if (kvm_check_request(KVM_REQ_SLEEP, vcpu))
1000 kvm_vcpu_sleep(vcpu);
1001
1002 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
1003 kvm_reset_vcpu(vcpu);
1004
1005 /*
1006 * Clear IRQ_PENDING requests that were made to guarantee
1007 * that a VCPU sees new virtual interrupts.
1008 */
1009 kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu);
1010
1011 if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu))
1012 kvm_update_stolen_time(vcpu);
1013
1014 if (kvm_check_request(KVM_REQ_RELOAD_GICv4, vcpu)) {
1015 /* The distributor enable bits were changed */
1016 preempt_disable();
1017 vgic_v4_put(vcpu);
1018 vgic_v4_load(vcpu);
1019 preempt_enable();
1020 }
1021
1022 if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu))
1023 kvm_vcpu_reload_pmu(vcpu);
1024
1025 if (kvm_check_request(KVM_REQ_RESYNC_PMU_EL0, vcpu))
1026 kvm_vcpu_pmu_restore_guest(vcpu);
1027
1028 if (kvm_check_request(KVM_REQ_SUSPEND, vcpu))
1029 return kvm_vcpu_suspend(vcpu);
1030
1031 if (kvm_dirty_ring_check_request(vcpu))
1032 return 0;
1033 }
1034
1035 return 1;
1036 }
1037
1038 static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu)
1039 {
1040 if (likely(!vcpu_mode_is_32bit(vcpu)))
1041 return false;
1042
1043 if (vcpu_has_nv(vcpu))
1044 return true;
1045
1046 return !kvm_supports_32bit_el0();
1047 }
1048
1049 /**
1050 * kvm_vcpu_exit_request - returns true if the VCPU should *not* enter the guest
1051 * @vcpu: The VCPU pointer
1052 * @ret: Pointer to write optional return code
1053 *
1054 * Returns: true if the VCPU needs to return to a preemptible + interruptible
1055 * and skip guest entry.
1056 *
1057 * This function disambiguates between two different types of exits: exits to a
1058 * preemptible + interruptible kernel context and exits to userspace. For an
1059 * exit to userspace, this function will write the return code to ret and return
1060 * true. For an exit to preemptible + interruptible kernel context (i.e. check
1061 * for pending work and re-enter), return true without writing to ret.
1062 */
1063 static bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu, int *ret)
1064 {
1065 struct kvm_run *run = vcpu->run;
1066
1067 /*
1068 * If we're using a userspace irqchip, then check if we need
1069 * to tell a userspace irqchip about timer or PMU level
1070 * changes and if so, exit to userspace (the actual level
1071 * state gets updated in kvm_timer_update_run and
1072 * kvm_pmu_update_run below).
1073 */
1074 if (static_branch_unlikely(&userspace_irqchip_in_use)) {
1075 if (kvm_timer_should_notify_user(vcpu) ||
1076 kvm_pmu_should_notify_user(vcpu)) {
1077 *ret = -EINTR;
1078 run->exit_reason = KVM_EXIT_INTR;
1079 return true;
1080 }
1081 }
1082
1083 if (unlikely(vcpu_on_unsupported_cpu(vcpu))) {
1084 run->exit_reason = KVM_EXIT_FAIL_ENTRY;
1085 run->fail_entry.hardware_entry_failure_reason = KVM_EXIT_FAIL_ENTRY_CPU_UNSUPPORTED;
1086 run->fail_entry.cpu = smp_processor_id();
1087 *ret = 0;
1088 return true;
1089 }
1090
1091 return kvm_request_pending(vcpu) ||
1092 xfer_to_guest_mode_work_pending();
1093 }
1094
1095 /*
1096 * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while
1097 * the vCPU is running.
1098 *
1099 * This must be noinstr as instrumentation may make use of RCU, and this is not
1100 * safe during the EQS.
1101 */
1102 static int noinstr kvm_arm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
1103 {
1104 int ret;
1105
1106 guest_state_enter_irqoff();
1107 ret = kvm_call_hyp_ret(__kvm_vcpu_run, vcpu);
1108 guest_state_exit_irqoff();
1109
1110 return ret;
1111 }
1112
1113 /**
1114 * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
1115 * @vcpu: The VCPU pointer
1116 *
1117 * This function is called through the VCPU_RUN ioctl called from user space. It
1118 * will execute VM code in a loop until the time slice for the process is used
1119 * or some emulation is needed from user space in which case the function will
1120 * return with return value 0 and with the kvm_run structure filled in with the
1121 * required data for the requested emulation.
1122 */
1123 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
1124 {
1125 struct kvm_run *run = vcpu->run;
1126 int ret;
1127
1128 if (run->exit_reason == KVM_EXIT_MMIO) {
1129 ret = kvm_handle_mmio_return(vcpu);
1130 if (ret <= 0)
1131 return ret;
1132 }
1133
1134 vcpu_load(vcpu);
1135
1136 if (!vcpu->wants_to_run) {
1137 ret = -EINTR;
1138 goto out;
1139 }
1140
1141 kvm_sigset_activate(vcpu);
1142
1143 ret = 1;
1144 run->exit_reason = KVM_EXIT_UNKNOWN;
1145 run->flags = 0;
1146 while (ret > 0) {
1147 /*
1148 * Check conditions before entering the guest
1149 */
1150 ret = xfer_to_guest_mode_handle_work(vcpu);
1151 if (!ret)
1152 ret = 1;
1153
1154 if (ret > 0)
1155 ret = check_vcpu_requests(vcpu);
1156
1157 /*
1158 * Preparing the interrupts to be injected also
1159 * involves poking the GIC, which must be done in a
1160 * non-preemptible context.
1161 */
1162 preempt_disable();
1163
1164 /*
1165 * The VMID allocator only tracks active VMIDs per
1166 * physical CPU, and therefore the VMID allocated may not be
1167 * preserved on VMID roll-over if the task was preempted,
1168 * making a thread's VMID inactive. So we need to call
1169 * kvm_arm_vmid_update() in non-premptible context.
1170 */
1171 if (kvm_arm_vmid_update(&vcpu->arch.hw_mmu->vmid) &&
1172 has_vhe())
1173 __load_stage2(vcpu->arch.hw_mmu,
1174 vcpu->arch.hw_mmu->arch);
1175
1176 kvm_pmu_flush_hwstate(vcpu);
1177
1178 local_irq_disable();
1179
1180 kvm_vgic_flush_hwstate(vcpu);
1181
1182 kvm_pmu_update_vcpu_events(vcpu);
1183
1184 /*
1185 * Ensure we set mode to IN_GUEST_MODE after we disable
1186 * interrupts and before the final VCPU requests check.
1187 * See the comment in kvm_vcpu_exiting_guest_mode() and
1188 * Documentation/virt/kvm/vcpu-requests.rst
1189 */
1190 smp_store_mb(vcpu->mode, IN_GUEST_MODE);
1191
1192 if (ret <= 0 || kvm_vcpu_exit_request(vcpu, &ret)) {
1193 vcpu->mode = OUTSIDE_GUEST_MODE;
1194 isb(); /* Ensure work in x_flush_hwstate is committed */
1195 kvm_pmu_sync_hwstate(vcpu);
1196 if (static_branch_unlikely(&userspace_irqchip_in_use))
1197 kvm_timer_sync_user(vcpu);
1198 kvm_vgic_sync_hwstate(vcpu);
1199 local_irq_enable();
1200 preempt_enable();
1201 continue;
1202 }
1203
1204 kvm_arm_setup_debug(vcpu);
1205 kvm_arch_vcpu_ctxflush_fp(vcpu);
1206
1207 /**************************************************************
1208 * Enter the guest
1209 */
1210 trace_kvm_entry(*vcpu_pc(vcpu));
1211 guest_timing_enter_irqoff();
1212
1213 ret = kvm_arm_vcpu_enter_exit(vcpu);
1214
1215 vcpu->mode = OUTSIDE_GUEST_MODE;
1216 vcpu->stat.exits++;
1217 /*
1218 * Back from guest
1219 *************************************************************/
1220
1221 kvm_arm_clear_debug(vcpu);
1222
1223 /*
1224 * We must sync the PMU state before the vgic state so
1225 * that the vgic can properly sample the updated state of the
1226 * interrupt line.
1227 */
1228 kvm_pmu_sync_hwstate(vcpu);
1229
1230 /*
1231 * Sync the vgic state before syncing the timer state because
1232 * the timer code needs to know if the virtual timer
1233 * interrupts are active.
1234 */
1235 kvm_vgic_sync_hwstate(vcpu);
1236
1237 /*
1238 * Sync the timer hardware state before enabling interrupts as
1239 * we don't want vtimer interrupts to race with syncing the
1240 * timer virtual interrupt state.
1241 */
1242 if (static_branch_unlikely(&userspace_irqchip_in_use))
1243 kvm_timer_sync_user(vcpu);
1244
1245 kvm_arch_vcpu_ctxsync_fp(vcpu);
1246
1247 /*
1248 * We must ensure that any pending interrupts are taken before
1249 * we exit guest timing so that timer ticks are accounted as
1250 * guest time. Transiently unmask interrupts so that any
1251 * pending interrupts are taken.
1252 *
1253 * Per ARM DDI 0487G.b section D1.13.4, an ISB (or other
1254 * context synchronization event) is necessary to ensure that
1255 * pending interrupts are taken.
1256 */
1257 if (ARM_EXCEPTION_CODE(ret) == ARM_EXCEPTION_IRQ) {
1258 local_irq_enable();
1259 isb();
1260 local_irq_disable();
1261 }
1262
1263 guest_timing_exit_irqoff();
1264
1265 local_irq_enable();
1266
1267 trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
1268
1269 /* Exit types that need handling before we can be preempted */
1270 handle_exit_early(vcpu, ret);
1271
1272 preempt_enable();
1273
1274 /*
1275 * The ARMv8 architecture doesn't give the hypervisor
1276 * a mechanism to prevent a guest from dropping to AArch32 EL0
1277 * if implemented by the CPU. If we spot the guest in such
1278 * state and that we decided it wasn't supposed to do so (like
1279 * with the asymmetric AArch32 case), return to userspace with
1280 * a fatal error.
1281 */
1282 if (vcpu_mode_is_bad_32bit(vcpu)) {
1283 /*
1284 * As we have caught the guest red-handed, decide that
1285 * it isn't fit for purpose anymore by making the vcpu
1286 * invalid. The VMM can try and fix it by issuing a
1287 * KVM_ARM_VCPU_INIT if it really wants to.
1288 */
1289 vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
1290 ret = ARM_EXCEPTION_IL;
1291 }
1292
1293 ret = handle_exit(vcpu, ret);
1294 }
1295
1296 /* Tell userspace about in-kernel device output levels */
1297 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
1298 kvm_timer_update_run(vcpu);
1299 kvm_pmu_update_run(vcpu);
1300 }
1301
1302 kvm_sigset_deactivate(vcpu);
1303
1304 out:
1305 /*
1306 * In the unlikely event that we are returning to userspace
1307 * with pending exceptions or PC adjustment, commit these
1308 * adjustments in order to give userspace a consistent view of
1309 * the vcpu state. Note that this relies on __kvm_adjust_pc()
1310 * being preempt-safe on VHE.
1311 */
1312 if (unlikely(vcpu_get_flag(vcpu, PENDING_EXCEPTION) ||
1313 vcpu_get_flag(vcpu, INCREMENT_PC)))
1314 kvm_call_hyp(__kvm_adjust_pc, vcpu);
1315
1316 vcpu_put(vcpu);
1317 return ret;
1318 }
1319
1320 static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
1321 {
1322 int bit_index;
1323 bool set;
1324 unsigned long *hcr;
1325
1326 if (number == KVM_ARM_IRQ_CPU_IRQ)
1327 bit_index = __ffs(HCR_VI);
1328 else /* KVM_ARM_IRQ_CPU_FIQ */
1329 bit_index = __ffs(HCR_VF);
1330
1331 hcr = vcpu_hcr(vcpu);
1332 if (level)
1333 set = test_and_set_bit(bit_index, hcr);
1334 else
1335 set = test_and_clear_bit(bit_index, hcr);
1336
1337 /*
1338 * If we didn't change anything, no need to wake up or kick other CPUs
1339 */
1340 if (set == level)
1341 return 0;
1342
1343 /*
1344 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
1345 * trigger a world-switch round on the running physical CPU to set the
1346 * virtual IRQ/FIQ fields in the HCR appropriately.
1347 */
1348 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
1349 kvm_vcpu_kick(vcpu);
1350
1351 return 0;
1352 }
1353
1354 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
1355 bool line_status)
1356 {
1357 u32 irq = irq_level->irq;
1358 unsigned int irq_type, vcpu_id, irq_num;
1359 struct kvm_vcpu *vcpu = NULL;
1360 bool level = irq_level->level;
1361
1362 irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
1363 vcpu_id = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
1364 vcpu_id += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);
1365 irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
1366
1367 trace_kvm_irq_line(irq_type, vcpu_id, irq_num, irq_level->level);
1368
1369 switch (irq_type) {
1370 case KVM_ARM_IRQ_TYPE_CPU:
1371 if (irqchip_in_kernel(kvm))
1372 return -ENXIO;
1373
1374 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
1375 if (!vcpu)
1376 return -EINVAL;
1377
1378 if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
1379 return -EINVAL;
1380
1381 return vcpu_interrupt_line(vcpu, irq_num, level);
1382 case KVM_ARM_IRQ_TYPE_PPI:
1383 if (!irqchip_in_kernel(kvm))
1384 return -ENXIO;
1385
1386 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
1387 if (!vcpu)
1388 return -EINVAL;
1389
1390 if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
1391 return -EINVAL;
1392
1393 return kvm_vgic_inject_irq(kvm, vcpu, irq_num, level, NULL);
1394 case KVM_ARM_IRQ_TYPE_SPI:
1395 if (!irqchip_in_kernel(kvm))
1396 return -ENXIO;
1397
1398 if (irq_num < VGIC_NR_PRIVATE_IRQS)
1399 return -EINVAL;
1400
1401 return kvm_vgic_inject_irq(kvm, NULL, irq_num, level, NULL);
1402 }
1403
1404 return -EINVAL;
1405 }
1406
1407 static unsigned long system_supported_vcpu_features(void)
1408 {
1409 unsigned long features = KVM_VCPU_VALID_FEATURES;
1410
1411 if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1))
1412 clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features);
1413
1414 if (!kvm_arm_support_pmu_v3())
1415 clear_bit(KVM_ARM_VCPU_PMU_V3, &features);
1416
1417 if (!system_supports_sve())
1418 clear_bit(KVM_ARM_VCPU_SVE, &features);
1419
1420 if (!kvm_has_full_ptr_auth()) {
1421 clear_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, &features);
1422 clear_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features);
1423 }
1424
1425 if (!cpus_have_final_cap(ARM64_HAS_NESTED_VIRT))
1426 clear_bit(KVM_ARM_VCPU_HAS_EL2, &features);
1427
1428 return features;
1429 }
1430
1431 static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu,
1432 const struct kvm_vcpu_init *init)
1433 {
1434 unsigned long features = init->features[0];
1435 int i;
1436
1437 if (features & ~KVM_VCPU_VALID_FEATURES)
1438 return -ENOENT;
1439
1440 for (i = 1; i < ARRAY_SIZE(init->features); i++) {
1441 if (init->features[i])
1442 return -ENOENT;
1443 }
1444
1445 if (features & ~system_supported_vcpu_features())
1446 return -EINVAL;
1447
1448 /*
1449 * For now make sure that both address/generic pointer authentication
1450 * features are requested by the userspace together.
1451 */
1452 if (test_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, &features) !=
1453 test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features))
1454 return -EINVAL;
1455
1456 if (!test_bit(KVM_ARM_VCPU_EL1_32BIT, &features))
1457 return 0;
1458
1459 /* MTE is incompatible with AArch32 */
1460 if (kvm_has_mte(vcpu->kvm))
1461 return -EINVAL;
1462
1463 /* NV is incompatible with AArch32 */
1464 if (test_bit(KVM_ARM_VCPU_HAS_EL2, &features))
1465 return -EINVAL;
1466
1467 return 0;
1468 }
1469
1470 static bool kvm_vcpu_init_changed(struct kvm_vcpu *vcpu,
1471 const struct kvm_vcpu_init *init)
1472 {
1473 unsigned long features = init->features[0];
1474
1475 return !bitmap_equal(vcpu->kvm->arch.vcpu_features, &features,
1476 KVM_VCPU_MAX_FEATURES);
1477 }
1478
1479 static int kvm_setup_vcpu(struct kvm_vcpu *vcpu)
1480 {
1481 struct kvm *kvm = vcpu->kvm;
1482 int ret = 0;
1483
1484 /*
1485 * When the vCPU has a PMU, but no PMU is set for the guest
1486 * yet, set the default one.
1487 */
1488 if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu)
1489 ret = kvm_arm_set_default_pmu(kvm);
1490
1491 /* Prepare for nested if required */
1492 if (!ret && vcpu_has_nv(vcpu))
1493 ret = kvm_vcpu_init_nested(vcpu);
1494
1495 return ret;
1496 }
1497
1498 static int __kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
1499 const struct kvm_vcpu_init *init)
1500 {
1501 unsigned long features = init->features[0];
1502 struct kvm *kvm = vcpu->kvm;
1503 int ret = -EINVAL;
1504
1505 mutex_lock(&kvm->arch.config_lock);
1506
1507 if (test_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags) &&
1508 kvm_vcpu_init_changed(vcpu, init))
1509 goto out_unlock;
1510
1511 bitmap_copy(kvm->arch.vcpu_features, &features, KVM_VCPU_MAX_FEATURES);
1512
1513 ret = kvm_setup_vcpu(vcpu);
1514 if (ret)
1515 goto out_unlock;
1516
1517 /* Now we know what it is, we can reset it. */
1518 kvm_reset_vcpu(vcpu);
1519
1520 set_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags);
1521 vcpu_set_flag(vcpu, VCPU_INITIALIZED);
1522 ret = 0;
1523 out_unlock:
1524 mutex_unlock(&kvm->arch.config_lock);
1525 return ret;
1526 }
1527
1528 static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
1529 const struct kvm_vcpu_init *init)
1530 {
1531 int ret;
1532
1533 if (init->target != KVM_ARM_TARGET_GENERIC_V8 &&
1534 init->target != kvm_target_cpu())
1535 return -EINVAL;
1536
1537 ret = kvm_vcpu_init_check_features(vcpu, init);
1538 if (ret)
1539 return ret;
1540
1541 if (!kvm_vcpu_initialized(vcpu))
1542 return __kvm_vcpu_set_target(vcpu, init);
1543
1544 if (kvm_vcpu_init_changed(vcpu, init))
1545 return -EINVAL;
1546
1547 kvm_reset_vcpu(vcpu);
1548 return 0;
1549 }
1550
1551 static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
1552 struct kvm_vcpu_init *init)
1553 {
1554 bool power_off = false;
1555 int ret;
1556
1557 /*
1558 * Treat the power-off vCPU feature as ephemeral. Clear the bit to avoid
1559 * reflecting it in the finalized feature set, thus limiting its scope
1560 * to a single KVM_ARM_VCPU_INIT call.
1561 */
1562 if (init->features[0] & BIT(KVM_ARM_VCPU_POWER_OFF)) {
1563 init->features[0] &= ~BIT(KVM_ARM_VCPU_POWER_OFF);
1564 power_off = true;
1565 }
1566
1567 ret = kvm_vcpu_set_target(vcpu, init);
1568 if (ret)
1569 return ret;
1570
1571 /*
1572 * Ensure a rebooted VM will fault in RAM pages and detect if the
1573 * guest MMU is turned off and flush the caches as needed.
1574 *
1575 * S2FWB enforces all memory accesses to RAM being cacheable,
1576 * ensuring that the data side is always coherent. We still
1577 * need to invalidate the I-cache though, as FWB does *not*
1578 * imply CTR_EL0.DIC.
1579 */
1580 if (vcpu_has_run_once(vcpu)) {
1581 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
1582 stage2_unmap_vm(vcpu->kvm);
1583 else
1584 icache_inval_all_pou();
1585 }
1586
1587 vcpu_reset_hcr(vcpu);
1588 vcpu->arch.cptr_el2 = kvm_get_reset_cptr_el2(vcpu);
1589
1590 /*
1591 * Handle the "start in power-off" case.
1592 */
1593 spin_lock(&vcpu->arch.mp_state_lock);
1594
1595 if (power_off)
1596 __kvm_arm_vcpu_power_off(vcpu);
1597 else
1598 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE);
1599
1600 spin_unlock(&vcpu->arch.mp_state_lock);
1601
1602 return 0;
1603 }
1604
1605 static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu,
1606 struct kvm_device_attr *attr)
1607 {
1608 int ret = -ENXIO;
1609
1610 switch (attr->group) {
1611 default:
1612 ret = kvm_arm_vcpu_arch_set_attr(vcpu, attr);
1613 break;
1614 }
1615
1616 return ret;
1617 }
1618
1619 static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu,
1620 struct kvm_device_attr *attr)
1621 {
1622 int ret = -ENXIO;
1623
1624 switch (attr->group) {
1625 default:
1626 ret = kvm_arm_vcpu_arch_get_attr(vcpu, attr);
1627 break;
1628 }
1629
1630 return ret;
1631 }
1632
1633 static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu,
1634 struct kvm_device_attr *attr)
1635 {
1636 int ret = -ENXIO;
1637
1638 switch (attr->group) {
1639 default:
1640 ret = kvm_arm_vcpu_arch_has_attr(vcpu, attr);
1641 break;
1642 }
1643
1644 return ret;
1645 }
1646
1647 static int kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
1648 struct kvm_vcpu_events *events)
1649 {
1650 memset(events, 0, sizeof(*events));
1651
1652 return __kvm_arm_vcpu_get_events(vcpu, events);
1653 }
1654
1655 static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
1656 struct kvm_vcpu_events *events)
1657 {
1658 int i;
1659
1660 /* check whether the reserved field is zero */
1661 for (i = 0; i < ARRAY_SIZE(events->reserved); i++)
1662 if (events->reserved[i])
1663 return -EINVAL;
1664
1665 /* check whether the pad field is zero */
1666 for (i = 0; i < ARRAY_SIZE(events->exception.pad); i++)
1667 if (events->exception.pad[i])
1668 return -EINVAL;
1669
1670 return __kvm_arm_vcpu_set_events(vcpu, events);
1671 }
1672
1673 long kvm_arch_vcpu_ioctl(struct file *filp,
1674 unsigned int ioctl, unsigned long arg)
1675 {
1676 struct kvm_vcpu *vcpu = filp->private_data;
1677 void __user *argp = (void __user *)arg;
1678 struct kvm_device_attr attr;
1679 long r;
1680
1681 switch (ioctl) {
1682 case KVM_ARM_VCPU_INIT: {
1683 struct kvm_vcpu_init init;
1684
1685 r = -EFAULT;
1686 if (copy_from_user(&init, argp, sizeof(init)))
1687 break;
1688
1689 r = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init);
1690 break;
1691 }
1692 case KVM_SET_ONE_REG:
1693 case KVM_GET_ONE_REG: {
1694 struct kvm_one_reg reg;
1695
1696 r = -ENOEXEC;
1697 if (unlikely(!kvm_vcpu_initialized(vcpu)))
1698 break;
1699
1700 r = -EFAULT;
1701 if (copy_from_user(®, argp, sizeof(reg)))
1702 break;
1703
1704 /*
1705 * We could owe a reset due to PSCI. Handle the pending reset
1706 * here to ensure userspace register accesses are ordered after
1707 * the reset.
1708 */
1709 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
1710 kvm_reset_vcpu(vcpu);
1711
1712 if (ioctl == KVM_SET_ONE_REG)
1713 r = kvm_arm_set_reg(vcpu, ®);
1714 else
1715 r = kvm_arm_get_reg(vcpu, ®);
1716 break;
1717 }
1718 case KVM_GET_REG_LIST: {
1719 struct kvm_reg_list __user *user_list = argp;
1720 struct kvm_reg_list reg_list;
1721 unsigned n;
1722
1723 r = -ENOEXEC;
1724 if (unlikely(!kvm_vcpu_initialized(vcpu)))
1725 break;
1726
1727 r = -EPERM;
1728 if (!kvm_arm_vcpu_is_finalized(vcpu))
1729 break;
1730
1731 r = -EFAULT;
1732 if (copy_from_user(®_list, user_list, sizeof(reg_list)))
1733 break;
1734 n = reg_list.n;
1735 reg_list.n = kvm_arm_num_regs(vcpu);
1736 if (copy_to_user(user_list, ®_list, sizeof(reg_list)))
1737 break;
1738 r = -E2BIG;
1739 if (n < reg_list.n)
1740 break;
1741 r = kvm_arm_copy_reg_indices(vcpu, user_list->reg);
1742 break;
1743 }
1744 case KVM_SET_DEVICE_ATTR: {
1745 r = -EFAULT;
1746 if (copy_from_user(&attr, argp, sizeof(attr)))
1747 break;
1748 r = kvm_arm_vcpu_set_attr(vcpu, &attr);
1749 break;
1750 }
1751 case KVM_GET_DEVICE_ATTR: {
1752 r = -EFAULT;
1753 if (copy_from_user(&attr, argp, sizeof(attr)))
1754 break;
1755 r = kvm_arm_vcpu_get_attr(vcpu, &attr);
1756 break;
1757 }
1758 case KVM_HAS_DEVICE_ATTR: {
1759 r = -EFAULT;
1760 if (copy_from_user(&attr, argp, sizeof(attr)))
1761 break;
1762 r = kvm_arm_vcpu_has_attr(vcpu, &attr);
1763 break;
1764 }
1765 case KVM_GET_VCPU_EVENTS: {
1766 struct kvm_vcpu_events events;
1767
1768 if (kvm_arm_vcpu_get_events(vcpu, &events))
1769 return -EINVAL;
1770
1771 if (copy_to_user(argp, &events, sizeof(events)))
1772 return -EFAULT;
1773
1774 return 0;
1775 }
1776 case KVM_SET_VCPU_EVENTS: {
1777 struct kvm_vcpu_events events;
1778
1779 if (copy_from_user(&events, argp, sizeof(events)))
1780 return -EFAULT;
1781
1782 return kvm_arm_vcpu_set_events(vcpu, &events);
1783 }
1784 case KVM_ARM_VCPU_FINALIZE: {
1785 int what;
1786
1787 if (!kvm_vcpu_initialized(vcpu))
1788 return -ENOEXEC;
1789
1790 if (get_user(what, (const int __user *)argp))
1791 return -EFAULT;
1792
1793 return kvm_arm_vcpu_finalize(vcpu, what);
1794 }
1795 default:
1796 r = -EINVAL;
1797 }
1798
1799 return r;
1800 }
1801
1802 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
1803 {
1804
1805 }
1806
1807 static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
1808 struct kvm_arm_device_addr *dev_addr)
1809 {
1810 switch (FIELD_GET(KVM_ARM_DEVICE_ID_MASK, dev_addr->id)) {
1811 case KVM_ARM_DEVICE_VGIC_V2:
1812 if (!vgic_present)
1813 return -ENXIO;
1814 return kvm_set_legacy_vgic_v2_addr(kvm, dev_addr);
1815 default:
1816 return -ENODEV;
1817 }
1818 }
1819
1820 static int kvm_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1821 {
1822 switch (attr->group) {
1823 case KVM_ARM_VM_SMCCC_CTRL:
1824 return kvm_vm_smccc_has_attr(kvm, attr);
1825 default:
1826 return -ENXIO;
1827 }
1828 }
1829
1830 static int kvm_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1831 {
1832 switch (attr->group) {
1833 case KVM_ARM_VM_SMCCC_CTRL:
1834 return kvm_vm_smccc_set_attr(kvm, attr);
1835 default:
1836 return -ENXIO;
1837 }
1838 }
1839
1840 int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
1841 {
1842 struct kvm *kvm = filp->private_data;
1843 void __user *argp = (void __user *)arg;
1844 struct kvm_device_attr attr;
1845
1846 switch (ioctl) {
1847 case KVM_CREATE_IRQCHIP: {
1848 int ret;
1849 if (!vgic_present)
1850 return -ENXIO;
1851 mutex_lock(&kvm->lock);
1852 ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
1853 mutex_unlock(&kvm->lock);
1854 return ret;
1855 }
1856 case KVM_ARM_SET_DEVICE_ADDR: {
1857 struct kvm_arm_device_addr dev_addr;
1858
1859 if (copy_from_user(&dev_addr, argp, sizeof(dev_addr)))
1860 return -EFAULT;
1861 return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
1862 }
1863 case KVM_ARM_PREFERRED_TARGET: {
1864 struct kvm_vcpu_init init = {
1865 .target = KVM_ARM_TARGET_GENERIC_V8,
1866 };
1867
1868 if (copy_to_user(argp, &init, sizeof(init)))
1869 return -EFAULT;
1870
1871 return 0;
1872 }
1873 case KVM_ARM_MTE_COPY_TAGS: {
1874 struct kvm_arm_copy_mte_tags copy_tags;
1875
1876 if (copy_from_user(©_tags, argp, sizeof(copy_tags)))
1877 return -EFAULT;
1878 return kvm_vm_ioctl_mte_copy_tags(kvm, ©_tags);
1879 }
1880 case KVM_ARM_SET_COUNTER_OFFSET: {
1881 struct kvm_arm_counter_offset offset;
1882
1883 if (copy_from_user(&offset, argp, sizeof(offset)))
1884 return -EFAULT;
1885 return kvm_vm_ioctl_set_counter_offset(kvm, &offset);
1886 }
1887 case KVM_HAS_DEVICE_ATTR: {
1888 if (copy_from_user(&attr, argp, sizeof(attr)))
1889 return -EFAULT;
1890
1891 return kvm_vm_has_attr(kvm, &attr);
1892 }
1893 case KVM_SET_DEVICE_ATTR: {
1894 if (copy_from_user(&attr, argp, sizeof(attr)))
1895 return -EFAULT;
1896
1897 return kvm_vm_set_attr(kvm, &attr);
1898 }
1899 case KVM_ARM_GET_REG_WRITABLE_MASKS: {
1900 struct reg_mask_range range;
1901
1902 if (copy_from_user(&range, argp, sizeof(range)))
1903 return -EFAULT;
1904 return kvm_vm_ioctl_get_reg_writable_masks(kvm, &range);
1905 }
1906 default:
1907 return -EINVAL;
1908 }
1909 }
1910
1911 /* unlocks vcpus from @vcpu_lock_idx and smaller */
1912 static void unlock_vcpus(struct kvm *kvm, int vcpu_lock_idx)
1913 {
1914 struct kvm_vcpu *tmp_vcpu;
1915
1916 for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
1917 tmp_vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx);
1918 mutex_unlock(&tmp_vcpu->mutex);
1919 }
1920 }
1921
1922 void unlock_all_vcpus(struct kvm *kvm)
1923 {
1924 lockdep_assert_held(&kvm->lock);
1925
1926 unlock_vcpus(kvm, atomic_read(&kvm->online_vcpus) - 1);
1927 }
1928
1929 /* Returns true if all vcpus were locked, false otherwise */
1930 bool lock_all_vcpus(struct kvm *kvm)
1931 {
1932 struct kvm_vcpu *tmp_vcpu;
1933 unsigned long c;
1934
1935 lockdep_assert_held(&kvm->lock);
1936
1937 /*
1938 * Any time a vcpu is in an ioctl (including running), the
1939 * core KVM code tries to grab the vcpu->mutex.
1940 *
1941 * By grabbing the vcpu->mutex of all VCPUs we ensure that no
1942 * other VCPUs can fiddle with the state while we access it.
1943 */
1944 kvm_for_each_vcpu(c, tmp_vcpu, kvm) {
1945 if (!mutex_trylock(&tmp_vcpu->mutex)) {
1946 unlock_vcpus(kvm, c - 1);
1947 return false;
1948 }
1949 }
1950
1951 return true;
1952 }
1953
1954 static unsigned long nvhe_percpu_size(void)
1955 {
1956 return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) -
1957 (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_start);
1958 }
1959
1960 static unsigned long nvhe_percpu_order(void)
1961 {
1962 unsigned long size = nvhe_percpu_size();
1963
1964 return size ? get_order(size) : 0;
1965 }
1966
1967 static size_t pkvm_host_sve_state_order(void)
1968 {
1969 return get_order(pkvm_host_sve_state_size());
1970 }
1971
1972 /* A lookup table holding the hypervisor VA for each vector slot */
1973 static void *hyp_spectre_vector_selector[BP_HARDEN_EL2_SLOTS];
1974
1975 static void kvm_init_vector_slot(void *base, enum arm64_hyp_spectre_vector slot)
1976 {
1977 hyp_spectre_vector_selector[slot] = __kvm_vector_slot2addr(base, slot);
1978 }
1979
1980 static int kvm_init_vector_slots(void)
1981 {
1982 int err;
1983 void *base;
1984
1985 base = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector));
1986 kvm_init_vector_slot(base, HYP_VECTOR_DIRECT);
1987
1988 base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs));
1989 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT);
1990
1991 if (kvm_system_needs_idmapped_vectors() &&
1992 !is_protected_kvm_enabled()) {
1993 err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs),
1994 __BP_HARDEN_HYP_VECS_SZ, &base);
1995 if (err)
1996 return err;
1997 }
1998
1999 kvm_init_vector_slot(base, HYP_VECTOR_INDIRECT);
2000 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_INDIRECT);
2001 return 0;
2002 }
2003
2004 static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits)
2005 {
2006 struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
2007 u64 mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
2008 unsigned long tcr;
2009
2010 /*
2011 * Calculate the raw per-cpu offset without a translation from the
2012 * kernel's mapping to the linear mapping, and store it in tpidr_el2
2013 * so that we can use adr_l to access per-cpu variables in EL2.
2014 * Also drop the KASAN tag which gets in the way...
2015 */
2016 params->tpidr_el2 = (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym(__per_cpu_start, cpu)) -
2017 (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start));
2018
2019 params->mair_el2 = read_sysreg(mair_el1);
2020
2021 tcr = read_sysreg(tcr_el1);
2022 if (cpus_have_final_cap(ARM64_KVM_HVHE)) {
2023 tcr |= TCR_EPD1_MASK;
2024 } else {
2025 tcr &= TCR_EL2_MASK;
2026 tcr |= TCR_EL2_RES1;
2027 }
2028 tcr &= ~TCR_T0SZ_MASK;
2029 tcr |= TCR_T0SZ(hyp_va_bits);
2030 tcr &= ~TCR_EL2_PS_MASK;
2031 tcr |= FIELD_PREP(TCR_EL2_PS_MASK, kvm_get_parange(mmfr0));
2032 if (kvm_lpa2_is_enabled())
2033 tcr |= TCR_EL2_DS;
2034 params->tcr_el2 = tcr;
2035
2036 params->pgd_pa = kvm_mmu_get_httbr();
2037 if (is_protected_kvm_enabled())
2038 params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS;
2039 else
2040 params->hcr_el2 = HCR_HOST_NVHE_FLAGS;
2041 if (cpus_have_final_cap(ARM64_KVM_HVHE))
2042 params->hcr_el2 |= HCR_E2H;
2043 params->vttbr = params->vtcr = 0;
2044
2045 /*
2046 * Flush the init params from the data cache because the struct will
2047 * be read while the MMU is off.
2048 */
2049 kvm_flush_dcache_to_poc(params, sizeof(*params));
2050 }
2051
2052 static void hyp_install_host_vector(void)
2053 {
2054 struct kvm_nvhe_init_params *params;
2055 struct arm_smccc_res res;
2056
2057 /* Switch from the HYP stub to our own HYP init vector */
2058 __hyp_set_vectors(kvm_get_idmap_vector());
2059
2060 /*
2061 * Call initialization code, and switch to the full blown HYP code.
2062 * If the cpucaps haven't been finalized yet, something has gone very
2063 * wrong, and hyp will crash and burn when it uses any
2064 * cpus_have_*_cap() wrapper.
2065 */
2066 BUG_ON(!system_capabilities_finalized());
2067 params = this_cpu_ptr_nvhe_sym(kvm_init_params);
2068 arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init), virt_to_phys(params), &res);
2069 WARN_ON(res.a0 != SMCCC_RET_SUCCESS);
2070 }
2071
2072 static void cpu_init_hyp_mode(void)
2073 {
2074 hyp_install_host_vector();
2075
2076 /*
2077 * Disabling SSBD on a non-VHE system requires us to enable SSBS
2078 * at EL2.
2079 */
2080 if (this_cpu_has_cap(ARM64_SSBS) &&
2081 arm64_get_spectre_v4_state() == SPECTRE_VULNERABLE) {
2082 kvm_call_hyp_nvhe(__kvm_enable_ssbs);
2083 }
2084 }
2085
2086 static void cpu_hyp_reset(void)
2087 {
2088 if (!is_kernel_in_hyp_mode())
2089 __hyp_reset_vectors();
2090 }
2091
2092 /*
2093 * EL2 vectors can be mapped and rerouted in a number of ways,
2094 * depending on the kernel configuration and CPU present:
2095 *
2096 * - If the CPU is affected by Spectre-v2, the hardening sequence is
2097 * placed in one of the vector slots, which is executed before jumping
2098 * to the real vectors.
2099 *
2100 * - If the CPU also has the ARM64_SPECTRE_V3A cap, the slot
2101 * containing the hardening sequence is mapped next to the idmap page,
2102 * and executed before jumping to the real vectors.
2103 *
2104 * - If the CPU only has the ARM64_SPECTRE_V3A cap, then an
2105 * empty slot is selected, mapped next to the idmap page, and
2106 * executed before jumping to the real vectors.
2107 *
2108 * Note that ARM64_SPECTRE_V3A is somewhat incompatible with
2109 * VHE, as we don't have hypervisor-specific mappings. If the system
2110 * is VHE and yet selects this capability, it will be ignored.
2111 */
2112 static void cpu_set_hyp_vector(void)
2113 {
2114 struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
2115 void *vector = hyp_spectre_vector_selector[data->slot];
2116
2117 if (!is_protected_kvm_enabled())
2118 *this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector;
2119 else
2120 kvm_call_hyp_nvhe(__pkvm_cpu_set_vector, data->slot);
2121 }
2122
2123 static void cpu_hyp_init_context(void)
2124 {
2125 kvm_init_host_cpu_context(host_data_ptr(host_ctxt));
2126
2127 if (!is_kernel_in_hyp_mode())
2128 cpu_init_hyp_mode();
2129 }
2130
2131 static void cpu_hyp_init_features(void)
2132 {
2133 cpu_set_hyp_vector();
2134 kvm_arm_init_debug();
2135
2136 if (is_kernel_in_hyp_mode())
2137 kvm_timer_init_vhe();
2138
2139 if (vgic_present)
2140 kvm_vgic_init_cpu_hardware();
2141 }
2142
2143 static void cpu_hyp_reinit(void)
2144 {
2145 cpu_hyp_reset();
2146 cpu_hyp_init_context();
2147 cpu_hyp_init_features();
2148 }
2149
2150 static void cpu_hyp_init(void *discard)
2151 {
2152 if (!__this_cpu_read(kvm_hyp_initialized)) {
2153 cpu_hyp_reinit();
2154 __this_cpu_write(kvm_hyp_initialized, 1);
2155 }
2156 }
2157
2158 static void cpu_hyp_uninit(void *discard)
2159 {
2160 if (__this_cpu_read(kvm_hyp_initialized)) {
2161 cpu_hyp_reset();
2162 __this_cpu_write(kvm_hyp_initialized, 0);
2163 }
2164 }
2165
2166 int kvm_arch_hardware_enable(void)
2167 {
2168 /*
2169 * Most calls to this function are made with migration
2170 * disabled, but not with preemption disabled. The former is
2171 * enough to ensure correctness, but most of the helpers
2172 * expect the later and will throw a tantrum otherwise.
2173 */
2174 preempt_disable();
2175
2176 cpu_hyp_init(NULL);
2177
2178 kvm_vgic_cpu_up();
2179 kvm_timer_cpu_up();
2180
2181 preempt_enable();
2182
2183 return 0;
2184 }
2185
2186 void kvm_arch_hardware_disable(void)
2187 {
2188 kvm_timer_cpu_down();
2189 kvm_vgic_cpu_down();
2190
2191 if (!is_protected_kvm_enabled())
2192 cpu_hyp_uninit(NULL);
2193 }
2194
2195 #ifdef CONFIG_CPU_PM
2196 static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
2197 unsigned long cmd,
2198 void *v)
2199 {
2200 /*
2201 * kvm_hyp_initialized is left with its old value over
2202 * PM_ENTER->PM_EXIT. It is used to indicate PM_EXIT should
2203 * re-enable hyp.
2204 */
2205 switch (cmd) {
2206 case CPU_PM_ENTER:
2207 if (__this_cpu_read(kvm_hyp_initialized))
2208 /*
2209 * don't update kvm_hyp_initialized here
2210 * so that the hyp will be re-enabled
2211 * when we resume. See below.
2212 */
2213 cpu_hyp_reset();
2214
2215 return NOTIFY_OK;
2216 case CPU_PM_ENTER_FAILED:
2217 case CPU_PM_EXIT:
2218 if (__this_cpu_read(kvm_hyp_initialized))
2219 /* The hyp was enabled before suspend. */
2220 cpu_hyp_reinit();
2221
2222 return NOTIFY_OK;
2223
2224 default:
2225 return NOTIFY_DONE;
2226 }
2227 }
2228
2229 static struct notifier_block hyp_init_cpu_pm_nb = {
2230 .notifier_call = hyp_init_cpu_pm_notifier,
2231 };
2232
2233 static void __init hyp_cpu_pm_init(void)
2234 {
2235 if (!is_protected_kvm_enabled())
2236 cpu_pm_register_notifier(&hyp_init_cpu_pm_nb);
2237 }
2238 static void __init hyp_cpu_pm_exit(void)
2239 {
2240 if (!is_protected_kvm_enabled())
2241 cpu_pm_unregister_notifier(&hyp_init_cpu_pm_nb);
2242 }
2243 #else
2244 static inline void __init hyp_cpu_pm_init(void)
2245 {
2246 }
2247 static inline void __init hyp_cpu_pm_exit(void)
2248 {
2249 }
2250 #endif
2251
2252 static void __init init_cpu_logical_map(void)
2253 {
2254 unsigned int cpu;
2255
2256 /*
2257 * Copy the MPIDR <-> logical CPU ID mapping to hyp.
2258 * Only copy the set of online CPUs whose features have been checked
2259 * against the finalized system capabilities. The hypervisor will not
2260 * allow any other CPUs from the `possible` set to boot.
2261 */
2262 for_each_online_cpu(cpu)
2263 hyp_cpu_logical_map[cpu] = cpu_logical_map(cpu);
2264 }
2265
2266 #define init_psci_0_1_impl_state(config, what) \
2267 config.psci_0_1_ ## what ## _implemented = psci_ops.what
2268
2269 static bool __init init_psci_relay(void)
2270 {
2271 /*
2272 * If PSCI has not been initialized, protected KVM cannot install
2273 * itself on newly booted CPUs.
2274 */
2275 if (!psci_ops.get_version) {
2276 kvm_err("Cannot initialize protected mode without PSCI\n");
2277 return false;
2278 }
2279
2280 kvm_host_psci_config.version = psci_ops.get_version();
2281 kvm_host_psci_config.smccc_version = arm_smccc_get_version();
2282
2283 if (kvm_host_psci_config.version == PSCI_VERSION(0, 1)) {
2284 kvm_host_psci_config.function_ids_0_1 = get_psci_0_1_function_ids();
2285 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_suspend);
2286 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_on);
2287 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_off);
2288 init_psci_0_1_impl_state(kvm_host_psci_config, migrate);
2289 }
2290 return true;
2291 }
2292
2293 static int __init init_subsystems(void)
2294 {
2295 int err = 0;
2296
2297 /*
2298 * Enable hardware so that subsystem initialisation can access EL2.
2299 */
2300 on_each_cpu(cpu_hyp_init, NULL, 1);
2301
2302 /*
2303 * Register CPU lower-power notifier
2304 */
2305 hyp_cpu_pm_init();
2306
2307 /*
2308 * Init HYP view of VGIC
2309 */
2310 err = kvm_vgic_hyp_init();
2311 switch (err) {
2312 case 0:
2313 vgic_present = true;
2314 break;
2315 case -ENODEV:
2316 case -ENXIO:
2317 vgic_present = false;
2318 err = 0;
2319 break;
2320 default:
2321 goto out;
2322 }
2323
2324 /*
2325 * Init HYP architected timer support
2326 */
2327 err = kvm_timer_hyp_init(vgic_present);
2328 if (err)
2329 goto out;
2330
2331 kvm_register_perf_callbacks(NULL);
2332
2333 out:
2334 if (err)
2335 hyp_cpu_pm_exit();
2336
2337 if (err || !is_protected_kvm_enabled())
2338 on_each_cpu(cpu_hyp_uninit, NULL, 1);
2339
2340 return err;
2341 }
2342
2343 static void __init teardown_subsystems(void)
2344 {
2345 kvm_unregister_perf_callbacks();
2346 hyp_cpu_pm_exit();
2347 }
2348
2349 static void __init teardown_hyp_mode(void)
2350 {
2351 bool free_sve = system_supports_sve() && is_protected_kvm_enabled();
2352 int cpu;
2353
2354 free_hyp_pgds();
2355 for_each_possible_cpu(cpu) {
2356 free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
2357 free_pages(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu], nvhe_percpu_order());
2358
2359 if (free_sve) {
2360 struct cpu_sve_state *sve_state;
2361
2362 sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state;
2363 free_pages((unsigned long) sve_state, pkvm_host_sve_state_order());
2364 }
2365 }
2366 }
2367
2368 static int __init do_pkvm_init(u32 hyp_va_bits)
2369 {
2370 void *per_cpu_base = kvm_ksym_ref(kvm_nvhe_sym(kvm_arm_hyp_percpu_base));
2371 int ret;
2372
2373 preempt_disable();
2374 cpu_hyp_init_context();
2375 ret = kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size,
2376 num_possible_cpus(), kern_hyp_va(per_cpu_base),
2377 hyp_va_bits);
2378 cpu_hyp_init_features();
2379
2380 /*
2381 * The stub hypercalls are now disabled, so set our local flag to
2382 * prevent a later re-init attempt in kvm_arch_hardware_enable().
2383 */
2384 __this_cpu_write(kvm_hyp_initialized, 1);
2385 preempt_enable();
2386
2387 return ret;
2388 }
2389
2390 static u64 get_hyp_id_aa64pfr0_el1(void)
2391 {
2392 /*
2393 * Track whether the system isn't affected by spectre/meltdown in the
2394 * hypervisor's view of id_aa64pfr0_el1, used for protected VMs.
2395 * Although this is per-CPU, we make it global for simplicity, e.g., not
2396 * to have to worry about vcpu migration.
2397 *
2398 * Unlike for non-protected VMs, userspace cannot override this for
2399 * protected VMs.
2400 */
2401 u64 val = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
2402
2403 val &= ~(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2) |
2404 ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3));
2405
2406 val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2),
2407 arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED);
2408 val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3),
2409 arm64_get_meltdown_state() == SPECTRE_UNAFFECTED);
2410
2411 return val;
2412 }
2413
2414 static void kvm_hyp_init_symbols(void)
2415 {
2416 kvm_nvhe_sym(id_aa64pfr0_el1_sys_val) = get_hyp_id_aa64pfr0_el1();
2417 kvm_nvhe_sym(id_aa64pfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1);
2418 kvm_nvhe_sym(id_aa64isar0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR0_EL1);
2419 kvm_nvhe_sym(id_aa64isar1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1);
2420 kvm_nvhe_sym(id_aa64isar2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
2421 kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
2422 kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
2423 kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR2_EL1);
2424 kvm_nvhe_sym(id_aa64smfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64SMFR0_EL1);
2425 kvm_nvhe_sym(__icache_flags) = __icache_flags;
2426 kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits;
2427 }
2428
2429 static int __init kvm_hyp_init_protection(u32 hyp_va_bits)
2430 {
2431 void *addr = phys_to_virt(hyp_mem_base);
2432 int ret;
2433
2434 ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP);
2435 if (ret)
2436 return ret;
2437
2438 ret = do_pkvm_init(hyp_va_bits);
2439 if (ret)
2440 return ret;
2441
2442 free_hyp_pgds();
2443
2444 return 0;
2445 }
2446
2447 static int init_pkvm_host_sve_state(void)
2448 {
2449 int cpu;
2450
2451 if (!system_supports_sve())
2452 return 0;
2453
2454 /* Allocate pages for host sve state in protected mode. */
2455 for_each_possible_cpu(cpu) {
2456 struct page *page = alloc_pages(GFP_KERNEL, pkvm_host_sve_state_order());
2457
2458 if (!page)
2459 return -ENOMEM;
2460
2461 per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = page_address(page);
2462 }
2463
2464 /*
2465 * Don't map the pages in hyp since these are only used in protected
2466 * mode, which will (re)create its own mapping when initialized.
2467 */
2468
2469 return 0;
2470 }
2471
2472 /*
2473 * Finalizes the initialization of hyp mode, once everything else is initialized
2474 * and the initialziation process cannot fail.
2475 */
2476 static void finalize_init_hyp_mode(void)
2477 {
2478 int cpu;
2479
2480 if (system_supports_sve() && is_protected_kvm_enabled()) {
2481 for_each_possible_cpu(cpu) {
2482 struct cpu_sve_state *sve_state;
2483
2484 sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state;
2485 per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state =
2486 kern_hyp_va(sve_state);
2487 }
2488 } else {
2489 for_each_possible_cpu(cpu) {
2490 struct user_fpsimd_state *fpsimd_state;
2491
2492 fpsimd_state = &per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->host_ctxt.fp_regs;
2493 per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->fpsimd_state =
2494 kern_hyp_va(fpsimd_state);
2495 }
2496 }
2497 }
2498
2499 static void pkvm_hyp_init_ptrauth(void)
2500 {
2501 struct kvm_cpu_context *hyp_ctxt;
2502 int cpu;
2503
2504 for_each_possible_cpu(cpu) {
2505 hyp_ctxt = per_cpu_ptr_nvhe_sym(kvm_hyp_ctxt, cpu);
2506 hyp_ctxt->sys_regs[APIAKEYLO_EL1] = get_random_long();
2507 hyp_ctxt->sys_regs[APIAKEYHI_EL1] = get_random_long();
2508 hyp_ctxt->sys_regs[APIBKEYLO_EL1] = get_random_long();
2509 hyp_ctxt->sys_regs[APIBKEYHI_EL1] = get_random_long();
2510 hyp_ctxt->sys_regs[APDAKEYLO_EL1] = get_random_long();
2511 hyp_ctxt->sys_regs[APDAKEYHI_EL1] = get_random_long();
2512 hyp_ctxt->sys_regs[APDBKEYLO_EL1] = get_random_long();
2513 hyp_ctxt->sys_regs[APDBKEYHI_EL1] = get_random_long();
2514 hyp_ctxt->sys_regs[APGAKEYLO_EL1] = get_random_long();
2515 hyp_ctxt->sys_regs[APGAKEYHI_EL1] = get_random_long();
2516 }
2517 }
2518
2519 /* Inits Hyp-mode on all online CPUs */
2520 static int __init init_hyp_mode(void)
2521 {
2522 u32 hyp_va_bits;
2523 int cpu;
2524 int err = -ENOMEM;
2525
2526 /*
2527 * The protected Hyp-mode cannot be initialized if the memory pool
2528 * allocation has failed.
2529 */
2530 if (is_protected_kvm_enabled() && !hyp_mem_base)
2531 goto out_err;
2532
2533 /*
2534 * Allocate Hyp PGD and setup Hyp identity mapping
2535 */
2536 err = kvm_mmu_init(&hyp_va_bits);
2537 if (err)
2538 goto out_err;
2539
2540 /*
2541 * Allocate stack pages for Hypervisor-mode
2542 */
2543 for_each_possible_cpu(cpu) {
2544 unsigned long stack_page;
2545
2546 stack_page = __get_free_page(GFP_KERNEL);
2547 if (!stack_page) {
2548 err = -ENOMEM;
2549 goto out_err;
2550 }
2551
2552 per_cpu(kvm_arm_hyp_stack_page, cpu) = stack_page;
2553 }
2554
2555 /*
2556 * Allocate and initialize pages for Hypervisor-mode percpu regions.
2557 */
2558 for_each_possible_cpu(cpu) {
2559 struct page *page;
2560 void *page_addr;
2561
2562 page = alloc_pages(GFP_KERNEL, nvhe_percpu_order());
2563 if (!page) {
2564 err = -ENOMEM;
2565 goto out_err;
2566 }
2567
2568 page_addr = page_address(page);
2569 memcpy(page_addr, CHOOSE_NVHE_SYM(__per_cpu_start), nvhe_percpu_size());
2570 kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu] = (unsigned long)page_addr;
2571 }
2572
2573 /*
2574 * Map the Hyp-code called directly from the host
2575 */
2576 err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start),
2577 kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC);
2578 if (err) {
2579 kvm_err("Cannot map world-switch code\n");
2580 goto out_err;
2581 }
2582
2583 err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start),
2584 kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO);
2585 if (err) {
2586 kvm_err("Cannot map .hyp.rodata section\n");
2587 goto out_err;
2588 }
2589
2590 err = create_hyp_mappings(kvm_ksym_ref(__start_rodata),
2591 kvm_ksym_ref(__end_rodata), PAGE_HYP_RO);
2592 if (err) {
2593 kvm_err("Cannot map rodata section\n");
2594 goto out_err;
2595 }
2596
2597 /*
2598 * .hyp.bss is guaranteed to be placed at the beginning of the .bss
2599 * section thanks to an assertion in the linker script. Map it RW and
2600 * the rest of .bss RO.
2601 */
2602 err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_start),
2603 kvm_ksym_ref(__hyp_bss_end), PAGE_HYP);
2604 if (err) {
2605 kvm_err("Cannot map hyp bss section: %d\n", err);
2606 goto out_err;
2607 }
2608
2609 err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_end),
2610 kvm_ksym_ref(__bss_stop), PAGE_HYP_RO);
2611 if (err) {
2612 kvm_err("Cannot map bss section\n");
2613 goto out_err;
2614 }
2615
2616 /*
2617 * Map the Hyp stack pages
2618 */
2619 for_each_possible_cpu(cpu) {
2620 struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
2621 char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu);
2622
2623 err = create_hyp_stack(__pa(stack_page), ¶ms->stack_hyp_va);
2624 if (err) {
2625 kvm_err("Cannot map hyp stack\n");
2626 goto out_err;
2627 }
2628
2629 /*
2630 * Save the stack PA in nvhe_init_params. This will be needed
2631 * to recreate the stack mapping in protected nVHE mode.
2632 * __hyp_pa() won't do the right thing there, since the stack
2633 * has been mapped in the flexible private VA space.
2634 */
2635 params->stack_pa = __pa(stack_page);
2636 }
2637
2638 for_each_possible_cpu(cpu) {
2639 char *percpu_begin = (char *)kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu];
2640 char *percpu_end = percpu_begin + nvhe_percpu_size();
2641
2642 /* Map Hyp percpu pages */
2643 err = create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP);
2644 if (err) {
2645 kvm_err("Cannot map hyp percpu region\n");
2646 goto out_err;
2647 }
2648
2649 /* Prepare the CPU initialization parameters */
2650 cpu_prepare_hyp_mode(cpu, hyp_va_bits);
2651 }
2652
2653 kvm_hyp_init_symbols();
2654
2655 if (is_protected_kvm_enabled()) {
2656 if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL) &&
2657 cpus_have_final_cap(ARM64_HAS_ADDRESS_AUTH))
2658 pkvm_hyp_init_ptrauth();
2659
2660 init_cpu_logical_map();
2661
2662 if (!init_psci_relay()) {
2663 err = -ENODEV;
2664 goto out_err;
2665 }
2666
2667 err = init_pkvm_host_sve_state();
2668 if (err)
2669 goto out_err;
2670
2671 err = kvm_hyp_init_protection(hyp_va_bits);
2672 if (err) {
2673 kvm_err("Failed to init hyp memory protection\n");
2674 goto out_err;
2675 }
2676 }
2677
2678 return 0;
2679
2680 out_err:
2681 teardown_hyp_mode();
2682 kvm_err("error initializing Hyp mode: %d\n", err);
2683 return err;
2684 }
2685
2686 struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
2687 {
2688 struct kvm_vcpu *vcpu = NULL;
2689 struct kvm_mpidr_data *data;
2690 unsigned long i;
2691
2692 mpidr &= MPIDR_HWID_BITMASK;
2693
2694 rcu_read_lock();
2695 data = rcu_dereference(kvm->arch.mpidr_data);
2696
2697 if (data) {
2698 u16 idx = kvm_mpidr_index(data, mpidr);
2699
2700 vcpu = kvm_get_vcpu(kvm, data->cmpidr_to_idx[idx]);
2701 if (mpidr != kvm_vcpu_get_mpidr_aff(vcpu))
2702 vcpu = NULL;
2703 }
2704
2705 rcu_read_unlock();
2706
2707 if (vcpu)
2708 return vcpu;
2709
2710 kvm_for_each_vcpu(i, vcpu, kvm) {
2711 if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))
2712 return vcpu;
2713 }
2714 return NULL;
2715 }
2716
2717 bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
2718 {
2719 return irqchip_in_kernel(kvm);
2720 }
2721
2722 bool kvm_arch_has_irq_bypass(void)
2723 {
2724 return true;
2725 }
2726
2727 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
2728 struct irq_bypass_producer *prod)
2729 {
2730 struct kvm_kernel_irqfd *irqfd =
2731 container_of(cons, struct kvm_kernel_irqfd, consumer);
2732
2733 return kvm_vgic_v4_set_forwarding(irqfd->kvm, prod->irq,
2734 &irqfd->irq_entry);
2735 }
2736 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
2737 struct irq_bypass_producer *prod)
2738 {
2739 struct kvm_kernel_irqfd *irqfd =
2740 container_of(cons, struct kvm_kernel_irqfd, consumer);
2741
2742 kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq,
2743 &irqfd->irq_entry);
2744 }
2745
2746 void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *cons)
2747 {
2748 struct kvm_kernel_irqfd *irqfd =
2749 container_of(cons, struct kvm_kernel_irqfd, consumer);
2750
2751 kvm_arm_halt_guest(irqfd->kvm);
2752 }
2753
2754 void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *cons)
2755 {
2756 struct kvm_kernel_irqfd *irqfd =
2757 container_of(cons, struct kvm_kernel_irqfd, consumer);
2758
2759 kvm_arm_resume_guest(irqfd->kvm);
2760 }
2761
2762 /* Initialize Hyp-mode and memory mappings on all CPUs */
2763 static __init int kvm_arm_init(void)
2764 {
2765 int err;
2766 bool in_hyp_mode;
2767
2768 if (!is_hyp_mode_available()) {
2769 kvm_info("HYP mode not available\n");
2770 return -ENODEV;
2771 }
2772
2773 if (kvm_get_mode() == KVM_MODE_NONE) {
2774 kvm_info("KVM disabled from command line\n");
2775 return -ENODEV;
2776 }
2777
2778 err = kvm_sys_reg_table_init();
2779 if (err) {
2780 kvm_info("Error initializing system register tables");
2781 return err;
2782 }
2783
2784 in_hyp_mode = is_kernel_in_hyp_mode();
2785
2786 if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) ||
2787 cpus_have_final_cap(ARM64_WORKAROUND_1508412))
2788 kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \
2789 "Only trusted guests should be used on this system.\n");
2790
2791 err = kvm_set_ipa_limit();
2792 if (err)
2793 return err;
2794
2795 err = kvm_arm_init_sve();
2796 if (err)
2797 return err;
2798
2799 err = kvm_arm_vmid_alloc_init();
2800 if (err) {
2801 kvm_err("Failed to initialize VMID allocator.\n");
2802 return err;
2803 }
2804
2805 if (!in_hyp_mode) {
2806 err = init_hyp_mode();
2807 if (err)
2808 goto out_err;
2809 }
2810
2811 err = kvm_init_vector_slots();
2812 if (err) {
2813 kvm_err("Cannot initialise vector slots\n");
2814 goto out_hyp;
2815 }
2816
2817 err = init_subsystems();
2818 if (err)
2819 goto out_hyp;
2820
2821 kvm_info("%s%sVHE mode initialized successfully\n",
2822 in_hyp_mode ? "" : (is_protected_kvm_enabled() ?
2823 "Protected " : "Hyp "),
2824 in_hyp_mode ? "" : (cpus_have_final_cap(ARM64_KVM_HVHE) ?
2825 "h" : "n"));
2826
2827 /*
2828 * FIXME: Do something reasonable if kvm_init() fails after pKVM
2829 * hypervisor protection is finalized.
2830 */
2831 err = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2832 if (err)
2833 goto out_subs;
2834
2835 /*
2836 * This should be called after initialization is done and failure isn't
2837 * possible anymore.
2838 */
2839 if (!in_hyp_mode)
2840 finalize_init_hyp_mode();
2841
2842 kvm_arm_initialised = true;
2843
2844 return 0;
2845
2846 out_subs:
2847 teardown_subsystems();
2848 out_hyp:
2849 if (!in_hyp_mode)
2850 teardown_hyp_mode();
2851 out_err:
2852 kvm_arm_vmid_alloc_free();
2853 return err;
2854 }
2855
2856 static int __init early_kvm_mode_cfg(char *arg)
2857 {
2858 if (!arg)
2859 return -EINVAL;
2860
2861 if (strcmp(arg, "none") == 0) {
2862 kvm_mode = KVM_MODE_NONE;
2863 return 0;
2864 }
2865
2866 if (!is_hyp_mode_available()) {
2867 pr_warn_once("KVM is not available. Ignoring kvm-arm.mode\n");
2868 return 0;
2869 }
2870
2871 if (strcmp(arg, "protected") == 0) {
2872 if (!is_kernel_in_hyp_mode())
2873 kvm_mode = KVM_MODE_PROTECTED;
2874 else
2875 pr_warn_once("Protected KVM not available with VHE\n");
2876
2877 return 0;
2878 }
2879
2880 if (strcmp(arg, "nvhe") == 0 && !WARN_ON(is_kernel_in_hyp_mode())) {
2881 kvm_mode = KVM_MODE_DEFAULT;
2882 return 0;
2883 }
2884
2885 if (strcmp(arg, "nested") == 0 && !WARN_ON(!is_kernel_in_hyp_mode())) {
2886 kvm_mode = KVM_MODE_NV;
2887 return 0;
2888 }
2889
2890 return -EINVAL;
2891 }
2892 early_param("kvm-arm.mode", early_kvm_mode_cfg);
2893
2894 static int __init early_kvm_wfx_trap_policy_cfg(char *arg, enum kvm_wfx_trap_policy *p)
2895 {
2896 if (!arg)
2897 return -EINVAL;
2898
2899 if (strcmp(arg, "trap") == 0) {
2900 *p = KVM_WFX_TRAP;
2901 return 0;
2902 }
2903
2904 if (strcmp(arg, "notrap") == 0) {
2905 *p = KVM_WFX_NOTRAP;
2906 return 0;
2907 }
2908
2909 return -EINVAL;
2910 }
2911
2912 static int __init early_kvm_wfi_trap_policy_cfg(char *arg)
2913 {
2914 return early_kvm_wfx_trap_policy_cfg(arg, &kvm_wfi_trap_policy);
2915 }
2916 early_param("kvm-arm.wfi_trap_policy", early_kvm_wfi_trap_policy_cfg);
2917
2918 static int __init early_kvm_wfe_trap_policy_cfg(char *arg)
2919 {
2920 return early_kvm_wfx_trap_policy_cfg(arg, &kvm_wfe_trap_policy);
2921 }
2922 early_param("kvm-arm.wfe_trap_policy", early_kvm_wfe_trap_policy_cfg);
2923
2924 enum kvm_mode kvm_get_mode(void)
2925 {
2926 return kvm_mode;
2927 }
2928
2929 module_init(kvm_arm_init);
2930
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.