1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2017 - Columbia University and Linaro Ltd.
4 * Author: Jintack Lim <jintack.lim@linaro.org>
5 */
6
7 #include <linux/bitfield.h>
8 #include <linux/kvm.h>
9 #include <linux/kvm_host.h>
10
11 #include <asm/kvm_arm.h>
12 #include <asm/kvm_emulate.h>
13 #include <asm/kvm_mmu.h>
14 #include <asm/kvm_nested.h>
15 #include <asm/sysreg.h>
16
17 #include "sys_regs.h"
18
19 /* Protection against the sysreg repainting madness... */
20 #define NV_FTR(r, f) ID_AA64##r##_EL1_##f
21
22 /*
23 * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between
24 * memory usage and potential number of different sets of S2 PTs in
25 * the guests. Running out of S2 MMUs only affects performance (we
26 * will invalidate them more often).
27 */
28 #define S2_MMU_PER_VCPU 2
29
30 void kvm_init_nested(struct kvm *kvm)
31 {
32 kvm->arch.nested_mmus = NULL;
33 kvm->arch.nested_mmus_size = 0;
34 }
35
36 static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)
37 {
38 /*
39 * We only initialise the IPA range on the canonical MMU, which
40 * defines the contract between KVM and userspace on where the
41 * "hardware" is in the IPA space. This affects the validity of MMIO
42 * exits forwarded to userspace, for example.
43 *
44 * For nested S2s, we use the PARange as exposed to the guest, as it
45 * is allowed to use it at will to expose whatever memory map it
46 * wants to its own guests as it would be on real HW.
47 */
48 return kvm_init_stage2_mmu(kvm, mmu, kvm_get_pa_bits(kvm));
49 }
50
51 int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu)
52 {
53 struct kvm *kvm = vcpu->kvm;
54 struct kvm_s2_mmu *tmp;
55 int num_mmus, ret = 0;
56
57 /*
58 * Let's treat memory allocation failures as benign: If we fail to
59 * allocate anything, return an error and keep the allocated array
60 * alive. Userspace may try to recover by intializing the vcpu
61 * again, and there is no reason to affect the whole VM for this.
62 */
63 num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU;
64 tmp = kvrealloc(kvm->arch.nested_mmus,
65 size_mul(sizeof(*kvm->arch.nested_mmus), kvm->arch.nested_mmus_size),
66 size_mul(sizeof(*kvm->arch.nested_mmus), num_mmus),
67 GFP_KERNEL_ACCOUNT | __GFP_ZERO);
68 if (!tmp)
69 return -ENOMEM;
70
71 /*
72 * If we went through a realocation, adjust the MMU back-pointers in
73 * the previously initialised kvm_pgtable structures.
74 */
75 if (kvm->arch.nested_mmus != tmp)
76 for (int i = 0; i < kvm->arch.nested_mmus_size; i++)
77 tmp[i].pgt->mmu = &tmp[i];
78
79 for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++)
80 ret = init_nested_s2_mmu(kvm, &tmp[i]);
81
82 if (ret) {
83 for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++)
84 kvm_free_stage2_pgd(&tmp[i]);
85
86 return ret;
87 }
88
89 kvm->arch.nested_mmus_size = num_mmus;
90 kvm->arch.nested_mmus = tmp;
91
92 return 0;
93 }
94
95 struct s2_walk_info {
96 int (*read_desc)(phys_addr_t pa, u64 *desc, void *data);
97 void *data;
98 u64 baddr;
99 unsigned int max_oa_bits;
100 unsigned int pgshift;
101 unsigned int sl;
102 unsigned int t0sz;
103 bool be;
104 };
105
106 static unsigned int ps_to_output_size(unsigned int ps)
107 {
108 switch (ps) {
109 case 0: return 32;
110 case 1: return 36;
111 case 2: return 40;
112 case 3: return 42;
113 case 4: return 44;
114 case 5:
115 default:
116 return 48;
117 }
118 }
119
120 static u32 compute_fsc(int level, u32 fsc)
121 {
122 return fsc | (level & 0x3);
123 }
124
125 static int esr_s2_fault(struct kvm_vcpu *vcpu, int level, u32 fsc)
126 {
127 u32 esr;
128
129 esr = kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC;
130 esr |= compute_fsc(level, fsc);
131 return esr;
132 }
133
134 static int get_ia_size(struct s2_walk_info *wi)
135 {
136 return 64 - wi->t0sz;
137 }
138
139 static int check_base_s2_limits(struct s2_walk_info *wi,
140 int level, int input_size, int stride)
141 {
142 int start_size, ia_size;
143
144 ia_size = get_ia_size(wi);
145
146 /* Check translation limits */
147 switch (BIT(wi->pgshift)) {
148 case SZ_64K:
149 if (level == 0 || (level == 1 && ia_size <= 42))
150 return -EFAULT;
151 break;
152 case SZ_16K:
153 if (level == 0 || (level == 1 && ia_size <= 40))
154 return -EFAULT;
155 break;
156 case SZ_4K:
157 if (level < 0 || (level == 0 && ia_size <= 42))
158 return -EFAULT;
159 break;
160 }
161
162 /* Check input size limits */
163 if (input_size > ia_size)
164 return -EFAULT;
165
166 /* Check number of entries in starting level table */
167 start_size = input_size - ((3 - level) * stride + wi->pgshift);
168 if (start_size < 1 || start_size > stride + 4)
169 return -EFAULT;
170
171 return 0;
172 }
173
174 /* Check if output is within boundaries */
175 static int check_output_size(struct s2_walk_info *wi, phys_addr_t output)
176 {
177 unsigned int output_size = wi->max_oa_bits;
178
179 if (output_size != 48 && (output & GENMASK_ULL(47, output_size)))
180 return -1;
181
182 return 0;
183 }
184
185 /*
186 * This is essentially a C-version of the pseudo code from the ARM ARM
187 * AArch64.TranslationTableWalk function. I strongly recommend looking at
188 * that pseudocode in trying to understand this.
189 *
190 * Must be called with the kvm->srcu read lock held
191 */
192 static int walk_nested_s2_pgd(phys_addr_t ipa,
193 struct s2_walk_info *wi, struct kvm_s2_trans *out)
194 {
195 int first_block_level, level, stride, input_size, base_lower_bound;
196 phys_addr_t base_addr;
197 unsigned int addr_top, addr_bottom;
198 u64 desc; /* page table entry */
199 int ret;
200 phys_addr_t paddr;
201
202 switch (BIT(wi->pgshift)) {
203 default:
204 case SZ_64K:
205 case SZ_16K:
206 level = 3 - wi->sl;
207 first_block_level = 2;
208 break;
209 case SZ_4K:
210 level = 2 - wi->sl;
211 first_block_level = 1;
212 break;
213 }
214
215 stride = wi->pgshift - 3;
216 input_size = get_ia_size(wi);
217 if (input_size > 48 || input_size < 25)
218 return -EFAULT;
219
220 ret = check_base_s2_limits(wi, level, input_size, stride);
221 if (WARN_ON(ret))
222 return ret;
223
224 base_lower_bound = 3 + input_size - ((3 - level) * stride +
225 wi->pgshift);
226 base_addr = wi->baddr & GENMASK_ULL(47, base_lower_bound);
227
228 if (check_output_size(wi, base_addr)) {
229 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ);
230 return 1;
231 }
232
233 addr_top = input_size - 1;
234
235 while (1) {
236 phys_addr_t index;
237
238 addr_bottom = (3 - level) * stride + wi->pgshift;
239 index = (ipa & GENMASK_ULL(addr_top, addr_bottom))
240 >> (addr_bottom - 3);
241
242 paddr = base_addr | index;
243 ret = wi->read_desc(paddr, &desc, wi->data);
244 if (ret < 0)
245 return ret;
246
247 /*
248 * Handle reversedescriptors if endianness differs between the
249 * host and the guest hypervisor.
250 */
251 if (wi->be)
252 desc = be64_to_cpu((__force __be64)desc);
253 else
254 desc = le64_to_cpu((__force __le64)desc);
255
256 /* Check for valid descriptor at this point */
257 if (!(desc & 1) || ((desc & 3) == 1 && level == 3)) {
258 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);
259 out->upper_attr = desc;
260 return 1;
261 }
262
263 /* We're at the final level or block translation level */
264 if ((desc & 3) == 1 || level == 3)
265 break;
266
267 if (check_output_size(wi, desc)) {
268 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ);
269 out->upper_attr = desc;
270 return 1;
271 }
272
273 base_addr = desc & GENMASK_ULL(47, wi->pgshift);
274
275 level += 1;
276 addr_top = addr_bottom - 1;
277 }
278
279 if (level < first_block_level) {
280 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);
281 out->upper_attr = desc;
282 return 1;
283 }
284
285 /*
286 * We don't use the contiguous bit in the stage-2 ptes, so skip check
287 * for misprogramming of the contiguous bit.
288 */
289
290 if (check_output_size(wi, desc)) {
291 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ);
292 out->upper_attr = desc;
293 return 1;
294 }
295
296 if (!(desc & BIT(10))) {
297 out->esr = compute_fsc(level, ESR_ELx_FSC_ACCESS);
298 out->upper_attr = desc;
299 return 1;
300 }
301
302 /* Calculate and return the result */
303 paddr = (desc & GENMASK_ULL(47, addr_bottom)) |
304 (ipa & GENMASK_ULL(addr_bottom - 1, 0));
305 out->output = paddr;
306 out->block_size = 1UL << ((3 - level) * stride + wi->pgshift);
307 out->readable = desc & (0b01 << 6);
308 out->writable = desc & (0b10 << 6);
309 out->level = level;
310 out->upper_attr = desc & GENMASK_ULL(63, 52);
311 return 0;
312 }
313
314 static int read_guest_s2_desc(phys_addr_t pa, u64 *desc, void *data)
315 {
316 struct kvm_vcpu *vcpu = data;
317
318 return kvm_read_guest(vcpu->kvm, pa, desc, sizeof(*desc));
319 }
320
321 static void vtcr_to_walk_info(u64 vtcr, struct s2_walk_info *wi)
322 {
323 wi->t0sz = vtcr & TCR_EL2_T0SZ_MASK;
324
325 switch (vtcr & VTCR_EL2_TG0_MASK) {
326 case VTCR_EL2_TG0_4K:
327 wi->pgshift = 12; break;
328 case VTCR_EL2_TG0_16K:
329 wi->pgshift = 14; break;
330 case VTCR_EL2_TG0_64K:
331 default: /* IMPDEF: treat any other value as 64k */
332 wi->pgshift = 16; break;
333 }
334
335 wi->sl = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);
336 /* Global limit for now, should eventually be per-VM */
337 wi->max_oa_bits = min(get_kvm_ipa_limit(),
338 ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr)));
339 }
340
341 int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
342 struct kvm_s2_trans *result)
343 {
344 u64 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
345 struct s2_walk_info wi;
346 int ret;
347
348 result->esr = 0;
349
350 if (!vcpu_has_nv(vcpu))
351 return 0;
352
353 wi.read_desc = read_guest_s2_desc;
354 wi.data = vcpu;
355 wi.baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
356
357 vtcr_to_walk_info(vtcr, &wi);
358
359 wi.be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE;
360
361 ret = walk_nested_s2_pgd(gipa, &wi, result);
362 if (ret)
363 result->esr |= (kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC);
364
365 return ret;
366 }
367
368 static unsigned int ttl_to_size(u8 ttl)
369 {
370 int level = ttl & 3;
371 int gran = (ttl >> 2) & 3;
372 unsigned int max_size = 0;
373
374 switch (gran) {
375 case TLBI_TTL_TG_4K:
376 switch (level) {
377 case 0:
378 break;
379 case 1:
380 max_size = SZ_1G;
381 break;
382 case 2:
383 max_size = SZ_2M;
384 break;
385 case 3:
386 max_size = SZ_4K;
387 break;
388 }
389 break;
390 case TLBI_TTL_TG_16K:
391 switch (level) {
392 case 0:
393 case 1:
394 break;
395 case 2:
396 max_size = SZ_32M;
397 break;
398 case 3:
399 max_size = SZ_16K;
400 break;
401 }
402 break;
403 case TLBI_TTL_TG_64K:
404 switch (level) {
405 case 0:
406 case 1:
407 /* No 52bit IPA support */
408 break;
409 case 2:
410 max_size = SZ_512M;
411 break;
412 case 3:
413 max_size = SZ_64K;
414 break;
415 }
416 break;
417 default: /* No size information */
418 break;
419 }
420
421 return max_size;
422 }
423
424 /*
425 * Compute the equivalent of the TTL field by parsing the shadow PT. The
426 * granule size is extracted from the cached VTCR_EL2.TG0 while the level is
427 * retrieved from first entry carrying the level as a tag.
428 */
429 static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr)
430 {
431 u64 tmp, sz = 0, vtcr = mmu->tlb_vtcr;
432 kvm_pte_t pte;
433 u8 ttl, level;
434
435 lockdep_assert_held_write(&kvm_s2_mmu_to_kvm(mmu)->mmu_lock);
436
437 switch (vtcr & VTCR_EL2_TG0_MASK) {
438 case VTCR_EL2_TG0_4K:
439 ttl = (TLBI_TTL_TG_4K << 2);
440 break;
441 case VTCR_EL2_TG0_16K:
442 ttl = (TLBI_TTL_TG_16K << 2);
443 break;
444 case VTCR_EL2_TG0_64K:
445 default: /* IMPDEF: treat any other value as 64k */
446 ttl = (TLBI_TTL_TG_64K << 2);
447 break;
448 }
449
450 tmp = addr;
451
452 again:
453 /* Iteratively compute the block sizes for a particular granule size */
454 switch (vtcr & VTCR_EL2_TG0_MASK) {
455 case VTCR_EL2_TG0_4K:
456 if (sz < SZ_4K) sz = SZ_4K;
457 else if (sz < SZ_2M) sz = SZ_2M;
458 else if (sz < SZ_1G) sz = SZ_1G;
459 else sz = 0;
460 break;
461 case VTCR_EL2_TG0_16K:
462 if (sz < SZ_16K) sz = SZ_16K;
463 else if (sz < SZ_32M) sz = SZ_32M;
464 else sz = 0;
465 break;
466 case VTCR_EL2_TG0_64K:
467 default: /* IMPDEF: treat any other value as 64k */
468 if (sz < SZ_64K) sz = SZ_64K;
469 else if (sz < SZ_512M) sz = SZ_512M;
470 else sz = 0;
471 break;
472 }
473
474 if (sz == 0)
475 return 0;
476
477 tmp &= ~(sz - 1);
478 if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL))
479 goto again;
480 if (!(pte & PTE_VALID))
481 goto again;
482 level = FIELD_GET(KVM_NV_GUEST_MAP_SZ, pte);
483 if (!level)
484 goto again;
485
486 ttl |= level;
487
488 /*
489 * We now have found some level information in the shadow S2. Check
490 * that the resulting range is actually including the original IPA.
491 */
492 sz = ttl_to_size(ttl);
493 if (addr < (tmp + sz))
494 return ttl;
495
496 return 0;
497 }
498
499 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val)
500 {
501 struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
502 unsigned long max_size;
503 u8 ttl;
504
505 ttl = FIELD_GET(TLBI_TTL_MASK, val);
506
507 if (!ttl || !kvm_has_feat(kvm, ID_AA64MMFR2_EL1, TTL, IMP)) {
508 /* No TTL, check the shadow S2 for a hint */
509 u64 addr = (val & GENMASK_ULL(35, 0)) << 12;
510 ttl = get_guest_mapping_ttl(mmu, addr);
511 }
512
513 max_size = ttl_to_size(ttl);
514
515 if (!max_size) {
516 /* Compute the maximum extent of the invalidation */
517 switch (mmu->tlb_vtcr & VTCR_EL2_TG0_MASK) {
518 case VTCR_EL2_TG0_4K:
519 max_size = SZ_1G;
520 break;
521 case VTCR_EL2_TG0_16K:
522 max_size = SZ_32M;
523 break;
524 case VTCR_EL2_TG0_64K:
525 default: /* IMPDEF: treat any other value as 64k */
526 /*
527 * No, we do not support 52bit IPA in nested yet. Once
528 * we do, this should be 4TB.
529 */
530 max_size = SZ_512M;
531 break;
532 }
533 }
534
535 WARN_ON(!max_size);
536 return max_size;
537 }
538
539 /*
540 * We can have multiple *different* MMU contexts with the same VMID:
541 *
542 * - S2 being enabled or not, hence differing by the HCR_EL2.VM bit
543 *
544 * - Multiple vcpus using private S2s (huh huh...), hence differing by the
545 * VBBTR_EL2.BADDR address
546 *
547 * - A combination of the above...
548 *
549 * We can always identify which MMU context to pick at run-time. However,
550 * TLB invalidation involving a VMID must take action on all the TLBs using
551 * this particular VMID. This translates into applying the same invalidation
552 * operation to all the contexts that are using this VMID. Moar phun!
553 */
554 void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid,
555 const union tlbi_info *info,
556 void (*tlbi_callback)(struct kvm_s2_mmu *,
557 const union tlbi_info *))
558 {
559 write_lock(&kvm->mmu_lock);
560
561 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) {
562 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
563
564 if (!kvm_s2_mmu_valid(mmu))
565 continue;
566
567 if (vmid == get_vmid(mmu->tlb_vttbr))
568 tlbi_callback(mmu, info);
569 }
570
571 write_unlock(&kvm->mmu_lock);
572 }
573
574 struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu)
575 {
576 struct kvm *kvm = vcpu->kvm;
577 bool nested_stage2_enabled;
578 u64 vttbr, vtcr, hcr;
579
580 lockdep_assert_held_write(&kvm->mmu_lock);
581
582 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
583 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
584 hcr = vcpu_read_sys_reg(vcpu, HCR_EL2);
585
586 nested_stage2_enabled = hcr & HCR_VM;
587
588 /* Don't consider the CnP bit for the vttbr match */
589 vttbr &= ~VTTBR_CNP_BIT;
590
591 /*
592 * Two possibilities when looking up a S2 MMU context:
593 *
594 * - either S2 is enabled in the guest, and we need a context that is
595 * S2-enabled and matches the full VTTBR (VMID+BADDR) and VTCR,
596 * which makes it safe from a TLB conflict perspective (a broken
597 * guest won't be able to generate them),
598 *
599 * - or S2 is disabled, and we need a context that is S2-disabled
600 * and matches the VMID only, as all TLBs are tagged by VMID even
601 * if S2 translation is disabled.
602 */
603 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) {
604 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
605
606 if (!kvm_s2_mmu_valid(mmu))
607 continue;
608
609 if (nested_stage2_enabled &&
610 mmu->nested_stage2_enabled &&
611 vttbr == mmu->tlb_vttbr &&
612 vtcr == mmu->tlb_vtcr)
613 return mmu;
614
615 if (!nested_stage2_enabled &&
616 !mmu->nested_stage2_enabled &&
617 get_vmid(vttbr) == get_vmid(mmu->tlb_vttbr))
618 return mmu;
619 }
620 return NULL;
621 }
622
623 static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu)
624 {
625 struct kvm *kvm = vcpu->kvm;
626 struct kvm_s2_mmu *s2_mmu;
627 int i;
628
629 lockdep_assert_held_write(&vcpu->kvm->mmu_lock);
630
631 s2_mmu = lookup_s2_mmu(vcpu);
632 if (s2_mmu)
633 goto out;
634
635 /*
636 * Make sure we don't always search from the same point, or we
637 * will always reuse a potentially active context, leaving
638 * free contexts unused.
639 */
640 for (i = kvm->arch.nested_mmus_next;
641 i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next);
642 i++) {
643 s2_mmu = &kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size];
644
645 if (atomic_read(&s2_mmu->refcnt) == 0)
646 break;
647 }
648 BUG_ON(atomic_read(&s2_mmu->refcnt)); /* We have struct MMUs to spare */
649
650 /* Set the scene for the next search */
651 kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size;
652
653 /* Clear the old state */
654 if (kvm_s2_mmu_valid(s2_mmu))
655 kvm_stage2_unmap_range(s2_mmu, 0, kvm_phys_size(s2_mmu));
656
657 /*
658 * The virtual VMID (modulo CnP) will be used as a key when matching
659 * an existing kvm_s2_mmu.
660 *
661 * We cache VTCR at allocation time, once and for all. It'd be great
662 * if the guest didn't screw that one up, as this is not very
663 * forgiving...
664 */
665 s2_mmu->tlb_vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2) & ~VTTBR_CNP_BIT;
666 s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
667 s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM;
668
669 out:
670 atomic_inc(&s2_mmu->refcnt);
671 return s2_mmu;
672 }
673
674 void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu)
675 {
676 /* CnP being set denotes an invalid entry */
677 mmu->tlb_vttbr = VTTBR_CNP_BIT;
678 mmu->nested_stage2_enabled = false;
679 atomic_set(&mmu->refcnt, 0);
680 }
681
682 void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu)
683 {
684 if (is_hyp_ctxt(vcpu)) {
685 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
686 } else {
687 write_lock(&vcpu->kvm->mmu_lock);
688 vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu);
689 write_unlock(&vcpu->kvm->mmu_lock);
690 }
691 }
692
693 void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu)
694 {
695 if (kvm_is_nested_s2_mmu(vcpu->kvm, vcpu->arch.hw_mmu)) {
696 atomic_dec(&vcpu->arch.hw_mmu->refcnt);
697 vcpu->arch.hw_mmu = NULL;
698 }
699 }
700
701 /*
702 * Returns non-zero if permission fault is handled by injecting it to the next
703 * level hypervisor.
704 */
705 int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, struct kvm_s2_trans *trans)
706 {
707 bool forward_fault = false;
708
709 trans->esr = 0;
710
711 if (!kvm_vcpu_trap_is_permission_fault(vcpu))
712 return 0;
713
714 if (kvm_vcpu_trap_is_iabt(vcpu)) {
715 forward_fault = !kvm_s2_trans_executable(trans);
716 } else {
717 bool write_fault = kvm_is_write_fault(vcpu);
718
719 forward_fault = ((write_fault && !trans->writable) ||
720 (!write_fault && !trans->readable));
721 }
722
723 if (forward_fault)
724 trans->esr = esr_s2_fault(vcpu, trans->level, ESR_ELx_FSC_PERM);
725
726 return forward_fault;
727 }
728
729 int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2)
730 {
731 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.far_el2, FAR_EL2);
732 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.hpfar_el2, HPFAR_EL2);
733
734 return kvm_inject_nested_sync(vcpu, esr_el2);
735 }
736
737 void kvm_nested_s2_wp(struct kvm *kvm)
738 {
739 int i;
740
741 lockdep_assert_held_write(&kvm->mmu_lock);
742
743 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
744 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
745
746 if (kvm_s2_mmu_valid(mmu))
747 kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu));
748 }
749 }
750
751 void kvm_nested_s2_unmap(struct kvm *kvm)
752 {
753 int i;
754
755 lockdep_assert_held_write(&kvm->mmu_lock);
756
757 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
758 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
759
760 if (kvm_s2_mmu_valid(mmu))
761 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu));
762 }
763 }
764
765 void kvm_nested_s2_flush(struct kvm *kvm)
766 {
767 int i;
768
769 lockdep_assert_held_write(&kvm->mmu_lock);
770
771 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
772 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
773
774 if (kvm_s2_mmu_valid(mmu))
775 kvm_stage2_flush_range(mmu, 0, kvm_phys_size(mmu));
776 }
777 }
778
779 void kvm_arch_flush_shadow_all(struct kvm *kvm)
780 {
781 int i;
782
783 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
784 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
785
786 if (!WARN_ON(atomic_read(&mmu->refcnt)))
787 kvm_free_stage2_pgd(mmu);
788 }
789 kvfree(kvm->arch.nested_mmus);
790 kvm->arch.nested_mmus = NULL;
791 kvm->arch.nested_mmus_size = 0;
792 kvm_uninit_stage2_mmu(kvm);
793 }
794
795 /*
796 * Our emulated CPU doesn't support all the possible features. For the
797 * sake of simplicity (and probably mental sanity), wipe out a number
798 * of feature bits we don't intend to support for the time being.
799 * This list should get updated as new features get added to the NV
800 * support, and new extension to the architecture.
801 */
802 static void limit_nv_id_regs(struct kvm *kvm)
803 {
804 u64 val, tmp;
805
806 /* Support everything but TME */
807 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64ISAR0_EL1);
808 val &= ~NV_FTR(ISAR0, TME);
809 kvm_set_vm_id_reg(kvm, SYS_ID_AA64ISAR0_EL1, val);
810
811 /* Support everything but Spec Invalidation and LS64 */
812 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64ISAR1_EL1);
813 val &= ~(NV_FTR(ISAR1, LS64) |
814 NV_FTR(ISAR1, SPECRES));
815 kvm_set_vm_id_reg(kvm, SYS_ID_AA64ISAR1_EL1, val);
816
817 /* No AMU, MPAM, S-EL2, or RAS */
818 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1);
819 val &= ~(GENMASK_ULL(55, 52) |
820 NV_FTR(PFR0, AMU) |
821 NV_FTR(PFR0, MPAM) |
822 NV_FTR(PFR0, SEL2) |
823 NV_FTR(PFR0, RAS) |
824 NV_FTR(PFR0, EL3) |
825 NV_FTR(PFR0, EL2) |
826 NV_FTR(PFR0, EL1));
827 /* 64bit EL1/EL2/EL3 only */
828 val |= FIELD_PREP(NV_FTR(PFR0, EL1), 0b0001);
829 val |= FIELD_PREP(NV_FTR(PFR0, EL2), 0b0001);
830 val |= FIELD_PREP(NV_FTR(PFR0, EL3), 0b0001);
831 kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1, val);
832
833 /* Only support BTI, SSBS, CSV2_frac */
834 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR1_EL1);
835 val &= (NV_FTR(PFR1, BT) |
836 NV_FTR(PFR1, SSBS) |
837 NV_FTR(PFR1, CSV2_frac));
838 kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR1_EL1, val);
839
840 /* Hide ECV, ExS, Secure Memory */
841 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1);
842 val &= ~(NV_FTR(MMFR0, ECV) |
843 NV_FTR(MMFR0, EXS) |
844 NV_FTR(MMFR0, TGRAN4_2) |
845 NV_FTR(MMFR0, TGRAN16_2) |
846 NV_FTR(MMFR0, TGRAN64_2) |
847 NV_FTR(MMFR0, SNSMEM));
848
849 /* Disallow unsupported S2 page sizes */
850 switch (PAGE_SIZE) {
851 case SZ_64K:
852 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN16_2), 0b0001);
853 fallthrough;
854 case SZ_16K:
855 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN4_2), 0b0001);
856 fallthrough;
857 case SZ_4K:
858 /* Support everything */
859 break;
860 }
861 /*
862 * Since we can't support a guest S2 page size smaller than
863 * the host's own page size (due to KVM only populating its
864 * own S2 using the kernel's page size), advertise the
865 * limitation using FEAT_GTG.
866 */
867 switch (PAGE_SIZE) {
868 case SZ_4K:
869 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN4_2), 0b0010);
870 fallthrough;
871 case SZ_16K:
872 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN16_2), 0b0010);
873 fallthrough;
874 case SZ_64K:
875 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN64_2), 0b0010);
876 break;
877 }
878 /* Cap PARange to 48bits */
879 tmp = FIELD_GET(NV_FTR(MMFR0, PARANGE), val);
880 if (tmp > 0b0101) {
881 val &= ~NV_FTR(MMFR0, PARANGE);
882 val |= FIELD_PREP(NV_FTR(MMFR0, PARANGE), 0b0101);
883 }
884 kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1, val);
885
886 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR1_EL1);
887 val &= (NV_FTR(MMFR1, HCX) |
888 NV_FTR(MMFR1, PAN) |
889 NV_FTR(MMFR1, LO) |
890 NV_FTR(MMFR1, HPDS) |
891 NV_FTR(MMFR1, VH) |
892 NV_FTR(MMFR1, VMIDBits));
893 kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR1_EL1, val);
894
895 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR2_EL1);
896 val &= ~(NV_FTR(MMFR2, BBM) |
897 NV_FTR(MMFR2, TTL) |
898 GENMASK_ULL(47, 44) |
899 NV_FTR(MMFR2, ST) |
900 NV_FTR(MMFR2, CCIDX) |
901 NV_FTR(MMFR2, VARange));
902
903 /* Force TTL support */
904 val |= FIELD_PREP(NV_FTR(MMFR2, TTL), 0b0001);
905 kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR2_EL1, val);
906
907 val = 0;
908 if (!cpus_have_final_cap(ARM64_HAS_HCR_NV1))
909 val |= FIELD_PREP(NV_FTR(MMFR4, E2H0),
910 ID_AA64MMFR4_EL1_E2H0_NI_NV1);
911 kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR4_EL1, val);
912
913 /* Only limited support for PMU, Debug, BPs and WPs */
914 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64DFR0_EL1);
915 val &= (NV_FTR(DFR0, PMUVer) |
916 NV_FTR(DFR0, WRPs) |
917 NV_FTR(DFR0, BRPs) |
918 NV_FTR(DFR0, DebugVer));
919
920 /* Cap Debug to ARMv8.1 */
921 tmp = FIELD_GET(NV_FTR(DFR0, DebugVer), val);
922 if (tmp > 0b0111) {
923 val &= ~NV_FTR(DFR0, DebugVer);
924 val |= FIELD_PREP(NV_FTR(DFR0, DebugVer), 0b0111);
925 }
926 kvm_set_vm_id_reg(kvm, SYS_ID_AA64DFR0_EL1, val);
927 }
928
929 u64 kvm_vcpu_sanitise_vncr_reg(const struct kvm_vcpu *vcpu, enum vcpu_sysreg sr)
930 {
931 u64 v = ctxt_sys_reg(&vcpu->arch.ctxt, sr);
932 struct kvm_sysreg_masks *masks;
933
934 masks = vcpu->kvm->arch.sysreg_masks;
935
936 if (masks) {
937 sr -= __VNCR_START__;
938
939 v &= ~masks->mask[sr].res0;
940 v |= masks->mask[sr].res1;
941 }
942
943 return v;
944 }
945
946 static void set_sysreg_masks(struct kvm *kvm, int sr, u64 res0, u64 res1)
947 {
948 int i = sr - __VNCR_START__;
949
950 kvm->arch.sysreg_masks->mask[i].res0 = res0;
951 kvm->arch.sysreg_masks->mask[i].res1 = res1;
952 }
953
954 int kvm_init_nv_sysregs(struct kvm *kvm)
955 {
956 u64 res0, res1;
957 int ret = 0;
958
959 mutex_lock(&kvm->arch.config_lock);
960
961 if (kvm->arch.sysreg_masks)
962 goto out;
963
964 kvm->arch.sysreg_masks = kzalloc(sizeof(*(kvm->arch.sysreg_masks)),
965 GFP_KERNEL_ACCOUNT);
966 if (!kvm->arch.sysreg_masks) {
967 ret = -ENOMEM;
968 goto out;
969 }
970
971 limit_nv_id_regs(kvm);
972
973 /* VTTBR_EL2 */
974 res0 = res1 = 0;
975 if (!kvm_has_feat_enum(kvm, ID_AA64MMFR1_EL1, VMIDBits, 16))
976 res0 |= GENMASK(63, 56);
977 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, CnP, IMP))
978 res0 |= VTTBR_CNP_BIT;
979 set_sysreg_masks(kvm, VTTBR_EL2, res0, res1);
980
981 /* VTCR_EL2 */
982 res0 = GENMASK(63, 32) | GENMASK(30, 20);
983 res1 = BIT(31);
984 set_sysreg_masks(kvm, VTCR_EL2, res0, res1);
985
986 /* VMPIDR_EL2 */
987 res0 = GENMASK(63, 40) | GENMASK(30, 24);
988 res1 = BIT(31);
989 set_sysreg_masks(kvm, VMPIDR_EL2, res0, res1);
990
991 /* HCR_EL2 */
992 res0 = BIT(48);
993 res1 = HCR_RW;
994 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, TWED, IMP))
995 res0 |= GENMASK(63, 59);
996 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, MTE, MTE2))
997 res0 |= (HCR_TID5 | HCR_DCT | HCR_ATA);
998 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, EVT, TTLBxS))
999 res0 |= (HCR_TTLBIS | HCR_TTLBOS);
1000 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, CSV2, CSV2_2) &&
1001 !kvm_has_feat(kvm, ID_AA64PFR1_EL1, CSV2_frac, CSV2_1p2))
1002 res0 |= HCR_ENSCXT;
1003 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, EVT, IMP))
1004 res0 |= (HCR_TOCU | HCR_TICAB | HCR_TID4);
1005 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, V1P1))
1006 res0 |= HCR_AMVOFFEN;
1007 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, V1P1))
1008 res0 |= HCR_FIEN;
1009 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, FWB, IMP))
1010 res0 |= HCR_FWB;
1011 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, NV, NV2))
1012 res0 |= HCR_NV2;
1013 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, NV, IMP))
1014 res0 |= (HCR_AT | HCR_NV1 | HCR_NV);
1015 if (!(__vcpu_has_feature(&kvm->arch, KVM_ARM_VCPU_PTRAUTH_ADDRESS) &&
1016 __vcpu_has_feature(&kvm->arch, KVM_ARM_VCPU_PTRAUTH_GENERIC)))
1017 res0 |= (HCR_API | HCR_APK);
1018 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TME, IMP))
1019 res0 |= BIT(39);
1020 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP))
1021 res0 |= (HCR_TEA | HCR_TERR);
1022 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP))
1023 res0 |= HCR_TLOR;
1024 if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, E2H0, IMP))
1025 res1 |= HCR_E2H;
1026 set_sysreg_masks(kvm, HCR_EL2, res0, res1);
1027
1028 /* HCRX_EL2 */
1029 res0 = HCRX_EL2_RES0;
1030 res1 = HCRX_EL2_RES1;
1031 if (!kvm_has_feat(kvm, ID_AA64ISAR3_EL1, PACM, TRIVIAL_IMP))
1032 res0 |= HCRX_EL2_PACMEn;
1033 if (!kvm_has_feat(kvm, ID_AA64PFR2_EL1, FPMR, IMP))
1034 res0 |= HCRX_EL2_EnFPM;
1035 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP))
1036 res0 |= HCRX_EL2_GCSEn;
1037 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, SYSREG_128, IMP))
1038 res0 |= HCRX_EL2_EnIDCP128;
1039 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, ADERR, DEV_ASYNC))
1040 res0 |= (HCRX_EL2_EnSDERR | HCRX_EL2_EnSNERR);
1041 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, DF2, IMP))
1042 res0 |= HCRX_EL2_TMEA;
1043 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, D128, IMP))
1044 res0 |= HCRX_EL2_D128En;
1045 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP))
1046 res0 |= HCRX_EL2_PTTWI;
1047 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, SCTLRX, IMP))
1048 res0 |= HCRX_EL2_SCTLR2En;
1049 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, TCRX, IMP))
1050 res0 |= HCRX_EL2_TCR2En;
1051 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, MOPS, IMP))
1052 res0 |= (HCRX_EL2_MSCEn | HCRX_EL2_MCE2);
1053 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, CMOW, IMP))
1054 res0 |= HCRX_EL2_CMOW;
1055 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, NMI, IMP))
1056 res0 |= (HCRX_EL2_VFNMI | HCRX_EL2_VINMI | HCRX_EL2_TALLINT);
1057 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, SME, IMP) ||
1058 !(read_sysreg_s(SYS_SMIDR_EL1) & SMIDR_EL1_SMPS))
1059 res0 |= HCRX_EL2_SMPME;
1060 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))
1061 res0 |= (HCRX_EL2_FGTnXS | HCRX_EL2_FnXS);
1062 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V))
1063 res0 |= HCRX_EL2_EnASR;
1064 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64))
1065 res0 |= HCRX_EL2_EnALS;
1066 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA))
1067 res0 |= HCRX_EL2_EnAS0;
1068 set_sysreg_masks(kvm, HCRX_EL2, res0, res1);
1069
1070 /* HFG[RW]TR_EL2 */
1071 res0 = res1 = 0;
1072 if (!(__vcpu_has_feature(&kvm->arch, KVM_ARM_VCPU_PTRAUTH_ADDRESS) &&
1073 __vcpu_has_feature(&kvm->arch, KVM_ARM_VCPU_PTRAUTH_GENERIC)))
1074 res0 |= (HFGxTR_EL2_APDAKey | HFGxTR_EL2_APDBKey |
1075 HFGxTR_EL2_APGAKey | HFGxTR_EL2_APIAKey |
1076 HFGxTR_EL2_APIBKey);
1077 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP))
1078 res0 |= (HFGxTR_EL2_LORC_EL1 | HFGxTR_EL2_LOREA_EL1 |
1079 HFGxTR_EL2_LORID_EL1 | HFGxTR_EL2_LORN_EL1 |
1080 HFGxTR_EL2_LORSA_EL1);
1081 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, CSV2, CSV2_2) &&
1082 !kvm_has_feat(kvm, ID_AA64PFR1_EL1, CSV2_frac, CSV2_1p2))
1083 res0 |= (HFGxTR_EL2_SCXTNUM_EL1 | HFGxTR_EL2_SCXTNUM_EL0);
1084 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, GIC, IMP))
1085 res0 |= HFGxTR_EL2_ICC_IGRPENn_EL1;
1086 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP))
1087 res0 |= (HFGxTR_EL2_ERRIDR_EL1 | HFGxTR_EL2_ERRSELR_EL1 |
1088 HFGxTR_EL2_ERXFR_EL1 | HFGxTR_EL2_ERXCTLR_EL1 |
1089 HFGxTR_EL2_ERXSTATUS_EL1 | HFGxTR_EL2_ERXMISCn_EL1 |
1090 HFGxTR_EL2_ERXPFGF_EL1 | HFGxTR_EL2_ERXPFGCTL_EL1 |
1091 HFGxTR_EL2_ERXPFGCDN_EL1 | HFGxTR_EL2_ERXADDR_EL1);
1092 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA))
1093 res0 |= HFGxTR_EL2_nACCDATA_EL1;
1094 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP))
1095 res0 |= (HFGxTR_EL2_nGCS_EL0 | HFGxTR_EL2_nGCS_EL1);
1096 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, SME, IMP))
1097 res0 |= (HFGxTR_EL2_nSMPRI_EL1 | HFGxTR_EL2_nTPIDR2_EL0);
1098 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP))
1099 res0 |= HFGxTR_EL2_nRCWMASK_EL1;
1100 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, S1PIE, IMP))
1101 res0 |= (HFGxTR_EL2_nPIRE0_EL1 | HFGxTR_EL2_nPIR_EL1);
1102 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, S1POE, IMP))
1103 res0 |= (HFGxTR_EL2_nPOR_EL0 | HFGxTR_EL2_nPOR_EL1);
1104 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, S2POE, IMP))
1105 res0 |= HFGxTR_EL2_nS2POR_EL1;
1106 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, AIE, IMP))
1107 res0 |= (HFGxTR_EL2_nMAIR2_EL1 | HFGxTR_EL2_nAMAIR2_EL1);
1108 set_sysreg_masks(kvm, HFGRTR_EL2, res0 | __HFGRTR_EL2_RES0, res1);
1109 set_sysreg_masks(kvm, HFGWTR_EL2, res0 | __HFGWTR_EL2_RES0, res1);
1110
1111 /* HDFG[RW]TR_EL2 */
1112 res0 = res1 = 0;
1113 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DoubleLock, IMP))
1114 res0 |= HDFGRTR_EL2_OSDLR_EL1;
1115 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP))
1116 res0 |= (HDFGRTR_EL2_PMEVCNTRn_EL0 | HDFGRTR_EL2_PMEVTYPERn_EL0 |
1117 HDFGRTR_EL2_PMCCFILTR_EL0 | HDFGRTR_EL2_PMCCNTR_EL0 |
1118 HDFGRTR_EL2_PMCNTEN | HDFGRTR_EL2_PMINTEN |
1119 HDFGRTR_EL2_PMOVS | HDFGRTR_EL2_PMSELR_EL0 |
1120 HDFGRTR_EL2_PMMIR_EL1 | HDFGRTR_EL2_PMUSERENR_EL0 |
1121 HDFGRTR_EL2_PMCEIDn_EL0);
1122 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, IMP))
1123 res0 |= (HDFGRTR_EL2_PMBLIMITR_EL1 | HDFGRTR_EL2_PMBPTR_EL1 |
1124 HDFGRTR_EL2_PMBSR_EL1 | HDFGRTR_EL2_PMSCR_EL1 |
1125 HDFGRTR_EL2_PMSEVFR_EL1 | HDFGRTR_EL2_PMSFCR_EL1 |
1126 HDFGRTR_EL2_PMSICR_EL1 | HDFGRTR_EL2_PMSIDR_EL1 |
1127 HDFGRTR_EL2_PMSIRR_EL1 | HDFGRTR_EL2_PMSLATFR_EL1 |
1128 HDFGRTR_EL2_PMBIDR_EL1);
1129 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceVer, IMP))
1130 res0 |= (HDFGRTR_EL2_TRC | HDFGRTR_EL2_TRCAUTHSTATUS |
1131 HDFGRTR_EL2_TRCAUXCTLR | HDFGRTR_EL2_TRCCLAIM |
1132 HDFGRTR_EL2_TRCCNTVRn | HDFGRTR_EL2_TRCID |
1133 HDFGRTR_EL2_TRCIMSPECn | HDFGRTR_EL2_TRCOSLSR |
1134 HDFGRTR_EL2_TRCPRGCTLR | HDFGRTR_EL2_TRCSEQSTR |
1135 HDFGRTR_EL2_TRCSSCSRn | HDFGRTR_EL2_TRCSTATR |
1136 HDFGRTR_EL2_TRCVICTLR);
1137 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, IMP))
1138 res0 |= (HDFGRTR_EL2_TRBBASER_EL1 | HDFGRTR_EL2_TRBIDR_EL1 |
1139 HDFGRTR_EL2_TRBLIMITR_EL1 | HDFGRTR_EL2_TRBMAR_EL1 |
1140 HDFGRTR_EL2_TRBPTR_EL1 | HDFGRTR_EL2_TRBSR_EL1 |
1141 HDFGRTR_EL2_TRBTRG_EL1);
1142 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, BRBE, IMP))
1143 res0 |= (HDFGRTR_EL2_nBRBIDR | HDFGRTR_EL2_nBRBCTL |
1144 HDFGRTR_EL2_nBRBDATA);
1145 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, V1P2))
1146 res0 |= HDFGRTR_EL2_nPMSNEVFR_EL1;
1147 set_sysreg_masks(kvm, HDFGRTR_EL2, res0 | HDFGRTR_EL2_RES0, res1);
1148
1149 /* Reuse the bits from the read-side and add the write-specific stuff */
1150 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP))
1151 res0 |= (HDFGWTR_EL2_PMCR_EL0 | HDFGWTR_EL2_PMSWINC_EL0);
1152 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceVer, IMP))
1153 res0 |= HDFGWTR_EL2_TRCOSLAR;
1154 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceFilt, IMP))
1155 res0 |= HDFGWTR_EL2_TRFCR_EL1;
1156 set_sysreg_masks(kvm, HFGWTR_EL2, res0 | HDFGWTR_EL2_RES0, res1);
1157
1158 /* HFGITR_EL2 */
1159 res0 = HFGITR_EL2_RES0;
1160 res1 = HFGITR_EL2_RES1;
1161 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, DPB, DPB2))
1162 res0 |= HFGITR_EL2_DCCVADP;
1163 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, PAN, PAN2))
1164 res0 |= (HFGITR_EL2_ATS1E1RP | HFGITR_EL2_ATS1E1WP);
1165 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
1166 res0 |= (HFGITR_EL2_TLBIRVAALE1OS | HFGITR_EL2_TLBIRVALE1OS |
1167 HFGITR_EL2_TLBIRVAAE1OS | HFGITR_EL2_TLBIRVAE1OS |
1168 HFGITR_EL2_TLBIVAALE1OS | HFGITR_EL2_TLBIVALE1OS |
1169 HFGITR_EL2_TLBIVAAE1OS | HFGITR_EL2_TLBIASIDE1OS |
1170 HFGITR_EL2_TLBIVAE1OS | HFGITR_EL2_TLBIVMALLE1OS);
1171 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
1172 res0 |= (HFGITR_EL2_TLBIRVAALE1 | HFGITR_EL2_TLBIRVALE1 |
1173 HFGITR_EL2_TLBIRVAAE1 | HFGITR_EL2_TLBIRVAE1 |
1174 HFGITR_EL2_TLBIRVAALE1IS | HFGITR_EL2_TLBIRVALE1IS |
1175 HFGITR_EL2_TLBIRVAAE1IS | HFGITR_EL2_TLBIRVAE1IS |
1176 HFGITR_EL2_TLBIRVAALE1OS | HFGITR_EL2_TLBIRVALE1OS |
1177 HFGITR_EL2_TLBIRVAAE1OS | HFGITR_EL2_TLBIRVAE1OS);
1178 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, SPECRES, IMP))
1179 res0 |= (HFGITR_EL2_CFPRCTX | HFGITR_EL2_DVPRCTX |
1180 HFGITR_EL2_CPPRCTX);
1181 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, BRBE, IMP))
1182 res0 |= (HFGITR_EL2_nBRBINJ | HFGITR_EL2_nBRBIALL);
1183 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP))
1184 res0 |= (HFGITR_EL2_nGCSPUSHM_EL1 | HFGITR_EL2_nGCSSTR_EL1 |
1185 HFGITR_EL2_nGCSEPP);
1186 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, SPECRES, COSP_RCTX))
1187 res0 |= HFGITR_EL2_COSPRCTX;
1188 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, ATS1A, IMP))
1189 res0 |= HFGITR_EL2_ATS1E1A;
1190 set_sysreg_masks(kvm, HFGITR_EL2, res0, res1);
1191
1192 /* HAFGRTR_EL2 - not a lot to see here */
1193 res0 = HAFGRTR_EL2_RES0;
1194 res1 = HAFGRTR_EL2_RES1;
1195 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, V1P1))
1196 res0 |= ~(res0 | res1);
1197 set_sysreg_masks(kvm, HAFGRTR_EL2, res0, res1);
1198 out:
1199 mutex_unlock(&kvm->arch.config_lock);
1200
1201 return ret;
1202 }
1203
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.