1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2012-2015 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7 #include <hyp/adjust_pc.h>
8
9 #include <linux/compiler.h>
10 #include <linux/irqchip/arm-gic-v3.h>
11 #include <linux/kvm_host.h>
12
13 #include <asm/kvm_emulate.h>
14 #include <asm/kvm_hyp.h>
15 #include <asm/kvm_mmu.h>
16
17 #define vtr_to_max_lr_idx(v) ((v) & 0xf)
18 #define vtr_to_nr_pre_bits(v) ((((u32)(v) >> 26) & 7) + 1)
19 #define vtr_to_nr_apr_regs(v) (1 << (vtr_to_nr_pre_bits(v) - 5))
20
21 static u64 __gic_v3_get_lr(unsigned int lr)
22 {
23 switch (lr & 0xf) {
24 case 0:
25 return read_gicreg(ICH_LR0_EL2);
26 case 1:
27 return read_gicreg(ICH_LR1_EL2);
28 case 2:
29 return read_gicreg(ICH_LR2_EL2);
30 case 3:
31 return read_gicreg(ICH_LR3_EL2);
32 case 4:
33 return read_gicreg(ICH_LR4_EL2);
34 case 5:
35 return read_gicreg(ICH_LR5_EL2);
36 case 6:
37 return read_gicreg(ICH_LR6_EL2);
38 case 7:
39 return read_gicreg(ICH_LR7_EL2);
40 case 8:
41 return read_gicreg(ICH_LR8_EL2);
42 case 9:
43 return read_gicreg(ICH_LR9_EL2);
44 case 10:
45 return read_gicreg(ICH_LR10_EL2);
46 case 11:
47 return read_gicreg(ICH_LR11_EL2);
48 case 12:
49 return read_gicreg(ICH_LR12_EL2);
50 case 13:
51 return read_gicreg(ICH_LR13_EL2);
52 case 14:
53 return read_gicreg(ICH_LR14_EL2);
54 case 15:
55 return read_gicreg(ICH_LR15_EL2);
56 }
57
58 unreachable();
59 }
60
61 static void __gic_v3_set_lr(u64 val, int lr)
62 {
63 switch (lr & 0xf) {
64 case 0:
65 write_gicreg(val, ICH_LR0_EL2);
66 break;
67 case 1:
68 write_gicreg(val, ICH_LR1_EL2);
69 break;
70 case 2:
71 write_gicreg(val, ICH_LR2_EL2);
72 break;
73 case 3:
74 write_gicreg(val, ICH_LR3_EL2);
75 break;
76 case 4:
77 write_gicreg(val, ICH_LR4_EL2);
78 break;
79 case 5:
80 write_gicreg(val, ICH_LR5_EL2);
81 break;
82 case 6:
83 write_gicreg(val, ICH_LR6_EL2);
84 break;
85 case 7:
86 write_gicreg(val, ICH_LR7_EL2);
87 break;
88 case 8:
89 write_gicreg(val, ICH_LR8_EL2);
90 break;
91 case 9:
92 write_gicreg(val, ICH_LR9_EL2);
93 break;
94 case 10:
95 write_gicreg(val, ICH_LR10_EL2);
96 break;
97 case 11:
98 write_gicreg(val, ICH_LR11_EL2);
99 break;
100 case 12:
101 write_gicreg(val, ICH_LR12_EL2);
102 break;
103 case 13:
104 write_gicreg(val, ICH_LR13_EL2);
105 break;
106 case 14:
107 write_gicreg(val, ICH_LR14_EL2);
108 break;
109 case 15:
110 write_gicreg(val, ICH_LR15_EL2);
111 break;
112 }
113 }
114
115 static void __vgic_v3_write_ap0rn(u32 val, int n)
116 {
117 switch (n) {
118 case 0:
119 write_gicreg(val, ICH_AP0R0_EL2);
120 break;
121 case 1:
122 write_gicreg(val, ICH_AP0R1_EL2);
123 break;
124 case 2:
125 write_gicreg(val, ICH_AP0R2_EL2);
126 break;
127 case 3:
128 write_gicreg(val, ICH_AP0R3_EL2);
129 break;
130 }
131 }
132
133 static void __vgic_v3_write_ap1rn(u32 val, int n)
134 {
135 switch (n) {
136 case 0:
137 write_gicreg(val, ICH_AP1R0_EL2);
138 break;
139 case 1:
140 write_gicreg(val, ICH_AP1R1_EL2);
141 break;
142 case 2:
143 write_gicreg(val, ICH_AP1R2_EL2);
144 break;
145 case 3:
146 write_gicreg(val, ICH_AP1R3_EL2);
147 break;
148 }
149 }
150
151 static u32 __vgic_v3_read_ap0rn(int n)
152 {
153 u32 val;
154
155 switch (n) {
156 case 0:
157 val = read_gicreg(ICH_AP0R0_EL2);
158 break;
159 case 1:
160 val = read_gicreg(ICH_AP0R1_EL2);
161 break;
162 case 2:
163 val = read_gicreg(ICH_AP0R2_EL2);
164 break;
165 case 3:
166 val = read_gicreg(ICH_AP0R3_EL2);
167 break;
168 default:
169 unreachable();
170 }
171
172 return val;
173 }
174
175 static u32 __vgic_v3_read_ap1rn(int n)
176 {
177 u32 val;
178
179 switch (n) {
180 case 0:
181 val = read_gicreg(ICH_AP1R0_EL2);
182 break;
183 case 1:
184 val = read_gicreg(ICH_AP1R1_EL2);
185 break;
186 case 2:
187 val = read_gicreg(ICH_AP1R2_EL2);
188 break;
189 case 3:
190 val = read_gicreg(ICH_AP1R3_EL2);
191 break;
192 default:
193 unreachable();
194 }
195
196 return val;
197 }
198
199 void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if)
200 {
201 u64 used_lrs = cpu_if->used_lrs;
202
203 /*
204 * Make sure stores to the GIC via the memory mapped interface
205 * are now visible to the system register interface when reading the
206 * LRs, and when reading back the VMCR on non-VHE systems.
207 */
208 if (used_lrs || !has_vhe()) {
209 if (!cpu_if->vgic_sre) {
210 dsb(sy);
211 isb();
212 }
213 }
214
215 if (used_lrs || cpu_if->its_vpe.its_vm) {
216 int i;
217 u32 elrsr;
218
219 elrsr = read_gicreg(ICH_ELRSR_EL2);
220
221 write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EN, ICH_HCR_EL2);
222
223 for (i = 0; i < used_lrs; i++) {
224 if (elrsr & (1 << i))
225 cpu_if->vgic_lr[i] &= ~ICH_LR_STATE;
226 else
227 cpu_if->vgic_lr[i] = __gic_v3_get_lr(i);
228
229 __gic_v3_set_lr(0, i);
230 }
231 }
232 }
233
234 void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if)
235 {
236 u64 used_lrs = cpu_if->used_lrs;
237 int i;
238
239 if (used_lrs || cpu_if->its_vpe.its_vm) {
240 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
241
242 for (i = 0; i < used_lrs; i++)
243 __gic_v3_set_lr(cpu_if->vgic_lr[i], i);
244 }
245
246 /*
247 * Ensure that writes to the LRs, and on non-VHE systems ensure that
248 * the write to the VMCR in __vgic_v3_activate_traps(), will have
249 * reached the (re)distributors. This ensure the guest will read the
250 * correct values from the memory-mapped interface.
251 */
252 if (used_lrs || !has_vhe()) {
253 if (!cpu_if->vgic_sre) {
254 isb();
255 dsb(sy);
256 }
257 }
258 }
259
260 void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if)
261 {
262 /*
263 * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a
264 * Group0 interrupt (as generated in GICv2 mode) to be
265 * delivered as a FIQ to the guest, with potentially fatal
266 * consequences. So we must make sure that ICC_SRE_EL1 has
267 * been actually programmed with the value we want before
268 * starting to mess with the rest of the GIC, and VMCR_EL2 in
269 * particular. This logic must be called before
270 * __vgic_v3_restore_state().
271 */
272 if (!cpu_if->vgic_sre) {
273 write_gicreg(0, ICC_SRE_EL1);
274 isb();
275 write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
276
277
278 if (has_vhe()) {
279 /*
280 * Ensure that the write to the VMCR will have reached
281 * the (re)distributors. This ensure the guest will
282 * read the correct values from the memory-mapped
283 * interface.
284 */
285 isb();
286 dsb(sy);
287 }
288 }
289
290 /*
291 * Prevent the guest from touching the GIC system registers if
292 * SRE isn't enabled for GICv3 emulation.
293 */
294 write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
295 ICC_SRE_EL2);
296
297 /*
298 * If we need to trap system registers, we must write
299 * ICH_HCR_EL2 anyway, even if no interrupts are being
300 * injected,
301 */
302 if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
303 cpu_if->its_vpe.its_vm)
304 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
305 }
306
307 void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if)
308 {
309 u64 val;
310
311 if (!cpu_if->vgic_sre) {
312 cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
313 }
314
315 val = read_gicreg(ICC_SRE_EL2);
316 write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
317
318 if (!cpu_if->vgic_sre) {
319 /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
320 isb();
321 write_gicreg(1, ICC_SRE_EL1);
322 }
323
324 /*
325 * If we were trapping system registers, we enabled the VGIC even if
326 * no interrupts were being injected, and we disable it again here.
327 */
328 if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
329 cpu_if->its_vpe.its_vm)
330 write_gicreg(0, ICH_HCR_EL2);
331 }
332
333 static void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if)
334 {
335 u64 val;
336 u32 nr_pre_bits;
337
338 val = read_gicreg(ICH_VTR_EL2);
339 nr_pre_bits = vtr_to_nr_pre_bits(val);
340
341 switch (nr_pre_bits) {
342 case 7:
343 cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3);
344 cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2);
345 fallthrough;
346 case 6:
347 cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1);
348 fallthrough;
349 default:
350 cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0);
351 }
352
353 switch (nr_pre_bits) {
354 case 7:
355 cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3);
356 cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2);
357 fallthrough;
358 case 6:
359 cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1);
360 fallthrough;
361 default:
362 cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0);
363 }
364 }
365
366 static void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if)
367 {
368 u64 val;
369 u32 nr_pre_bits;
370
371 val = read_gicreg(ICH_VTR_EL2);
372 nr_pre_bits = vtr_to_nr_pre_bits(val);
373
374 switch (nr_pre_bits) {
375 case 7:
376 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3);
377 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2);
378 fallthrough;
379 case 6:
380 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1);
381 fallthrough;
382 default:
383 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0);
384 }
385
386 switch (nr_pre_bits) {
387 case 7:
388 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3);
389 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2);
390 fallthrough;
391 case 6:
392 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1);
393 fallthrough;
394 default:
395 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0);
396 }
397 }
398
399 void __vgic_v3_init_lrs(void)
400 {
401 int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2));
402 int i;
403
404 for (i = 0; i <= max_lr_idx; i++)
405 __gic_v3_set_lr(0, i);
406 }
407
408 /*
409 * Return the GIC CPU configuration:
410 * - [31:0] ICH_VTR_EL2
411 * - [62:32] RES0
412 * - [63] MMIO (GICv2) capable
413 */
414 u64 __vgic_v3_get_gic_config(void)
415 {
416 u64 val, sre = read_gicreg(ICC_SRE_EL1);
417 unsigned long flags = 0;
418
419 /*
420 * To check whether we have a MMIO-based (GICv2 compatible)
421 * CPU interface, we need to disable the system register
422 * view. To do that safely, we have to prevent any interrupt
423 * from firing (which would be deadly).
424 *
425 * Note that this only makes sense on VHE, as interrupts are
426 * already masked for nVHE as part of the exception entry to
427 * EL2.
428 */
429 if (has_vhe())
430 flags = local_daif_save();
431
432 /*
433 * Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates
434 * that to be able to set ICC_SRE_EL1.SRE to 0, all the
435 * interrupt overrides must be set. You've got to love this.
436 */
437 sysreg_clear_set(hcr_el2, 0, HCR_AMO | HCR_FMO | HCR_IMO);
438 isb();
439 write_gicreg(0, ICC_SRE_EL1);
440 isb();
441
442 val = read_gicreg(ICC_SRE_EL1);
443
444 write_gicreg(sre, ICC_SRE_EL1);
445 isb();
446 sysreg_clear_set(hcr_el2, HCR_AMO | HCR_FMO | HCR_IMO, 0);
447 isb();
448
449 if (has_vhe())
450 local_daif_restore(flags);
451
452 val = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63);
453 val |= read_gicreg(ICH_VTR_EL2);
454
455 return val;
456 }
457
458 static u64 __vgic_v3_read_vmcr(void)
459 {
460 return read_gicreg(ICH_VMCR_EL2);
461 }
462
463 static void __vgic_v3_write_vmcr(u32 vmcr)
464 {
465 write_gicreg(vmcr, ICH_VMCR_EL2);
466 }
467
468 void __vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
469 {
470 __vgic_v3_save_aprs(cpu_if);
471 if (cpu_if->vgic_sre)
472 cpu_if->vgic_vmcr = __vgic_v3_read_vmcr();
473 }
474
475 void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
476 {
477 /*
478 * If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen
479 * is dependent on ICC_SRE_EL1.SRE, and we have to perform the
480 * VMCR_EL2 save/restore in the world switch.
481 */
482 if (cpu_if->vgic_sre)
483 __vgic_v3_write_vmcr(cpu_if->vgic_vmcr);
484 __vgic_v3_restore_aprs(cpu_if);
485 }
486
487 static int __vgic_v3_bpr_min(void)
488 {
489 /* See Pseudocode for VPriorityGroup */
490 return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2));
491 }
492
493 static int __vgic_v3_get_group(struct kvm_vcpu *vcpu)
494 {
495 u64 esr = kvm_vcpu_get_esr(vcpu);
496 u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
497
498 return crm != 8;
499 }
500
501 #define GICv3_IDLE_PRIORITY 0xff
502
503 static int __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, u32 vmcr,
504 u64 *lr_val)
505 {
506 unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
507 u8 priority = GICv3_IDLE_PRIORITY;
508 int i, lr = -1;
509
510 for (i = 0; i < used_lrs; i++) {
511 u64 val = __gic_v3_get_lr(i);
512 u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
513
514 /* Not pending in the state? */
515 if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT)
516 continue;
517
518 /* Group-0 interrupt, but Group-0 disabled? */
519 if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK))
520 continue;
521
522 /* Group-1 interrupt, but Group-1 disabled? */
523 if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK))
524 continue;
525
526 /* Not the highest priority? */
527 if (lr_prio >= priority)
528 continue;
529
530 /* This is a candidate */
531 priority = lr_prio;
532 *lr_val = val;
533 lr = i;
534 }
535
536 if (lr == -1)
537 *lr_val = ICC_IAR1_EL1_SPURIOUS;
538
539 return lr;
540 }
541
542 static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid,
543 u64 *lr_val)
544 {
545 unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
546 int i;
547
548 for (i = 0; i < used_lrs; i++) {
549 u64 val = __gic_v3_get_lr(i);
550
551 if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid &&
552 (val & ICH_LR_ACTIVE_BIT)) {
553 *lr_val = val;
554 return i;
555 }
556 }
557
558 *lr_val = ICC_IAR1_EL1_SPURIOUS;
559 return -1;
560 }
561
562 static int __vgic_v3_get_highest_active_priority(void)
563 {
564 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
565 u32 hap = 0;
566 int i;
567
568 for (i = 0; i < nr_apr_regs; i++) {
569 u32 val;
570
571 /*
572 * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers
573 * contain the active priority levels for this VCPU
574 * for the maximum number of supported priority
575 * levels, and we return the full priority level only
576 * if the BPR is programmed to its minimum, otherwise
577 * we return a combination of the priority level and
578 * subpriority, as determined by the setting of the
579 * BPR, but without the full subpriority.
580 */
581 val = __vgic_v3_read_ap0rn(i);
582 val |= __vgic_v3_read_ap1rn(i);
583 if (!val) {
584 hap += 32;
585 continue;
586 }
587
588 return (hap + __ffs(val)) << __vgic_v3_bpr_min();
589 }
590
591 return GICv3_IDLE_PRIORITY;
592 }
593
594 static unsigned int __vgic_v3_get_bpr0(u32 vmcr)
595 {
596 return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT;
597 }
598
599 static unsigned int __vgic_v3_get_bpr1(u32 vmcr)
600 {
601 unsigned int bpr;
602
603 if (vmcr & ICH_VMCR_CBPR_MASK) {
604 bpr = __vgic_v3_get_bpr0(vmcr);
605 if (bpr < 7)
606 bpr++;
607 } else {
608 bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT;
609 }
610
611 return bpr;
612 }
613
614 /*
615 * Convert a priority to a preemption level, taking the relevant BPR
616 * into account by zeroing the sub-priority bits.
617 */
618 static u8 __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp)
619 {
620 unsigned int bpr;
621
622 if (!grp)
623 bpr = __vgic_v3_get_bpr0(vmcr) + 1;
624 else
625 bpr = __vgic_v3_get_bpr1(vmcr);
626
627 return pri & (GENMASK(7, 0) << bpr);
628 }
629
630 /*
631 * The priority value is independent of any of the BPR values, so we
632 * normalize it using the minimal BPR value. This guarantees that no
633 * matter what the guest does with its BPR, we can always set/get the
634 * same value of a priority.
635 */
636 static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp)
637 {
638 u8 pre, ap;
639 u32 val;
640 int apr;
641
642 pre = __vgic_v3_pri_to_pre(pri, vmcr, grp);
643 ap = pre >> __vgic_v3_bpr_min();
644 apr = ap / 32;
645
646 if (!grp) {
647 val = __vgic_v3_read_ap0rn(apr);
648 __vgic_v3_write_ap0rn(val | BIT(ap % 32), apr);
649 } else {
650 val = __vgic_v3_read_ap1rn(apr);
651 __vgic_v3_write_ap1rn(val | BIT(ap % 32), apr);
652 }
653 }
654
655 static int __vgic_v3_clear_highest_active_priority(void)
656 {
657 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
658 u32 hap = 0;
659 int i;
660
661 for (i = 0; i < nr_apr_regs; i++) {
662 u32 ap0, ap1;
663 int c0, c1;
664
665 ap0 = __vgic_v3_read_ap0rn(i);
666 ap1 = __vgic_v3_read_ap1rn(i);
667 if (!ap0 && !ap1) {
668 hap += 32;
669 continue;
670 }
671
672 c0 = ap0 ? __ffs(ap0) : 32;
673 c1 = ap1 ? __ffs(ap1) : 32;
674
675 /* Always clear the LSB, which is the highest priority */
676 if (c0 < c1) {
677 ap0 &= ~BIT(c0);
678 __vgic_v3_write_ap0rn(ap0, i);
679 hap += c0;
680 } else {
681 ap1 &= ~BIT(c1);
682 __vgic_v3_write_ap1rn(ap1, i);
683 hap += c1;
684 }
685
686 /* Rescale to 8 bits of priority */
687 return hap << __vgic_v3_bpr_min();
688 }
689
690 return GICv3_IDLE_PRIORITY;
691 }
692
693 static void __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
694 {
695 u64 lr_val;
696 u8 lr_prio, pmr;
697 int lr, grp;
698
699 grp = __vgic_v3_get_group(vcpu);
700
701 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
702 if (lr < 0)
703 goto spurious;
704
705 if (grp != !!(lr_val & ICH_LR_GROUP))
706 goto spurious;
707
708 pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT;
709 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
710 if (pmr <= lr_prio)
711 goto spurious;
712
713 if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp))
714 goto spurious;
715
716 lr_val &= ~ICH_LR_STATE;
717 lr_val |= ICH_LR_ACTIVE_BIT;
718 __gic_v3_set_lr(lr_val, lr);
719 __vgic_v3_set_active_priority(lr_prio, vmcr, grp);
720 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
721 return;
722
723 spurious:
724 vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS);
725 }
726
727 static void __vgic_v3_clear_active_lr(int lr, u64 lr_val)
728 {
729 lr_val &= ~ICH_LR_ACTIVE_BIT;
730 if (lr_val & ICH_LR_HW) {
731 u32 pid;
732
733 pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT;
734 gic_write_dir(pid);
735 }
736
737 __gic_v3_set_lr(lr_val, lr);
738 }
739
740 static void __vgic_v3_bump_eoicount(void)
741 {
742 u32 hcr;
743
744 hcr = read_gicreg(ICH_HCR_EL2);
745 hcr += 1 << ICH_HCR_EOIcount_SHIFT;
746 write_gicreg(hcr, ICH_HCR_EL2);
747 }
748
749 static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
750 {
751 u32 vid = vcpu_get_reg(vcpu, rt);
752 u64 lr_val;
753 int lr;
754
755 /* EOImode == 0, nothing to be done here */
756 if (!(vmcr & ICH_VMCR_EOIM_MASK))
757 return;
758
759 /* No deactivate to be performed on an LPI */
760 if (vid >= VGIC_MIN_LPI)
761 return;
762
763 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
764 if (lr == -1) {
765 __vgic_v3_bump_eoicount();
766 return;
767 }
768
769 __vgic_v3_clear_active_lr(lr, lr_val);
770 }
771
772 static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
773 {
774 u32 vid = vcpu_get_reg(vcpu, rt);
775 u64 lr_val;
776 u8 lr_prio, act_prio;
777 int lr, grp;
778
779 grp = __vgic_v3_get_group(vcpu);
780
781 /* Drop priority in any case */
782 act_prio = __vgic_v3_clear_highest_active_priority();
783
784 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
785 if (lr == -1) {
786 /* Do not bump EOIcount for LPIs that aren't in the LRs */
787 if (!(vid >= VGIC_MIN_LPI))
788 __vgic_v3_bump_eoicount();
789 return;
790 }
791
792 /* EOImode == 1 and not an LPI, nothing to be done here */
793 if ((vmcr & ICH_VMCR_EOIM_MASK) && !(vid >= VGIC_MIN_LPI))
794 return;
795
796 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
797
798 /* If priorities or group do not match, the guest has fscked-up. */
799 if (grp != !!(lr_val & ICH_LR_GROUP) ||
800 __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio)
801 return;
802
803 /* Let's now perform the deactivation */
804 __vgic_v3_clear_active_lr(lr, lr_val);
805 }
806
807 static void __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
808 {
809 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK));
810 }
811
812 static void __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
813 {
814 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK));
815 }
816
817 static void __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
818 {
819 u64 val = vcpu_get_reg(vcpu, rt);
820
821 if (val & 1)
822 vmcr |= ICH_VMCR_ENG0_MASK;
823 else
824 vmcr &= ~ICH_VMCR_ENG0_MASK;
825
826 __vgic_v3_write_vmcr(vmcr);
827 }
828
829 static void __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
830 {
831 u64 val = vcpu_get_reg(vcpu, rt);
832
833 if (val & 1)
834 vmcr |= ICH_VMCR_ENG1_MASK;
835 else
836 vmcr &= ~ICH_VMCR_ENG1_MASK;
837
838 __vgic_v3_write_vmcr(vmcr);
839 }
840
841 static void __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
842 {
843 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr));
844 }
845
846 static void __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
847 {
848 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr));
849 }
850
851 static void __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
852 {
853 u64 val = vcpu_get_reg(vcpu, rt);
854 u8 bpr_min = __vgic_v3_bpr_min() - 1;
855
856 /* Enforce BPR limiting */
857 if (val < bpr_min)
858 val = bpr_min;
859
860 val <<= ICH_VMCR_BPR0_SHIFT;
861 val &= ICH_VMCR_BPR0_MASK;
862 vmcr &= ~ICH_VMCR_BPR0_MASK;
863 vmcr |= val;
864
865 __vgic_v3_write_vmcr(vmcr);
866 }
867
868 static void __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
869 {
870 u64 val = vcpu_get_reg(vcpu, rt);
871 u8 bpr_min = __vgic_v3_bpr_min();
872
873 if (vmcr & ICH_VMCR_CBPR_MASK)
874 return;
875
876 /* Enforce BPR limiting */
877 if (val < bpr_min)
878 val = bpr_min;
879
880 val <<= ICH_VMCR_BPR1_SHIFT;
881 val &= ICH_VMCR_BPR1_MASK;
882 vmcr &= ~ICH_VMCR_BPR1_MASK;
883 vmcr |= val;
884
885 __vgic_v3_write_vmcr(vmcr);
886 }
887
888 static void __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
889 {
890 u32 val;
891
892 if (!__vgic_v3_get_group(vcpu))
893 val = __vgic_v3_read_ap0rn(n);
894 else
895 val = __vgic_v3_read_ap1rn(n);
896
897 vcpu_set_reg(vcpu, rt, val);
898 }
899
900 static void __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
901 {
902 u32 val = vcpu_get_reg(vcpu, rt);
903
904 if (!__vgic_v3_get_group(vcpu))
905 __vgic_v3_write_ap0rn(val, n);
906 else
907 __vgic_v3_write_ap1rn(val, n);
908 }
909
910 static void __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu,
911 u32 vmcr, int rt)
912 {
913 __vgic_v3_read_apxrn(vcpu, rt, 0);
914 }
915
916 static void __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu,
917 u32 vmcr, int rt)
918 {
919 __vgic_v3_read_apxrn(vcpu, rt, 1);
920 }
921
922 static void __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
923 {
924 __vgic_v3_read_apxrn(vcpu, rt, 2);
925 }
926
927 static void __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
928 {
929 __vgic_v3_read_apxrn(vcpu, rt, 3);
930 }
931
932 static void __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
933 {
934 __vgic_v3_write_apxrn(vcpu, rt, 0);
935 }
936
937 static void __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
938 {
939 __vgic_v3_write_apxrn(vcpu, rt, 1);
940 }
941
942 static void __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
943 {
944 __vgic_v3_write_apxrn(vcpu, rt, 2);
945 }
946
947 static void __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
948 {
949 __vgic_v3_write_apxrn(vcpu, rt, 3);
950 }
951
952 static void __vgic_v3_read_hppir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
953 {
954 u64 lr_val;
955 int lr, lr_grp, grp;
956
957 grp = __vgic_v3_get_group(vcpu);
958
959 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
960 if (lr == -1)
961 goto spurious;
962
963 lr_grp = !!(lr_val & ICH_LR_GROUP);
964 if (lr_grp != grp)
965 lr_val = ICC_IAR1_EL1_SPURIOUS;
966
967 spurious:
968 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
969 }
970
971 static void __vgic_v3_read_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
972 {
973 vmcr &= ICH_VMCR_PMR_MASK;
974 vmcr >>= ICH_VMCR_PMR_SHIFT;
975 vcpu_set_reg(vcpu, rt, vmcr);
976 }
977
978 static void __vgic_v3_write_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
979 {
980 u32 val = vcpu_get_reg(vcpu, rt);
981
982 val <<= ICH_VMCR_PMR_SHIFT;
983 val &= ICH_VMCR_PMR_MASK;
984 vmcr &= ~ICH_VMCR_PMR_MASK;
985 vmcr |= val;
986
987 write_gicreg(vmcr, ICH_VMCR_EL2);
988 }
989
990 static void __vgic_v3_read_rpr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
991 {
992 u32 val = __vgic_v3_get_highest_active_priority();
993 vcpu_set_reg(vcpu, rt, val);
994 }
995
996 static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
997 {
998 u32 vtr, val;
999
1000 vtr = read_gicreg(ICH_VTR_EL2);
1001 /* PRIbits */
1002 val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT;
1003 /* IDbits */
1004 val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT;
1005 /* SEIS */
1006 if (kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_SEIS_MASK)
1007 val |= BIT(ICC_CTLR_EL1_SEIS_SHIFT);
1008 /* A3V */
1009 val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT;
1010 /* EOImode */
1011 val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT;
1012 /* CBPR */
1013 val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT;
1014
1015 vcpu_set_reg(vcpu, rt, val);
1016 }
1017
1018 static void __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1019 {
1020 u32 val = vcpu_get_reg(vcpu, rt);
1021
1022 if (val & ICC_CTLR_EL1_CBPR_MASK)
1023 vmcr |= ICH_VMCR_CBPR_MASK;
1024 else
1025 vmcr &= ~ICH_VMCR_CBPR_MASK;
1026
1027 if (val & ICC_CTLR_EL1_EOImode_MASK)
1028 vmcr |= ICH_VMCR_EOIM_MASK;
1029 else
1030 vmcr &= ~ICH_VMCR_EOIM_MASK;
1031
1032 write_gicreg(vmcr, ICH_VMCR_EL2);
1033 }
1034
1035 int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu)
1036 {
1037 int rt;
1038 u64 esr;
1039 u32 vmcr;
1040 void (*fn)(struct kvm_vcpu *, u32, int);
1041 bool is_read;
1042 u32 sysreg;
1043
1044 esr = kvm_vcpu_get_esr(vcpu);
1045 if (vcpu_mode_is_32bit(vcpu)) {
1046 if (!kvm_condition_valid(vcpu)) {
1047 __kvm_skip_instr(vcpu);
1048 return 1;
1049 }
1050
1051 sysreg = esr_cp15_to_sysreg(esr);
1052 } else {
1053 sysreg = esr_sys64_to_sysreg(esr);
1054 }
1055
1056 is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
1057
1058 switch (sysreg) {
1059 case SYS_ICC_IAR0_EL1:
1060 case SYS_ICC_IAR1_EL1:
1061 if (unlikely(!is_read))
1062 return 0;
1063 fn = __vgic_v3_read_iar;
1064 break;
1065 case SYS_ICC_EOIR0_EL1:
1066 case SYS_ICC_EOIR1_EL1:
1067 if (unlikely(is_read))
1068 return 0;
1069 fn = __vgic_v3_write_eoir;
1070 break;
1071 case SYS_ICC_IGRPEN1_EL1:
1072 if (is_read)
1073 fn = __vgic_v3_read_igrpen1;
1074 else
1075 fn = __vgic_v3_write_igrpen1;
1076 break;
1077 case SYS_ICC_BPR1_EL1:
1078 if (is_read)
1079 fn = __vgic_v3_read_bpr1;
1080 else
1081 fn = __vgic_v3_write_bpr1;
1082 break;
1083 case SYS_ICC_AP0Rn_EL1(0):
1084 case SYS_ICC_AP1Rn_EL1(0):
1085 if (is_read)
1086 fn = __vgic_v3_read_apxr0;
1087 else
1088 fn = __vgic_v3_write_apxr0;
1089 break;
1090 case SYS_ICC_AP0Rn_EL1(1):
1091 case SYS_ICC_AP1Rn_EL1(1):
1092 if (is_read)
1093 fn = __vgic_v3_read_apxr1;
1094 else
1095 fn = __vgic_v3_write_apxr1;
1096 break;
1097 case SYS_ICC_AP0Rn_EL1(2):
1098 case SYS_ICC_AP1Rn_EL1(2):
1099 if (is_read)
1100 fn = __vgic_v3_read_apxr2;
1101 else
1102 fn = __vgic_v3_write_apxr2;
1103 break;
1104 case SYS_ICC_AP0Rn_EL1(3):
1105 case SYS_ICC_AP1Rn_EL1(3):
1106 if (is_read)
1107 fn = __vgic_v3_read_apxr3;
1108 else
1109 fn = __vgic_v3_write_apxr3;
1110 break;
1111 case SYS_ICC_HPPIR0_EL1:
1112 case SYS_ICC_HPPIR1_EL1:
1113 if (unlikely(!is_read))
1114 return 0;
1115 fn = __vgic_v3_read_hppir;
1116 break;
1117 case SYS_ICC_IGRPEN0_EL1:
1118 if (is_read)
1119 fn = __vgic_v3_read_igrpen0;
1120 else
1121 fn = __vgic_v3_write_igrpen0;
1122 break;
1123 case SYS_ICC_BPR0_EL1:
1124 if (is_read)
1125 fn = __vgic_v3_read_bpr0;
1126 else
1127 fn = __vgic_v3_write_bpr0;
1128 break;
1129 case SYS_ICC_DIR_EL1:
1130 if (unlikely(is_read))
1131 return 0;
1132 fn = __vgic_v3_write_dir;
1133 break;
1134 case SYS_ICC_RPR_EL1:
1135 if (unlikely(!is_read))
1136 return 0;
1137 fn = __vgic_v3_read_rpr;
1138 break;
1139 case SYS_ICC_CTLR_EL1:
1140 if (is_read)
1141 fn = __vgic_v3_read_ctlr;
1142 else
1143 fn = __vgic_v3_write_ctlr;
1144 break;
1145 case SYS_ICC_PMR_EL1:
1146 if (is_read)
1147 fn = __vgic_v3_read_pmr;
1148 else
1149 fn = __vgic_v3_write_pmr;
1150 break;
1151 default:
1152 return 0;
1153 }
1154
1155 vmcr = __vgic_v3_read_vmcr();
1156 rt = kvm_vcpu_sys_get_rt(vcpu);
1157 fn(vcpu, vmcr, rt);
1158
1159 __kvm_skip_instr(vcpu);
1160
1161 return 1;
1162 }
1163
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.