1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright 2017 Benjamin Herrenschmidt, IBM Corporation.
4 */
5
6 #define pr_fmt(fmt) "xive-kvm: " fmt
7
8 #include <linux/kernel.h>
9 #include <linux/kvm_host.h>
10 #include <linux/err.h>
11 #include <linux/gfp.h>
12 #include <linux/spinlock.h>
13 #include <linux/delay.h>
14 #include <linux/percpu.h>
15 #include <linux/cpumask.h>
16 #include <linux/uaccess.h>
17 #include <linux/irqdomain.h>
18 #include <asm/kvm_book3s.h>
19 #include <asm/kvm_ppc.h>
20 #include <asm/hvcall.h>
21 #include <asm/xics.h>
22 #include <asm/xive.h>
23 #include <asm/xive-regs.h>
24 #include <asm/debug.h>
25 #include <asm/time.h>
26 #include <asm/opal.h>
27
28 #include <linux/debugfs.h>
29 #include <linux/seq_file.h>
30
31 #include "book3s_xive.h"
32
33 #define __x_eoi_page(xd) ((void __iomem *)((xd)->eoi_mmio))
34 #define __x_trig_page(xd) ((void __iomem *)((xd)->trig_mmio))
35
36 /* Dummy interrupt used when taking interrupts out of a queue in H_CPPR */
37 #define XICS_DUMMY 1
38
39 static void xive_vm_ack_pending(struct kvmppc_xive_vcpu *xc)
40 {
41 u8 cppr;
42 u16 ack;
43
44 /*
45 * Ensure any previous store to CPPR is ordered vs.
46 * the subsequent loads from PIPR or ACK.
47 */
48 eieio();
49
50 /* Perform the acknowledge OS to register cycle. */
51 ack = be16_to_cpu(__raw_readw(xive_tima + TM_SPC_ACK_OS_REG));
52
53 /* Synchronize subsequent queue accesses */
54 mb();
55
56 /* XXX Check grouping level */
57
58 /* Anything ? */
59 if (!((ack >> 8) & TM_QW1_NSR_EO))
60 return;
61
62 /* Grab CPPR of the most favored pending interrupt */
63 cppr = ack & 0xff;
64 if (cppr < 8)
65 xc->pending |= 1 << cppr;
66
67 /* Check consistency */
68 if (cppr >= xc->hw_cppr)
69 pr_warn("KVM-XIVE: CPU %d odd ack CPPR, got %d at %d\n",
70 smp_processor_id(), cppr, xc->hw_cppr);
71
72 /*
73 * Update our image of the HW CPPR. We don't yet modify
74 * xc->cppr, this will be done as we scan for interrupts
75 * in the queues.
76 */
77 xc->hw_cppr = cppr;
78 }
79
80 static u8 xive_vm_esb_load(struct xive_irq_data *xd, u32 offset)
81 {
82 u64 val;
83
84 if (offset == XIVE_ESB_SET_PQ_10 && xd->flags & XIVE_IRQ_FLAG_STORE_EOI)
85 offset |= XIVE_ESB_LD_ST_MO;
86
87 val = __raw_readq(__x_eoi_page(xd) + offset);
88 #ifdef __LITTLE_ENDIAN__
89 val >>= 64-8;
90 #endif
91 return (u8)val;
92 }
93
94
95 static void xive_vm_source_eoi(u32 hw_irq, struct xive_irq_data *xd)
96 {
97 /* If the XIVE supports the new "store EOI facility, use it */
98 if (xd->flags & XIVE_IRQ_FLAG_STORE_EOI)
99 __raw_writeq(0, __x_eoi_page(xd) + XIVE_ESB_STORE_EOI);
100 else if (xd->flags & XIVE_IRQ_FLAG_LSI) {
101 /*
102 * For LSIs the HW EOI cycle is used rather than PQ bits,
103 * as they are automatically re-triggred in HW when still
104 * pending.
105 */
106 __raw_readq(__x_eoi_page(xd) + XIVE_ESB_LOAD_EOI);
107 } else {
108 uint64_t eoi_val;
109
110 /*
111 * Otherwise for EOI, we use the special MMIO that does
112 * a clear of both P and Q and returns the old Q,
113 * except for LSIs where we use the "EOI cycle" special
114 * load.
115 *
116 * This allows us to then do a re-trigger if Q was set
117 * rather than synthetizing an interrupt in software
118 */
119 eoi_val = xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_00);
120
121 /* Re-trigger if needed */
122 if ((eoi_val & 1) && __x_trig_page(xd))
123 __raw_writeq(0, __x_trig_page(xd));
124 }
125 }
126
127 enum {
128 scan_fetch,
129 scan_poll,
130 scan_eoi,
131 };
132
133 static u32 xive_vm_scan_interrupts(struct kvmppc_xive_vcpu *xc,
134 u8 pending, int scan_type)
135 {
136 u32 hirq = 0;
137 u8 prio = 0xff;
138
139 /* Find highest pending priority */
140 while ((xc->mfrr != 0xff || pending != 0) && hirq == 0) {
141 struct xive_q *q;
142 u32 idx, toggle;
143 __be32 *qpage;
144
145 /*
146 * If pending is 0 this will return 0xff which is what
147 * we want
148 */
149 prio = ffs(pending) - 1;
150
151 /* Don't scan past the guest cppr */
152 if (prio >= xc->cppr || prio > 7) {
153 if (xc->mfrr < xc->cppr) {
154 prio = xc->mfrr;
155 hirq = XICS_IPI;
156 }
157 break;
158 }
159
160 /* Grab queue and pointers */
161 q = &xc->queues[prio];
162 idx = q->idx;
163 toggle = q->toggle;
164
165 /*
166 * Snapshot the queue page. The test further down for EOI
167 * must use the same "copy" that was used by __xive_read_eq
168 * since qpage can be set concurrently and we don't want
169 * to miss an EOI.
170 */
171 qpage = READ_ONCE(q->qpage);
172
173 skip_ipi:
174 /*
175 * Try to fetch from the queue. Will return 0 for a
176 * non-queueing priority (ie, qpage = 0).
177 */
178 hirq = __xive_read_eq(qpage, q->msk, &idx, &toggle);
179
180 /*
181 * If this was a signal for an MFFR change done by
182 * H_IPI we skip it. Additionally, if we were fetching
183 * we EOI it now, thus re-enabling reception of a new
184 * such signal.
185 *
186 * We also need to do that if prio is 0 and we had no
187 * page for the queue. In this case, we have non-queued
188 * IPI that needs to be EOId.
189 *
190 * This is safe because if we have another pending MFRR
191 * change that wasn't observed above, the Q bit will have
192 * been set and another occurrence of the IPI will trigger.
193 */
194 if (hirq == XICS_IPI || (prio == 0 && !qpage)) {
195 if (scan_type == scan_fetch) {
196 xive_vm_source_eoi(xc->vp_ipi,
197 &xc->vp_ipi_data);
198 q->idx = idx;
199 q->toggle = toggle;
200 }
201 /* Loop back on same queue with updated idx/toggle */
202 WARN_ON(hirq && hirq != XICS_IPI);
203 if (hirq)
204 goto skip_ipi;
205 }
206
207 /* If it's the dummy interrupt, continue searching */
208 if (hirq == XICS_DUMMY)
209 goto skip_ipi;
210
211 /* Clear the pending bit if the queue is now empty */
212 if (!hirq) {
213 pending &= ~(1 << prio);
214
215 /*
216 * Check if the queue count needs adjusting due to
217 * interrupts being moved away.
218 */
219 if (atomic_read(&q->pending_count)) {
220 int p = atomic_xchg(&q->pending_count, 0);
221
222 if (p) {
223 WARN_ON(p > atomic_read(&q->count));
224 atomic_sub(p, &q->count);
225 }
226 }
227 }
228
229 /*
230 * If the most favoured prio we found pending is less
231 * favored (or equal) than a pending IPI, we return
232 * the IPI instead.
233 */
234 if (prio >= xc->mfrr && xc->mfrr < xc->cppr) {
235 prio = xc->mfrr;
236 hirq = XICS_IPI;
237 break;
238 }
239
240 /* If fetching, update queue pointers */
241 if (scan_type == scan_fetch) {
242 q->idx = idx;
243 q->toggle = toggle;
244 }
245 }
246
247 /* If we are just taking a "peek", do nothing else */
248 if (scan_type == scan_poll)
249 return hirq;
250
251 /* Update the pending bits */
252 xc->pending = pending;
253
254 /*
255 * If this is an EOI that's it, no CPPR adjustment done here,
256 * all we needed was cleanup the stale pending bits and check
257 * if there's anything left.
258 */
259 if (scan_type == scan_eoi)
260 return hirq;
261
262 /*
263 * If we found an interrupt, adjust what the guest CPPR should
264 * be as if we had just fetched that interrupt from HW.
265 *
266 * Note: This can only make xc->cppr smaller as the previous
267 * loop will only exit with hirq != 0 if prio is lower than
268 * the current xc->cppr. Thus we don't need to re-check xc->mfrr
269 * for pending IPIs.
270 */
271 if (hirq)
272 xc->cppr = prio;
273 /*
274 * If it was an IPI the HW CPPR might have been lowered too much
275 * as the HW interrupt we use for IPIs is routed to priority 0.
276 *
277 * We re-sync it here.
278 */
279 if (xc->cppr != xc->hw_cppr) {
280 xc->hw_cppr = xc->cppr;
281 __raw_writeb(xc->cppr, xive_tima + TM_QW1_OS + TM_CPPR);
282 }
283
284 return hirq;
285 }
286
287 static unsigned long xive_vm_h_xirr(struct kvm_vcpu *vcpu)
288 {
289 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
290 u8 old_cppr;
291 u32 hirq;
292
293 pr_devel("H_XIRR\n");
294
295 xc->stat_vm_h_xirr++;
296
297 /* First collect pending bits from HW */
298 xive_vm_ack_pending(xc);
299
300 pr_devel(" new pending=0x%02x hw_cppr=%d cppr=%d\n",
301 xc->pending, xc->hw_cppr, xc->cppr);
302
303 /* Grab previous CPPR and reverse map it */
304 old_cppr = xive_prio_to_guest(xc->cppr);
305
306 /* Scan for actual interrupts */
307 hirq = xive_vm_scan_interrupts(xc, xc->pending, scan_fetch);
308
309 pr_devel(" got hirq=0x%x hw_cppr=%d cppr=%d\n",
310 hirq, xc->hw_cppr, xc->cppr);
311
312 /* That should never hit */
313 if (hirq & 0xff000000)
314 pr_warn("XIVE: Weird guest interrupt number 0x%08x\n", hirq);
315
316 /*
317 * XXX We could check if the interrupt is masked here and
318 * filter it. If we chose to do so, we would need to do:
319 *
320 * if (masked) {
321 * lock();
322 * if (masked) {
323 * old_Q = true;
324 * hirq = 0;
325 * }
326 * unlock();
327 * }
328 */
329
330 /* Return interrupt and old CPPR in GPR4 */
331 kvmppc_set_gpr(vcpu, 4, hirq | (old_cppr << 24));
332
333 return H_SUCCESS;
334 }
335
336 static unsigned long xive_vm_h_ipoll(struct kvm_vcpu *vcpu, unsigned long server)
337 {
338 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
339 u8 pending = xc->pending;
340 u32 hirq;
341
342 pr_devel("H_IPOLL(server=%ld)\n", server);
343
344 xc->stat_vm_h_ipoll++;
345
346 /* Grab the target VCPU if not the current one */
347 if (xc->server_num != server) {
348 vcpu = kvmppc_xive_find_server(vcpu->kvm, server);
349 if (!vcpu)
350 return H_PARAMETER;
351 xc = vcpu->arch.xive_vcpu;
352
353 /* Scan all priorities */
354 pending = 0xff;
355 } else {
356 /* Grab pending interrupt if any */
357 __be64 qw1 = __raw_readq(xive_tima + TM_QW1_OS);
358 u8 pipr = be64_to_cpu(qw1) & 0xff;
359
360 if (pipr < 8)
361 pending |= 1 << pipr;
362 }
363
364 hirq = xive_vm_scan_interrupts(xc, pending, scan_poll);
365
366 /* Return interrupt and old CPPR in GPR4 */
367 kvmppc_set_gpr(vcpu, 4, hirq | (xc->cppr << 24));
368
369 return H_SUCCESS;
370 }
371
372 static void xive_vm_push_pending_to_hw(struct kvmppc_xive_vcpu *xc)
373 {
374 u8 pending, prio;
375
376 pending = xc->pending;
377 if (xc->mfrr != 0xff) {
378 if (xc->mfrr < 8)
379 pending |= 1 << xc->mfrr;
380 else
381 pending |= 0x80;
382 }
383 if (!pending)
384 return;
385 prio = ffs(pending) - 1;
386
387 __raw_writeb(prio, xive_tima + TM_SPC_SET_OS_PENDING);
388 }
389
390 static void xive_vm_scan_for_rerouted_irqs(struct kvmppc_xive *xive,
391 struct kvmppc_xive_vcpu *xc)
392 {
393 unsigned int prio;
394
395 /* For each priority that is now masked */
396 for (prio = xc->cppr; prio < KVMPPC_XIVE_Q_COUNT; prio++) {
397 struct xive_q *q = &xc->queues[prio];
398 struct kvmppc_xive_irq_state *state;
399 struct kvmppc_xive_src_block *sb;
400 u32 idx, toggle, entry, irq, hw_num;
401 struct xive_irq_data *xd;
402 __be32 *qpage;
403 u16 src;
404
405 idx = q->idx;
406 toggle = q->toggle;
407 qpage = READ_ONCE(q->qpage);
408 if (!qpage)
409 continue;
410
411 /* For each interrupt in the queue */
412 for (;;) {
413 entry = be32_to_cpup(qpage + idx);
414
415 /* No more ? */
416 if ((entry >> 31) == toggle)
417 break;
418 irq = entry & 0x7fffffff;
419
420 /* Skip dummies and IPIs */
421 if (irq == XICS_DUMMY || irq == XICS_IPI)
422 goto next;
423 sb = kvmppc_xive_find_source(xive, irq, &src);
424 if (!sb)
425 goto next;
426 state = &sb->irq_state[src];
427
428 /* Has it been rerouted ? */
429 if (xc->server_num == state->act_server)
430 goto next;
431
432 /*
433 * Allright, it *has* been re-routed, kill it from
434 * the queue.
435 */
436 qpage[idx] = cpu_to_be32((entry & 0x80000000) | XICS_DUMMY);
437
438 /* Find the HW interrupt */
439 kvmppc_xive_select_irq(state, &hw_num, &xd);
440
441 /* If it's not an LSI, set PQ to 11 the EOI will force a resend */
442 if (!(xd->flags & XIVE_IRQ_FLAG_LSI))
443 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_11);
444
445 /* EOI the source */
446 xive_vm_source_eoi(hw_num, xd);
447
448 next:
449 idx = (idx + 1) & q->msk;
450 if (idx == 0)
451 toggle ^= 1;
452 }
453 }
454 }
455
456 static int xive_vm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr)
457 {
458 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
459 struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
460 u8 old_cppr;
461
462 pr_devel("H_CPPR(cppr=%ld)\n", cppr);
463
464 xc->stat_vm_h_cppr++;
465
466 /* Map CPPR */
467 cppr = xive_prio_from_guest(cppr);
468
469 /* Remember old and update SW state */
470 old_cppr = xc->cppr;
471 xc->cppr = cppr;
472
473 /*
474 * Order the above update of xc->cppr with the subsequent
475 * read of xc->mfrr inside push_pending_to_hw()
476 */
477 smp_mb();
478
479 if (cppr > old_cppr) {
480 /*
481 * We are masking less, we need to look for pending things
482 * to deliver and set VP pending bits accordingly to trigger
483 * a new interrupt otherwise we might miss MFRR changes for
484 * which we have optimized out sending an IPI signal.
485 */
486 xive_vm_push_pending_to_hw(xc);
487 } else {
488 /*
489 * We are masking more, we need to check the queue for any
490 * interrupt that has been routed to another CPU, take
491 * it out (replace it with the dummy) and retrigger it.
492 *
493 * This is necessary since those interrupts may otherwise
494 * never be processed, at least not until this CPU restores
495 * its CPPR.
496 *
497 * This is in theory racy vs. HW adding new interrupts to
498 * the queue. In practice this works because the interesting
499 * cases are when the guest has done a set_xive() to move the
500 * interrupt away, which flushes the xive, followed by the
501 * target CPU doing a H_CPPR. So any new interrupt coming into
502 * the queue must still be routed to us and isn't a source
503 * of concern.
504 */
505 xive_vm_scan_for_rerouted_irqs(xive, xc);
506 }
507
508 /* Apply new CPPR */
509 xc->hw_cppr = cppr;
510 __raw_writeb(cppr, xive_tima + TM_QW1_OS + TM_CPPR);
511
512 return H_SUCCESS;
513 }
514
515 static int xive_vm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr)
516 {
517 struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
518 struct kvmppc_xive_src_block *sb;
519 struct kvmppc_xive_irq_state *state;
520 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
521 struct xive_irq_data *xd;
522 u8 new_cppr = xirr >> 24;
523 u32 irq = xirr & 0x00ffffff, hw_num;
524 u16 src;
525 int rc = 0;
526
527 pr_devel("H_EOI(xirr=%08lx)\n", xirr);
528
529 xc->stat_vm_h_eoi++;
530
531 xc->cppr = xive_prio_from_guest(new_cppr);
532
533 /*
534 * IPIs are synthesized from MFRR and thus don't need
535 * any special EOI handling. The underlying interrupt
536 * used to signal MFRR changes is EOId when fetched from
537 * the queue.
538 */
539 if (irq == XICS_IPI || irq == 0) {
540 /*
541 * This barrier orders the setting of xc->cppr vs.
542 * subsequent test of xc->mfrr done inside
543 * scan_interrupts and push_pending_to_hw
544 */
545 smp_mb();
546 goto bail;
547 }
548
549 /* Find interrupt source */
550 sb = kvmppc_xive_find_source(xive, irq, &src);
551 if (!sb) {
552 pr_devel(" source not found !\n");
553 rc = H_PARAMETER;
554 /* Same as above */
555 smp_mb();
556 goto bail;
557 }
558 state = &sb->irq_state[src];
559 kvmppc_xive_select_irq(state, &hw_num, &xd);
560
561 state->in_eoi = true;
562
563 /*
564 * This barrier orders both setting of in_eoi above vs,
565 * subsequent test of guest_priority, and the setting
566 * of xc->cppr vs. subsequent test of xc->mfrr done inside
567 * scan_interrupts and push_pending_to_hw
568 */
569 smp_mb();
570
571 again:
572 if (state->guest_priority == MASKED) {
573 arch_spin_lock(&sb->lock);
574 if (state->guest_priority != MASKED) {
575 arch_spin_unlock(&sb->lock);
576 goto again;
577 }
578 pr_devel(" EOI on saved P...\n");
579
580 /* Clear old_p, that will cause unmask to perform an EOI */
581 state->old_p = false;
582
583 arch_spin_unlock(&sb->lock);
584 } else {
585 pr_devel(" EOI on source...\n");
586
587 /* Perform EOI on the source */
588 xive_vm_source_eoi(hw_num, xd);
589
590 /* If it's an emulated LSI, check level and resend */
591 if (state->lsi && state->asserted)
592 __raw_writeq(0, __x_trig_page(xd));
593
594 }
595
596 /*
597 * This barrier orders the above guest_priority check
598 * and spin_lock/unlock with clearing in_eoi below.
599 *
600 * It also has to be a full mb() as it must ensure
601 * the MMIOs done in source_eoi() are completed before
602 * state->in_eoi is visible.
603 */
604 mb();
605 state->in_eoi = false;
606 bail:
607
608 /* Re-evaluate pending IRQs and update HW */
609 xive_vm_scan_interrupts(xc, xc->pending, scan_eoi);
610 xive_vm_push_pending_to_hw(xc);
611 pr_devel(" after scan pending=%02x\n", xc->pending);
612
613 /* Apply new CPPR */
614 xc->hw_cppr = xc->cppr;
615 __raw_writeb(xc->cppr, xive_tima + TM_QW1_OS + TM_CPPR);
616
617 return rc;
618 }
619
620 static int xive_vm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server,
621 unsigned long mfrr)
622 {
623 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
624
625 pr_devel("H_IPI(server=%08lx,mfrr=%ld)\n", server, mfrr);
626
627 xc->stat_vm_h_ipi++;
628
629 /* Find target */
630 vcpu = kvmppc_xive_find_server(vcpu->kvm, server);
631 if (!vcpu)
632 return H_PARAMETER;
633 xc = vcpu->arch.xive_vcpu;
634
635 /* Locklessly write over MFRR */
636 xc->mfrr = mfrr;
637
638 /*
639 * The load of xc->cppr below and the subsequent MMIO store
640 * to the IPI must happen after the above mfrr update is
641 * globally visible so that:
642 *
643 * - Synchronize with another CPU doing an H_EOI or a H_CPPR
644 * updating xc->cppr then reading xc->mfrr.
645 *
646 * - The target of the IPI sees the xc->mfrr update
647 */
648 mb();
649
650 /* Shoot the IPI if most favored than target cppr */
651 if (mfrr < xc->cppr)
652 __raw_writeq(0, __x_trig_page(&xc->vp_ipi_data));
653
654 return H_SUCCESS;
655 }
656
657 /*
658 * We leave a gap of a couple of interrupts in the queue to
659 * account for the IPI and additional safety guard.
660 */
661 #define XIVE_Q_GAP 2
662
663 static bool kvmppc_xive_vcpu_has_save_restore(struct kvm_vcpu *vcpu)
664 {
665 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
666
667 /* Check enablement at VP level */
668 return xc->vp_cam & TM_QW1W2_HO;
669 }
670
671 bool kvmppc_xive_check_save_restore(struct kvm_vcpu *vcpu)
672 {
673 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
674 struct kvmppc_xive *xive = xc->xive;
675
676 if (xive->flags & KVMPPC_XIVE_FLAG_SAVE_RESTORE)
677 return kvmppc_xive_vcpu_has_save_restore(vcpu);
678
679 return true;
680 }
681
682 /*
683 * Push a vcpu's context to the XIVE on guest entry.
684 * This assumes we are in virtual mode (MMU on)
685 */
686 void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu)
687 {
688 void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt;
689 u64 pq;
690
691 /*
692 * Nothing to do if the platform doesn't have a XIVE
693 * or this vCPU doesn't have its own XIVE context
694 * (e.g. because it's not using an in-kernel interrupt controller).
695 */
696 if (!tima || !vcpu->arch.xive_cam_word)
697 return;
698
699 eieio();
700 if (!kvmppc_xive_vcpu_has_save_restore(vcpu))
701 __raw_writeq(vcpu->arch.xive_saved_state.w01, tima + TM_QW1_OS);
702 __raw_writel(vcpu->arch.xive_cam_word, tima + TM_QW1_OS + TM_WORD2);
703 vcpu->arch.xive_pushed = 1;
704 eieio();
705
706 /*
707 * We clear the irq_pending flag. There is a small chance of a
708 * race vs. the escalation interrupt happening on another
709 * processor setting it again, but the only consequence is to
710 * cause a spurious wakeup on the next H_CEDE, which is not an
711 * issue.
712 */
713 vcpu->arch.irq_pending = 0;
714
715 /*
716 * In single escalation mode, if the escalation interrupt is
717 * on, we mask it.
718 */
719 if (vcpu->arch.xive_esc_on) {
720 pq = __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr +
721 XIVE_ESB_SET_PQ_01));
722 mb();
723
724 /*
725 * We have a possible subtle race here: The escalation
726 * interrupt might have fired and be on its way to the
727 * host queue while we mask it, and if we unmask it
728 * early enough (re-cede right away), there is a
729 * theoretical possibility that it fires again, thus
730 * landing in the target queue more than once which is
731 * a big no-no.
732 *
733 * Fortunately, solving this is rather easy. If the
734 * above load setting PQ to 01 returns a previous
735 * value where P is set, then we know the escalation
736 * interrupt is somewhere on its way to the host. In
737 * that case we simply don't clear the xive_esc_on
738 * flag below. It will be eventually cleared by the
739 * handler for the escalation interrupt.
740 *
741 * Then, when doing a cede, we check that flag again
742 * before re-enabling the escalation interrupt, and if
743 * set, we abort the cede.
744 */
745 if (!(pq & XIVE_ESB_VAL_P))
746 /* Now P is 0, we can clear the flag */
747 vcpu->arch.xive_esc_on = 0;
748 }
749 }
750 EXPORT_SYMBOL_GPL(kvmppc_xive_push_vcpu);
751
752 /*
753 * Pull a vcpu's context from the XIVE on guest exit.
754 * This assumes we are in virtual mode (MMU on)
755 */
756 void kvmppc_xive_pull_vcpu(struct kvm_vcpu *vcpu)
757 {
758 void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt;
759
760 if (!vcpu->arch.xive_pushed)
761 return;
762
763 /*
764 * Should not have been pushed if there is no tima
765 */
766 if (WARN_ON(!tima))
767 return;
768
769 eieio();
770 /* First load to pull the context, we ignore the value */
771 __raw_readl(tima + TM_SPC_PULL_OS_CTX);
772 /* Second load to recover the context state (Words 0 and 1) */
773 if (!kvmppc_xive_vcpu_has_save_restore(vcpu))
774 vcpu->arch.xive_saved_state.w01 = __raw_readq(tima + TM_QW1_OS);
775
776 /* Fixup some of the state for the next load */
777 vcpu->arch.xive_saved_state.lsmfb = 0;
778 vcpu->arch.xive_saved_state.ack = 0xff;
779 vcpu->arch.xive_pushed = 0;
780 eieio();
781 }
782 EXPORT_SYMBOL_GPL(kvmppc_xive_pull_vcpu);
783
784 bool kvmppc_xive_rearm_escalation(struct kvm_vcpu *vcpu)
785 {
786 void __iomem *esc_vaddr = (void __iomem *)vcpu->arch.xive_esc_vaddr;
787 bool ret = true;
788
789 if (!esc_vaddr)
790 return ret;
791
792 /* we are using XIVE with single escalation */
793
794 if (vcpu->arch.xive_esc_on) {
795 /*
796 * If we still have a pending escalation, abort the cede,
797 * and we must set PQ to 10 rather than 00 so that we don't
798 * potentially end up with two entries for the escalation
799 * interrupt in the XIVE interrupt queue. In that case
800 * we also don't want to set xive_esc_on to 1 here in
801 * case we race with xive_esc_irq().
802 */
803 ret = false;
804 /*
805 * The escalation interrupts are special as we don't EOI them.
806 * There is no need to use the load-after-store ordering offset
807 * to set PQ to 10 as we won't use StoreEOI.
808 */
809 __raw_readq(esc_vaddr + XIVE_ESB_SET_PQ_10);
810 } else {
811 vcpu->arch.xive_esc_on = true;
812 mb();
813 __raw_readq(esc_vaddr + XIVE_ESB_SET_PQ_00);
814 }
815 mb();
816
817 return ret;
818 }
819 EXPORT_SYMBOL_GPL(kvmppc_xive_rearm_escalation);
820
821 /*
822 * This is a simple trigger for a generic XIVE IRQ. This must
823 * only be called for interrupts that support a trigger page
824 */
825 static bool xive_irq_trigger(struct xive_irq_data *xd)
826 {
827 /* This should be only for MSIs */
828 if (WARN_ON(xd->flags & XIVE_IRQ_FLAG_LSI))
829 return false;
830
831 /* Those interrupts should always have a trigger page */
832 if (WARN_ON(!xd->trig_mmio))
833 return false;
834
835 out_be64(xd->trig_mmio, 0);
836
837 return true;
838 }
839
840 static irqreturn_t xive_esc_irq(int irq, void *data)
841 {
842 struct kvm_vcpu *vcpu = data;
843
844 vcpu->arch.irq_pending = 1;
845 smp_mb();
846 if (vcpu->arch.ceded || vcpu->arch.nested)
847 kvmppc_fast_vcpu_kick(vcpu);
848
849 /* Since we have the no-EOI flag, the interrupt is effectively
850 * disabled now. Clearing xive_esc_on means we won't bother
851 * doing so on the next entry.
852 *
853 * This also allows the entry code to know that if a PQ combination
854 * of 10 is observed while xive_esc_on is true, it means the queue
855 * contains an unprocessed escalation interrupt. We don't make use of
856 * that knowledge today but might (see comment in book3s_hv_rmhandler.S)
857 */
858 vcpu->arch.xive_esc_on = false;
859
860 /* This orders xive_esc_on = false vs. subsequent stale_p = true */
861 smp_wmb(); /* goes with smp_mb() in cleanup_single_escalation */
862
863 return IRQ_HANDLED;
864 }
865
866 int kvmppc_xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio,
867 bool single_escalation)
868 {
869 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
870 struct xive_q *q = &xc->queues[prio];
871 char *name = NULL;
872 int rc;
873
874 /* Already there ? */
875 if (xc->esc_virq[prio])
876 return 0;
877
878 /* Hook up the escalation interrupt */
879 xc->esc_virq[prio] = irq_create_mapping(NULL, q->esc_irq);
880 if (!xc->esc_virq[prio]) {
881 pr_err("Failed to map escalation interrupt for queue %d of VCPU %d\n",
882 prio, xc->server_num);
883 return -EIO;
884 }
885
886 if (single_escalation)
887 name = kasprintf(GFP_KERNEL, "kvm-%lld-%d",
888 vcpu->kvm->arch.lpid, xc->server_num);
889 else
890 name = kasprintf(GFP_KERNEL, "kvm-%lld-%d-%d",
891 vcpu->kvm->arch.lpid, xc->server_num, prio);
892 if (!name) {
893 pr_err("Failed to allocate escalation irq name for queue %d of VCPU %d\n",
894 prio, xc->server_num);
895 rc = -ENOMEM;
896 goto error;
897 }
898
899 pr_devel("Escalation %s irq %d (prio %d)\n", name, xc->esc_virq[prio], prio);
900
901 rc = request_irq(xc->esc_virq[prio], xive_esc_irq,
902 IRQF_NO_THREAD, name, vcpu);
903 if (rc) {
904 pr_err("Failed to request escalation interrupt for queue %d of VCPU %d\n",
905 prio, xc->server_num);
906 goto error;
907 }
908 xc->esc_virq_names[prio] = name;
909
910 /* In single escalation mode, we grab the ESB MMIO of the
911 * interrupt and mask it. Also populate the VCPU v/raddr
912 * of the ESB page for use by asm entry/exit code. Finally
913 * set the XIVE_IRQ_FLAG_NO_EOI flag which will prevent the
914 * core code from performing an EOI on the escalation
915 * interrupt, thus leaving it effectively masked after
916 * it fires once.
917 */
918 if (single_escalation) {
919 struct irq_data *d = irq_get_irq_data(xc->esc_virq[prio]);
920 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
921
922 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01);
923 vcpu->arch.xive_esc_raddr = xd->eoi_page;
924 vcpu->arch.xive_esc_vaddr = (__force u64)xd->eoi_mmio;
925 xd->flags |= XIVE_IRQ_FLAG_NO_EOI;
926 }
927
928 return 0;
929 error:
930 irq_dispose_mapping(xc->esc_virq[prio]);
931 xc->esc_virq[prio] = 0;
932 kfree(name);
933 return rc;
934 }
935
936 static int xive_provision_queue(struct kvm_vcpu *vcpu, u8 prio)
937 {
938 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
939 struct kvmppc_xive *xive = xc->xive;
940 struct xive_q *q = &xc->queues[prio];
941 void *qpage;
942 int rc;
943
944 if (WARN_ON(q->qpage))
945 return 0;
946
947 /* Allocate the queue and retrieve infos on current node for now */
948 qpage = (__be32 *)__get_free_pages(GFP_KERNEL, xive->q_page_order);
949 if (!qpage) {
950 pr_err("Failed to allocate queue %d for VCPU %d\n",
951 prio, xc->server_num);
952 return -ENOMEM;
953 }
954 memset(qpage, 0, 1 << xive->q_order);
955
956 /*
957 * Reconfigure the queue. This will set q->qpage only once the
958 * queue is fully configured. This is a requirement for prio 0
959 * as we will stop doing EOIs for every IPI as soon as we observe
960 * qpage being non-NULL, and instead will only EOI when we receive
961 * corresponding queue 0 entries
962 */
963 rc = xive_native_configure_queue(xc->vp_id, q, prio, qpage,
964 xive->q_order, true);
965 if (rc)
966 pr_err("Failed to configure queue %d for VCPU %d\n",
967 prio, xc->server_num);
968 return rc;
969 }
970
971 /* Called with xive->lock held */
972 static int xive_check_provisioning(struct kvm *kvm, u8 prio)
973 {
974 struct kvmppc_xive *xive = kvm->arch.xive;
975 struct kvm_vcpu *vcpu;
976 unsigned long i;
977 int rc;
978
979 lockdep_assert_held(&xive->lock);
980
981 /* Already provisioned ? */
982 if (xive->qmap & (1 << prio))
983 return 0;
984
985 pr_devel("Provisioning prio... %d\n", prio);
986
987 /* Provision each VCPU and enable escalations if needed */
988 kvm_for_each_vcpu(i, vcpu, kvm) {
989 if (!vcpu->arch.xive_vcpu)
990 continue;
991 rc = xive_provision_queue(vcpu, prio);
992 if (rc == 0 && !kvmppc_xive_has_single_escalation(xive))
993 kvmppc_xive_attach_escalation(vcpu, prio,
994 kvmppc_xive_has_single_escalation(xive));
995 if (rc)
996 return rc;
997 }
998
999 /* Order previous stores and mark it as provisioned */
1000 mb();
1001 xive->qmap |= (1 << prio);
1002 return 0;
1003 }
1004
1005 static void xive_inc_q_pending(struct kvm *kvm, u32 server, u8 prio)
1006 {
1007 struct kvm_vcpu *vcpu;
1008 struct kvmppc_xive_vcpu *xc;
1009 struct xive_q *q;
1010
1011 /* Locate target server */
1012 vcpu = kvmppc_xive_find_server(kvm, server);
1013 if (!vcpu) {
1014 pr_warn("%s: Can't find server %d\n", __func__, server);
1015 return;
1016 }
1017 xc = vcpu->arch.xive_vcpu;
1018 if (WARN_ON(!xc))
1019 return;
1020
1021 q = &xc->queues[prio];
1022 atomic_inc(&q->pending_count);
1023 }
1024
1025 static int xive_try_pick_queue(struct kvm_vcpu *vcpu, u8 prio)
1026 {
1027 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1028 struct xive_q *q;
1029 u32 max;
1030
1031 if (WARN_ON(!xc))
1032 return -ENXIO;
1033 if (!xc->valid)
1034 return -ENXIO;
1035
1036 q = &xc->queues[prio];
1037 if (WARN_ON(!q->qpage))
1038 return -ENXIO;
1039
1040 /* Calculate max number of interrupts in that queue. */
1041 max = (q->msk + 1) - XIVE_Q_GAP;
1042 return atomic_add_unless(&q->count, 1, max) ? 0 : -EBUSY;
1043 }
1044
1045 int kvmppc_xive_select_target(struct kvm *kvm, u32 *server, u8 prio)
1046 {
1047 struct kvm_vcpu *vcpu;
1048 unsigned long i;
1049 int rc;
1050
1051 /* Locate target server */
1052 vcpu = kvmppc_xive_find_server(kvm, *server);
1053 if (!vcpu) {
1054 pr_devel("Can't find server %d\n", *server);
1055 return -EINVAL;
1056 }
1057
1058 pr_devel("Finding irq target on 0x%x/%d...\n", *server, prio);
1059
1060 /* Try pick it */
1061 rc = xive_try_pick_queue(vcpu, prio);
1062 if (rc == 0)
1063 return rc;
1064
1065 pr_devel(" .. failed, looking up candidate...\n");
1066
1067 /* Failed, pick another VCPU */
1068 kvm_for_each_vcpu(i, vcpu, kvm) {
1069 if (!vcpu->arch.xive_vcpu)
1070 continue;
1071 rc = xive_try_pick_queue(vcpu, prio);
1072 if (rc == 0) {
1073 *server = vcpu->arch.xive_vcpu->server_num;
1074 pr_devel(" found on 0x%x/%d\n", *server, prio);
1075 return rc;
1076 }
1077 }
1078 pr_devel(" no available target !\n");
1079
1080 /* No available target ! */
1081 return -EBUSY;
1082 }
1083
1084 static u8 xive_lock_and_mask(struct kvmppc_xive *xive,
1085 struct kvmppc_xive_src_block *sb,
1086 struct kvmppc_xive_irq_state *state)
1087 {
1088 struct xive_irq_data *xd;
1089 u32 hw_num;
1090 u8 old_prio;
1091 u64 val;
1092
1093 /*
1094 * Take the lock, set masked, try again if racing
1095 * with H_EOI
1096 */
1097 for (;;) {
1098 arch_spin_lock(&sb->lock);
1099 old_prio = state->guest_priority;
1100 state->guest_priority = MASKED;
1101 mb();
1102 if (!state->in_eoi)
1103 break;
1104 state->guest_priority = old_prio;
1105 arch_spin_unlock(&sb->lock);
1106 }
1107
1108 /* No change ? Bail */
1109 if (old_prio == MASKED)
1110 return old_prio;
1111
1112 /* Get the right irq */
1113 kvmppc_xive_select_irq(state, &hw_num, &xd);
1114
1115 /* Set PQ to 10, return old P and old Q and remember them */
1116 val = xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_10);
1117 state->old_p = !!(val & 2);
1118 state->old_q = !!(val & 1);
1119
1120 /*
1121 * Synchronize hardware to sensure the queues are updated when
1122 * masking
1123 */
1124 xive_native_sync_source(hw_num);
1125
1126 return old_prio;
1127 }
1128
1129 static void xive_lock_for_unmask(struct kvmppc_xive_src_block *sb,
1130 struct kvmppc_xive_irq_state *state)
1131 {
1132 /*
1133 * Take the lock try again if racing with H_EOI
1134 */
1135 for (;;) {
1136 arch_spin_lock(&sb->lock);
1137 if (!state->in_eoi)
1138 break;
1139 arch_spin_unlock(&sb->lock);
1140 }
1141 }
1142
1143 static void xive_finish_unmask(struct kvmppc_xive *xive,
1144 struct kvmppc_xive_src_block *sb,
1145 struct kvmppc_xive_irq_state *state,
1146 u8 prio)
1147 {
1148 struct xive_irq_data *xd;
1149 u32 hw_num;
1150
1151 /* If we aren't changing a thing, move on */
1152 if (state->guest_priority != MASKED)
1153 goto bail;
1154
1155 /* Get the right irq */
1156 kvmppc_xive_select_irq(state, &hw_num, &xd);
1157
1158 /* Old Q set, set PQ to 11 */
1159 if (state->old_q)
1160 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_11);
1161
1162 /*
1163 * If not old P, then perform an "effective" EOI,
1164 * on the source. This will handle the cases where
1165 * FW EOI is needed.
1166 */
1167 if (!state->old_p)
1168 xive_vm_source_eoi(hw_num, xd);
1169
1170 /* Synchronize ordering and mark unmasked */
1171 mb();
1172 bail:
1173 state->guest_priority = prio;
1174 }
1175
1176 /*
1177 * Target an interrupt to a given server/prio, this will fallback
1178 * to another server if necessary and perform the HW targetting
1179 * updates as needed
1180 *
1181 * NOTE: Must be called with the state lock held
1182 */
1183 static int xive_target_interrupt(struct kvm *kvm,
1184 struct kvmppc_xive_irq_state *state,
1185 u32 server, u8 prio)
1186 {
1187 struct kvmppc_xive *xive = kvm->arch.xive;
1188 u32 hw_num;
1189 int rc;
1190
1191 /*
1192 * This will return a tentative server and actual
1193 * priority. The count for that new target will have
1194 * already been incremented.
1195 */
1196 rc = kvmppc_xive_select_target(kvm, &server, prio);
1197
1198 /*
1199 * We failed to find a target ? Not much we can do
1200 * at least until we support the GIQ.
1201 */
1202 if (rc)
1203 return rc;
1204
1205 /*
1206 * Increment the old queue pending count if there
1207 * was one so that the old queue count gets adjusted later
1208 * when observed to be empty.
1209 */
1210 if (state->act_priority != MASKED)
1211 xive_inc_q_pending(kvm,
1212 state->act_server,
1213 state->act_priority);
1214 /*
1215 * Update state and HW
1216 */
1217 state->act_priority = prio;
1218 state->act_server = server;
1219
1220 /* Get the right irq */
1221 kvmppc_xive_select_irq(state, &hw_num, NULL);
1222
1223 return xive_native_configure_irq(hw_num,
1224 kvmppc_xive_vp(xive, server),
1225 prio, state->number);
1226 }
1227
1228 /*
1229 * Targetting rules: In order to avoid losing track of
1230 * pending interrupts across mask and unmask, which would
1231 * allow queue overflows, we implement the following rules:
1232 *
1233 * - Unless it was never enabled (or we run out of capacity)
1234 * an interrupt is always targetted at a valid server/queue
1235 * pair even when "masked" by the guest. This pair tends to
1236 * be the last one used but it can be changed under some
1237 * circumstances. That allows us to separate targetting
1238 * from masking, we only handle accounting during (re)targetting,
1239 * this also allows us to let an interrupt drain into its target
1240 * queue after masking, avoiding complex schemes to remove
1241 * interrupts out of remote processor queues.
1242 *
1243 * - When masking, we set PQ to 10 and save the previous value
1244 * of P and Q.
1245 *
1246 * - When unmasking, if saved Q was set, we set PQ to 11
1247 * otherwise we leave PQ to the HW state which will be either
1248 * 10 if nothing happened or 11 if the interrupt fired while
1249 * masked. Effectively we are OR'ing the previous Q into the
1250 * HW Q.
1251 *
1252 * Then if saved P is clear, we do an effective EOI (Q->P->Trigger)
1253 * which will unmask the interrupt and shoot a new one if Q was
1254 * set.
1255 *
1256 * Otherwise (saved P is set) we leave PQ unchanged (so 10 or 11,
1257 * effectively meaning an H_EOI from the guest is still expected
1258 * for that interrupt).
1259 *
1260 * - If H_EOI occurs while masked, we clear the saved P.
1261 *
1262 * - When changing target, we account on the new target and
1263 * increment a separate "pending" counter on the old one.
1264 * This pending counter will be used to decrement the old
1265 * target's count when its queue has been observed empty.
1266 */
1267
1268 int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server,
1269 u32 priority)
1270 {
1271 struct kvmppc_xive *xive = kvm->arch.xive;
1272 struct kvmppc_xive_src_block *sb;
1273 struct kvmppc_xive_irq_state *state;
1274 u8 new_act_prio;
1275 int rc = 0;
1276 u16 idx;
1277
1278 if (!xive)
1279 return -ENODEV;
1280
1281 pr_devel("set_xive ! irq 0x%x server 0x%x prio %d\n",
1282 irq, server, priority);
1283
1284 /* First, check provisioning of queues */
1285 if (priority != MASKED) {
1286 mutex_lock(&xive->lock);
1287 rc = xive_check_provisioning(xive->kvm,
1288 xive_prio_from_guest(priority));
1289 mutex_unlock(&xive->lock);
1290 }
1291 if (rc) {
1292 pr_devel(" provisioning failure %d !\n", rc);
1293 return rc;
1294 }
1295
1296 sb = kvmppc_xive_find_source(xive, irq, &idx);
1297 if (!sb)
1298 return -EINVAL;
1299 state = &sb->irq_state[idx];
1300
1301 /*
1302 * We first handle masking/unmasking since the locking
1303 * might need to be retried due to EOIs, we'll handle
1304 * targetting changes later. These functions will return
1305 * with the SB lock held.
1306 *
1307 * xive_lock_and_mask() will also set state->guest_priority
1308 * but won't otherwise change other fields of the state.
1309 *
1310 * xive_lock_for_unmask will not actually unmask, this will
1311 * be done later by xive_finish_unmask() once the targetting
1312 * has been done, so we don't try to unmask an interrupt
1313 * that hasn't yet been targetted.
1314 */
1315 if (priority == MASKED)
1316 xive_lock_and_mask(xive, sb, state);
1317 else
1318 xive_lock_for_unmask(sb, state);
1319
1320
1321 /*
1322 * Then we handle targetting.
1323 *
1324 * First calculate a new "actual priority"
1325 */
1326 new_act_prio = state->act_priority;
1327 if (priority != MASKED)
1328 new_act_prio = xive_prio_from_guest(priority);
1329
1330 pr_devel(" new_act_prio=%x act_server=%x act_prio=%x\n",
1331 new_act_prio, state->act_server, state->act_priority);
1332
1333 /*
1334 * Then check if we actually need to change anything,
1335 *
1336 * The condition for re-targetting the interrupt is that
1337 * we have a valid new priority (new_act_prio is not 0xff)
1338 * and either the server or the priority changed.
1339 *
1340 * Note: If act_priority was ff and the new priority is
1341 * also ff, we don't do anything and leave the interrupt
1342 * untargetted. An attempt of doing an int_on on an
1343 * untargetted interrupt will fail. If that is a problem
1344 * we could initialize interrupts with valid default
1345 */
1346
1347 if (new_act_prio != MASKED &&
1348 (state->act_server != server ||
1349 state->act_priority != new_act_prio))
1350 rc = xive_target_interrupt(kvm, state, server, new_act_prio);
1351
1352 /*
1353 * Perform the final unmasking of the interrupt source
1354 * if necessary
1355 */
1356 if (priority != MASKED)
1357 xive_finish_unmask(xive, sb, state, priority);
1358
1359 /*
1360 * Finally Update saved_priority to match. Only int_on/off
1361 * set this field to a different value.
1362 */
1363 state->saved_priority = priority;
1364
1365 arch_spin_unlock(&sb->lock);
1366 return rc;
1367 }
1368
1369 int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server,
1370 u32 *priority)
1371 {
1372 struct kvmppc_xive *xive = kvm->arch.xive;
1373 struct kvmppc_xive_src_block *sb;
1374 struct kvmppc_xive_irq_state *state;
1375 u16 idx;
1376
1377 if (!xive)
1378 return -ENODEV;
1379
1380 sb = kvmppc_xive_find_source(xive, irq, &idx);
1381 if (!sb)
1382 return -EINVAL;
1383 state = &sb->irq_state[idx];
1384 arch_spin_lock(&sb->lock);
1385 *server = state->act_server;
1386 *priority = state->guest_priority;
1387 arch_spin_unlock(&sb->lock);
1388
1389 return 0;
1390 }
1391
1392 int kvmppc_xive_int_on(struct kvm *kvm, u32 irq)
1393 {
1394 struct kvmppc_xive *xive = kvm->arch.xive;
1395 struct kvmppc_xive_src_block *sb;
1396 struct kvmppc_xive_irq_state *state;
1397 u16 idx;
1398
1399 if (!xive)
1400 return -ENODEV;
1401
1402 sb = kvmppc_xive_find_source(xive, irq, &idx);
1403 if (!sb)
1404 return -EINVAL;
1405 state = &sb->irq_state[idx];
1406
1407 pr_devel("int_on(irq=0x%x)\n", irq);
1408
1409 /*
1410 * Check if interrupt was not targetted
1411 */
1412 if (state->act_priority == MASKED) {
1413 pr_devel("int_on on untargetted interrupt\n");
1414 return -EINVAL;
1415 }
1416
1417 /* If saved_priority is 0xff, do nothing */
1418 if (state->saved_priority == MASKED)
1419 return 0;
1420
1421 /*
1422 * Lock and unmask it.
1423 */
1424 xive_lock_for_unmask(sb, state);
1425 xive_finish_unmask(xive, sb, state, state->saved_priority);
1426 arch_spin_unlock(&sb->lock);
1427
1428 return 0;
1429 }
1430
1431 int kvmppc_xive_int_off(struct kvm *kvm, u32 irq)
1432 {
1433 struct kvmppc_xive *xive = kvm->arch.xive;
1434 struct kvmppc_xive_src_block *sb;
1435 struct kvmppc_xive_irq_state *state;
1436 u16 idx;
1437
1438 if (!xive)
1439 return -ENODEV;
1440
1441 sb = kvmppc_xive_find_source(xive, irq, &idx);
1442 if (!sb)
1443 return -EINVAL;
1444 state = &sb->irq_state[idx];
1445
1446 pr_devel("int_off(irq=0x%x)\n", irq);
1447
1448 /*
1449 * Lock and mask
1450 */
1451 state->saved_priority = xive_lock_and_mask(xive, sb, state);
1452 arch_spin_unlock(&sb->lock);
1453
1454 return 0;
1455 }
1456
1457 static bool xive_restore_pending_irq(struct kvmppc_xive *xive, u32 irq)
1458 {
1459 struct kvmppc_xive_src_block *sb;
1460 struct kvmppc_xive_irq_state *state;
1461 u16 idx;
1462
1463 sb = kvmppc_xive_find_source(xive, irq, &idx);
1464 if (!sb)
1465 return false;
1466 state = &sb->irq_state[idx];
1467 if (!state->valid)
1468 return false;
1469
1470 /*
1471 * Trigger the IPI. This assumes we never restore a pass-through
1472 * interrupt which should be safe enough
1473 */
1474 xive_irq_trigger(&state->ipi_data);
1475
1476 return true;
1477 }
1478
1479 u64 kvmppc_xive_get_icp(struct kvm_vcpu *vcpu)
1480 {
1481 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1482
1483 if (!xc)
1484 return 0;
1485
1486 /* Return the per-cpu state for state saving/migration */
1487 return (u64)xc->cppr << KVM_REG_PPC_ICP_CPPR_SHIFT |
1488 (u64)xc->mfrr << KVM_REG_PPC_ICP_MFRR_SHIFT |
1489 (u64)0xff << KVM_REG_PPC_ICP_PPRI_SHIFT;
1490 }
1491
1492 int kvmppc_xive_set_icp(struct kvm_vcpu *vcpu, u64 icpval)
1493 {
1494 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1495 struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
1496 u8 cppr, mfrr;
1497 u32 xisr;
1498
1499 if (!xc || !xive)
1500 return -ENOENT;
1501
1502 /* Grab individual state fields. We don't use pending_pri */
1503 cppr = icpval >> KVM_REG_PPC_ICP_CPPR_SHIFT;
1504 xisr = (icpval >> KVM_REG_PPC_ICP_XISR_SHIFT) &
1505 KVM_REG_PPC_ICP_XISR_MASK;
1506 mfrr = icpval >> KVM_REG_PPC_ICP_MFRR_SHIFT;
1507
1508 pr_devel("set_icp vcpu %d cppr=0x%x mfrr=0x%x xisr=0x%x\n",
1509 xc->server_num, cppr, mfrr, xisr);
1510
1511 /*
1512 * We can't update the state of a "pushed" VCPU, but that
1513 * shouldn't happen because the vcpu->mutex makes running a
1514 * vcpu mutually exclusive with doing one_reg get/set on it.
1515 */
1516 if (WARN_ON(vcpu->arch.xive_pushed))
1517 return -EIO;
1518
1519 /* Update VCPU HW saved state */
1520 vcpu->arch.xive_saved_state.cppr = cppr;
1521 xc->hw_cppr = xc->cppr = cppr;
1522
1523 /*
1524 * Update MFRR state. If it's not 0xff, we mark the VCPU as
1525 * having a pending MFRR change, which will re-evaluate the
1526 * target. The VCPU will thus potentially get a spurious
1527 * interrupt but that's not a big deal.
1528 */
1529 xc->mfrr = mfrr;
1530 if (mfrr < cppr)
1531 xive_irq_trigger(&xc->vp_ipi_data);
1532
1533 /*
1534 * Now saved XIRR is "interesting". It means there's something in
1535 * the legacy "1 element" queue... for an IPI we simply ignore it,
1536 * as the MFRR restore will handle that. For anything else we need
1537 * to force a resend of the source.
1538 * However the source may not have been setup yet. If that's the
1539 * case, we keep that info and increment a counter in the xive to
1540 * tell subsequent xive_set_source() to go look.
1541 */
1542 if (xisr > XICS_IPI && !xive_restore_pending_irq(xive, xisr)) {
1543 xc->delayed_irq = xisr;
1544 xive->delayed_irqs++;
1545 pr_devel(" xisr restore delayed\n");
1546 }
1547
1548 return 0;
1549 }
1550
1551 int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq,
1552 unsigned long host_irq)
1553 {
1554 struct kvmppc_xive *xive = kvm->arch.xive;
1555 struct kvmppc_xive_src_block *sb;
1556 struct kvmppc_xive_irq_state *state;
1557 struct irq_data *host_data =
1558 irq_domain_get_irq_data(irq_get_default_host(), host_irq);
1559 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(host_data);
1560 u16 idx;
1561 u8 prio;
1562 int rc;
1563
1564 if (!xive)
1565 return -ENODEV;
1566
1567 pr_debug("%s: GIRQ 0x%lx host IRQ %ld XIVE HW IRQ 0x%x\n",
1568 __func__, guest_irq, host_irq, hw_irq);
1569
1570 sb = kvmppc_xive_find_source(xive, guest_irq, &idx);
1571 if (!sb)
1572 return -EINVAL;
1573 state = &sb->irq_state[idx];
1574
1575 /*
1576 * Mark the passed-through interrupt as going to a VCPU,
1577 * this will prevent further EOIs and similar operations
1578 * from the XIVE code. It will also mask the interrupt
1579 * to either PQ=10 or 11 state, the latter if the interrupt
1580 * is pending. This will allow us to unmask or retrigger it
1581 * after routing it to the guest with a simple EOI.
1582 *
1583 * The "state" argument is a "token", all it needs is to be
1584 * non-NULL to switch to passed-through or NULL for the
1585 * other way around. We may not yet have an actual VCPU
1586 * target here and we don't really care.
1587 */
1588 rc = irq_set_vcpu_affinity(host_irq, state);
1589 if (rc) {
1590 pr_err("Failed to set VCPU affinity for host IRQ %ld\n", host_irq);
1591 return rc;
1592 }
1593
1594 /*
1595 * Mask and read state of IPI. We need to know if its P bit
1596 * is set as that means it's potentially already using a
1597 * queue entry in the target
1598 */
1599 prio = xive_lock_and_mask(xive, sb, state);
1600 pr_devel(" old IPI prio %02x P:%d Q:%d\n", prio,
1601 state->old_p, state->old_q);
1602
1603 /* Turn the IPI hard off */
1604 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
1605
1606 /*
1607 * Reset ESB guest mapping. Needed when ESB pages are exposed
1608 * to the guest in XIVE native mode
1609 */
1610 if (xive->ops && xive->ops->reset_mapped)
1611 xive->ops->reset_mapped(kvm, guest_irq);
1612
1613 /* Grab info about irq */
1614 state->pt_number = hw_irq;
1615 state->pt_data = irq_data_get_irq_handler_data(host_data);
1616
1617 /*
1618 * Configure the IRQ to match the existing configuration of
1619 * the IPI if it was already targetted. Otherwise this will
1620 * mask the interrupt in a lossy way (act_priority is 0xff)
1621 * which is fine for a never started interrupt.
1622 */
1623 xive_native_configure_irq(hw_irq,
1624 kvmppc_xive_vp(xive, state->act_server),
1625 state->act_priority, state->number);
1626
1627 /*
1628 * We do an EOI to enable the interrupt (and retrigger if needed)
1629 * if the guest has the interrupt unmasked and the P bit was *not*
1630 * set in the IPI. If it was set, we know a slot may still be in
1631 * use in the target queue thus we have to wait for a guest
1632 * originated EOI
1633 */
1634 if (prio != MASKED && !state->old_p)
1635 xive_vm_source_eoi(hw_irq, state->pt_data);
1636
1637 /* Clear old_p/old_q as they are no longer relevant */
1638 state->old_p = state->old_q = false;
1639
1640 /* Restore guest prio (unlocks EOI) */
1641 mb();
1642 state->guest_priority = prio;
1643 arch_spin_unlock(&sb->lock);
1644
1645 return 0;
1646 }
1647 EXPORT_SYMBOL_GPL(kvmppc_xive_set_mapped);
1648
1649 int kvmppc_xive_clr_mapped(struct kvm *kvm, unsigned long guest_irq,
1650 unsigned long host_irq)
1651 {
1652 struct kvmppc_xive *xive = kvm->arch.xive;
1653 struct kvmppc_xive_src_block *sb;
1654 struct kvmppc_xive_irq_state *state;
1655 u16 idx;
1656 u8 prio;
1657 int rc;
1658
1659 if (!xive)
1660 return -ENODEV;
1661
1662 pr_debug("%s: GIRQ 0x%lx host IRQ %ld\n", __func__, guest_irq, host_irq);
1663
1664 sb = kvmppc_xive_find_source(xive, guest_irq, &idx);
1665 if (!sb)
1666 return -EINVAL;
1667 state = &sb->irq_state[idx];
1668
1669 /*
1670 * Mask and read state of IRQ. We need to know if its P bit
1671 * is set as that means it's potentially already using a
1672 * queue entry in the target
1673 */
1674 prio = xive_lock_and_mask(xive, sb, state);
1675 pr_devel(" old IRQ prio %02x P:%d Q:%d\n", prio,
1676 state->old_p, state->old_q);
1677
1678 /*
1679 * If old_p is set, the interrupt is pending, we switch it to
1680 * PQ=11. This will force a resend in the host so the interrupt
1681 * isn't lost to whatever host driver may pick it up
1682 */
1683 if (state->old_p)
1684 xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_11);
1685
1686 /* Release the passed-through interrupt to the host */
1687 rc = irq_set_vcpu_affinity(host_irq, NULL);
1688 if (rc) {
1689 pr_err("Failed to clr VCPU affinity for host IRQ %ld\n", host_irq);
1690 return rc;
1691 }
1692
1693 /* Forget about the IRQ */
1694 state->pt_number = 0;
1695 state->pt_data = NULL;
1696
1697 /*
1698 * Reset ESB guest mapping. Needed when ESB pages are exposed
1699 * to the guest in XIVE native mode
1700 */
1701 if (xive->ops && xive->ops->reset_mapped) {
1702 xive->ops->reset_mapped(kvm, guest_irq);
1703 }
1704
1705 /* Reconfigure the IPI */
1706 xive_native_configure_irq(state->ipi_number,
1707 kvmppc_xive_vp(xive, state->act_server),
1708 state->act_priority, state->number);
1709
1710 /*
1711 * If old_p is set (we have a queue entry potentially
1712 * occupied) or the interrupt is masked, we set the IPI
1713 * to PQ=10 state. Otherwise we just re-enable it (PQ=00).
1714 */
1715 if (prio == MASKED || state->old_p)
1716 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_10);
1717 else
1718 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_00);
1719
1720 /* Restore guest prio (unlocks EOI) */
1721 mb();
1722 state->guest_priority = prio;
1723 arch_spin_unlock(&sb->lock);
1724
1725 return 0;
1726 }
1727 EXPORT_SYMBOL_GPL(kvmppc_xive_clr_mapped);
1728
1729 void kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu *vcpu)
1730 {
1731 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1732 struct kvm *kvm = vcpu->kvm;
1733 struct kvmppc_xive *xive = kvm->arch.xive;
1734 int i, j;
1735
1736 for (i = 0; i <= xive->max_sbid; i++) {
1737 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
1738
1739 if (!sb)
1740 continue;
1741 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) {
1742 struct kvmppc_xive_irq_state *state = &sb->irq_state[j];
1743
1744 if (!state->valid)
1745 continue;
1746 if (state->act_priority == MASKED)
1747 continue;
1748 if (state->act_server != xc->server_num)
1749 continue;
1750
1751 /* Clean it up */
1752 arch_spin_lock(&sb->lock);
1753 state->act_priority = MASKED;
1754 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
1755 xive_native_configure_irq(state->ipi_number, 0, MASKED, 0);
1756 if (state->pt_number) {
1757 xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_01);
1758 xive_native_configure_irq(state->pt_number, 0, MASKED, 0);
1759 }
1760 arch_spin_unlock(&sb->lock);
1761 }
1762 }
1763
1764 /* Disable vcpu's escalation interrupt */
1765 if (vcpu->arch.xive_esc_on) {
1766 __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr +
1767 XIVE_ESB_SET_PQ_01));
1768 vcpu->arch.xive_esc_on = false;
1769 }
1770
1771 /*
1772 * Clear pointers to escalation interrupt ESB.
1773 * This is safe because the vcpu->mutex is held, preventing
1774 * any other CPU from concurrently executing a KVM_RUN ioctl.
1775 */
1776 vcpu->arch.xive_esc_vaddr = 0;
1777 vcpu->arch.xive_esc_raddr = 0;
1778 }
1779
1780 /*
1781 * In single escalation mode, the escalation interrupt is marked so
1782 * that EOI doesn't re-enable it, but just sets the stale_p flag to
1783 * indicate that the P bit has already been dealt with. However, the
1784 * assembly code that enters the guest sets PQ to 00 without clearing
1785 * stale_p (because it has no easy way to address it). Hence we have
1786 * to adjust stale_p before shutting down the interrupt.
1787 */
1788 void xive_cleanup_single_escalation(struct kvm_vcpu *vcpu, int irq)
1789 {
1790 struct irq_data *d = irq_get_irq_data(irq);
1791 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
1792
1793 /*
1794 * This slightly odd sequence gives the right result
1795 * (i.e. stale_p set if xive_esc_on is false) even if
1796 * we race with xive_esc_irq() and xive_irq_eoi().
1797 */
1798 xd->stale_p = false;
1799 smp_mb(); /* paired with smb_wmb in xive_esc_irq */
1800 if (!vcpu->arch.xive_esc_on)
1801 xd->stale_p = true;
1802 }
1803
1804 void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu)
1805 {
1806 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1807 struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
1808 int i;
1809
1810 if (!kvmppc_xics_enabled(vcpu))
1811 return;
1812
1813 if (!xc)
1814 return;
1815
1816 pr_devel("cleanup_vcpu(cpu=%d)\n", xc->server_num);
1817
1818 /* Ensure no interrupt is still routed to that VP */
1819 xc->valid = false;
1820 kvmppc_xive_disable_vcpu_interrupts(vcpu);
1821
1822 /* Mask the VP IPI */
1823 xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_01);
1824
1825 /* Free escalations */
1826 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1827 if (xc->esc_virq[i]) {
1828 if (kvmppc_xive_has_single_escalation(xc->xive))
1829 xive_cleanup_single_escalation(vcpu, xc->esc_virq[i]);
1830 free_irq(xc->esc_virq[i], vcpu);
1831 irq_dispose_mapping(xc->esc_virq[i]);
1832 kfree(xc->esc_virq_names[i]);
1833 }
1834 }
1835
1836 /* Disable the VP */
1837 xive_native_disable_vp(xc->vp_id);
1838
1839 /* Clear the cam word so guest entry won't try to push context */
1840 vcpu->arch.xive_cam_word = 0;
1841
1842 /* Free the queues */
1843 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1844 struct xive_q *q = &xc->queues[i];
1845
1846 xive_native_disable_queue(xc->vp_id, q, i);
1847 if (q->qpage) {
1848 free_pages((unsigned long)q->qpage,
1849 xive->q_page_order);
1850 q->qpage = NULL;
1851 }
1852 }
1853
1854 /* Free the IPI */
1855 if (xc->vp_ipi) {
1856 xive_cleanup_irq_data(&xc->vp_ipi_data);
1857 xive_native_free_irq(xc->vp_ipi);
1858 }
1859 /* Free the VP */
1860 kfree(xc);
1861
1862 /* Cleanup the vcpu */
1863 vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT;
1864 vcpu->arch.xive_vcpu = NULL;
1865 }
1866
1867 static bool kvmppc_xive_vcpu_id_valid(struct kvmppc_xive *xive, u32 cpu)
1868 {
1869 /* We have a block of xive->nr_servers VPs. We just need to check
1870 * packed vCPU ids are below that.
1871 */
1872 return kvmppc_pack_vcpu_id(xive->kvm, cpu) < xive->nr_servers;
1873 }
1874
1875 int kvmppc_xive_compute_vp_id(struct kvmppc_xive *xive, u32 cpu, u32 *vp)
1876 {
1877 u32 vp_id;
1878
1879 if (!kvmppc_xive_vcpu_id_valid(xive, cpu)) {
1880 pr_devel("Out of bounds !\n");
1881 return -EINVAL;
1882 }
1883
1884 if (xive->vp_base == XIVE_INVALID_VP) {
1885 xive->vp_base = xive_native_alloc_vp_block(xive->nr_servers);
1886 pr_devel("VP_Base=%x nr_servers=%d\n", xive->vp_base, xive->nr_servers);
1887
1888 if (xive->vp_base == XIVE_INVALID_VP)
1889 return -ENOSPC;
1890 }
1891
1892 vp_id = kvmppc_xive_vp(xive, cpu);
1893 if (kvmppc_xive_vp_in_use(xive->kvm, vp_id)) {
1894 pr_devel("Duplicate !\n");
1895 return -EEXIST;
1896 }
1897
1898 *vp = vp_id;
1899
1900 return 0;
1901 }
1902
1903 int kvmppc_xive_connect_vcpu(struct kvm_device *dev,
1904 struct kvm_vcpu *vcpu, u32 cpu)
1905 {
1906 struct kvmppc_xive *xive = dev->private;
1907 struct kvmppc_xive_vcpu *xc;
1908 int i, r = -EBUSY;
1909 u32 vp_id;
1910
1911 pr_devel("connect_vcpu(cpu=%d)\n", cpu);
1912
1913 if (dev->ops != &kvm_xive_ops) {
1914 pr_devel("Wrong ops !\n");
1915 return -EPERM;
1916 }
1917 if (xive->kvm != vcpu->kvm)
1918 return -EPERM;
1919 if (vcpu->arch.irq_type != KVMPPC_IRQ_DEFAULT)
1920 return -EBUSY;
1921
1922 /* We need to synchronize with queue provisioning */
1923 mutex_lock(&xive->lock);
1924
1925 r = kvmppc_xive_compute_vp_id(xive, cpu, &vp_id);
1926 if (r)
1927 goto bail;
1928
1929 xc = kzalloc(sizeof(*xc), GFP_KERNEL);
1930 if (!xc) {
1931 r = -ENOMEM;
1932 goto bail;
1933 }
1934
1935 vcpu->arch.xive_vcpu = xc;
1936 xc->xive = xive;
1937 xc->vcpu = vcpu;
1938 xc->server_num = cpu;
1939 xc->vp_id = vp_id;
1940 xc->mfrr = 0xff;
1941 xc->valid = true;
1942
1943 r = xive_native_get_vp_info(xc->vp_id, &xc->vp_cam, &xc->vp_chip_id);
1944 if (r)
1945 goto bail;
1946
1947 if (!kvmppc_xive_check_save_restore(vcpu)) {
1948 pr_err("inconsistent save-restore setup for VCPU %d\n", cpu);
1949 r = -EIO;
1950 goto bail;
1951 }
1952
1953 /* Configure VCPU fields for use by assembly push/pull */
1954 vcpu->arch.xive_saved_state.w01 = cpu_to_be64(0xff000000);
1955 vcpu->arch.xive_cam_word = cpu_to_be32(xc->vp_cam | TM_QW1W2_VO);
1956
1957 /* Allocate IPI */
1958 xc->vp_ipi = xive_native_alloc_irq();
1959 if (!xc->vp_ipi) {
1960 pr_err("Failed to allocate xive irq for VCPU IPI\n");
1961 r = -EIO;
1962 goto bail;
1963 }
1964 pr_devel(" IPI=0x%x\n", xc->vp_ipi);
1965
1966 r = xive_native_populate_irq_data(xc->vp_ipi, &xc->vp_ipi_data);
1967 if (r)
1968 goto bail;
1969
1970 /*
1971 * Enable the VP first as the single escalation mode will
1972 * affect escalation interrupts numbering
1973 */
1974 r = xive_native_enable_vp(xc->vp_id, kvmppc_xive_has_single_escalation(xive));
1975 if (r) {
1976 pr_err("Failed to enable VP in OPAL, err %d\n", r);
1977 goto bail;
1978 }
1979
1980 /*
1981 * Initialize queues. Initially we set them all for no queueing
1982 * and we enable escalation for queue 0 only which we'll use for
1983 * our mfrr change notifications. If the VCPU is hot-plugged, we
1984 * do handle provisioning however based on the existing "map"
1985 * of enabled queues.
1986 */
1987 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1988 struct xive_q *q = &xc->queues[i];
1989
1990 /* Single escalation, no queue 7 */
1991 if (i == 7 && kvmppc_xive_has_single_escalation(xive))
1992 break;
1993
1994 /* Is queue already enabled ? Provision it */
1995 if (xive->qmap & (1 << i)) {
1996 r = xive_provision_queue(vcpu, i);
1997 if (r == 0 && !kvmppc_xive_has_single_escalation(xive))
1998 kvmppc_xive_attach_escalation(
1999 vcpu, i, kvmppc_xive_has_single_escalation(xive));
2000 if (r)
2001 goto bail;
2002 } else {
2003 r = xive_native_configure_queue(xc->vp_id,
2004 q, i, NULL, 0, true);
2005 if (r) {
2006 pr_err("Failed to configure queue %d for VCPU %d\n",
2007 i, cpu);
2008 goto bail;
2009 }
2010 }
2011 }
2012
2013 /* If not done above, attach priority 0 escalation */
2014 r = kvmppc_xive_attach_escalation(vcpu, 0, kvmppc_xive_has_single_escalation(xive));
2015 if (r)
2016 goto bail;
2017
2018 /* Route the IPI */
2019 r = xive_native_configure_irq(xc->vp_ipi, xc->vp_id, 0, XICS_IPI);
2020 if (!r)
2021 xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_00);
2022
2023 bail:
2024 mutex_unlock(&xive->lock);
2025 if (r) {
2026 kvmppc_xive_cleanup_vcpu(vcpu);
2027 return r;
2028 }
2029
2030 vcpu->arch.irq_type = KVMPPC_IRQ_XICS;
2031 return 0;
2032 }
2033
2034 /*
2035 * Scanning of queues before/after migration save
2036 */
2037 static void xive_pre_save_set_queued(struct kvmppc_xive *xive, u32 irq)
2038 {
2039 struct kvmppc_xive_src_block *sb;
2040 struct kvmppc_xive_irq_state *state;
2041 u16 idx;
2042
2043 sb = kvmppc_xive_find_source(xive, irq, &idx);
2044 if (!sb)
2045 return;
2046
2047 state = &sb->irq_state[idx];
2048
2049 /* Some sanity checking */
2050 if (!state->valid) {
2051 pr_err("invalid irq 0x%x in cpu queue!\n", irq);
2052 return;
2053 }
2054
2055 /*
2056 * If the interrupt is in a queue it should have P set.
2057 * We warn so that gets reported. A backtrace isn't useful
2058 * so no need to use a WARN_ON.
2059 */
2060 if (!state->saved_p)
2061 pr_err("Interrupt 0x%x is marked in a queue but P not set !\n", irq);
2062
2063 /* Set flag */
2064 state->in_queue = true;
2065 }
2066
2067 static void xive_pre_save_mask_irq(struct kvmppc_xive *xive,
2068 struct kvmppc_xive_src_block *sb,
2069 u32 irq)
2070 {
2071 struct kvmppc_xive_irq_state *state = &sb->irq_state[irq];
2072
2073 if (!state->valid)
2074 return;
2075
2076 /* Mask and save state, this will also sync HW queues */
2077 state->saved_scan_prio = xive_lock_and_mask(xive, sb, state);
2078
2079 /* Transfer P and Q */
2080 state->saved_p = state->old_p;
2081 state->saved_q = state->old_q;
2082
2083 /* Unlock */
2084 arch_spin_unlock(&sb->lock);
2085 }
2086
2087 static void xive_pre_save_unmask_irq(struct kvmppc_xive *xive,
2088 struct kvmppc_xive_src_block *sb,
2089 u32 irq)
2090 {
2091 struct kvmppc_xive_irq_state *state = &sb->irq_state[irq];
2092
2093 if (!state->valid)
2094 return;
2095
2096 /*
2097 * Lock / exclude EOI (not technically necessary if the
2098 * guest isn't running concurrently. If this becomes a
2099 * performance issue we can probably remove the lock.
2100 */
2101 xive_lock_for_unmask(sb, state);
2102
2103 /* Restore mask/prio if it wasn't masked */
2104 if (state->saved_scan_prio != MASKED)
2105 xive_finish_unmask(xive, sb, state, state->saved_scan_prio);
2106
2107 /* Unlock */
2108 arch_spin_unlock(&sb->lock);
2109 }
2110
2111 static void xive_pre_save_queue(struct kvmppc_xive *xive, struct xive_q *q)
2112 {
2113 u32 idx = q->idx;
2114 u32 toggle = q->toggle;
2115 u32 irq;
2116
2117 do {
2118 irq = __xive_read_eq(q->qpage, q->msk, &idx, &toggle);
2119 if (irq > XICS_IPI)
2120 xive_pre_save_set_queued(xive, irq);
2121 } while(irq);
2122 }
2123
2124 static void xive_pre_save_scan(struct kvmppc_xive *xive)
2125 {
2126 struct kvm_vcpu *vcpu = NULL;
2127 unsigned long i;
2128 int j;
2129
2130 /*
2131 * See comment in xive_get_source() about how this
2132 * work. Collect a stable state for all interrupts
2133 */
2134 for (i = 0; i <= xive->max_sbid; i++) {
2135 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
2136 if (!sb)
2137 continue;
2138 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
2139 xive_pre_save_mask_irq(xive, sb, j);
2140 }
2141
2142 /* Then scan the queues and update the "in_queue" flag */
2143 kvm_for_each_vcpu(i, vcpu, xive->kvm) {
2144 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
2145 if (!xc)
2146 continue;
2147 for (j = 0; j < KVMPPC_XIVE_Q_COUNT; j++) {
2148 if (xc->queues[j].qpage)
2149 xive_pre_save_queue(xive, &xc->queues[j]);
2150 }
2151 }
2152
2153 /* Finally restore interrupt states */
2154 for (i = 0; i <= xive->max_sbid; i++) {
2155 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
2156 if (!sb)
2157 continue;
2158 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
2159 xive_pre_save_unmask_irq(xive, sb, j);
2160 }
2161 }
2162
2163 static void xive_post_save_scan(struct kvmppc_xive *xive)
2164 {
2165 u32 i, j;
2166
2167 /* Clear all the in_queue flags */
2168 for (i = 0; i <= xive->max_sbid; i++) {
2169 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
2170 if (!sb)
2171 continue;
2172 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
2173 sb->irq_state[j].in_queue = false;
2174 }
2175
2176 /* Next get_source() will do a new scan */
2177 xive->saved_src_count = 0;
2178 }
2179
2180 /*
2181 * This returns the source configuration and state to user space.
2182 */
2183 static int xive_get_source(struct kvmppc_xive *xive, long irq, u64 addr)
2184 {
2185 struct kvmppc_xive_src_block *sb;
2186 struct kvmppc_xive_irq_state *state;
2187 u64 __user *ubufp = (u64 __user *) addr;
2188 u64 val, prio;
2189 u16 idx;
2190
2191 sb = kvmppc_xive_find_source(xive, irq, &idx);
2192 if (!sb)
2193 return -ENOENT;
2194
2195 state = &sb->irq_state[idx];
2196
2197 if (!state->valid)
2198 return -ENOENT;
2199
2200 pr_devel("get_source(%ld)...\n", irq);
2201
2202 /*
2203 * So to properly save the state into something that looks like a
2204 * XICS migration stream we cannot treat interrupts individually.
2205 *
2206 * We need, instead, mask them all (& save their previous PQ state)
2207 * to get a stable state in the HW, then sync them to ensure that
2208 * any interrupt that had already fired hits its queue, and finally
2209 * scan all the queues to collect which interrupts are still present
2210 * in the queues, so we can set the "pending" flag on them and
2211 * they can be resent on restore.
2212 *
2213 * So we do it all when the "first" interrupt gets saved, all the
2214 * state is collected at that point, the rest of xive_get_source()
2215 * will merely collect and convert that state to the expected
2216 * userspace bit mask.
2217 */
2218 if (xive->saved_src_count == 0)
2219 xive_pre_save_scan(xive);
2220 xive->saved_src_count++;
2221
2222 /* Convert saved state into something compatible with xics */
2223 val = state->act_server;
2224 prio = state->saved_scan_prio;
2225
2226 if (prio == MASKED) {
2227 val |= KVM_XICS_MASKED;
2228 prio = state->saved_priority;
2229 }
2230 val |= prio << KVM_XICS_PRIORITY_SHIFT;
2231 if (state->lsi) {
2232 val |= KVM_XICS_LEVEL_SENSITIVE;
2233 if (state->saved_p)
2234 val |= KVM_XICS_PENDING;
2235 } else {
2236 if (state->saved_p)
2237 val |= KVM_XICS_PRESENTED;
2238
2239 if (state->saved_q)
2240 val |= KVM_XICS_QUEUED;
2241
2242 /*
2243 * We mark it pending (which will attempt a re-delivery)
2244 * if we are in a queue *or* we were masked and had
2245 * Q set which is equivalent to the XICS "masked pending"
2246 * state
2247 */
2248 if (state->in_queue || (prio == MASKED && state->saved_q))
2249 val |= KVM_XICS_PENDING;
2250 }
2251
2252 /*
2253 * If that was the last interrupt saved, reset the
2254 * in_queue flags
2255 */
2256 if (xive->saved_src_count == xive->src_count)
2257 xive_post_save_scan(xive);
2258
2259 /* Copy the result to userspace */
2260 if (put_user(val, ubufp))
2261 return -EFAULT;
2262
2263 return 0;
2264 }
2265
2266 struct kvmppc_xive_src_block *kvmppc_xive_create_src_block(
2267 struct kvmppc_xive *xive, int irq)
2268 {
2269 struct kvmppc_xive_src_block *sb;
2270 int i, bid;
2271
2272 bid = irq >> KVMPPC_XICS_ICS_SHIFT;
2273
2274 mutex_lock(&xive->lock);
2275
2276 /* block already exists - somebody else got here first */
2277 if (xive->src_blocks[bid])
2278 goto out;
2279
2280 /* Create the ICS */
2281 sb = kzalloc(sizeof(*sb), GFP_KERNEL);
2282 if (!sb)
2283 goto out;
2284
2285 sb->id = bid;
2286
2287 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
2288 sb->irq_state[i].number = (bid << KVMPPC_XICS_ICS_SHIFT) | i;
2289 sb->irq_state[i].eisn = 0;
2290 sb->irq_state[i].guest_priority = MASKED;
2291 sb->irq_state[i].saved_priority = MASKED;
2292 sb->irq_state[i].act_priority = MASKED;
2293 }
2294 smp_wmb();
2295 xive->src_blocks[bid] = sb;
2296
2297 if (bid > xive->max_sbid)
2298 xive->max_sbid = bid;
2299
2300 out:
2301 mutex_unlock(&xive->lock);
2302 return xive->src_blocks[bid];
2303 }
2304
2305 static bool xive_check_delayed_irq(struct kvmppc_xive *xive, u32 irq)
2306 {
2307 struct kvm *kvm = xive->kvm;
2308 struct kvm_vcpu *vcpu = NULL;
2309 unsigned long i;
2310
2311 kvm_for_each_vcpu(i, vcpu, kvm) {
2312 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
2313
2314 if (!xc)
2315 continue;
2316
2317 if (xc->delayed_irq == irq) {
2318 xc->delayed_irq = 0;
2319 xive->delayed_irqs--;
2320 return true;
2321 }
2322 }
2323 return false;
2324 }
2325
2326 static int xive_set_source(struct kvmppc_xive *xive, long irq, u64 addr)
2327 {
2328 struct kvmppc_xive_src_block *sb;
2329 struct kvmppc_xive_irq_state *state;
2330 u64 __user *ubufp = (u64 __user *) addr;
2331 u16 idx;
2332 u64 val;
2333 u8 act_prio, guest_prio;
2334 u32 server;
2335 int rc = 0;
2336
2337 if (irq < KVMPPC_XICS_FIRST_IRQ || irq >= KVMPPC_XICS_NR_IRQS)
2338 return -ENOENT;
2339
2340 pr_devel("set_source(irq=0x%lx)\n", irq);
2341
2342 /* Find the source */
2343 sb = kvmppc_xive_find_source(xive, irq, &idx);
2344 if (!sb) {
2345 pr_devel("No source, creating source block...\n");
2346 sb = kvmppc_xive_create_src_block(xive, irq);
2347 if (!sb) {
2348 pr_devel("Failed to create block...\n");
2349 return -ENOMEM;
2350 }
2351 }
2352 state = &sb->irq_state[idx];
2353
2354 /* Read user passed data */
2355 if (get_user(val, ubufp)) {
2356 pr_devel("fault getting user info !\n");
2357 return -EFAULT;
2358 }
2359
2360 server = val & KVM_XICS_DESTINATION_MASK;
2361 guest_prio = val >> KVM_XICS_PRIORITY_SHIFT;
2362
2363 pr_devel(" val=0x016%llx (server=0x%x, guest_prio=%d)\n",
2364 val, server, guest_prio);
2365
2366 /*
2367 * If the source doesn't already have an IPI, allocate
2368 * one and get the corresponding data
2369 */
2370 if (!state->ipi_number) {
2371 state->ipi_number = xive_native_alloc_irq();
2372 if (state->ipi_number == 0) {
2373 pr_devel("Failed to allocate IPI !\n");
2374 return -ENOMEM;
2375 }
2376 xive_native_populate_irq_data(state->ipi_number, &state->ipi_data);
2377 pr_devel(" src_ipi=0x%x\n", state->ipi_number);
2378 }
2379
2380 /*
2381 * We use lock_and_mask() to set us in the right masked
2382 * state. We will override that state from the saved state
2383 * further down, but this will handle the cases of interrupts
2384 * that need FW masking. We set the initial guest_priority to
2385 * 0 before calling it to ensure it actually performs the masking.
2386 */
2387 state->guest_priority = 0;
2388 xive_lock_and_mask(xive, sb, state);
2389
2390 /*
2391 * Now, we select a target if we have one. If we don't we
2392 * leave the interrupt untargetted. It means that an interrupt
2393 * can become "untargetted" across migration if it was masked
2394 * by set_xive() but there is little we can do about it.
2395 */
2396
2397 /* First convert prio and mark interrupt as untargetted */
2398 act_prio = xive_prio_from_guest(guest_prio);
2399 state->act_priority = MASKED;
2400
2401 /*
2402 * We need to drop the lock due to the mutex below. Hopefully
2403 * nothing is touching that interrupt yet since it hasn't been
2404 * advertized to a running guest yet
2405 */
2406 arch_spin_unlock(&sb->lock);
2407
2408 /* If we have a priority target the interrupt */
2409 if (act_prio != MASKED) {
2410 /* First, check provisioning of queues */
2411 mutex_lock(&xive->lock);
2412 rc = xive_check_provisioning(xive->kvm, act_prio);
2413 mutex_unlock(&xive->lock);
2414
2415 /* Target interrupt */
2416 if (rc == 0)
2417 rc = xive_target_interrupt(xive->kvm, state,
2418 server, act_prio);
2419 /*
2420 * If provisioning or targetting failed, leave it
2421 * alone and masked. It will remain disabled until
2422 * the guest re-targets it.
2423 */
2424 }
2425
2426 /*
2427 * Find out if this was a delayed irq stashed in an ICP,
2428 * in which case, treat it as pending
2429 */
2430 if (xive->delayed_irqs && xive_check_delayed_irq(xive, irq)) {
2431 val |= KVM_XICS_PENDING;
2432 pr_devel(" Found delayed ! forcing PENDING !\n");
2433 }
2434
2435 /* Cleanup the SW state */
2436 state->old_p = false;
2437 state->old_q = false;
2438 state->lsi = false;
2439 state->asserted = false;
2440
2441 /* Restore LSI state */
2442 if (val & KVM_XICS_LEVEL_SENSITIVE) {
2443 state->lsi = true;
2444 if (val & KVM_XICS_PENDING)
2445 state->asserted = true;
2446 pr_devel(" LSI ! Asserted=%d\n", state->asserted);
2447 }
2448
2449 /*
2450 * Restore P and Q. If the interrupt was pending, we
2451 * force Q and !P, which will trigger a resend.
2452 *
2453 * That means that a guest that had both an interrupt
2454 * pending (queued) and Q set will restore with only
2455 * one instance of that interrupt instead of 2, but that
2456 * is perfectly fine as coalescing interrupts that haven't
2457 * been presented yet is always allowed.
2458 */
2459 if (val & KVM_XICS_PRESENTED && !(val & KVM_XICS_PENDING))
2460 state->old_p = true;
2461 if (val & KVM_XICS_QUEUED || val & KVM_XICS_PENDING)
2462 state->old_q = true;
2463
2464 pr_devel(" P=%d, Q=%d\n", state->old_p, state->old_q);
2465
2466 /*
2467 * If the interrupt was unmasked, update guest priority and
2468 * perform the appropriate state transition and do a
2469 * re-trigger if necessary.
2470 */
2471 if (val & KVM_XICS_MASKED) {
2472 pr_devel(" masked, saving prio\n");
2473 state->guest_priority = MASKED;
2474 state->saved_priority = guest_prio;
2475 } else {
2476 pr_devel(" unmasked, restoring to prio %d\n", guest_prio);
2477 xive_finish_unmask(xive, sb, state, guest_prio);
2478 state->saved_priority = guest_prio;
2479 }
2480
2481 /* Increment the number of valid sources and mark this one valid */
2482 if (!state->valid)
2483 xive->src_count++;
2484 state->valid = true;
2485
2486 return 0;
2487 }
2488
2489 int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
2490 bool line_status)
2491 {
2492 struct kvmppc_xive *xive = kvm->arch.xive;
2493 struct kvmppc_xive_src_block *sb;
2494 struct kvmppc_xive_irq_state *state;
2495 u16 idx;
2496
2497 if (!xive)
2498 return -ENODEV;
2499
2500 sb = kvmppc_xive_find_source(xive, irq, &idx);
2501 if (!sb)
2502 return -EINVAL;
2503
2504 /* Perform locklessly .... (we need to do some RCUisms here...) */
2505 state = &sb->irq_state[idx];
2506 if (!state->valid)
2507 return -EINVAL;
2508
2509 /* We don't allow a trigger on a passed-through interrupt */
2510 if (state->pt_number)
2511 return -EINVAL;
2512
2513 if ((level == 1 && state->lsi) || level == KVM_INTERRUPT_SET_LEVEL)
2514 state->asserted = true;
2515 else if (level == 0 || level == KVM_INTERRUPT_UNSET) {
2516 state->asserted = false;
2517 return 0;
2518 }
2519
2520 /* Trigger the IPI */
2521 xive_irq_trigger(&state->ipi_data);
2522
2523 return 0;
2524 }
2525
2526 int kvmppc_xive_set_nr_servers(struct kvmppc_xive *xive, u64 addr)
2527 {
2528 u32 __user *ubufp = (u32 __user *) addr;
2529 u32 nr_servers;
2530 int rc = 0;
2531
2532 if (get_user(nr_servers, ubufp))
2533 return -EFAULT;
2534
2535 pr_devel("%s nr_servers=%u\n", __func__, nr_servers);
2536
2537 if (!nr_servers || nr_servers > KVM_MAX_VCPU_IDS)
2538 return -EINVAL;
2539
2540 mutex_lock(&xive->lock);
2541 if (xive->vp_base != XIVE_INVALID_VP)
2542 /* The VP block is allocated once and freed when the device
2543 * is released. Better not allow to change its size since its
2544 * used by connect_vcpu to validate vCPU ids are valid (eg,
2545 * setting it back to a higher value could allow connect_vcpu
2546 * to come up with a VP id that goes beyond the VP block, which
2547 * is likely to cause a crash in OPAL).
2548 */
2549 rc = -EBUSY;
2550 else if (nr_servers > KVM_MAX_VCPUS)
2551 /* We don't need more servers. Higher vCPU ids get packed
2552 * down below KVM_MAX_VCPUS by kvmppc_pack_vcpu_id().
2553 */
2554 xive->nr_servers = KVM_MAX_VCPUS;
2555 else
2556 xive->nr_servers = nr_servers;
2557
2558 mutex_unlock(&xive->lock);
2559
2560 return rc;
2561 }
2562
2563 static int xive_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2564 {
2565 struct kvmppc_xive *xive = dev->private;
2566
2567 /* We honor the existing XICS ioctl */
2568 switch (attr->group) {
2569 case KVM_DEV_XICS_GRP_SOURCES:
2570 return xive_set_source(xive, attr->attr, attr->addr);
2571 case KVM_DEV_XICS_GRP_CTRL:
2572 switch (attr->attr) {
2573 case KVM_DEV_XICS_NR_SERVERS:
2574 return kvmppc_xive_set_nr_servers(xive, attr->addr);
2575 }
2576 }
2577 return -ENXIO;
2578 }
2579
2580 static int xive_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2581 {
2582 struct kvmppc_xive *xive = dev->private;
2583
2584 /* We honor the existing XICS ioctl */
2585 switch (attr->group) {
2586 case KVM_DEV_XICS_GRP_SOURCES:
2587 return xive_get_source(xive, attr->attr, attr->addr);
2588 }
2589 return -ENXIO;
2590 }
2591
2592 static int xive_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2593 {
2594 /* We honor the same limits as XICS, at least for now */
2595 switch (attr->group) {
2596 case KVM_DEV_XICS_GRP_SOURCES:
2597 if (attr->attr >= KVMPPC_XICS_FIRST_IRQ &&
2598 attr->attr < KVMPPC_XICS_NR_IRQS)
2599 return 0;
2600 break;
2601 case KVM_DEV_XICS_GRP_CTRL:
2602 switch (attr->attr) {
2603 case KVM_DEV_XICS_NR_SERVERS:
2604 return 0;
2605 }
2606 }
2607 return -ENXIO;
2608 }
2609
2610 static void kvmppc_xive_cleanup_irq(u32 hw_num, struct xive_irq_data *xd)
2611 {
2612 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01);
2613 xive_native_configure_irq(hw_num, 0, MASKED, 0);
2614 }
2615
2616 void kvmppc_xive_free_sources(struct kvmppc_xive_src_block *sb)
2617 {
2618 int i;
2619
2620 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
2621 struct kvmppc_xive_irq_state *state = &sb->irq_state[i];
2622
2623 if (!state->valid)
2624 continue;
2625
2626 kvmppc_xive_cleanup_irq(state->ipi_number, &state->ipi_data);
2627 xive_cleanup_irq_data(&state->ipi_data);
2628 xive_native_free_irq(state->ipi_number);
2629
2630 /* Pass-through, cleanup too but keep IRQ hw data */
2631 if (state->pt_number)
2632 kvmppc_xive_cleanup_irq(state->pt_number, state->pt_data);
2633
2634 state->valid = false;
2635 }
2636 }
2637
2638 /*
2639 * Called when device fd is closed. kvm->lock is held.
2640 */
2641 static void kvmppc_xive_release(struct kvm_device *dev)
2642 {
2643 struct kvmppc_xive *xive = dev->private;
2644 struct kvm *kvm = xive->kvm;
2645 struct kvm_vcpu *vcpu;
2646 unsigned long i;
2647
2648 pr_devel("Releasing xive device\n");
2649
2650 /*
2651 * Since this is the device release function, we know that
2652 * userspace does not have any open fd referring to the
2653 * device. Therefore there can not be any of the device
2654 * attribute set/get functions being executed concurrently,
2655 * and similarly, the connect_vcpu and set/clr_mapped
2656 * functions also cannot be being executed.
2657 */
2658
2659 debugfs_remove(xive->dentry);
2660
2661 /*
2662 * We should clean up the vCPU interrupt presenters first.
2663 */
2664 kvm_for_each_vcpu(i, vcpu, kvm) {
2665 /*
2666 * Take vcpu->mutex to ensure that no one_reg get/set ioctl
2667 * (i.e. kvmppc_xive_[gs]et_icp) can be done concurrently.
2668 * Holding the vcpu->mutex also means that the vcpu cannot
2669 * be executing the KVM_RUN ioctl, and therefore it cannot
2670 * be executing the XIVE push or pull code or accessing
2671 * the XIVE MMIO regions.
2672 */
2673 mutex_lock(&vcpu->mutex);
2674 kvmppc_xive_cleanup_vcpu(vcpu);
2675 mutex_unlock(&vcpu->mutex);
2676 }
2677
2678 /*
2679 * Now that we have cleared vcpu->arch.xive_vcpu, vcpu->arch.irq_type
2680 * and vcpu->arch.xive_esc_[vr]addr on each vcpu, we are safe
2681 * against xive code getting called during vcpu execution or
2682 * set/get one_reg operations.
2683 */
2684 kvm->arch.xive = NULL;
2685
2686 /* Mask and free interrupts */
2687 for (i = 0; i <= xive->max_sbid; i++) {
2688 if (xive->src_blocks[i])
2689 kvmppc_xive_free_sources(xive->src_blocks[i]);
2690 kfree(xive->src_blocks[i]);
2691 xive->src_blocks[i] = NULL;
2692 }
2693
2694 if (xive->vp_base != XIVE_INVALID_VP)
2695 xive_native_free_vp_block(xive->vp_base);
2696
2697 /*
2698 * A reference of the kvmppc_xive pointer is now kept under
2699 * the xive_devices struct of the machine for reuse. It is
2700 * freed when the VM is destroyed for now until we fix all the
2701 * execution paths.
2702 */
2703
2704 kfree(dev);
2705 }
2706
2707 /*
2708 * When the guest chooses the interrupt mode (XICS legacy or XIVE
2709 * native), the VM will switch of KVM device. The previous device will
2710 * be "released" before the new one is created.
2711 *
2712 * Until we are sure all execution paths are well protected, provide a
2713 * fail safe (transitional) method for device destruction, in which
2714 * the XIVE device pointer is recycled and not directly freed.
2715 */
2716 struct kvmppc_xive *kvmppc_xive_get_device(struct kvm *kvm, u32 type)
2717 {
2718 struct kvmppc_xive **kvm_xive_device = type == KVM_DEV_TYPE_XIVE ?
2719 &kvm->arch.xive_devices.native :
2720 &kvm->arch.xive_devices.xics_on_xive;
2721 struct kvmppc_xive *xive = *kvm_xive_device;
2722
2723 if (!xive) {
2724 xive = kzalloc(sizeof(*xive), GFP_KERNEL);
2725 *kvm_xive_device = xive;
2726 } else {
2727 memset(xive, 0, sizeof(*xive));
2728 }
2729
2730 return xive;
2731 }
2732
2733 /*
2734 * Create a XICS device with XIVE backend. kvm->lock is held.
2735 */
2736 static int kvmppc_xive_create(struct kvm_device *dev, u32 type)
2737 {
2738 struct kvmppc_xive *xive;
2739 struct kvm *kvm = dev->kvm;
2740
2741 pr_devel("Creating xive for partition\n");
2742
2743 /* Already there ? */
2744 if (kvm->arch.xive)
2745 return -EEXIST;
2746
2747 xive = kvmppc_xive_get_device(kvm, type);
2748 if (!xive)
2749 return -ENOMEM;
2750
2751 dev->private = xive;
2752 xive->dev = dev;
2753 xive->kvm = kvm;
2754 mutex_init(&xive->lock);
2755
2756 /* We use the default queue size set by the host */
2757 xive->q_order = xive_native_default_eq_shift();
2758 if (xive->q_order < PAGE_SHIFT)
2759 xive->q_page_order = 0;
2760 else
2761 xive->q_page_order = xive->q_order - PAGE_SHIFT;
2762
2763 /* VP allocation is delayed to the first call to connect_vcpu */
2764 xive->vp_base = XIVE_INVALID_VP;
2765 /* KVM_MAX_VCPUS limits the number of VMs to roughly 64 per sockets
2766 * on a POWER9 system.
2767 */
2768 xive->nr_servers = KVM_MAX_VCPUS;
2769
2770 if (xive_native_has_single_escalation())
2771 xive->flags |= KVMPPC_XIVE_FLAG_SINGLE_ESCALATION;
2772
2773 if (xive_native_has_save_restore())
2774 xive->flags |= KVMPPC_XIVE_FLAG_SAVE_RESTORE;
2775
2776 kvm->arch.xive = xive;
2777 return 0;
2778 }
2779
2780 int kvmppc_xive_xics_hcall(struct kvm_vcpu *vcpu, u32 req)
2781 {
2782 /* The VM should have configured XICS mode before doing XICS hcalls. */
2783 if (!kvmppc_xics_enabled(vcpu))
2784 return H_TOO_HARD;
2785
2786 switch (req) {
2787 case H_XIRR:
2788 return xive_vm_h_xirr(vcpu);
2789 case H_CPPR:
2790 return xive_vm_h_cppr(vcpu, kvmppc_get_gpr(vcpu, 4));
2791 case H_EOI:
2792 return xive_vm_h_eoi(vcpu, kvmppc_get_gpr(vcpu, 4));
2793 case H_IPI:
2794 return xive_vm_h_ipi(vcpu, kvmppc_get_gpr(vcpu, 4),
2795 kvmppc_get_gpr(vcpu, 5));
2796 case H_IPOLL:
2797 return xive_vm_h_ipoll(vcpu, kvmppc_get_gpr(vcpu, 4));
2798 case H_XIRR_X:
2799 xive_vm_h_xirr(vcpu);
2800 kvmppc_set_gpr(vcpu, 5, get_tb() + kvmppc_get_tb_offset(vcpu));
2801 return H_SUCCESS;
2802 }
2803
2804 return H_UNSUPPORTED;
2805 }
2806 EXPORT_SYMBOL_GPL(kvmppc_xive_xics_hcall);
2807
2808 int kvmppc_xive_debug_show_queues(struct seq_file *m, struct kvm_vcpu *vcpu)
2809 {
2810 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
2811 unsigned int i;
2812
2813 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
2814 struct xive_q *q = &xc->queues[i];
2815 u32 i0, i1, idx;
2816
2817 if (!q->qpage && !xc->esc_virq[i])
2818 continue;
2819
2820 if (q->qpage) {
2821 seq_printf(m, " q[%d]: ", i);
2822 idx = q->idx;
2823 i0 = be32_to_cpup(q->qpage + idx);
2824 idx = (idx + 1) & q->msk;
2825 i1 = be32_to_cpup(q->qpage + idx);
2826 seq_printf(m, "T=%d %08x %08x...\n", q->toggle,
2827 i0, i1);
2828 }
2829 if (xc->esc_virq[i]) {
2830 struct irq_data *d = irq_get_irq_data(xc->esc_virq[i]);
2831 struct xive_irq_data *xd =
2832 irq_data_get_irq_handler_data(d);
2833 u64 pq = xive_vm_esb_load(xd, XIVE_ESB_GET);
2834
2835 seq_printf(m, " ESC %d %c%c EOI @%llx",
2836 xc->esc_virq[i],
2837 (pq & XIVE_ESB_VAL_P) ? 'P' : '-',
2838 (pq & XIVE_ESB_VAL_Q) ? 'Q' : '-',
2839 xd->eoi_page);
2840 seq_puts(m, "\n");
2841 }
2842 }
2843 return 0;
2844 }
2845
2846 void kvmppc_xive_debug_show_sources(struct seq_file *m,
2847 struct kvmppc_xive_src_block *sb)
2848 {
2849 int i;
2850
2851 seq_puts(m, " LISN HW/CHIP TYPE PQ EISN CPU/PRIO\n");
2852 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
2853 struct kvmppc_xive_irq_state *state = &sb->irq_state[i];
2854 struct xive_irq_data *xd;
2855 u64 pq;
2856 u32 hw_num;
2857
2858 if (!state->valid)
2859 continue;
2860
2861 kvmppc_xive_select_irq(state, &hw_num, &xd);
2862
2863 pq = xive_vm_esb_load(xd, XIVE_ESB_GET);
2864
2865 seq_printf(m, "%08x %08x/%02x", state->number, hw_num,
2866 xd->src_chip);
2867 if (state->lsi)
2868 seq_printf(m, " %cLSI", state->asserted ? '^' : ' ');
2869 else
2870 seq_puts(m, " MSI");
2871
2872 seq_printf(m, " %s %c%c %08x % 4d/%d",
2873 state->ipi_number == hw_num ? "IPI" : " PT",
2874 pq & XIVE_ESB_VAL_P ? 'P' : '-',
2875 pq & XIVE_ESB_VAL_Q ? 'Q' : '-',
2876 state->eisn, state->act_server,
2877 state->act_priority);
2878
2879 seq_puts(m, "\n");
2880 }
2881 }
2882
2883 static int xive_debug_show(struct seq_file *m, void *private)
2884 {
2885 struct kvmppc_xive *xive = m->private;
2886 struct kvm *kvm = xive->kvm;
2887 struct kvm_vcpu *vcpu;
2888 u64 t_rm_h_xirr = 0;
2889 u64 t_rm_h_ipoll = 0;
2890 u64 t_rm_h_cppr = 0;
2891 u64 t_rm_h_eoi = 0;
2892 u64 t_rm_h_ipi = 0;
2893 u64 t_vm_h_xirr = 0;
2894 u64 t_vm_h_ipoll = 0;
2895 u64 t_vm_h_cppr = 0;
2896 u64 t_vm_h_eoi = 0;
2897 u64 t_vm_h_ipi = 0;
2898 unsigned long i;
2899
2900 if (!kvm)
2901 return 0;
2902
2903 seq_puts(m, "=========\nVCPU state\n=========\n");
2904
2905 kvm_for_each_vcpu(i, vcpu, kvm) {
2906 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
2907
2908 if (!xc)
2909 continue;
2910
2911 seq_printf(m, "VCPU %d: VP:%#x/%02x\n"
2912 " CPPR:%#x HWCPPR:%#x MFRR:%#x PEND:%#x h_xirr: R=%lld V=%lld\n",
2913 xc->server_num, xc->vp_id, xc->vp_chip_id,
2914 xc->cppr, xc->hw_cppr,
2915 xc->mfrr, xc->pending,
2916 xc->stat_rm_h_xirr, xc->stat_vm_h_xirr);
2917
2918 kvmppc_xive_debug_show_queues(m, vcpu);
2919
2920 t_rm_h_xirr += xc->stat_rm_h_xirr;
2921 t_rm_h_ipoll += xc->stat_rm_h_ipoll;
2922 t_rm_h_cppr += xc->stat_rm_h_cppr;
2923 t_rm_h_eoi += xc->stat_rm_h_eoi;
2924 t_rm_h_ipi += xc->stat_rm_h_ipi;
2925 t_vm_h_xirr += xc->stat_vm_h_xirr;
2926 t_vm_h_ipoll += xc->stat_vm_h_ipoll;
2927 t_vm_h_cppr += xc->stat_vm_h_cppr;
2928 t_vm_h_eoi += xc->stat_vm_h_eoi;
2929 t_vm_h_ipi += xc->stat_vm_h_ipi;
2930 }
2931
2932 seq_puts(m, "Hcalls totals\n");
2933 seq_printf(m, " H_XIRR R=%10lld V=%10lld\n", t_rm_h_xirr, t_vm_h_xirr);
2934 seq_printf(m, " H_IPOLL R=%10lld V=%10lld\n", t_rm_h_ipoll, t_vm_h_ipoll);
2935 seq_printf(m, " H_CPPR R=%10lld V=%10lld\n", t_rm_h_cppr, t_vm_h_cppr);
2936 seq_printf(m, " H_EOI R=%10lld V=%10lld\n", t_rm_h_eoi, t_vm_h_eoi);
2937 seq_printf(m, " H_IPI R=%10lld V=%10lld\n", t_rm_h_ipi, t_vm_h_ipi);
2938
2939 seq_puts(m, "=========\nSources\n=========\n");
2940
2941 for (i = 0; i <= xive->max_sbid; i++) {
2942 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
2943
2944 if (sb) {
2945 arch_spin_lock(&sb->lock);
2946 kvmppc_xive_debug_show_sources(m, sb);
2947 arch_spin_unlock(&sb->lock);
2948 }
2949 }
2950
2951 return 0;
2952 }
2953
2954 DEFINE_SHOW_ATTRIBUTE(xive_debug);
2955
2956 static void xive_debugfs_init(struct kvmppc_xive *xive)
2957 {
2958 xive->dentry = debugfs_create_file("xive", S_IRUGO, xive->kvm->debugfs_dentry,
2959 xive, &xive_debug_fops);
2960
2961 pr_debug("%s: created\n", __func__);
2962 }
2963
2964 static void kvmppc_xive_init(struct kvm_device *dev)
2965 {
2966 struct kvmppc_xive *xive = dev->private;
2967
2968 /* Register some debug interfaces */
2969 xive_debugfs_init(xive);
2970 }
2971
2972 struct kvm_device_ops kvm_xive_ops = {
2973 .name = "kvm-xive",
2974 .create = kvmppc_xive_create,
2975 .init = kvmppc_xive_init,
2976 .release = kvmppc_xive_release,
2977 .set_attr = xive_set_attr,
2978 .get_attr = xive_get_attr,
2979 .has_attr = xive_has_attr,
2980 };
2981
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.