1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
4 * Copyright (C) 2022 Ventana Micro Systems Inc.
5 *
6 * Authors:
7 * Anup Patel <apatel@ventanamicro.com>
8 */
9
10 #include <linux/atomic.h>
11 #include <linux/bitmap.h>
12 #include <linux/irqchip/riscv-imsic.h>
13 #include <linux/kvm_host.h>
14 #include <linux/math.h>
15 #include <linux/spinlock.h>
16 #include <linux/swab.h>
17 #include <kvm/iodev.h>
18 #include <asm/csr.h>
19
20 #define IMSIC_MAX_EIX (IMSIC_MAX_ID / BITS_PER_TYPE(u64))
21
22 struct imsic_mrif_eix {
23 unsigned long eip[BITS_PER_TYPE(u64) / BITS_PER_LONG];
24 unsigned long eie[BITS_PER_TYPE(u64) / BITS_PER_LONG];
25 };
26
27 struct imsic_mrif {
28 struct imsic_mrif_eix eix[IMSIC_MAX_EIX];
29 unsigned long eithreshold;
30 unsigned long eidelivery;
31 };
32
33 struct imsic {
34 struct kvm_io_device iodev;
35
36 u32 nr_msis;
37 u32 nr_eix;
38 u32 nr_hw_eix;
39
40 /*
41 * At any point in time, the register state is in
42 * one of the following places:
43 *
44 * 1) Hardware: IMSIC VS-file (vsfile_cpu >= 0)
45 * 2) Software: IMSIC SW-file (vsfile_cpu < 0)
46 */
47
48 /* IMSIC VS-file */
49 rwlock_t vsfile_lock;
50 int vsfile_cpu;
51 int vsfile_hgei;
52 void __iomem *vsfile_va;
53 phys_addr_t vsfile_pa;
54
55 /* IMSIC SW-file */
56 struct imsic_mrif *swfile;
57 phys_addr_t swfile_pa;
58 spinlock_t swfile_extirq_lock;
59 };
60
61 #define imsic_vs_csr_read(__c) \
62 ({ \
63 unsigned long __r; \
64 csr_write(CSR_VSISELECT, __c); \
65 __r = csr_read(CSR_VSIREG); \
66 __r; \
67 })
68
69 #define imsic_read_switchcase(__ireg) \
70 case __ireg: \
71 return imsic_vs_csr_read(__ireg);
72 #define imsic_read_switchcase_2(__ireg) \
73 imsic_read_switchcase(__ireg + 0) \
74 imsic_read_switchcase(__ireg + 1)
75 #define imsic_read_switchcase_4(__ireg) \
76 imsic_read_switchcase_2(__ireg + 0) \
77 imsic_read_switchcase_2(__ireg + 2)
78 #define imsic_read_switchcase_8(__ireg) \
79 imsic_read_switchcase_4(__ireg + 0) \
80 imsic_read_switchcase_4(__ireg + 4)
81 #define imsic_read_switchcase_16(__ireg) \
82 imsic_read_switchcase_8(__ireg + 0) \
83 imsic_read_switchcase_8(__ireg + 8)
84 #define imsic_read_switchcase_32(__ireg) \
85 imsic_read_switchcase_16(__ireg + 0) \
86 imsic_read_switchcase_16(__ireg + 16)
87 #define imsic_read_switchcase_64(__ireg) \
88 imsic_read_switchcase_32(__ireg + 0) \
89 imsic_read_switchcase_32(__ireg + 32)
90
91 static unsigned long imsic_eix_read(int ireg)
92 {
93 switch (ireg) {
94 imsic_read_switchcase_64(IMSIC_EIP0)
95 imsic_read_switchcase_64(IMSIC_EIE0)
96 }
97
98 return 0;
99 }
100
101 #define imsic_vs_csr_swap(__c, __v) \
102 ({ \
103 unsigned long __r; \
104 csr_write(CSR_VSISELECT, __c); \
105 __r = csr_swap(CSR_VSIREG, __v); \
106 __r; \
107 })
108
109 #define imsic_swap_switchcase(__ireg, __v) \
110 case __ireg: \
111 return imsic_vs_csr_swap(__ireg, __v);
112 #define imsic_swap_switchcase_2(__ireg, __v) \
113 imsic_swap_switchcase(__ireg + 0, __v) \
114 imsic_swap_switchcase(__ireg + 1, __v)
115 #define imsic_swap_switchcase_4(__ireg, __v) \
116 imsic_swap_switchcase_2(__ireg + 0, __v) \
117 imsic_swap_switchcase_2(__ireg + 2, __v)
118 #define imsic_swap_switchcase_8(__ireg, __v) \
119 imsic_swap_switchcase_4(__ireg + 0, __v) \
120 imsic_swap_switchcase_4(__ireg + 4, __v)
121 #define imsic_swap_switchcase_16(__ireg, __v) \
122 imsic_swap_switchcase_8(__ireg + 0, __v) \
123 imsic_swap_switchcase_8(__ireg + 8, __v)
124 #define imsic_swap_switchcase_32(__ireg, __v) \
125 imsic_swap_switchcase_16(__ireg + 0, __v) \
126 imsic_swap_switchcase_16(__ireg + 16, __v)
127 #define imsic_swap_switchcase_64(__ireg, __v) \
128 imsic_swap_switchcase_32(__ireg + 0, __v) \
129 imsic_swap_switchcase_32(__ireg + 32, __v)
130
131 static unsigned long imsic_eix_swap(int ireg, unsigned long val)
132 {
133 switch (ireg) {
134 imsic_swap_switchcase_64(IMSIC_EIP0, val)
135 imsic_swap_switchcase_64(IMSIC_EIE0, val)
136 }
137
138 return 0;
139 }
140
141 #define imsic_vs_csr_write(__c, __v) \
142 do { \
143 csr_write(CSR_VSISELECT, __c); \
144 csr_write(CSR_VSIREG, __v); \
145 } while (0)
146
147 #define imsic_write_switchcase(__ireg, __v) \
148 case __ireg: \
149 imsic_vs_csr_write(__ireg, __v); \
150 break;
151 #define imsic_write_switchcase_2(__ireg, __v) \
152 imsic_write_switchcase(__ireg + 0, __v) \
153 imsic_write_switchcase(__ireg + 1, __v)
154 #define imsic_write_switchcase_4(__ireg, __v) \
155 imsic_write_switchcase_2(__ireg + 0, __v) \
156 imsic_write_switchcase_2(__ireg + 2, __v)
157 #define imsic_write_switchcase_8(__ireg, __v) \
158 imsic_write_switchcase_4(__ireg + 0, __v) \
159 imsic_write_switchcase_4(__ireg + 4, __v)
160 #define imsic_write_switchcase_16(__ireg, __v) \
161 imsic_write_switchcase_8(__ireg + 0, __v) \
162 imsic_write_switchcase_8(__ireg + 8, __v)
163 #define imsic_write_switchcase_32(__ireg, __v) \
164 imsic_write_switchcase_16(__ireg + 0, __v) \
165 imsic_write_switchcase_16(__ireg + 16, __v)
166 #define imsic_write_switchcase_64(__ireg, __v) \
167 imsic_write_switchcase_32(__ireg + 0, __v) \
168 imsic_write_switchcase_32(__ireg + 32, __v)
169
170 static void imsic_eix_write(int ireg, unsigned long val)
171 {
172 switch (ireg) {
173 imsic_write_switchcase_64(IMSIC_EIP0, val)
174 imsic_write_switchcase_64(IMSIC_EIE0, val)
175 }
176 }
177
178 #define imsic_vs_csr_set(__c, __v) \
179 do { \
180 csr_write(CSR_VSISELECT, __c); \
181 csr_set(CSR_VSIREG, __v); \
182 } while (0)
183
184 #define imsic_set_switchcase(__ireg, __v) \
185 case __ireg: \
186 imsic_vs_csr_set(__ireg, __v); \
187 break;
188 #define imsic_set_switchcase_2(__ireg, __v) \
189 imsic_set_switchcase(__ireg + 0, __v) \
190 imsic_set_switchcase(__ireg + 1, __v)
191 #define imsic_set_switchcase_4(__ireg, __v) \
192 imsic_set_switchcase_2(__ireg + 0, __v) \
193 imsic_set_switchcase_2(__ireg + 2, __v)
194 #define imsic_set_switchcase_8(__ireg, __v) \
195 imsic_set_switchcase_4(__ireg + 0, __v) \
196 imsic_set_switchcase_4(__ireg + 4, __v)
197 #define imsic_set_switchcase_16(__ireg, __v) \
198 imsic_set_switchcase_8(__ireg + 0, __v) \
199 imsic_set_switchcase_8(__ireg + 8, __v)
200 #define imsic_set_switchcase_32(__ireg, __v) \
201 imsic_set_switchcase_16(__ireg + 0, __v) \
202 imsic_set_switchcase_16(__ireg + 16, __v)
203 #define imsic_set_switchcase_64(__ireg, __v) \
204 imsic_set_switchcase_32(__ireg + 0, __v) \
205 imsic_set_switchcase_32(__ireg + 32, __v)
206
207 static void imsic_eix_set(int ireg, unsigned long val)
208 {
209 switch (ireg) {
210 imsic_set_switchcase_64(IMSIC_EIP0, val)
211 imsic_set_switchcase_64(IMSIC_EIE0, val)
212 }
213 }
214
215 static unsigned long imsic_mrif_atomic_rmw(struct imsic_mrif *mrif,
216 unsigned long *ptr,
217 unsigned long new_val,
218 unsigned long wr_mask)
219 {
220 unsigned long old_val = 0, tmp = 0;
221
222 __asm__ __volatile__ (
223 "0: lr.w.aq %1, %0\n"
224 " and %2, %1, %3\n"
225 " or %2, %2, %4\n"
226 " sc.w.rl %2, %2, %0\n"
227 " bnez %2, 0b"
228 : "+A" (*ptr), "+r" (old_val), "+r" (tmp)
229 : "r" (~wr_mask), "r" (new_val & wr_mask)
230 : "memory");
231
232 return old_val;
233 }
234
235 static unsigned long imsic_mrif_atomic_or(struct imsic_mrif *mrif,
236 unsigned long *ptr,
237 unsigned long val)
238 {
239 return atomic_long_fetch_or(val, (atomic_long_t *)ptr);
240 }
241
242 #define imsic_mrif_atomic_write(__mrif, __ptr, __new_val) \
243 imsic_mrif_atomic_rmw(__mrif, __ptr, __new_val, -1UL)
244 #define imsic_mrif_atomic_read(__mrif, __ptr) \
245 imsic_mrif_atomic_or(__mrif, __ptr, 0)
246
247 static u32 imsic_mrif_topei(struct imsic_mrif *mrif, u32 nr_eix, u32 nr_msis)
248 {
249 struct imsic_mrif_eix *eix;
250 u32 i, imin, imax, ei, max_msi;
251 unsigned long eipend[BITS_PER_TYPE(u64) / BITS_PER_LONG];
252 unsigned long eithreshold = imsic_mrif_atomic_read(mrif,
253 &mrif->eithreshold);
254
255 max_msi = (eithreshold && (eithreshold <= nr_msis)) ?
256 eithreshold : nr_msis;
257 for (ei = 0; ei < nr_eix; ei++) {
258 eix = &mrif->eix[ei];
259 eipend[0] = imsic_mrif_atomic_read(mrif, &eix->eie[0]) &
260 imsic_mrif_atomic_read(mrif, &eix->eip[0]);
261 #ifdef CONFIG_32BIT
262 eipend[1] = imsic_mrif_atomic_read(mrif, &eix->eie[1]) &
263 imsic_mrif_atomic_read(mrif, &eix->eip[1]);
264 if (!eipend[0] && !eipend[1])
265 #else
266 if (!eipend[0])
267 #endif
268 continue;
269
270 imin = ei * BITS_PER_TYPE(u64);
271 imax = ((imin + BITS_PER_TYPE(u64)) < max_msi) ?
272 imin + BITS_PER_TYPE(u64) : max_msi;
273 for (i = (!imin) ? 1 : imin; i < imax; i++) {
274 if (test_bit(i - imin, eipend))
275 return (i << TOPEI_ID_SHIFT) | i;
276 }
277 }
278
279 return 0;
280 }
281
282 static int imsic_mrif_isel_check(u32 nr_eix, unsigned long isel)
283 {
284 u32 num = 0;
285
286 switch (isel) {
287 case IMSIC_EIDELIVERY:
288 case IMSIC_EITHRESHOLD:
289 break;
290 case IMSIC_EIP0 ... IMSIC_EIP63:
291 num = isel - IMSIC_EIP0;
292 break;
293 case IMSIC_EIE0 ... IMSIC_EIE63:
294 num = isel - IMSIC_EIE0;
295 break;
296 default:
297 return -ENOENT;
298 }
299 #ifndef CONFIG_32BIT
300 if (num & 0x1)
301 return -EINVAL;
302 #endif
303 if ((num / 2) >= nr_eix)
304 return -EINVAL;
305
306 return 0;
307 }
308
309 static int imsic_mrif_rmw(struct imsic_mrif *mrif, u32 nr_eix,
310 unsigned long isel, unsigned long *val,
311 unsigned long new_val, unsigned long wr_mask)
312 {
313 bool pend;
314 struct imsic_mrif_eix *eix;
315 unsigned long *ei, num, old_val = 0;
316
317 switch (isel) {
318 case IMSIC_EIDELIVERY:
319 old_val = imsic_mrif_atomic_rmw(mrif, &mrif->eidelivery,
320 new_val, wr_mask & 0x1);
321 break;
322 case IMSIC_EITHRESHOLD:
323 old_val = imsic_mrif_atomic_rmw(mrif, &mrif->eithreshold,
324 new_val, wr_mask & (IMSIC_MAX_ID - 1));
325 break;
326 case IMSIC_EIP0 ... IMSIC_EIP63:
327 case IMSIC_EIE0 ... IMSIC_EIE63:
328 if (isel >= IMSIC_EIP0 && isel <= IMSIC_EIP63) {
329 pend = true;
330 num = isel - IMSIC_EIP0;
331 } else {
332 pend = false;
333 num = isel - IMSIC_EIE0;
334 }
335
336 if ((num / 2) >= nr_eix)
337 return -EINVAL;
338 eix = &mrif->eix[num / 2];
339
340 #ifndef CONFIG_32BIT
341 if (num & 0x1)
342 return -EINVAL;
343 ei = (pend) ? &eix->eip[0] : &eix->eie[0];
344 #else
345 ei = (pend) ? &eix->eip[num & 0x1] : &eix->eie[num & 0x1];
346 #endif
347
348 /* Bit0 of EIP0 or EIE0 is read-only */
349 if (!num)
350 wr_mask &= ~BIT(0);
351
352 old_val = imsic_mrif_atomic_rmw(mrif, ei, new_val, wr_mask);
353 break;
354 default:
355 return -ENOENT;
356 }
357
358 if (val)
359 *val = old_val;
360
361 return 0;
362 }
363
364 struct imsic_vsfile_read_data {
365 int hgei;
366 u32 nr_eix;
367 bool clear;
368 struct imsic_mrif *mrif;
369 };
370
371 static void imsic_vsfile_local_read(void *data)
372 {
373 u32 i;
374 struct imsic_mrif_eix *eix;
375 struct imsic_vsfile_read_data *idata = data;
376 struct imsic_mrif *mrif = idata->mrif;
377 unsigned long new_hstatus, old_hstatus, old_vsiselect;
378
379 old_vsiselect = csr_read(CSR_VSISELECT);
380 old_hstatus = csr_read(CSR_HSTATUS);
381 new_hstatus = old_hstatus & ~HSTATUS_VGEIN;
382 new_hstatus |= ((unsigned long)idata->hgei) << HSTATUS_VGEIN_SHIFT;
383 csr_write(CSR_HSTATUS, new_hstatus);
384
385 /*
386 * We don't use imsic_mrif_atomic_xyz() functions to store
387 * values in MRIF because imsic_vsfile_read() is always called
388 * with pointer to temporary MRIF on stack.
389 */
390
391 if (idata->clear) {
392 mrif->eidelivery = imsic_vs_csr_swap(IMSIC_EIDELIVERY, 0);
393 mrif->eithreshold = imsic_vs_csr_swap(IMSIC_EITHRESHOLD, 0);
394 for (i = 0; i < idata->nr_eix; i++) {
395 eix = &mrif->eix[i];
396 eix->eip[0] = imsic_eix_swap(IMSIC_EIP0 + i * 2, 0);
397 eix->eie[0] = imsic_eix_swap(IMSIC_EIE0 + i * 2, 0);
398 #ifdef CONFIG_32BIT
399 eix->eip[1] = imsic_eix_swap(IMSIC_EIP0 + i * 2 + 1, 0);
400 eix->eie[1] = imsic_eix_swap(IMSIC_EIE0 + i * 2 + 1, 0);
401 #endif
402 }
403 } else {
404 mrif->eidelivery = imsic_vs_csr_read(IMSIC_EIDELIVERY);
405 mrif->eithreshold = imsic_vs_csr_read(IMSIC_EITHRESHOLD);
406 for (i = 0; i < idata->nr_eix; i++) {
407 eix = &mrif->eix[i];
408 eix->eip[0] = imsic_eix_read(IMSIC_EIP0 + i * 2);
409 eix->eie[0] = imsic_eix_read(IMSIC_EIE0 + i * 2);
410 #ifdef CONFIG_32BIT
411 eix->eip[1] = imsic_eix_read(IMSIC_EIP0 + i * 2 + 1);
412 eix->eie[1] = imsic_eix_read(IMSIC_EIE0 + i * 2 + 1);
413 #endif
414 }
415 }
416
417 csr_write(CSR_HSTATUS, old_hstatus);
418 csr_write(CSR_VSISELECT, old_vsiselect);
419 }
420
421 static void imsic_vsfile_read(int vsfile_hgei, int vsfile_cpu, u32 nr_eix,
422 bool clear, struct imsic_mrif *mrif)
423 {
424 struct imsic_vsfile_read_data idata;
425
426 /* We can only read clear if we have a IMSIC VS-file */
427 if (vsfile_cpu < 0 || vsfile_hgei <= 0)
428 return;
429
430 /* We can only read clear on local CPU */
431 idata.hgei = vsfile_hgei;
432 idata.nr_eix = nr_eix;
433 idata.clear = clear;
434 idata.mrif = mrif;
435 on_each_cpu_mask(cpumask_of(vsfile_cpu),
436 imsic_vsfile_local_read, &idata, 1);
437 }
438
439 struct imsic_vsfile_rw_data {
440 int hgei;
441 int isel;
442 bool write;
443 unsigned long val;
444 };
445
446 static void imsic_vsfile_local_rw(void *data)
447 {
448 struct imsic_vsfile_rw_data *idata = data;
449 unsigned long new_hstatus, old_hstatus, old_vsiselect;
450
451 old_vsiselect = csr_read(CSR_VSISELECT);
452 old_hstatus = csr_read(CSR_HSTATUS);
453 new_hstatus = old_hstatus & ~HSTATUS_VGEIN;
454 new_hstatus |= ((unsigned long)idata->hgei) << HSTATUS_VGEIN_SHIFT;
455 csr_write(CSR_HSTATUS, new_hstatus);
456
457 switch (idata->isel) {
458 case IMSIC_EIDELIVERY:
459 if (idata->write)
460 imsic_vs_csr_write(IMSIC_EIDELIVERY, idata->val);
461 else
462 idata->val = imsic_vs_csr_read(IMSIC_EIDELIVERY);
463 break;
464 case IMSIC_EITHRESHOLD:
465 if (idata->write)
466 imsic_vs_csr_write(IMSIC_EITHRESHOLD, idata->val);
467 else
468 idata->val = imsic_vs_csr_read(IMSIC_EITHRESHOLD);
469 break;
470 case IMSIC_EIP0 ... IMSIC_EIP63:
471 case IMSIC_EIE0 ... IMSIC_EIE63:
472 #ifndef CONFIG_32BIT
473 if (idata->isel & 0x1)
474 break;
475 #endif
476 if (idata->write)
477 imsic_eix_write(idata->isel, idata->val);
478 else
479 idata->val = imsic_eix_read(idata->isel);
480 break;
481 default:
482 break;
483 }
484
485 csr_write(CSR_HSTATUS, old_hstatus);
486 csr_write(CSR_VSISELECT, old_vsiselect);
487 }
488
489 static int imsic_vsfile_rw(int vsfile_hgei, int vsfile_cpu, u32 nr_eix,
490 unsigned long isel, bool write,
491 unsigned long *val)
492 {
493 int rc;
494 struct imsic_vsfile_rw_data rdata;
495
496 /* We can only access register if we have a IMSIC VS-file */
497 if (vsfile_cpu < 0 || vsfile_hgei <= 0)
498 return -EINVAL;
499
500 /* Check IMSIC register iselect */
501 rc = imsic_mrif_isel_check(nr_eix, isel);
502 if (rc)
503 return rc;
504
505 /* We can only access register on local CPU */
506 rdata.hgei = vsfile_hgei;
507 rdata.isel = isel;
508 rdata.write = write;
509 rdata.val = (write) ? *val : 0;
510 on_each_cpu_mask(cpumask_of(vsfile_cpu),
511 imsic_vsfile_local_rw, &rdata, 1);
512
513 if (!write)
514 *val = rdata.val;
515
516 return 0;
517 }
518
519 static void imsic_vsfile_local_clear(int vsfile_hgei, u32 nr_eix)
520 {
521 u32 i;
522 unsigned long new_hstatus, old_hstatus, old_vsiselect;
523
524 /* We can only zero-out if we have a IMSIC VS-file */
525 if (vsfile_hgei <= 0)
526 return;
527
528 old_vsiselect = csr_read(CSR_VSISELECT);
529 old_hstatus = csr_read(CSR_HSTATUS);
530 new_hstatus = old_hstatus & ~HSTATUS_VGEIN;
531 new_hstatus |= ((unsigned long)vsfile_hgei) << HSTATUS_VGEIN_SHIFT;
532 csr_write(CSR_HSTATUS, new_hstatus);
533
534 imsic_vs_csr_write(IMSIC_EIDELIVERY, 0);
535 imsic_vs_csr_write(IMSIC_EITHRESHOLD, 0);
536 for (i = 0; i < nr_eix; i++) {
537 imsic_eix_write(IMSIC_EIP0 + i * 2, 0);
538 imsic_eix_write(IMSIC_EIE0 + i * 2, 0);
539 #ifdef CONFIG_32BIT
540 imsic_eix_write(IMSIC_EIP0 + i * 2 + 1, 0);
541 imsic_eix_write(IMSIC_EIE0 + i * 2 + 1, 0);
542 #endif
543 }
544
545 csr_write(CSR_HSTATUS, old_hstatus);
546 csr_write(CSR_VSISELECT, old_vsiselect);
547 }
548
549 static void imsic_vsfile_local_update(int vsfile_hgei, u32 nr_eix,
550 struct imsic_mrif *mrif)
551 {
552 u32 i;
553 struct imsic_mrif_eix *eix;
554 unsigned long new_hstatus, old_hstatus, old_vsiselect;
555
556 /* We can only update if we have a HW IMSIC context */
557 if (vsfile_hgei <= 0)
558 return;
559
560 /*
561 * We don't use imsic_mrif_atomic_xyz() functions to read values
562 * from MRIF in this function because it is always called with
563 * pointer to temporary MRIF on stack.
564 */
565
566 old_vsiselect = csr_read(CSR_VSISELECT);
567 old_hstatus = csr_read(CSR_HSTATUS);
568 new_hstatus = old_hstatus & ~HSTATUS_VGEIN;
569 new_hstatus |= ((unsigned long)vsfile_hgei) << HSTATUS_VGEIN_SHIFT;
570 csr_write(CSR_HSTATUS, new_hstatus);
571
572 for (i = 0; i < nr_eix; i++) {
573 eix = &mrif->eix[i];
574 imsic_eix_set(IMSIC_EIP0 + i * 2, eix->eip[0]);
575 imsic_eix_set(IMSIC_EIE0 + i * 2, eix->eie[0]);
576 #ifdef CONFIG_32BIT
577 imsic_eix_set(IMSIC_EIP0 + i * 2 + 1, eix->eip[1]);
578 imsic_eix_set(IMSIC_EIE0 + i * 2 + 1, eix->eie[1]);
579 #endif
580 }
581 imsic_vs_csr_write(IMSIC_EITHRESHOLD, mrif->eithreshold);
582 imsic_vs_csr_write(IMSIC_EIDELIVERY, mrif->eidelivery);
583
584 csr_write(CSR_HSTATUS, old_hstatus);
585 csr_write(CSR_VSISELECT, old_vsiselect);
586 }
587
588 static void imsic_vsfile_cleanup(struct imsic *imsic)
589 {
590 int old_vsfile_hgei, old_vsfile_cpu;
591 unsigned long flags;
592
593 /*
594 * We don't use imsic_mrif_atomic_xyz() functions to clear the
595 * SW-file in this function because it is always called when the
596 * VCPU is being destroyed.
597 */
598
599 write_lock_irqsave(&imsic->vsfile_lock, flags);
600 old_vsfile_hgei = imsic->vsfile_hgei;
601 old_vsfile_cpu = imsic->vsfile_cpu;
602 imsic->vsfile_cpu = imsic->vsfile_hgei = -1;
603 imsic->vsfile_va = NULL;
604 imsic->vsfile_pa = 0;
605 write_unlock_irqrestore(&imsic->vsfile_lock, flags);
606
607 memset(imsic->swfile, 0, sizeof(*imsic->swfile));
608
609 if (old_vsfile_cpu >= 0)
610 kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei);
611 }
612
613 static void imsic_swfile_extirq_update(struct kvm_vcpu *vcpu)
614 {
615 struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
616 struct imsic_mrif *mrif = imsic->swfile;
617 unsigned long flags;
618
619 /*
620 * The critical section is necessary during external interrupt
621 * updates to avoid the risk of losing interrupts due to potential
622 * interruptions between reading topei and updating pending status.
623 */
624
625 spin_lock_irqsave(&imsic->swfile_extirq_lock, flags);
626
627 if (imsic_mrif_atomic_read(mrif, &mrif->eidelivery) &&
628 imsic_mrif_topei(mrif, imsic->nr_eix, imsic->nr_msis))
629 kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT);
630 else
631 kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT);
632
633 spin_unlock_irqrestore(&imsic->swfile_extirq_lock, flags);
634 }
635
636 static void imsic_swfile_read(struct kvm_vcpu *vcpu, bool clear,
637 struct imsic_mrif *mrif)
638 {
639 struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
640
641 /*
642 * We don't use imsic_mrif_atomic_xyz() functions to read and
643 * write SW-file and MRIF in this function because it is always
644 * called when VCPU is not using SW-file and the MRIF points to
645 * a temporary MRIF on stack.
646 */
647
648 memcpy(mrif, imsic->swfile, sizeof(*mrif));
649 if (clear) {
650 memset(imsic->swfile, 0, sizeof(*imsic->swfile));
651 kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT);
652 }
653 }
654
655 static void imsic_swfile_update(struct kvm_vcpu *vcpu,
656 struct imsic_mrif *mrif)
657 {
658 u32 i;
659 struct imsic_mrif_eix *seix, *eix;
660 struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
661 struct imsic_mrif *smrif = imsic->swfile;
662
663 imsic_mrif_atomic_write(smrif, &smrif->eidelivery, mrif->eidelivery);
664 imsic_mrif_atomic_write(smrif, &smrif->eithreshold, mrif->eithreshold);
665 for (i = 0; i < imsic->nr_eix; i++) {
666 seix = &smrif->eix[i];
667 eix = &mrif->eix[i];
668 imsic_mrif_atomic_or(smrif, &seix->eip[0], eix->eip[0]);
669 imsic_mrif_atomic_or(smrif, &seix->eie[0], eix->eie[0]);
670 #ifdef CONFIG_32BIT
671 imsic_mrif_atomic_or(smrif, &seix->eip[1], eix->eip[1]);
672 imsic_mrif_atomic_or(smrif, &seix->eie[1], eix->eie[1]);
673 #endif
674 }
675
676 imsic_swfile_extirq_update(vcpu);
677 }
678
679 void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu)
680 {
681 unsigned long flags;
682 struct imsic_mrif tmrif;
683 int old_vsfile_hgei, old_vsfile_cpu;
684 struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
685
686 /* Read and clear IMSIC VS-file details */
687 write_lock_irqsave(&imsic->vsfile_lock, flags);
688 old_vsfile_hgei = imsic->vsfile_hgei;
689 old_vsfile_cpu = imsic->vsfile_cpu;
690 imsic->vsfile_cpu = imsic->vsfile_hgei = -1;
691 imsic->vsfile_va = NULL;
692 imsic->vsfile_pa = 0;
693 write_unlock_irqrestore(&imsic->vsfile_lock, flags);
694
695 /* Do nothing, if no IMSIC VS-file to release */
696 if (old_vsfile_cpu < 0)
697 return;
698
699 /*
700 * At this point, all interrupt producers are still using
701 * the old IMSIC VS-file so we first re-direct all interrupt
702 * producers.
703 */
704
705 /* Purge the G-stage mapping */
706 kvm_riscv_gstage_iounmap(vcpu->kvm,
707 vcpu->arch.aia_context.imsic_addr,
708 IMSIC_MMIO_PAGE_SZ);
709
710 /* TODO: Purge the IOMMU mapping ??? */
711
712 /*
713 * At this point, all interrupt producers have been re-directed
714 * to somewhere else so we move register state from the old IMSIC
715 * VS-file to the IMSIC SW-file.
716 */
717
718 /* Read and clear register state from old IMSIC VS-file */
719 memset(&tmrif, 0, sizeof(tmrif));
720 imsic_vsfile_read(old_vsfile_hgei, old_vsfile_cpu, imsic->nr_hw_eix,
721 true, &tmrif);
722
723 /* Update register state in IMSIC SW-file */
724 imsic_swfile_update(vcpu, &tmrif);
725
726 /* Free-up old IMSIC VS-file */
727 kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei);
728 }
729
730 int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu)
731 {
732 unsigned long flags;
733 phys_addr_t new_vsfile_pa;
734 struct imsic_mrif tmrif;
735 void __iomem *new_vsfile_va;
736 struct kvm *kvm = vcpu->kvm;
737 struct kvm_run *run = vcpu->run;
738 struct kvm_vcpu_aia *vaia = &vcpu->arch.aia_context;
739 struct imsic *imsic = vaia->imsic_state;
740 int ret = 0, new_vsfile_hgei = -1, old_vsfile_hgei, old_vsfile_cpu;
741
742 /* Do nothing for emulation mode */
743 if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_EMUL)
744 return 1;
745
746 /* Read old IMSIC VS-file details */
747 read_lock_irqsave(&imsic->vsfile_lock, flags);
748 old_vsfile_hgei = imsic->vsfile_hgei;
749 old_vsfile_cpu = imsic->vsfile_cpu;
750 read_unlock_irqrestore(&imsic->vsfile_lock, flags);
751
752 /* Do nothing if we are continuing on same CPU */
753 if (old_vsfile_cpu == vcpu->cpu)
754 return 1;
755
756 /* Allocate new IMSIC VS-file */
757 ret = kvm_riscv_aia_alloc_hgei(vcpu->cpu, vcpu,
758 &new_vsfile_va, &new_vsfile_pa);
759 if (ret <= 0) {
760 /* For HW acceleration mode, we can't continue */
761 if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_HWACCEL) {
762 run->fail_entry.hardware_entry_failure_reason =
763 CSR_HSTATUS;
764 run->fail_entry.cpu = vcpu->cpu;
765 run->exit_reason = KVM_EXIT_FAIL_ENTRY;
766 return 0;
767 }
768
769 /* Release old IMSIC VS-file */
770 if (old_vsfile_cpu >= 0)
771 kvm_riscv_vcpu_aia_imsic_release(vcpu);
772
773 /* For automatic mode, we continue */
774 goto done;
775 }
776 new_vsfile_hgei = ret;
777
778 /*
779 * At this point, all interrupt producers are still using
780 * to the old IMSIC VS-file so we first move all interrupt
781 * producers to the new IMSIC VS-file.
782 */
783
784 /* Zero-out new IMSIC VS-file */
785 imsic_vsfile_local_clear(new_vsfile_hgei, imsic->nr_hw_eix);
786
787 /* Update G-stage mapping for the new IMSIC VS-file */
788 ret = kvm_riscv_gstage_ioremap(kvm, vcpu->arch.aia_context.imsic_addr,
789 new_vsfile_pa, IMSIC_MMIO_PAGE_SZ,
790 true, true);
791 if (ret)
792 goto fail_free_vsfile_hgei;
793
794 /* TODO: Update the IOMMU mapping ??? */
795
796 /* Update new IMSIC VS-file details in IMSIC context */
797 write_lock_irqsave(&imsic->vsfile_lock, flags);
798 imsic->vsfile_hgei = new_vsfile_hgei;
799 imsic->vsfile_cpu = vcpu->cpu;
800 imsic->vsfile_va = new_vsfile_va;
801 imsic->vsfile_pa = new_vsfile_pa;
802 write_unlock_irqrestore(&imsic->vsfile_lock, flags);
803
804 /*
805 * At this point, all interrupt producers have been moved
806 * to the new IMSIC VS-file so we move register state from
807 * the old IMSIC VS/SW-file to the new IMSIC VS-file.
808 */
809
810 memset(&tmrif, 0, sizeof(tmrif));
811 if (old_vsfile_cpu >= 0) {
812 /* Read and clear register state from old IMSIC VS-file */
813 imsic_vsfile_read(old_vsfile_hgei, old_vsfile_cpu,
814 imsic->nr_hw_eix, true, &tmrif);
815
816 /* Free-up old IMSIC VS-file */
817 kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei);
818 } else {
819 /* Read and clear register state from IMSIC SW-file */
820 imsic_swfile_read(vcpu, true, &tmrif);
821 }
822
823 /* Restore register state in the new IMSIC VS-file */
824 imsic_vsfile_local_update(new_vsfile_hgei, imsic->nr_hw_eix, &tmrif);
825
826 done:
827 /* Set VCPU HSTATUS.VGEIN to new IMSIC VS-file */
828 vcpu->arch.guest_context.hstatus &= ~HSTATUS_VGEIN;
829 if (new_vsfile_hgei > 0)
830 vcpu->arch.guest_context.hstatus |=
831 ((unsigned long)new_vsfile_hgei) << HSTATUS_VGEIN_SHIFT;
832
833 /* Continue run-loop */
834 return 1;
835
836 fail_free_vsfile_hgei:
837 kvm_riscv_aia_free_hgei(vcpu->cpu, new_vsfile_hgei);
838 return ret;
839 }
840
841 int kvm_riscv_vcpu_aia_imsic_rmw(struct kvm_vcpu *vcpu, unsigned long isel,
842 unsigned long *val, unsigned long new_val,
843 unsigned long wr_mask)
844 {
845 u32 topei;
846 struct imsic_mrif_eix *eix;
847 int r, rc = KVM_INSN_CONTINUE_NEXT_SEPC;
848 struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
849
850 if (isel == KVM_RISCV_AIA_IMSIC_TOPEI) {
851 /* Read pending and enabled interrupt with highest priority */
852 topei = imsic_mrif_topei(imsic->swfile, imsic->nr_eix,
853 imsic->nr_msis);
854 if (val)
855 *val = topei;
856
857 /* Writes ignore value and clear top pending interrupt */
858 if (topei && wr_mask) {
859 topei >>= TOPEI_ID_SHIFT;
860 if (topei) {
861 eix = &imsic->swfile->eix[topei /
862 BITS_PER_TYPE(u64)];
863 clear_bit(topei & (BITS_PER_TYPE(u64) - 1),
864 eix->eip);
865 }
866 }
867 } else {
868 r = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix, isel,
869 val, new_val, wr_mask);
870 /* Forward unknown IMSIC register to user-space */
871 if (r)
872 rc = (r == -ENOENT) ? 0 : KVM_INSN_ILLEGAL_TRAP;
873 }
874
875 if (wr_mask)
876 imsic_swfile_extirq_update(vcpu);
877
878 return rc;
879 }
880
881 int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type,
882 bool write, unsigned long *val)
883 {
884 u32 isel, vcpu_id;
885 unsigned long flags;
886 struct imsic *imsic;
887 struct kvm_vcpu *vcpu;
888 int rc, vsfile_hgei, vsfile_cpu;
889
890 if (!kvm_riscv_aia_initialized(kvm))
891 return -ENODEV;
892
893 vcpu_id = KVM_DEV_RISCV_AIA_IMSIC_GET_VCPU(type);
894 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
895 if (!vcpu)
896 return -ENODEV;
897
898 isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type);
899 imsic = vcpu->arch.aia_context.imsic_state;
900
901 read_lock_irqsave(&imsic->vsfile_lock, flags);
902
903 rc = 0;
904 vsfile_hgei = imsic->vsfile_hgei;
905 vsfile_cpu = imsic->vsfile_cpu;
906 if (vsfile_cpu < 0) {
907 if (write) {
908 rc = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix,
909 isel, NULL, *val, -1UL);
910 imsic_swfile_extirq_update(vcpu);
911 } else
912 rc = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix,
913 isel, val, 0, 0);
914 }
915
916 read_unlock_irqrestore(&imsic->vsfile_lock, flags);
917
918 if (!rc && vsfile_cpu >= 0)
919 rc = imsic_vsfile_rw(vsfile_hgei, vsfile_cpu, imsic->nr_eix,
920 isel, write, val);
921
922 return rc;
923 }
924
925 int kvm_riscv_aia_imsic_has_attr(struct kvm *kvm, unsigned long type)
926 {
927 u32 isel, vcpu_id;
928 struct imsic *imsic;
929 struct kvm_vcpu *vcpu;
930
931 if (!kvm_riscv_aia_initialized(kvm))
932 return -ENODEV;
933
934 vcpu_id = KVM_DEV_RISCV_AIA_IMSIC_GET_VCPU(type);
935 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
936 if (!vcpu)
937 return -ENODEV;
938
939 isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type);
940 imsic = vcpu->arch.aia_context.imsic_state;
941 return imsic_mrif_isel_check(imsic->nr_eix, isel);
942 }
943
944 void kvm_riscv_vcpu_aia_imsic_reset(struct kvm_vcpu *vcpu)
945 {
946 struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
947
948 if (!imsic)
949 return;
950
951 kvm_riscv_vcpu_aia_imsic_release(vcpu);
952
953 memset(imsic->swfile, 0, sizeof(*imsic->swfile));
954 }
955
956 int kvm_riscv_vcpu_aia_imsic_inject(struct kvm_vcpu *vcpu,
957 u32 guest_index, u32 offset, u32 iid)
958 {
959 unsigned long flags;
960 struct imsic_mrif_eix *eix;
961 struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
962
963 /* We only emulate one IMSIC MMIO page for each Guest VCPU */
964 if (!imsic || !iid || guest_index ||
965 (offset != IMSIC_MMIO_SETIPNUM_LE &&
966 offset != IMSIC_MMIO_SETIPNUM_BE))
967 return -ENODEV;
968
969 iid = (offset == IMSIC_MMIO_SETIPNUM_BE) ? __swab32(iid) : iid;
970 if (imsic->nr_msis <= iid)
971 return -EINVAL;
972
973 read_lock_irqsave(&imsic->vsfile_lock, flags);
974
975 if (imsic->vsfile_cpu >= 0) {
976 writel(iid, imsic->vsfile_va + IMSIC_MMIO_SETIPNUM_LE);
977 kvm_vcpu_kick(vcpu);
978 } else {
979 eix = &imsic->swfile->eix[iid / BITS_PER_TYPE(u64)];
980 set_bit(iid & (BITS_PER_TYPE(u64) - 1), eix->eip);
981 imsic_swfile_extirq_update(vcpu);
982 }
983
984 read_unlock_irqrestore(&imsic->vsfile_lock, flags);
985
986 return 0;
987 }
988
989 static int imsic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
990 gpa_t addr, int len, void *val)
991 {
992 if (len != 4 || (addr & 0x3) != 0)
993 return -EOPNOTSUPP;
994
995 *((u32 *)val) = 0;
996
997 return 0;
998 }
999
1000 static int imsic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
1001 gpa_t addr, int len, const void *val)
1002 {
1003 struct kvm_msi msi = { 0 };
1004
1005 if (len != 4 || (addr & 0x3) != 0)
1006 return -EOPNOTSUPP;
1007
1008 msi.address_hi = addr >> 32;
1009 msi.address_lo = (u32)addr;
1010 msi.data = *((const u32 *)val);
1011 kvm_riscv_aia_inject_msi(vcpu->kvm, &msi);
1012
1013 return 0;
1014 };
1015
1016 static struct kvm_io_device_ops imsic_iodoev_ops = {
1017 .read = imsic_mmio_read,
1018 .write = imsic_mmio_write,
1019 };
1020
1021 int kvm_riscv_vcpu_aia_imsic_init(struct kvm_vcpu *vcpu)
1022 {
1023 int ret = 0;
1024 struct imsic *imsic;
1025 struct page *swfile_page;
1026 struct kvm *kvm = vcpu->kvm;
1027
1028 /* Fail if we have zero IDs */
1029 if (!kvm->arch.aia.nr_ids)
1030 return -EINVAL;
1031
1032 /* Allocate IMSIC context */
1033 imsic = kzalloc(sizeof(*imsic), GFP_KERNEL);
1034 if (!imsic)
1035 return -ENOMEM;
1036 vcpu->arch.aia_context.imsic_state = imsic;
1037
1038 /* Setup IMSIC context */
1039 imsic->nr_msis = kvm->arch.aia.nr_ids + 1;
1040 rwlock_init(&imsic->vsfile_lock);
1041 imsic->nr_eix = BITS_TO_U64(imsic->nr_msis);
1042 imsic->nr_hw_eix = BITS_TO_U64(kvm_riscv_aia_max_ids);
1043 imsic->vsfile_hgei = imsic->vsfile_cpu = -1;
1044
1045 /* Setup IMSIC SW-file */
1046 swfile_page = alloc_pages(GFP_KERNEL | __GFP_ZERO,
1047 get_order(sizeof(*imsic->swfile)));
1048 if (!swfile_page) {
1049 ret = -ENOMEM;
1050 goto fail_free_imsic;
1051 }
1052 imsic->swfile = page_to_virt(swfile_page);
1053 imsic->swfile_pa = page_to_phys(swfile_page);
1054 spin_lock_init(&imsic->swfile_extirq_lock);
1055
1056 /* Setup IO device */
1057 kvm_iodevice_init(&imsic->iodev, &imsic_iodoev_ops);
1058 mutex_lock(&kvm->slots_lock);
1059 ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS,
1060 vcpu->arch.aia_context.imsic_addr,
1061 KVM_DEV_RISCV_IMSIC_SIZE,
1062 &imsic->iodev);
1063 mutex_unlock(&kvm->slots_lock);
1064 if (ret)
1065 goto fail_free_swfile;
1066
1067 return 0;
1068
1069 fail_free_swfile:
1070 free_pages((unsigned long)imsic->swfile,
1071 get_order(sizeof(*imsic->swfile)));
1072 fail_free_imsic:
1073 vcpu->arch.aia_context.imsic_state = NULL;
1074 kfree(imsic);
1075 return ret;
1076 }
1077
1078 void kvm_riscv_vcpu_aia_imsic_cleanup(struct kvm_vcpu *vcpu)
1079 {
1080 struct kvm *kvm = vcpu->kvm;
1081 struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
1082
1083 if (!imsic)
1084 return;
1085
1086 imsic_vsfile_cleanup(imsic);
1087
1088 mutex_lock(&kvm->slots_lock);
1089 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &imsic->iodev);
1090 mutex_unlock(&kvm->slots_lock);
1091
1092 free_pages((unsigned long)imsic->swfile,
1093 get_order(sizeof(*imsic->swfile)));
1094
1095 vcpu->arch.aia_context.imsic_state = NULL;
1096 kfree(imsic);
1097 }
1098
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.