1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2023 Rivos Inc
4 *
5 * Authors:
6 * Atish Patra <atishp@rivosinc.com>
7 */
8
9 #define pr_fmt(fmt) "riscv-kvm-pmu: " fmt
10 #include <linux/errno.h>
11 #include <linux/err.h>
12 #include <linux/kvm_host.h>
13 #include <linux/perf/riscv_pmu.h>
14 #include <asm/csr.h>
15 #include <asm/kvm_vcpu_sbi.h>
16 #include <asm/kvm_vcpu_pmu.h>
17 #include <asm/sbi.h>
18 #include <linux/bitops.h>
19
20 #define kvm_pmu_num_counters(pmu) ((pmu)->num_hw_ctrs + (pmu)->num_fw_ctrs)
21 #define get_event_type(x) (((x) & SBI_PMU_EVENT_IDX_TYPE_MASK) >> 16)
22 #define get_event_code(x) ((x) & SBI_PMU_EVENT_IDX_CODE_MASK)
23
24 static enum perf_hw_id hw_event_perf_map[SBI_PMU_HW_GENERAL_MAX] = {
25 [SBI_PMU_HW_CPU_CYCLES] = PERF_COUNT_HW_CPU_CYCLES,
26 [SBI_PMU_HW_INSTRUCTIONS] = PERF_COUNT_HW_INSTRUCTIONS,
27 [SBI_PMU_HW_CACHE_REFERENCES] = PERF_COUNT_HW_CACHE_REFERENCES,
28 [SBI_PMU_HW_CACHE_MISSES] = PERF_COUNT_HW_CACHE_MISSES,
29 [SBI_PMU_HW_BRANCH_INSTRUCTIONS] = PERF_COUNT_HW_BRANCH_INSTRUCTIONS,
30 [SBI_PMU_HW_BRANCH_MISSES] = PERF_COUNT_HW_BRANCH_MISSES,
31 [SBI_PMU_HW_BUS_CYCLES] = PERF_COUNT_HW_BUS_CYCLES,
32 [SBI_PMU_HW_STALLED_CYCLES_FRONTEND] = PERF_COUNT_HW_STALLED_CYCLES_FRONTEND,
33 [SBI_PMU_HW_STALLED_CYCLES_BACKEND] = PERF_COUNT_HW_STALLED_CYCLES_BACKEND,
34 [SBI_PMU_HW_REF_CPU_CYCLES] = PERF_COUNT_HW_REF_CPU_CYCLES,
35 };
36
37 static u64 kvm_pmu_get_sample_period(struct kvm_pmc *pmc)
38 {
39 u64 counter_val_mask = GENMASK(pmc->cinfo.width, 0);
40 u64 sample_period;
41
42 if (!pmc->counter_val)
43 sample_period = counter_val_mask;
44 else
45 sample_period = (-pmc->counter_val) & counter_val_mask;
46
47 return sample_period;
48 }
49
50 static u32 kvm_pmu_get_perf_event_type(unsigned long eidx)
51 {
52 enum sbi_pmu_event_type etype = get_event_type(eidx);
53 u32 type = PERF_TYPE_MAX;
54
55 switch (etype) {
56 case SBI_PMU_EVENT_TYPE_HW:
57 type = PERF_TYPE_HARDWARE;
58 break;
59 case SBI_PMU_EVENT_TYPE_CACHE:
60 type = PERF_TYPE_HW_CACHE;
61 break;
62 case SBI_PMU_EVENT_TYPE_RAW:
63 case SBI_PMU_EVENT_TYPE_FW:
64 type = PERF_TYPE_RAW;
65 break;
66 default:
67 break;
68 }
69
70 return type;
71 }
72
73 static bool kvm_pmu_is_fw_event(unsigned long eidx)
74 {
75 return get_event_type(eidx) == SBI_PMU_EVENT_TYPE_FW;
76 }
77
78 static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc)
79 {
80 if (pmc->perf_event) {
81 perf_event_disable(pmc->perf_event);
82 perf_event_release_kernel(pmc->perf_event);
83 pmc->perf_event = NULL;
84 }
85 }
86
87 static u64 kvm_pmu_get_perf_event_hw_config(u32 sbi_event_code)
88 {
89 return hw_event_perf_map[sbi_event_code];
90 }
91
92 static u64 kvm_pmu_get_perf_event_cache_config(u32 sbi_event_code)
93 {
94 u64 config = U64_MAX;
95 unsigned int cache_type, cache_op, cache_result;
96
97 /* All the cache event masks lie within 0xFF. No separate masking is necessary */
98 cache_type = (sbi_event_code & SBI_PMU_EVENT_CACHE_ID_CODE_MASK) >>
99 SBI_PMU_EVENT_CACHE_ID_SHIFT;
100 cache_op = (sbi_event_code & SBI_PMU_EVENT_CACHE_OP_ID_CODE_MASK) >>
101 SBI_PMU_EVENT_CACHE_OP_SHIFT;
102 cache_result = sbi_event_code & SBI_PMU_EVENT_CACHE_RESULT_ID_CODE_MASK;
103
104 if (cache_type >= PERF_COUNT_HW_CACHE_MAX ||
105 cache_op >= PERF_COUNT_HW_CACHE_OP_MAX ||
106 cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
107 return config;
108
109 config = cache_type | (cache_op << 8) | (cache_result << 16);
110
111 return config;
112 }
113
114 static u64 kvm_pmu_get_perf_event_config(unsigned long eidx, uint64_t evt_data)
115 {
116 enum sbi_pmu_event_type etype = get_event_type(eidx);
117 u32 ecode = get_event_code(eidx);
118 u64 config = U64_MAX;
119
120 switch (etype) {
121 case SBI_PMU_EVENT_TYPE_HW:
122 if (ecode < SBI_PMU_HW_GENERAL_MAX)
123 config = kvm_pmu_get_perf_event_hw_config(ecode);
124 break;
125 case SBI_PMU_EVENT_TYPE_CACHE:
126 config = kvm_pmu_get_perf_event_cache_config(ecode);
127 break;
128 case SBI_PMU_EVENT_TYPE_RAW:
129 config = evt_data & RISCV_PMU_RAW_EVENT_MASK;
130 break;
131 case SBI_PMU_EVENT_TYPE_FW:
132 if (ecode < SBI_PMU_FW_MAX)
133 config = (1ULL << 63) | ecode;
134 break;
135 default:
136 break;
137 }
138
139 return config;
140 }
141
142 static int kvm_pmu_get_fixed_pmc_index(unsigned long eidx)
143 {
144 u32 etype = kvm_pmu_get_perf_event_type(eidx);
145 u32 ecode = get_event_code(eidx);
146
147 if (etype != SBI_PMU_EVENT_TYPE_HW)
148 return -EINVAL;
149
150 if (ecode == SBI_PMU_HW_CPU_CYCLES)
151 return 0;
152 else if (ecode == SBI_PMU_HW_INSTRUCTIONS)
153 return 2;
154 else
155 return -EINVAL;
156 }
157
158 static int kvm_pmu_get_programmable_pmc_index(struct kvm_pmu *kvpmu, unsigned long eidx,
159 unsigned long cbase, unsigned long cmask)
160 {
161 int ctr_idx = -1;
162 int i, pmc_idx;
163 int min, max;
164
165 if (kvm_pmu_is_fw_event(eidx)) {
166 /* Firmware counters are mapped 1:1 starting from num_hw_ctrs for simplicity */
167 min = kvpmu->num_hw_ctrs;
168 max = min + kvpmu->num_fw_ctrs;
169 } else {
170 /* First 3 counters are reserved for fixed counters */
171 min = 3;
172 max = kvpmu->num_hw_ctrs;
173 }
174
175 for_each_set_bit(i, &cmask, BITS_PER_LONG) {
176 pmc_idx = i + cbase;
177 if ((pmc_idx >= min && pmc_idx < max) &&
178 !test_bit(pmc_idx, kvpmu->pmc_in_use)) {
179 ctr_idx = pmc_idx;
180 break;
181 }
182 }
183
184 return ctr_idx;
185 }
186
187 static int pmu_get_pmc_index(struct kvm_pmu *pmu, unsigned long eidx,
188 unsigned long cbase, unsigned long cmask)
189 {
190 int ret;
191
192 /* Fixed counters need to be have fixed mapping as they have different width */
193 ret = kvm_pmu_get_fixed_pmc_index(eidx);
194 if (ret >= 0)
195 return ret;
196
197 return kvm_pmu_get_programmable_pmc_index(pmu, eidx, cbase, cmask);
198 }
199
200 static int pmu_fw_ctr_read_hi(struct kvm_vcpu *vcpu, unsigned long cidx,
201 unsigned long *out_val)
202 {
203 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
204 struct kvm_pmc *pmc;
205 int fevent_code;
206
207 if (!IS_ENABLED(CONFIG_32BIT)) {
208 pr_warn("%s: should be invoked for only RV32\n", __func__);
209 return -EINVAL;
210 }
211
212 if (cidx >= kvm_pmu_num_counters(kvpmu) || cidx == 1) {
213 pr_warn("Invalid counter id [%ld]during read\n", cidx);
214 return -EINVAL;
215 }
216
217 pmc = &kvpmu->pmc[cidx];
218
219 if (pmc->cinfo.type != SBI_PMU_CTR_TYPE_FW)
220 return -EINVAL;
221
222 fevent_code = get_event_code(pmc->event_idx);
223 pmc->counter_val = kvpmu->fw_event[fevent_code].value;
224
225 *out_val = pmc->counter_val >> 32;
226
227 return 0;
228 }
229
230 static int pmu_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx,
231 unsigned long *out_val)
232 {
233 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
234 struct kvm_pmc *pmc;
235 u64 enabled, running;
236 int fevent_code;
237
238 if (cidx >= kvm_pmu_num_counters(kvpmu) || cidx == 1) {
239 pr_warn("Invalid counter id [%ld] during read\n", cidx);
240 return -EINVAL;
241 }
242
243 pmc = &kvpmu->pmc[cidx];
244
245 if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) {
246 fevent_code = get_event_code(pmc->event_idx);
247 pmc->counter_val = kvpmu->fw_event[fevent_code].value;
248 } else if (pmc->perf_event) {
249 pmc->counter_val += perf_event_read_value(pmc->perf_event, &enabled, &running);
250 } else {
251 return -EINVAL;
252 }
253 *out_val = pmc->counter_val;
254
255 return 0;
256 }
257
258 static int kvm_pmu_validate_counter_mask(struct kvm_pmu *kvpmu, unsigned long ctr_base,
259 unsigned long ctr_mask)
260 {
261 /* Make sure the we have a valid counter mask requested from the caller */
262 if (!ctr_mask || (ctr_base + __fls(ctr_mask) >= kvm_pmu_num_counters(kvpmu)))
263 return -EINVAL;
264
265 return 0;
266 }
267
268 static void kvm_riscv_pmu_overflow(struct perf_event *perf_event,
269 struct perf_sample_data *data,
270 struct pt_regs *regs)
271 {
272 struct kvm_pmc *pmc = perf_event->overflow_handler_context;
273 struct kvm_vcpu *vcpu = pmc->vcpu;
274 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
275 struct riscv_pmu *rpmu = to_riscv_pmu(perf_event->pmu);
276 u64 period;
277
278 /*
279 * Stop the event counting by directly accessing the perf_event.
280 * Otherwise, this needs to deferred via a workqueue.
281 * That will introduce skew in the counter value because the actual
282 * physical counter would start after returning from this function.
283 * It will be stopped again once the workqueue is scheduled
284 */
285 rpmu->pmu.stop(perf_event, PERF_EF_UPDATE);
286
287 /*
288 * The hw counter would start automatically when this function returns.
289 * Thus, the host may continue to interrupt and inject it to the guest
290 * even without the guest configuring the next event. Depending on the hardware
291 * the host may have some sluggishness only if privilege mode filtering is not
292 * available. In an ideal world, where qemu is not the only capable hardware,
293 * this can be removed.
294 * FYI: ARM64 does this way while x86 doesn't do anything as such.
295 * TODO: Should we keep it for RISC-V ?
296 */
297 period = -(local64_read(&perf_event->count));
298
299 local64_set(&perf_event->hw.period_left, 0);
300 perf_event->attr.sample_period = period;
301 perf_event->hw.sample_period = period;
302
303 set_bit(pmc->idx, kvpmu->pmc_overflown);
304 kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_PMU_OVF);
305
306 rpmu->pmu.start(perf_event, PERF_EF_RELOAD);
307 }
308
309 static long kvm_pmu_create_perf_event(struct kvm_pmc *pmc, struct perf_event_attr *attr,
310 unsigned long flags, unsigned long eidx,
311 unsigned long evtdata)
312 {
313 struct perf_event *event;
314
315 kvm_pmu_release_perf_event(pmc);
316 attr->config = kvm_pmu_get_perf_event_config(eidx, evtdata);
317 if (flags & SBI_PMU_CFG_FLAG_CLEAR_VALUE) {
318 //TODO: Do we really want to clear the value in hardware counter
319 pmc->counter_val = 0;
320 }
321
322 /*
323 * Set the default sample_period for now. The guest specified value
324 * will be updated in the start call.
325 */
326 attr->sample_period = kvm_pmu_get_sample_period(pmc);
327
328 event = perf_event_create_kernel_counter(attr, -1, current, kvm_riscv_pmu_overflow, pmc);
329 if (IS_ERR(event)) {
330 pr_debug("kvm pmu event creation failed for eidx %lx: %ld\n", eidx, PTR_ERR(event));
331 return PTR_ERR(event);
332 }
333
334 pmc->perf_event = event;
335 if (flags & SBI_PMU_CFG_FLAG_AUTO_START)
336 perf_event_enable(pmc->perf_event);
337
338 return 0;
339 }
340
341 int kvm_riscv_vcpu_pmu_incr_fw(struct kvm_vcpu *vcpu, unsigned long fid)
342 {
343 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
344 struct kvm_fw_event *fevent;
345
346 if (!kvpmu || fid >= SBI_PMU_FW_MAX)
347 return -EINVAL;
348
349 fevent = &kvpmu->fw_event[fid];
350 if (fevent->started)
351 fevent->value++;
352
353 return 0;
354 }
355
356 int kvm_riscv_vcpu_pmu_read_hpm(struct kvm_vcpu *vcpu, unsigned int csr_num,
357 unsigned long *val, unsigned long new_val,
358 unsigned long wr_mask)
359 {
360 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
361 int cidx, ret = KVM_INSN_CONTINUE_NEXT_SEPC;
362
363 if (!kvpmu || !kvpmu->init_done) {
364 /*
365 * In absence of sscofpmf in the platform, the guest OS may use
366 * the legacy PMU driver to read cycle/instret. In that case,
367 * just return 0 to avoid any illegal trap. However, any other
368 * hpmcounter access should result in illegal trap as they must
369 * be access through SBI PMU only.
370 */
371 if (csr_num == CSR_CYCLE || csr_num == CSR_INSTRET) {
372 *val = 0;
373 return ret;
374 } else {
375 return KVM_INSN_ILLEGAL_TRAP;
376 }
377 }
378
379 /* The counter CSR are read only. Thus, any write should result in illegal traps */
380 if (wr_mask)
381 return KVM_INSN_ILLEGAL_TRAP;
382
383 cidx = csr_num - CSR_CYCLE;
384
385 if (pmu_ctr_read(vcpu, cidx, val) < 0)
386 return KVM_INSN_ILLEGAL_TRAP;
387
388 return ret;
389 }
390
391 static void kvm_pmu_clear_snapshot_area(struct kvm_vcpu *vcpu)
392 {
393 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
394
395 kfree(kvpmu->sdata);
396 kvpmu->sdata = NULL;
397 kvpmu->snapshot_addr = INVALID_GPA;
398 }
399
400 int kvm_riscv_vcpu_pmu_snapshot_set_shmem(struct kvm_vcpu *vcpu, unsigned long saddr_low,
401 unsigned long saddr_high, unsigned long flags,
402 struct kvm_vcpu_sbi_return *retdata)
403 {
404 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
405 int snapshot_area_size = sizeof(struct riscv_pmu_snapshot_data);
406 int sbiret = 0;
407 gpa_t saddr;
408 unsigned long hva;
409 bool writable;
410
411 if (!kvpmu || flags) {
412 sbiret = SBI_ERR_INVALID_PARAM;
413 goto out;
414 }
415
416 if (saddr_low == SBI_SHMEM_DISABLE && saddr_high == SBI_SHMEM_DISABLE) {
417 kvm_pmu_clear_snapshot_area(vcpu);
418 return 0;
419 }
420
421 saddr = saddr_low;
422
423 if (saddr_high != 0) {
424 if (IS_ENABLED(CONFIG_32BIT))
425 saddr |= ((gpa_t)saddr_high << 32);
426 else
427 sbiret = SBI_ERR_INVALID_ADDRESS;
428 goto out;
429 }
430
431 hva = kvm_vcpu_gfn_to_hva_prot(vcpu, saddr >> PAGE_SHIFT, &writable);
432 if (kvm_is_error_hva(hva) || !writable) {
433 sbiret = SBI_ERR_INVALID_ADDRESS;
434 goto out;
435 }
436
437 kvpmu->sdata = kzalloc(snapshot_area_size, GFP_ATOMIC);
438 if (!kvpmu->sdata)
439 return -ENOMEM;
440
441 if (kvm_vcpu_write_guest(vcpu, saddr, kvpmu->sdata, snapshot_area_size)) {
442 kfree(kvpmu->sdata);
443 sbiret = SBI_ERR_FAILURE;
444 goto out;
445 }
446
447 kvpmu->snapshot_addr = saddr;
448
449 out:
450 retdata->err_val = sbiret;
451
452 return 0;
453 }
454
455 int kvm_riscv_vcpu_pmu_num_ctrs(struct kvm_vcpu *vcpu,
456 struct kvm_vcpu_sbi_return *retdata)
457 {
458 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
459
460 retdata->out_val = kvm_pmu_num_counters(kvpmu);
461
462 return 0;
463 }
464
465 int kvm_riscv_vcpu_pmu_ctr_info(struct kvm_vcpu *vcpu, unsigned long cidx,
466 struct kvm_vcpu_sbi_return *retdata)
467 {
468 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
469
470 if (cidx > RISCV_KVM_MAX_COUNTERS || cidx == 1) {
471 retdata->err_val = SBI_ERR_INVALID_PARAM;
472 return 0;
473 }
474
475 retdata->out_val = kvpmu->pmc[cidx].cinfo.value;
476
477 return 0;
478 }
479
480 int kvm_riscv_vcpu_pmu_ctr_start(struct kvm_vcpu *vcpu, unsigned long ctr_base,
481 unsigned long ctr_mask, unsigned long flags, u64 ival,
482 struct kvm_vcpu_sbi_return *retdata)
483 {
484 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
485 int i, pmc_index, sbiret = 0;
486 struct kvm_pmc *pmc;
487 int fevent_code;
488 bool snap_flag_set = flags & SBI_PMU_START_FLAG_INIT_SNAPSHOT;
489
490 if (kvm_pmu_validate_counter_mask(kvpmu, ctr_base, ctr_mask) < 0) {
491 sbiret = SBI_ERR_INVALID_PARAM;
492 goto out;
493 }
494
495 if (snap_flag_set) {
496 if (kvpmu->snapshot_addr == INVALID_GPA) {
497 sbiret = SBI_ERR_NO_SHMEM;
498 goto out;
499 }
500 if (kvm_vcpu_read_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata,
501 sizeof(struct riscv_pmu_snapshot_data))) {
502 pr_warn("Unable to read snapshot shared memory while starting counters\n");
503 sbiret = SBI_ERR_FAILURE;
504 goto out;
505 }
506 }
507 /* Start the counters that have been configured and requested by the guest */
508 for_each_set_bit(i, &ctr_mask, RISCV_MAX_COUNTERS) {
509 pmc_index = i + ctr_base;
510 if (!test_bit(pmc_index, kvpmu->pmc_in_use))
511 continue;
512 /* The guest started the counter again. Reset the overflow status */
513 clear_bit(pmc_index, kvpmu->pmc_overflown);
514 pmc = &kvpmu->pmc[pmc_index];
515 if (flags & SBI_PMU_START_FLAG_SET_INIT_VALUE) {
516 pmc->counter_val = ival;
517 } else if (snap_flag_set) {
518 /* The counter index in the snapshot are relative to the counter base */
519 pmc->counter_val = kvpmu->sdata->ctr_values[i];
520 }
521
522 if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) {
523 fevent_code = get_event_code(pmc->event_idx);
524 if (fevent_code >= SBI_PMU_FW_MAX) {
525 sbiret = SBI_ERR_INVALID_PARAM;
526 goto out;
527 }
528
529 /* Check if the counter was already started for some reason */
530 if (kvpmu->fw_event[fevent_code].started) {
531 sbiret = SBI_ERR_ALREADY_STARTED;
532 continue;
533 }
534
535 kvpmu->fw_event[fevent_code].started = true;
536 kvpmu->fw_event[fevent_code].value = pmc->counter_val;
537 } else if (pmc->perf_event) {
538 if (unlikely(pmc->started)) {
539 sbiret = SBI_ERR_ALREADY_STARTED;
540 continue;
541 }
542 perf_event_period(pmc->perf_event, kvm_pmu_get_sample_period(pmc));
543 perf_event_enable(pmc->perf_event);
544 pmc->started = true;
545 } else {
546 sbiret = SBI_ERR_INVALID_PARAM;
547 }
548 }
549
550 out:
551 retdata->err_val = sbiret;
552
553 return 0;
554 }
555
556 int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base,
557 unsigned long ctr_mask, unsigned long flags,
558 struct kvm_vcpu_sbi_return *retdata)
559 {
560 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
561 int i, pmc_index, sbiret = 0;
562 u64 enabled, running;
563 struct kvm_pmc *pmc;
564 int fevent_code;
565 bool snap_flag_set = flags & SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT;
566 bool shmem_needs_update = false;
567
568 if (kvm_pmu_validate_counter_mask(kvpmu, ctr_base, ctr_mask) < 0) {
569 sbiret = SBI_ERR_INVALID_PARAM;
570 goto out;
571 }
572
573 if (snap_flag_set && kvpmu->snapshot_addr == INVALID_GPA) {
574 sbiret = SBI_ERR_NO_SHMEM;
575 goto out;
576 }
577
578 /* Stop the counters that have been configured and requested by the guest */
579 for_each_set_bit(i, &ctr_mask, RISCV_MAX_COUNTERS) {
580 pmc_index = i + ctr_base;
581 if (!test_bit(pmc_index, kvpmu->pmc_in_use))
582 continue;
583 pmc = &kvpmu->pmc[pmc_index];
584 if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) {
585 fevent_code = get_event_code(pmc->event_idx);
586 if (fevent_code >= SBI_PMU_FW_MAX) {
587 sbiret = SBI_ERR_INVALID_PARAM;
588 goto out;
589 }
590
591 if (!kvpmu->fw_event[fevent_code].started)
592 sbiret = SBI_ERR_ALREADY_STOPPED;
593
594 kvpmu->fw_event[fevent_code].started = false;
595 } else if (pmc->perf_event) {
596 if (pmc->started) {
597 /* Stop counting the counter */
598 perf_event_disable(pmc->perf_event);
599 pmc->started = false;
600 } else {
601 sbiret = SBI_ERR_ALREADY_STOPPED;
602 }
603
604 if (flags & SBI_PMU_STOP_FLAG_RESET)
605 /* Release the counter if this is a reset request */
606 kvm_pmu_release_perf_event(pmc);
607 } else {
608 sbiret = SBI_ERR_INVALID_PARAM;
609 }
610
611 if (snap_flag_set && !sbiret) {
612 if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW)
613 pmc->counter_val = kvpmu->fw_event[fevent_code].value;
614 else if (pmc->perf_event)
615 pmc->counter_val += perf_event_read_value(pmc->perf_event,
616 &enabled, &running);
617 /*
618 * The counter and overflow indicies in the snapshot region are w.r.to
619 * cbase. Modify the set bit in the counter mask instead of the pmc_index
620 * which indicates the absolute counter index.
621 */
622 if (test_bit(pmc_index, kvpmu->pmc_overflown))
623 kvpmu->sdata->ctr_overflow_mask |= BIT(i);
624 kvpmu->sdata->ctr_values[i] = pmc->counter_val;
625 shmem_needs_update = true;
626 }
627
628 if (flags & SBI_PMU_STOP_FLAG_RESET) {
629 pmc->event_idx = SBI_PMU_EVENT_IDX_INVALID;
630 clear_bit(pmc_index, kvpmu->pmc_in_use);
631 clear_bit(pmc_index, kvpmu->pmc_overflown);
632 if (snap_flag_set) {
633 /*
634 * Only clear the given counter as the caller is responsible to
635 * validate both the overflow mask and configured counters.
636 */
637 kvpmu->sdata->ctr_overflow_mask &= ~BIT(i);
638 shmem_needs_update = true;
639 }
640 }
641 }
642
643 if (shmem_needs_update)
644 kvm_vcpu_write_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata,
645 sizeof(struct riscv_pmu_snapshot_data));
646
647 out:
648 retdata->err_val = sbiret;
649
650 return 0;
651 }
652
653 int kvm_riscv_vcpu_pmu_ctr_cfg_match(struct kvm_vcpu *vcpu, unsigned long ctr_base,
654 unsigned long ctr_mask, unsigned long flags,
655 unsigned long eidx, u64 evtdata,
656 struct kvm_vcpu_sbi_return *retdata)
657 {
658 int ctr_idx, sbiret = 0;
659 long ret;
660 bool is_fevent;
661 unsigned long event_code;
662 u32 etype = kvm_pmu_get_perf_event_type(eidx);
663 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
664 struct kvm_pmc *pmc = NULL;
665 struct perf_event_attr attr = {
666 .type = etype,
667 .size = sizeof(struct perf_event_attr),
668 .pinned = true,
669 /*
670 * It should never reach here if the platform doesn't support the sscofpmf
671 * extension as mode filtering won't work without it.
672 */
673 .exclude_host = true,
674 .exclude_hv = true,
675 .exclude_user = !!(flags & SBI_PMU_CFG_FLAG_SET_UINH),
676 .exclude_kernel = !!(flags & SBI_PMU_CFG_FLAG_SET_SINH),
677 .config1 = RISCV_PMU_CONFIG1_GUEST_EVENTS,
678 };
679
680 if (kvm_pmu_validate_counter_mask(kvpmu, ctr_base, ctr_mask) < 0) {
681 sbiret = SBI_ERR_INVALID_PARAM;
682 goto out;
683 }
684
685 event_code = get_event_code(eidx);
686 is_fevent = kvm_pmu_is_fw_event(eidx);
687 if (is_fevent && event_code >= SBI_PMU_FW_MAX) {
688 sbiret = SBI_ERR_NOT_SUPPORTED;
689 goto out;
690 }
691
692 /*
693 * SKIP_MATCH flag indicates the caller is aware of the assigned counter
694 * for this event. Just do a sanity check if it already marked used.
695 */
696 if (flags & SBI_PMU_CFG_FLAG_SKIP_MATCH) {
697 if (!test_bit(ctr_base + __ffs(ctr_mask), kvpmu->pmc_in_use)) {
698 sbiret = SBI_ERR_FAILURE;
699 goto out;
700 }
701 ctr_idx = ctr_base + __ffs(ctr_mask);
702 } else {
703 ctr_idx = pmu_get_pmc_index(kvpmu, eidx, ctr_base, ctr_mask);
704 if (ctr_idx < 0) {
705 sbiret = SBI_ERR_NOT_SUPPORTED;
706 goto out;
707 }
708 }
709
710 pmc = &kvpmu->pmc[ctr_idx];
711 pmc->idx = ctr_idx;
712
713 if (is_fevent) {
714 if (flags & SBI_PMU_CFG_FLAG_AUTO_START)
715 kvpmu->fw_event[event_code].started = true;
716 } else {
717 ret = kvm_pmu_create_perf_event(pmc, &attr, flags, eidx, evtdata);
718 if (ret) {
719 sbiret = SBI_ERR_NOT_SUPPORTED;
720 goto out;
721 }
722 }
723
724 set_bit(ctr_idx, kvpmu->pmc_in_use);
725 pmc->event_idx = eidx;
726 retdata->out_val = ctr_idx;
727 out:
728 retdata->err_val = sbiret;
729
730 return 0;
731 }
732
733 int kvm_riscv_vcpu_pmu_fw_ctr_read_hi(struct kvm_vcpu *vcpu, unsigned long cidx,
734 struct kvm_vcpu_sbi_return *retdata)
735 {
736 int ret;
737
738 ret = pmu_fw_ctr_read_hi(vcpu, cidx, &retdata->out_val);
739 if (ret == -EINVAL)
740 retdata->err_val = SBI_ERR_INVALID_PARAM;
741
742 return 0;
743 }
744
745 int kvm_riscv_vcpu_pmu_fw_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx,
746 struct kvm_vcpu_sbi_return *retdata)
747 {
748 int ret;
749
750 ret = pmu_ctr_read(vcpu, cidx, &retdata->out_val);
751 if (ret == -EINVAL)
752 retdata->err_val = SBI_ERR_INVALID_PARAM;
753
754 return 0;
755 }
756
757 void kvm_riscv_vcpu_pmu_init(struct kvm_vcpu *vcpu)
758 {
759 int i = 0, ret, num_hw_ctrs = 0, hpm_width = 0;
760 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
761 struct kvm_pmc *pmc;
762
763 /*
764 * PMU functionality should be only available to guests if privilege mode
765 * filtering is available in the host. Otherwise, guest will always count
766 * events while the execution is in hypervisor mode.
767 */
768 if (!riscv_isa_extension_available(NULL, SSCOFPMF))
769 return;
770
771 ret = riscv_pmu_get_hpm_info(&hpm_width, &num_hw_ctrs);
772 if (ret < 0 || !hpm_width || !num_hw_ctrs)
773 return;
774
775 /*
776 * Increase the number of hardware counters to offset the time counter.
777 */
778 kvpmu->num_hw_ctrs = num_hw_ctrs + 1;
779 kvpmu->num_fw_ctrs = SBI_PMU_FW_MAX;
780 memset(&kvpmu->fw_event, 0, SBI_PMU_FW_MAX * sizeof(struct kvm_fw_event));
781 kvpmu->snapshot_addr = INVALID_GPA;
782
783 if (kvpmu->num_hw_ctrs > RISCV_KVM_MAX_HW_CTRS) {
784 pr_warn_once("Limiting the hardware counters to 32 as specified by the ISA");
785 kvpmu->num_hw_ctrs = RISCV_KVM_MAX_HW_CTRS;
786 }
787
788 /*
789 * There is no correlation between the logical hardware counter and virtual counters.
790 * However, we need to encode a hpmcounter CSR in the counter info field so that
791 * KVM can trap n emulate the read. This works well in the migration use case as
792 * KVM doesn't care if the actual hpmcounter is available in the hardware or not.
793 */
794 for (i = 0; i < kvm_pmu_num_counters(kvpmu); i++) {
795 /* TIME CSR shouldn't be read from perf interface */
796 if (i == 1)
797 continue;
798 pmc = &kvpmu->pmc[i];
799 pmc->idx = i;
800 pmc->event_idx = SBI_PMU_EVENT_IDX_INVALID;
801 pmc->vcpu = vcpu;
802 if (i < kvpmu->num_hw_ctrs) {
803 pmc->cinfo.type = SBI_PMU_CTR_TYPE_HW;
804 if (i < 3)
805 /* CY, IR counters */
806 pmc->cinfo.width = 63;
807 else
808 pmc->cinfo.width = hpm_width;
809 /*
810 * The CSR number doesn't have any relation with the logical
811 * hardware counters. The CSR numbers are encoded sequentially
812 * to avoid maintaining a map between the virtual counter
813 * and CSR number.
814 */
815 pmc->cinfo.csr = CSR_CYCLE + i;
816 } else {
817 pmc->cinfo.type = SBI_PMU_CTR_TYPE_FW;
818 pmc->cinfo.width = 63;
819 }
820 }
821
822 kvpmu->init_done = true;
823 }
824
825 void kvm_riscv_vcpu_pmu_deinit(struct kvm_vcpu *vcpu)
826 {
827 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
828 struct kvm_pmc *pmc;
829 int i;
830
831 if (!kvpmu)
832 return;
833
834 for_each_set_bit(i, kvpmu->pmc_in_use, RISCV_KVM_MAX_COUNTERS) {
835 pmc = &kvpmu->pmc[i];
836 pmc->counter_val = 0;
837 kvm_pmu_release_perf_event(pmc);
838 pmc->event_idx = SBI_PMU_EVENT_IDX_INVALID;
839 }
840 bitmap_zero(kvpmu->pmc_in_use, RISCV_KVM_MAX_COUNTERS);
841 bitmap_zero(kvpmu->pmc_overflown, RISCV_KVM_MAX_COUNTERS);
842 memset(&kvpmu->fw_event, 0, SBI_PMU_FW_MAX * sizeof(struct kvm_fw_event));
843 kvm_pmu_clear_snapshot_area(vcpu);
844 }
845
846 void kvm_riscv_vcpu_pmu_reset(struct kvm_vcpu *vcpu)
847 {
848 kvm_riscv_vcpu_pmu_deinit(vcpu);
849 }
850
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.