1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright 2019 Arm Limited
4 * Author: Andrew Murray <Andrew.Murray@arm.com>
5 */
6 #include <linux/kvm_host.h>
7 #include <linux/perf_event.h>
8
9 static DEFINE_PER_CPU(struct kvm_pmu_events, kvm_pmu_events);
10
11 /*
12 * Given the perf event attributes and system type, determine
13 * if we are going to need to switch counters at guest entry/exit.
14 */
15 static bool kvm_pmu_switch_needed(struct perf_event_attr *attr)
16 {
17 /**
18 * With VHE the guest kernel runs at EL1 and the host at EL2,
19 * where user (EL0) is excluded then we have no reason to switch
20 * counters.
21 */
22 if (has_vhe() && attr->exclude_user)
23 return false;
24
25 /* Only switch if attributes are different */
26 return (attr->exclude_host != attr->exclude_guest);
27 }
28
29 struct kvm_pmu_events *kvm_get_pmu_events(void)
30 {
31 return this_cpu_ptr(&kvm_pmu_events);
32 }
33
34 /*
35 * Add events to track that we may want to switch at guest entry/exit
36 * time.
37 */
38 void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr)
39 {
40 struct kvm_pmu_events *pmu = kvm_get_pmu_events();
41
42 if (!kvm_arm_support_pmu_v3() || !kvm_pmu_switch_needed(attr))
43 return;
44
45 if (!attr->exclude_host)
46 pmu->events_host |= set;
47 if (!attr->exclude_guest)
48 pmu->events_guest |= set;
49 }
50
51 /*
52 * Stop tracking events
53 */
54 void kvm_clr_pmu_events(u32 clr)
55 {
56 struct kvm_pmu_events *pmu = kvm_get_pmu_events();
57
58 if (!kvm_arm_support_pmu_v3())
59 return;
60
61 pmu->events_host &= ~clr;
62 pmu->events_guest &= ~clr;
63 }
64
65 #define PMEVTYPER_READ_CASE(idx) \
66 case idx: \
67 return read_sysreg(pmevtyper##idx##_el0)
68
69 #define PMEVTYPER_WRITE_CASE(idx) \
70 case idx: \
71 write_sysreg(val, pmevtyper##idx##_el0); \
72 break
73
74 #define PMEVTYPER_CASES(readwrite) \
75 PMEVTYPER_##readwrite##_CASE(0); \
76 PMEVTYPER_##readwrite##_CASE(1); \
77 PMEVTYPER_##readwrite##_CASE(2); \
78 PMEVTYPER_##readwrite##_CASE(3); \
79 PMEVTYPER_##readwrite##_CASE(4); \
80 PMEVTYPER_##readwrite##_CASE(5); \
81 PMEVTYPER_##readwrite##_CASE(6); \
82 PMEVTYPER_##readwrite##_CASE(7); \
83 PMEVTYPER_##readwrite##_CASE(8); \
84 PMEVTYPER_##readwrite##_CASE(9); \
85 PMEVTYPER_##readwrite##_CASE(10); \
86 PMEVTYPER_##readwrite##_CASE(11); \
87 PMEVTYPER_##readwrite##_CASE(12); \
88 PMEVTYPER_##readwrite##_CASE(13); \
89 PMEVTYPER_##readwrite##_CASE(14); \
90 PMEVTYPER_##readwrite##_CASE(15); \
91 PMEVTYPER_##readwrite##_CASE(16); \
92 PMEVTYPER_##readwrite##_CASE(17); \
93 PMEVTYPER_##readwrite##_CASE(18); \
94 PMEVTYPER_##readwrite##_CASE(19); \
95 PMEVTYPER_##readwrite##_CASE(20); \
96 PMEVTYPER_##readwrite##_CASE(21); \
97 PMEVTYPER_##readwrite##_CASE(22); \
98 PMEVTYPER_##readwrite##_CASE(23); \
99 PMEVTYPER_##readwrite##_CASE(24); \
100 PMEVTYPER_##readwrite##_CASE(25); \
101 PMEVTYPER_##readwrite##_CASE(26); \
102 PMEVTYPER_##readwrite##_CASE(27); \
103 PMEVTYPER_##readwrite##_CASE(28); \
104 PMEVTYPER_##readwrite##_CASE(29); \
105 PMEVTYPER_##readwrite##_CASE(30)
106
107 /*
108 * Read a value direct from PMEVTYPER<idx> where idx is 0-30
109 * or PMCCFILTR_EL0 where idx is ARMV8_PMU_CYCLE_IDX (31).
110 */
111 static u64 kvm_vcpu_pmu_read_evtype_direct(int idx)
112 {
113 switch (idx) {
114 PMEVTYPER_CASES(READ);
115 case ARMV8_PMU_CYCLE_IDX:
116 return read_sysreg(pmccfiltr_el0);
117 default:
118 WARN_ON(1);
119 }
120
121 return 0;
122 }
123
124 /*
125 * Write a value direct to PMEVTYPER<idx> where idx is 0-30
126 * or PMCCFILTR_EL0 where idx is ARMV8_PMU_CYCLE_IDX (31).
127 */
128 static void kvm_vcpu_pmu_write_evtype_direct(int idx, u32 val)
129 {
130 switch (idx) {
131 PMEVTYPER_CASES(WRITE);
132 case ARMV8_PMU_CYCLE_IDX:
133 write_sysreg(val, pmccfiltr_el0);
134 break;
135 default:
136 WARN_ON(1);
137 }
138 }
139
140 /*
141 * Modify ARMv8 PMU events to include EL0 counting
142 */
143 static void kvm_vcpu_pmu_enable_el0(unsigned long events)
144 {
145 u64 typer;
146 u32 counter;
147
148 for_each_set_bit(counter, &events, 32) {
149 typer = kvm_vcpu_pmu_read_evtype_direct(counter);
150 typer &= ~ARMV8_PMU_EXCLUDE_EL0;
151 kvm_vcpu_pmu_write_evtype_direct(counter, typer);
152 }
153 }
154
155 /*
156 * Modify ARMv8 PMU events to exclude EL0 counting
157 */
158 static void kvm_vcpu_pmu_disable_el0(unsigned long events)
159 {
160 u64 typer;
161 u32 counter;
162
163 for_each_set_bit(counter, &events, 32) {
164 typer = kvm_vcpu_pmu_read_evtype_direct(counter);
165 typer |= ARMV8_PMU_EXCLUDE_EL0;
166 kvm_vcpu_pmu_write_evtype_direct(counter, typer);
167 }
168 }
169
170 /*
171 * On VHE ensure that only guest events have EL0 counting enabled.
172 * This is called from both vcpu_{load,put} and the sysreg handling.
173 * Since the latter is preemptible, special care must be taken to
174 * disable preemption.
175 */
176 void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu)
177 {
178 struct kvm_pmu_events *pmu;
179 u32 events_guest, events_host;
180
181 if (!kvm_arm_support_pmu_v3() || !has_vhe())
182 return;
183
184 preempt_disable();
185 pmu = kvm_get_pmu_events();
186 events_guest = pmu->events_guest;
187 events_host = pmu->events_host;
188
189 kvm_vcpu_pmu_enable_el0(events_guest);
190 kvm_vcpu_pmu_disable_el0(events_host);
191 preempt_enable();
192 }
193
194 /*
195 * On VHE ensure that only host events have EL0 counting enabled
196 */
197 void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu)
198 {
199 struct kvm_pmu_events *pmu;
200 u32 events_guest, events_host;
201
202 if (!kvm_arm_support_pmu_v3() || !has_vhe())
203 return;
204
205 pmu = kvm_get_pmu_events();
206 events_guest = pmu->events_guest;
207 events_host = pmu->events_host;
208
209 kvm_vcpu_pmu_enable_el0(events_host);
210 kvm_vcpu_pmu_disable_el0(events_guest);
211 }
212
213 /*
214 * With VHE, keep track of the PMUSERENR_EL0 value for the host EL0 on the pCPU
215 * where PMUSERENR_EL0 for the guest is loaded, since PMUSERENR_EL0 is switched
216 * to the value for the guest on vcpu_load(). The value for the host EL0
217 * will be restored on vcpu_put(), before returning to userspace.
218 * This isn't necessary for nVHE, as the register is context switched for
219 * every guest enter/exit.
220 *
221 * Return true if KVM takes care of the register. Otherwise return false.
222 */
223 bool kvm_set_pmuserenr(u64 val)
224 {
225 struct kvm_cpu_context *hctxt;
226 struct kvm_vcpu *vcpu;
227
228 if (!kvm_arm_support_pmu_v3() || !has_vhe())
229 return false;
230
231 vcpu = kvm_get_running_vcpu();
232 if (!vcpu || !vcpu_get_flag(vcpu, PMUSERENR_ON_CPU))
233 return false;
234
235 hctxt = host_data_ptr(host_ctxt);
236 ctxt_sys_reg(hctxt, PMUSERENR_EL0) = val;
237 return true;
238 }
239
240 /*
241 * If we interrupted the guest to update the host PMU context, make
242 * sure we re-apply the guest EL0 state.
243 */
244 void kvm_vcpu_pmu_resync_el0(void)
245 {
246 struct kvm_vcpu *vcpu;
247
248 if (!has_vhe() || !in_interrupt())
249 return;
250
251 vcpu = kvm_get_running_vcpu();
252 if (!vcpu)
253 return;
254
255 kvm_make_request(KVM_REQ_RESYNC_PMU_EL0, vcpu);
256 }
257
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.