1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * The hwprobe interface, for allowing userspace to probe to see which features
4 * are supported by the hardware. See Documentation/arch/riscv/hwprobe.rst for
5 * more details.
6 */
7 #include <linux/syscalls.h>
8 #include <asm/cacheflush.h>
9 #include <asm/cpufeature.h>
10 #include <asm/hwprobe.h>
11 #include <asm/processor.h>
12 #include <asm/delay.h>
13 #include <asm/sbi.h>
14 #include <asm/switch_to.h>
15 #include <asm/uaccess.h>
16 #include <asm/unistd.h>
17 #include <asm/vector.h>
18 #include <vdso/vsyscall.h>
19
20
21 static void hwprobe_arch_id(struct riscv_hwprobe *pair,
22 const struct cpumask *cpus)
23 {
24 u64 id = -1ULL;
25 bool first = true;
26 int cpu;
27
28 for_each_cpu(cpu, cpus) {
29 u64 cpu_id;
30
31 switch (pair->key) {
32 case RISCV_HWPROBE_KEY_MVENDORID:
33 cpu_id = riscv_cached_mvendorid(cpu);
34 break;
35 case RISCV_HWPROBE_KEY_MIMPID:
36 cpu_id = riscv_cached_mimpid(cpu);
37 break;
38 case RISCV_HWPROBE_KEY_MARCHID:
39 cpu_id = riscv_cached_marchid(cpu);
40 break;
41 }
42
43 if (first) {
44 id = cpu_id;
45 first = false;
46 }
47
48 /*
49 * If there's a mismatch for the given set, return -1 in the
50 * value.
51 */
52 if (id != cpu_id) {
53 id = -1ULL;
54 break;
55 }
56 }
57
58 pair->value = id;
59 }
60
61 static void hwprobe_isa_ext0(struct riscv_hwprobe *pair,
62 const struct cpumask *cpus)
63 {
64 int cpu;
65 u64 missing = 0;
66
67 pair->value = 0;
68 if (has_fpu())
69 pair->value |= RISCV_HWPROBE_IMA_FD;
70
71 if (riscv_isa_extension_available(NULL, c))
72 pair->value |= RISCV_HWPROBE_IMA_C;
73
74 if (has_vector() && riscv_isa_extension_available(NULL, v))
75 pair->value |= RISCV_HWPROBE_IMA_V;
76
77 /*
78 * Loop through and record extensions that 1) anyone has, and 2) anyone
79 * doesn't have.
80 */
81 for_each_cpu(cpu, cpus) {
82 struct riscv_isainfo *isainfo = &hart_isa[cpu];
83
84 #define EXT_KEY(ext) \
85 do { \
86 if (__riscv_isa_extension_available(isainfo->isa, RISCV_ISA_EXT_##ext)) \
87 pair->value |= RISCV_HWPROBE_EXT_##ext; \
88 else \
89 missing |= RISCV_HWPROBE_EXT_##ext; \
90 } while (false)
91
92 /*
93 * Only use EXT_KEY() for extensions which can be exposed to userspace,
94 * regardless of the kernel's configuration, as no other checks, besides
95 * presence in the hart_isa bitmap, are made.
96 */
97 EXT_KEY(ZACAS);
98 EXT_KEY(ZAWRS);
99 EXT_KEY(ZBA);
100 EXT_KEY(ZBB);
101 EXT_KEY(ZBC);
102 EXT_KEY(ZBKB);
103 EXT_KEY(ZBKC);
104 EXT_KEY(ZBKX);
105 EXT_KEY(ZBS);
106 EXT_KEY(ZCA);
107 EXT_KEY(ZCB);
108 EXT_KEY(ZCMOP);
109 EXT_KEY(ZICBOZ);
110 EXT_KEY(ZICOND);
111 EXT_KEY(ZIHINTNTL);
112 EXT_KEY(ZIHINTPAUSE);
113 EXT_KEY(ZIMOP);
114 EXT_KEY(ZKND);
115 EXT_KEY(ZKNE);
116 EXT_KEY(ZKNH);
117 EXT_KEY(ZKSED);
118 EXT_KEY(ZKSH);
119 EXT_KEY(ZKT);
120 EXT_KEY(ZTSO);
121
122 /*
123 * All the following extensions must depend on the kernel
124 * support of V.
125 */
126 if (has_vector()) {
127 EXT_KEY(ZVBB);
128 EXT_KEY(ZVBC);
129 EXT_KEY(ZVE32F);
130 EXT_KEY(ZVE32X);
131 EXT_KEY(ZVE64D);
132 EXT_KEY(ZVE64F);
133 EXT_KEY(ZVE64X);
134 EXT_KEY(ZVFH);
135 EXT_KEY(ZVFHMIN);
136 EXT_KEY(ZVKB);
137 EXT_KEY(ZVKG);
138 EXT_KEY(ZVKNED);
139 EXT_KEY(ZVKNHA);
140 EXT_KEY(ZVKNHB);
141 EXT_KEY(ZVKSED);
142 EXT_KEY(ZVKSH);
143 EXT_KEY(ZVKT);
144 }
145
146 if (has_fpu()) {
147 EXT_KEY(ZCD);
148 EXT_KEY(ZCF);
149 EXT_KEY(ZFA);
150 EXT_KEY(ZFH);
151 EXT_KEY(ZFHMIN);
152 }
153 #undef EXT_KEY
154 }
155
156 /* Now turn off reporting features if any CPU is missing it. */
157 pair->value &= ~missing;
158 }
159
160 static bool hwprobe_ext0_has(const struct cpumask *cpus, unsigned long ext)
161 {
162 struct riscv_hwprobe pair;
163
164 hwprobe_isa_ext0(&pair, cpus);
165 return (pair.value & ext);
166 }
167
168 #if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS)
169 static u64 hwprobe_misaligned(const struct cpumask *cpus)
170 {
171 int cpu;
172 u64 perf = -1ULL;
173
174 for_each_cpu(cpu, cpus) {
175 int this_perf = per_cpu(misaligned_access_speed, cpu);
176
177 if (perf == -1ULL)
178 perf = this_perf;
179
180 if (perf != this_perf) {
181 perf = RISCV_HWPROBE_MISALIGNED_UNKNOWN;
182 break;
183 }
184 }
185
186 if (perf == -1ULL)
187 return RISCV_HWPROBE_MISALIGNED_UNKNOWN;
188
189 return perf;
190 }
191 #else
192 static u64 hwprobe_misaligned(const struct cpumask *cpus)
193 {
194 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_UNALIGNED_ACCESS))
195 return RISCV_HWPROBE_MISALIGNED_FAST;
196
197 if (IS_ENABLED(CONFIG_RISCV_EMULATED_UNALIGNED_ACCESS) && unaligned_ctl_available())
198 return RISCV_HWPROBE_MISALIGNED_EMULATED;
199
200 return RISCV_HWPROBE_MISALIGNED_SLOW;
201 }
202 #endif
203
204 static void hwprobe_one_pair(struct riscv_hwprobe *pair,
205 const struct cpumask *cpus)
206 {
207 switch (pair->key) {
208 case RISCV_HWPROBE_KEY_MVENDORID:
209 case RISCV_HWPROBE_KEY_MARCHID:
210 case RISCV_HWPROBE_KEY_MIMPID:
211 hwprobe_arch_id(pair, cpus);
212 break;
213 /*
214 * The kernel already assumes that the base single-letter ISA
215 * extensions are supported on all harts, and only supports the
216 * IMA base, so just cheat a bit here and tell that to
217 * userspace.
218 */
219 case RISCV_HWPROBE_KEY_BASE_BEHAVIOR:
220 pair->value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA;
221 break;
222
223 case RISCV_HWPROBE_KEY_IMA_EXT_0:
224 hwprobe_isa_ext0(pair, cpus);
225 break;
226
227 case RISCV_HWPROBE_KEY_CPUPERF_0:
228 pair->value = hwprobe_misaligned(cpus);
229 break;
230
231 case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE:
232 pair->value = 0;
233 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ))
234 pair->value = riscv_cboz_block_size;
235 break;
236 case RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS:
237 pair->value = user_max_virt_addr();
238 break;
239
240 case RISCV_HWPROBE_KEY_TIME_CSR_FREQ:
241 pair->value = riscv_timebase;
242 break;
243
244 /*
245 * For forward compatibility, unknown keys don't fail the whole
246 * call, but get their element key set to -1 and value set to 0
247 * indicating they're unrecognized.
248 */
249 default:
250 pair->key = -1;
251 pair->value = 0;
252 break;
253 }
254 }
255
256 static int hwprobe_get_values(struct riscv_hwprobe __user *pairs,
257 size_t pair_count, size_t cpusetsize,
258 unsigned long __user *cpus_user,
259 unsigned int flags)
260 {
261 size_t out;
262 int ret;
263 cpumask_t cpus;
264
265 /* Check the reserved flags. */
266 if (flags != 0)
267 return -EINVAL;
268
269 /*
270 * The interface supports taking in a CPU mask, and returns values that
271 * are consistent across that mask. Allow userspace to specify NULL and
272 * 0 as a shortcut to all online CPUs.
273 */
274 cpumask_clear(&cpus);
275 if (!cpusetsize && !cpus_user) {
276 cpumask_copy(&cpus, cpu_online_mask);
277 } else {
278 if (cpusetsize > cpumask_size())
279 cpusetsize = cpumask_size();
280
281 ret = copy_from_user(&cpus, cpus_user, cpusetsize);
282 if (ret)
283 return -EFAULT;
284
285 /*
286 * Userspace must provide at least one online CPU, without that
287 * there's no way to define what is supported.
288 */
289 cpumask_and(&cpus, &cpus, cpu_online_mask);
290 if (cpumask_empty(&cpus))
291 return -EINVAL;
292 }
293
294 for (out = 0; out < pair_count; out++, pairs++) {
295 struct riscv_hwprobe pair;
296
297 if (get_user(pair.key, &pairs->key))
298 return -EFAULT;
299
300 pair.value = 0;
301 hwprobe_one_pair(&pair, &cpus);
302 ret = put_user(pair.key, &pairs->key);
303 if (ret == 0)
304 ret = put_user(pair.value, &pairs->value);
305
306 if (ret)
307 return -EFAULT;
308 }
309
310 return 0;
311 }
312
313 static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs,
314 size_t pair_count, size_t cpusetsize,
315 unsigned long __user *cpus_user,
316 unsigned int flags)
317 {
318 cpumask_t cpus, one_cpu;
319 bool clear_all = false;
320 size_t i;
321 int ret;
322
323 if (flags != RISCV_HWPROBE_WHICH_CPUS)
324 return -EINVAL;
325
326 if (!cpusetsize || !cpus_user)
327 return -EINVAL;
328
329 if (cpusetsize > cpumask_size())
330 cpusetsize = cpumask_size();
331
332 ret = copy_from_user(&cpus, cpus_user, cpusetsize);
333 if (ret)
334 return -EFAULT;
335
336 if (cpumask_empty(&cpus))
337 cpumask_copy(&cpus, cpu_online_mask);
338
339 cpumask_and(&cpus, &cpus, cpu_online_mask);
340
341 cpumask_clear(&one_cpu);
342
343 for (i = 0; i < pair_count; i++) {
344 struct riscv_hwprobe pair, tmp;
345 int cpu;
346
347 ret = copy_from_user(&pair, &pairs[i], sizeof(pair));
348 if (ret)
349 return -EFAULT;
350
351 if (!riscv_hwprobe_key_is_valid(pair.key)) {
352 clear_all = true;
353 pair = (struct riscv_hwprobe){ .key = -1, };
354 ret = copy_to_user(&pairs[i], &pair, sizeof(pair));
355 if (ret)
356 return -EFAULT;
357 }
358
359 if (clear_all)
360 continue;
361
362 tmp = (struct riscv_hwprobe){ .key = pair.key, };
363
364 for_each_cpu(cpu, &cpus) {
365 cpumask_set_cpu(cpu, &one_cpu);
366
367 hwprobe_one_pair(&tmp, &one_cpu);
368
369 if (!riscv_hwprobe_pair_cmp(&tmp, &pair))
370 cpumask_clear_cpu(cpu, &cpus);
371
372 cpumask_clear_cpu(cpu, &one_cpu);
373 }
374 }
375
376 if (clear_all)
377 cpumask_clear(&cpus);
378
379 ret = copy_to_user(cpus_user, &cpus, cpusetsize);
380 if (ret)
381 return -EFAULT;
382
383 return 0;
384 }
385
386 static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs,
387 size_t pair_count, size_t cpusetsize,
388 unsigned long __user *cpus_user,
389 unsigned int flags)
390 {
391 if (flags & RISCV_HWPROBE_WHICH_CPUS)
392 return hwprobe_get_cpus(pairs, pair_count, cpusetsize,
393 cpus_user, flags);
394
395 return hwprobe_get_values(pairs, pair_count, cpusetsize,
396 cpus_user, flags);
397 }
398
399 #ifdef CONFIG_MMU
400
401 static int __init init_hwprobe_vdso_data(void)
402 {
403 struct vdso_data *vd = __arch_get_k_vdso_data();
404 struct arch_vdso_data *avd = &vd->arch_data;
405 u64 id_bitsmash = 0;
406 struct riscv_hwprobe pair;
407 int key;
408
409 /*
410 * Initialize vDSO data with the answers for the "all CPUs" case, to
411 * save a syscall in the common case.
412 */
413 for (key = 0; key <= RISCV_HWPROBE_MAX_KEY; key++) {
414 pair.key = key;
415 hwprobe_one_pair(&pair, cpu_online_mask);
416
417 WARN_ON_ONCE(pair.key < 0);
418
419 avd->all_cpu_hwprobe_values[key] = pair.value;
420 /*
421 * Smash together the vendor, arch, and impl IDs to see if
422 * they're all 0 or any negative.
423 */
424 if (key <= RISCV_HWPROBE_KEY_MIMPID)
425 id_bitsmash |= pair.value;
426 }
427
428 /*
429 * If the arch, vendor, and implementation ID are all the same across
430 * all harts, then assume all CPUs are the same, and allow the vDSO to
431 * answer queries for arbitrary masks. However if all values are 0 (not
432 * populated) or any value returns -1 (varies across CPUs), then the
433 * vDSO should defer to the kernel for exotic cpu masks.
434 */
435 avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1;
436 return 0;
437 }
438
439 arch_initcall_sync(init_hwprobe_vdso_data);
440
441 #endif /* CONFIG_MMU */
442
443 SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs,
444 size_t, pair_count, size_t, cpusetsize, unsigned long __user *,
445 cpus, unsigned int, flags)
446 {
447 return do_riscv_hwprobe(pairs, pair_count, cpusetsize,
448 cpus, flags);
449 }
450
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.