1 // SPDX-License-Identifier: GPL-2.0-only 1
2 /*
3 * Copyright (C) 2020 Western Digital Corporat
4 */
5 #include <linux/kernel.h>
6 #include <linux/init.h>
7 #include <linux/mm.h>
8 #include <linux/module.h>
9 #include <linux/perf_event.h>
10 #include <linux/irq.h>
11 #include <linux/stringify.h>
12
13 #include <asm/processor.h>
14 #include <asm/ptrace.h>
15 #include <asm/csr.h>
16 #include <asm/entry-common.h>
17 #include <asm/hwprobe.h>
18 #include <asm/cpufeature.h>
19
20 #define INSN_MATCH_LB 0x3
21 #define INSN_MASK_LB 0x707f
22 #define INSN_MATCH_LH 0x1003
23 #define INSN_MASK_LH 0x707f
24 #define INSN_MATCH_LW 0x2003
25 #define INSN_MASK_LW 0x707f
26 #define INSN_MATCH_LD 0x3003
27 #define INSN_MASK_LD 0x707f
28 #define INSN_MATCH_LBU 0x4003
29 #define INSN_MASK_LBU 0x707f
30 #define INSN_MATCH_LHU 0x5003
31 #define INSN_MASK_LHU 0x707f
32 #define INSN_MATCH_LWU 0x6003
33 #define INSN_MASK_LWU 0x707f
34 #define INSN_MATCH_SB 0x23
35 #define INSN_MASK_SB 0x707f
36 #define INSN_MATCH_SH 0x1023
37 #define INSN_MASK_SH 0x707f
38 #define INSN_MATCH_SW 0x2023
39 #define INSN_MASK_SW 0x707f
40 #define INSN_MATCH_SD 0x3023
41 #define INSN_MASK_SD 0x707f
42
43 #define INSN_MATCH_FLW 0x2007
44 #define INSN_MASK_FLW 0x707f
45 #define INSN_MATCH_FLD 0x3007
46 #define INSN_MASK_FLD 0x707f
47 #define INSN_MATCH_FLQ 0x4007
48 #define INSN_MASK_FLQ 0x707f
49 #define INSN_MATCH_FSW 0x2027
50 #define INSN_MASK_FSW 0x707f
51 #define INSN_MATCH_FSD 0x3027
52 #define INSN_MASK_FSD 0x707f
53 #define INSN_MATCH_FSQ 0x4027
54 #define INSN_MASK_FSQ 0x707f
55
56 #define INSN_MATCH_C_LD 0x6000
57 #define INSN_MASK_C_LD 0xe003
58 #define INSN_MATCH_C_SD 0xe000
59 #define INSN_MASK_C_SD 0xe003
60 #define INSN_MATCH_C_LW 0x4000
61 #define INSN_MASK_C_LW 0xe003
62 #define INSN_MATCH_C_SW 0xc000
63 #define INSN_MASK_C_SW 0xe003
64 #define INSN_MATCH_C_LDSP 0x6002
65 #define INSN_MASK_C_LDSP 0xe003
66 #define INSN_MATCH_C_SDSP 0xe002
67 #define INSN_MASK_C_SDSP 0xe003
68 #define INSN_MATCH_C_LWSP 0x4002
69 #define INSN_MASK_C_LWSP 0xe003
70 #define INSN_MATCH_C_SWSP 0xc002
71 #define INSN_MASK_C_SWSP 0xe003
72
73 #define INSN_MATCH_C_FLD 0x2000
74 #define INSN_MASK_C_FLD 0xe003
75 #define INSN_MATCH_C_FLW 0x6000
76 #define INSN_MASK_C_FLW 0xe003
77 #define INSN_MATCH_C_FSD 0xa000
78 #define INSN_MASK_C_FSD 0xe003
79 #define INSN_MATCH_C_FSW 0xe000
80 #define INSN_MASK_C_FSW 0xe003
81 #define INSN_MATCH_C_FLDSP 0x2002
82 #define INSN_MASK_C_FLDSP 0xe003
83 #define INSN_MATCH_C_FSDSP 0xa002
84 #define INSN_MASK_C_FSDSP 0xe003
85 #define INSN_MATCH_C_FLWSP 0x6002
86 #define INSN_MASK_C_FLWSP 0xe003
87 #define INSN_MATCH_C_FSWSP 0xe002
88 #define INSN_MASK_C_FSWSP 0xe003
89
90 #define INSN_LEN(insn) ((((in
91
92 #if defined(CONFIG_64BIT)
93 #define LOG_REGBYTES 3
94 #define XLEN 64
95 #else
96 #define LOG_REGBYTES 2
97 #define XLEN 32
98 #endif
99 #define REGBYTES (1 <<
100 #define XLEN_MINUS_16 ((XLEN
101
102 #define SH_RD 7
103 #define SH_RS1 15
104 #define SH_RS2 20
105 #define SH_RS2C 2
106
107 #define RV_X(x, s, n) (((x)
108 #define RVC_LW_IMM(x) ((RV_X
109 (RV_X
110 (RV_X
111 #define RVC_LD_IMM(x) ((RV_X
112 (RV_X
113 #define RVC_LWSP_IMM(x) ((RV_X
114 (RV_X
115 (RV_X
116 #define RVC_LDSP_IMM(x) ((RV_X
117 (RV_X
118 (RV_X
119 #define RVC_SWSP_IMM(x) ((RV_X
120 (RV_X
121 #define RVC_SDSP_IMM(x) ((RV_X
122 (RV_X
123 #define RVC_RS1S(insn) (8 + R
124 #define RVC_RS2S(insn) (8 + R
125 #define RVC_RS2(insn) RV_X(i
126
127 #define SHIFT_RIGHT(x, y) \
128 ((y) < 0 ? ((x) << -(y)) : ((x) >> (y)
129
130 #define REG_MASK \
131 ((1 << (5 + LOG_REGBYTES)) - (1 << LOG
132
133 #define REG_OFFSET(insn, pos) \
134 (SHIFT_RIGHT((insn), (pos) - LOG_REGBY
135
136 #define REG_PTR(insn, pos, regs) \
137 (ulong *)((ulong)(regs) + REG_OFFSET(i
138
139 #define GET_RS1(insn, regs) (*REG_
140 #define GET_RS2(insn, regs) (*REG_
141 #define GET_RS1S(insn, regs) (*REG_
142 #define GET_RS2S(insn, regs) (*REG_
143 #define GET_RS2C(insn, regs) (*REG_
144 #define GET_SP(regs) (*REG_
145 #define SET_RD(insn, regs, val) (*REG_
146 #define IMM_I(insn) ((s32)
147 #define IMM_S(insn) (((s32
148 (s32)
149 #define MASK_FUNCT3 0x7000
150
151 #define GET_PRECISION(insn) (((insn) >> 25) &
152 #define GET_RM(insn) (((insn) >> 12) & 7)
153 #define PRECISION_S 0
154 #define PRECISION_D 1
155
156 #ifdef CONFIG_FPU
157
158 #define FP_GET_RD(insn) (insn >> 7 & 0
159
160 extern void put_f32_reg(unsigned long fp_reg,
161
162 static int set_f32_rd(unsigned long insn, stru
163 unsigned long val)
164 {
165 unsigned long fp_reg = FP_GET_RD(insn)
166
167 put_f32_reg(fp_reg, val);
168 regs->status |= SR_FS_DIRTY;
169
170 return 0;
171 }
172
173 extern void put_f64_reg(unsigned long fp_reg,
174
175 static int set_f64_rd(unsigned long insn, stru
176 {
177 unsigned long fp_reg = FP_GET_RD(insn)
178 unsigned long value;
179
180 #if __riscv_xlen == 32
181 value = (unsigned long) &val;
182 #else
183 value = val;
184 #endif
185 put_f64_reg(fp_reg, value);
186 regs->status |= SR_FS_DIRTY;
187
188 return 0;
189 }
190
191 #if __riscv_xlen == 32
192 extern void get_f64_reg(unsigned long fp_reg,
193
194 static u64 get_f64_rs(unsigned long insn, u8 f
195 struct pt_regs *regs)
196 {
197 unsigned long fp_reg = (insn >> fp_reg
198 u64 val;
199
200 get_f64_reg(fp_reg, &val);
201 regs->status |= SR_FS_DIRTY;
202
203 return val;
204 }
205 #else
206
207 extern unsigned long get_f64_reg(unsigned long
208
209 static unsigned long get_f64_rs(unsigned long
210 struct pt_regs
211 {
212 unsigned long fp_reg = (insn >> fp_reg
213 unsigned long val;
214
215 val = get_f64_reg(fp_reg);
216 regs->status |= SR_FS_DIRTY;
217
218 return val;
219 }
220
221 #endif
222
223 extern unsigned long get_f32_reg(unsigned long
224
225 static unsigned long get_f32_rs(unsigned long
226 struct pt_regs
227 {
228 unsigned long fp_reg = (insn >> fp_reg
229 unsigned long val;
230
231 val = get_f32_reg(fp_reg);
232 regs->status |= SR_FS_DIRTY;
233
234 return val;
235 }
236
237 #else /* CONFIG_FPU */
238 static void set_f32_rd(unsigned long insn, str
239 unsigned long val) {}
240
241 static void set_f64_rd(unsigned long insn, str
242
243 static unsigned long get_f64_rs(unsigned long
244 struct pt_regs
245 {
246 return 0;
247 }
248
249 static unsigned long get_f32_rs(unsigned long
250 struct pt_regs
251 {
252 return 0;
253 }
254
255 #endif
256
257 #define GET_F64_RS2(insn, regs) (get_f64_rs(in
258 #define GET_F64_RS2C(insn, regs) (get_f64_rs(i
259 #define GET_F64_RS2S(insn, regs) (get_f64_rs(R
260
261 #define GET_F32_RS2(insn, regs) (get_f32_rs(in
262 #define GET_F32_RS2C(insn, regs) (get_f32_rs(i
263 #define GET_F32_RS2S(insn, regs) (get_f32_rs(R
264
265 #define __read_insn(regs, insn, insn_addr, typ
266 ({
267 int __ret;
268
269 if (user_mode(regs)) {
270 __ret = __get_user(insn, (type
271 } else {
272 insn = *(type *)insn_addr;
273 __ret = 0;
274 }
275
276 __ret;
277 })
278
279 static inline int get_insn(struct pt_regs *reg
280 {
281 ulong insn = 0;
282
283 if (epc & 0x2) {
284 ulong tmp = 0;
285
286 if (__read_insn(regs, insn, ep
287 return -EFAULT;
288 /* __get_user() uses regular "
289 * value make sure to clear hi
290 * below with the upper 16 bit
291 */
292 insn &= GENMASK(15, 0);
293 if ((insn & __INSN_LENGTH_MASK
294 *r_insn = insn;
295 return 0;
296 }
297 epc += sizeof(u16);
298 if (__read_insn(regs, tmp, epc
299 return -EFAULT;
300 *r_insn = (tmp << 16) | insn;
301
302 return 0;
303 } else {
304 if (__read_insn(regs, insn, ep
305 return -EFAULT;
306 if ((insn & __INSN_LENGTH_MASK
307 *r_insn = insn;
308 return 0;
309 }
310 insn &= GENMASK(15, 0);
311 *r_insn = insn;
312
313 return 0;
314 }
315 }
316
317 union reg_data {
318 u8 data_bytes[8];
319 ulong data_ulong;
320 u64 data_u64;
321 };
322
323 static bool unaligned_ctl __read_mostly;
324
325 /* sysctl hooks */
326 int unaligned_enabled __read_mostly = 1;
327
328 int handle_misaligned_load(struct pt_regs *reg
329 {
330 union reg_data val;
331 unsigned long epc = regs->epc;
332 unsigned long insn;
333 unsigned long addr = regs->badaddr;
334 int fp = 0, shift = 0, len = 0;
335
336 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_
337
338 #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
339 *this_cpu_ptr(&misaligned_access_speed
340 #endif
341
342 if (!unaligned_enabled)
343 return -1;
344
345 if (user_mode(regs) && (current->threa
346 return -1;
347
348 if (get_insn(regs, epc, &insn))
349 return -1;
350
351 regs->epc = 0;
352
353 if ((insn & INSN_MASK_LW) == INSN_MATC
354 len = 4;
355 shift = 8 * (sizeof(unsigned l
356 #if defined(CONFIG_64BIT)
357 } else if ((insn & INSN_MASK_LD) == IN
358 len = 8;
359 shift = 8 * (sizeof(unsigned l
360 } else if ((insn & INSN_MASK_LWU) == I
361 len = 4;
362 #endif
363 } else if ((insn & INSN_MASK_FLD) == I
364 fp = 1;
365 len = 8;
366 } else if ((insn & INSN_MASK_FLW) == I
367 fp = 1;
368 len = 4;
369 } else if ((insn & INSN_MASK_LH) == IN
370 len = 2;
371 shift = 8 * (sizeof(unsigned l
372 } else if ((insn & INSN_MASK_LHU) == I
373 len = 2;
374 #if defined(CONFIG_64BIT)
375 } else if ((insn & INSN_MASK_C_LD) ==
376 len = 8;
377 shift = 8 * (sizeof(unsigned l
378 insn = RVC_RS2S(insn) << SH_RD
379 } else if ((insn & INSN_MASK_C_LDSP) =
380 ((insn >> SH_RD) & 0x1f)) {
381 len = 8;
382 shift = 8 * (sizeof(unsigned l
383 #endif
384 } else if ((insn & INSN_MASK_C_LW) ==
385 len = 4;
386 shift = 8 * (sizeof(unsigned l
387 insn = RVC_RS2S(insn) << SH_RD
388 } else if ((insn & INSN_MASK_C_LWSP) =
389 ((insn >> SH_RD) & 0x1f)) {
390 len = 4;
391 shift = 8 * (sizeof(unsigned l
392 } else if ((insn & INSN_MASK_C_FLD) ==
393 fp = 1;
394 len = 8;
395 insn = RVC_RS2S(insn) << SH_RD
396 } else if ((insn & INSN_MASK_C_FLDSP)
397 fp = 1;
398 len = 8;
399 #if defined(CONFIG_32BIT)
400 } else if ((insn & INSN_MASK_C_FLW) ==
401 fp = 1;
402 len = 4;
403 insn = RVC_RS2S(insn) << SH_RD
404 } else if ((insn & INSN_MASK_C_FLWSP)
405 fp = 1;
406 len = 4;
407 #endif
408 } else {
409 regs->epc = epc;
410 return -1;
411 }
412
413 if (!IS_ENABLED(CONFIG_FPU) && fp)
414 return -EOPNOTSUPP;
415
416 val.data_u64 = 0;
417 if (user_mode(regs)) {
418 if (copy_from_user(&val, (u8 _
419 return -1;
420 } else {
421 memcpy(&val, (u8 *)addr, len);
422 }
423
424 if (!fp)
425 SET_RD(insn, regs, val.data_ul
426 else if (len == 8)
427 set_f64_rd(insn, regs, val.dat
428 else
429 set_f32_rd(insn, regs, val.dat
430
431 regs->epc = epc + INSN_LEN(insn);
432
433 return 0;
434 }
435
436 int handle_misaligned_store(struct pt_regs *re
437 {
438 union reg_data val;
439 unsigned long epc = regs->epc;
440 unsigned long insn;
441 unsigned long addr = regs->badaddr;
442 int len = 0, fp = 0;
443
444 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_
445
446 if (!unaligned_enabled)
447 return -1;
448
449 if (user_mode(regs) && (current->threa
450 return -1;
451
452 if (get_insn(regs, epc, &insn))
453 return -1;
454
455 regs->epc = 0;
456
457 val.data_ulong = GET_RS2(insn, regs);
458
459 if ((insn & INSN_MASK_SW) == INSN_MATC
460 len = 4;
461 #if defined(CONFIG_64BIT)
462 } else if ((insn & INSN_MASK_SD) == IN
463 len = 8;
464 #endif
465 } else if ((insn & INSN_MASK_FSD) == I
466 fp = 1;
467 len = 8;
468 val.data_u64 = GET_F64_RS2(ins
469 } else if ((insn & INSN_MASK_FSW) == I
470 fp = 1;
471 len = 4;
472 val.data_ulong = GET_F32_RS2(i
473 } else if ((insn & INSN_MASK_SH) == IN
474 len = 2;
475 #if defined(CONFIG_64BIT)
476 } else if ((insn & INSN_MASK_C_SD) ==
477 len = 8;
478 val.data_ulong = GET_RS2S(insn
479 } else if ((insn & INSN_MASK_C_SDSP) =
480 len = 8;
481 val.data_ulong = GET_RS2C(insn
482 #endif
483 } else if ((insn & INSN_MASK_C_SW) ==
484 len = 4;
485 val.data_ulong = GET_RS2S(insn
486 } else if ((insn & INSN_MASK_C_SWSP) =
487 len = 4;
488 val.data_ulong = GET_RS2C(insn
489 } else if ((insn & INSN_MASK_C_FSD) ==
490 fp = 1;
491 len = 8;
492 val.data_u64 = GET_F64_RS2S(in
493 } else if ((insn & INSN_MASK_C_FSDSP)
494 fp = 1;
495 len = 8;
496 val.data_u64 = GET_F64_RS2C(in
497 #if !defined(CONFIG_64BIT)
498 } else if ((insn & INSN_MASK_C_FSW) ==
499 fp = 1;
500 len = 4;
501 val.data_ulong = GET_F32_RS2S(
502 } else if ((insn & INSN_MASK_C_FSWSP)
503 fp = 1;
504 len = 4;
505 val.data_ulong = GET_F32_RS2C(
506 #endif
507 } else {
508 regs->epc = epc;
509 return -1;
510 }
511
512 if (!IS_ENABLED(CONFIG_FPU) && fp)
513 return -EOPNOTSUPP;
514
515 if (user_mode(regs)) {
516 if (copy_to_user((u8 __user *)
517 return -1;
518 } else {
519 memcpy((u8 *)addr, &val, len);
520 }
521
522 regs->epc = epc + INSN_LEN(insn);
523
524 return 0;
525 }
526
527 static bool check_unaligned_access_emulated(in
528 {
529 long *mas_ptr = per_cpu_ptr(&misaligne
530 unsigned long tmp_var, tmp_val;
531 bool misaligned_emu_detected;
532
533 *mas_ptr = RISCV_HWPROBE_MISALIGNED_SC
534
535 __asm__ __volatile__ (
536 " "REG_L" %[tmp], 1(%[pt
537 : [tmp] "=r" (tmp_val) : [ptr]
538
539 misaligned_emu_detected = (*mas_ptr ==
540 /*
541 * If unaligned_ctl is already set, th
542 * CPUS uses emulated misaligned acces
543 * when hotplugging the new cpu, this
544 */
545 if (unlikely(unaligned_ctl && !misalig
546 pr_crit("CPU misaligned access
547 while (true)
548 cpu_relax();
549 }
550
551 return misaligned_emu_detected;
552 }
553
554 bool check_unaligned_access_emulated_all_cpus(
555 {
556 int cpu;
557
558 /*
559 * We can only support PR_UNALIGN cont
560 * accesses emulated since tasks reque
561 * CPU.
562 */
563 for_each_online_cpu(cpu)
564 if (!check_unaligned_access_em
565 return false;
566
567 unaligned_ctl = true;
568 return true;
569 }
570
571 bool unaligned_ctl_available(void)
572 {
573 return unaligned_ctl;
574 }
575
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.