1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Author: Lu Zeng <zenglu@loongson.cn>
4 * Pei Huang <huangpei@loongson.cn>
5 * Huacai Chen <chenhuacai@loongson.cn>
6 *
7 * Copyright (C) 2020-2022 Loongson Technology
8 */
9 #include <linux/export.h>
10 #include <asm/asm.h>
11 #include <asm/asmmacro.h>
12 #include <asm/asm-extable.h>
13 #include <asm/asm-offsets.h>
14 #include <asm/errno.h>
15 #include <asm/fpregdef.h>
16 #include <asm/loongarch.h>
17 #include <asm/regdef.h>
18 #include <asm/unwind_hints.h>
19
20 #define FPU_REG_WIDTH 8
21 #define LSX_REG_WIDTH 16
22 #define LASX_REG_WIDTH 32
23
24 .macro EX insn, reg, src, offs
25 .ex\@: \insn \reg, \src, \offs
26 _asm_extable .ex\@, .L_fpu_fault
27 .endm
28
29 .macro sc_save_fp base
30 EX fst.d $f0, \base, (0 * FPU_
31 EX fst.d $f1, \base, (1 * FPU_
32 EX fst.d $f2, \base, (2 * FPU_
33 EX fst.d $f3, \base, (3 * FPU_
34 EX fst.d $f4, \base, (4 * FPU_
35 EX fst.d $f5, \base, (5 * FPU_
36 EX fst.d $f6, \base, (6 * FPU_
37 EX fst.d $f7, \base, (7 * FPU_
38 EX fst.d $f8, \base, (8 * FPU_
39 EX fst.d $f9, \base, (9 * FPU_
40 EX fst.d $f10, \base, (10 * FPU
41 EX fst.d $f11, \base, (11 * FPU
42 EX fst.d $f12, \base, (12 * FPU
43 EX fst.d $f13, \base, (13 * FPU
44 EX fst.d $f14, \base, (14 * FPU
45 EX fst.d $f15, \base, (15 * FPU
46 EX fst.d $f16, \base, (16 * FPU
47 EX fst.d $f17, \base, (17 * FPU
48 EX fst.d $f18, \base, (18 * FPU
49 EX fst.d $f19, \base, (19 * FPU
50 EX fst.d $f20, \base, (20 * FPU
51 EX fst.d $f21, \base, (21 * FPU
52 EX fst.d $f22, \base, (22 * FPU
53 EX fst.d $f23, \base, (23 * FPU
54 EX fst.d $f24, \base, (24 * FPU
55 EX fst.d $f25, \base, (25 * FPU
56 EX fst.d $f26, \base, (26 * FPU
57 EX fst.d $f27, \base, (27 * FPU
58 EX fst.d $f28, \base, (28 * FPU
59 EX fst.d $f29, \base, (29 * FPU
60 EX fst.d $f30, \base, (30 * FPU
61 EX fst.d $f31, \base, (31 * FPU
62 .endm
63
64 .macro sc_restore_fp base
65 EX fld.d $f0, \base, (0 * FPU_
66 EX fld.d $f1, \base, (1 * FPU_
67 EX fld.d $f2, \base, (2 * FPU_
68 EX fld.d $f3, \base, (3 * FPU_
69 EX fld.d $f4, \base, (4 * FPU_
70 EX fld.d $f5, \base, (5 * FPU_
71 EX fld.d $f6, \base, (6 * FPU_
72 EX fld.d $f7, \base, (7 * FPU_
73 EX fld.d $f8, \base, (8 * FPU_
74 EX fld.d $f9, \base, (9 * FPU_
75 EX fld.d $f10, \base, (10 * FPU
76 EX fld.d $f11, \base, (11 * FPU
77 EX fld.d $f12, \base, (12 * FPU
78 EX fld.d $f13, \base, (13 * FPU
79 EX fld.d $f14, \base, (14 * FPU
80 EX fld.d $f15, \base, (15 * FPU
81 EX fld.d $f16, \base, (16 * FPU
82 EX fld.d $f17, \base, (17 * FPU
83 EX fld.d $f18, \base, (18 * FPU
84 EX fld.d $f19, \base, (19 * FPU
85 EX fld.d $f20, \base, (20 * FPU
86 EX fld.d $f21, \base, (21 * FPU
87 EX fld.d $f22, \base, (22 * FPU
88 EX fld.d $f23, \base, (23 * FPU
89 EX fld.d $f24, \base, (24 * FPU
90 EX fld.d $f25, \base, (25 * FPU
91 EX fld.d $f26, \base, (26 * FPU
92 EX fld.d $f27, \base, (27 * FPU
93 EX fld.d $f28, \base, (28 * FPU
94 EX fld.d $f29, \base, (29 * FPU
95 EX fld.d $f30, \base, (30 * FPU
96 EX fld.d $f31, \base, (31 * FPU
97 .endm
98
99 .macro sc_save_fcc base, tmp0, tmp1
100 movcf2gr \tmp0, $fcc0
101 move \tmp1, \tmp0
102 movcf2gr \tmp0, $fcc1
103 bstrins.d \tmp1, \tmp0, 15, 8
104 movcf2gr \tmp0, $fcc2
105 bstrins.d \tmp1, \tmp0, 23, 16
106 movcf2gr \tmp0, $fcc3
107 bstrins.d \tmp1, \tmp0, 31, 24
108 movcf2gr \tmp0, $fcc4
109 bstrins.d \tmp1, \tmp0, 39, 32
110 movcf2gr \tmp0, $fcc5
111 bstrins.d \tmp1, \tmp0, 47, 40
112 movcf2gr \tmp0, $fcc6
113 bstrins.d \tmp1, \tmp0, 55, 48
114 movcf2gr \tmp0, $fcc7
115 bstrins.d \tmp1, \tmp0, 63, 56
116 EX st.d \tmp1, \base, 0
117 .endm
118
119 .macro sc_restore_fcc base, tmp0, tmp1
120 EX ld.d \tmp0, \base, 0
121 bstrpick.d \tmp1, \tmp0, 7, 0
122 movgr2cf $fcc0, \tmp1
123 bstrpick.d \tmp1, \tmp0, 15, 8
124 movgr2cf $fcc1, \tmp1
125 bstrpick.d \tmp1, \tmp0, 23, 16
126 movgr2cf $fcc2, \tmp1
127 bstrpick.d \tmp1, \tmp0, 31, 24
128 movgr2cf $fcc3, \tmp1
129 bstrpick.d \tmp1, \tmp0, 39, 32
130 movgr2cf $fcc4, \tmp1
131 bstrpick.d \tmp1, \tmp0, 47, 40
132 movgr2cf $fcc5, \tmp1
133 bstrpick.d \tmp1, \tmp0, 55, 48
134 movgr2cf $fcc6, \tmp1
135 bstrpick.d \tmp1, \tmp0, 63, 56
136 movgr2cf $fcc7, \tmp1
137 .endm
138
139 .macro sc_save_fcsr base, tmp0
140 movfcsr2gr \tmp0, fcsr0
141 EX st.w \tmp0, \base, 0
142 #if defined(CONFIG_CPU_HAS_LBT)
143 /* TM bit is always 0 if LBT not suppo
144 andi \tmp0, \tmp0, FPU_CSR_
145 beqz \tmp0, 1f
146 x86clrtm
147 1:
148 #endif
149 .endm
150
151 .macro sc_restore_fcsr base, tmp0
152 EX ld.w \tmp0, \base, 0
153 movgr2fcsr fcsr0, \tmp0
154 .endm
155
156 .macro sc_save_lsx base
157 #ifdef CONFIG_CPU_HAS_LSX
158 EX vst $vr0, \base, (0 * LSX
159 EX vst $vr1, \base, (1 * LSX
160 EX vst $vr2, \base, (2 * LSX
161 EX vst $vr3, \base, (3 * LSX
162 EX vst $vr4, \base, (4 * LSX
163 EX vst $vr5, \base, (5 * LSX
164 EX vst $vr6, \base, (6 * LSX
165 EX vst $vr7, \base, (7 * LSX
166 EX vst $vr8, \base, (8 * LSX
167 EX vst $vr9, \base, (9 * LSX
168 EX vst $vr10, \base, (10 * LS
169 EX vst $vr11, \base, (11 * LS
170 EX vst $vr12, \base, (12 * LS
171 EX vst $vr13, \base, (13 * LS
172 EX vst $vr14, \base, (14 * LS
173 EX vst $vr15, \base, (15 * LS
174 EX vst $vr16, \base, (16 * LS
175 EX vst $vr17, \base, (17 * LS
176 EX vst $vr18, \base, (18 * LS
177 EX vst $vr19, \base, (19 * LS
178 EX vst $vr20, \base, (20 * LS
179 EX vst $vr21, \base, (21 * LS
180 EX vst $vr22, \base, (22 * LS
181 EX vst $vr23, \base, (23 * LS
182 EX vst $vr24, \base, (24 * LS
183 EX vst $vr25, \base, (25 * LS
184 EX vst $vr26, \base, (26 * LS
185 EX vst $vr27, \base, (27 * LS
186 EX vst $vr28, \base, (28 * LS
187 EX vst $vr29, \base, (29 * LS
188 EX vst $vr30, \base, (30 * LS
189 EX vst $vr31, \base, (31 * LS
190 #endif
191 .endm
192
193 .macro sc_restore_lsx base
194 #ifdef CONFIG_CPU_HAS_LSX
195 EX vld $vr0, \base, (0 * LSX
196 EX vld $vr1, \base, (1 * LSX
197 EX vld $vr2, \base, (2 * LSX
198 EX vld $vr3, \base, (3 * LSX
199 EX vld $vr4, \base, (4 * LSX
200 EX vld $vr5, \base, (5 * LSX
201 EX vld $vr6, \base, (6 * LSX
202 EX vld $vr7, \base, (7 * LSX
203 EX vld $vr8, \base, (8 * LSX
204 EX vld $vr9, \base, (9 * LSX
205 EX vld $vr10, \base, (10 * LS
206 EX vld $vr11, \base, (11 * LS
207 EX vld $vr12, \base, (12 * LS
208 EX vld $vr13, \base, (13 * LS
209 EX vld $vr14, \base, (14 * LS
210 EX vld $vr15, \base, (15 * LS
211 EX vld $vr16, \base, (16 * LS
212 EX vld $vr17, \base, (17 * LS
213 EX vld $vr18, \base, (18 * LS
214 EX vld $vr19, \base, (19 * LS
215 EX vld $vr20, \base, (20 * LS
216 EX vld $vr21, \base, (21 * LS
217 EX vld $vr22, \base, (22 * LS
218 EX vld $vr23, \base, (23 * LS
219 EX vld $vr24, \base, (24 * LS
220 EX vld $vr25, \base, (25 * LS
221 EX vld $vr26, \base, (26 * LS
222 EX vld $vr27, \base, (27 * LS
223 EX vld $vr28, \base, (28 * LS
224 EX vld $vr29, \base, (29 * LS
225 EX vld $vr30, \base, (30 * LS
226 EX vld $vr31, \base, (31 * LS
227 #endif
228 .endm
229
230 .macro sc_save_lasx base
231 #ifdef CONFIG_CPU_HAS_LASX
232 EX xvst $xr0, \base, (0 * LAS
233 EX xvst $xr1, \base, (1 * LAS
234 EX xvst $xr2, \base, (2 * LAS
235 EX xvst $xr3, \base, (3 * LAS
236 EX xvst $xr4, \base, (4 * LAS
237 EX xvst $xr5, \base, (5 * LAS
238 EX xvst $xr6, \base, (6 * LAS
239 EX xvst $xr7, \base, (7 * LAS
240 EX xvst $xr8, \base, (8 * LAS
241 EX xvst $xr9, \base, (9 * LAS
242 EX xvst $xr10, \base, (10 * LA
243 EX xvst $xr11, \base, (11 * LA
244 EX xvst $xr12, \base, (12 * LA
245 EX xvst $xr13, \base, (13 * LA
246 EX xvst $xr14, \base, (14 * LA
247 EX xvst $xr15, \base, (15 * LA
248 EX xvst $xr16, \base, (16 * LA
249 EX xvst $xr17, \base, (17 * LA
250 EX xvst $xr18, \base, (18 * LA
251 EX xvst $xr19, \base, (19 * LA
252 EX xvst $xr20, \base, (20 * LA
253 EX xvst $xr21, \base, (21 * LA
254 EX xvst $xr22, \base, (22 * LA
255 EX xvst $xr23, \base, (23 * LA
256 EX xvst $xr24, \base, (24 * LA
257 EX xvst $xr25, \base, (25 * LA
258 EX xvst $xr26, \base, (26 * LA
259 EX xvst $xr27, \base, (27 * LA
260 EX xvst $xr28, \base, (28 * LA
261 EX xvst $xr29, \base, (29 * LA
262 EX xvst $xr30, \base, (30 * LA
263 EX xvst $xr31, \base, (31 * LA
264 #endif
265 .endm
266
267 .macro sc_restore_lasx base
268 #ifdef CONFIG_CPU_HAS_LASX
269 EX xvld $xr0, \base, (0 * LAS
270 EX xvld $xr1, \base, (1 * LAS
271 EX xvld $xr2, \base, (2 * LAS
272 EX xvld $xr3, \base, (3 * LAS
273 EX xvld $xr4, \base, (4 * LAS
274 EX xvld $xr5, \base, (5 * LAS
275 EX xvld $xr6, \base, (6 * LAS
276 EX xvld $xr7, \base, (7 * LAS
277 EX xvld $xr8, \base, (8 * LAS
278 EX xvld $xr9, \base, (9 * LAS
279 EX xvld $xr10, \base, (10 * LA
280 EX xvld $xr11, \base, (11 * LA
281 EX xvld $xr12, \base, (12 * LA
282 EX xvld $xr13, \base, (13 * LA
283 EX xvld $xr14, \base, (14 * LA
284 EX xvld $xr15, \base, (15 * LA
285 EX xvld $xr16, \base, (16 * LA
286 EX xvld $xr17, \base, (17 * LA
287 EX xvld $xr18, \base, (18 * LA
288 EX xvld $xr19, \base, (19 * LA
289 EX xvld $xr20, \base, (20 * LA
290 EX xvld $xr21, \base, (21 * LA
291 EX xvld $xr22, \base, (22 * LA
292 EX xvld $xr23, \base, (23 * LA
293 EX xvld $xr24, \base, (24 * LA
294 EX xvld $xr25, \base, (25 * LA
295 EX xvld $xr26, \base, (26 * LA
296 EX xvld $xr27, \base, (27 * LA
297 EX xvld $xr28, \base, (28 * LA
298 EX xvld $xr29, \base, (29 * LA
299 EX xvld $xr30, \base, (30 * LA
300 EX xvld $xr31, \base, (31 * LA
301 #endif
302 .endm
303
304 /*
305 * Save a thread's fp context.
306 */
307 SYM_FUNC_START(_save_fp)
308 fpu_save_csr a0 t1
309 fpu_save_double a0 t1
310 fpu_save_cc a0 t1 t2
311 jr ra
312 SYM_FUNC_END(_save_fp)
313 EXPORT_SYMBOL(_save_fp)
314
315 /*
316 * Restore a thread's fp context.
317 */
318 SYM_FUNC_START(_restore_fp)
319 fpu_restore_double a0 t1
320 fpu_restore_csr a0 t1 t2
321 fpu_restore_cc a0 t1 t2
322 jr ra
323 SYM_FUNC_END(_restore_fp)
324
325 #ifdef CONFIG_CPU_HAS_LSX
326
327 /*
328 * Save a thread's LSX vector context.
329 */
330 SYM_FUNC_START(_save_lsx)
331 lsx_save_all a0 t1 t2
332 jr ra
333 SYM_FUNC_END(_save_lsx)
334 EXPORT_SYMBOL(_save_lsx)
335
336 /*
337 * Restore a thread's LSX vector context.
338 */
339 SYM_FUNC_START(_restore_lsx)
340 lsx_restore_all a0 t1 t2
341 jr ra
342 SYM_FUNC_END(_restore_lsx)
343
344 SYM_FUNC_START(_save_lsx_upper)
345 lsx_save_all_upper a0 t0 t1
346 jr ra
347 SYM_FUNC_END(_save_lsx_upper)
348
349 SYM_FUNC_START(_restore_lsx_upper)
350 lsx_restore_all_upper a0 t0 t1
351 jr ra
352 SYM_FUNC_END(_restore_lsx_upper)
353 EXPORT_SYMBOL(_restore_lsx_upper)
354
355 SYM_FUNC_START(_init_lsx_upper)
356 lsx_init_all_upper t1
357 jr ra
358 SYM_FUNC_END(_init_lsx_upper)
359 #endif
360
361 #ifdef CONFIG_CPU_HAS_LASX
362
363 /*
364 * Save a thread's LASX vector context.
365 */
366 SYM_FUNC_START(_save_lasx)
367 lasx_save_all a0 t1 t2
368 jr ra
369 SYM_FUNC_END(_save_lasx)
370 EXPORT_SYMBOL(_save_lasx)
371
372 /*
373 * Restore a thread's LASX vector context.
374 */
375 SYM_FUNC_START(_restore_lasx)
376 lasx_restore_all a0 t1 t2
377 jr ra
378 SYM_FUNC_END(_restore_lasx)
379
380 SYM_FUNC_START(_save_lasx_upper)
381 lasx_save_all_upper a0 t0 t1
382 jr ra
383 SYM_FUNC_END(_save_lasx_upper)
384
385 SYM_FUNC_START(_restore_lasx_upper)
386 lasx_restore_all_upper a0 t0 t1
387 jr ra
388 SYM_FUNC_END(_restore_lasx_upper)
389 EXPORT_SYMBOL(_restore_lasx_upper)
390
391 SYM_FUNC_START(_init_lasx_upper)
392 lasx_init_all_upper t1
393 jr ra
394 SYM_FUNC_END(_init_lasx_upper)
395 #endif
396
397 /*
398 * Load the FPU with signalling NANS. This bi
399 * the property that no matter whether conside
400 * precision represents signaling NANS.
401 *
402 * The value to initialize fcsr0 to comes in $
403 */
404
405 SYM_FUNC_START(_init_fpu)
406 li.w t1, CSR_EUEN_FPEN
407 csrxchg t1, t1, LOONGARCH_CSR_EUEN
408
409 movgr2fcsr fcsr0, a0
410
411 li.w t1, -1
412
413 movgr2fr.d $f0, t1
414 movgr2fr.d $f1, t1
415 movgr2fr.d $f2, t1
416 movgr2fr.d $f3, t1
417 movgr2fr.d $f4, t1
418 movgr2fr.d $f5, t1
419 movgr2fr.d $f6, t1
420 movgr2fr.d $f7, t1
421 movgr2fr.d $f8, t1
422 movgr2fr.d $f9, t1
423 movgr2fr.d $f10, t1
424 movgr2fr.d $f11, t1
425 movgr2fr.d $f12, t1
426 movgr2fr.d $f13, t1
427 movgr2fr.d $f14, t1
428 movgr2fr.d $f15, t1
429 movgr2fr.d $f16, t1
430 movgr2fr.d $f17, t1
431 movgr2fr.d $f18, t1
432 movgr2fr.d $f19, t1
433 movgr2fr.d $f20, t1
434 movgr2fr.d $f21, t1
435 movgr2fr.d $f22, t1
436 movgr2fr.d $f23, t1
437 movgr2fr.d $f24, t1
438 movgr2fr.d $f25, t1
439 movgr2fr.d $f26, t1
440 movgr2fr.d $f27, t1
441 movgr2fr.d $f28, t1
442 movgr2fr.d $f29, t1
443 movgr2fr.d $f30, t1
444 movgr2fr.d $f31, t1
445
446 jr ra
447 SYM_FUNC_END(_init_fpu)
448
449 /*
450 * a0: fpregs
451 * a1: fcc
452 * a2: fcsr
453 */
454 SYM_FUNC_START(_save_fp_context)
455 sc_save_fcc a1 t1 t2
456 sc_save_fcsr a2 t1
457 sc_save_fp a0
458 li.w a0, 0
459 jr ra
460 SYM_FUNC_END(_save_fp_context)
461
462 /*
463 * a0: fpregs
464 * a1: fcc
465 * a2: fcsr
466 */
467 SYM_FUNC_START(_restore_fp_context)
468 sc_restore_fp a0
469 sc_restore_fcc a1 t1 t2
470 sc_restore_fcsr a2 t1
471 li.w a0, 0
472 jr ra
473 SYM_FUNC_END(_restore_fp_context)
474
475 /*
476 * a0: fpregs
477 * a1: fcc
478 * a2: fcsr
479 */
480 SYM_FUNC_START(_save_lsx_context)
481 sc_save_fcc a1, t0, t1
482 sc_save_fcsr a2, t0
483 sc_save_lsx a0
484 li.w a0, 0
485 jr ra
486 SYM_FUNC_END(_save_lsx_context)
487
488 /*
489 * a0: fpregs
490 * a1: fcc
491 * a2: fcsr
492 */
493 SYM_FUNC_START(_restore_lsx_context)
494 sc_restore_lsx a0
495 sc_restore_fcc a1, t1, t2
496 sc_restore_fcsr a2, t1
497 li.w a0, 0
498 jr ra
499 SYM_FUNC_END(_restore_lsx_context)
500
501 /*
502 * a0: fpregs
503 * a1: fcc
504 * a2: fcsr
505 */
506 SYM_FUNC_START(_save_lasx_context)
507 sc_save_fcc a1, t0, t1
508 sc_save_fcsr a2, t0
509 sc_save_lasx a0
510 li.w a0, 0
511 jr ra
512 SYM_FUNC_END(_save_lasx_context)
513
514 /*
515 * a0: fpregs
516 * a1: fcc
517 * a2: fcsr
518 */
519 SYM_FUNC_START(_restore_lasx_context)
520 sc_restore_lasx a0
521 sc_restore_fcc a1, t1, t2
522 sc_restore_fcsr a2, t1
523 li.w a0, 0
524 jr ra
525 SYM_FUNC_END(_restore_lasx_context)
526
527 .L_fpu_fault:
528 li.w a0, -EFAULT
529 jr ra
530
531 #ifdef CONFIG_CPU_HAS_LBT
532 STACK_FRAME_NON_STANDARD _restore_fp
533 #ifdef CONFIG_CPU_HAS_LSX
534 STACK_FRAME_NON_STANDARD _restore_lsx
535 #endif
536 #ifdef CONFIG_CPU_HAS_LASX
537 STACK_FRAME_NON_STANDARD _restore_lasx
538 #endif
539 #endif
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.