1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* !! 2 /* memmove.S: Simple memmove implementation.
3 * Copyright (C) 2020-2022 Loongson Technology !! 3 *
>> 4 * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com)
>> 5 * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz)
4 */ 6 */
5 7
6 #include <linux/export.h> 8 #include <linux/export.h>
7 #include <asm/alternative-asm.h> !! 9 #include <linux/linkage.h>
8 #include <asm/asm.h> <<
9 #include <asm/asmmacro.h> <<
10 #include <asm/cpu.h> <<
11 #include <asm/regdef.h> <<
12 <<
13 .section .noinstr.text, "ax" <<
14 <<
15 SYM_FUNC_START(memmove) <<
16 blt a0, a1, __memcpy /* dst <<
17 blt a1, a0, __rmemcpy /* src <<
18 jr ra /* dst <<
19 SYM_FUNC_END(memmove) <<
20 SYM_FUNC_ALIAS(__memmove, memmove) <<
21 <<
22 EXPORT_SYMBOL(memmove) <<
23 EXPORT_SYMBOL(__memmove) <<
24 10
25 _ASM_NOKPROBE(memmove) !! 11 .text
26 _ASM_NOKPROBE(__memmove) !! 12 ENTRY(memmove) /* o0=dst o1=src o2=len */
27 !! 13 brz,pn %o2, 99f
28 SYM_FUNC_START(__rmemcpy) !! 14 mov %o0, %g1
29 /* !! 15
30 * Some CPUs support hardware unaligne !! 16 cmp %o0, %o1
>> 17 bleu,pt %xcc, 2f
>> 18 add %o1, %o2, %g7
>> 19 cmp %g7, %o0
>> 20 bleu,pt %xcc, memcpy
>> 21 add %o0, %o2, %o5
>> 22 sub %g7, 1, %o1
>> 23
>> 24 sub %o5, 1, %o0
>> 25 1: ldub [%o1], %g7
>> 26 subcc %o2, 1, %o2
>> 27 sub %o1, 1, %o1
>> 28 stb %g7, [%o0]
>> 29 bne,pt %icc, 1b
>> 30 sub %o0, 1, %o0
>> 31 99:
>> 32 retl
>> 33 mov %g1, %o0
>> 34
>> 35 /* We can't just call memcpy for these memmove cases. On some
>> 36 * chips the memcpy uses cache initializing stores and when dst
>> 37 * and src are close enough, those can clobber the source data
>> 38 * before we've loaded it in.
31 */ 39 */
32 ALTERNATIVE "b __rmemcpy_generic", !! 40 2: or %o0, %o1, %g7
33 "b __rmemcpy_fast", CP !! 41 or %o2, %g7, %g7
34 SYM_FUNC_END(__rmemcpy) !! 42 andcc %g7, 0x7, %g0
35 _ASM_NOKPROBE(__rmemcpy) !! 43 bne,pn %xcc, 4f
36 !! 44 nop
37 /* !! 45
38 * void *__rmemcpy_generic(void *dst, const vo !! 46 3: ldx [%o1], %g7
39 * !! 47 add %o1, 8, %o1
40 * a0: dst !! 48 subcc %o2, 8, %o2
41 * a1: src !! 49 add %o0, 8, %o0
42 * a2: n !! 50 bne,pt %icc, 3b
43 */ !! 51 stx %g7, [%o0 - 0x8]
44 SYM_FUNC_START(__rmemcpy_generic) !! 52 ba,a,pt %xcc, 99b
45 move a3, a0 !! 53
46 beqz a2, 2f !! 54 4: ldub [%o1], %g7
47 !! 55 add %o1, 1, %o1
48 add.d a0, a0, a2 !! 56 subcc %o2, 1, %o2
49 add.d a1, a1, a2 !! 57 add %o0, 1, %o0
50 !! 58 bne,pt %icc, 4b
51 1: ld.b t0, a1, -1 !! 59 stb %g7, [%o0 - 0x1]
52 st.b t0, a0, -1 !! 60 ba,a,pt %xcc, 99b
53 addi.d a0, a0, -1 !! 61 ENDPROC(memmove)
54 addi.d a1, a1, -1 !! 62 EXPORT_SYMBOL(memmove)
55 addi.d a2, a2, -1 <<
56 bgt a2, zero, 1b <<
57 <<
58 2: move a0, a3 <<
59 jr ra <<
60 SYM_FUNC_END(__rmemcpy_generic) <<
61 _ASM_NOKPROBE(__rmemcpy_generic) <<
62 <<
63 /* <<
64 * void *__rmemcpy_fast(void *dst, const void <<
65 * <<
66 * a0: dst <<
67 * a1: src <<
68 * a2: n <<
69 */ <<
70 SYM_FUNC_START(__rmemcpy_fast) <<
71 sltui t0, a2, 9 <<
72 bnez t0, __memcpy_small <<
73 <<
74 add.d a3, a1, a2 <<
75 add.d a2, a0, a2 <<
76 ld.d a6, a1, 0 <<
77 ld.d a7, a3, -8 <<
78 <<
79 /* align up destination address */ <<
80 andi t1, a2, 7 <<
81 sub.d a3, a3, t1 <<
82 sub.d a5, a2, t1 <<
83 <<
84 addi.d a4, a1, 64 <<
85 bgeu a4, a3, .Llt64 <<
86 <<
87 /* copy 64 bytes at a time */ <<
88 .Lloop64: <<
89 ld.d t0, a3, -8 <<
90 ld.d t1, a3, -16 <<
91 ld.d t2, a3, -24 <<
92 ld.d t3, a3, -32 <<
93 ld.d t4, a3, -40 <<
94 ld.d t5, a3, -48 <<
95 ld.d t6, a3, -56 <<
96 ld.d t7, a3, -64 <<
97 addi.d a3, a3, -64 <<
98 st.d t0, a5, -8 <<
99 st.d t1, a5, -16 <<
100 st.d t2, a5, -24 <<
101 st.d t3, a5, -32 <<
102 st.d t4, a5, -40 <<
103 st.d t5, a5, -48 <<
104 st.d t6, a5, -56 <<
105 st.d t7, a5, -64 <<
106 addi.d a5, a5, -64 <<
107 bltu a4, a3, .Lloop64 <<
108 <<
109 /* copy the remaining bytes */ <<
110 .Llt64: <<
111 addi.d a4, a1, 32 <<
112 bgeu a4, a3, .Llt32 <<
113 ld.d t0, a3, -8 <<
114 ld.d t1, a3, -16 <<
115 ld.d t2, a3, -24 <<
116 ld.d t3, a3, -32 <<
117 addi.d a3, a3, -32 <<
118 st.d t0, a5, -8 <<
119 st.d t1, a5, -16 <<
120 st.d t2, a5, -24 <<
121 st.d t3, a5, -32 <<
122 addi.d a5, a5, -32 <<
123 <<
124 .Llt32: <<
125 addi.d a4, a1, 16 <<
126 bgeu a4, a3, .Llt16 <<
127 ld.d t0, a3, -8 <<
128 ld.d t1, a3, -16 <<
129 addi.d a3, a3, -16 <<
130 st.d t0, a5, -8 <<
131 st.d t1, a5, -16 <<
132 addi.d a5, a5, -16 <<
133 <<
134 .Llt16: <<
135 addi.d a4, a1, 8 <<
136 bgeu a4, a3, .Llt8 <<
137 ld.d t0, a3, -8 <<
138 st.d t0, a5, -8 <<
139 <<
140 .Llt8: <<
141 st.d a6, a0, 0 <<
142 st.d a7, a2, -8 <<
143 <<
144 /* return */ <<
145 jr ra <<
146 SYM_FUNC_END(__rmemcpy_fast) <<
147 _ASM_NOKPROBE(__rmemcpy_fast) <<
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.