1 /* !! 1 /* SPDX-License-Identifier: GPL-2.0 */
2 * This file is subject to the terms and condi !! 2 /* linux/arch/sparc/lib/memset.S: Sparc optimized memset, bzero and clear_user code
3 * License. See the file "COPYING" in the mai !! 3 * Copyright (C) 1991,1996 Free Software Foundation
4 * for more details. !! 4 * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
>> 5 * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
5 * 6 *
6 * Copyright (C) 1998, 1999, 2000 by Ralf Baec !! 7 * Calls to memset returns initial %o0. Calls to bzero returns 0, if ok, and
7 * Copyright (C) 1999, 2000 Silicon Graphics, !! 8 * number of bytes not yet set if exception occurs and we were called as
8 * Copyright (C) 2007 by Maciej W. Rozycki !! 9 * clear_user.
9 * Copyright (C) 2011, 2012 MIPS Technologies, <<
10 */ 10 */
>> 11
11 #include <linux/export.h> 12 #include <linux/export.h>
12 #include <asm/asm.h> !! 13 #include <asm/ptrace.h>
13 #include <asm/asm-offsets.h> <<
14 #include <asm/regdef.h> <<
15 <<
16 #if LONGSIZE == 4 <<
17 #define LONG_S_L swl <<
18 #define LONG_S_R swr <<
19 #else <<
20 #define LONG_S_L sdl <<
21 #define LONG_S_R sdr <<
22 #endif <<
23 <<
24 #ifdef CONFIG_CPU_MICROMIPS <<
25 #define STORSIZE (LONGSIZE * 2) <<
26 #define STORMASK (STORSIZE - 1) <<
27 #define FILL64RG t8 <<
28 #define FILLPTRG t7 <<
29 #undef LONG_S <<
30 #define LONG_S LONG_SP <<
31 #else <<
32 #define STORSIZE LONGSIZE <<
33 #define STORMASK LONGMASK <<
34 #define FILL64RG a1 <<
35 #define FILLPTRG t0 <<
36 #endif <<
37 <<
38 #define LEGACY_MODE 1 <<
39 #define EVA_MODE 2 <<
40 <<
41 /* <<
42 * No need to protect it with EVA #ifdefery. T <<
43 * will never be assembled if EVA is not enabl <<
44 */ <<
45 #define __EVAFY(insn, reg, addr) __BUILD_EVA_I <<
46 #define ___BUILD_EVA_INSN(insn, reg, addr) __E <<
47 14
48 #define EX(insn,reg,addr,handler) !! 15 /* Work around cpp -rob */
49 .if \mode == LEGACY_MODE; !! 16 #define ALLOC #alloc
50 9: insn reg, addr; !! 17 #define EXECINSTR #execinstr
51 .else; !! 18 #define EX(x,y,a,b) \
52 9: ___BUILD_EVA_INSN(insn, reg, a !! 19 98: x,y; \
53 .endif; !! 20 .section .fixup,ALLOC,EXECINSTR; \
54 .section __ex_table,"a"; !! 21 .align 4; \
55 PTR_WD 9b, handler; !! 22 99: retl; \
56 .previous !! 23 a, b, %o0; \
57 !! 24 .section __ex_table,ALLOC; \
58 .macro f_fill64 dst, offset, val, fix !! 25 .align 4; \
59 EX(LONG_S, \val, (\offset + 0 * STORS !! 26 .word 98b, 99b; \
60 EX(LONG_S, \val, (\offset + 1 * STORS !! 27 .text; \
61 EX(LONG_S, \val, (\offset + 2 * STORS !! 28 .align 4
62 EX(LONG_S, \val, (\offset + 3 * STORS !! 29
63 #if ((defined(CONFIG_CPU_MICROMIPS) && (LONGSI !! 30 #define STORE(source, base, offset, n) \
64 EX(LONG_S, \val, (\offset + 4 * STORS !! 31 98: std source, [base + offset + n]; \
65 EX(LONG_S, \val, (\offset + 5 * STORS !! 32 .section .fixup,ALLOC,EXECINSTR; \
66 EX(LONG_S, \val, (\offset + 6 * STORS !! 33 .align 4; \
67 EX(LONG_S, \val, (\offset + 7 * STORS !! 34 99: ba 30f; \
68 #endif !! 35 sub %o3, n - offset, %o3; \
69 #if (!defined(CONFIG_CPU_MICROMIPS) && (LONGSI !! 36 .section __ex_table,ALLOC; \
70 EX(LONG_S, \val, (\offset + 8 * STORS !! 37 .align 4; \
71 EX(LONG_S, \val, (\offset + 9 * STORS !! 38 .word 98b, 99b; \
72 EX(LONG_S, \val, (\offset + 10 * STORS !! 39 .text; \
73 EX(LONG_S, \val, (\offset + 11 * STORS !! 40 .align 4;
74 EX(LONG_S, \val, (\offset + 12 * STORS !! 41
75 EX(LONG_S, \val, (\offset + 13 * STORS !! 42 #define STORE_LAST(source, base, offset, n) \
76 EX(LONG_S, \val, (\offset + 14 * STORS !! 43 EX(std source, [base - offset - n], \
77 EX(LONG_S, \val, (\offset + 15 * STORS !! 44 add %o1, offset + n);
78 #endif !! 45
79 .endm !! 46 /* Please don't change these macros, unless you change the logic
80 !! 47 * in the .fixup section below as well.
81 .align 5 !! 48 * Store 64 bytes at (BASE + OFFSET) using value SOURCE. */
82 !! 49 #define ZERO_BIG_BLOCK(base, offset, source) \
83 /* !! 50 STORE(source, base, offset, 0x00); \
84 * Macro to generate the __bzero{,_use !! 51 STORE(source, base, offset, 0x08); \
85 * Arguments: !! 52 STORE(source, base, offset, 0x10); \
86 * mode: LEGACY_MODE or EVA_MODE !! 53 STORE(source, base, offset, 0x18); \
87 */ !! 54 STORE(source, base, offset, 0x20); \
88 .macro __BUILD_BZERO mode !! 55 STORE(source, base, offset, 0x28); \
89 /* Initialize __memset if this is the !! 56 STORE(source, base, offset, 0x30); \
90 .ifnotdef __memset !! 57 STORE(source, base, offset, 0x38);
91 .set __memset, 1 !! 58
92 .hidden __memset /* Make sure it does !! 59 #define ZERO_LAST_BLOCKS(base, offset, source) \
93 .endif !! 60 STORE_LAST(source, base, offset, 0x38); \
94 !! 61 STORE_LAST(source, base, offset, 0x30); \
95 sltiu t0, a2, STORSIZE !! 62 STORE_LAST(source, base, offset, 0x28); \
96 .set noreorder !! 63 STORE_LAST(source, base, offset, 0x20); \
97 bnez t0, .Lsmall_memset\@ !! 64 STORE_LAST(source, base, offset, 0x18); \
98 andi t0, a0, STORMASK !! 65 STORE_LAST(source, base, offset, 0x10); \
99 .set reorder !! 66 STORE_LAST(source, base, offset, 0x08); \
100 !! 67 STORE_LAST(source, base, offset, 0x00);
101 #ifdef CONFIG_CPU_MICROMIPS !! 68
102 move t8, a1 !! 69 .text
103 move t9, a1 !! 70 .align 4
104 #endif !! 71
105 .set noreorder !! 72 .globl __bzero_begin
106 #ifndef CONFIG_CPU_DADDI_WORKAROUNDS !! 73 __bzero_begin:
107 beqz t0, 1f !! 74
108 PTR_SUBU t0, STORSIZE !! 75 .globl __bzero
109 #else !! 76 .type __bzero,#function
110 .set noat !! 77 .globl memset
111 li AT, STORSIZE !! 78 EXPORT_SYMBOL(__bzero)
112 beqz t0, 1f !! 79 EXPORT_SYMBOL(memset)
113 PTR_SUBU t0, AT !! 80 memset:
114 .set at !! 81 mov %o0, %g1
115 #endif !! 82 mov 1, %g4
116 .set reorder !! 83 and %o1, 0xff, %g3
117 !! 84 sll %g3, 8, %g2
118 #ifndef CONFIG_CPU_NO_LOAD_STORE_LR !! 85 or %g3, %g2, %g3
119 R10KCBARRIER(0(ra)) !! 86 sll %g3, 16, %g2
120 #ifdef __MIPSEB__ !! 87 or %g3, %g2, %g3
121 EX(LONG_S_L, a1, (a0), .Lfirst_fixup\@ !! 88 b 1f
122 #else !! 89 mov %o2, %o1
123 EX(LONG_S_R, a1, (a0), .Lfirst_fixup\@ !! 90 3:
124 #endif !! 91 cmp %o2, 3
125 PTR_SUBU a0, t0 !! 92 be 2f
126 PTR_ADDU a2, t0 !! 93 EX(stb %g3, [%o0], sub %o1, 0)
127 !! 94
128 #else /* CONFIG_CPU_NO_LOAD_STORE_LR */ !! 95 cmp %o2, 2
129 #define STORE_BYTE(N) !! 96 be 2f
130 EX(sb, a1, N(a0), .Lbyte_fixup\@); !! 97 EX(stb %g3, [%o0 + 0x01], sub %o1, 1)
131 .set noreorder; !! 98
132 beqz t0, 0f; !! 99 EX(stb %g3, [%o0 + 0x02], sub %o1, 2)
133 PTR_ADDU t0, 1; !! 100 2:
134 .set reorder; !! 101 sub %o2, 4, %o2
135 !! 102 add %o1, %o2, %o1
136 PTR_ADDU a2, t0 !! 103 b 4f
137 PTR_ADDU t0, 1 !! 104 sub %o0, %o2, %o0
138 STORE_BYTE(0) !! 105
139 STORE_BYTE(1) !! 106 __bzero:
140 #if LONGSIZE == 4 !! 107 clr %g4
141 EX(sb, a1, 2(a0), .Lbyte_fixup\@) !! 108 mov %g0, %g3
142 #else !! 109 1:
143 STORE_BYTE(2) !! 110 cmp %o1, 7
144 STORE_BYTE(3) !! 111 bleu 7f
145 STORE_BYTE(4) !! 112 andcc %o0, 3, %o2
146 STORE_BYTE(5) !! 113
147 EX(sb, a1, 6(a0), .Lbyte_fixup\@) !! 114 bne 3b
148 #endif !! 115 4:
149 0: !! 116 andcc %o0, 4, %g0
150 ori a0, STORMASK !! 117
151 xori a0, STORMASK !! 118 be 2f
152 PTR_ADDIU a0, STORSIZE !! 119 mov %g3, %g2
153 #endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ !! 120
154 1: ori t1, a2, 0x3f !! 121 EX(st %g3, [%o0], sub %o1, 0)
155 xori t1, 0x3f !! 122 sub %o1, 4, %o1
156 andi t0, a2, 0x40-STORSIZE !! 123 add %o0, 4, %o0
157 beqz t1, .Lmemset_partial\@ !! 124 2:
158 !! 125 andcc %o1, 0xffffff80, %o3 ! Now everything is 8 aligned and o1 is len to run
159 PTR_ADDU t1, a0 !! 126 be 9f
160 1: PTR_ADDIU a0, 64 !! 127 andcc %o1, 0x78, %o2
161 R10KCBARRIER(0(ra)) !! 128 10:
162 f_fill64 a0, -64, FILL64RG, .Lfwd_fixu !! 129 ZERO_BIG_BLOCK(%o0, 0x00, %g2)
163 bne t1, a0, 1b !! 130 subcc %o3, 128, %o3
164 !! 131 ZERO_BIG_BLOCK(%o0, 0x40, %g2)
165 .Lmemset_partial\@: !! 132 bne 10b
166 R10KCBARRIER(0(ra)) !! 133 add %o0, 128, %o0
167 PTR_LA t1, 2f !! 134
168 #ifdef CONFIG_CPU_MICROMIPS !! 135 orcc %o2, %g0, %g0
169 LONG_SRL t7, t0, 1 !! 136 9:
170 #endif !! 137 be 13f
171 #if LONGSIZE == 4 !! 138 andcc %o1, 7, %o1
172 PTR_SUBU t1, FILLPTRG !! 139
173 #else !! 140 srl %o2, 1, %o3
174 .set noat !! 141 set 13f, %o4
175 LONG_SRL AT, FILLPTRG, 1 !! 142 sub %o4, %o3, %o4
176 PTR_SUBU t1, AT !! 143 jmp %o4
177 .set at !! 144 add %o0, %o2, %o0
178 #endif !! 145
179 PTR_ADDU a0, t0 !! 146 ZERO_LAST_BLOCKS(%o0, 0x48, %g2)
180 jr t1 !! 147 ZERO_LAST_BLOCKS(%o0, 0x08, %g2)
181 !! 148 13:
182 /* ... but first do longs ... */ !! 149 be 8f
183 f_fill64 a0, -64, FILL64RG, .Lpartial_ !! 150 andcc %o1, 4, %g0
184 2: andi a2, STORMASK <<
185 <<
186 .set noreorder <<
187 beqz a2, 1f <<
188 #ifndef CONFIG_CPU_NO_LOAD_STORE_LR <<
189 PTR_ADDU a0, a2 <<
190 .set reorder <<
191 R10KCBARRIER(0(ra)) <<
192 #ifdef __MIPSEB__ <<
193 EX(LONG_S_R, a1, -1(a0), .Llast_fixup\ <<
194 #else <<
195 EX(LONG_S_L, a1, -1(a0), .Llast_fixup\ <<
196 #endif <<
197 #else /* CONFIG_CPU_NO_LOAD_STORE_LR */ <<
198 PTR_SUBU t0, $0, a2 <<
199 .set reorder <<
200 move a2, zero <<
201 PTR_ADDIU t0, 1 <<
202 STORE_BYTE(0) <<
203 STORE_BYTE(1) <<
204 #if LONGSIZE == 4 <<
205 EX(sb, a1, 2(a0), .Lbyte_fixup\@) <<
206 #else <<
207 STORE_BYTE(2) <<
208 STORE_BYTE(3) <<
209 STORE_BYTE(4) <<
210 STORE_BYTE(5) <<
211 EX(sb, a1, 6(a0), .Lbyte_fixup\@) <<
212 #endif <<
213 0: <<
214 #endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ <<
215 1: move a2, zero <<
216 jr ra <<
217 <<
218 .Lsmall_memset\@: <<
219 PTR_ADDU t1, a0, a2 <<
220 beqz a2, 2f <<
221 <<
222 1: PTR_ADDIU a0, 1 <<
223 R10KCBARRIER(0(ra)) <<
224 .set noreorder <<
225 bne t1, a0, 1b <<
226 EX(sb, a1, -1(a0), .Lsmall_fixup\@) <<
227 .set reorder <<
228 <<
229 2: move a2, zero <<
230 jr ra <<
231 .if __memset == 1 <<
232 END(memset) <<
233 .set __memset, 0 <<
234 .hidden __memset <<
235 .endif <<
236 <<
237 #ifdef CONFIG_CPU_NO_LOAD_STORE_LR <<
238 .Lbyte_fixup\@: <<
239 /* <<
240 * unset_bytes = (#bytes - (#unaligned <<
241 * a2 = a2 <<
242 */ <<
243 PTR_SUBU a2, t0 <<
244 PTR_ADDIU a2, 1 <<
245 jr ra <<
246 #endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ <<
247 <<
248 .Lfirst_fixup\@: <<
249 /* unset_bytes already in a2 */ <<
250 jr ra <<
251 <<
252 .Lfwd_fixup\@: <<
253 /* <<
254 * unset_bytes = partial_start_addr + <<
255 * a2 = t1 + <<
256 */ <<
257 PTR_L t0, TI_TASK($28) <<
258 andi a2, 0x3f <<
259 LONG_L t0, THREAD_BUADDR(t0) <<
260 LONG_ADDU a2, t1 <<
261 LONG_SUBU a2, t0 <<
262 jr ra <<
263 <<
264 .Lpartial_fixup\@: <<
265 /* <<
266 * unset_bytes = partial_end_addr + <<
267 * a2 = a0 + (a <<
268 */ <<
269 PTR_L t0, TI_TASK($28) <<
270 andi a2, STORMASK <<
271 LONG_L t0, THREAD_BUADDR(t0) <<
272 LONG_ADDU a2, a0 <<
273 LONG_SUBU a2, t0 <<
274 jr ra <<
275 <<
276 .Llast_fixup\@: <<
277 /* unset_bytes already in a2 */ <<
278 jr ra <<
279 <<
280 .Lsmall_fixup\@: <<
281 /* <<
282 * unset_bytes = end_addr - current_ad <<
283 * a2 = t1 - a0 <<
284 */ <<
285 PTR_SUBU a2, t1, a0 <<
286 PTR_ADDIU a2, 1 <<
287 jr ra <<
288 151
289 .endm !! 152 be 1f
>> 153 andcc %o1, 2, %g0
290 154
291 /* !! 155 EX(st %g3, [%o0], and %o1, 7)
292 * memset(void *s, int c, size_t n) !! 156 add %o0, 4, %o0
293 * !! 157 1:
294 * a0: start of area to clear !! 158 be 1f
295 * a1: char to fill with !! 159 andcc %o1, 1, %g0
296 * a2: size of area to clear <<
297 */ <<
298 160
299 LEAF(memset) !! 161 EX(sth %g3, [%o0], and %o1, 3)
300 EXPORT_SYMBOL(memset) !! 162 add %o0, 2, %o0
301 move v0, a0 <<
302 beqz a1, 1f <<
303 <<
304 andi a1, 0xff <<
305 LONG_SLL t1, a1, 8 <<
306 or a1, t1 <<
307 LONG_SLL t1, a1, 16 <<
308 #if LONGSIZE == 8 <<
309 or a1, t1 <<
310 LONG_SLL t1, a1, 32 <<
311 #endif <<
312 or a1, t1 <<
313 1: 163 1:
314 #ifndef CONFIG_EVA !! 164 bne,a 8f
315 FEXPORT(__bzero) !! 165 EX(stb %g3, [%o0], and %o1, 1)
316 EXPORT_SYMBOL(__bzero) !! 166 8:
317 #endif !! 167 b 0f
318 __BUILD_BZERO LEGACY_MODE !! 168 nop
319 !! 169 7:
320 #ifdef CONFIG_EVA !! 170 be 13b
321 LEAF(__bzero) !! 171 orcc %o1, 0, %g0
322 EXPORT_SYMBOL(__bzero) !! 172
323 __BUILD_BZERO EVA_MODE !! 173 be 0f
324 END(__bzero) !! 174 8:
325 #endif !! 175 add %o0, 1, %o0
>> 176 subcc %o1, 1, %o1
>> 177 bne 8b
>> 178 EX(stb %g3, [%o0 - 1], add %o1, 1)
>> 179 0:
>> 180 andcc %g4, 1, %g0
>> 181 be 5f
>> 182 nop
>> 183 retl
>> 184 mov %g1, %o0
>> 185 5:
>> 186 retl
>> 187 clr %o0
>> 188
>> 189 .section .fixup,#alloc,#execinstr
>> 190 .align 4
>> 191 30:
>> 192 and %o1, 0x7f, %o1
>> 193 retl
>> 194 add %o3, %o1, %o0
>> 195
>> 196 .globl __bzero_end
>> 197 __bzero_end:
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.