1 /* SPDX-License-Identifier: GPL-2.0-only */ <<
2 /* 1 /*
3 * Copyright (c) 2011, The Linux Foundation. A !! 2 * This file is subject to the terms and conditions of the GNU General Public
>> 3 * License. See the file "COPYING" in the main directory of this archive
>> 4 * for more details.
>> 5 *
>> 6 * Copyright (C) 1998, 1999, 2000 by Ralf Baechle
>> 7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
>> 8 * Copyright (C) 2007 by Maciej W. Rozycki
>> 9 * Copyright (C) 2011, 2012 MIPS Technologies, Inc.
4 */ 10 */
>> 11 #include <asm/asm.h>
>> 12 #include <asm/asm-offsets.h>
>> 13 #include <asm/export.h>
>> 14 #include <asm/regdef.h>
>> 15
>> 16 #if LONGSIZE == 4
>> 17 #define LONG_S_L swl
>> 18 #define LONG_S_R swr
>> 19 #else
>> 20 #define LONG_S_L sdl
>> 21 #define LONG_S_R sdr
>> 22 #endif
5 23
>> 24 #ifdef CONFIG_CPU_MICROMIPS
>> 25 #define STORSIZE (LONGSIZE * 2)
>> 26 #define STORMASK (STORSIZE - 1)
>> 27 #define FILL64RG t8
>> 28 #define FILLPTRG t7
>> 29 #undef LONG_S
>> 30 #define LONG_S LONG_SP
>> 31 #else
>> 32 #define STORSIZE LONGSIZE
>> 33 #define STORMASK LONGMASK
>> 34 #define FILL64RG a1
>> 35 #define FILLPTRG t0
>> 36 #endif
6 37
7 /* HEXAGON assembly optimized memset */ !! 38 #define LEGACY_MODE 1
8 /* Replaces the standard library function mems !! 39 #define EVA_MODE 2
9 40
>> 41 /*
>> 42 * No need to protect it with EVA #ifdefery. The generated block of code
>> 43 * will never be assembled if EVA is not enabled.
>> 44 */
>> 45 #define __EVAFY(insn, reg, addr) __BUILD_EVA_INSN(insn##e, reg, addr)
>> 46 #define ___BUILD_EVA_INSN(insn, reg, addr) __EVAFY(insn, reg, addr)
10 47
11 .macro HEXAGON_OPT_FUNC_BEGIN name !! 48 #define EX(insn,reg,addr,handler) \
12 .text !! 49 .if \mode == LEGACY_MODE; \
13 .p2align 4 !! 50 9: insn reg, addr; \
14 .globl \name !! 51 .else; \
15 .type \name, @function !! 52 9: ___BUILD_EVA_INSN(insn, reg, addr); \
16 \name: !! 53 .endif; \
>> 54 .section __ex_table,"a"; \
>> 55 PTR 9b, handler; \
>> 56 .previous
>> 57
>> 58 .macro f_fill64 dst, offset, val, fixup, mode
>> 59 EX(LONG_S, \val, (\offset + 0 * STORSIZE)(\dst), \fixup)
>> 60 EX(LONG_S, \val, (\offset + 1 * STORSIZE)(\dst), \fixup)
>> 61 EX(LONG_S, \val, (\offset + 2 * STORSIZE)(\dst), \fixup)
>> 62 EX(LONG_S, \val, (\offset + 3 * STORSIZE)(\dst), \fixup)
>> 63 #if ((defined(CONFIG_CPU_MICROMIPS) && (LONGSIZE == 4)) || !defined(CONFIG_CPU_MICROMIPS))
>> 64 EX(LONG_S, \val, (\offset + 4 * STORSIZE)(\dst), \fixup)
>> 65 EX(LONG_S, \val, (\offset + 5 * STORSIZE)(\dst), \fixup)
>> 66 EX(LONG_S, \val, (\offset + 6 * STORSIZE)(\dst), \fixup)
>> 67 EX(LONG_S, \val, (\offset + 7 * STORSIZE)(\dst), \fixup)
>> 68 #endif
>> 69 #if (!defined(CONFIG_CPU_MICROMIPS) && (LONGSIZE == 4))
>> 70 EX(LONG_S, \val, (\offset + 8 * STORSIZE)(\dst), \fixup)
>> 71 EX(LONG_S, \val, (\offset + 9 * STORSIZE)(\dst), \fixup)
>> 72 EX(LONG_S, \val, (\offset + 10 * STORSIZE)(\dst), \fixup)
>> 73 EX(LONG_S, \val, (\offset + 11 * STORSIZE)(\dst), \fixup)
>> 74 EX(LONG_S, \val, (\offset + 12 * STORSIZE)(\dst), \fixup)
>> 75 EX(LONG_S, \val, (\offset + 13 * STORSIZE)(\dst), \fixup)
>> 76 EX(LONG_S, \val, (\offset + 14 * STORSIZE)(\dst), \fixup)
>> 77 EX(LONG_S, \val, (\offset + 15 * STORSIZE)(\dst), \fixup)
>> 78 #endif
17 .endm 79 .endm
18 80
19 .macro HEXAGON_OPT_FUNC_FINISH name !! 81 .set noreorder
20 .size \name, . - \name !! 82 .align 5
>> 83
>> 84 /*
>> 85 * Macro to generate the __bzero{,_user} symbol
>> 86 * Arguments:
>> 87 * mode: LEGACY_MODE or EVA_MODE
>> 88 */
>> 89 .macro __BUILD_BZERO mode
>> 90 /* Initialize __memset if this is the first time we call this macro */
>> 91 .ifnotdef __memset
>> 92 .set __memset, 1
>> 93 .hidden __memset /* Make sure it does not leak */
>> 94 .endif
>> 95
>> 96 sltiu t0, a2, STORSIZE /* very small region? */
>> 97 bnez t0, .Lsmall_memset\@
>> 98 andi t0, a0, STORMASK /* aligned? */
>> 99
>> 100 #ifdef CONFIG_CPU_MICROMIPS
>> 101 move t8, a1 /* used by 'swp' instruction */
>> 102 move t9, a1
>> 103 #endif
>> 104 #ifndef CONFIG_CPU_DADDI_WORKAROUNDS
>> 105 beqz t0, 1f
>> 106 PTR_SUBU t0, STORSIZE /* alignment in bytes */
>> 107 #else
>> 108 .set noat
>> 109 li AT, STORSIZE
>> 110 beqz t0, 1f
>> 111 PTR_SUBU t0, AT /* alignment in bytes */
>> 112 .set at
>> 113 #endif
>> 114
>> 115 #ifndef CONFIG_CPU_MIPSR6
>> 116 R10KCBARRIER(0(ra))
>> 117 #ifdef __MIPSEB__
>> 118 EX(LONG_S_L, a1, (a0), .Lfirst_fixup\@) /* make word/dword aligned */
>> 119 #else
>> 120 EX(LONG_S_R, a1, (a0), .Lfirst_fixup\@) /* make word/dword aligned */
>> 121 #endif
>> 122 PTR_SUBU a0, t0 /* long align ptr */
>> 123 PTR_ADDU a2, t0 /* correct size */
>> 124
>> 125 #else /* CONFIG_CPU_MIPSR6 */
>> 126 #define STORE_BYTE(N) \
>> 127 EX(sb, a1, N(a0), .Lbyte_fixup\@); \
>> 128 beqz t0, 0f; \
>> 129 PTR_ADDU t0, 1;
>> 130
>> 131 PTR_ADDU a2, t0 /* correct size */
>> 132 PTR_ADDU t0, 1
>> 133 STORE_BYTE(0)
>> 134 STORE_BYTE(1)
>> 135 #if LONGSIZE == 4
>> 136 EX(sb, a1, 2(a0), .Lbyte_fixup\@)
>> 137 #else
>> 138 STORE_BYTE(2)
>> 139 STORE_BYTE(3)
>> 140 STORE_BYTE(4)
>> 141 STORE_BYTE(5)
>> 142 EX(sb, a1, 6(a0), .Lbyte_fixup\@)
>> 143 #endif
>> 144 0:
>> 145 ori a0, STORMASK
>> 146 xori a0, STORMASK
>> 147 PTR_ADDIU a0, STORSIZE
>> 148 #endif /* CONFIG_CPU_MIPSR6 */
>> 149 1: ori t1, a2, 0x3f /* # of full blocks */
>> 150 xori t1, 0x3f
>> 151 beqz t1, .Lmemset_partial\@ /* no block to fill */
>> 152 andi t0, a2, 0x40-STORSIZE
>> 153
>> 154 PTR_ADDU t1, a0 /* end address */
>> 155 .set reorder
>> 156 1: PTR_ADDIU a0, 64
>> 157 R10KCBARRIER(0(ra))
>> 158 f_fill64 a0, -64, FILL64RG, .Lfwd_fixup\@, \mode
>> 159 bne t1, a0, 1b
>> 160 .set noreorder
>> 161
>> 162 .Lmemset_partial\@:
>> 163 R10KCBARRIER(0(ra))
>> 164 PTR_LA t1, 2f /* where to start */
>> 165 #ifdef CONFIG_CPU_MICROMIPS
>> 166 LONG_SRL t7, t0, 1
>> 167 #endif
>> 168 #if LONGSIZE == 4
>> 169 PTR_SUBU t1, FILLPTRG
>> 170 #else
>> 171 .set noat
>> 172 LONG_SRL AT, FILLPTRG, 1
>> 173 PTR_SUBU t1, AT
>> 174 .set at
>> 175 #endif
>> 176 jr t1
>> 177 PTR_ADDU a0, t0 /* dest ptr */
>> 178
>> 179 .set push
>> 180 .set noreorder
>> 181 .set nomacro
>> 182 /* ... but first do longs ... */
>> 183 f_fill64 a0, -64, FILL64RG, .Lpartial_fixup\@, \mode
>> 184 2: .set pop
>> 185 andi a2, STORMASK /* At most one long to go */
>> 186
>> 187 beqz a2, 1f
>> 188 #ifndef CONFIG_CPU_MIPSR6
>> 189 PTR_ADDU a0, a2 /* What's left */
>> 190 R10KCBARRIER(0(ra))
>> 191 #ifdef __MIPSEB__
>> 192 EX(LONG_S_R, a1, -1(a0), .Llast_fixup\@)
>> 193 #else
>> 194 EX(LONG_S_L, a1, -1(a0), .Llast_fixup\@)
>> 195 #endif
>> 196 #else
>> 197 PTR_SUBU t0, $0, a2
>> 198 PTR_ADDIU t0, 1
>> 199 STORE_BYTE(0)
>> 200 STORE_BYTE(1)
>> 201 #if LONGSIZE == 4
>> 202 EX(sb, a1, 2(a0), .Lbyte_fixup\@)
>> 203 #else
>> 204 STORE_BYTE(2)
>> 205 STORE_BYTE(3)
>> 206 STORE_BYTE(4)
>> 207 STORE_BYTE(5)
>> 208 EX(sb, a1, 6(a0), .Lbyte_fixup\@)
>> 209 #endif
>> 210 0:
>> 211 #endif
>> 212 1: jr ra
>> 213 move a2, zero
>> 214
>> 215 .Lsmall_memset\@:
>> 216 beqz a2, 2f
>> 217 PTR_ADDU t1, a0, a2
>> 218
>> 219 1: PTR_ADDIU a0, 1 /* fill bytewise */
>> 220 R10KCBARRIER(0(ra))
>> 221 bne t1, a0, 1b
>> 222 sb a1, -1(a0)
>> 223
>> 224 2: jr ra /* done */
>> 225 move a2, zero
>> 226 .if __memset == 1
>> 227 END(memset)
>> 228 .set __memset, 0
>> 229 .hidden __memset
>> 230 .endif
>> 231
>> 232 #ifdef CONFIG_CPU_MIPSR6
>> 233 .Lbyte_fixup\@:
>> 234 PTR_SUBU a2, $0, t0
>> 235 jr ra
>> 236 PTR_ADDIU a2, 1
>> 237 #endif /* CONFIG_CPU_MIPSR6 */
>> 238
>> 239 .Lfirst_fixup\@:
>> 240 jr ra
>> 241 nop
>> 242
>> 243 .Lfwd_fixup\@:
>> 244 PTR_L t0, TI_TASK($28)
>> 245 andi a2, 0x3f
>> 246 LONG_L t0, THREAD_BUADDR(t0)
>> 247 LONG_ADDU a2, t1
>> 248 jr ra
>> 249 LONG_SUBU a2, t0
>> 250
>> 251 .Lpartial_fixup\@:
>> 252 PTR_L t0, TI_TASK($28)
>> 253 andi a2, STORMASK
>> 254 LONG_L t0, THREAD_BUADDR(t0)
>> 255 LONG_ADDU a2, t1
>> 256 jr ra
>> 257 LONG_SUBU a2, t0
>> 258
>> 259 .Llast_fixup\@:
>> 260 jr ra
>> 261 andi v1, a2, STORMASK
>> 262
21 .endm 263 .endm
22 264
23 /* FUNCTION: memset (v2 version) */ !! 265 /*
24 #if __HEXAGON_ARCH__ < 3 !! 266 * memset(void *s, int c, size_t n)
25 HEXAGON_OPT_FUNC_BEGIN memset !! 267 *
26 { !! 268 * a0: start of area to clear
27 r6 = #8 !! 269 * a1: char to fill with
28 r7 = extractu(r0, #3 , #0) !! 270 * a2: size of area to clear
29 p0 = cmp.eq(r2, #0) !! 271 */
30 p1 = cmp.gtu(r2, #7) !! 272
31 } !! 273 LEAF(memset)
32 { !! 274 EXPORT_SYMBOL(memset)
33 r4 = vsplatb(r1) !! 275 beqz a1, 1f
34 r8 = r0 /* leave r0 !! 276 move v0, a0 /* result */
35 r9 = sub(r6, r7) /* bytes unt !! 277
36 if p0 jumpr r31 /* count == !! 278 andi a1, 0xff /* spread fillword */
37 } !! 279 LONG_SLL t1, a1, 8
38 { !! 280 or a1, t1
39 r3 = #0 !! 281 LONG_SLL t1, a1, 16
40 r7 = #0 !! 282 #if LONGSIZE == 8
41 p0 = tstbit(r9, #0) !! 283 or a1, t1
42 if p1 jump 2f /* skip byte loo !! 284 LONG_SLL t1, a1, 32
43 } !! 285 #endif
44 !! 286 or a1, t1
45 /* less than 8 bytes to set, so just set a byt !! 287 1:
46 !! 288 #ifndef CONFIG_EVA
47 loop0(1f, r2) /* byte loop */ !! 289 FEXPORT(__bzero)
48 .falign !! 290 EXPORT_SYMBOL(__bzero)
49 1: /* byte loop */ !! 291 #else
50 { !! 292 FEXPORT(__bzero_kernel)
51 memb(r8++#1) = r4 !! 293 EXPORT_SYMBOL(__bzero_kernel)
52 }:endloop0 !! 294 #endif
53 jumpr r31 !! 295 __BUILD_BZERO LEGACY_MODE
54 .falign !! 296
55 2: /* skip byte loop */ !! 297 #ifdef CONFIG_EVA
56 { !! 298 LEAF(__bzero)
57 r6 = #1 !! 299 EXPORT_SYMBOL(__bzero)
58 p0 = tstbit(r9, #1) !! 300 __BUILD_BZERO EVA_MODE
59 p1 = cmp.eq(r2, #1) !! 301 END(__bzero)
60 if !p0 jump 3f /* skip initial <<
61 } <<
62 { <<
63 memb(r8++#1) = r4 <<
64 r3:2 = sub(r3:2, r7:6) <<
65 if p1 jumpr r31 <<
66 } <<
67 .falign <<
68 3: /* skip initial byte store */ <<
69 { <<
70 r6 = #2 <<
71 p0 = tstbit(r9, #2) <<
72 p1 = cmp.eq(r2, #2) <<
73 if !p0 jump 4f /* skip initial <<
74 } <<
75 { <<
76 memh(r8++#2) = r4 <<
77 r3:2 = sub(r3:2, r7:6) <<
78 if p1 jumpr r31 <<
79 } <<
80 .falign <<
81 4: /* skip initial half store */ <<
82 { <<
83 r6 = #4 <<
84 p0 = cmp.gtu(r2, #7) <<
85 p1 = cmp.eq(r2, #4) <<
86 if !p0 jump 5f /* skip initial <<
87 } <<
88 { <<
89 memw(r8++#4) = r4 <<
90 r3:2 = sub(r3:2, r7:6) <<
91 p0 = cmp.gtu(r2, #11) <<
92 if p1 jumpr r31 <<
93 } <<
94 .falign <<
95 5: /* skip initial word store */ <<
96 { <<
97 r10 = lsr(r2, #3) <<
98 p1 = cmp.eq(r3, #1) <<
99 if !p0 jump 7f /* skip double <<
100 } <<
101 { <<
102 r5 = r4 <<
103 r6 = #8 <<
104 loop0(6f, r10) /* double loop <<
105 } <<
106 <<
107 /* set bytes a double word at a time */ <<
108 <<
109 .falign <<
110 6: /* double loop */ <<
111 { <<
112 memd(r8++#8) = r5:4 <<
113 r3:2 = sub(r3:2, r7:6) <<
114 p1 = cmp.eq(r2, #8) <<
115 }:endloop0 <<
116 .falign <<
117 7: /* skip double loop */ <<
118 { <<
119 p0 = tstbit(r2, #2) <<
120 if p1 jumpr r31 <<
121 } <<
122 { <<
123 r6 = #4 <<
124 p0 = tstbit(r2, #1) <<
125 p1 = cmp.eq(r2, #4) <<
126 if !p0 jump 8f /* skip final w <<
127 } <<
128 { <<
129 memw(r8++#4) = r4 <<
130 r3:2 = sub(r3:2, r7:6) <<
131 if p1 jumpr r31 <<
132 } <<
133 .falign <<
134 8: /* skip final word store */ <<
135 { <<
136 p1 = cmp.eq(r2, #2) <<
137 if !p0 jump 9f /* skip final h <<
138 } <<
139 { <<
140 memh(r8++#2) = r4 <<
141 if p1 jumpr r31 <<
142 } <<
143 .falign <<
144 9: /* skip final half store */ <<
145 { <<
146 memb(r8++#1) = r4 <<
147 jumpr r31 <<
148 } <<
149 HEXAGON_OPT_FUNC_FINISH memset <<
150 #endif <<
151 <<
152 <<
153 /* FUNCTION: memset (v3 and higher version) <<
154 #if __HEXAGON_ARCH__ >= 3 <<
155 HEXAGON_OPT_FUNC_BEGIN memset <<
156 { <<
157 r7=vsplatb(r1) <<
158 r6 = r0 <<
159 if (r2==#0) jump:nt .L1 <<
160 } <<
161 { <<
162 r5:4=combine(r7,r7) <<
163 p0 = cmp.gtu(r2,#8) <<
164 if (p0.new) jump:nt .L3 <<
165 } <<
166 { <<
167 r3 = r0 <<
168 loop0(.L47,r2) <<
169 } <<
170 .falign <<
171 .L47: <<
172 { <<
173 memb(r3++#1) = r1 <<
174 }:endloop0 /* start=.L47 */ <<
175 jumpr r31 <<
176 .L3: <<
177 { <<
178 p0 = tstbit(r0,#0) <<
179 if (!p0.new) jump:nt .L8 <<
180 p1 = cmp.eq(r2, #1) <<
181 } <<
182 { <<
183 r6 = add(r0, #1) <<
184 r2 = add(r2,#-1) <<
185 memb(r0) = r1 <<
186 if (p1) jump .L1 <<
187 } <<
188 .L8: <<
189 { <<
190 p0 = tstbit(r6,#1) <<
191 if (!p0.new) jump:nt .L10 <<
192 } <<
193 { <<
194 r2 = add(r2,#-2) <<
195 memh(r6++#2) = r7 <<
196 p0 = cmp.eq(r2, #2) <<
197 if (p0.new) jump:nt .L1 <<
198 } <<
199 .L10: <<
200 { <<
201 p0 = tstbit(r6,#2) <<
202 if (!p0.new) jump:nt .L12 <<
203 } <<
204 { <<
205 r2 = add(r2,#-4) <<
206 memw(r6++#4) = r7 <<
207 p0 = cmp.eq(r2, #4) <<
208 if (p0.new) jump:nt .L1 <<
209 } <<
210 .L12: <<
211 { <<
212 p0 = cmp.gtu(r2,#127) <<
213 if (!p0.new) jump:nt .L14 <<
214 } <<
215 r3 = and(r6,#31) <<
216 if (r3==#0) jump:nt .L17 <<
217 { <<
218 memd(r6++#8) = r5:4 <<
219 r2 = add(r2,#-8) <<
220 } <<
221 r3 = and(r6,#31) <<
222 if (r3==#0) jump:nt .L17 <<
223 { <<
224 memd(r6++#8) = r5:4 <<
225 r2 = add(r2,#-8) <<
226 } <<
227 r3 = and(r6,#31) <<
228 if (r3==#0) jump:nt .L17 <<
229 { <<
230 memd(r6++#8) = r5:4 <<
231 r2 = add(r2,#-8) <<
232 } <<
233 .L17: <<
234 { <<
235 r3 = lsr(r2,#5) <<
236 if (r1!=#0) jump:nt .L18 <<
237 } <<
238 { <<
239 r8 = r3 <<
240 r3 = r6 <<
241 loop0(.L46,r3) <<
242 } <<
243 .falign <<
244 .L46: <<
245 { <<
246 dczeroa(r6) <<
247 r6 = add(r6,#32) <<
248 r2 = add(r2,#-32) <<
249 }:endloop0 /* start=.L46 */ <<
250 .L14: <<
251 { <<
252 p0 = cmp.gtu(r2,#7) <<
253 if (!p0.new) jump:nt .L28 <<
254 r8 = lsr(r2,#3) <<
255 } <<
256 loop0(.L44,r8) <<
257 .falign <<
258 .L44: <<
259 { <<
260 memd(r6++#8) = r5:4 <<
261 r2 = add(r2,#-8) <<
262 }:endloop0 /* start=.L44 */ <<
263 .L28: <<
264 { <<
265 p0 = tstbit(r2,#2) <<
266 if (!p0.new) jump:nt .L33 <<
267 } <<
268 { <<
269 r2 = add(r2,#-4) <<
270 memw(r6++#4) = r7 <<
271 } <<
272 .L33: <<
273 { <<
274 p0 = tstbit(r2,#1) <<
275 if (!p0.new) jump:nt .L35 <<
276 } <<
277 { <<
278 r2 = add(r2,#-2) <<
279 memh(r6++#2) = r7 <<
280 } <<
281 .L35: <<
282 p0 = cmp.eq(r2,#1) <<
283 if (p0) memb(r6) = r1 <<
284 .L1: <<
285 jumpr r31 <<
286 .L18: <<
287 loop0(.L45,r3) <<
288 .falign <<
289 .L45: <<
290 dczeroa(r6) <<
291 { <<
292 memd(r6++#8) = r5:4 <<
293 r2 = add(r2,#-32) <<
294 } <<
295 memd(r6++#8) = r5:4 <<
296 memd(r6++#8) = r5:4 <<
297 { <<
298 memd(r6++#8) = r5:4 <<
299 }:endloop0 /* start=.L45 */ <<
300 jump .L14 <<
301 HEXAGON_OPT_FUNC_FINISH memset <<
302 #endif 302 #endif
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.