1 /* SPDX-License-Identifier: GPL-2.0-or-later * <<
2 /* 1 /*
3 * OpenRISC memset.S !! 2 * This file is subject to the terms and conditions of the GNU General Public
>> 3 * License. See the file "COPYING" in the main directory of this archive
>> 4 * for more details.
4 * 5 *
5 * Hand-optimized assembler version of memset !! 6 * Copyright (C) 1998, 1999, 2000 by Ralf Baechle
6 * Algorithm inspired by several other arch-sp !! 7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
7 * in the kernel tree !! 8 * Copyright (C) 2007 by Maciej W. Rozycki
8 * !! 9 * Copyright (C) 2011, 2012 MIPS Technologies, Inc.
9 * Copyright (C) 2015 Olof Kindgren <olof.kindg !! 10 */
>> 11 #include <asm/asm.h>
>> 12 #include <asm/asm-offsets.h>
>> 13 #include <asm/export.h>
>> 14 #include <asm/regdef.h>
>> 15
>> 16 #if LONGSIZE == 4
>> 17 #define LONG_S_L swl
>> 18 #define LONG_S_R swr
>> 19 #else
>> 20 #define LONG_S_L sdl
>> 21 #define LONG_S_R sdr
>> 22 #endif
>> 23
>> 24 #ifdef CONFIG_CPU_MICROMIPS
>> 25 #define STORSIZE (LONGSIZE * 2)
>> 26 #define STORMASK (STORSIZE - 1)
>> 27 #define FILL64RG t8
>> 28 #define FILLPTRG t7
>> 29 #undef LONG_S
>> 30 #define LONG_S LONG_SP
>> 31 #else
>> 32 #define STORSIZE LONGSIZE
>> 33 #define STORMASK LONGMASK
>> 34 #define FILL64RG a1
>> 35 #define FILLPTRG t0
>> 36 #endif
>> 37
>> 38 #define LEGACY_MODE 1
>> 39 #define EVA_MODE 2
>> 40
>> 41 /*
>> 42 * No need to protect it with EVA #ifdefery. The generated block of code
>> 43 * will never be assembled if EVA is not enabled.
10 */ 44 */
>> 45 #define __EVAFY(insn, reg, addr) __BUILD_EVA_INSN(insn##e, reg, addr)
>> 46 #define ___BUILD_EVA_INSN(insn, reg, addr) __EVAFY(insn, reg, addr)
11 47
12 .global memset !! 48 #define EX(insn,reg,addr,handler) \
13 .type memset, @function !! 49 .if \mode == LEGACY_MODE; \
14 memset: !! 50 9: insn reg, addr; \
15 /* arguments: !! 51 .else; \
16 * r3 = *s !! 52 9: ___BUILD_EVA_INSN(insn, reg, addr); \
17 * r4 = c !! 53 .endif; \
18 * r5 = n !! 54 .section __ex_table,"a"; \
19 * r13, r15, r17, r19 used as temp reg !! 55 PTR 9b, handler; \
20 */ !! 56 .previous
21 !! 57
22 /* Exit if n == 0 */ !! 58 .macro f_fill64 dst, offset, val, fixup, mode
23 l.sfeqi r5, 0 !! 59 EX(LONG_S, \val, (\offset + 0 * STORSIZE)(\dst), \fixup)
24 l.bf 4f !! 60 EX(LONG_S, \val, (\offset + 1 * STORSIZE)(\dst), \fixup)
25 !! 61 EX(LONG_S, \val, (\offset + 2 * STORSIZE)(\dst), \fixup)
26 /* Truncate c to char */ !! 62 EX(LONG_S, \val, (\offset + 3 * STORSIZE)(\dst), \fixup)
27 l.andi r13, r4, 0xff !! 63 #if ((defined(CONFIG_CPU_MICROMIPS) && (LONGSIZE == 4)) || !defined(CONFIG_CPU_MICROMIPS))
28 !! 64 EX(LONG_S, \val, (\offset + 4 * STORSIZE)(\dst), \fixup)
29 /* Skip word extension if c is 0 */ !! 65 EX(LONG_S, \val, (\offset + 5 * STORSIZE)(\dst), \fixup)
30 l.sfeqi r13, 0 !! 66 EX(LONG_S, \val, (\offset + 6 * STORSIZE)(\dst), \fixup)
31 l.bf 1f !! 67 EX(LONG_S, \val, (\offset + 7 * STORSIZE)(\dst), \fixup)
32 /* Check for at least two whole words !! 68 #endif
33 l.sfleui r5, 7 !! 69 #if (!defined(CONFIG_CPU_MICROMIPS) && (LONGSIZE == 4))
34 !! 70 EX(LONG_S, \val, (\offset + 8 * STORSIZE)(\dst), \fixup)
35 /* Extend char c to 32-bit word cccc i !! 71 EX(LONG_S, \val, (\offset + 9 * STORSIZE)(\dst), \fixup)
36 l.slli r15, r13, 16 // r13 = !! 72 EX(LONG_S, \val, (\offset + 10 * STORSIZE)(\dst), \fixup)
37 l.or r13, r13, r15 // r13 = !! 73 EX(LONG_S, \val, (\offset + 11 * STORSIZE)(\dst), \fixup)
38 l.slli r15, r13, 8 // r13 = !! 74 EX(LONG_S, \val, (\offset + 12 * STORSIZE)(\dst), \fixup)
39 l.or r13, r13, r15 // r13 = !! 75 EX(LONG_S, \val, (\offset + 13 * STORSIZE)(\dst), \fixup)
40 !! 76 EX(LONG_S, \val, (\offset + 14 * STORSIZE)(\dst), \fixup)
41 1: l.addi r19, r3, 0 // Set r19 !! 77 EX(LONG_S, \val, (\offset + 15 * STORSIZE)(\dst), \fixup)
42 /* Jump to byte copy loop if less than !! 78 #endif
43 l.bf 3f !! 79 .endm
44 l.or r17, r5, r0 // Set r17 !! 80
45 !! 81 .set noreorder
46 /* Mask out two LSBs to check alignmen !! 82 .align 5
47 l.andi r15, r3, 0x3 !! 83
48 !! 84 /*
49 /* lsb == 00, jump to word copy loop * !! 85 * Macro to generate the __bzero{,_user} symbol
50 l.sfeqi r15, 0 !! 86 * Arguments:
51 l.bf 2f !! 87 * mode: LEGACY_MODE or EVA_MODE
52 l.addi r19, r3, 0 // Set r19 !! 88 */
53 !! 89 .macro __BUILD_BZERO mode
54 /* lsb == 01,10 or 11 */ !! 90 /* Initialize __memset if this is the first time we call this macro */
55 l.sb 0(r3), r13 // *src = !! 91 .ifnotdef __memset
56 l.addi r17, r17, -1 // Decrea !! 92 .set __memset, 1
57 !! 93 .hidden __memset /* Make sure it does not leak */
58 l.sfeqi r15, 3 !! 94 .endif
59 l.bf 2f !! 95
60 l.addi r19, r3, 1 // src += !! 96 sltiu t0, a2, STORSIZE /* very small region? */
61 !! 97 bnez t0, .Lsmall_memset\@
62 /* lsb == 01 or 10 */ !! 98 andi t0, a0, STORMASK /* aligned? */
63 l.sb 1(r3), r13 // *(src+ !! 99
64 l.addi r17, r17, -1 // Decrea !! 100 #ifdef CONFIG_CPU_MICROMIPS
65 !! 101 move t8, a1 /* used by 'swp' instruction */
66 l.sfeqi r15, 2 !! 102 move t9, a1
67 l.bf 2f !! 103 #endif
68 l.addi r19, r3, 2 // src += !! 104 #ifndef CONFIG_CPU_DADDI_WORKAROUNDS
69 !! 105 beqz t0, 1f
70 /* lsb == 01 */ !! 106 PTR_SUBU t0, STORSIZE /* alignment in bytes */
71 l.sb 2(r3), r13 // *(src+ !! 107 #else
72 l.addi r17, r17, -1 // Decrea !! 108 .set noat
73 l.addi r19, r3, 3 // src += !! 109 li AT, STORSIZE
74 !! 110 beqz t0, 1f
75 /* Word copy loop */ !! 111 PTR_SUBU t0, AT /* alignment in bytes */
76 2: l.sw 0(r19), r13 // *src = !! 112 .set at
77 l.addi r17, r17, -4 // Decrea !! 113 #endif
78 l.sfgeui r17, 4 !! 114
79 l.bf 2b !! 115 #ifndef CONFIG_CPU_MIPSR6
80 l.addi r19, r19, 4 // Increa !! 116 R10KCBARRIER(0(ra))
81 !! 117 #ifdef __MIPSEB__
82 /* When n > 0, copy the remaining byte !! 118 EX(LONG_S_L, a1, (a0), .Lfirst_fixup\@) /* make word/dword aligned */
83 l.sfeqi r17, 0 !! 119 #else
84 l.bf 4f !! 120 EX(LONG_S_R, a1, (a0), .Lfirst_fixup\@) /* make word/dword aligned */
85 !! 121 #endif
86 /* Byte copy loop */ !! 122 PTR_SUBU a0, t0 /* long align ptr */
87 3: l.addi r17, r17, -1 // Decrea !! 123 PTR_ADDU a2, t0 /* correct size */
88 l.sb 0(r19), r13 // *src = !! 124
89 l.sfnei r17, 0 !! 125 #else /* CONFIG_CPU_MIPSR6 */
90 l.bf 3b !! 126 #define STORE_BYTE(N) \
91 l.addi r19, r19, 1 // Increa !! 127 EX(sb, a1, N(a0), .Lbyte_fixup\@); \
>> 128 beqz t0, 0f; \
>> 129 PTR_ADDU t0, 1;
>> 130
>> 131 PTR_ADDU a2, t0 /* correct size */
>> 132 PTR_ADDU t0, 1
>> 133 STORE_BYTE(0)
>> 134 STORE_BYTE(1)
>> 135 #if LONGSIZE == 4
>> 136 EX(sb, a1, 2(a0), .Lbyte_fixup\@)
>> 137 #else
>> 138 STORE_BYTE(2)
>> 139 STORE_BYTE(3)
>> 140 STORE_BYTE(4)
>> 141 STORE_BYTE(5)
>> 142 EX(sb, a1, 6(a0), .Lbyte_fixup\@)
>> 143 #endif
>> 144 0:
>> 145 ori a0, STORMASK
>> 146 xori a0, STORMASK
>> 147 PTR_ADDIU a0, STORSIZE
>> 148 #endif /* CONFIG_CPU_MIPSR6 */
>> 149 1: ori t1, a2, 0x3f /* # of full blocks */
>> 150 xori t1, 0x3f
>> 151 beqz t1, .Lmemset_partial\@ /* no block to fill */
>> 152 andi t0, a2, 0x40-STORSIZE
>> 153
>> 154 PTR_ADDU t1, a0 /* end address */
>> 155 .set reorder
>> 156 1: PTR_ADDIU a0, 64
>> 157 R10KCBARRIER(0(ra))
>> 158 f_fill64 a0, -64, FILL64RG, .Lfwd_fixup\@, \mode
>> 159 bne t1, a0, 1b
>> 160 .set noreorder
>> 161
>> 162 .Lmemset_partial\@:
>> 163 R10KCBARRIER(0(ra))
>> 164 PTR_LA t1, 2f /* where to start */
>> 165 #ifdef CONFIG_CPU_MICROMIPS
>> 166 LONG_SRL t7, t0, 1
>> 167 #endif
>> 168 #if LONGSIZE == 4
>> 169 PTR_SUBU t1, FILLPTRG
>> 170 #else
>> 171 .set noat
>> 172 LONG_SRL AT, FILLPTRG, 1
>> 173 PTR_SUBU t1, AT
>> 174 .set at
>> 175 #endif
>> 176 jr t1
>> 177 PTR_ADDU a0, t0 /* dest ptr */
>> 178
>> 179 .set push
>> 180 .set noreorder
>> 181 .set nomacro
>> 182 /* ... but first do longs ... */
>> 183 f_fill64 a0, -64, FILL64RG, .Lpartial_fixup\@, \mode
>> 184 2: .set pop
>> 185 andi a2, STORMASK /* At most one long to go */
>> 186
>> 187 beqz a2, 1f
>> 188 #ifndef CONFIG_CPU_MIPSR6
>> 189 PTR_ADDU a0, a2 /* What's left */
>> 190 R10KCBARRIER(0(ra))
>> 191 #ifdef __MIPSEB__
>> 192 EX(LONG_S_R, a1, -1(a0), .Llast_fixup\@)
>> 193 #else
>> 194 EX(LONG_S_L, a1, -1(a0), .Llast_fixup\@)
>> 195 #endif
>> 196 #else
>> 197 PTR_SUBU t0, $0, a2
>> 198 PTR_ADDIU t0, 1
>> 199 STORE_BYTE(0)
>> 200 STORE_BYTE(1)
>> 201 #if LONGSIZE == 4
>> 202 EX(sb, a1, 2(a0), .Lbyte_fixup\@)
>> 203 #else
>> 204 STORE_BYTE(2)
>> 205 STORE_BYTE(3)
>> 206 STORE_BYTE(4)
>> 207 STORE_BYTE(5)
>> 208 EX(sb, a1, 6(a0), .Lbyte_fixup\@)
>> 209 #endif
>> 210 0:
>> 211 #endif
>> 212 1: jr ra
>> 213 move a2, zero
>> 214
>> 215 .Lsmall_memset\@:
>> 216 beqz a2, 2f
>> 217 PTR_ADDU t1, a0, a2
>> 218
>> 219 1: PTR_ADDIU a0, 1 /* fill bytewise */
>> 220 R10KCBARRIER(0(ra))
>> 221 bne t1, a0, 1b
>> 222 sb a1, -1(a0)
>> 223
>> 224 2: jr ra /* done */
>> 225 move a2, zero
>> 226 .if __memset == 1
>> 227 END(memset)
>> 228 .set __memset, 0
>> 229 .hidden __memset
>> 230 .endif
>> 231
>> 232 #ifdef CONFIG_CPU_MIPSR6
>> 233 .Lbyte_fixup\@:
>> 234 PTR_SUBU a2, $0, t0
>> 235 jr ra
>> 236 PTR_ADDIU a2, 1
>> 237 #endif /* CONFIG_CPU_MIPSR6 */
>> 238
>> 239 .Lfirst_fixup\@:
>> 240 jr ra
>> 241 nop
>> 242
>> 243 .Lfwd_fixup\@:
>> 244 PTR_L t0, TI_TASK($28)
>> 245 andi a2, 0x3f
>> 246 LONG_L t0, THREAD_BUADDR(t0)
>> 247 LONG_ADDU a2, t1
>> 248 jr ra
>> 249 LONG_SUBU a2, t0
>> 250
>> 251 .Lpartial_fixup\@:
>> 252 PTR_L t0, TI_TASK($28)
>> 253 andi a2, STORMASK
>> 254 LONG_L t0, THREAD_BUADDR(t0)
>> 255 LONG_ADDU a2, t1
>> 256 jr ra
>> 257 LONG_SUBU a2, t0
>> 258
>> 259 .Llast_fixup\@:
>> 260 jr ra
>> 261 andi v1, a2, STORMASK
>> 262
>> 263 .endm
>> 264
>> 265 /*
>> 266 * memset(void *s, int c, size_t n)
>> 267 *
>> 268 * a0: start of area to clear
>> 269 * a1: char to fill with
>> 270 * a2: size of area to clear
>> 271 */
92 272
93 4: l.jr r9 !! 273 LEAF(memset)
94 l.ori r11, r3, 0 !! 274 EXPORT_SYMBOL(memset)
>> 275 beqz a1, 1f
>> 276 move v0, a0 /* result */
>> 277
>> 278 andi a1, 0xff /* spread fillword */
>> 279 LONG_SLL t1, a1, 8
>> 280 or a1, t1
>> 281 LONG_SLL t1, a1, 16
>> 282 #if LONGSIZE == 8
>> 283 or a1, t1
>> 284 LONG_SLL t1, a1, 32
>> 285 #endif
>> 286 or a1, t1
>> 287 1:
>> 288 #ifndef CONFIG_EVA
>> 289 FEXPORT(__bzero)
>> 290 EXPORT_SYMBOL(__bzero)
>> 291 #else
>> 292 FEXPORT(__bzero_kernel)
>> 293 EXPORT_SYMBOL(__bzero_kernel)
>> 294 #endif
>> 295 __BUILD_BZERO LEGACY_MODE
>> 296
>> 297 #ifdef CONFIG_EVA
>> 298 LEAF(__bzero)
>> 299 EXPORT_SYMBOL(__bzero)
>> 300 __BUILD_BZERO EVA_MODE
>> 301 END(__bzero)
>> 302 #endif
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.