1 /* SPDX-License-Identifier: GPL-2.0 */ <<
2 /* 1 /*
3 * Copyright (C) 2020-2022 Loongson Technology !! 2 * This file is subject to the terms and conditions of the GNU General Public
>> 3 * License. See the file "COPYING" in the main directory of this archive
>> 4 * for more details.
>> 5 *
>> 6 * Copyright (C) 1998, 1999, 2000 by Ralf Baechle
>> 7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
>> 8 * Copyright (C) 2007 by Maciej W. Rozycki
>> 9 * Copyright (C) 2011, 2012 MIPS Technologies, Inc.
4 */ 10 */
5 <<
6 #include <linux/export.h> <<
7 #include <asm/alternative-asm.h> <<
8 #include <asm/asm.h> 11 #include <asm/asm.h>
9 #include <asm/asmmacro.h> !! 12 #include <asm/asm-offsets.h>
10 #include <asm/cpu.h> <<
11 #include <asm/regdef.h> 13 #include <asm/regdef.h>
12 #include <asm/unwind_hints.h> <<
13 <<
14 .macro fill_to_64 r0 <<
15 bstrins.d \r0, \r0, 15, 8 <<
16 bstrins.d \r0, \r0, 31, 16 <<
17 bstrins.d \r0, \r0, 63, 32 <<
18 .endm <<
19 <<
20 .section .noinstr.text, "ax" <<
21 <<
22 SYM_FUNC_START(memset) <<
23 /* <<
24 * Some CPUs support hardware unaligne <<
25 */ <<
26 ALTERNATIVE "b __memset_generic", <<
27 "b __memset_fast", CPU <<
28 SYM_FUNC_END(memset) <<
29 SYM_FUNC_ALIAS(__memset, memset) <<
30 <<
31 EXPORT_SYMBOL(memset) <<
32 EXPORT_SYMBOL(__memset) <<
33 <<
34 _ASM_NOKPROBE(memset) <<
35 _ASM_NOKPROBE(__memset) <<
36 <<
37 /* <<
38 * void *__memset_generic(void *s, int c, size <<
39 * <<
40 * a0: s <<
41 * a1: c <<
42 * a2: n <<
43 */ <<
44 SYM_FUNC_START(__memset_generic) <<
45 move a3, a0 <<
46 beqz a2, 2f <<
47 <<
48 1: st.b a1, a0, 0 <<
49 addi.d a0, a0, 1 <<
50 addi.d a2, a2, -1 <<
51 bgt a2, zero, 1b <<
52 14
53 2: move a0, a3 !! 15 #if LONGSIZE == 4
54 jr ra !! 16 #define LONG_S_L swl
55 SYM_FUNC_END(__memset_generic) !! 17 #define LONG_S_R swr
56 _ASM_NOKPROBE(__memset_generic) !! 18 #else
>> 19 #define LONG_S_L sdl
>> 20 #define LONG_S_R sdr
>> 21 #endif
>> 22
>> 23 #ifdef CONFIG_CPU_MICROMIPS
>> 24 #define STORSIZE (LONGSIZE * 2)
>> 25 #define STORMASK (STORSIZE - 1)
>> 26 #define FILL64RG t8
>> 27 #define FILLPTRG t7
>> 28 #undef LONG_S
>> 29 #define LONG_S LONG_SP
>> 30 #else
>> 31 #define STORSIZE LONGSIZE
>> 32 #define STORMASK LONGMASK
>> 33 #define FILL64RG a1
>> 34 #define FILLPTRG t0
>> 35 #endif
>> 36
>> 37 #define EX(insn,reg,addr,handler) \
>> 38 9: insn reg, addr; \
>> 39 .section __ex_table,"a"; \
>> 40 PTR 9b, handler; \
>> 41 .previous
>> 42
>> 43 .macro f_fill64 dst, offset, val, fixup
>> 44 EX(LONG_S, \val, (\offset + 0 * STORSIZE)(\dst), \fixup)
>> 45 EX(LONG_S, \val, (\offset + 1 * STORSIZE)(\dst), \fixup)
>> 46 EX(LONG_S, \val, (\offset + 2 * STORSIZE)(\dst), \fixup)
>> 47 EX(LONG_S, \val, (\offset + 3 * STORSIZE)(\dst), \fixup)
>> 48 #if ((defined(CONFIG_CPU_MICROMIPS) && (LONGSIZE == 4)) || !defined(CONFIG_CPU_MICROMIPS))
>> 49 EX(LONG_S, \val, (\offset + 4 * STORSIZE)(\dst), \fixup)
>> 50 EX(LONG_S, \val, (\offset + 5 * STORSIZE)(\dst), \fixup)
>> 51 EX(LONG_S, \val, (\offset + 6 * STORSIZE)(\dst), \fixup)
>> 52 EX(LONG_S, \val, (\offset + 7 * STORSIZE)(\dst), \fixup)
>> 53 #endif
>> 54 #if (!defined(CONFIG_CPU_MICROMIPS) && (LONGSIZE == 4))
>> 55 EX(LONG_S, \val, (\offset + 8 * STORSIZE)(\dst), \fixup)
>> 56 EX(LONG_S, \val, (\offset + 9 * STORSIZE)(\dst), \fixup)
>> 57 EX(LONG_S, \val, (\offset + 10 * STORSIZE)(\dst), \fixup)
>> 58 EX(LONG_S, \val, (\offset + 11 * STORSIZE)(\dst), \fixup)
>> 59 EX(LONG_S, \val, (\offset + 12 * STORSIZE)(\dst), \fixup)
>> 60 EX(LONG_S, \val, (\offset + 13 * STORSIZE)(\dst), \fixup)
>> 61 EX(LONG_S, \val, (\offset + 14 * STORSIZE)(\dst), \fixup)
>> 62 EX(LONG_S, \val, (\offset + 15 * STORSIZE)(\dst), \fixup)
>> 63 #endif
>> 64 .endm
57 65
58 /* 66 /*
59 * void *__memset_fast(void *s, int c, size_t !! 67 * memset(void *s, int c, size_t n)
60 * 68 *
61 * a0: s !! 69 * a0: start of area to clear
62 * a1: c !! 70 * a1: char to fill with
63 * a2: n !! 71 * a2: size of area to clear
64 */ 72 */
65 SYM_FUNC_START(__memset_fast) !! 73 .set noreorder
66 /* fill a1 to 64 bits */ !! 74 .align 5
67 fill_to_64 a1 !! 75 LEAF(memset)
68 !! 76 beqz a1, 1f
69 sltui t0, a2, 9 !! 77 move v0, a0 /* result */
70 bnez t0, .Lsmall !! 78
71 !! 79 andi a1, 0xff /* spread fillword */
72 add.d a2, a0, a2 !! 80 LONG_SLL t1, a1, 8
73 st.d a1, a0, 0 !! 81 or a1, t1
74 !! 82 LONG_SLL t1, a1, 16
75 /* align up address */ !! 83 #if LONGSIZE == 8
76 addi.d a3, a0, 8 !! 84 or a1, t1
77 bstrins.d a3, zero, 2, 0 !! 85 LONG_SLL t1, a1, 32
78 !! 86 #endif
79 addi.d a4, a2, -64 !! 87 or a1, t1
80 bgeu a3, a4, .Llt64 !! 88 1:
81 !! 89
82 /* set 64 bytes at a time */ !! 90 FEXPORT(__bzero)
83 .Lloop64: !! 91 sltiu t0, a2, STORSIZE /* very small region? */
84 st.d a1, a3, 0 !! 92 bnez t0, .Lsmall_memset
85 st.d a1, a3, 8 !! 93 andi t0, a0, STORMASK /* aligned? */
86 st.d a1, a3, 16 !! 94
87 st.d a1, a3, 24 !! 95 #ifdef CONFIG_CPU_MICROMIPS
88 st.d a1, a3, 32 !! 96 move t8, a1 /* used by 'swp' instruction */
89 st.d a1, a3, 40 !! 97 move t9, a1
90 st.d a1, a3, 48 !! 98 #endif
91 st.d a1, a3, 56 !! 99 #ifndef CONFIG_CPU_DADDI_WORKAROUNDS
92 addi.d a3, a3, 64 !! 100 beqz t0, 1f
93 bltu a3, a4, .Lloop64 !! 101 PTR_SUBU t0, STORSIZE /* alignment in bytes */
94 !! 102 #else
95 /* set the remaining bytes */ !! 103 .set noat
96 .Llt64: !! 104 li AT, STORSIZE
97 addi.d a4, a2, -32 !! 105 beqz t0, 1f
98 bgeu a3, a4, .Llt32 !! 106 PTR_SUBU t0, AT /* alignment in bytes */
99 st.d a1, a3, 0 !! 107 .set at
100 st.d a1, a3, 8 !! 108 #endif
101 st.d a1, a3, 16 !! 109
102 st.d a1, a3, 24 !! 110 R10KCBARRIER(0(ra))
103 addi.d a3, a3, 32 !! 111 #ifdef __MIPSEB__
104 !! 112 EX(LONG_S_L, a1, (a0), .Lfirst_fixup) /* make word/dword aligned */
105 .Llt32: !! 113 #endif
106 addi.d a4, a2, -16 !! 114 #ifdef __MIPSEL__
107 bgeu a3, a4, .Llt16 !! 115 EX(LONG_S_R, a1, (a0), .Lfirst_fixup) /* make word/dword aligned */
108 st.d a1, a3, 0 !! 116 #endif
109 st.d a1, a3, 8 !! 117 PTR_SUBU a0, t0 /* long align ptr */
110 addi.d a3, a3, 16 !! 118 PTR_ADDU a2, t0 /* correct size */
111 !! 119
112 .Llt16: !! 120 1: ori t1, a2, 0x3f /* # of full blocks */
113 addi.d a4, a2, -8 !! 121 xori t1, 0x3f
114 bgeu a3, a4, .Llt8 !! 122 beqz t1, .Lmemset_partial /* no block to fill */
115 st.d a1, a3, 0 !! 123 andi t0, a2, 0x40-STORSIZE
116 !! 124
117 .Llt8: !! 125 PTR_ADDU t1, a0 /* end address */
118 st.d a1, a2, -8 !! 126 .set reorder
119 !! 127 1: PTR_ADDIU a0, 64
120 /* return */ !! 128 R10KCBARRIER(0(ra))
121 jr ra !! 129 f_fill64 a0, -64, FILL64RG, .Lfwd_fixup
122 !! 130 bne t1, a0, 1b
123 .align 4 !! 131 .set noreorder
124 .Lsmall: !! 132
125 pcaddi t0, 4 !! 133 .Lmemset_partial:
126 slli.d a2, a2, 4 !! 134 R10KCBARRIER(0(ra))
127 add.d t0, t0, a2 !! 135 PTR_LA t1, 2f /* where to start */
128 jr t0 !! 136 #ifdef CONFIG_CPU_MICROMIPS
129 !! 137 LONG_SRL t7, t0, 1
130 .align 4 !! 138 #endif
131 0: jr ra !! 139 #if LONGSIZE == 4
132 !! 140 PTR_SUBU t1, FILLPTRG
133 .align 4 !! 141 #else
134 1: st.b a1, a0, 0 !! 142 .set noat
135 jr ra !! 143 LONG_SRL AT, FILLPTRG, 1
136 !! 144 PTR_SUBU t1, AT
137 .align 4 !! 145 .set at
138 2: st.h a1, a0, 0 !! 146 #endif
139 jr ra !! 147 jr t1
140 !! 148 PTR_ADDU a0, t0 /* dest ptr */
141 .align 4 !! 149
142 3: st.h a1, a0, 0 !! 150 .set push
143 st.b a1, a0, 2 !! 151 .set noreorder
144 jr ra !! 152 .set nomacro
145 !! 153 f_fill64 a0, -64, FILL64RG, .Lpartial_fixup /* ... but first do longs ... */
146 .align 4 !! 154 2: .set pop
147 4: st.w a1, a0, 0 !! 155 andi a2, STORMASK /* At most one long to go */
148 jr ra !! 156
149 !! 157 beqz a2, 1f
150 .align 4 !! 158 PTR_ADDU a0, a2 /* What's left */
151 5: st.w a1, a0, 0 !! 159 R10KCBARRIER(0(ra))
152 st.b a1, a0, 4 !! 160 #ifdef __MIPSEB__
153 jr ra !! 161 EX(LONG_S_R, a1, -1(a0), .Llast_fixup)
154 !! 162 #endif
155 .align 4 !! 163 #ifdef __MIPSEL__
156 6: st.w a1, a0, 0 !! 164 EX(LONG_S_L, a1, -1(a0), .Llast_fixup)
157 st.h a1, a0, 4 !! 165 #endif
158 jr ra !! 166 1: jr ra
159 !! 167 move a2, zero
160 .align 4 !! 168
161 7: st.w a1, a0, 0 !! 169 .Lsmall_memset:
162 st.w a1, a0, 3 !! 170 beqz a2, 2f
163 jr ra !! 171 PTR_ADDU t1, a0, a2
164 !! 172
165 .align 4 !! 173 1: PTR_ADDIU a0, 1 /* fill bytewise */
166 8: st.d a1, a0, 0 !! 174 R10KCBARRIER(0(ra))
167 jr ra !! 175 bne t1, a0, 1b
168 SYM_FUNC_END(__memset_fast) !! 176 sb a1, -1(a0)
169 _ASM_NOKPROBE(__memset_fast) !! 177
170 !! 178 2: jr ra /* done */
171 STACK_FRAME_NON_STANDARD __memset_fast !! 179 move a2, zero
>> 180 END(memset)
>> 181
>> 182 .Lfirst_fixup:
>> 183 jr ra
>> 184 nop
>> 185
>> 186 .Lfwd_fixup:
>> 187 PTR_L t0, TI_TASK($28)
>> 188 andi a2, 0x3f
>> 189 LONG_L t0, THREAD_BUADDR(t0)
>> 190 LONG_ADDU a2, t1
>> 191 jr ra
>> 192 LONG_SUBU a2, t0
>> 193
>> 194 .Lpartial_fixup:
>> 195 PTR_L t0, TI_TASK($28)
>> 196 andi a2, STORMASK
>> 197 LONG_L t0, THREAD_BUADDR(t0)
>> 198 LONG_ADDU a2, t1
>> 199 jr ra
>> 200 LONG_SUBU a2, t0
>> 201
>> 202 .Llast_fixup:
>> 203 jr ra
>> 204 andi v1, a2, STORMASK
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.