1 /* SPDX-License-Identifier: GPL-2.0 */ <<
2 /* linux/arch/sparc/lib/memset.S: Sparc optimi 1 /* linux/arch/sparc/lib/memset.S: Sparc optimized memset, bzero and clear_user code
3 * Copyright (C) 1991,1996 Free Software Found 2 * Copyright (C) 1991,1996 Free Software Foundation
4 * Copyright (C) 1996,1997 Jakub Jelinek (jj@s 3 * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
5 * Copyright (C) 1996 David S. Miller (davem@c 4 * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
6 * 5 *
7 * Calls to memset returns initial %o0. Calls 6 * Calls to memset returns initial %o0. Calls to bzero returns 0, if ok, and
8 * number of bytes not yet set if exception oc 7 * number of bytes not yet set if exception occurs and we were called as
9 * clear_user. 8 * clear_user.
10 */ 9 */
11 10
12 #include <linux/export.h> <<
13 #include <asm/ptrace.h> 11 #include <asm/ptrace.h>
14 12
15 /* Work around cpp -rob */ 13 /* Work around cpp -rob */
16 #define ALLOC #alloc 14 #define ALLOC #alloc
17 #define EXECINSTR #execinstr 15 #define EXECINSTR #execinstr
18 #define EX(x,y,a,b) 16 #define EX(x,y,a,b) \
19 98: x,y; 17 98: x,y; \
20 .section .fixup,ALLOC,EXECINSTR; 18 .section .fixup,ALLOC,EXECINSTR; \
21 .align 4; 19 .align 4; \
22 99: retl; !! 20 99: ba 30f; \
23 a, b, %o0; 21 a, b, %o0; \
24 .section __ex_table,ALLOC; 22 .section __ex_table,ALLOC; \
25 .align 4; 23 .align 4; \
26 .word 98b, 99b; 24 .word 98b, 99b; \
27 .text; 25 .text; \
28 .align 4 26 .align 4
29 27
30 #define STORE(source, base, offset, n) !! 28 #define EXT(start,end,handler) \
31 98: std source, [base + offset + n]; <<
32 .section .fixup,ALLOC,EXECINSTR; <<
33 .align 4; <<
34 99: ba 30f; <<
35 sub %o3, n - offset, %o3; <<
36 .section __ex_table,ALLOC; 29 .section __ex_table,ALLOC; \
37 .align 4; 30 .align 4; \
38 .word 98b, 99b; !! 31 .word start, 0, end, handler; \
39 .text; 32 .text; \
40 .align 4; !! 33 .align 4
41 <<
42 #define STORE_LAST(source, base, offset, n) <<
43 EX(std source, [base - offset - n], <<
44 add %o1, offset + n); <<
45 34
46 /* Please don't change these macros, unless yo 35 /* Please don't change these macros, unless you change the logic
47 * in the .fixup section below as well. 36 * in the .fixup section below as well.
48 * Store 64 bytes at (BASE + OFFSET) using val 37 * Store 64 bytes at (BASE + OFFSET) using value SOURCE. */
49 #define ZERO_BIG_BLOCK(base, offset, source) !! 38 #define ZERO_BIG_BLOCK(base, offset, source) \
50 STORE(source, base, offset, 0x00); !! 39 std source, [base + offset + 0x00]; \
51 STORE(source, base, offset, 0x08); !! 40 std source, [base + offset + 0x08]; \
52 STORE(source, base, offset, 0x10); !! 41 std source, [base + offset + 0x10]; \
53 STORE(source, base, offset, 0x18); !! 42 std source, [base + offset + 0x18]; \
54 STORE(source, base, offset, 0x20); !! 43 std source, [base + offset + 0x20]; \
55 STORE(source, base, offset, 0x28); !! 44 std source, [base + offset + 0x28]; \
56 STORE(source, base, offset, 0x30); !! 45 std source, [base + offset + 0x30]; \
57 STORE(source, base, offset, 0x38); !! 46 std source, [base + offset + 0x38];
58 47
59 #define ZERO_LAST_BLOCKS(base, offset, source) 48 #define ZERO_LAST_BLOCKS(base, offset, source) \
60 STORE_LAST(source, base, offset, 0x38) !! 49 std source, [base - offset - 0x38]; \
61 STORE_LAST(source, base, offset, 0x30) !! 50 std source, [base - offset - 0x30]; \
62 STORE_LAST(source, base, offset, 0x28) !! 51 std source, [base - offset - 0x28]; \
63 STORE_LAST(source, base, offset, 0x20) !! 52 std source, [base - offset - 0x20]; \
64 STORE_LAST(source, base, offset, 0x18) !! 53 std source, [base - offset - 0x18]; \
65 STORE_LAST(source, base, offset, 0x10) !! 54 std source, [base - offset - 0x10]; \
66 STORE_LAST(source, base, offset, 0x08) !! 55 std source, [base - offset - 0x08]; \
67 STORE_LAST(source, base, offset, 0x00) !! 56 std source, [base - offset - 0x00];
68 57
69 .text 58 .text
70 .align 4 59 .align 4
71 60
72 .globl __bzero_begin 61 .globl __bzero_begin
73 __bzero_begin: 62 __bzero_begin:
74 63
75 .globl __bzero 64 .globl __bzero
76 .type __bzero,#function <<
77 .globl memset 65 .globl memset
78 EXPORT_SYMBOL(__bzero) !! 66 .globl __memset_start, __memset_end
79 EXPORT_SYMBOL(memset) !! 67 __memset_start:
80 memset: 68 memset:
81 mov %o0, %g1 69 mov %o0, %g1
82 mov 1, %g4 70 mov 1, %g4
83 and %o1, 0xff, %g3 71 and %o1, 0xff, %g3
84 sll %g3, 8, %g2 72 sll %g3, 8, %g2
85 or %g3, %g2, %g3 73 or %g3, %g2, %g3
86 sll %g3, 16, %g2 74 sll %g3, 16, %g2
87 or %g3, %g2, %g3 75 or %g3, %g2, %g3
88 b 1f 76 b 1f
89 mov %o2, %o1 77 mov %o2, %o1
90 3: 78 3:
91 cmp %o2, 3 79 cmp %o2, 3
92 be 2f 80 be 2f
93 EX(stb %g3, [%o0], sub %o1, 0) 81 EX(stb %g3, [%o0], sub %o1, 0)
94 82
95 cmp %o2, 2 83 cmp %o2, 2
96 be 2f 84 be 2f
97 EX(stb %g3, [%o0 + 0x01], sub %o1, 1) 85 EX(stb %g3, [%o0 + 0x01], sub %o1, 1)
98 86
99 EX(stb %g3, [%o0 + 0x02], sub %o1, 2) 87 EX(stb %g3, [%o0 + 0x02], sub %o1, 2)
100 2: 88 2:
101 sub %o2, 4, %o2 89 sub %o2, 4, %o2
102 add %o1, %o2, %o1 90 add %o1, %o2, %o1
103 b 4f 91 b 4f
104 sub %o0, %o2, %o0 92 sub %o0, %o2, %o0
105 93
106 __bzero: 94 __bzero:
107 clr %g4 95 clr %g4
108 mov %g0, %g3 96 mov %g0, %g3
109 1: 97 1:
110 cmp %o1, 7 98 cmp %o1, 7
111 bleu 7f 99 bleu 7f
112 andcc %o0, 3, %o2 100 andcc %o0, 3, %o2
113 101
114 bne 3b 102 bne 3b
115 4: 103 4:
116 andcc %o0, 4, %g0 104 andcc %o0, 4, %g0
117 105
118 be 2f 106 be 2f
119 mov %g3, %g2 107 mov %g3, %g2
120 108
121 EX(st %g3, [%o0], sub %o1, 0) 109 EX(st %g3, [%o0], sub %o1, 0)
122 sub %o1, 4, %o1 110 sub %o1, 4, %o1
123 add %o0, 4, %o0 111 add %o0, 4, %o0
124 2: 112 2:
125 andcc %o1, 0xffffff80, %o3 ! Now 113 andcc %o1, 0xffffff80, %o3 ! Now everything is 8 aligned and o1 is len to run
126 be 9f 114 be 9f
127 andcc %o1, 0x78, %o2 115 andcc %o1, 0x78, %o2
128 10: 116 10:
129 ZERO_BIG_BLOCK(%o0, 0x00, %g2) 117 ZERO_BIG_BLOCK(%o0, 0x00, %g2)
130 subcc %o3, 128, %o3 118 subcc %o3, 128, %o3
131 ZERO_BIG_BLOCK(%o0, 0x40, %g2) 119 ZERO_BIG_BLOCK(%o0, 0x40, %g2)
>> 120 11:
>> 121 EXT(10b, 11b, 20f)
132 bne 10b 122 bne 10b
133 add %o0, 128, %o0 123 add %o0, 128, %o0
134 124
135 orcc %o2, %g0, %g0 125 orcc %o2, %g0, %g0
136 9: 126 9:
137 be 13f 127 be 13f
138 andcc %o1, 7, %o1 128 andcc %o1, 7, %o1
139 129
140 srl %o2, 1, %o3 130 srl %o2, 1, %o3
141 set 13f, %o4 131 set 13f, %o4
142 sub %o4, %o3, %o4 132 sub %o4, %o3, %o4
143 jmp %o4 133 jmp %o4
144 add %o0, %o2, %o0 134 add %o0, %o2, %o0
145 135
>> 136 12:
146 ZERO_LAST_BLOCKS(%o0, 0x48, %g2) 137 ZERO_LAST_BLOCKS(%o0, 0x48, %g2)
147 ZERO_LAST_BLOCKS(%o0, 0x08, %g2) 138 ZERO_LAST_BLOCKS(%o0, 0x08, %g2)
148 13: 139 13:
>> 140 EXT(12b, 13b, 21f)
149 be 8f 141 be 8f
150 andcc %o1, 4, %g0 142 andcc %o1, 4, %g0
151 143
152 be 1f 144 be 1f
153 andcc %o1, 2, %g0 145 andcc %o1, 2, %g0
154 146
155 EX(st %g3, [%o0], and %o1, 7) 147 EX(st %g3, [%o0], and %o1, 7)
156 add %o0, 4, %o0 148 add %o0, 4, %o0
157 1: 149 1:
158 be 1f 150 be 1f
159 andcc %o1, 1, %g0 151 andcc %o1, 1, %g0
160 152
161 EX(sth %g3, [%o0], and %o1, 3) 153 EX(sth %g3, [%o0], and %o1, 3)
162 add %o0, 2, %o0 154 add %o0, 2, %o0
163 1: 155 1:
164 bne,a 8f 156 bne,a 8f
165 EX(stb %g3, [%o0], and %o1, 1) 157 EX(stb %g3, [%o0], and %o1, 1)
166 8: 158 8:
167 b 0f 159 b 0f
168 nop 160 nop
169 7: 161 7:
170 be 13b 162 be 13b
171 orcc %o1, 0, %g0 163 orcc %o1, 0, %g0
172 164
173 be 0f 165 be 0f
174 8: 166 8:
175 add %o0, 1, %o0 167 add %o0, 1, %o0
176 subcc %o1, 1, %o1 168 subcc %o1, 1, %o1
177 bne 8b 169 bne 8b
178 EX(stb %g3, [%o0 - 1], add %o1, 1) 170 EX(stb %g3, [%o0 - 1], add %o1, 1)
179 0: 171 0:
180 andcc %g4, 1, %g0 172 andcc %g4, 1, %g0
181 be 5f 173 be 5f
182 nop 174 nop
183 retl 175 retl
184 mov %g1, %o0 176 mov %g1, %o0
185 5: 177 5:
186 retl 178 retl
187 clr %o0 179 clr %o0
>> 180 __memset_end:
188 181
189 .section .fixup,#alloc,#execinstr 182 .section .fixup,#alloc,#execinstr
190 .align 4 183 .align 4
>> 184 20:
>> 185 cmp %g2, 8
>> 186 bleu 1f
>> 187 and %o1, 0x7f, %o1
>> 188 sub %g2, 9, %g2
>> 189 add %o3, 64, %o3
>> 190 1:
>> 191 sll %g2, 3, %g2
>> 192 add %o3, %o1, %o0
>> 193 b 30f
>> 194 sub %o0, %g2, %o0
>> 195 21:
>> 196 mov 8, %o0
>> 197 and %o1, 7, %o1
>> 198 sub %o0, %g2, %o0
>> 199 sll %o0, 3, %o0
>> 200 b 30f
>> 201 add %o0, %o1, %o0
191 30: 202 30:
192 and %o1, 0x7f, %o1 !! 203 /* %o4 is faulting address, %o5 is %pc where fault occurred */
193 retl !! 204 save %sp, -104, %sp
194 add %o3, %o1, %o0 !! 205 mov %i5, %o0
>> 206 mov %i7, %o1
>> 207 call lookup_fault
>> 208 mov %i4, %o2
>> 209 ret
>> 210 restore
195 211
196 .globl __bzero_end 212 .globl __bzero_end
197 __bzero_end: 213 __bzero_end:
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.