1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* 2 /*
3 * arch/alpha/lib/ev6-stxcpy.S 3 * arch/alpha/lib/ev6-stxcpy.S
4 * 21264 version contributed by Rick Gorton <ri 4 * 21264 version contributed by Rick Gorton <rick.gorton@alpha-processor.com>
5 * 5 *
6 * Copy a null-terminated string from SRC to D 6 * Copy a null-terminated string from SRC to DST.
7 * 7 *
8 * This is an internal routine used by strcpy, 8 * This is an internal routine used by strcpy, stpcpy, and strcat.
9 * As such, it uses special linkage convention 9 * As such, it uses special linkage conventions to make implementation
10 * of these public functions more efficient. 10 * of these public functions more efficient.
11 * 11 *
12 * On input: 12 * On input:
13 * t9 = return address 13 * t9 = return address
14 * a0 = DST 14 * a0 = DST
15 * a1 = SRC 15 * a1 = SRC
16 * 16 *
17 * On output: 17 * On output:
18 * t12 = bitmask (with one bit set) indic 18 * t12 = bitmask (with one bit set) indicating the last byte written
19 * a0 = unaligned address of the last *w 19 * a0 = unaligned address of the last *word* written
20 * 20 *
21 * Furthermore, v0, a3-a5, t11, and t12 are un 21 * Furthermore, v0, a3-a5, t11, and t12 are untouched.
22 * 22 *
23 * Much of the information about 21264 schedul 23 * Much of the information about 21264 scheduling/coding comes from:
24 * Compiler Writer's Guide for the Alpha 24 * Compiler Writer's Guide for the Alpha 21264
25 * abbreviated as 'CWG' in other comments 25 * abbreviated as 'CWG' in other comments here
26 * ftp.digital.com/pub/Digital/info/semic 26 * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
27 * Scheduling notation: 27 * Scheduling notation:
28 * E - either cluster 28 * E - either cluster
29 * U - upper subcluster; U0 - subcl 29 * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1
30 * L - lower subcluster; L0 - subcl 30 * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1
31 * Try not to change the actual algorithm if p 31 * Try not to change the actual algorithm if possible for consistency.
32 */ 32 */
33 33
34 #include <asm/regdef.h> 34 #include <asm/regdef.h>
35 35
36 .set noat 36 .set noat
37 .set noreorder 37 .set noreorder
38 38
39 .text 39 .text
40 40
41 /* There is a problem with either gdb (as of 4 41 /* There is a problem with either gdb (as of 4.16) or gas (as of 2.7) that
42 doesn't like putting the entry point for a 42 doesn't like putting the entry point for a procedure somewhere in the
43 middle of the procedure descriptor. Work a 43 middle of the procedure descriptor. Work around this by putting the
44 aligned copy in its own procedure descripto 44 aligned copy in its own procedure descriptor */
45 45
46 46
47 .ent stxcpy_aligned 47 .ent stxcpy_aligned
48 .align 4 48 .align 4
49 stxcpy_aligned: 49 stxcpy_aligned:
50 .frame sp, 0, t9 50 .frame sp, 0, t9
51 .prologue 0 51 .prologue 0
52 52
53 /* On entry to this basic block: 53 /* On entry to this basic block:
54 t0 == the first destination word fo 54 t0 == the first destination word for masking back in
55 t1 == the first source word. */ 55 t1 == the first source word. */
56 56
57 /* Create the 1st output word and dete 57 /* Create the 1st output word and detect 0's in the 1st input word. */
58 lda t2, -1 # E : build a 58 lda t2, -1 # E : build a mask against false zero
59 mskqh t2, a1, t2 # U : detect 59 mskqh t2, a1, t2 # U : detection in the src word (stall)
60 mskqh t1, a1, t3 # U : 60 mskqh t1, a1, t3 # U :
61 ornot t1, t2, t2 # E : (stall) 61 ornot t1, t2, t2 # E : (stall)
62 62
63 mskql t0, a1, t0 # U : assemble 63 mskql t0, a1, t0 # U : assemble the first output word
64 cmpbge zero, t2, t8 # E : bits set 64 cmpbge zero, t2, t8 # E : bits set iff null found
65 or t0, t3, t1 # E : (stall) 65 or t0, t3, t1 # E : (stall)
66 bne t8, $a_eos # U : (stall) 66 bne t8, $a_eos # U : (stall)
67 67
68 /* On entry to this basic block: 68 /* On entry to this basic block:
69 t0 == the first destination word fo 69 t0 == the first destination word for masking back in
70 t1 == a source word not containing 70 t1 == a source word not containing a null. */
71 /* Nops here to separate store quads f 71 /* Nops here to separate store quads from load quads */
72 72
73 $a_loop: 73 $a_loop:
74 stq_u t1, 0(a0) # L : 74 stq_u t1, 0(a0) # L :
75 addq a0, 8, a0 # E : 75 addq a0, 8, a0 # E :
76 nop 76 nop
77 nop 77 nop
78 78
79 ldq_u t1, 0(a1) # L : Latency= 79 ldq_u t1, 0(a1) # L : Latency=3
80 addq a1, 8, a1 # E : 80 addq a1, 8, a1 # E :
81 cmpbge zero, t1, t8 # E : (3 cycle 81 cmpbge zero, t1, t8 # E : (3 cycle stall)
82 beq t8, $a_loop # U : (stall f 82 beq t8, $a_loop # U : (stall for t8)
83 83
84 /* Take care of the final (partial) wo 84 /* Take care of the final (partial) word store.
85 On entry to this basic block we hav 85 On entry to this basic block we have:
86 t1 == the source word containing th 86 t1 == the source word containing the null
87 t8 == the cmpbge mask that found it 87 t8 == the cmpbge mask that found it. */
88 $a_eos: 88 $a_eos:
89 negq t8, t6 # E : find low 89 negq t8, t6 # E : find low bit set
90 and t8, t6, t12 # E : (stall) 90 and t8, t6, t12 # E : (stall)
91 /* For the sake of the cache, don't re 91 /* For the sake of the cache, don't read a destination word
92 if we're not going to need it. */ 92 if we're not going to need it. */
93 and t12, 0x80, t6 # E : (stall) 93 and t12, 0x80, t6 # E : (stall)
94 bne t6, 1f # U : (stall) 94 bne t6, 1f # U : (stall)
95 95
96 /* We're doing a partial word store an 96 /* We're doing a partial word store and so need to combine
97 our source and original destination 97 our source and original destination words. */
98 ldq_u t0, 0(a0) # L : Latency= 98 ldq_u t0, 0(a0) # L : Latency=3
99 subq t12, 1, t6 # E : 99 subq t12, 1, t6 # E :
100 zapnot t1, t6, t1 # U : clear sr 100 zapnot t1, t6, t1 # U : clear src bytes >= null (stall)
101 or t12, t6, t8 # E : (stall) 101 or t12, t6, t8 # E : (stall)
102 102
103 zap t0, t8, t0 # E : clear ds 103 zap t0, t8, t0 # E : clear dst bytes <= null
104 or t0, t1, t1 # E : (stall) 104 or t0, t1, t1 # E : (stall)
105 nop 105 nop
106 nop 106 nop
107 107
108 1: stq_u t1, 0(a0) # L : 108 1: stq_u t1, 0(a0) # L :
109 ret (t9) # L0 : Latency 109 ret (t9) # L0 : Latency=3
110 nop 110 nop
111 nop 111 nop
112 112
113 .end stxcpy_aligned 113 .end stxcpy_aligned
114 114
115 .align 4 115 .align 4
116 .ent __stxcpy 116 .ent __stxcpy
117 .globl __stxcpy 117 .globl __stxcpy
118 __stxcpy: 118 __stxcpy:
119 .frame sp, 0, t9 119 .frame sp, 0, t9
120 .prologue 0 120 .prologue 0
121 121
122 /* Are source and destination co-align 122 /* Are source and destination co-aligned? */
123 xor a0, a1, t0 # E : 123 xor a0, a1, t0 # E :
124 unop # E : 124 unop # E :
125 and t0, 7, t0 # E : (stall) 125 and t0, 7, t0 # E : (stall)
126 bne t0, $unaligned # U : (stall) 126 bne t0, $unaligned # U : (stall)
127 127
128 /* We are co-aligned; take care of a p 128 /* We are co-aligned; take care of a partial first word. */
129 ldq_u t1, 0(a1) # L : 129 ldq_u t1, 0(a1) # L : load first src word
130 and a0, 7, t0 # E : 130 and a0, 7, t0 # E : take care not to load a word ...
131 addq a1, 8, a1 # E : 131 addq a1, 8, a1 # E :
132 beq t0, stxcpy_aligned # U : 132 beq t0, stxcpy_aligned # U : ... if we wont need it (stall)
133 133
134 ldq_u t0, 0(a0) # L : 134 ldq_u t0, 0(a0) # L :
135 br stxcpy_aligned # L0 : Latency 135 br stxcpy_aligned # L0 : Latency=3
136 nop 136 nop
137 nop 137 nop
138 138
139 139
140 /* The source and destination are not co-align 140 /* The source and destination are not co-aligned. Align the destination
141 and cope. We have to be very careful about 141 and cope. We have to be very careful about not reading too much and
142 causing a SEGV. */ 142 causing a SEGV. */
143 143
144 .align 4 144 .align 4
145 $u_head: 145 $u_head:
146 /* We know just enough now to be able 146 /* We know just enough now to be able to assemble the first
147 full source word. We can still fin 147 full source word. We can still find a zero at the end of it
148 that prevents us from outputting th 148 that prevents us from outputting the whole thing.
149 149
150 On entry to this basic block: 150 On entry to this basic block:
151 t0 == the first dest word, for mask 151 t0 == the first dest word, for masking back in, if needed else 0
152 t1 == the low bits of the first sou 152 t1 == the low bits of the first source word
153 t6 == bytemask that is -1 in dest w 153 t6 == bytemask that is -1 in dest word bytes */
154 154
155 ldq_u t2, 8(a1) # L : 155 ldq_u t2, 8(a1) # L :
156 addq a1, 8, a1 # E : 156 addq a1, 8, a1 # E :
157 extql t1, a1, t1 # U : (stall o 157 extql t1, a1, t1 # U : (stall on a1)
158 extqh t2, a1, t4 # U : (stall o 158 extqh t2, a1, t4 # U : (stall on a1)
159 159
160 mskql t0, a0, t0 # U : 160 mskql t0, a0, t0 # U :
161 or t1, t4, t1 # E : 161 or t1, t4, t1 # E :
162 mskqh t1, a0, t1 # U : (stall o 162 mskqh t1, a0, t1 # U : (stall on t1)
163 or t0, t1, t1 # E : (stall o 163 or t0, t1, t1 # E : (stall on t1)
164 164
165 or t1, t6, t6 # E : 165 or t1, t6, t6 # E :
166 cmpbge zero, t6, t8 # E : (stall) 166 cmpbge zero, t6, t8 # E : (stall)
167 lda t6, -1 # E : for mask 167 lda t6, -1 # E : for masking just below
168 bne t8, $u_final # U : (stall) 168 bne t8, $u_final # U : (stall)
169 169
170 mskql t6, a1, t6 # U : 170 mskql t6, a1, t6 # U : mask out the bits we have
171 or t6, t2, t2 # E : 171 or t6, t2, t2 # E : already extracted before (stall)
172 cmpbge zero, t2, t8 # E : 172 cmpbge zero, t2, t8 # E : testing eos (stall)
173 bne t8, $u_late_head_exit # U : 173 bne t8, $u_late_head_exit # U : (stall)
174 174
175 /* Finally, we've got all the stupid l 175 /* Finally, we've got all the stupid leading edge cases taken care
176 of and we can set up to enter the m 176 of and we can set up to enter the main loop. */
177 177
178 stq_u t1, 0(a0) # L : store fi 178 stq_u t1, 0(a0) # L : store first output word
179 addq a0, 8, a0 # E : 179 addq a0, 8, a0 # E :
180 extql t2, a1, t0 # U : position 180 extql t2, a1, t0 # U : position ho-bits of lo word
181 ldq_u t2, 8(a1) # U : read nex 181 ldq_u t2, 8(a1) # U : read next high-order source word
182 182
183 addq a1, 8, a1 # E : 183 addq a1, 8, a1 # E :
184 cmpbge zero, t2, t8 # E : (stall f 184 cmpbge zero, t2, t8 # E : (stall for t2)
185 nop # E : 185 nop # E :
186 bne t8, $u_eos # U : (stall) 186 bne t8, $u_eos # U : (stall)
187 187
188 /* Unaligned copy main loop. In order 188 /* Unaligned copy main loop. In order to avoid reading too much,
189 the loop is structured to detect ze 189 the loop is structured to detect zeros in aligned source words.
190 This has, unfortunately, effectivel 190 This has, unfortunately, effectively pulled half of a loop
191 iteration out into the head and hal 191 iteration out into the head and half into the tail, but it does
192 prevent nastiness from accumulating 192 prevent nastiness from accumulating in the very thing we want
193 to run as fast as possible. 193 to run as fast as possible.
194 194
195 On entry to this basic block: 195 On entry to this basic block:
196 t0 == the shifted high-order bits f 196 t0 == the shifted high-order bits from the previous source word
197 t2 == the unshifted current source 197 t2 == the unshifted current source word
198 198
199 We further know that t2 does not co 199 We further know that t2 does not contain a null terminator. */
200 200
201 .align 3 201 .align 3
202 $u_loop: 202 $u_loop:
203 extqh t2, a1, t1 # U : extract 203 extqh t2, a1, t1 # U : extract high bits for current word
204 addq a1, 8, a1 # E : (stall) 204 addq a1, 8, a1 # E : (stall)
205 extql t2, a1, t3 # U : extract 205 extql t2, a1, t3 # U : extract low bits for next time (stall)
206 addq a0, 8, a0 # E : 206 addq a0, 8, a0 # E :
207 207
208 or t0, t1, t1 # E : current 208 or t0, t1, t1 # E : current dst word now complete
209 ldq_u t2, 0(a1) # L : Latency= 209 ldq_u t2, 0(a1) # L : Latency=3 load high word for next time
210 stq_u t1, -8(a0) # L : save the 210 stq_u t1, -8(a0) # L : save the current word (stall)
211 mov t3, t0 # E : 211 mov t3, t0 # E :
212 212
213 cmpbge zero, t2, t8 # E : test new 213 cmpbge zero, t2, t8 # E : test new word for eos
214 beq t8, $u_loop # U : (stall) 214 beq t8, $u_loop # U : (stall)
215 nop 215 nop
216 nop 216 nop
217 217
218 /* We've found a zero somewhere in the 218 /* We've found a zero somewhere in the source word we just read.
219 If it resides in the lower half, we 219 If it resides in the lower half, we have one (probably partial)
220 word to write out, and if it reside 220 word to write out, and if it resides in the upper half, we
221 have one full and one partial word 221 have one full and one partial word left to write out.
222 222
223 On entry to this basic block: 223 On entry to this basic block:
224 t0 == the shifted high-order bits f 224 t0 == the shifted high-order bits from the previous source word
225 t2 == the unshifted current source 225 t2 == the unshifted current source word. */
226 $u_eos: 226 $u_eos:
227 extqh t2, a1, t1 # U : 227 extqh t2, a1, t1 # U :
228 or t0, t1, t1 # E : first (p 228 or t0, t1, t1 # E : first (partial) source word complete (stall)
229 cmpbge zero, t1, t8 # E : is the n 229 cmpbge zero, t1, t8 # E : is the null in this first bit? (stall)
230 bne t8, $u_final # U : (stall) 230 bne t8, $u_final # U : (stall)
231 231
232 $u_late_head_exit: 232 $u_late_head_exit:
233 stq_u t1, 0(a0) # L : the null 233 stq_u t1, 0(a0) # L : the null was in the high-order bits
234 addq a0, 8, a0 # E : 234 addq a0, 8, a0 # E :
235 extql t2, a1, t1 # U : 235 extql t2, a1, t1 # U :
236 cmpbge zero, t1, t8 # E : (stall) 236 cmpbge zero, t1, t8 # E : (stall)
237 237
238 /* Take care of a final (probably part 238 /* Take care of a final (probably partial) result word.
239 On entry to this basic block: 239 On entry to this basic block:
240 t1 == assembled source word 240 t1 == assembled source word
241 t8 == cmpbge mask that found the nu 241 t8 == cmpbge mask that found the null. */
242 $u_final: 242 $u_final:
243 negq t8, t6 # E : isolate 243 negq t8, t6 # E : isolate low bit set
244 and t6, t8, t12 # E : (stall) 244 and t6, t8, t12 # E : (stall)
245 and t12, 0x80, t6 # E : avoid de 245 and t12, 0x80, t6 # E : avoid dest word load if we can (stall)
246 bne t6, 1f # U : (stall) 246 bne t6, 1f # U : (stall)
247 247
248 ldq_u t0, 0(a0) # E : 248 ldq_u t0, 0(a0) # E :
249 subq t12, 1, t6 # E : 249 subq t12, 1, t6 # E :
250 or t6, t12, t8 # E : (stall) 250 or t6, t12, t8 # E : (stall)
251 zapnot t1, t6, t1 # U : kill sou 251 zapnot t1, t6, t1 # U : kill source bytes >= null (stall)
252 252
253 zap t0, t8, t0 # U : kill des 253 zap t0, t8, t0 # U : kill dest bytes <= null (2 cycle data stall)
254 or t0, t1, t1 # E : (stall) 254 or t0, t1, t1 # E : (stall)
255 nop 255 nop
256 nop 256 nop
257 257
258 1: stq_u t1, 0(a0) # L : 258 1: stq_u t1, 0(a0) # L :
259 ret (t9) # L0 : Latency 259 ret (t9) # L0 : Latency=3
260 nop 260 nop
261 nop 261 nop
262 262
263 /* Unaligned copy entry point. */ 263 /* Unaligned copy entry point. */
264 .align 4 264 .align 4
265 $unaligned: 265 $unaligned:
266 266
267 ldq_u t1, 0(a1) # L : load fir 267 ldq_u t1, 0(a1) # L : load first source word
268 and a0, 7, t4 # E : find des 268 and a0, 7, t4 # E : find dest misalignment
269 and a1, 7, t5 # E : find src 269 and a1, 7, t5 # E : find src misalignment
270 /* Conditionally load the first destin 270 /* Conditionally load the first destination word and a bytemask
271 with 0xff indicating that the desti 271 with 0xff indicating that the destination byte is sacrosanct. */
272 mov zero, t0 # E : 272 mov zero, t0 # E :
273 273
274 mov zero, t6 # E : 274 mov zero, t6 # E :
275 beq t4, 1f # U : 275 beq t4, 1f # U :
276 ldq_u t0, 0(a0) # L : 276 ldq_u t0, 0(a0) # L :
277 lda t6, -1 # E : 277 lda t6, -1 # E :
278 278
279 mskql t6, a0, t6 # U : 279 mskql t6, a0, t6 # U :
280 nop 280 nop
281 nop 281 nop
282 nop 282 nop
283 1: 283 1:
284 subq a1, t4, a1 # E : sub dest 284 subq a1, t4, a1 # E : sub dest misalignment from src addr
285 /* If source misalignment is larger th 285 /* If source misalignment is larger than dest misalignment, we need
286 extra startup checks to avoid SEGV. 286 extra startup checks to avoid SEGV. */
287 cmplt t4, t5, t12 # E : 287 cmplt t4, t5, t12 # E :
288 beq t12, $u_head # U : 288 beq t12, $u_head # U :
289 lda t2, -1 # E : mask out 289 lda t2, -1 # E : mask out leading garbage in source
290 290
291 mskqh t2, t5, t2 # U : 291 mskqh t2, t5, t2 # U :
292 ornot t1, t2, t3 # E : (stall) 292 ornot t1, t2, t3 # E : (stall)
293 cmpbge zero, t3, t8 # E : is there 293 cmpbge zero, t3, t8 # E : is there a zero? (stall)
294 beq t8, $u_head # U : (stall) 294 beq t8, $u_head # U : (stall)
295 295
296 /* At this point we've found a zero in 296 /* At this point we've found a zero in the first partial word of
297 the source. We need to isolate the 297 the source. We need to isolate the valid source data and mask
298 it into the original destination da 298 it into the original destination data. (Incidentally, we know
299 that we'll need at least one byte o 299 that we'll need at least one byte of that original dest word.) */
300 300
301 ldq_u t0, 0(a0) # L : 301 ldq_u t0, 0(a0) # L :
302 negq t8, t6 # E : build bi 302 negq t8, t6 # E : build bitmask of bytes <= zero
303 and t6, t8, t12 # E : (stall) 303 and t6, t8, t12 # E : (stall)
304 and a1, 7, t5 # E : 304 and a1, 7, t5 # E :
305 305
306 subq t12, 1, t6 # E : 306 subq t12, 1, t6 # E :
307 or t6, t12, t8 # E : (stall) 307 or t6, t12, t8 # E : (stall)
308 srl t12, t5, t12 # U : adjust f 308 srl t12, t5, t12 # U : adjust final null return value
309 zapnot t2, t8, t2 # U : prepare 309 zapnot t2, t8, t2 # U : prepare source word; mirror changes (stall)
310 310
311 and t1, t2, t1 # E : to sourc 311 and t1, t2, t1 # E : to source validity mask
312 extql t2, a1, t2 # U : 312 extql t2, a1, t2 # U :
313 extql t1, a1, t1 # U : (stall) 313 extql t1, a1, t1 # U : (stall)
314 andnot t0, t2, t0 # .. e1 : zero 314 andnot t0, t2, t0 # .. e1 : zero place for source to reside (stall)
315 315
316 or t0, t1, t1 # e1 : and 316 or t0, t1, t1 # e1 : and put it there
317 stq_u t1, 0(a0) # .. e0 : (sta 317 stq_u t1, 0(a0) # .. e0 : (stall)
318 ret (t9) # e1 : 318 ret (t9) # e1 :
319 nop 319 nop
320 320
321 .end __stxcpy 321 .end __stxcpy
322 322
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.