1 /* SPDX-License-Identifier: GPL-2.0 */ <<
2 /* 1 /*
3 * arch/alpha/lib/stxcpy.S 2 * arch/alpha/lib/stxcpy.S
4 * Contributed by Richard Henderson (rth@tamu. 3 * Contributed by Richard Henderson (rth@tamu.edu)
5 * 4 *
6 * Copy a null-terminated string from SRC to D 5 * Copy a null-terminated string from SRC to DST.
7 * 6 *
8 * This is an internal routine used by strcpy, 7 * This is an internal routine used by strcpy, stpcpy, and strcat.
9 * As such, it uses special linkage convention 8 * As such, it uses special linkage conventions to make implementation
10 * of these public functions more efficient. 9 * of these public functions more efficient.
11 * 10 *
12 * On input: 11 * On input:
13 * t9 = return address 12 * t9 = return address
14 * a0 = DST 13 * a0 = DST
15 * a1 = SRC 14 * a1 = SRC
16 * 15 *
17 * On output: 16 * On output:
18 * t12 = bitmask (with one bit set) indic 17 * t12 = bitmask (with one bit set) indicating the last byte written
19 * a0 = unaligned address of the last *w 18 * a0 = unaligned address of the last *word* written
20 * 19 *
21 * Furthermore, v0, a3-a5, t11, and t12 are un 20 * Furthermore, v0, a3-a5, t11, and t12 are untouched.
22 */ 21 */
23 22
24 #include <asm/regdef.h> 23 #include <asm/regdef.h>
25 24
26 .set noat 25 .set noat
27 .set noreorder 26 .set noreorder
28 27
29 .text 28 .text
30 29
31 /* There is a problem with either gdb (as of 4 30 /* There is a problem with either gdb (as of 4.16) or gas (as of 2.7) that
32 doesn't like putting the entry point for a 31 doesn't like putting the entry point for a procedure somewhere in the
33 middle of the procedure descriptor. Work a 32 middle of the procedure descriptor. Work around this by putting the
34 aligned copy in its own procedure descripto 33 aligned copy in its own procedure descriptor */
35 34
36 .ent stxcpy_aligned 35 .ent stxcpy_aligned
37 .align 3 36 .align 3
38 stxcpy_aligned: 37 stxcpy_aligned:
39 .frame sp, 0, t9 38 .frame sp, 0, t9
40 .prologue 0 39 .prologue 0
41 40
42 /* On entry to this basic block: 41 /* On entry to this basic block:
43 t0 == the first destination word fo 42 t0 == the first destination word for masking back in
44 t1 == the first source word. */ 43 t1 == the first source word. */
45 44
46 /* Create the 1st output word and dete 45 /* Create the 1st output word and detect 0's in the 1st input word. */
47 lda t2, -1 # e1 : buil 46 lda t2, -1 # e1 : build a mask against false zero
48 mskqh t2, a1, t2 # e0 : de 47 mskqh t2, a1, t2 # e0 : detection in the src word
49 mskqh t1, a1, t3 # e0 : 48 mskqh t1, a1, t3 # e0 :
50 ornot t1, t2, t2 # .. e1 : 49 ornot t1, t2, t2 # .. e1 :
51 mskql t0, a1, t0 # e0 : asse 50 mskql t0, a1, t0 # e0 : assemble the first output word
52 cmpbge zero, t2, t8 # .. e1 : bits 51 cmpbge zero, t2, t8 # .. e1 : bits set iff null found
53 or t0, t3, t1 # e0 : 52 or t0, t3, t1 # e0 :
54 bne t8, $a_eos # .. e1 : 53 bne t8, $a_eos # .. e1 :
55 54
56 /* On entry to this basic block: 55 /* On entry to this basic block:
57 t0 == the first destination word fo 56 t0 == the first destination word for masking back in
58 t1 == a source word not containing 57 t1 == a source word not containing a null. */
59 58
60 $a_loop: 59 $a_loop:
61 stq_u t1, 0(a0) # e0 : 60 stq_u t1, 0(a0) # e0 :
62 addq a0, 8, a0 # .. e1 : 61 addq a0, 8, a0 # .. e1 :
63 ldq_u t1, 0(a1) # e0 : 62 ldq_u t1, 0(a1) # e0 :
64 addq a1, 8, a1 # .. e1 : 63 addq a1, 8, a1 # .. e1 :
65 cmpbge zero, t1, t8 # e0 (stall) 64 cmpbge zero, t1, t8 # e0 (stall)
66 beq t8, $a_loop # .. e1 (zdb) 65 beq t8, $a_loop # .. e1 (zdb)
67 66
68 /* Take care of the final (partial) wo 67 /* Take care of the final (partial) word store.
69 On entry to this basic block we hav 68 On entry to this basic block we have:
70 t1 == the source word containing th 69 t1 == the source word containing the null
71 t8 == the cmpbge mask that found it 70 t8 == the cmpbge mask that found it. */
72 $a_eos: 71 $a_eos:
73 negq t8, t6 # e0 : find 72 negq t8, t6 # e0 : find low bit set
74 and t8, t6, t12 # e1 (stall) 73 and t8, t6, t12 # e1 (stall)
75 74
76 /* For the sake of the cache, don't re 75 /* For the sake of the cache, don't read a destination word
77 if we're not going to need it. */ 76 if we're not going to need it. */
78 and t12, 0x80, t6 # e0 : 77 and t12, 0x80, t6 # e0 :
79 bne t6, 1f # .. e1 (zdb) 78 bne t6, 1f # .. e1 (zdb)
80 79
81 /* We're doing a partial word store an 80 /* We're doing a partial word store and so need to combine
82 our source and original destination 81 our source and original destination words. */
83 ldq_u t0, 0(a0) # e0 : 82 ldq_u t0, 0(a0) # e0 :
84 subq t12, 1, t6 # .. e1 : 83 subq t12, 1, t6 # .. e1 :
85 zapnot t1, t6, t1 # e0 : clea 84 zapnot t1, t6, t1 # e0 : clear src bytes >= null
86 or t12, t6, t8 # .. e1 : 85 or t12, t6, t8 # .. e1 :
87 zap t0, t8, t0 # e0 : clea 86 zap t0, t8, t0 # e0 : clear dst bytes <= null
88 or t0, t1, t1 # e1 : 87 or t0, t1, t1 # e1 :
89 88
90 1: stq_u t1, 0(a0) # e0 : 89 1: stq_u t1, 0(a0) # e0 :
91 ret (t9) # .. e1 : 90 ret (t9) # .. e1 :
92 91
93 .end stxcpy_aligned 92 .end stxcpy_aligned
94 93
95 .align 3 94 .align 3
96 .ent __stxcpy 95 .ent __stxcpy
97 .globl __stxcpy 96 .globl __stxcpy
98 __stxcpy: 97 __stxcpy:
99 .frame sp, 0, t9 98 .frame sp, 0, t9
100 .prologue 0 99 .prologue 0
101 100
102 /* Are source and destination co-align 101 /* Are source and destination co-aligned? */
103 xor a0, a1, t0 # e0 : 102 xor a0, a1, t0 # e0 :
104 unop # : 103 unop # :
105 and t0, 7, t0 # e0 : 104 and t0, 7, t0 # e0 :
106 bne t0, $unaligned # .. e1 : 105 bne t0, $unaligned # .. e1 :
107 106
108 /* We are co-aligned; take care of a p 107 /* We are co-aligned; take care of a partial first word. */
109 ldq_u t1, 0(a1) # e0 : load 108 ldq_u t1, 0(a1) # e0 : load first src word
110 and a0, 7, t0 # .. e1 : take 109 and a0, 7, t0 # .. e1 : take care not to load a word ...
111 addq a1, 8, a1 # e0 110 addq a1, 8, a1 # e0 :
112 beq t0, stxcpy_aligned # .. e 111 beq t0, stxcpy_aligned # .. e1 : ... if we wont need it
113 ldq_u t0, 0(a0) # e0 : 112 ldq_u t0, 0(a0) # e0 :
114 br stxcpy_aligned # .. e1 : 113 br stxcpy_aligned # .. e1 :
115 114
116 115
117 /* The source and destination are not co-align 116 /* The source and destination are not co-aligned. Align the destination
118 and cope. We have to be very careful about 117 and cope. We have to be very careful about not reading too much and
119 causing a SEGV. */ 118 causing a SEGV. */
120 119
121 .align 3 120 .align 3
122 $u_head: 121 $u_head:
123 /* We know just enough now to be able 122 /* We know just enough now to be able to assemble the first
124 full source word. We can still fin 123 full source word. We can still find a zero at the end of it
125 that prevents us from outputting th 124 that prevents us from outputting the whole thing.
126 125
127 On entry to this basic block: 126 On entry to this basic block:
128 t0 == the first dest word, for mask 127 t0 == the first dest word, for masking back in, if needed else 0
129 t1 == the low bits of the first sou 128 t1 == the low bits of the first source word
130 t6 == bytemask that is -1 in dest w 129 t6 == bytemask that is -1 in dest word bytes */
131 130
132 ldq_u t2, 8(a1) # e0 : 131 ldq_u t2, 8(a1) # e0 :
133 addq a1, 8, a1 # .. e1 : 132 addq a1, 8, a1 # .. e1 :
134 133
135 extql t1, a1, t1 # e0 : 134 extql t1, a1, t1 # e0 :
136 extqh t2, a1, t4 # e0 : 135 extqh t2, a1, t4 # e0 :
137 mskql t0, a0, t0 # e0 : 136 mskql t0, a0, t0 # e0 :
138 or t1, t4, t1 # .. e1 : 137 or t1, t4, t1 # .. e1 :
139 mskqh t1, a0, t1 # e0 : 138 mskqh t1, a0, t1 # e0 :
140 or t0, t1, t1 # e1 : 139 or t0, t1, t1 # e1 :
141 140
142 or t1, t6, t6 # e0 : 141 or t1, t6, t6 # e0 :
143 cmpbge zero, t6, t8 # .. e1 : 142 cmpbge zero, t6, t8 # .. e1 :
144 lda t6, -1 # e0 : for 143 lda t6, -1 # e0 : for masking just below
145 bne t8, $u_final # .. e1 : 144 bne t8, $u_final # .. e1 :
146 145
147 mskql t6, a1, t6 # e0 146 mskql t6, a1, t6 # e0 : mask out the bits we have
148 or t6, t2, t2 # e1 147 or t6, t2, t2 # e1 : already extracted before
149 cmpbge zero, t2, t8 # e0 148 cmpbge zero, t2, t8 # e0 : testing eos
150 bne t8, $u_late_head_exit # .. e 149 bne t8, $u_late_head_exit # .. e1 (zdb)
151 150
152 /* Finally, we've got all the stupid l 151 /* Finally, we've got all the stupid leading edge cases taken care
153 of and we can set up to enter the m 152 of and we can set up to enter the main loop. */
154 153
155 stq_u t1, 0(a0) # e0 : stor 154 stq_u t1, 0(a0) # e0 : store first output word
156 addq a0, 8, a0 # .. e1 : 155 addq a0, 8, a0 # .. e1 :
157 extql t2, a1, t0 # e0 : posi 156 extql t2, a1, t0 # e0 : position ho-bits of lo word
158 ldq_u t2, 8(a1) # .. e1 : read 157 ldq_u t2, 8(a1) # .. e1 : read next high-order source word
159 addq a1, 8, a1 # e0 : 158 addq a1, 8, a1 # e0 :
160 cmpbge zero, t2, t8 # .. e1 : 159 cmpbge zero, t2, t8 # .. e1 :
161 nop # e0 : 160 nop # e0 :
162 bne t8, $u_eos # .. e1 : 161 bne t8, $u_eos # .. e1 :
163 162
164 /* Unaligned copy main loop. In order 163 /* Unaligned copy main loop. In order to avoid reading too much,
165 the loop is structured to detect ze 164 the loop is structured to detect zeros in aligned source words.
166 This has, unfortunately, effectivel 165 This has, unfortunately, effectively pulled half of a loop
167 iteration out into the head and hal 166 iteration out into the head and half into the tail, but it does
168 prevent nastiness from accumulating 167 prevent nastiness from accumulating in the very thing we want
169 to run as fast as possible. 168 to run as fast as possible.
170 169
171 On entry to this basic block: 170 On entry to this basic block:
172 t0 == the shifted high-order bits f 171 t0 == the shifted high-order bits from the previous source word
173 t2 == the unshifted current source 172 t2 == the unshifted current source word
174 173
175 We further know that t2 does not co 174 We further know that t2 does not contain a null terminator. */
176 175
177 .align 3 176 .align 3
178 $u_loop: 177 $u_loop:
179 extqh t2, a1, t1 # e0 : extr 178 extqh t2, a1, t1 # e0 : extract high bits for current word
180 addq a1, 8, a1 # .. e1 : 179 addq a1, 8, a1 # .. e1 :
181 extql t2, a1, t3 # e0 : extr 180 extql t2, a1, t3 # e0 : extract low bits for next time
182 addq a0, 8, a0 # .. e1 : 181 addq a0, 8, a0 # .. e1 :
183 or t0, t1, t1 # e0 : curr 182 or t0, t1, t1 # e0 : current dst word now complete
184 ldq_u t2, 0(a1) # .. e1 : load 183 ldq_u t2, 0(a1) # .. e1 : load high word for next time
185 stq_u t1, -8(a0) # e0 : save 184 stq_u t1, -8(a0) # e0 : save the current word
186 mov t3, t0 # .. e1 : 185 mov t3, t0 # .. e1 :
187 cmpbge zero, t2, t8 # e0 : test 186 cmpbge zero, t2, t8 # e0 : test new word for eos
188 beq t8, $u_loop # .. e1 : 187 beq t8, $u_loop # .. e1 :
189 188
190 /* We've found a zero somewhere in the 189 /* We've found a zero somewhere in the source word we just read.
191 If it resides in the lower half, we 190 If it resides in the lower half, we have one (probably partial)
192 word to write out, and if it reside 191 word to write out, and if it resides in the upper half, we
193 have one full and one partial word 192 have one full and one partial word left to write out.
194 193
195 On entry to this basic block: 194 On entry to this basic block:
196 t0 == the shifted high-order bits f 195 t0 == the shifted high-order bits from the previous source word
197 t2 == the unshifted current source 196 t2 == the unshifted current source word. */
198 $u_eos: 197 $u_eos:
199 extqh t2, a1, t1 # e0 : 198 extqh t2, a1, t1 # e0 :
200 or t0, t1, t1 # e1 : firs 199 or t0, t1, t1 # e1 : first (partial) source word complete
201 200
202 cmpbge zero, t1, t8 # e0 : is t 201 cmpbge zero, t1, t8 # e0 : is the null in this first bit?
203 bne t8, $u_final # .. e1 (zdb) 202 bne t8, $u_final # .. e1 (zdb)
204 203
205 $u_late_head_exit: 204 $u_late_head_exit:
206 stq_u t1, 0(a0) # e0 : the 205 stq_u t1, 0(a0) # e0 : the null was in the high-order bits
207 addq a0, 8, a0 # .. e1 : 206 addq a0, 8, a0 # .. e1 :
208 extql t2, a1, t1 # e0 : 207 extql t2, a1, t1 # e0 :
209 cmpbge zero, t1, t8 # .. e1 : 208 cmpbge zero, t1, t8 # .. e1 :
210 209
211 /* Take care of a final (probably part 210 /* Take care of a final (probably partial) result word.
212 On entry to this basic block: 211 On entry to this basic block:
213 t1 == assembled source word 212 t1 == assembled source word
214 t8 == cmpbge mask that found the nu 213 t8 == cmpbge mask that found the null. */
215 $u_final: 214 $u_final:
216 negq t8, t6 # e0 : isol 215 negq t8, t6 # e0 : isolate low bit set
217 and t6, t8, t12 # e1 : 216 and t6, t8, t12 # e1 :
218 217
219 and t12, 0x80, t6 # e0 : avoi 218 and t12, 0x80, t6 # e0 : avoid dest word load if we can
220 bne t6, 1f # .. e1 (zdb) 219 bne t6, 1f # .. e1 (zdb)
221 220
222 ldq_u t0, 0(a0) # e0 : 221 ldq_u t0, 0(a0) # e0 :
223 subq t12, 1, t6 # .. e1 : 222 subq t12, 1, t6 # .. e1 :
224 or t6, t12, t8 # e0 : 223 or t6, t12, t8 # e0 :
225 zapnot t1, t6, t1 # .. e1 : kill 224 zapnot t1, t6, t1 # .. e1 : kill source bytes >= null
226 zap t0, t8, t0 # e0 : kill 225 zap t0, t8, t0 # e0 : kill dest bytes <= null
227 or t0, t1, t1 # e1 : 226 or t0, t1, t1 # e1 :
228 227
229 1: stq_u t1, 0(a0) # e0 : 228 1: stq_u t1, 0(a0) # e0 :
230 ret (t9) # .. e1 : 229 ret (t9) # .. e1 :
231 230
232 /* Unaligned copy entry point. */ 231 /* Unaligned copy entry point. */
233 .align 3 232 .align 3
234 $unaligned: 233 $unaligned:
235 234
236 ldq_u t1, 0(a1) # e0 : load 235 ldq_u t1, 0(a1) # e0 : load first source word
237 236
238 and a0, 7, t4 # .. e1 : find 237 and a0, 7, t4 # .. e1 : find dest misalignment
239 and a1, 7, t5 # e0 : find 238 and a1, 7, t5 # e0 : find src misalignment
240 239
241 /* Conditionally load the first destin 240 /* Conditionally load the first destination word and a bytemask
242 with 0xff indicating that the desti 241 with 0xff indicating that the destination byte is sacrosanct. */
243 242
244 mov zero, t0 # .. e1 : 243 mov zero, t0 # .. e1 :
245 mov zero, t6 # e0 : 244 mov zero, t6 # e0 :
246 beq t4, 1f # .. e1 : 245 beq t4, 1f # .. e1 :
247 ldq_u t0, 0(a0) # e0 : 246 ldq_u t0, 0(a0) # e0 :
248 lda t6, -1 # .. e1 : 247 lda t6, -1 # .. e1 :
249 mskql t6, a0, t6 # e0 : 248 mskql t6, a0, t6 # e0 :
250 1: 249 1:
251 subq a1, t4, a1 # .. e1 : sub 250 subq a1, t4, a1 # .. e1 : sub dest misalignment from src addr
252 251
253 /* If source misalignment is larger th 252 /* If source misalignment is larger than dest misalignment, we need
254 extra startup checks to avoid SEGV. 253 extra startup checks to avoid SEGV. */
255 254
256 cmplt t4, t5, t12 # e0 : 255 cmplt t4, t5, t12 # e0 :
257 beq t12, $u_head # .. e1 (zdb) 256 beq t12, $u_head # .. e1 (zdb)
258 257
259 lda t2, -1 # e1 : mask 258 lda t2, -1 # e1 : mask out leading garbage in source
260 mskqh t2, t5, t2 # e0 : 259 mskqh t2, t5, t2 # e0 :
261 nop # e0 : 260 nop # e0 :
262 ornot t1, t2, t3 # .. e1 : 261 ornot t1, t2, t3 # .. e1 :
263 cmpbge zero, t3, t8 # e0 : is t 262 cmpbge zero, t3, t8 # e0 : is there a zero?
264 beq t8, $u_head # .. e1 (zdb) 263 beq t8, $u_head # .. e1 (zdb)
265 264
266 /* At this point we've found a zero in 265 /* At this point we've found a zero in the first partial word of
267 the source. We need to isolate the 266 the source. We need to isolate the valid source data and mask
268 it into the original destination da 267 it into the original destination data. (Incidentally, we know
269 that we'll need at least one byte o 268 that we'll need at least one byte of that original dest word.) */
270 269
271 ldq_u t0, 0(a0) # e0 : 270 ldq_u t0, 0(a0) # e0 :
272 271
273 negq t8, t6 # .. e1 : buil 272 negq t8, t6 # .. e1 : build bitmask of bytes <= zero
274 and t6, t8, t12 # e0 : 273 and t6, t8, t12 # e0 :
275 and a1, 7, t5 # .. e1 : 274 and a1, 7, t5 # .. e1 :
276 subq t12, 1, t6 # e0 : 275 subq t12, 1, t6 # e0 :
277 or t6, t12, t8 # e1 : 276 or t6, t12, t8 # e1 :
278 srl t12, t5, t12 # e0 : adju 277 srl t12, t5, t12 # e0 : adjust final null return value
279 278
280 zapnot t2, t8, t2 # .. e1 : prep 279 zapnot t2, t8, t2 # .. e1 : prepare source word; mirror changes
281 and t1, t2, t1 # e1 : to s 280 and t1, t2, t1 # e1 : to source validity mask
282 extql t2, a1, t2 # .. e0 : 281 extql t2, a1, t2 # .. e0 :
283 extql t1, a1, t1 # e0 : 282 extql t1, a1, t1 # e0 :
284 283
285 andnot t0, t2, t0 # .. e1 : zero 284 andnot t0, t2, t0 # .. e1 : zero place for source to reside
286 or t0, t1, t1 # e1 : and 285 or t0, t1, t1 # e1 : and put it there
287 stq_u t1, 0(a0) # .. e0 : 286 stq_u t1, 0(a0) # .. e0 :
288 ret (t9) # e1 : 287 ret (t9) # e1 :
289 288
290 .end __stxcpy 289 .end __stxcpy
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.