1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * arch/alpha/lib/ev6-stxncpy.S 4 * 21264 version contributed by Rick Gorton <rick.gorton@api-networks.com> 5 * 6 * Copy no more than COUNT bytes of the null-terminated string from 7 * SRC to DST. 8 * 9 * This is an internal routine used by strncpy, stpncpy, and strncat. 10 * As such, it uses special linkage conventions to make implementation 11 * of these public functions more efficient. 12 * 13 * On input: 14 * t9 = return address 15 * a0 = DST 16 * a1 = SRC 17 * a2 = COUNT 18 * 19 * Furthermore, COUNT may not be zero. 20 * 21 * On output: 22 * t0 = last word written 23 * t10 = bitmask (with one bit set) indicating the byte position of 24 * the end of the range specified by COUNT 25 * t12 = bitmask (with one bit set) indicating the last byte written 26 * a0 = unaligned address of the last *word* written 27 * a2 = the number of full words left in COUNT 28 * 29 * Furthermore, v0, a3-a5, t11, and $at are untouched. 30 * 31 * Much of the information about 21264 scheduling/coding comes from: 32 * Compiler Writer's Guide for the Alpha 21264 33 * abbreviated as 'CWG' in other comments here 34 * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html 35 * Scheduling notation: 36 * E - either cluster 37 * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 38 * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 39 * Try not to change the actual algorithm if possible for consistency. 40 */ 41 42 #include <asm/regdef.h> 43 44 .set noat 45 .set noreorder 46 47 .text 48 49 /* There is a problem with either gdb (as of 4.16) or gas (as of 2.7) that 50 doesn't like putting the entry point for a procedure somewhere in the 51 middle of the procedure descriptor. Work around this by putting the 52 aligned copy in its own procedure descriptor */ 53 54 55 .ent stxncpy_aligned 56 .align 4 57 stxncpy_aligned: 58 .frame sp, 0, t9, 0 59 .prologue 0 60 61 /* On entry to this basic block: 62 t0 == the first destination word for masking back in 63 t1 == the first source word. */ 64 65 /* Create the 1st output word and detect 0's in the 1st input word. */ 66 lda t2, -1 # E : build a mask against false zero 67 mskqh t2, a1, t2 # U : detection in the src word (stall) 68 mskqh t1, a1, t3 # U : 69 ornot t1, t2, t2 # E : (stall) 70 71 mskql t0, a1, t0 # U : assemble the first output word 72 cmpbge zero, t2, t8 # E : bits set iff null found 73 or t0, t3, t0 # E : (stall) 74 beq a2, $a_eoc # U : 75 76 bne t8, $a_eos # U : 77 nop 78 nop 79 nop 80 81 /* On entry to this basic block: 82 t0 == a source word not containing a null. */ 83 84 /* 85 * nops here to: 86 * separate store quads from load quads 87 * limit of 1 bcond/quad to permit training 88 */ 89 $a_loop: 90 stq_u t0, 0(a0) # L : 91 addq a0, 8, a0 # E : 92 subq a2, 1, a2 # E : 93 nop 94 95 ldq_u t0, 0(a1) # L : 96 addq a1, 8, a1 # E : 97 cmpbge zero, t0, t8 # E : 98 beq a2, $a_eoc # U : 99 100 beq t8, $a_loop # U : 101 nop 102 nop 103 nop 104 105 /* Take care of the final (partial) word store. At this point 106 the end-of-count bit is set in t8 iff it applies. 107 108 On entry to this basic block we have: 109 t0 == the source word containing the null 110 t8 == the cmpbge mask that found it. */ 111 112 $a_eos: 113 negq t8, t12 # E : find low bit set 114 and t8, t12, t12 # E : (stall) 115 /* For the sake of the cache, don't read a destination word 116 if we're not going to need it. */ 117 and t12, 0x80, t6 # E : (stall) 118 bne t6, 1f # U : (stall) 119 120 /* We're doing a partial word store and so need to combine 121 our source and original destination words. */ 122 ldq_u t1, 0(a0) # L : 123 subq t12, 1, t6 # E : 124 or t12, t6, t8 # E : (stall) 125 zapnot t0, t8, t0 # U : clear src bytes > null (stall) 126 127 zap t1, t8, t1 # .. e1 : clear dst bytes <= null 128 or t0, t1, t0 # e1 : (stall) 129 nop 130 nop 131 132 1: stq_u t0, 0(a0) # L : 133 ret (t9) # L0 : Latency=3 134 nop 135 nop 136 137 /* Add the end-of-count bit to the eos detection bitmask. */ 138 $a_eoc: 139 or t10, t8, t8 # E : 140 br $a_eos # L0 : Latency=3 141 nop 142 nop 143 144 .end stxncpy_aligned 145 146 .align 4 147 .ent __stxncpy 148 .globl __stxncpy 149 __stxncpy: 150 .frame sp, 0, t9, 0 151 .prologue 0 152 153 /* Are source and destination co-aligned? */ 154 xor a0, a1, t1 # E : 155 and a0, 7, t0 # E : find dest misalignment 156 and t1, 7, t1 # E : (stall) 157 addq a2, t0, a2 # E : bias count by dest misalignment (stall) 158 159 subq a2, 1, a2 # E : 160 and a2, 7, t2 # E : (stall) 161 srl a2, 3, a2 # U : a2 = loop counter = (count - 1)/8 (stall) 162 addq zero, 1, t10 # E : 163 164 sll t10, t2, t10 # U : t10 = bitmask of last count byte 165 bne t1, $unaligned # U : 166 /* We are co-aligned; take care of a partial first word. */ 167 ldq_u t1, 0(a1) # L : load first src word 168 addq a1, 8, a1 # E : 169 170 beq t0, stxncpy_aligned # U : avoid loading dest word if not needed 171 ldq_u t0, 0(a0) # L : 172 nop 173 nop 174 175 br stxncpy_aligned # .. e1 : 176 nop 177 nop 178 nop 179 180 181 182 /* The source and destination are not co-aligned. Align the destination 183 and cope. We have to be very careful about not reading too much and 184 causing a SEGV. */ 185 186 .align 4 187 $u_head: 188 /* We know just enough now to be able to assemble the first 189 full source word. We can still find a zero at the end of it 190 that prevents us from outputting the whole thing. 191 192 On entry to this basic block: 193 t0 == the first dest word, unmasked 194 t1 == the shifted low bits of the first source word 195 t6 == bytemask that is -1 in dest word bytes */ 196 197 ldq_u t2, 8(a1) # L : Latency=3 load second src word 198 addq a1, 8, a1 # E : 199 mskql t0, a0, t0 # U : mask trailing garbage in dst 200 extqh t2, a1, t4 # U : (3 cycle stall on t2) 201 202 or t1, t4, t1 # E : first aligned src word complete (stall) 203 mskqh t1, a0, t1 # U : mask leading garbage in src (stall) 204 or t0, t1, t0 # E : first output word complete (stall) 205 or t0, t6, t6 # E : mask original data for zero test (stall) 206 207 cmpbge zero, t6, t8 # E : 208 beq a2, $u_eocfin # U : 209 lda t6, -1 # E : 210 nop 211 212 bne t8, $u_final # U : 213 mskql t6, a1, t6 # U : mask out bits already seen 214 stq_u t0, 0(a0) # L : store first output word 215 or t6, t2, t2 # E : (stall) 216 217 cmpbge zero, t2, t8 # E : find nulls in second partial 218 addq a0, 8, a0 # E : 219 subq a2, 1, a2 # E : 220 bne t8, $u_late_head_exit # U : 221 222 /* Finally, we've got all the stupid leading edge cases taken care 223 of and we can set up to enter the main loop. */ 224 extql t2, a1, t1 # U : position hi-bits of lo word 225 beq a2, $u_eoc # U : 226 ldq_u t2, 8(a1) # L : read next high-order source word 227 addq a1, 8, a1 # E : 228 229 extqh t2, a1, t0 # U : position lo-bits of hi word (stall) 230 cmpbge zero, t2, t8 # E : 231 nop 232 bne t8, $u_eos # U : 233 234 /* Unaligned copy main loop. In order to avoid reading too much, 235 the loop is structured to detect zeros in aligned source words. 236 This has, unfortunately, effectively pulled half of a loop 237 iteration out into the head and half into the tail, but it does 238 prevent nastiness from accumulating in the very thing we want 239 to run as fast as possible. 240 241 On entry to this basic block: 242 t0 == the shifted low-order bits from the current source word 243 t1 == the shifted high-order bits from the previous source word 244 t2 == the unshifted current source word 245 246 We further know that t2 does not contain a null terminator. */ 247 248 .align 4 249 $u_loop: 250 or t0, t1, t0 # E : current dst word now complete 251 subq a2, 1, a2 # E : decrement word count 252 extql t2, a1, t1 # U : extract low bits for next time 253 addq a0, 8, a0 # E : 254 255 stq_u t0, -8(a0) # U : save the current word 256 beq a2, $u_eoc # U : 257 ldq_u t2, 8(a1) # U : Latency=3 load high word for next time 258 addq a1, 8, a1 # E : 259 260 extqh t2, a1, t0 # U : extract low bits (2 cycle stall) 261 cmpbge zero, t2, t8 # E : test new word for eos 262 nop 263 beq t8, $u_loop # U : 264 265 /* We've found a zero somewhere in the source word we just read. 266 If it resides in the lower half, we have one (probably partial) 267 word to write out, and if it resides in the upper half, we 268 have one full and one partial word left to write out. 269 270 On entry to this basic block: 271 t0 == the shifted low-order bits from the current source word 272 t1 == the shifted high-order bits from the previous source word 273 t2 == the unshifted current source word. */ 274 $u_eos: 275 or t0, t1, t0 # E : first (partial) source word complete 276 nop 277 cmpbge zero, t0, t8 # E : is the null in this first bit? (stall) 278 bne t8, $u_final # U : (stall) 279 280 stq_u t0, 0(a0) # L : the null was in the high-order bits 281 addq a0, 8, a0 # E : 282 subq a2, 1, a2 # E : 283 nop 284 285 $u_late_head_exit: 286 extql t2, a1, t0 # U : 287 cmpbge zero, t0, t8 # E : 288 or t8, t10, t6 # E : (stall) 289 cmoveq a2, t6, t8 # E : Latency=2, extra map slot (stall) 290 291 /* Take care of a final (probably partial) result word. 292 On entry to this basic block: 293 t0 == assembled source word 294 t8 == cmpbge mask that found the null. */ 295 $u_final: 296 negq t8, t6 # E : isolate low bit set 297 and t6, t8, t12 # E : (stall) 298 and t12, 0x80, t6 # E : avoid dest word load if we can (stall) 299 bne t6, 1f # U : (stall) 300 301 ldq_u t1, 0(a0) # L : 302 subq t12, 1, t6 # E : 303 or t6, t12, t8 # E : (stall) 304 zapnot t0, t8, t0 # U : kill source bytes > null 305 306 zap t1, t8, t1 # U : kill dest bytes <= null 307 or t0, t1, t0 # E : (stall) 308 nop 309 nop 310 311 1: stq_u t0, 0(a0) # L : 312 ret (t9) # L0 : Latency=3 313 314 /* Got to end-of-count before end of string. 315 On entry to this basic block: 316 t1 == the shifted high-order bits from the previous source word */ 317 $u_eoc: 318 and a1, 7, t6 # E : avoid final load if possible 319 sll t10, t6, t6 # U : (stall) 320 and t6, 0xff, t6 # E : (stall) 321 bne t6, 1f # U : (stall) 322 323 ldq_u t2, 8(a1) # L : load final src word 324 nop 325 extqh t2, a1, t0 # U : extract low bits for last word (stall) 326 or t1, t0, t1 # E : (stall) 327 328 1: cmpbge zero, t1, t8 # E : 329 mov t1, t0 # E : 330 331 $u_eocfin: # end-of-count, final word 332 or t10, t8, t8 # E : 333 br $u_final # L0 : Latency=3 334 335 /* Unaligned copy entry point. */ 336 .align 4 337 $unaligned: 338 339 ldq_u t1, 0(a1) # L : load first source word 340 and a0, 7, t4 # E : find dest misalignment 341 and a1, 7, t5 # E : find src misalignment 342 /* Conditionally load the first destination word and a bytemask 343 with 0xff indicating that the destination byte is sacrosanct. */ 344 mov zero, t0 # E : 345 346 mov zero, t6 # E : 347 beq t4, 1f # U : 348 ldq_u t0, 0(a0) # L : 349 lda t6, -1 # E : 350 351 mskql t6, a0, t6 # U : 352 nop 353 nop 354 subq a1, t4, a1 # E : sub dest misalignment from src addr 355 356 /* If source misalignment is larger than dest misalignment, we need 357 extra startup checks to avoid SEGV. */ 358 359 1: cmplt t4, t5, t12 # E : 360 extql t1, a1, t1 # U : shift src into place 361 lda t2, -1 # E : for creating masks later 362 beq t12, $u_head # U : (stall) 363 364 extql t2, a1, t2 # U : 365 cmpbge zero, t1, t8 # E : is there a zero? 366 andnot t2, t6, t2 # E : dest mask for a single word copy 367 or t8, t10, t5 # E : test for end-of-count too 368 369 cmpbge zero, t2, t3 # E : 370 cmoveq a2, t5, t8 # E : Latency=2, extra map slot 371 nop # E : keep with cmoveq 372 andnot t8, t3, t8 # E : (stall) 373 374 beq t8, $u_head # U : 375 /* At this point we've found a zero in the first partial word of 376 the source. We need to isolate the valid source data and mask 377 it into the original destination data. (Incidentally, we know 378 that we'll need at least one byte of that original dest word.) */ 379 ldq_u t0, 0(a0) # L : 380 negq t8, t6 # E : build bitmask of bytes <= zero 381 mskqh t1, t4, t1 # U : 382 383 and t6, t8, t12 # E : 384 subq t12, 1, t6 # E : (stall) 385 or t6, t12, t8 # E : (stall) 386 zapnot t2, t8, t2 # U : prepare source word; mirror changes (stall) 387 388 zapnot t1, t8, t1 # U : to source validity mask 389 andnot t0, t2, t0 # E : zero place for source to reside 390 or t0, t1, t0 # E : and put it there (stall both t0, t1) 391 stq_u t0, 0(a0) # L : (stall) 392 393 ret (t9) # L0 : Latency=3 394 nop 395 nop 396 nop 397 398 .end __stxncpy
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.