1 /* SPDX-License-Identifier: GPL-2.0+ !! 1 /* checksum.S: Sparc optimized checksum code.
2 * 2 *
3 * $Id: checksum.S,v 1.10 2001/07/06 13:11:32 !! 3 * Copyright(C) 1995 Linus Torvalds
>> 4 * Copyright(C) 1995 Miguel de Icaza
>> 5 * Copyright(C) 1996 David S. Miller
>> 6 * Copyright(C) 1997 Jakub Jelinek
4 * 7 *
5 * INET An implementation of the TCP/I !! 8 * derived from:
6 * operating system. INET is imp !! 9 * Linux/Alpha checksum c-code
7 * interface as the means of comm !! 10 * Linux/ix86 inline checksum assembly
8 * !! 11 * RFC1071 Computing the Internet Checksum (esp. Jacobsons m68k code)
9 * IP/TCP/UDP checksumming routin !! 12 * David Mosberger-Tang for optimized reference c-code
10 * !! 13 * BSD4.4 portable checksum routine
11 * Authors: Jorge Cwik, <jorge@laser.satlin <<
12 * Arnt Gulbrandsen, <agulbra@nvg. <<
13 * Tom May, <ftom@netcom.com> <<
14 * Pentium Pro/II routines: <<
15 * Alexander Kjeldaas <astor@guard <<
16 * Finn Arne Gangstad <finnag@guar <<
17 * Lots of code moved from tcp.c <<
18 * for more names. <<
19 * <<
20 * Changes: Ingo Molnar, converted csum_pa <<
21 * handling. <<
22 * Andi Kleen, add zeroing on er <<
23 * converted to pure assembl <<
24 * <<
25 * SuperH version: Copyright (C) 1999 Niibe <<
26 */ 14 */
27 15
>> 16 #include <asm/cprefix.h>
28 #include <asm/errno.h> 17 #include <asm/errno.h>
29 #include <linux/linkage.h> <<
30 18
31 /* !! 19 #define CSUM_BIGCHUNK(buf, offset, sum, t0, t1, t2, t3, t4, t5) \
32 * computes a partial checksum, e.g. for TCP/U !! 20 ldd [buf + offset + 0x00], t0; \
33 */ !! 21 ldd [buf + offset + 0x08], t2; \
34 !! 22 addxcc t0, sum, sum; \
35 /* !! 23 addxcc t1, sum, sum; \
36 * unsigned int csum_partial(const unsigned ch !! 24 ldd [buf + offset + 0x10], t4; \
37 * unsigned int sum) !! 25 addxcc t2, sum, sum; \
38 */ !! 26 addxcc t3, sum, sum; \
39 !! 27 ldd [buf + offset + 0x18], t0; \
40 .text !! 28 addxcc t4, sum, sum; \
41 ENTRY(csum_partial) !! 29 addxcc t5, sum, sum; \
42 /* !! 30 addxcc t0, sum, sum; \
43 * Experiments with Ethernet and SLI !! 31 addxcc t1, sum, sum;
44 * is aligned on either a 2-byte or !! 32
45 * least a twofold speedup on 486 an !! 33 #define CSUM_LASTCHUNK(buf, offset, sum, t0, t1, t2, t3) \
46 * Fortunately, it is easy to conver !! 34 ldd [buf - offset - 0x08], t0; \
47 * alignment for the unrolled loop. !! 35 ldd [buf - offset - 0x00], t2; \
48 */ !! 36 addxcc t0, sum, sum; \
49 mov r5, r1 !! 37 addxcc t1, sum, sum; \
50 mov r4, r0 !! 38 addxcc t2, sum, sum; \
51 tst #2, r0 ! Check alignm !! 39 addxcc t3, sum, sum;
52 bt 2f ! Jump if alig !! 40
53 ! !! 41 /* Do end cruft out of band to get better cache patterns. */
54 add #-2, r5 ! Alignment us !! 42 csum_partial_end_cruft:
55 cmp/pz r5 ! !! 43 be 1f ! caller asks %o1 & 0x8
56 bt/s 1f ! Jump if we h !! 44 andcc %o1, 4, %g0 ! nope, check for word remaining
57 clrt !! 45 ldd [%o0], %g2 ! load two
58 bra 6f !! 46 addcc %g2, %o2, %o2 ! add first word to sum
59 add #2, r5 ! r5 was < 2. !! 47 addxcc %g3, %o2, %o2 ! add second word as well
>> 48 add %o0, 8, %o0 ! advance buf ptr
>> 49 addx %g0, %o2, %o2 ! add in final carry
>> 50 andcc %o1, 4, %g0 ! check again for word remaining
>> 51 1: be 1f ! nope, skip this code
>> 52 andcc %o1, 3, %o1 ! check for trailing bytes
>> 53 ld [%o0], %g2 ! load it
>> 54 addcc %g2, %o2, %o2 ! add to sum
>> 55 add %o0, 4, %o0 ! advance buf ptr
>> 56 addx %g0, %o2, %o2 ! add in final carry
>> 57 andcc %o1, 3, %g0 ! check again for trailing bytes
>> 58 1: be 1f ! no trailing bytes, return
>> 59 addcc %o1, -1, %g0 ! only one byte remains?
>> 60 bne 2f ! at least two bytes more
>> 61 subcc %o1, 2, %o1 ! only two bytes more?
>> 62 b 4f ! only one byte remains
>> 63 or %g0, %g0, %o4 ! clear fake hword value
>> 64 2: lduh [%o0], %o4 ! get hword
>> 65 be 6f ! jmp if only hword remains
>> 66 add %o0, 2, %o0 ! advance buf ptr either way
>> 67 sll %o4, 16, %o4 ! create upper hword
>> 68 4: ldub [%o0], %o5 ! get final byte
>> 69 sll %o5, 8, %o5 ! put into place
>> 70 or %o5, %o4, %o4 ! coalese with hword (if any)
>> 71 6: addcc %o4, %o2, %o2 ! add to sum
>> 72 1: retl ! get outta here
>> 73 addx %g0, %o2, %o0 ! add final carry into retval
>> 74
>> 75 /* Also do alignment out of band to get better cache patterns. */
>> 76 csum_partial_fix_alignment:
>> 77 cmp %o1, 6
>> 78 bl cpte - 0x4
>> 79 andcc %o0, 0x2, %g0
>> 80 be 1f
>> 81 andcc %o0, 0x4, %g0
>> 82 lduh [%o0 + 0x00], %g2
>> 83 sub %o1, 2, %o1
>> 84 add %o0, 2, %o0
>> 85 sll %g2, 16, %g2
>> 86 addcc %g2, %o2, %o2
>> 87 srl %o2, 16, %g3
>> 88 addx %g0, %g3, %g2
>> 89 sll %o2, 16, %o2
>> 90 sll %g2, 16, %g3
>> 91 srl %o2, 16, %o2
>> 92 andcc %o0, 0x4, %g0
>> 93 or %g3, %o2, %o2
>> 94 1: be cpa
>> 95 andcc %o1, 0xffffff80, %o3
>> 96 ld [%o0 + 0x00], %g2
>> 97 sub %o1, 4, %o1
>> 98 addcc %g2, %o2, %o2
>> 99 add %o0, 4, %o0
>> 100 addx %g0, %o2, %o2
>> 101 b cpa
>> 102 andcc %o1, 0xffffff80, %o3
>> 103
>> 104 /* The common case is to get called with a nicely aligned
>> 105 * buffer of size 0x20. Follow the code path for that case.
>> 106 */
>> 107 .globl C_LABEL(csum_partial)
>> 108 C_LABEL(csum_partial): /* %o0=buf, %o1=len, %o2=sum */
>> 109 andcc %o0, 0x7, %g0 ! alignment problems?
>> 110 bne csum_partial_fix_alignment ! yep, handle it
>> 111 sethi %hi(cpte - 8), %g7 ! prepare table jmp ptr
>> 112 andcc %o1, 0xffffff80, %o3 ! num loop iterations
>> 113 cpa: be 3f ! none to do
>> 114 andcc %o1, 0x70, %g1 ! clears carry flag too
>> 115 5: CSUM_BIGCHUNK(%o0, 0x00, %o2, %o4, %o5, %g2, %g3, %g4, %g5)
>> 116 CSUM_BIGCHUNK(%o0, 0x20, %o2, %o4, %o5, %g2, %g3, %g4, %g5)
>> 117 CSUM_BIGCHUNK(%o0, 0x40, %o2, %o4, %o5, %g2, %g3, %g4, %g5)
>> 118 CSUM_BIGCHUNK(%o0, 0x60, %o2, %o4, %o5, %g2, %g3, %g4, %g5)
>> 119 addx %g0, %o2, %o2 ! sink in final carry
>> 120 subcc %o3, 128, %o3 ! detract from loop iters
>> 121 bne 5b ! more to do
>> 122 add %o0, 128, %o0 ! advance buf ptr
>> 123 andcc %o1, 0x70, %g1 ! clears carry flag too
>> 124 3: be cpte ! nope
>> 125 andcc %o1, 0xf, %g0 ! anything left at all?
>> 126 srl %g1, 1, %o4 ! compute offset
>> 127 sub %g7, %g1, %g7 ! adjust jmp ptr
>> 128 sub %g7, %o4, %g7 ! final jmp ptr adjust
>> 129 jmp %g7 + %lo(cpte - 8) ! enter the table
>> 130 add %o0, %g1, %o0 ! advance buf ptr
>> 131 cptbl: CSUM_LASTCHUNK(%o0, 0x68, %o2, %g2, %g3, %g4, %g5)
>> 132 CSUM_LASTCHUNK(%o0, 0x58, %o2, %g2, %g3, %g4, %g5)
>> 133 CSUM_LASTCHUNK(%o0, 0x48, %o2, %g2, %g3, %g4, %g5)
>> 134 CSUM_LASTCHUNK(%o0, 0x38, %o2, %g2, %g3, %g4, %g5)
>> 135 CSUM_LASTCHUNK(%o0, 0x28, %o2, %g2, %g3, %g4, %g5)
>> 136 CSUM_LASTCHUNK(%o0, 0x18, %o2, %g2, %g3, %g4, %g5)
>> 137 CSUM_LASTCHUNK(%o0, 0x08, %o2, %g2, %g3, %g4, %g5)
>> 138 addx %g0, %o2, %o2 ! fetch final carry
>> 139 andcc %o1, 0xf, %g0 ! anything left at all?
>> 140 cpte: bne csum_partial_end_cruft ! yep, handle it
>> 141 andcc %o1, 8, %g0 ! check how much
>> 142 cpout: retl ! get outta here
>> 143 mov %o2, %o0 ! return computed csum
>> 144
>> 145 .globl C_LABEL(__csum_partial_copy_start), C_LABEL(__csum_partial_copy_end)
>> 146 C_LABEL(__csum_partial_copy_start):
>> 147
>> 148 /* Work around cpp -rob */
>> 149 #define ALLOC #alloc
>> 150 #define EXECINSTR #execinstr
>> 151 #define EX(x,y,a,b) \
>> 152 98: x,y; \
>> 153 .section .fixup,ALLOC,EXECINSTR; \
>> 154 .align 4; \
>> 155 99: ba 30f; \
>> 156 a, b, %o3; \
>> 157 .section __ex_table,ALLOC; \
>> 158 .align 4; \
>> 159 .word 98b, 99b; \
>> 160 .text; \
>> 161 .align 4
>> 162
>> 163 #define EX2(x,y) \
>> 164 98: x,y; \
>> 165 .section __ex_table,ALLOC; \
>> 166 .align 4; \
>> 167 .word 98b, 30f; \
>> 168 .text; \
>> 169 .align 4
>> 170
>> 171 #define EX3(x,y) \
>> 172 98: x,y; \
>> 173 .section __ex_table,ALLOC; \
>> 174 .align 4; \
>> 175 .word 98b, 96f; \
>> 176 .text; \
>> 177 .align 4
>> 178
>> 179 #define EXT(start,end,handler) \
>> 180 .section __ex_table,ALLOC; \
>> 181 .align 4; \
>> 182 .word start, 0, end, handler; \
>> 183 .text; \
>> 184 .align 4
>> 185
>> 186 /* This aligned version executes typically in 8.5 superscalar cycles, this
>> 187 * is the best I can do. I say 8.5 because the final add will pair with
>> 188 * the next ldd in the main unrolled loop. Thus the pipe is always full.
>> 189 * If you change these macros (including order of instructions),
>> 190 * please check the fixup code below as well.
>> 191 */
>> 192 #define CSUMCOPY_BIGCHUNK_ALIGNED(src, dst, sum, off, t0, t1, t2, t3, t4, t5, t6, t7) \
>> 193 ldd [src + off + 0x00], t0; \
>> 194 ldd [src + off + 0x08], t2; \
>> 195 addxcc t0, sum, sum; \
>> 196 ldd [src + off + 0x10], t4; \
>> 197 addxcc t1, sum, sum; \
>> 198 ldd [src + off + 0x18], t6; \
>> 199 addxcc t2, sum, sum; \
>> 200 std t0, [dst + off + 0x00]; \
>> 201 addxcc t3, sum, sum; \
>> 202 std t2, [dst + off + 0x08]; \
>> 203 addxcc t4, sum, sum; \
>> 204 std t4, [dst + off + 0x10]; \
>> 205 addxcc t5, sum, sum; \
>> 206 std t6, [dst + off + 0x18]; \
>> 207 addxcc t6, sum, sum; \
>> 208 addxcc t7, sum, sum;
>> 209
>> 210 /* 12 superscalar cycles seems to be the limit for this case,
>> 211 * because of this we thus do all the ldd's together to get
>> 212 * Viking MXCC into streaming mode. Ho hum...
>> 213 */
>> 214 #define CSUMCOPY_BIGCHUNK(src, dst, sum, off, t0, t1, t2, t3, t4, t5, t6, t7) \
>> 215 ldd [src + off + 0x00], t0; \
>> 216 ldd [src + off + 0x08], t2; \
>> 217 ldd [src + off + 0x10], t4; \
>> 218 ldd [src + off + 0x18], t6; \
>> 219 st t0, [dst + off + 0x00]; \
>> 220 addxcc t0, sum, sum; \
>> 221 st t1, [dst + off + 0x04]; \
>> 222 addxcc t1, sum, sum; \
>> 223 st t2, [dst + off + 0x08]; \
>> 224 addxcc t2, sum, sum; \
>> 225 st t3, [dst + off + 0x0c]; \
>> 226 addxcc t3, sum, sum; \
>> 227 st t4, [dst + off + 0x10]; \
>> 228 addxcc t4, sum, sum; \
>> 229 st t5, [dst + off + 0x14]; \
>> 230 addxcc t5, sum, sum; \
>> 231 st t6, [dst + off + 0x18]; \
>> 232 addxcc t6, sum, sum; \
>> 233 st t7, [dst + off + 0x1c]; \
>> 234 addxcc t7, sum, sum;
>> 235
>> 236 /* Yuck, 6 superscalar cycles... */
>> 237 #define CSUMCOPY_LASTCHUNK(src, dst, sum, off, t0, t1, t2, t3) \
>> 238 ldd [src - off - 0x08], t0; \
>> 239 ldd [src - off - 0x00], t2; \
>> 240 addxcc t0, sum, sum; \
>> 241 st t0, [dst - off - 0x08]; \
>> 242 addxcc t1, sum, sum; \
>> 243 st t1, [dst - off - 0x04]; \
>> 244 addxcc t2, sum, sum; \
>> 245 st t2, [dst - off - 0x00]; \
>> 246 addxcc t3, sum, sum; \
>> 247 st t3, [dst - off + 0x04];
>> 248
>> 249 /* Handle the end cruft code out of band for better cache patterns. */
>> 250 cc_end_cruft:
>> 251 be 1f
>> 252 andcc %o3, 4, %g0
>> 253 EX(ldd [%o0 + 0x00], %g2, and %o3, 0xf)
>> 254 add %o1, 8, %o1
>> 255 addcc %g2, %g7, %g7
>> 256 add %o0, 8, %o0
>> 257 addxcc %g3, %g7, %g7
>> 258 EX2(st %g2, [%o1 - 0x08])
>> 259 addx %g0, %g7, %g7
>> 260 andcc %o3, 4, %g0
>> 261 EX2(st %g3, [%o1 - 0x04])
>> 262 1: be 1f
>> 263 andcc %o3, 3, %o3
>> 264 EX(ld [%o0 + 0x00], %g2, add %o3, 4)
>> 265 add %o1, 4, %o1
>> 266 addcc %g2, %g7, %g7
>> 267 EX2(st %g2, [%o1 - 0x04])
>> 268 addx %g0, %g7, %g7
>> 269 andcc %o3, 3, %g0
>> 270 add %o0, 4, %o0
>> 271 1: be 1f
>> 272 addcc %o3, -1, %g0
>> 273 bne 2f
>> 274 subcc %o3, 2, %o3
>> 275 b 4f
>> 276 or %g0, %g0, %o4
>> 277 2: EX(lduh [%o0 + 0x00], %o4, add %o3, 2)
>> 278 add %o0, 2, %o0
>> 279 EX2(sth %o4, [%o1 + 0x00])
>> 280 be 6f
>> 281 add %o1, 2, %o1
>> 282 sll %o4, 16, %o4
>> 283 4: EX(ldub [%o0 + 0x00], %o5, add %g0, 1)
>> 284 EX2(stb %o5, [%o1 + 0x00])
>> 285 sll %o5, 8, %o5
>> 286 or %o5, %o4, %o4
>> 287 6: addcc %o4, %g7, %g7
>> 288 1: retl
>> 289 addx %g0, %g7, %o0
>> 290
>> 291 /* Also, handle the alignment code out of band. */
>> 292 cc_dword_align:
>> 293 cmp %g1, 6
>> 294 bl,a ccte
>> 295 andcc %g1, 0xf, %o3
>> 296 andcc %o0, 0x1, %g0
>> 297 bne ccslow
>> 298 andcc %o0, 0x2, %g0
>> 299 be 1f
>> 300 andcc %o0, 0x4, %g0
>> 301 EX(lduh [%o0 + 0x00], %g4, add %g1, 0)
>> 302 sub %g1, 2, %g1
>> 303 EX2(sth %g4, [%o1 + 0x00])
>> 304 add %o0, 2, %o0
>> 305 sll %g4, 16, %g4
>> 306 addcc %g4, %g7, %g7
>> 307 add %o1, 2, %o1
>> 308 srl %g7, 16, %g3
>> 309 addx %g0, %g3, %g4
>> 310 sll %g7, 16, %g7
>> 311 sll %g4, 16, %g3
>> 312 srl %g7, 16, %g7
>> 313 andcc %o0, 0x4, %g0
>> 314 or %g3, %g7, %g7
>> 315 1: be 3f
>> 316 andcc %g1, 0xffffff80, %g0
>> 317 EX(ld [%o0 + 0x00], %g4, add %g1, 0)
>> 318 sub %g1, 4, %g1
>> 319 EX2(st %g4, [%o1 + 0x00])
>> 320 add %o0, 4, %o0
>> 321 addcc %g4, %g7, %g7
>> 322 add %o1, 4, %o1
>> 323 addx %g0, %g7, %g7
>> 324 b 3f
>> 325 andcc %g1, 0xffffff80, %g0
>> 326
>> 327 /* Sun, you just can't beat me, you just can't. Stop trying,
>> 328 * give up. I'm serious, I am going to kick the living shit
>> 329 * out of you, game over, lights out.
>> 330 */
>> 331 .align 8
>> 332 .globl C_LABEL(__csum_partial_copy_sparc_generic)
>> 333 C_LABEL(__csum_partial_copy_sparc_generic):
>> 334 /* %o0=src, %o1=dest, %g1=len, %g7=sum */
>> 335 xor %o0, %o1, %o4 ! get changing bits
>> 336 andcc %o4, 3, %g0 ! check for mismatched alignment
>> 337 bne ccslow ! better this than unaligned/fixups
>> 338 andcc %o0, 7, %g0 ! need to align things?
>> 339 bne cc_dword_align ! yes, we check for short lengths there
>> 340 andcc %g1, 0xffffff80, %g0 ! can we use unrolled loop?
>> 341 3: be 3f ! nope, less than one loop remains
>> 342 andcc %o1, 4, %g0 ! dest aligned on 4 or 8 byte boundary?
>> 343 be ccdbl + 4 ! 8 byte aligned, kick ass
>> 344 5: CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x00,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3)
>> 345 CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x20,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3)
>> 346 CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x40,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3)
>> 347 CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x60,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3)
>> 348 10: EXT(5b, 10b, 20f) ! note for exception handling
>> 349 sub %g1, 128, %g1 ! detract from length
>> 350 addx %g0, %g7, %g7 ! add in last carry bit
>> 351 andcc %g1, 0xffffff80, %g0 ! more to csum?
>> 352 add %o0, 128, %o0 ! advance src ptr
>> 353 bne 5b ! we did not go negative, continue looping
>> 354 add %o1, 128, %o1 ! advance dest ptr
>> 355 3: andcc %g1, 0x70, %o2 ! can use table?
>> 356 ccmerge:be ccte ! nope, go and check for end cruft
>> 357 andcc %g1, 0xf, %o3 ! get low bits of length (clears carry btw)
>> 358 srl %o2, 1, %o4 ! begin negative offset computation
>> 359 sethi %hi(12f), %o5 ! set up table ptr end
>> 360 add %o0, %o2, %o0 ! advance src ptr
>> 361 sub %o5, %o4, %o5 ! continue table calculation
>> 362 sll %o2, 1, %g2 ! constant multiplies are fun...
>> 363 sub %o5, %g2, %o5 ! some more adjustments
>> 364 jmp %o5 + %lo(12f) ! jump into it, duff style, wheee...
>> 365 add %o1, %o2, %o1 ! advance dest ptr (carry is clear btw)
>> 366 cctbl: CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x68,%g2,%g3,%g4,%g5)
>> 367 CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x58,%g2,%g3,%g4,%g5)
>> 368 CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x48,%g2,%g3,%g4,%g5)
>> 369 CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x38,%g2,%g3,%g4,%g5)
>> 370 CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x28,%g2,%g3,%g4,%g5)
>> 371 CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x18,%g2,%g3,%g4,%g5)
>> 372 CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x08,%g2,%g3,%g4,%g5)
>> 373 12: EXT(cctbl, 12b, 22f) ! note for exception table handling
>> 374 addx %g0, %g7, %g7
>> 375 andcc %o3, 0xf, %g0 ! check for low bits set
>> 376 ccte: bne cc_end_cruft ! something left, handle it out of band
>> 377 andcc %o3, 8, %g0 ! begin checks for that code
>> 378 retl ! return
>> 379 mov %g7, %o0 ! give em the computed checksum
>> 380 ccdbl: CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x00,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3)
>> 381 CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x20,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3)
>> 382 CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x40,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3)
>> 383 CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x60,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3)
>> 384 11: EXT(ccdbl, 11b, 21f) ! note for exception table handling
>> 385 sub %g1, 128, %g1 ! detract from length
>> 386 addx %g0, %g7, %g7 ! add in last carry bit
>> 387 andcc %g1, 0xffffff80, %g0 ! more to csum?
>> 388 add %o0, 128, %o0 ! advance src ptr
>> 389 bne ccdbl ! we did not go negative, continue looping
>> 390 add %o1, 128, %o1 ! advance dest ptr
>> 391 b ccmerge ! finish it off, above
>> 392 andcc %g1, 0x70, %o2 ! can use table? (clears carry btw)
>> 393
>> 394 ccslow: cmp %g1, 0
>> 395 mov 0, %g5
>> 396 bleu 4f
>> 397 andcc %o0, 1, %o5
>> 398 be,a 1f
>> 399 srl %g1, 1, %g4
>> 400 sub %g1, 1, %g1
>> 401 EX(ldub [%o0], %g5, add %g1, 1)
>> 402 add %o0, 1, %o0
>> 403 EX2(stb %g5, [%o1])
>> 404 srl %g1, 1, %g4
>> 405 add %o1, 1, %o1
>> 406 1: cmp %g4, 0
>> 407 be,a 3f
>> 408 andcc %g1, 1, %g0
>> 409 andcc %o0, 2, %g0
>> 410 be,a 1f
>> 411 srl %g4, 1, %g4
>> 412 EX(lduh [%o0], %o4, add %g1, 0)
>> 413 sub %g1, 2, %g1
>> 414 srl %o4, 8, %g2
>> 415 sub %g4, 1, %g4
>> 416 EX2(stb %g2, [%o1])
>> 417 add %o4, %g5, %g5
>> 418 EX2(stb %o4, [%o1 + 1])
>> 419 add %o0, 2, %o0
>> 420 srl %g4, 1, %g4
>> 421 add %o1, 2, %o1
>> 422 1: cmp %g4, 0
>> 423 be,a 2f
>> 424 andcc %g1, 2, %g0
>> 425 EX3(ld [%o0], %o4)
>> 426 5: srl %o4, 24, %g2
>> 427 srl %o4, 16, %g3
>> 428 EX2(stb %g2, [%o1])
>> 429 srl %o4, 8, %g2
>> 430 EX2(stb %g3, [%o1 + 1])
>> 431 add %o0, 4, %o0
>> 432 EX2(stb %g2, [%o1 + 2])
>> 433 addcc %o4, %g5, %g5
>> 434 EX2(stb %o4, [%o1 + 3])
>> 435 addx %g5, %g0, %g5 ! I am now to lazy to optimize this (question it
>> 436 add %o1, 4, %o1 ! is worthy). Maybe some day - with the sll/srl
>> 437 subcc %g4, 1, %g4 ! tricks
>> 438 bne,a 5b
>> 439 EX3(ld [%o0], %o4)
>> 440 sll %g5, 16, %g2
>> 441 srl %g5, 16, %g5
>> 442 srl %g2, 16, %g2
>> 443 andcc %g1, 2, %g0
>> 444 add %g2, %g5, %g5
>> 445 2: be,a 3f
>> 446 andcc %g1, 1, %g0
>> 447 EX(lduh [%o0], %o4, and %g1, 3)
>> 448 andcc %g1, 1, %g0
>> 449 srl %o4, 8, %g2
>> 450 add %o0, 2, %o0
>> 451 EX2(stb %g2, [%o1])
>> 452 add %g5, %o4, %g5
>> 453 EX2(stb %o4, [%o1 + 1])
>> 454 add %o1, 2, %o1
>> 455 3: be,a 1f
>> 456 sll %g5, 16, %o4
>> 457 EX(ldub [%o0], %g2, add %g0, 1)
>> 458 sll %g2, 8, %o4
>> 459 EX2(stb %g2, [%o1])
>> 460 add %g5, %o4, %g5
>> 461 sll %g5, 16, %o4
>> 462 1: addcc %o4, %g5, %g5
>> 463 srl %g5, 16, %o4
>> 464 addx %g0, %o4, %g5
>> 465 orcc %o5, %g0, %g0
>> 466 be 4f
>> 467 srl %g5, 8, %o4
>> 468 and %g5, 0xff, %g2
>> 469 and %o4, 0xff, %o4
>> 470 sll %g2, 8, %g2
>> 471 or %g2, %o4, %g5
>> 472 4: addcc %g7, %g5, %g7
>> 473 retl
>> 474 addx %g0, %g7, %o0
>> 475 C_LABEL(__csum_partial_copy_end):
>> 476
>> 477 /* We do these strange calculations for the csum_*_from_user case only, ie.
>> 478 * we only bother with faults on loads... */
>> 479
>> 480 /* o2 = ((g2%20)&3)*8
>> 481 * o3 = g1 - (g2/20)*32 - o2 */
>> 482 20:
>> 483 cmp %g2, 20
>> 484 blu,a 1f
>> 485 and %g2, 3, %o2
>> 486 sub %g1, 32, %g1
>> 487 b 20b
>> 488 sub %g2, 20, %g2
60 1: 489 1:
61 mov r5, r1 ! Save new len !! 490 sll %o2, 3, %o2
62 mov.w @r4+, r0 !! 491 b 31f
63 extu.w r0, r0 !! 492 sub %g1, %o2, %o3
64 addc r0, r6 !! 493
65 bf 2f !! 494 /* o2 = (!(g2 & 15) ? 0 : (((g2 & 15) + 1) & ~1)*8)
66 add #1, r6 !! 495 * o3 = g1 - (g2/16)*32 - o2 */
67 2: !! 496 21:
68 mov #-5, r0 !! 497 andcc %g2, 15, %o3
69 shld r0, r5 !! 498 srl %g2, 4, %g2
70 tst r5, r5 !! 499 be,a 1f
71 bt/s 4f ! if it's =0, !! 500 clr %o2
72 clrt !! 501 add %o3, 1, %o3
73 .align 2 !! 502 and %o3, 14, %o3
74 3: !! 503 sll %o3, 3, %o2
75 mov.l @r4+, r0 !! 504 1:
76 mov.l @r4+, r2 !! 505 sll %g2, 5, %g2
77 mov.l @r4+, r3 !! 506 sub %g1, %g2, %o3
78 addc r0, r6 !! 507 b 31f
79 mov.l @r4+, r0 !! 508 sub %o3, %o2, %o3
80 addc r2, r6 !! 509
81 mov.l @r4+, r2 !! 510 /* o0 += (g2/10)*16 - 0x70
82 addc r3, r6 !! 511 * 01 += (g2/10)*16 - 0x70
83 mov.l @r4+, r3 !! 512 * o2 = (g2 % 10) ? 8 : 0
84 addc r0, r6 !! 513 * o3 += 0x70 - (g2/10)*16 - o2 */
85 mov.l @r4+, r0 !! 514 22:
86 addc r2, r6 !! 515 cmp %g2, 10
87 mov.l @r4+, r2 !! 516 blu,a 1f
88 addc r3, r6 !! 517 sub %o0, 0x70, %o0
89 addc r0, r6 !! 518 add %o0, 16, %o0
90 addc r2, r6 !! 519 add %o1, 16, %o1
91 movt r0 !! 520 sub %o3, 16, %o3
92 dt r5 !! 521 b 22b
93 bf/s 3b !! 522 sub %g2, 10, %g2
94 cmp/eq #1, r0 !! 523 1:
95 ! here, we know r5==0 !! 524 sub %o1, 0x70, %o1
96 addc r5, r6 ! add !! 525 add %o3, 0x70, %o3
97 4: !! 526 clr %o2
98 mov r1, r0 !! 527 tst %g2
99 and #0x1c, r0 !! 528 bne,a 1f
100 tst r0, r0 !! 529 mov 8, %o2
101 bt/s 6f !! 530 1:
102 mov r0, r5 !! 531 b 31f
103 shlr2 r5 !! 532 sub %o3, %o2, %o3
104 mov #0, r2 !! 533 96:
105 5: !! 534 and %g1, 3, %g1
106 addc r2, r6 !! 535 sll %g4, 2, %g4
107 mov.l @r4+, r2 !! 536 add %g1, %g4, %o3
108 movt r0 !! 537 30:
109 dt r5 !! 538 /* %o1 is dst
110 bf/s 5b !! 539 * %o3 is # bytes to zero out
111 cmp/eq #1, r0 !! 540 * %o4 is faulting address
112 addc r2, r6 !! 541 * %o5 is %pc where fault occurred */
113 addc r5, r6 ! r5==0 here, !! 542 clr %o2
114 6: !! 543 31:
115 mov r1, r5 !! 544 /* %o0 is src
116 mov #3, r0 !! 545 * %o1 is dst
117 and r0, r5 !! 546 * %o2 is # of bytes to copy from src to dst
118 tst r5, r5 !! 547 * %o3 is # bytes to zero out
119 bt 9f ! if it's =0 g !! 548 * %o4 is faulting address
120 mov #2, r1 !! 549 * %o5 is %pc where fault occurred */
121 cmp/hs r1, r5 !! 550 save %sp, -104, %sp
122 bf 7f !! 551 mov %i5, %o0
123 mov.w @r4+, r0 !! 552 mov %i7, %o1
124 extu.w r0, r0 !! 553 mov %i4, %o2
125 cmp/eq r1, r5 !! 554 call C_LABEL(lookup_fault)
126 bt/s 8f !! 555 mov %g7, %i4
127 clrt !! 556 cmp %o0, 2
128 shll16 r0 !! 557 bne 1f
129 addc r0, r6 !! 558 add %g0, -EFAULT, %i5
130 7: !! 559 tst %i2
131 mov.b @r4+, r0 !! 560 be 2f
132 extu.b r0, r0 !! 561 mov %i0, %o1
133 #ifndef __LITTLE_ENDIAN__ !! 562 mov %i1, %o0
134 shll8 r0 <<
135 #endif <<
136 8: <<
137 addc r0, r6 <<
138 mov #0, r0 <<
139 addc r0, r6 <<
140 9: <<
141 rts <<
142 mov r6, r0 <<
143 <<
144 /* <<
145 unsigned int csum_partial_copy_generic (const <<
146 */ <<
147 <<
148 /* <<
149 * Copy from ds while checksumming, otherwise <<
150 * sum being ~0U <<
151 */ <<
152 <<
153 #define EXC(...) \ <<
154 9999: __VA_ARGS__ ; \ <<
155 .section __ex_table, "a"; \ <<
156 .long 9999b, 6001f ; \ <<
157 .previous <<
158 <<
159 ! <<
160 ! r4: const char *SRC <<
161 ! r5: char *DST <<
162 ! r6: int LEN <<
163 ! <<
164 ENTRY(csum_partial_copy_generic) <<
165 mov #-1,r7 <<
166 mov #3,r0 ! Check src an <<
167 mov r4,r1 <<
168 and r0,r1 <<
169 and r5,r0 <<
170 cmp/eq r1,r0 <<
171 bf 3f ! Different al <<
172 tst #1,r0 ! Check dest w <<
173 bf 3f ! If not, do i <<
174 <<
175 mov #2,r0 <<
176 tst r0,r5 ! Check dest a <<
177 bt 2f ! Jump if alig <<
178 add #-2,r6 ! Alignment us <<
179 cmp/pz r6 ! Jump if we h <<
180 bt/s 1f <<
181 clrt <<
182 add #2,r6 ! r6 was < 2. <<
183 bra 4f <<
184 mov r6,r2 <<
185 <<
186 3: ! Handle different src and dest alignm <<
187 ! This is not common, so simple byte b <<
188 mov r6,r2 <<
189 shlr r6 <<
190 tst r6,r6 <<
191 bt 4f <<
192 clrt <<
193 .align 2 <<
194 5: 563 5:
195 EXC( mov.b @r4+,r1 ) !! 564 call C_LABEL(__memcpy)
196 EXC( mov.b @r4+,r0 ) !! 565 mov %i2, %o2
197 extu.b r1,r1 !! 566 tst %o0
198 EXC( mov.b r1,@r5 ) !! 567 bne,a 2f
199 EXC( mov.b r0,@(1,r5) ) !! 568 add %i3, %i2, %i3
200 extu.b r0,r0 !! 569 add %i1, %i2, %i1
201 add #2,r5 <<
202 <<
203 #ifdef __LITTLE_ENDIAN__ <<
204 shll8 r0 <<
205 #else <<
206 shll8 r1 <<
207 #endif <<
208 or r1,r0 <<
209 <<
210 addc r0,r7 <<
211 movt r0 <<
212 dt r6 <<
213 bf/s 5b <<
214 cmp/eq #1,r0 <<
215 mov #0,r0 <<
216 addc r0, r7 <<
217 <<
218 mov r2, r0 <<
219 tst #1, r0 <<
220 bt 7f <<
221 bra 5f <<
222 clrt <<
223 <<
224 ! src and dest equally aligned, but to <<
225 ! Handle first two bytes as a special <<
226 .align 2 <<
227 1: <<
228 EXC( mov.w @r4+,r0 ) <<
229 EXC( mov.w r0,@r5 ) <<
230 add #2,r5 <<
231 extu.w r0,r0 <<
232 addc r0,r7 <<
233 mov #0,r0 <<
234 addc r0,r7 <<
235 2: 570 2:
236 mov r6,r2 !! 571 mov %i1, %o0
237 mov #-5,r0 !! 572 6:
238 shld r0,r6 !! 573 call C_LABEL(__bzero)
239 tst r6,r6 !! 574 mov %i3, %o1
240 bt/s 2f !! 575 1:
241 clrt !! 576 ld [%sp + 168], %o2 ! struct_ptr of parent
242 .align 2 !! 577 st %i5, [%o2]
243 1: !! 578 ret
244 EXC( mov.l @r4+,r0 ) !! 579 restore
245 EXC( mov.l @r4+,r1 ) !! 580
246 addc r0,r7 !! 581 .section __ex_table,#alloc
247 EXC( mov.l r0,@r5 ) !! 582 .align 4
248 EXC( mov.l r1,@(4,r5) ) !! 583 .word 5b,2
249 addc r1,r7 !! 584 .word 6b,2
250 <<
251 EXC( mov.l @r4+,r0 ) <<
252 EXC( mov.l @r4+,r1 ) <<
253 addc r0,r7 <<
254 EXC( mov.l r0,@(8,r5) ) <<
255 EXC( mov.l r1,@(12,r5) ) <<
256 addc r1,r7 <<
257 <<
258 EXC( mov.l @r4+,r0 ) <<
259 EXC( mov.l @r4+,r1 ) <<
260 addc r0,r7 <<
261 EXC( mov.l r0,@(16,r5) ) <<
262 EXC( mov.l r1,@(20,r5) ) <<
263 addc r1,r7 <<
264 <<
265 EXC( mov.l @r4+,r0 ) <<
266 EXC( mov.l @r4+,r1 ) <<
267 addc r0,r7 <<
268 EXC( mov.l r0,@(24,r5) ) <<
269 EXC( mov.l r1,@(28,r5) ) <<
270 addc r1,r7 <<
271 add #32,r5 <<
272 movt r0 <<
273 dt r6 <<
274 bf/s 1b <<
275 cmp/eq #1,r0 <<
276 mov #0,r0 <<
277 addc r0,r7 <<
278 <<
279 2: mov r2,r6 <<
280 mov #0x1c,r0 <<
281 and r0,r6 <<
282 cmp/pl r6 <<
283 bf/s 4f <<
284 clrt <<
285 shlr2 r6 <<
286 3: <<
287 EXC( mov.l @r4+,r0 ) <<
288 addc r0,r7 <<
289 EXC( mov.l r0,@r5 ) <<
290 add #4,r5 <<
291 movt r0 <<
292 dt r6 <<
293 bf/s 3b <<
294 cmp/eq #1,r0 <<
295 mov #0,r0 <<
296 addc r0,r7 <<
297 4: mov r2,r6 <<
298 mov #3,r0 <<
299 and r0,r6 <<
300 cmp/pl r6 <<
301 bf 7f <<
302 mov #2,r1 <<
303 cmp/hs r1,r6 <<
304 bf 5f <<
305 EXC( mov.w @r4+,r0 ) <<
306 EXC( mov.w r0,@r5 ) <<
307 extu.w r0,r0 <<
308 add #2,r5 <<
309 cmp/eq r1,r6 <<
310 bt/s 6f <<
311 clrt <<
312 shll16 r0 <<
313 addc r0,r7 <<
314 5: <<
315 EXC( mov.b @r4+,r0 ) <<
316 EXC( mov.b r0,@r5 ) <<
317 extu.b r0,r0 <<
318 #ifndef __LITTLE_ENDIAN__ <<
319 shll8 r0 <<
320 #endif <<
321 6: addc r0,r7 <<
322 mov #0,r0 <<
323 addc r0,r7 <<
324 7: <<
325 <<
326 # Exception handler: <<
327 .section .fixup, "ax" <<
328 <<
329 6001: <<
330 rts <<
331 mov #0,r0 <<
332 .previous <<
333 rts <<
334 mov r7,r0 <<
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.