1 /* SPDX-License-Identifier: GPL-2.0 */ <<
2 /* memcpy.S: Sparc optimized memcpy and memmov 1 /* memcpy.S: Sparc optimized memcpy and memmove code
3 * Hand optimized from GNU libc's memcpy and m 2 * Hand optimized from GNU libc's memcpy and memmove
4 * Copyright (C) 1991,1996 Free Software Found 3 * Copyright (C) 1991,1996 Free Software Foundation
5 * Copyright (C) 1995 Linus Torvalds (Linus.To 4 * Copyright (C) 1995 Linus Torvalds (Linus.Torvalds@helsinki.fi)
6 * Copyright (C) 1996 David S. Miller (davem@c 5 * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
7 * Copyright (C) 1996 Eddie C. Dost (ecd@skyne 6 * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
8 * Copyright (C) 1996 Jakub Jelinek (jj@sunsit 7 * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
9 */ 8 */
10 9
11 #include <linux/export.h> !! 10 #ifdef __KERNEL__
12 11
13 #define FUNC(x) \ !! 12 #define FUNC(x) \
14 .globl x; \ 13 .globl x; \
15 .type x,@function; \ 14 .type x,@function; \
16 .align 4; \ !! 15 .align 4; \
17 x: 16 x:
18 17
>> 18 #undef FASTER_REVERSE
>> 19 #undef FASTER_NONALIGNED
>> 20 #define FASTER_ALIGNED
>> 21
>> 22 /* In kernel these functions don't return a value.
>> 23 * One should use macros in asm/string.h for that purpose.
>> 24 * We return 0, so that bugs are more apparent.
>> 25 */
>> 26 #define SETUP_RETL
>> 27 #define RETL_INSN clr %o0
>> 28
>> 29 #else
>> 30
>> 31 /* libc */
>> 32
>> 33 #include "DEFS.h"
>> 34
>> 35 #define FASTER_REVERSE
>> 36 #define FASTER_NONALIGNED
>> 37 #define FASTER_ALIGNED
>> 38
>> 39 #define SETUP_RETL mov %o0, %g6
>> 40 #define RETL_INSN mov %g6, %o0
>> 41
>> 42 #endif
>> 43
19 /* Both these macros have to start with exactl 44 /* Both these macros have to start with exactly the same insn */
20 #define MOVE_BIGCHUNK(src, dst, offset, t0, t1 45 #define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
21 ldd [%src + (offset) + 0x00], %t0; 46 ldd [%src + (offset) + 0x00], %t0; \
22 ldd [%src + (offset) + 0x08], %t2; 47 ldd [%src + (offset) + 0x08], %t2; \
23 ldd [%src + (offset) + 0x10], %t4; 48 ldd [%src + (offset) + 0x10], %t4; \
24 ldd [%src + (offset) + 0x18], %t6; 49 ldd [%src + (offset) + 0x18], %t6; \
25 st %t0, [%dst + (offset) + 0x00]; 50 st %t0, [%dst + (offset) + 0x00]; \
26 st %t1, [%dst + (offset) + 0x04]; 51 st %t1, [%dst + (offset) + 0x04]; \
27 st %t2, [%dst + (offset) + 0x08]; 52 st %t2, [%dst + (offset) + 0x08]; \
28 st %t3, [%dst + (offset) + 0x0c]; 53 st %t3, [%dst + (offset) + 0x0c]; \
29 st %t4, [%dst + (offset) + 0x10]; 54 st %t4, [%dst + (offset) + 0x10]; \
30 st %t5, [%dst + (offset) + 0x14]; 55 st %t5, [%dst + (offset) + 0x14]; \
31 st %t6, [%dst + (offset) + 0x18]; 56 st %t6, [%dst + (offset) + 0x18]; \
32 st %t7, [%dst + (offset) + 0x1c]; 57 st %t7, [%dst + (offset) + 0x1c];
33 58
34 #define MOVE_BIGALIGNCHUNK(src, dst, offset, t 59 #define MOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
35 ldd [%src + (offset) + 0x00], %t0; 60 ldd [%src + (offset) + 0x00], %t0; \
36 ldd [%src + (offset) + 0x08], %t2; 61 ldd [%src + (offset) + 0x08], %t2; \
37 ldd [%src + (offset) + 0x10], %t4; 62 ldd [%src + (offset) + 0x10], %t4; \
38 ldd [%src + (offset) + 0x18], %t6; 63 ldd [%src + (offset) + 0x18], %t6; \
39 std %t0, [%dst + (offset) + 0x00]; 64 std %t0, [%dst + (offset) + 0x00]; \
40 std %t2, [%dst + (offset) + 0x08]; 65 std %t2, [%dst + (offset) + 0x08]; \
41 std %t4, [%dst + (offset) + 0x10]; 66 std %t4, [%dst + (offset) + 0x10]; \
42 std %t6, [%dst + (offset) + 0x18]; 67 std %t6, [%dst + (offset) + 0x18];
43 68
44 #define MOVE_LASTCHUNK(src, dst, offset, t0, t 69 #define MOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \
45 ldd [%src - (offset) - 0x10], %t0; 70 ldd [%src - (offset) - 0x10], %t0; \
46 ldd [%src - (offset) - 0x08], %t2; 71 ldd [%src - (offset) - 0x08], %t2; \
47 st %t0, [%dst - (offset) - 0x10]; 72 st %t0, [%dst - (offset) - 0x10]; \
48 st %t1, [%dst - (offset) - 0x0c]; 73 st %t1, [%dst - (offset) - 0x0c]; \
49 st %t2, [%dst - (offset) - 0x08]; 74 st %t2, [%dst - (offset) - 0x08]; \
50 st %t3, [%dst - (offset) - 0x04]; 75 st %t3, [%dst - (offset) - 0x04];
51 76
52 #define MOVE_LASTALIGNCHUNK(src, dst, offset, 77 #define MOVE_LASTALIGNCHUNK(src, dst, offset, t0, t1, t2, t3) \
53 ldd [%src - (offset) - 0x10], %t0; 78 ldd [%src - (offset) - 0x10], %t0; \
54 ldd [%src - (offset) - 0x08], %t2; 79 ldd [%src - (offset) - 0x08], %t2; \
55 std %t0, [%dst - (offset) - 0x10]; 80 std %t0, [%dst - (offset) - 0x10]; \
56 std %t2, [%dst - (offset) - 0x08]; 81 std %t2, [%dst - (offset) - 0x08];
57 82
58 #define MOVE_SHORTCHUNK(src, dst, offset, t0, 83 #define MOVE_SHORTCHUNK(src, dst, offset, t0, t1) \
59 ldub [%src - (offset) - 0x02], %t0; 84 ldub [%src - (offset) - 0x02], %t0; \
60 ldub [%src - (offset) - 0x01], %t1; 85 ldub [%src - (offset) - 0x01], %t1; \
61 stb %t0, [%dst - (offset) - 0x02]; 86 stb %t0, [%dst - (offset) - 0x02]; \
62 stb %t1, [%dst - (offset) - 0x01]; 87 stb %t1, [%dst - (offset) - 0x01];
63 88
>> 89 /* Both these macros have to start with exactly the same insn */
>> 90 #define RMOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
>> 91 ldd [%src - (offset) - 0x20], %t0; \
>> 92 ldd [%src - (offset) - 0x18], %t2; \
>> 93 ldd [%src - (offset) - 0x10], %t4; \
>> 94 ldd [%src - (offset) - 0x08], %t6; \
>> 95 st %t0, [%dst - (offset) - 0x20]; \
>> 96 st %t1, [%dst - (offset) - 0x1c]; \
>> 97 st %t2, [%dst - (offset) - 0x18]; \
>> 98 st %t3, [%dst - (offset) - 0x14]; \
>> 99 st %t4, [%dst - (offset) - 0x10]; \
>> 100 st %t5, [%dst - (offset) - 0x0c]; \
>> 101 st %t6, [%dst - (offset) - 0x08]; \
>> 102 st %t7, [%dst - (offset) - 0x04];
>> 103
>> 104 #define RMOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
>> 105 ldd [%src - (offset) - 0x20], %t0; \
>> 106 ldd [%src - (offset) - 0x18], %t2; \
>> 107 ldd [%src - (offset) - 0x10], %t4; \
>> 108 ldd [%src - (offset) - 0x08], %t6; \
>> 109 std %t0, [%dst - (offset) - 0x20]; \
>> 110 std %t2, [%dst - (offset) - 0x18]; \
>> 111 std %t4, [%dst - (offset) - 0x10]; \
>> 112 std %t6, [%dst - (offset) - 0x08];
>> 113
>> 114 #define RMOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \
>> 115 ldd [%src + (offset) + 0x00], %t0; \
>> 116 ldd [%src + (offset) + 0x08], %t2; \
>> 117 st %t0, [%dst + (offset) + 0x00]; \
>> 118 st %t1, [%dst + (offset) + 0x04]; \
>> 119 st %t2, [%dst + (offset) + 0x08]; \
>> 120 st %t3, [%dst + (offset) + 0x0c];
>> 121
>> 122 #define RMOVE_SHORTCHUNK(src, dst, offset, t0, t1) \
>> 123 ldub [%src + (offset) + 0x00], %t0; \
>> 124 ldub [%src + (offset) + 0x01], %t1; \
>> 125 stb %t0, [%dst + (offset) + 0x00]; \
>> 126 stb %t1, [%dst + (offset) + 0x01];
>> 127
>> 128 #define SMOVE_CHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, prev, shil, shir, offset2) \
>> 129 ldd [%src + (offset) + 0x00], %t0; \
>> 130 ldd [%src + (offset) + 0x08], %t2; \
>> 131 srl %t0, shir, %t5; \
>> 132 srl %t1, shir, %t6; \
>> 133 sll %t0, shil, %t0; \
>> 134 or %t5, %prev, %t5; \
>> 135 sll %t1, shil, %prev; \
>> 136 or %t6, %t0, %t0; \
>> 137 srl %t2, shir, %t1; \
>> 138 srl %t3, shir, %t6; \
>> 139 sll %t2, shil, %t2; \
>> 140 or %t1, %prev, %t1; \
>> 141 std %t4, [%dst + (offset) + (offset2) - 0x04]; \
>> 142 std %t0, [%dst + (offset) + (offset2) + 0x04]; \
>> 143 sll %t3, shil, %prev; \
>> 144 or %t6, %t2, %t4;
>> 145
>> 146 #define SMOVE_ALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, prev, shil, shir, offset2) \
>> 147 ldd [%src + (offset) + 0x00], %t0; \
>> 148 ldd [%src + (offset) + 0x08], %t2; \
>> 149 srl %t0, shir, %t4; \
>> 150 srl %t1, shir, %t5; \
>> 151 sll %t0, shil, %t6; \
>> 152 or %t4, %prev, %t0; \
>> 153 sll %t1, shil, %prev; \
>> 154 or %t5, %t6, %t1; \
>> 155 srl %t2, shir, %t4; \
>> 156 srl %t3, shir, %t5; \
>> 157 sll %t2, shil, %t6; \
>> 158 or %t4, %prev, %t2; \
>> 159 sll %t3, shil, %prev; \
>> 160 or %t5, %t6, %t3; \
>> 161 std %t0, [%dst + (offset) + (offset2) + 0x00]; \
>> 162 std %t2, [%dst + (offset) + (offset2) + 0x08];
>> 163
64 .text 164 .text
65 .align 4 165 .align 4
66 166
>> 167 #ifdef FASTER_REVERSE
>> 168
>> 169 70: /* rdword_align */
>> 170
>> 171 andcc %o1, 1, %g0
>> 172 be 4f
>> 173 andcc %o1, 2, %g0
>> 174
>> 175 ldub [%o1 - 1], %g2
>> 176 sub %o1, 1, %o1
>> 177 stb %g2, [%o0 - 1]
>> 178 sub %o2, 1, %o2
>> 179 be 3f
>> 180 sub %o0, 1, %o0
>> 181 4:
>> 182 lduh [%o1 - 2], %g2
>> 183 sub %o1, 2, %o1
>> 184 sth %g2, [%o0 - 2]
>> 185 sub %o2, 2, %o2
>> 186 b 3f
>> 187 sub %o0, 2, %o0
>> 188
>> 189 #endif /* FASTER_REVERSE */
>> 190
>> 191 0:
>> 192 retl
>> 193 nop ! Only bcopy returns here and it retuns void...
>> 194
>> 195 #ifdef __KERNEL__
>> 196 FUNC(amemmove)
>> 197 FUNC(__memmove)
>> 198 #endif
67 FUNC(memmove) 199 FUNC(memmove)
68 EXPORT_SYMBOL(memmove) <<
69 cmp %o0, %o1 200 cmp %o0, %o1
70 mov %o0, %g7 !! 201 SETUP_RETL
71 bleu 9f 202 bleu 9f
72 sub %o0, %o1, %o4 203 sub %o0, %o1, %o4
73 204
74 add %o1, %o2, %o3 205 add %o1, %o2, %o3
75 cmp %o3, %o0 206 cmp %o3, %o0
76 bleu 0f 207 bleu 0f
77 andcc %o4, 3, %o5 208 andcc %o4, 3, %o5
78 209
>> 210 #ifndef FASTER_REVERSE
>> 211
79 add %o1, %o2, %o1 212 add %o1, %o2, %o1
80 add %o0, %o2, %o0 213 add %o0, %o2, %o0
81 sub %o1, 1, %o1 214 sub %o1, 1, %o1
82 sub %o0, 1, %o0 215 sub %o0, 1, %o0
83 216
84 1: /* reverse_bytes */ 217 1: /* reverse_bytes */
85 218
86 ldub [%o1], %o4 219 ldub [%o1], %o4
87 subcc %o2, 1, %o2 220 subcc %o2, 1, %o2
88 stb %o4, [%o0] 221 stb %o4, [%o0]
89 sub %o1, 1, %o1 222 sub %o1, 1, %o1
90 bne 1b 223 bne 1b
91 sub %o0, 1, %o0 224 sub %o0, 1, %o0
92 225
93 retl 226 retl
94 mov %g7, %o0 !! 227 RETL_INSN
>> 228
>> 229 #else /* FASTER_REVERSE */
>> 230
>> 231 add %o1, %o2, %o1
>> 232 add %o0, %o2, %o0
>> 233 bne 77f
>> 234 cmp %o2, 15
>> 235 bleu 91f
>> 236 andcc %o1, 3, %g0
>> 237 bne 70b
>> 238 3:
>> 239 andcc %o1, 4, %g0
>> 240
>> 241 be 2f
>> 242 mov %o2, %g1
>> 243
>> 244 ld [%o1 - 4], %o4
>> 245 sub %g1, 4, %g1
>> 246 st %o4, [%o0 - 4]
>> 247 sub %o1, 4, %o1
>> 248 sub %o0, 4, %o0
>> 249 2:
>> 250 andcc %g1, 0xffffff80, %g7
>> 251 be 3f
>> 252 andcc %o0, 4, %g0
>> 253
>> 254 be 74f + 4
>> 255 5:
>> 256 RMOVE_BIGCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5)
>> 257 RMOVE_BIGCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
>> 258 RMOVE_BIGCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
>> 259 RMOVE_BIGCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
>> 260 subcc %g7, 128, %g7
>> 261 sub %o1, 128, %o1
>> 262 bne 5b
>> 263 sub %o0, 128, %o0
>> 264 3:
>> 265 andcc %g1, 0x70, %g7
>> 266 be 72f
>> 267 andcc %g1, 8, %g0
>> 268
>> 269 sethi %hi(72f), %o5
>> 270 srl %g7, 1, %o4
>> 271 add %g7, %o4, %o4
>> 272 sub %o1, %g7, %o1
>> 273 sub %o5, %o4, %o5
>> 274 jmpl %o5 + %lo(72f), %g0
>> 275 sub %o0, %g7, %o0
>> 276
>> 277 71: /* rmemcpy_table */
>> 278 RMOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g4, g5)
>> 279 RMOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g4, g5)
>> 280 RMOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g4, g5)
>> 281 RMOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g4, g5)
>> 282 RMOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g4, g5)
>> 283 RMOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g4, g5)
>> 284 RMOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g4, g5)
>> 285
>> 286 72: /* rmemcpy_table_end */
>> 287
>> 288 be 73f
>> 289 andcc %g1, 4, %g0
>> 290
>> 291 ldd [%o1 - 0x08], %g2
>> 292 sub %o0, 8, %o0
>> 293 sub %o1, 8, %o1
>> 294 st %g2, [%o0]
>> 295 st %g3, [%o0 + 0x04]
>> 296
>> 297 73: /* rmemcpy_last7 */
>> 298
>> 299 be 1f
>> 300 andcc %g1, 2, %g0
>> 301
>> 302 ld [%o1 - 4], %g2
>> 303 sub %o1, 4, %o1
>> 304 st %g2, [%o0 - 4]
>> 305 sub %o0, 4, %o0
>> 306 1:
>> 307 be 1f
>> 308 andcc %g1, 1, %g0
>> 309
>> 310 lduh [%o1 - 2], %g2
>> 311 sub %o1, 2, %o1
>> 312 sth %g2, [%o0 - 2]
>> 313 sub %o0, 2, %o0
>> 314 1:
>> 315 be 1f
>> 316 nop
>> 317
>> 318 ldub [%o1 - 1], %g2
>> 319 stb %g2, [%o0 - 1]
>> 320 1:
>> 321 retl
>> 322 RETL_INSN
>> 323
>> 324 74: /* rldd_std */
>> 325 RMOVE_BIGALIGNCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5)
>> 326 RMOVE_BIGALIGNCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
>> 327 RMOVE_BIGALIGNCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
>> 328 RMOVE_BIGALIGNCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
>> 329 subcc %g7, 128, %g7
>> 330 sub %o1, 128, %o1
>> 331 bne 74b
>> 332 sub %o0, 128, %o0
>> 333
>> 334 andcc %g1, 0x70, %g7
>> 335 be 72b
>> 336 andcc %g1, 8, %g0
>> 337
>> 338 sethi %hi(72b), %o5
>> 339 srl %g7, 1, %o4
>> 340 add %g7, %o4, %o4
>> 341 sub %o1, %g7, %o1
>> 342 sub %o5, %o4, %o5
>> 343 jmpl %o5 + %lo(72b), %g0
>> 344 sub %o0, %g7, %o0
>> 345
>> 346 75: /* rshort_end */
>> 347
>> 348 and %o2, 0xe, %o3
>> 349 2:
>> 350 sethi %hi(76f), %o5
>> 351 sll %o3, 3, %o4
>> 352 sub %o0, %o3, %o0
>> 353 sub %o5, %o4, %o5
>> 354 sub %o1, %o3, %o1
>> 355 jmpl %o5 + %lo(76f), %g0
>> 356 andcc %o2, 1, %g0
>> 357
>> 358 RMOVE_SHORTCHUNK(o1, o0, 0x0c, g2, g3)
>> 359 RMOVE_SHORTCHUNK(o1, o0, 0x0a, g2, g3)
>> 360 RMOVE_SHORTCHUNK(o1, o0, 0x08, g2, g3)
>> 361 RMOVE_SHORTCHUNK(o1, o0, 0x06, g2, g3)
>> 362 RMOVE_SHORTCHUNK(o1, o0, 0x04, g2, g3)
>> 363 RMOVE_SHORTCHUNK(o1, o0, 0x02, g2, g3)
>> 364 RMOVE_SHORTCHUNK(o1, o0, 0x00, g2, g3)
>> 365
>> 366 76: /* rshort_table_end */
>> 367
>> 368 be 1f
>> 369 nop
>> 370 ldub [%o1 - 1], %g2
>> 371 stb %g2, [%o0 - 1]
>> 372 1:
>> 373 retl
>> 374 RETL_INSN
>> 375
>> 376 91: /* rshort_aligned_end */
>> 377
>> 378 bne 75b
>> 379 andcc %o2, 8, %g0
>> 380
>> 381 be 1f
>> 382 andcc %o2, 4, %g0
>> 383
>> 384 ld [%o1 - 0x08], %g2
>> 385 ld [%o1 - 0x04], %g3
>> 386 sub %o1, 8, %o1
>> 387 st %g2, [%o0 - 0x08]
>> 388 st %g3, [%o0 - 0x04]
>> 389 sub %o0, 8, %o0
>> 390 1:
>> 391 b 73b
>> 392 mov %o2, %g1
>> 393
>> 394 77: /* rnon_aligned */
>> 395 cmp %o2, 15
>> 396 bleu 75b
>> 397 andcc %o0, 3, %g0
>> 398 be 64f
>> 399 andcc %o0, 1, %g0
>> 400 be 63f
>> 401 andcc %o0, 2, %g0
>> 402 ldub [%o1 - 1], %g5
>> 403 sub %o1, 1, %o1
>> 404 stb %g5, [%o0 - 1]
>> 405 sub %o0, 1, %o0
>> 406 be 64f
>> 407 sub %o2, 1, %o2
>> 408 63:
>> 409 ldub [%o1 - 1], %g5
>> 410 sub %o1, 2, %o1
>> 411 stb %g5, [%o0 - 1]
>> 412 sub %o0, 2, %o0
>> 413 ldub [%o1], %g5
>> 414 sub %o2, 2, %o2
>> 415 stb %g5, [%o0]
>> 416 64:
>> 417 and %o1, 3, %g2
>> 418 and %o1, -4, %o1
>> 419 and %o2, 0xc, %g3
>> 420 add %o1, 4, %o1
>> 421 cmp %g3, 4
>> 422 sll %g2, 3, %g4
>> 423 mov 32, %g2
>> 424 be 4f
>> 425 sub %g2, %g4, %g7
>> 426
>> 427 blu 3f
>> 428 cmp %g3, 8
>> 429
>> 430 be 2f
>> 431 srl %o2, 2, %g3
>> 432
>> 433 ld [%o1 - 4], %o3
>> 434 add %o0, -8, %o0
>> 435 ld [%o1 - 8], %o4
>> 436 add %o1, -16, %o1
>> 437 b 7f
>> 438 add %g3, 1, %g3
>> 439 2:
>> 440 ld [%o1 - 4], %o4
>> 441 add %o0, -4, %o0
>> 442 ld [%o1 - 8], %g1
>> 443 add %o1, -12, %o1
>> 444 b 8f
>> 445 add %g3, 2, %g3
>> 446 3:
>> 447 ld [%o1 - 4], %o5
>> 448 add %o0, -12, %o0
>> 449 ld [%o1 - 8], %o3
>> 450 add %o1, -20, %o1
>> 451 b 6f
>> 452 srl %o2, 2, %g3
>> 453 4:
>> 454 ld [%o1 - 4], %g1
>> 455 srl %o2, 2, %g3
>> 456 ld [%o1 - 8], %o5
>> 457 add %o1, -24, %o1
>> 458 add %o0, -16, %o0
>> 459 add %g3, -1, %g3
>> 460
>> 461 ld [%o1 + 12], %o3
>> 462 5:
>> 463 sll %o5, %g4, %g2
>> 464 srl %g1, %g7, %g5
>> 465 or %g2, %g5, %g2
>> 466 st %g2, [%o0 + 12]
>> 467 6:
>> 468 ld [%o1 + 8], %o4
>> 469 sll %o3, %g4, %g2
>> 470 srl %o5, %g7, %g5
>> 471 or %g2, %g5, %g2
>> 472 st %g2, [%o0 + 8]
>> 473 7:
>> 474 ld [%o1 + 4], %g1
>> 475 sll %o4, %g4, %g2
>> 476 srl %o3, %g7, %g5
>> 477 or %g2, %g5, %g2
>> 478 st %g2, [%o0 + 4]
>> 479 8:
>> 480 ld [%o1], %o5
>> 481 sll %g1, %g4, %g2
>> 482 srl %o4, %g7, %g5
>> 483 addcc %g3, -4, %g3
>> 484 or %g2, %g5, %g2
>> 485 add %o1, -16, %o1
>> 486 st %g2, [%o0]
>> 487 add %o0, -16, %o0
>> 488 bne,a 5b
>> 489 ld [%o1 + 12], %o3
>> 490 sll %o5, %g4, %g2
>> 491 srl %g1, %g7, %g5
>> 492 srl %g4, 3, %g3
>> 493 or %g2, %g5, %g2
>> 494 add %o1, %g3, %o1
>> 495 andcc %o2, 2, %g0
>> 496 st %g2, [%o0 + 12]
>> 497 be 1f
>> 498 andcc %o2, 1, %g0
>> 499
>> 500 ldub [%o1 + 15], %g5
>> 501 add %o1, -2, %o1
>> 502 stb %g5, [%o0 + 11]
>> 503 add %o0, -2, %o0
>> 504 ldub [%o1 + 16], %g5
>> 505 stb %g5, [%o0 + 12]
>> 506 1:
>> 507 be 1f
>> 508 nop
>> 509 ldub [%o1 + 15], %g5
>> 510 stb %g5, [%o0 + 11]
>> 511 1:
>> 512 retl
>> 513 RETL_INSN
>> 514
>> 515 #endif /* FASTER_REVERSE */
95 516
96 /* NOTE: This code is executed just for the ca 517 /* NOTE: This code is executed just for the cases,
97 where %src (=%o1) & 3 is != 0. 518 where %src (=%o1) & 3 is != 0.
98 We need to align it to 4. So, for (%s 519 We need to align it to 4. So, for (%src & 3)
99 1 we need to do ldub,lduh 520 1 we need to do ldub,lduh
100 2 lduh 521 2 lduh
101 3 just ldub 522 3 just ldub
102 so even if it looks weird, the branch 523 so even if it looks weird, the branches
103 are correct here. -jj 524 are correct here. -jj
104 */ 525 */
105 78: /* dword_align */ 526 78: /* dword_align */
106 527
107 andcc %o1, 1, %g0 528 andcc %o1, 1, %g0
108 be 4f 529 be 4f
109 andcc %o1, 2, %g0 530 andcc %o1, 2, %g0
110 531
111 ldub [%o1], %g2 532 ldub [%o1], %g2
112 add %o1, 1, %o1 533 add %o1, 1, %o1
113 stb %g2, [%o0] 534 stb %g2, [%o0]
114 sub %o2, 1, %o2 535 sub %o2, 1, %o2
115 bne 3f 536 bne 3f
116 add %o0, 1, %o0 537 add %o0, 1, %o0
117 4: 538 4:
118 lduh [%o1], %g2 539 lduh [%o1], %g2
119 add %o1, 2, %o1 540 add %o1, 2, %o1
120 sth %g2, [%o0] 541 sth %g2, [%o0]
121 sub %o2, 2, %o2 542 sub %o2, 2, %o2
122 b 3f 543 b 3f
123 add %o0, 2, %o0 544 add %o0, 2, %o0
124 545
>> 546 #ifdef __KERNEL__
>> 547 FUNC(__memcpy)
>> 548 #endif
125 FUNC(memcpy) /* %o0=dst %o1=src %o2=len */ 549 FUNC(memcpy) /* %o0=dst %o1=src %o2=len */
126 EXPORT_SYMBOL(memcpy) <<
127 550
128 sub %o0, %o1, %o4 551 sub %o0, %o1, %o4
129 mov %o0, %g7 !! 552 SETUP_RETL
130 9: 553 9:
131 andcc %o4, 3, %o5 554 andcc %o4, 3, %o5
132 0: 555 0:
133 bne 86f 556 bne 86f
134 cmp %o2, 15 557 cmp %o2, 15
135 558
136 bleu 90f 559 bleu 90f
137 andcc %o1, 3, %g0 560 andcc %o1, 3, %g0
138 561
139 bne 78b 562 bne 78b
140 3: 563 3:
141 andcc %o1, 4, %g0 564 andcc %o1, 4, %g0
142 565
143 be 2f 566 be 2f
144 mov %o2, %g1 567 mov %o2, %g1
145 568
146 ld [%o1], %o4 569 ld [%o1], %o4
147 sub %g1, 4, %g1 570 sub %g1, 4, %g1
148 st %o4, [%o0] 571 st %o4, [%o0]
149 add %o1, 4, %o1 572 add %o1, 4, %o1
150 add %o0, 4, %o0 573 add %o0, 4, %o0
151 2: 574 2:
152 andcc %g1, 0xffffff80, %g0 !! 575 andcc %g1, 0xffffff80, %g7
153 be 3f 576 be 3f
154 andcc %o0, 4, %g0 577 andcc %o0, 4, %g0
155 578
156 be 82f + 4 579 be 82f + 4
157 5: 580 5:
158 MOVE_BIGCHUNK(o1, o0, 0x00, o2, o3, o4 581 MOVE_BIGCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5)
159 MOVE_BIGCHUNK(o1, o0, 0x20, o2, o3, o4 582 MOVE_BIGCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
160 MOVE_BIGCHUNK(o1, o0, 0x40, o2, o3, o4 583 MOVE_BIGCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
161 MOVE_BIGCHUNK(o1, o0, 0x60, o2, o3, o4 584 MOVE_BIGCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
162 sub %g1, 128, %g1 !! 585 subcc %g7, 128, %g7
163 add %o1, 128, %o1 586 add %o1, 128, %o1
164 cmp %g1, 128 !! 587 bne 5b
165 bge 5b <<
166 add %o0, 128, %o0 588 add %o0, 128, %o0
167 3: 589 3:
168 andcc %g1, 0x70, %g4 !! 590 andcc %g1, 0x70, %g7
169 be 80f 591 be 80f
170 andcc %g1, 8, %g0 592 andcc %g1, 8, %g0
171 593
172 sethi %hi(80f), %o5 594 sethi %hi(80f), %o5
173 srl %g4, 1, %o4 !! 595 srl %g7, 1, %o4
174 add %g4, %o4, %o4 !! 596 add %g7, %o4, %o4
175 add %o1, %g4, %o1 !! 597 add %o1, %g7, %o1
176 sub %o5, %o4, %o5 598 sub %o5, %o4, %o5
177 jmpl %o5 + %lo(80f), %g0 599 jmpl %o5 + %lo(80f), %g0
178 add %o0, %g4, %o0 !! 600 add %o0, %g7, %o0
179 601
180 79: /* memcpy_table */ 602 79: /* memcpy_table */
181 603
182 MOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g 604 MOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g4, g5)
183 MOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g 605 MOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g4, g5)
184 MOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g 606 MOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g4, g5)
185 MOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g 607 MOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g4, g5)
186 MOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g 608 MOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g4, g5)
187 MOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g 609 MOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g4, g5)
188 MOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g 610 MOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g4, g5)
189 611
190 80: /* memcpy_table_end */ 612 80: /* memcpy_table_end */
191 be 81f 613 be 81f
192 andcc %g1, 4, %g0 614 andcc %g1, 4, %g0
193 615
194 ldd [%o1], %g2 616 ldd [%o1], %g2
195 add %o0, 8, %o0 617 add %o0, 8, %o0
196 st %g2, [%o0 - 0x08] 618 st %g2, [%o0 - 0x08]
197 add %o1, 8, %o1 619 add %o1, 8, %o1
198 st %g3, [%o0 - 0x04] 620 st %g3, [%o0 - 0x04]
199 621
200 81: /* memcpy_last7 */ 622 81: /* memcpy_last7 */
201 623
202 be 1f 624 be 1f
203 andcc %g1, 2, %g0 625 andcc %g1, 2, %g0
204 626
205 ld [%o1], %g2 627 ld [%o1], %g2
206 add %o1, 4, %o1 628 add %o1, 4, %o1
207 st %g2, [%o0] 629 st %g2, [%o0]
208 add %o0, 4, %o0 630 add %o0, 4, %o0
209 1: 631 1:
210 be 1f 632 be 1f
211 andcc %g1, 1, %g0 633 andcc %g1, 1, %g0
212 634
213 lduh [%o1], %g2 635 lduh [%o1], %g2
214 add %o1, 2, %o1 636 add %o1, 2, %o1
215 sth %g2, [%o0] 637 sth %g2, [%o0]
216 add %o0, 2, %o0 638 add %o0, 2, %o0
217 1: 639 1:
218 be 1f 640 be 1f
219 nop 641 nop
220 642
221 ldub [%o1], %g2 643 ldub [%o1], %g2
222 stb %g2, [%o0] 644 stb %g2, [%o0]
223 1: 645 1:
224 retl 646 retl
225 mov %g7, %o0 !! 647 RETL_INSN
226 648
227 82: /* ldd_std */ 649 82: /* ldd_std */
228 MOVE_BIGALIGNCHUNK(o1, o0, 0x00, o2, o 650 MOVE_BIGALIGNCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5)
229 MOVE_BIGALIGNCHUNK(o1, o0, 0x20, o2, o 651 MOVE_BIGALIGNCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
230 MOVE_BIGALIGNCHUNK(o1, o0, 0x40, o2, o 652 MOVE_BIGALIGNCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
231 MOVE_BIGALIGNCHUNK(o1, o0, 0x60, o2, o 653 MOVE_BIGALIGNCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
232 subcc %g1, 128, %g1 !! 654 subcc %g7, 128, %g7
233 add %o1, 128, %o1 655 add %o1, 128, %o1
234 cmp %g1, 128 !! 656 bne 82b
235 bge 82b <<
236 add %o0, 128, %o0 657 add %o0, 128, %o0
237 658
238 andcc %g1, 0x70, %g4 !! 659 #ifndef FASTER_ALIGNED
>> 660
>> 661 andcc %g1, 0x70, %g7
>> 662 be 80b
>> 663 andcc %g1, 8, %g0
>> 664
>> 665 sethi %hi(80b), %o5
>> 666 srl %g7, 1, %o4
>> 667 add %g7, %o4, %o4
>> 668 add %o1, %g7, %o1
>> 669 sub %o5, %o4, %o5
>> 670 jmpl %o5 + %lo(80b), %g0
>> 671 add %o0, %g7, %o0
>> 672
>> 673 #else /* FASTER_ALIGNED */
>> 674
>> 675 andcc %g1, 0x70, %g7
239 be 84f 676 be 84f
240 andcc %g1, 8, %g0 677 andcc %g1, 8, %g0
241 678
242 sethi %hi(84f), %o5 679 sethi %hi(84f), %o5
243 add %o1, %g4, %o1 !! 680 add %o1, %g7, %o1
244 sub %o5, %g4, %o5 !! 681 sub %o5, %g7, %o5
245 jmpl %o5 + %lo(84f), %g0 682 jmpl %o5 + %lo(84f), %g0
246 add %o0, %g4, %o0 !! 683 add %o0, %g7, %o0
247 684
248 83: /* amemcpy_table */ 685 83: /* amemcpy_table */
249 686
250 MOVE_LASTALIGNCHUNK(o1, o0, 0x60, g2, 687 MOVE_LASTALIGNCHUNK(o1, o0, 0x60, g2, g3, g4, g5)
251 MOVE_LASTALIGNCHUNK(o1, o0, 0x50, g2, 688 MOVE_LASTALIGNCHUNK(o1, o0, 0x50, g2, g3, g4, g5)
252 MOVE_LASTALIGNCHUNK(o1, o0, 0x40, g2, 689 MOVE_LASTALIGNCHUNK(o1, o0, 0x40, g2, g3, g4, g5)
253 MOVE_LASTALIGNCHUNK(o1, o0, 0x30, g2, 690 MOVE_LASTALIGNCHUNK(o1, o0, 0x30, g2, g3, g4, g5)
254 MOVE_LASTALIGNCHUNK(o1, o0, 0x20, g2, 691 MOVE_LASTALIGNCHUNK(o1, o0, 0x20, g2, g3, g4, g5)
255 MOVE_LASTALIGNCHUNK(o1, o0, 0x10, g2, 692 MOVE_LASTALIGNCHUNK(o1, o0, 0x10, g2, g3, g4, g5)
256 MOVE_LASTALIGNCHUNK(o1, o0, 0x00, g2, 693 MOVE_LASTALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5)
257 694
258 84: /* amemcpy_table_end */ 695 84: /* amemcpy_table_end */
259 be 85f 696 be 85f
260 andcc %g1, 4, %g0 697 andcc %g1, 4, %g0
261 698
262 ldd [%o1], %g2 699 ldd [%o1], %g2
263 add %o0, 8, %o0 700 add %o0, 8, %o0
264 std %g2, [%o0 - 0x08] 701 std %g2, [%o0 - 0x08]
265 add %o1, 8, %o1 702 add %o1, 8, %o1
266 85: /* amemcpy_last7 */ 703 85: /* amemcpy_last7 */
267 be 1f 704 be 1f
268 andcc %g1, 2, %g0 705 andcc %g1, 2, %g0
269 706
270 ld [%o1], %g2 707 ld [%o1], %g2
271 add %o1, 4, %o1 708 add %o1, 4, %o1
272 st %g2, [%o0] 709 st %g2, [%o0]
273 add %o0, 4, %o0 710 add %o0, 4, %o0
274 1: 711 1:
275 be 1f 712 be 1f
276 andcc %g1, 1, %g0 713 andcc %g1, 1, %g0
277 714
278 lduh [%o1], %g2 715 lduh [%o1], %g2
279 add %o1, 2, %o1 716 add %o1, 2, %o1
280 sth %g2, [%o0] 717 sth %g2, [%o0]
281 add %o0, 2, %o0 718 add %o0, 2, %o0
282 1: 719 1:
283 be 1f 720 be 1f
284 nop 721 nop
285 722
286 ldub [%o1], %g2 723 ldub [%o1], %g2
287 stb %g2, [%o0] 724 stb %g2, [%o0]
288 1: 725 1:
289 retl 726 retl
290 mov %g7, %o0 !! 727 RETL_INSN
>> 728
>> 729 #endif /* FASTER_ALIGNED */
291 730
292 86: /* non_aligned */ 731 86: /* non_aligned */
293 cmp %o2, 6 732 cmp %o2, 6
294 bleu 88f 733 bleu 88f
295 nop <<
296 734
297 save %sp, -96, %sp !! 735 #ifdef FASTER_NONALIGNED
298 andcc %i0, 3, %g0 !! 736
>> 737 cmp %o2, 256
>> 738 bcc 87f
>> 739
>> 740 #endif /* FASTER_NONALIGNED */
>> 741
>> 742 andcc %o0, 3, %g0
299 be 61f 743 be 61f
300 andcc %i0, 1, %g0 !! 744 andcc %o0, 1, %g0
301 be 60f 745 be 60f
302 andcc %i0, 2, %g0 !! 746 andcc %o0, 2, %g0
303 747
304 ldub [%i1], %g5 !! 748 ldub [%o1], %g5
305 add %i1, 1, %i1 !! 749 add %o1, 1, %o1
306 stb %g5, [%i0] !! 750 stb %g5, [%o0]
307 sub %i2, 1, %i2 !! 751 sub %o2, 1, %o2
308 bne 61f 752 bne 61f
309 add %i0, 1, %i0 !! 753 add %o0, 1, %o0
310 60: 754 60:
311 ldub [%i1], %g3 !! 755 ldub [%o1], %g3
312 add %i1, 2, %i1 !! 756 add %o1, 2, %o1
313 stb %g3, [%i0] !! 757 stb %g3, [%o0]
314 sub %i2, 2, %i2 !! 758 sub %o2, 2, %o2
315 ldub [%i1 - 1], %g3 !! 759 ldub [%o1 - 1], %g3
316 add %i0, 2, %i0 !! 760 add %o0, 2, %o0
317 stb %g3, [%i0 - 1] !! 761 stb %g3, [%o0 - 1]
318 61: 762 61:
319 and %i1, 3, %g2 !! 763 and %o1, 3, %g2
320 and %i2, 0xc, %g3 !! 764 and %o2, 0xc, %g3
321 and %i1, -4, %i1 !! 765 and %o1, -4, %o1
322 cmp %g3, 4 766 cmp %g3, 4
323 sll %g2, 3, %g4 767 sll %g2, 3, %g4
324 mov 32, %g2 768 mov 32, %g2
325 be 4f 769 be 4f
326 sub %g2, %g4, %l0 !! 770 sub %g2, %g4, %g7
327 771
328 blu 3f 772 blu 3f
329 cmp %g3, 0x8 773 cmp %g3, 0x8
330 774
331 be 2f 775 be 2f
332 srl %i2, 2, %g3 !! 776 srl %o2, 2, %g3
333 777
334 ld [%i1], %i3 !! 778 ld [%o1], %o3
335 add %i0, -8, %i0 !! 779 add %o0, -8, %o0
336 ld [%i1 + 4], %i4 !! 780 ld [%o1 + 4], %o4
337 b 8f 781 b 8f
338 add %g3, 1, %g3 782 add %g3, 1, %g3
339 2: 783 2:
340 ld [%i1], %i4 !! 784 ld [%o1], %o4
341 add %i0, -12, %i0 !! 785 add %o0, -12, %o0
342 ld [%i1 + 4], %i5 !! 786 ld [%o1 + 4], %o5
343 add %g3, 2, %g3 787 add %g3, 2, %g3
344 b 9f 788 b 9f
345 add %i1, -4, %i1 !! 789 add %o1, -4, %o1
346 3: 790 3:
347 ld [%i1], %g1 !! 791 ld [%o1], %g1
348 add %i0, -4, %i0 !! 792 add %o0, -4, %o0
349 ld [%i1 + 4], %i3 !! 793 ld [%o1 + 4], %o3
350 srl %i2, 2, %g3 !! 794 srl %o2, 2, %g3
351 b 7f 795 b 7f
352 add %i1, 4, %i1 !! 796 add %o1, 4, %o1
353 4: 797 4:
354 ld [%i1], %i5 !! 798 ld [%o1], %o5
355 cmp %i2, 7 !! 799 cmp %o2, 7
356 ld [%i1 + 4], %g1 !! 800 ld [%o1 + 4], %g1
357 srl %i2, 2, %g3 !! 801 srl %o2, 2, %g3
358 bleu 10f 802 bleu 10f
359 add %i1, 8, %i1 !! 803 add %o1, 8, %o1
360 804
361 ld [%i1], %i3 !! 805 ld [%o1], %o3
362 add %g3, -1, %g3 806 add %g3, -1, %g3
363 5: 807 5:
364 sll %i5, %g4, %g2 !! 808 sll %o5, %g4, %g2
365 srl %g1, %l0, %g5 !! 809 srl %g1, %g7, %g5
366 or %g2, %g5, %g2 810 or %g2, %g5, %g2
367 st %g2, [%i0] !! 811 st %g2, [%o0]
368 7: 812 7:
369 ld [%i1 + 4], %i4 !! 813 ld [%o1 + 4], %o4
370 sll %g1, %g4, %g2 814 sll %g1, %g4, %g2
371 srl %i3, %l0, %g5 !! 815 srl %o3, %g7, %g5
372 or %g2, %g5, %g2 816 or %g2, %g5, %g2
373 st %g2, [%i0 + 4] !! 817 st %g2, [%o0 + 4]
374 8: 818 8:
375 ld [%i1 + 8], %i5 !! 819 ld [%o1 + 8], %o5
376 sll %i3, %g4, %g2 !! 820 sll %o3, %g4, %g2
377 srl %i4, %l0, %g5 !! 821 srl %o4, %g7, %g5
378 or %g2, %g5, %g2 822 or %g2, %g5, %g2
379 st %g2, [%i0 + 8] !! 823 st %g2, [%o0 + 8]
380 9: 824 9:
381 ld [%i1 + 12], %g1 !! 825 ld [%o1 + 12], %g1
382 sll %i4, %g4, %g2 !! 826 sll %o4, %g4, %g2
383 srl %i5, %l0, %g5 !! 827 srl %o5, %g7, %g5
384 addcc %g3, -4, %g3 828 addcc %g3, -4, %g3
385 or %g2, %g5, %g2 829 or %g2, %g5, %g2
386 add %i1, 16, %i1 !! 830 add %o1, 16, %o1
387 st %g2, [%i0 + 12] !! 831 st %g2, [%o0 + 12]
388 add %i0, 16, %i0 !! 832 add %o0, 16, %o0
389 bne,a 5b 833 bne,a 5b
390 ld [%i1], %i3 !! 834 ld [%o1], %o3
391 10: 835 10:
392 sll %i5, %g4, %g2 !! 836 sll %o5, %g4, %g2
393 srl %g1, %l0, %g5 !! 837 srl %g1, %g7, %g5
394 srl %l0, 3, %g3 !! 838 srl %g7, 3, %g3
395 or %g2, %g5, %g2 839 or %g2, %g5, %g2
396 sub %i1, %g3, %i1 !! 840 sub %o1, %g3, %o1
397 andcc %i2, 2, %g0 !! 841 andcc %o2, 2, %g0
398 st %g2, [%i0] !! 842 st %g2, [%o0]
399 be 1f !! 843 be 1f
400 andcc %i2, 1, %g0 !! 844 andcc %o2, 1, %g0
401 !! 845
402 ldub [%i1], %g2 !! 846 ldub [%o1], %g2
403 add %i1, 2, %i1 !! 847 add %o1, 2, %o1
404 stb %g2, [%i0 + 4] !! 848 stb %g2, [%o0 + 4]
405 add %i0, 2, %i0 !! 849 add %o0, 2, %o0
406 ldub [%i1 - 1], %g2 !! 850 ldub [%o1 - 1], %g2
407 stb %g2, [%i0 + 3] !! 851 stb %g2, [%o0 + 3]
408 1: 852 1:
409 be 1f 853 be 1f
410 nop 854 nop
411 ldub [%i1], %g2 !! 855 ldub [%o1], %g2
412 stb %g2, [%i0 + 4] !! 856 stb %g2, [%o0 + 4]
413 1: 857 1:
414 ret !! 858 retl
415 restore %g7, %g0, %o0 !! 859 RETL_INSN
>> 860
>> 861 #ifdef FASTER_NONALIGNED
>> 862
>> 863 87: /* faster_nonaligned */
>> 864
>> 865 andcc %o1, 3, %g0
>> 866 be 3f
>> 867 andcc %o1, 1, %g0
>> 868
>> 869 be 4f
>> 870 andcc %o1, 2, %g0
>> 871
>> 872 ldub [%o1], %g2
>> 873 add %o1, 1, %o1
>> 874 stb %g2, [%o0]
>> 875 sub %o2, 1, %o2
>> 876 bne 3f
>> 877 add %o0, 1, %o0
>> 878 4:
>> 879 lduh [%o1], %g2
>> 880 add %o1, 2, %o1
>> 881 srl %g2, 8, %g3
>> 882 sub %o2, 2, %o2
>> 883 stb %g3, [%o0]
>> 884 add %o0, 2, %o0
>> 885 stb %g2, [%o0 - 1]
>> 886 3:
>> 887 andcc %o1, 4, %g0
>> 888
>> 889 bne 2f
>> 890 cmp %o5, 1
>> 891
>> 892 ld [%o1], %o4
>> 893 srl %o4, 24, %g2
>> 894 stb %g2, [%o0]
>> 895 srl %o4, 16, %g3
>> 896 stb %g3, [%o0 + 1]
>> 897 srl %o4, 8, %g2
>> 898 stb %g2, [%o0 + 2]
>> 899 sub %o2, 4, %o2
>> 900 stb %o4, [%o0 + 3]
>> 901 add %o1, 4, %o1
>> 902 add %o0, 4, %o0
>> 903 2:
>> 904 be 33f
>> 905 cmp %o5, 2
>> 906 be 32f
>> 907 sub %o2, 4, %o2
>> 908 31:
>> 909 ld [%o1], %g2
>> 910 add %o1, 4, %o1
>> 911 srl %g2, 24, %g3
>> 912 and %o0, 7, %g5
>> 913 stb %g3, [%o0]
>> 914 cmp %g5, 7
>> 915 sll %g2, 8, %g1
>> 916 add %o0, 4, %o0
>> 917 be 41f
>> 918 and %o2, 0xffffffc0, %o3
>> 919 ld [%o0 - 7], %o4
>> 920 4:
>> 921 SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
>> 922 SMOVE_CHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
>> 923 SMOVE_CHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
>> 924 SMOVE_CHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
>> 925 subcc %o3, 64, %o3
>> 926 add %o1, 64, %o1
>> 927 bne 4b
>> 928 add %o0, 64, %o0
>> 929
>> 930 andcc %o2, 0x30, %o3
>> 931 be,a 1f
>> 932 srl %g1, 16, %g2
>> 933 4:
>> 934 SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
>> 935 subcc %o3, 16, %o3
>> 936 add %o1, 16, %o1
>> 937 bne 4b
>> 938 add %o0, 16, %o0
>> 939
>> 940 srl %g1, 16, %g2
>> 941 1:
>> 942 st %o4, [%o0 - 7]
>> 943 sth %g2, [%o0 - 3]
>> 944 srl %g1, 8, %g4
>> 945 b 88f
>> 946 stb %g4, [%o0 - 1]
>> 947 32:
>> 948 ld [%o1], %g2
>> 949 add %o1, 4, %o1
>> 950 srl %g2, 16, %g3
>> 951 and %o0, 7, %g5
>> 952 sth %g3, [%o0]
>> 953 cmp %g5, 6
>> 954 sll %g2, 16, %g1
>> 955 add %o0, 4, %o0
>> 956 be 42f
>> 957 and %o2, 0xffffffc0, %o3
>> 958 ld [%o0 - 6], %o4
>> 959 4:
>> 960 SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
>> 961 SMOVE_CHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
>> 962 SMOVE_CHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
>> 963 SMOVE_CHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
>> 964 subcc %o3, 64, %o3
>> 965 add %o1, 64, %o1
>> 966 bne 4b
>> 967 add %o0, 64, %o0
>> 968
>> 969 andcc %o2, 0x30, %o3
>> 970 be,a 1f
>> 971 srl %g1, 16, %g2
>> 972 4:
>> 973 SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
>> 974 subcc %o3, 16, %o3
>> 975 add %o1, 16, %o1
>> 976 bne 4b
>> 977 add %o0, 16, %o0
>> 978
>> 979 srl %g1, 16, %g2
>> 980 1:
>> 981 st %o4, [%o0 - 6]
>> 982 b 88f
>> 983 sth %g2, [%o0 - 2]
>> 984 33:
>> 985 ld [%o1], %g2
>> 986 sub %o2, 4, %o2
>> 987 srl %g2, 24, %g3
>> 988 and %o0, 7, %g5
>> 989 stb %g3, [%o0]
>> 990 cmp %g5, 5
>> 991 srl %g2, 8, %g4
>> 992 sll %g2, 24, %g1
>> 993 sth %g4, [%o0 + 1]
>> 994 add %o1, 4, %o1
>> 995 be 43f
>> 996 and %o2, 0xffffffc0, %o3
>> 997
>> 998 ld [%o0 - 1], %o4
>> 999 add %o0, 4, %o0
>> 1000 4:
>> 1001 SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, -1)
>> 1002 SMOVE_CHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, -1)
>> 1003 SMOVE_CHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, -1)
>> 1004 SMOVE_CHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, -1)
>> 1005 subcc %o3, 64, %o3
>> 1006 add %o1, 64, %o1
>> 1007 bne 4b
>> 1008 add %o0, 64, %o0
>> 1009
>> 1010 andcc %o2, 0x30, %o3
>> 1011 be,a 1f
>> 1012 srl %g1, 24, %g2
>> 1013 4:
>> 1014 SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, -1)
>> 1015 subcc %o3, 16, %o3
>> 1016 add %o1, 16, %o1
>> 1017 bne 4b
>> 1018 add %o0, 16, %o0
>> 1019
>> 1020 srl %g1, 24, %g2
>> 1021 1:
>> 1022 st %o4, [%o0 - 5]
>> 1023 b 88f
>> 1024 stb %g2, [%o0 - 1]
>> 1025 41:
>> 1026 SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
>> 1027 SMOVE_ALIGNCHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
>> 1028 SMOVE_ALIGNCHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
>> 1029 SMOVE_ALIGNCHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
>> 1030 subcc %o3, 64, %o3
>> 1031 add %o1, 64, %o1
>> 1032 bne 41b
>> 1033 add %o0, 64, %o0
>> 1034
>> 1035 andcc %o2, 0x30, %o3
>> 1036 be,a 1f
>> 1037 srl %g1, 16, %g2
>> 1038 4:
>> 1039 SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
>> 1040 subcc %o3, 16, %o3
>> 1041 add %o1, 16, %o1
>> 1042 bne 4b
>> 1043 add %o0, 16, %o0
>> 1044
>> 1045 srl %g1, 16, %g2
>> 1046 1:
>> 1047 sth %g2, [%o0 - 3]
>> 1048 srl %g1, 8, %g4
>> 1049 b 88f
>> 1050 stb %g4, [%o0 - 1]
>> 1051 43:
>> 1052 SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, 3)
>> 1053 SMOVE_ALIGNCHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, 3)
>> 1054 SMOVE_ALIGNCHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, 3)
>> 1055 SMOVE_ALIGNCHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, 3)
>> 1056 subcc %o3, 64, %o3
>> 1057 add %o1, 64, %o1
>> 1058 bne 43b
>> 1059 add %o0, 64, %o0
>> 1060
>> 1061 andcc %o2, 0x30, %o3
>> 1062 be,a 1f
>> 1063 srl %g1, 24, %g2
>> 1064 4:
>> 1065 SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, 3)
>> 1066 subcc %o3, 16, %o3
>> 1067 add %o1, 16, %o1
>> 1068 bne 4b
>> 1069 add %o0, 16, %o0
>> 1070
>> 1071 srl %g1, 24, %g2
>> 1072 1:
>> 1073 stb %g2, [%o0 + 3]
>> 1074 b 88f
>> 1075 add %o0, 4, %o0
>> 1076 42:
>> 1077 SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
>> 1078 SMOVE_ALIGNCHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
>> 1079 SMOVE_ALIGNCHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
>> 1080 SMOVE_ALIGNCHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
>> 1081 subcc %o3, 64, %o3
>> 1082 add %o1, 64, %o1
>> 1083 bne 42b
>> 1084 add %o0, 64, %o0
>> 1085
>> 1086 andcc %o2, 0x30, %o3
>> 1087 be,a 1f
>> 1088 srl %g1, 16, %g2
>> 1089 4:
>> 1090 SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
>> 1091 subcc %o3, 16, %o3
>> 1092 add %o1, 16, %o1
>> 1093 bne 4b
>> 1094 add %o0, 16, %o0
>> 1095
>> 1096 srl %g1, 16, %g2
>> 1097 1:
>> 1098 sth %g2, [%o0 - 2]
>> 1099
>> 1100 /* Fall through */
>> 1101
>> 1102 #endif /* FASTER_NONALIGNED */
416 1103
417 88: /* short_end */ 1104 88: /* short_end */
418 1105
419 and %o2, 0xe, %o3 1106 and %o2, 0xe, %o3
420 20: 1107 20:
421 sethi %hi(89f), %o5 1108 sethi %hi(89f), %o5
422 sll %o3, 3, %o4 1109 sll %o3, 3, %o4
423 add %o0, %o3, %o0 1110 add %o0, %o3, %o0
424 sub %o5, %o4, %o5 1111 sub %o5, %o4, %o5
425 add %o1, %o3, %o1 1112 add %o1, %o3, %o1
426 jmpl %o5 + %lo(89f), %g0 1113 jmpl %o5 + %lo(89f), %g0
427 andcc %o2, 1, %g0 1114 andcc %o2, 1, %g0
428 1115
429 MOVE_SHORTCHUNK(o1, o0, 0x0c, g2, g3) 1116 MOVE_SHORTCHUNK(o1, o0, 0x0c, g2, g3)
430 MOVE_SHORTCHUNK(o1, o0, 0x0a, g2, g3) 1117 MOVE_SHORTCHUNK(o1, o0, 0x0a, g2, g3)
431 MOVE_SHORTCHUNK(o1, o0, 0x08, g2, g3) 1118 MOVE_SHORTCHUNK(o1, o0, 0x08, g2, g3)
432 MOVE_SHORTCHUNK(o1, o0, 0x06, g2, g3) 1119 MOVE_SHORTCHUNK(o1, o0, 0x06, g2, g3)
433 MOVE_SHORTCHUNK(o1, o0, 0x04, g2, g3) 1120 MOVE_SHORTCHUNK(o1, o0, 0x04, g2, g3)
434 MOVE_SHORTCHUNK(o1, o0, 0x02, g2, g3) 1121 MOVE_SHORTCHUNK(o1, o0, 0x02, g2, g3)
435 MOVE_SHORTCHUNK(o1, o0, 0x00, g2, g3) 1122 MOVE_SHORTCHUNK(o1, o0, 0x00, g2, g3)
436 1123
437 89: /* short_table_end */ 1124 89: /* short_table_end */
438 1125
439 be 1f 1126 be 1f
440 nop 1127 nop
441 1128
442 ldub [%o1], %g2 1129 ldub [%o1], %g2
443 stb %g2, [%o0] 1130 stb %g2, [%o0]
444 1: 1131 1:
445 retl 1132 retl
446 mov %g7, %o0 !! 1133 RETL_INSN
447 1134
448 90: /* short_aligned_end */ 1135 90: /* short_aligned_end */
449 bne 88b 1136 bne 88b
450 andcc %o2, 8, %g0 1137 andcc %o2, 8, %g0
451 1138
452 be 1f 1139 be 1f
453 andcc %o2, 4, %g0 1140 andcc %o2, 4, %g0
454 1141
455 ld [%o1 + 0x00], %g2 1142 ld [%o1 + 0x00], %g2
456 ld [%o1 + 0x04], %g3 1143 ld [%o1 + 0x04], %g3
457 add %o1, 8, %o1 1144 add %o1, 8, %o1
458 st %g2, [%o0 + 0x00] 1145 st %g2, [%o0 + 0x00]
459 st %g3, [%o0 + 0x04] 1146 st %g3, [%o0 + 0x04]
460 add %o0, 8, %o0 1147 add %o0, 8, %o0
461 1: 1148 1:
462 b 81b 1149 b 81b
463 mov %o2, %g1 1150 mov %o2, %g1
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.