1 /* SPDX-License-Identifier: GPL-2.0 */ <<
2 /* U1memcpy.S: UltraSPARC-I/II/IIi/IIe optimiz 1 /* U1memcpy.S: UltraSPARC-I/II/IIi/IIe optimized memcpy.
3 * 2 *
4 * Copyright (C) 1997, 2004 David S. Miller (d 3 * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com)
5 * Copyright (C) 1996, 1997, 1998, 1999 Jakub 4 * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz)
6 */ 5 */
7 6
8 #ifdef __KERNEL__ 7 #ifdef __KERNEL__
9 #include <linux/export.h> <<
10 #include <linux/linkage.h> 8 #include <linux/linkage.h>
11 #include <asm/visasm.h> 9 #include <asm/visasm.h>
12 #include <asm/asi.h> 10 #include <asm/asi.h>
>> 11 #include <asm/export.h>
13 #define GLOBAL_SPARE g7 12 #define GLOBAL_SPARE g7
14 #else 13 #else
15 #define GLOBAL_SPARE g5 14 #define GLOBAL_SPARE g5
16 #define ASI_BLK_P 0xf0 15 #define ASI_BLK_P 0xf0
17 #define FPRS_FEF 0x04 16 #define FPRS_FEF 0x04
18 #ifdef MEMCPY_DEBUG 17 #ifdef MEMCPY_DEBUG
19 #define VISEntry rd %fprs, %o5; wr %g0, FPRS_F 18 #define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \
20 clr %g1; clr %g2; clr %g3; su 19 clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0;
21 #define VISExit and %o5, FPRS_FEF, %o5; wr %o5 20 #define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
22 #else 21 #else
23 #define VISEntry rd %fprs, %o5; wr %g0, FPRS_F 22 #define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
24 #define VISExit and %o5, FPRS_FEF, %o5; wr %o5 23 #define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
25 #endif 24 #endif
26 #endif 25 #endif
27 26
28 #ifndef EX_LD 27 #ifndef EX_LD
29 #define EX_LD(x,y) x 28 #define EX_LD(x,y) x
30 #endif 29 #endif
31 #ifndef EX_LD_FP 30 #ifndef EX_LD_FP
32 #define EX_LD_FP(x,y) x 31 #define EX_LD_FP(x,y) x
33 #endif 32 #endif
34 33
35 #ifndef EX_ST 34 #ifndef EX_ST
36 #define EX_ST(x,y) x 35 #define EX_ST(x,y) x
37 #endif 36 #endif
38 #ifndef EX_ST_FP 37 #ifndef EX_ST_FP
39 #define EX_ST_FP(x,y) x 38 #define EX_ST_FP(x,y) x
40 #endif 39 #endif
41 40
42 #ifndef LOAD 41 #ifndef LOAD
43 #define LOAD(type,addr,dest) type [addr], d 42 #define LOAD(type,addr,dest) type [addr], dest
44 #endif 43 #endif
45 44
46 #ifndef LOAD_BLK 45 #ifndef LOAD_BLK
47 #define LOAD_BLK(addr,dest) ldda [addr] AS 46 #define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_P, dest
48 #endif 47 #endif
49 48
50 #ifndef STORE 49 #ifndef STORE
51 #define STORE(type,src,addr) type src, [add 50 #define STORE(type,src,addr) type src, [addr]
52 #endif 51 #endif
53 52
54 #ifndef STORE_BLK 53 #ifndef STORE_BLK
55 #define STORE_BLK(src,addr) stda src, [add 54 #define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P
56 #endif 55 #endif
57 56
58 #ifndef FUNC_NAME 57 #ifndef FUNC_NAME
59 #define FUNC_NAME memcpy 58 #define FUNC_NAME memcpy
60 #endif 59 #endif
61 60
62 #ifndef PREAMBLE 61 #ifndef PREAMBLE
63 #define PREAMBLE 62 #define PREAMBLE
64 #endif 63 #endif
65 64
66 #ifndef XCC 65 #ifndef XCC
67 #define XCC xcc 66 #define XCC xcc
68 #endif 67 #endif
69 68
70 #define FREG_FROB(f1, f2, f3, f4, f5, f6, f7, 69 #define FREG_FROB(f1, f2, f3, f4, f5, f6, f7, f8, f9) \
71 faligndata %f1, %f2, %f48 70 faligndata %f1, %f2, %f48; \
72 faligndata %f2, %f3, %f50 71 faligndata %f2, %f3, %f50; \
73 faligndata %f3, %f4, %f52 72 faligndata %f3, %f4, %f52; \
74 faligndata %f4, %f5, %f54 73 faligndata %f4, %f5, %f54; \
75 faligndata %f5, %f6, %f56 74 faligndata %f5, %f6, %f56; \
76 faligndata %f6, %f7, %f58 75 faligndata %f6, %f7, %f58; \
77 faligndata %f7, %f8, %f60 76 faligndata %f7, %f8, %f60; \
78 faligndata %f8, %f9, %f62 77 faligndata %f8, %f9, %f62;
79 78
80 #define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc 79 #define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, jmptgt) \
81 EX_LD_FP(LOAD_BLK(%src, %fdest), U1_gs 80 EX_LD_FP(LOAD_BLK(%src, %fdest), U1_gs_80_fp); \
82 EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_g 81 EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \
83 add %src, 0x40, %s 82 add %src, 0x40, %src; \
84 subcc %GLOBAL_SPARE, 83 subcc %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE; \
85 be,pn %xcc, jmptgt; 84 be,pn %xcc, jmptgt; \
86 add %dest, 0x40, % 85 add %dest, 0x40, %dest; \
87 86
88 #define LOOP_CHUNK1(src, dest, branch_dest) 87 #define LOOP_CHUNK1(src, dest, branch_dest) \
89 MAIN_LOOP_CHUNK(src, dest, f0, f48, b 88 MAIN_LOOP_CHUNK(src, dest, f0, f48, branch_dest)
90 #define LOOP_CHUNK2(src, dest, branch_dest) 89 #define LOOP_CHUNK2(src, dest, branch_dest) \
91 MAIN_LOOP_CHUNK(src, dest, f16, f48, b 90 MAIN_LOOP_CHUNK(src, dest, f16, f48, branch_dest)
92 #define LOOP_CHUNK3(src, dest, branch_dest) 91 #define LOOP_CHUNK3(src, dest, branch_dest) \
93 MAIN_LOOP_CHUNK(src, dest, f32, f48, b 92 MAIN_LOOP_CHUNK(src, dest, f32, f48, branch_dest)
94 93
95 #define DO_SYNC membar #Sync; 94 #define DO_SYNC membar #Sync;
96 #define STORE_SYNC(dest, fsrc) 95 #define STORE_SYNC(dest, fsrc) \
97 EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_g 96 EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \
98 add %dest, 0x40, % 97 add %dest, 0x40, %dest; \
99 DO_SYNC 98 DO_SYNC
100 99
101 #define STORE_JUMP(dest, fsrc, target) 100 #define STORE_JUMP(dest, fsrc, target) \
102 EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_g 101 EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_40_fp); \
103 add %dest, 0x40, % 102 add %dest, 0x40, %dest; \
104 ba,pt %xcc, target; 103 ba,pt %xcc, target; \
105 nop; 104 nop;
106 105
107 #define FINISH_VISCHUNK(dest, f0, f1) 106 #define FINISH_VISCHUNK(dest, f0, f1) \
108 subcc %g3, 8, %g3; 107 subcc %g3, 8, %g3; \
109 bl,pn %xcc, 95f; 108 bl,pn %xcc, 95f; \
110 faligndata %f0, %f1, %f48 109 faligndata %f0, %f1, %f48; \
111 EX_ST_FP(STORE(std, %f48, %dest), U1_g 110 EX_ST_FP(STORE(std, %f48, %dest), U1_g3_8_fp); \
112 add %dest, 8, %des 111 add %dest, 8, %dest;
113 112
114 #define UNEVEN_VISCHUNK_LAST(dest, f0, f1) 113 #define UNEVEN_VISCHUNK_LAST(dest, f0, f1) \
115 subcc %g3, 8, %g3; 114 subcc %g3, 8, %g3; \
116 bl,pn %xcc, 95f; 115 bl,pn %xcc, 95f; \
117 fsrc2 %f0, %f1; 116 fsrc2 %f0, %f1;
118 117
119 #define UNEVEN_VISCHUNK(dest, f0, f1) 118 #define UNEVEN_VISCHUNK(dest, f0, f1) \
120 UNEVEN_VISCHUNK_LAST(dest, f0, f1) 119 UNEVEN_VISCHUNK_LAST(dest, f0, f1) \
121 ba,a,pt %xcc, 93f; 120 ba,a,pt %xcc, 93f;
122 121
123 .register %g2,#scratch 122 .register %g2,#scratch
124 .register %g3,#scratch 123 .register %g3,#scratch
125 124
126 .text 125 .text
127 #ifndef EX_RETVAL 126 #ifndef EX_RETVAL
128 #define EX_RETVAL(x) x 127 #define EX_RETVAL(x) x
129 ENTRY(U1_g1_1_fp) 128 ENTRY(U1_g1_1_fp)
130 VISExitHalf 129 VISExitHalf
131 add %g1, 1, %g1 130 add %g1, 1, %g1
132 add %g1, %g2, %g1 131 add %g1, %g2, %g1
133 retl 132 retl
134 add %g1, %o2, %o0 133 add %g1, %o2, %o0
135 ENDPROC(U1_g1_1_fp) 134 ENDPROC(U1_g1_1_fp)
136 ENTRY(U1_g2_0_fp) 135 ENTRY(U1_g2_0_fp)
137 VISExitHalf 136 VISExitHalf
138 retl 137 retl
139 add %g2, %o2, %o0 138 add %g2, %o2, %o0
140 ENDPROC(U1_g2_0_fp) 139 ENDPROC(U1_g2_0_fp)
141 ENTRY(U1_g2_8_fp) 140 ENTRY(U1_g2_8_fp)
142 VISExitHalf 141 VISExitHalf
143 add %g2, 8, %g2 142 add %g2, 8, %g2
144 retl 143 retl
145 add %g2, %o2, %o0 144 add %g2, %o2, %o0
146 ENDPROC(U1_g2_8_fp) 145 ENDPROC(U1_g2_8_fp)
147 ENTRY(U1_gs_0_fp) 146 ENTRY(U1_gs_0_fp)
148 VISExitHalf 147 VISExitHalf
149 add %GLOBAL_SPARE, %g3, %o 148 add %GLOBAL_SPARE, %g3, %o0
150 retl 149 retl
151 add %o0, %o2, %o0 150 add %o0, %o2, %o0
152 ENDPROC(U1_gs_0_fp) 151 ENDPROC(U1_gs_0_fp)
153 ENTRY(U1_gs_80_fp) 152 ENTRY(U1_gs_80_fp)
154 VISExitHalf 153 VISExitHalf
155 add %GLOBAL_SPARE, 0x80, % 154 add %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
156 add %GLOBAL_SPARE, %g3, %o 155 add %GLOBAL_SPARE, %g3, %o0
157 retl 156 retl
158 add %o0, %o2, %o0 157 add %o0, %o2, %o0
159 ENDPROC(U1_gs_80_fp) 158 ENDPROC(U1_gs_80_fp)
160 ENTRY(U1_gs_40_fp) 159 ENTRY(U1_gs_40_fp)
161 VISExitHalf 160 VISExitHalf
162 add %GLOBAL_SPARE, 0x40, % 161 add %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE
163 add %GLOBAL_SPARE, %g3, %o 162 add %GLOBAL_SPARE, %g3, %o0
164 retl 163 retl
165 add %o0, %o2, %o0 164 add %o0, %o2, %o0
166 ENDPROC(U1_gs_40_fp) 165 ENDPROC(U1_gs_40_fp)
167 ENTRY(U1_g3_0_fp) 166 ENTRY(U1_g3_0_fp)
168 VISExitHalf 167 VISExitHalf
169 retl 168 retl
170 add %g3, %o2, %o0 169 add %g3, %o2, %o0
171 ENDPROC(U1_g3_0_fp) 170 ENDPROC(U1_g3_0_fp)
172 ENTRY(U1_g3_8_fp) 171 ENTRY(U1_g3_8_fp)
173 VISExitHalf 172 VISExitHalf
174 add %g3, 8, %g3 173 add %g3, 8, %g3
175 retl 174 retl
176 add %g3, %o2, %o0 175 add %g3, %o2, %o0
177 ENDPROC(U1_g3_8_fp) 176 ENDPROC(U1_g3_8_fp)
178 ENTRY(U1_o2_0_fp) 177 ENTRY(U1_o2_0_fp)
179 VISExitHalf 178 VISExitHalf
180 retl 179 retl
181 mov %o2, %o0 180 mov %o2, %o0
182 ENDPROC(U1_o2_0_fp) 181 ENDPROC(U1_o2_0_fp)
183 ENTRY(U1_o2_1_fp) 182 ENTRY(U1_o2_1_fp)
184 VISExitHalf 183 VISExitHalf
185 retl 184 retl
186 add %o2, 1, %o0 185 add %o2, 1, %o0
187 ENDPROC(U1_o2_1_fp) 186 ENDPROC(U1_o2_1_fp)
188 ENTRY(U1_gs_0) 187 ENTRY(U1_gs_0)
189 VISExitHalf 188 VISExitHalf
190 retl 189 retl
191 add %GLOBAL_SPARE, %o2, %o 190 add %GLOBAL_SPARE, %o2, %o0
192 ENDPROC(U1_gs_0) 191 ENDPROC(U1_gs_0)
193 ENTRY(U1_gs_8) 192 ENTRY(U1_gs_8)
194 VISExitHalf 193 VISExitHalf
195 add %GLOBAL_SPARE, %o2, %G 194 add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE
196 retl 195 retl
197 add %GLOBAL_SPARE, 0x8, %o 196 add %GLOBAL_SPARE, 0x8, %o0
198 ENDPROC(U1_gs_8) 197 ENDPROC(U1_gs_8)
199 ENTRY(U1_gs_10) 198 ENTRY(U1_gs_10)
200 VISExitHalf 199 VISExitHalf
201 add %GLOBAL_SPARE, %o2, %G 200 add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE
202 retl 201 retl
203 add %GLOBAL_SPARE, 0x10, % 202 add %GLOBAL_SPARE, 0x10, %o0
204 ENDPROC(U1_gs_10) 203 ENDPROC(U1_gs_10)
205 ENTRY(U1_o2_0) 204 ENTRY(U1_o2_0)
206 retl 205 retl
207 mov %o2, %o0 206 mov %o2, %o0
208 ENDPROC(U1_o2_0) 207 ENDPROC(U1_o2_0)
209 ENTRY(U1_o2_8) 208 ENTRY(U1_o2_8)
210 retl 209 retl
211 add %o2, 8, %o0 210 add %o2, 8, %o0
212 ENDPROC(U1_o2_8) 211 ENDPROC(U1_o2_8)
213 ENTRY(U1_o2_4) 212 ENTRY(U1_o2_4)
214 retl 213 retl
215 add %o2, 4, %o0 214 add %o2, 4, %o0
216 ENDPROC(U1_o2_4) 215 ENDPROC(U1_o2_4)
217 ENTRY(U1_o2_1) 216 ENTRY(U1_o2_1)
218 retl 217 retl
219 add %o2, 1, %o0 218 add %o2, 1, %o0
220 ENDPROC(U1_o2_1) 219 ENDPROC(U1_o2_1)
221 ENTRY(U1_g1_0) 220 ENTRY(U1_g1_0)
222 retl 221 retl
223 add %g1, %o2, %o0 222 add %g1, %o2, %o0
224 ENDPROC(U1_g1_0) 223 ENDPROC(U1_g1_0)
225 ENTRY(U1_g1_1) 224 ENTRY(U1_g1_1)
226 add %g1, 1, %g1 225 add %g1, 1, %g1
227 retl 226 retl
228 add %g1, %o2, %o0 227 add %g1, %o2, %o0
229 ENDPROC(U1_g1_1) 228 ENDPROC(U1_g1_1)
230 ENTRY(U1_gs_0_o2_adj) 229 ENTRY(U1_gs_0_o2_adj)
231 and %o2, 7, %o2 230 and %o2, 7, %o2
232 retl 231 retl
233 add %GLOBAL_SPARE, %o2, %o 232 add %GLOBAL_SPARE, %o2, %o0
234 ENDPROC(U1_gs_0_o2_adj) 233 ENDPROC(U1_gs_0_o2_adj)
235 ENTRY(U1_gs_8_o2_adj) 234 ENTRY(U1_gs_8_o2_adj)
236 and %o2, 7, %o2 235 and %o2, 7, %o2
237 add %GLOBAL_SPARE, 8, %GLO 236 add %GLOBAL_SPARE, 8, %GLOBAL_SPARE
238 retl 237 retl
239 add %GLOBAL_SPARE, %o2, %o 238 add %GLOBAL_SPARE, %o2, %o0
240 ENDPROC(U1_gs_8_o2_adj) 239 ENDPROC(U1_gs_8_o2_adj)
241 #endif 240 #endif
242 241
243 .align 64 242 .align 64
244 243
245 .globl FUNC_NAME 244 .globl FUNC_NAME
246 .type FUNC_NAME,#function 245 .type FUNC_NAME,#function
247 FUNC_NAME: /* %o0=dst, %o1=src, % 246 FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
248 srlx %o2, 31, %g2 247 srlx %o2, 31, %g2
249 cmp %g2, 0 248 cmp %g2, 0
250 tne %xcc, 5 249 tne %xcc, 5
251 PREAMBLE 250 PREAMBLE
252 mov %o0, %o4 251 mov %o0, %o4
253 cmp %o2, 0 252 cmp %o2, 0
254 be,pn %XCC, 85f 253 be,pn %XCC, 85f
255 or %o0, %o1, %o3 254 or %o0, %o1, %o3
256 cmp %o2, 16 255 cmp %o2, 16
257 blu,a,pn %XCC, 80f 256 blu,a,pn %XCC, 80f
258 or %o3, %o2, %o3 257 or %o3, %o2, %o3
259 258
260 cmp %o2, (5 * 64) 259 cmp %o2, (5 * 64)
261 blu,pt %XCC, 70f 260 blu,pt %XCC, 70f
262 andcc %o3, 0x7, %g0 261 andcc %o3, 0x7, %g0
263 262
264 /* Clobbers o5/g1/g2/g3/g7/icc/xcc. * 263 /* Clobbers o5/g1/g2/g3/g7/icc/xcc. */
265 VISEntry 264 VISEntry
266 265
267 /* Is 'dst' already aligned on an 64-b 266 /* Is 'dst' already aligned on an 64-byte boundary? */
268 andcc %o0, 0x3f, %g2 267 andcc %o0, 0x3f, %g2
269 be,pt %XCC, 2f 268 be,pt %XCC, 2f
270 269
271 /* Compute abs((dst & 0x3f) - 0x40) in 270 /* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number
272 * of bytes to copy to make 'dst' 64-b 271 * of bytes to copy to make 'dst' 64-byte aligned. We pre-
273 * subtract this from 'len'. 272 * subtract this from 'len'.
274 */ 273 */
275 sub %o0, %o1, %GLOBAL_SPAR 274 sub %o0, %o1, %GLOBAL_SPARE
276 sub %g2, 0x40, %g2 275 sub %g2, 0x40, %g2
277 sub %g0, %g2, %g2 276 sub %g0, %g2, %g2
278 sub %o2, %g2, %o2 277 sub %o2, %g2, %o2
279 andcc %g2, 0x7, %g1 278 andcc %g2, 0x7, %g1
280 be,pt %icc, 2f 279 be,pt %icc, 2f
281 and %g2, 0x38, %g2 280 and %g2, 0x38, %g2
282 281
283 1: subcc %g1, 0x1, %g1 282 1: subcc %g1, 0x1, %g1
284 EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), 283 EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U1_g1_1_fp)
285 EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL 284 EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE), U1_g1_1_fp)
286 bgu,pt %XCC, 1b 285 bgu,pt %XCC, 1b
287 add %o1, 0x1, %o1 286 add %o1, 0x1, %o1
288 287
289 add %o1, %GLOBAL_SPARE, %o 288 add %o1, %GLOBAL_SPARE, %o0
290 289
291 2: cmp %g2, 0x0 290 2: cmp %g2, 0x0
292 and %o1, 0x7, %g1 291 and %o1, 0x7, %g1
293 be,pt %icc, 3f 292 be,pt %icc, 3f
294 alignaddr %o1, %g0, %o1 293 alignaddr %o1, %g0, %o1
295 294
296 EX_LD_FP(LOAD(ldd, %o1, %f4), U1_g2_0_ 295 EX_LD_FP(LOAD(ldd, %o1, %f4), U1_g2_0_fp)
297 1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U1 296 1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U1_g2_0_fp)
298 add %o1, 0x8, %o1 297 add %o1, 0x8, %o1
299 subcc %g2, 0x8, %g2 298 subcc %g2, 0x8, %g2
300 faligndata %f4, %f6, %f0 299 faligndata %f4, %f6, %f0
301 EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8 300 EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp)
302 be,pn %icc, 3f 301 be,pn %icc, 3f
303 add %o0, 0x8, %o0 302 add %o0, 0x8, %o0
304 303
305 EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U1 304 EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U1_g2_0_fp)
306 add %o1, 0x8, %o1 305 add %o1, 0x8, %o1
307 subcc %g2, 0x8, %g2 306 subcc %g2, 0x8, %g2
308 faligndata %f6, %f4, %f0 307 faligndata %f6, %f4, %f0
309 EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8 308 EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp)
310 bne,pt %icc, 1b 309 bne,pt %icc, 1b
311 add %o0, 0x8, %o0 310 add %o0, 0x8, %o0
312 311
313 /* Destination is 64-byte aligned. */ 312 /* Destination is 64-byte aligned. */
314 3: 313 3:
315 membar #LoadStore | #StoreS 314 membar #LoadStore | #StoreStore | #StoreLoad
316 315
317 subcc %o2, 0x40, %GLOBAL_SPA 316 subcc %o2, 0x40, %GLOBAL_SPARE
318 add %o1, %g1, %g1 317 add %o1, %g1, %g1
319 andncc %GLOBAL_SPARE, (0x40 - 318 andncc %GLOBAL_SPARE, (0x40 - 1), %GLOBAL_SPARE
320 srl %g1, 3, %g2 319 srl %g1, 3, %g2
321 sub %o2, %GLOBAL_SPARE, %g 320 sub %o2, %GLOBAL_SPARE, %g3
322 andn %o1, (0x40 - 1), %o1 321 andn %o1, (0x40 - 1), %o1
323 and %g2, 7, %g2 322 and %g2, 7, %g2
324 andncc %g3, 0x7, %g3 323 andncc %g3, 0x7, %g3
325 fsrc2 %f0, %f2 324 fsrc2 %f0, %f2
326 sub %g3, 0x8, %g3 325 sub %g3, 0x8, %g3
327 sub %o2, %GLOBAL_SPARE, %o 326 sub %o2, %GLOBAL_SPARE, %o2
328 327
329 add %g1, %GLOBAL_SPARE, %g 328 add %g1, %GLOBAL_SPARE, %g1
330 subcc %o2, %g3, %o2 329 subcc %o2, %g3, %o2
331 330
332 EX_LD_FP(LOAD_BLK(%o1, %f0), U1_gs_0_f 331 EX_LD_FP(LOAD_BLK(%o1, %f0), U1_gs_0_fp)
333 add %o1, 0x40, %o1 332 add %o1, 0x40, %o1
334 add %g1, %g3, %g1 333 add %g1, %g3, %g1
335 EX_LD_FP(LOAD_BLK(%o1, %f16), U1_gs_0_ 334 EX_LD_FP(LOAD_BLK(%o1, %f16), U1_gs_0_fp)
336 add %o1, 0x40, %o1 335 add %o1, 0x40, %o1
337 sub %GLOBAL_SPARE, 0x80, % 336 sub %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
338 EX_LD_FP(LOAD_BLK(%o1, %f32), U1_gs_80 337 EX_LD_FP(LOAD_BLK(%o1, %f32), U1_gs_80_fp)
339 add %o1, 0x40, %o1 338 add %o1, 0x40, %o1
340 339
341 /* There are 8 instances of the unroll 340 /* There are 8 instances of the unrolled loop,
342 * one for each possible alignment of 341 * one for each possible alignment of the
343 * source buffer. Each loop instance 342 * source buffer. Each loop instance is 452
344 * bytes. 343 * bytes.
345 */ 344 */
346 sll %g2, 3, %o3 345 sll %g2, 3, %o3
347 sub %o3, %g2, %o3 346 sub %o3, %g2, %o3
348 sllx %o3, 4, %o3 347 sllx %o3, 4, %o3
349 add %o3, %g2, %o3 348 add %o3, %g2, %o3
350 sllx %o3, 2, %g2 349 sllx %o3, 2, %g2
351 1: rd %pc, %o3 350 1: rd %pc, %o3
352 add %o3, %lo(1f - 1b), %o3 351 add %o3, %lo(1f - 1b), %o3
353 jmpl %o3 + %g2, %g0 352 jmpl %o3 + %g2, %g0
354 nop 353 nop
355 354
356 .align 64 355 .align 64
357 1: FREG_FROB(f0, f2, f4, f6, f8, f10,f12, 356 1: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
358 LOOP_CHUNK1(o1, o0, 1f) 357 LOOP_CHUNK1(o1, o0, 1f)
359 FREG_FROB(f16,f18,f20,f22,f24,f26,f28, 358 FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
360 LOOP_CHUNK2(o1, o0, 2f) 359 LOOP_CHUNK2(o1, o0, 2f)
361 FREG_FROB(f32,f34,f36,f38,f40,f42,f44, 360 FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
362 LOOP_CHUNK3(o1, o0, 3f) 361 LOOP_CHUNK3(o1, o0, 3f)
363 ba,pt %xcc, 1b+4 362 ba,pt %xcc, 1b+4
364 faligndata %f0, %f2, %f48 363 faligndata %f0, %f2, %f48
365 1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28, 364 1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
366 STORE_SYNC(o0, f48) 365 STORE_SYNC(o0, f48)
367 FREG_FROB(f32,f34,f36,f38,f40,f42,f44, 366 FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
368 STORE_JUMP(o0, f48, 40f) 367 STORE_JUMP(o0, f48, 40f)
369 2: FREG_FROB(f32,f34,f36,f38,f40,f42,f44, 368 2: FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
370 STORE_SYNC(o0, f48) 369 STORE_SYNC(o0, f48)
371 FREG_FROB(f0, f2, f4, f6, f8, f10,f12, 370 FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
372 STORE_JUMP(o0, f48, 48f) 371 STORE_JUMP(o0, f48, 48f)
373 3: FREG_FROB(f0, f2, f4, f6, f8, f10,f12, 372 3: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
374 STORE_SYNC(o0, f48) 373 STORE_SYNC(o0, f48)
375 FREG_FROB(f16,f18,f20,f22,f24,f26,f28, 374 FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
376 STORE_JUMP(o0, f48, 56f) 375 STORE_JUMP(o0, f48, 56f)
377 376
378 1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14, 377 1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
379 LOOP_CHUNK1(o1, o0, 1f) 378 LOOP_CHUNK1(o1, o0, 1f)
380 FREG_FROB(f18,f20,f22,f24,f26,f28,f30, 379 FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
381 LOOP_CHUNK2(o1, o0, 2f) 380 LOOP_CHUNK2(o1, o0, 2f)
382 FREG_FROB(f34,f36,f38,f40,f42,f44,f46, 381 FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
383 LOOP_CHUNK3(o1, o0, 3f) 382 LOOP_CHUNK3(o1, o0, 3f)
384 ba,pt %xcc, 1b+4 383 ba,pt %xcc, 1b+4
385 faligndata %f2, %f4, %f48 384 faligndata %f2, %f4, %f48
386 1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30, 385 1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
387 STORE_SYNC(o0, f48) 386 STORE_SYNC(o0, f48)
388 FREG_FROB(f34,f36,f38,f40,f42,f44,f46, 387 FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
389 STORE_JUMP(o0, f48, 41f) 388 STORE_JUMP(o0, f48, 41f)
390 2: FREG_FROB(f34,f36,f38,f40,f42,f44,f46, 389 2: FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
391 STORE_SYNC(o0, f48) 390 STORE_SYNC(o0, f48)
392 FREG_FROB(f2, f4, f6, f8, f10,f12,f14, 391 FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
393 STORE_JUMP(o0, f48, 49f) 392 STORE_JUMP(o0, f48, 49f)
394 3: FREG_FROB(f2, f4, f6, f8, f10,f12,f14, 393 3: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
395 STORE_SYNC(o0, f48) 394 STORE_SYNC(o0, f48)
396 FREG_FROB(f18,f20,f22,f24,f26,f28,f30, 395 FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
397 STORE_JUMP(o0, f48, 57f) 396 STORE_JUMP(o0, f48, 57f)
398 397
399 1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16, 398 1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
400 LOOP_CHUNK1(o1, o0, 1f) 399 LOOP_CHUNK1(o1, o0, 1f)
401 FREG_FROB(f20,f22,f24,f26,f28,f30,f32, 400 FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
402 LOOP_CHUNK2(o1, o0, 2f) 401 LOOP_CHUNK2(o1, o0, 2f)
403 FREG_FROB(f36,f38,f40,f42,f44,f46,f0, 402 FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
404 LOOP_CHUNK3(o1, o0, 3f) 403 LOOP_CHUNK3(o1, o0, 3f)
405 ba,pt %xcc, 1b+4 404 ba,pt %xcc, 1b+4
406 faligndata %f4, %f6, %f48 405 faligndata %f4, %f6, %f48
407 1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32, 406 1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
408 STORE_SYNC(o0, f48) 407 STORE_SYNC(o0, f48)
409 FREG_FROB(f36,f38,f40,f42,f44,f46,f0, 408 FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
410 STORE_JUMP(o0, f48, 42f) 409 STORE_JUMP(o0, f48, 42f)
411 2: FREG_FROB(f36,f38,f40,f42,f44,f46,f0, 410 2: FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
412 STORE_SYNC(o0, f48) 411 STORE_SYNC(o0, f48)
413 FREG_FROB(f4, f6, f8, f10,f12,f14,f16, 412 FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
414 STORE_JUMP(o0, f48, 50f) 413 STORE_JUMP(o0, f48, 50f)
415 3: FREG_FROB(f4, f6, f8, f10,f12,f14,f16, 414 3: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
416 STORE_SYNC(o0, f48) 415 STORE_SYNC(o0, f48)
417 FREG_FROB(f20,f22,f24,f26,f28,f30,f32, 416 FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
418 STORE_JUMP(o0, f48, 58f) 417 STORE_JUMP(o0, f48, 58f)
419 418
420 1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18, 419 1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
421 LOOP_CHUNK1(o1, o0, 1f) 420 LOOP_CHUNK1(o1, o0, 1f)
422 FREG_FROB(f22,f24,f26,f28,f30,f32,f34, 421 FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
423 LOOP_CHUNK2(o1, o0, 2f) 422 LOOP_CHUNK2(o1, o0, 2f)
424 FREG_FROB(f38,f40,f42,f44,f46,f0, f2, 423 FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
425 LOOP_CHUNK3(o1, o0, 3f) 424 LOOP_CHUNK3(o1, o0, 3f)
426 ba,pt %xcc, 1b+4 425 ba,pt %xcc, 1b+4
427 faligndata %f6, %f8, %f48 426 faligndata %f6, %f8, %f48
428 1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34, 427 1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
429 STORE_SYNC(o0, f48) 428 STORE_SYNC(o0, f48)
430 FREG_FROB(f38,f40,f42,f44,f46,f0, f2, 429 FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
431 STORE_JUMP(o0, f48, 43f) 430 STORE_JUMP(o0, f48, 43f)
432 2: FREG_FROB(f38,f40,f42,f44,f46,f0, f2, 431 2: FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
433 STORE_SYNC(o0, f48) 432 STORE_SYNC(o0, f48)
434 FREG_FROB(f6, f8, f10,f12,f14,f16,f18, 433 FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
435 STORE_JUMP(o0, f48, 51f) 434 STORE_JUMP(o0, f48, 51f)
436 3: FREG_FROB(f6, f8, f10,f12,f14,f16,f18, 435 3: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
437 STORE_SYNC(o0, f48) 436 STORE_SYNC(o0, f48)
438 FREG_FROB(f22,f24,f26,f28,f30,f32,f34, 437 FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
439 STORE_JUMP(o0, f48, 59f) 438 STORE_JUMP(o0, f48, 59f)
440 439
441 1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20, 440 1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
442 LOOP_CHUNK1(o1, o0, 1f) 441 LOOP_CHUNK1(o1, o0, 1f)
443 FREG_FROB(f24,f26,f28,f30,f32,f34,f36, 442 FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
444 LOOP_CHUNK2(o1, o0, 2f) 443 LOOP_CHUNK2(o1, o0, 2f)
445 FREG_FROB(f40,f42,f44,f46,f0, f2, f4, 444 FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
446 LOOP_CHUNK3(o1, o0, 3f) 445 LOOP_CHUNK3(o1, o0, 3f)
447 ba,pt %xcc, 1b+4 446 ba,pt %xcc, 1b+4
448 faligndata %f8, %f10, %f48 447 faligndata %f8, %f10, %f48
449 1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36, 448 1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
450 STORE_SYNC(o0, f48) 449 STORE_SYNC(o0, f48)
451 FREG_FROB(f40,f42,f44,f46,f0, f2, f4, 450 FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
452 STORE_JUMP(o0, f48, 44f) 451 STORE_JUMP(o0, f48, 44f)
453 2: FREG_FROB(f40,f42,f44,f46,f0, f2, f4, 452 2: FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
454 STORE_SYNC(o0, f48) 453 STORE_SYNC(o0, f48)
455 FREG_FROB(f8, f10,f12,f14,f16,f18,f20, 454 FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
456 STORE_JUMP(o0, f48, 52f) 455 STORE_JUMP(o0, f48, 52f)
457 3: FREG_FROB(f8, f10,f12,f14,f16,f18,f20, 456 3: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
458 STORE_SYNC(o0, f48) 457 STORE_SYNC(o0, f48)
459 FREG_FROB(f24,f26,f28,f30,f32,f34,f36, 458 FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
460 STORE_JUMP(o0, f48, 60f) 459 STORE_JUMP(o0, f48, 60f)
461 460
462 1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22, 461 1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
463 LOOP_CHUNK1(o1, o0, 1f) 462 LOOP_CHUNK1(o1, o0, 1f)
464 FREG_FROB(f26,f28,f30,f32,f34,f36,f38, 463 FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
465 LOOP_CHUNK2(o1, o0, 2f) 464 LOOP_CHUNK2(o1, o0, 2f)
466 FREG_FROB(f42,f44,f46,f0, f2, f4, f6, 465 FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
467 LOOP_CHUNK3(o1, o0, 3f) 466 LOOP_CHUNK3(o1, o0, 3f)
468 ba,pt %xcc, 1b+4 467 ba,pt %xcc, 1b+4
469 faligndata %f10, %f12, %f48 468 faligndata %f10, %f12, %f48
470 1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38, 469 1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
471 STORE_SYNC(o0, f48) 470 STORE_SYNC(o0, f48)
472 FREG_FROB(f42,f44,f46,f0, f2, f4, f6, 471 FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
473 STORE_JUMP(o0, f48, 45f) 472 STORE_JUMP(o0, f48, 45f)
474 2: FREG_FROB(f42,f44,f46,f0, f2, f4, f6, 473 2: FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
475 STORE_SYNC(o0, f48) 474 STORE_SYNC(o0, f48)
476 FREG_FROB(f10,f12,f14,f16,f18,f20,f22, 475 FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
477 STORE_JUMP(o0, f48, 53f) 476 STORE_JUMP(o0, f48, 53f)
478 3: FREG_FROB(f10,f12,f14,f16,f18,f20,f22, 477 3: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
479 STORE_SYNC(o0, f48) 478 STORE_SYNC(o0, f48)
480 FREG_FROB(f26,f28,f30,f32,f34,f36,f38, 479 FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
481 STORE_JUMP(o0, f48, 61f) 480 STORE_JUMP(o0, f48, 61f)
482 481
483 1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24, 482 1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
484 LOOP_CHUNK1(o1, o0, 1f) 483 LOOP_CHUNK1(o1, o0, 1f)
485 FREG_FROB(f28,f30,f32,f34,f36,f38,f40, 484 FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
486 LOOP_CHUNK2(o1, o0, 2f) 485 LOOP_CHUNK2(o1, o0, 2f)
487 FREG_FROB(f44,f46,f0, f2, f4, f6, f8, 486 FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
488 LOOP_CHUNK3(o1, o0, 3f) 487 LOOP_CHUNK3(o1, o0, 3f)
489 ba,pt %xcc, 1b+4 488 ba,pt %xcc, 1b+4
490 faligndata %f12, %f14, %f48 489 faligndata %f12, %f14, %f48
491 1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40, 490 1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
492 STORE_SYNC(o0, f48) 491 STORE_SYNC(o0, f48)
493 FREG_FROB(f44,f46,f0, f2, f4, f6, f8, 492 FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
494 STORE_JUMP(o0, f48, 46f) 493 STORE_JUMP(o0, f48, 46f)
495 2: FREG_FROB(f44,f46,f0, f2, f4, f6, f8, 494 2: FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
496 STORE_SYNC(o0, f48) 495 STORE_SYNC(o0, f48)
497 FREG_FROB(f12,f14,f16,f18,f20,f22,f24, 496 FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
498 STORE_JUMP(o0, f48, 54f) 497 STORE_JUMP(o0, f48, 54f)
499 3: FREG_FROB(f12,f14,f16,f18,f20,f22,f24, 498 3: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
500 STORE_SYNC(o0, f48) 499 STORE_SYNC(o0, f48)
501 FREG_FROB(f28,f30,f32,f34,f36,f38,f40, 500 FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
502 STORE_JUMP(o0, f48, 62f) 501 STORE_JUMP(o0, f48, 62f)
503 502
504 1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26, 503 1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
505 LOOP_CHUNK1(o1, o0, 1f) 504 LOOP_CHUNK1(o1, o0, 1f)
506 FREG_FROB(f30,f32,f34,f36,f38,f40,f42, 505 FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
507 LOOP_CHUNK2(o1, o0, 2f) 506 LOOP_CHUNK2(o1, o0, 2f)
508 FREG_FROB(f46,f0, f2, f4, f6, f8, f10, 507 FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
509 LOOP_CHUNK3(o1, o0, 3f) 508 LOOP_CHUNK3(o1, o0, 3f)
510 ba,pt %xcc, 1b+4 509 ba,pt %xcc, 1b+4
511 faligndata %f14, %f16, %f48 510 faligndata %f14, %f16, %f48
512 1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42, 511 1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
513 STORE_SYNC(o0, f48) 512 STORE_SYNC(o0, f48)
514 FREG_FROB(f46,f0, f2, f4, f6, f8, f10, 513 FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
515 STORE_JUMP(o0, f48, 47f) 514 STORE_JUMP(o0, f48, 47f)
516 2: FREG_FROB(f46,f0, f2, f4, f6, f8, f10, 515 2: FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
517 STORE_SYNC(o0, f48) 516 STORE_SYNC(o0, f48)
518 FREG_FROB(f14,f16,f18,f20,f22,f24,f26, 517 FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
519 STORE_JUMP(o0, f48, 55f) 518 STORE_JUMP(o0, f48, 55f)
520 3: FREG_FROB(f14,f16,f18,f20,f22,f24,f26, 519 3: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
521 STORE_SYNC(o0, f48) 520 STORE_SYNC(o0, f48)
522 FREG_FROB(f30,f32,f34,f36,f38,f40,f42, 521 FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
523 STORE_JUMP(o0, f48, 63f) 522 STORE_JUMP(o0, f48, 63f)
524 523
525 40: FINISH_VISCHUNK(o0, f0, f2) 524 40: FINISH_VISCHUNK(o0, f0, f2)
526 41: FINISH_VISCHUNK(o0, f2, f4) 525 41: FINISH_VISCHUNK(o0, f2, f4)
527 42: FINISH_VISCHUNK(o0, f4, f6) 526 42: FINISH_VISCHUNK(o0, f4, f6)
528 43: FINISH_VISCHUNK(o0, f6, f8) 527 43: FINISH_VISCHUNK(o0, f6, f8)
529 44: FINISH_VISCHUNK(o0, f8, f10) 528 44: FINISH_VISCHUNK(o0, f8, f10)
530 45: FINISH_VISCHUNK(o0, f10, f12) 529 45: FINISH_VISCHUNK(o0, f10, f12)
531 46: FINISH_VISCHUNK(o0, f12, f14) 530 46: FINISH_VISCHUNK(o0, f12, f14)
532 47: UNEVEN_VISCHUNK(o0, f14, f0) 531 47: UNEVEN_VISCHUNK(o0, f14, f0)
533 48: FINISH_VISCHUNK(o0, f16, f18) 532 48: FINISH_VISCHUNK(o0, f16, f18)
534 49: FINISH_VISCHUNK(o0, f18, f20) 533 49: FINISH_VISCHUNK(o0, f18, f20)
535 50: FINISH_VISCHUNK(o0, f20, f22) 534 50: FINISH_VISCHUNK(o0, f20, f22)
536 51: FINISH_VISCHUNK(o0, f22, f24) 535 51: FINISH_VISCHUNK(o0, f22, f24)
537 52: FINISH_VISCHUNK(o0, f24, f26) 536 52: FINISH_VISCHUNK(o0, f24, f26)
538 53: FINISH_VISCHUNK(o0, f26, f28) 537 53: FINISH_VISCHUNK(o0, f26, f28)
539 54: FINISH_VISCHUNK(o0, f28, f30) 538 54: FINISH_VISCHUNK(o0, f28, f30)
540 55: UNEVEN_VISCHUNK(o0, f30, f0) 539 55: UNEVEN_VISCHUNK(o0, f30, f0)
541 56: FINISH_VISCHUNK(o0, f32, f34) 540 56: FINISH_VISCHUNK(o0, f32, f34)
542 57: FINISH_VISCHUNK(o0, f34, f36) 541 57: FINISH_VISCHUNK(o0, f34, f36)
543 58: FINISH_VISCHUNK(o0, f36, f38) 542 58: FINISH_VISCHUNK(o0, f36, f38)
544 59: FINISH_VISCHUNK(o0, f38, f40) 543 59: FINISH_VISCHUNK(o0, f38, f40)
545 60: FINISH_VISCHUNK(o0, f40, f42) 544 60: FINISH_VISCHUNK(o0, f40, f42)
546 61: FINISH_VISCHUNK(o0, f42, f44) 545 61: FINISH_VISCHUNK(o0, f42, f44)
547 62: FINISH_VISCHUNK(o0, f44, f46) 546 62: FINISH_VISCHUNK(o0, f44, f46)
548 63: UNEVEN_VISCHUNK_LAST(o0, f46, f0) 547 63: UNEVEN_VISCHUNK_LAST(o0, f46, f0)
549 548
550 93: EX_LD_FP(LOAD(ldd, %o1, %f2), U1_g3_0_ 549 93: EX_LD_FP(LOAD(ldd, %o1, %f2), U1_g3_0_fp)
551 add %o1, 8, %o1 550 add %o1, 8, %o1
552 subcc %g3, 8, %g3 551 subcc %g3, 8, %g3
553 faligndata %f0, %f2, %f8 552 faligndata %f0, %f2, %f8
554 EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8 553 EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp)
555 bl,pn %xcc, 95f 554 bl,pn %xcc, 95f
556 add %o0, 8, %o0 555 add %o0, 8, %o0
557 EX_LD_FP(LOAD(ldd, %o1, %f0), U1_g3_0_ 556 EX_LD_FP(LOAD(ldd, %o1, %f0), U1_g3_0_fp)
558 add %o1, 8, %o1 557 add %o1, 8, %o1
559 subcc %g3, 8, %g3 558 subcc %g3, 8, %g3
560 faligndata %f2, %f0, %f8 559 faligndata %f2, %f0, %f8
561 EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8 560 EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp)
562 bge,pt %xcc, 93b 561 bge,pt %xcc, 93b
563 add %o0, 8, %o0 562 add %o0, 8, %o0
564 563
565 95: brz,pt %o2, 2f 564 95: brz,pt %o2, 2f
566 mov %g1, %o1 565 mov %g1, %o1
567 566
568 1: EX_LD_FP(LOAD(ldub, %o1, %o3), U1_o2_0 567 1: EX_LD_FP(LOAD(ldub, %o1, %o3), U1_o2_0_fp)
569 add %o1, 1, %o1 568 add %o1, 1, %o1
570 subcc %o2, 1, %o2 569 subcc %o2, 1, %o2
571 EX_ST_FP(STORE(stb, %o3, %o0), U1_o2_1 570 EX_ST_FP(STORE(stb, %o3, %o0), U1_o2_1_fp)
572 bne,pt %xcc, 1b 571 bne,pt %xcc, 1b
573 add %o0, 1, %o0 572 add %o0, 1, %o0
574 573
575 2: membar #StoreLoad | #StoreSto 574 2: membar #StoreLoad | #StoreStore
576 VISExit 575 VISExit
577 retl 576 retl
578 mov EX_RETVAL(%o4), %o0 577 mov EX_RETVAL(%o4), %o0
579 578
580 .align 64 579 .align 64
581 70: /* 16 < len <= (5 * 64) */ 580 70: /* 16 < len <= (5 * 64) */
582 bne,pn %XCC, 75f 581 bne,pn %XCC, 75f
583 sub %o0, %o1, %o3 582 sub %o0, %o1, %o3
584 583
585 72: andn %o2, 0xf, %GLOBAL_SPAR 584 72: andn %o2, 0xf, %GLOBAL_SPARE
586 and %o2, 0xf, %o2 585 and %o2, 0xf, %o2
587 1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U1_g 586 1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U1_gs_0)
588 EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U1_g 587 EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U1_gs_0)
589 subcc %GLOBAL_SPARE, 0x10, % 588 subcc %GLOBAL_SPARE, 0x10, %GLOBAL_SPARE
590 EX_ST(STORE(stx, %o5, %o1 + %o3), U1_g 589 EX_ST(STORE(stx, %o5, %o1 + %o3), U1_gs_10)
591 add %o1, 0x8, %o1 590 add %o1, 0x8, %o1
592 EX_ST(STORE(stx, %g1, %o1 + %o3), U1_g 591 EX_ST(STORE(stx, %g1, %o1 + %o3), U1_gs_8)
593 bgu,pt %XCC, 1b 592 bgu,pt %XCC, 1b
594 add %o1, 0x8, %o1 593 add %o1, 0x8, %o1
595 73: andcc %o2, 0x8, %g0 594 73: andcc %o2, 0x8, %g0
596 be,pt %XCC, 1f 595 be,pt %XCC, 1f
597 nop 596 nop
598 EX_LD(LOAD(ldx, %o1, %o5), U1_o2_0) 597 EX_LD(LOAD(ldx, %o1, %o5), U1_o2_0)
599 sub %o2, 0x8, %o2 598 sub %o2, 0x8, %o2
600 EX_ST(STORE(stx, %o5, %o1 + %o3), U1_o 599 EX_ST(STORE(stx, %o5, %o1 + %o3), U1_o2_8)
601 add %o1, 0x8, %o1 600 add %o1, 0x8, %o1
602 1: andcc %o2, 0x4, %g0 601 1: andcc %o2, 0x4, %g0
603 be,pt %XCC, 1f 602 be,pt %XCC, 1f
604 nop 603 nop
605 EX_LD(LOAD(lduw, %o1, %o5), U1_o2_0) 604 EX_LD(LOAD(lduw, %o1, %o5), U1_o2_0)
606 sub %o2, 0x4, %o2 605 sub %o2, 0x4, %o2
607 EX_ST(STORE(stw, %o5, %o1 + %o3), U1_o 606 EX_ST(STORE(stw, %o5, %o1 + %o3), U1_o2_4)
608 add %o1, 0x4, %o1 607 add %o1, 0x4, %o1
609 1: cmp %o2, 0 608 1: cmp %o2, 0
610 be,pt %XCC, 85f 609 be,pt %XCC, 85f
611 nop 610 nop
612 ba,pt %xcc, 90f 611 ba,pt %xcc, 90f
613 nop 612 nop
614 613
615 75: andcc %o0, 0x7, %g1 614 75: andcc %o0, 0x7, %g1
616 sub %g1, 0x8, %g1 615 sub %g1, 0x8, %g1
617 be,pn %icc, 2f 616 be,pn %icc, 2f
618 sub %g0, %g1, %g1 617 sub %g0, %g1, %g1
619 sub %o2, %g1, %o2 618 sub %o2, %g1, %o2
620 619
621 1: EX_LD(LOAD(ldub, %o1, %o5), U1_g1_0) 620 1: EX_LD(LOAD(ldub, %o1, %o5), U1_g1_0)
622 subcc %g1, 1, %g1 621 subcc %g1, 1, %g1
623 EX_ST(STORE(stb, %o5, %o1 + %o3), U1_g 622 EX_ST(STORE(stb, %o5, %o1 + %o3), U1_g1_1)
624 bgu,pt %icc, 1b 623 bgu,pt %icc, 1b
625 add %o1, 1, %o1 624 add %o1, 1, %o1
626 625
627 2: add %o1, %o3, %o0 626 2: add %o1, %o3, %o0
628 andcc %o1, 0x7, %g1 627 andcc %o1, 0x7, %g1
629 bne,pt %icc, 8f 628 bne,pt %icc, 8f
630 sll %g1, 3, %g1 629 sll %g1, 3, %g1
631 630
632 cmp %o2, 16 631 cmp %o2, 16
633 bgeu,pt %icc, 72b 632 bgeu,pt %icc, 72b
634 nop 633 nop
635 ba,a,pt %xcc, 73b 634 ba,a,pt %xcc, 73b
636 635
637 8: mov 64, %o3 636 8: mov 64, %o3
638 andn %o1, 0x7, %o1 637 andn %o1, 0x7, %o1
639 EX_LD(LOAD(ldx, %o1, %g2), U1_o2_0) 638 EX_LD(LOAD(ldx, %o1, %g2), U1_o2_0)
640 sub %o3, %g1, %o3 639 sub %o3, %g1, %o3
641 andn %o2, 0x7, %GLOBAL_SPAR 640 andn %o2, 0x7, %GLOBAL_SPARE
642 sllx %g2, %g1, %g2 641 sllx %g2, %g1, %g2
643 1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U1_gs 642 1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U1_gs_0_o2_adj)
644 subcc %GLOBAL_SPARE, 0x8, %G 643 subcc %GLOBAL_SPARE, 0x8, %GLOBAL_SPARE
645 add %o1, 0x8, %o1 644 add %o1, 0x8, %o1
646 srlx %g3, %o3, %o5 645 srlx %g3, %o3, %o5
647 or %o5, %g2, %o5 646 or %o5, %g2, %o5
648 EX_ST(STORE(stx, %o5, %o0), U1_gs_8_o2 647 EX_ST(STORE(stx, %o5, %o0), U1_gs_8_o2_adj)
649 add %o0, 0x8, %o0 648 add %o0, 0x8, %o0
650 bgu,pt %icc, 1b 649 bgu,pt %icc, 1b
651 sllx %g3, %g1, %g2 650 sllx %g3, %g1, %g2
652 651
653 srl %g1, 3, %g1 652 srl %g1, 3, %g1
654 andcc %o2, 0x7, %o2 653 andcc %o2, 0x7, %o2
655 be,pn %icc, 85f 654 be,pn %icc, 85f
656 add %o1, %g1, %o1 655 add %o1, %g1, %o1
657 ba,pt %xcc, 90f 656 ba,pt %xcc, 90f
658 sub %o0, %o1, %o3 657 sub %o0, %o1, %o3
659 658
660 .align 64 659 .align 64
661 80: /* 0 < len <= 16 */ 660 80: /* 0 < len <= 16 */
662 andcc %o3, 0x3, %g0 661 andcc %o3, 0x3, %g0
663 bne,pn %XCC, 90f 662 bne,pn %XCC, 90f
664 sub %o0, %o1, %o3 663 sub %o0, %o1, %o3
665 664
666 1: EX_LD(LOAD(lduw, %o1, %g1), U1_o2_0) 665 1: EX_LD(LOAD(lduw, %o1, %g1), U1_o2_0)
667 subcc %o2, 4, %o2 666 subcc %o2, 4, %o2
668 EX_ST(STORE(stw, %g1, %o1 + %o3), U1_o 667 EX_ST(STORE(stw, %g1, %o1 + %o3), U1_o2_4)
669 bgu,pt %XCC, 1b 668 bgu,pt %XCC, 1b
670 add %o1, 4, %o1 669 add %o1, 4, %o1
671 670
672 85: retl 671 85: retl
673 mov EX_RETVAL(%o4), %o0 672 mov EX_RETVAL(%o4), %o0
674 673
675 .align 32 674 .align 32
676 90: EX_LD(LOAD(ldub, %o1, %g1), U1_o2_0) 675 90: EX_LD(LOAD(ldub, %o1, %g1), U1_o2_0)
677 subcc %o2, 1, %o2 676 subcc %o2, 1, %o2
678 EX_ST(STORE(stb, %g1, %o1 + %o3), U1_o 677 EX_ST(STORE(stb, %g1, %o1 + %o3), U1_o2_1)
679 bgu,pt %XCC, 90b 678 bgu,pt %XCC, 90b
680 add %o1, 1, %o1 679 add %o1, 1, %o1
681 retl 680 retl
682 mov EX_RETVAL(%o4), %o0 681 mov EX_RETVAL(%o4), %o0
683 682
684 .size FUNC_NAME, .-FUNC_NAME 683 .size FUNC_NAME, .-FUNC_NAME
685 EXPORT_SYMBOL(FUNC_NAME) 684 EXPORT_SYMBOL(FUNC_NAME)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.