1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* !! 2 /* copy_user.S: Sparc optimized copy_from_user and copy_to_user code.
3 * arch/alpha/lib/copy_user.S <<
4 * 3 *
5 * Copy to/from user space, handling exception !! 4 * Copyright(C) 1995 Linus Torvalds
6 * isn't exactly pretty. !! 5 * Copyright(C) 1996 David S. Miller
>> 6 * Copyright(C) 1996 Eddie C. Dost
>> 7 * Copyright(C) 1996,1998 Jakub Jelinek
7 * 8 *
8 * This is essentially the same as "memcpy()", !! 9 * derived from:
9 * Notably, we have to make sure that $0 is al !! 10 * e-mail between David and Eddie.
10 * contains the right "bytes left to copy" val !! 11 *
11 * only _after_ a successful copy). There is a !! 12 * Returns 0 if successful, otherwise count of bytes not copied yet
12 * exception setup stuff.. <<
13 */ 13 */
14 14
15 #include <linux/export.h> 15 #include <linux/export.h>
>> 16 #include <asm/ptrace.h>
>> 17 #include <asm/asmmacro.h>
>> 18 #include <asm/page.h>
>> 19 #include <asm/thread_info.h>
>> 20
>> 21 /* Work around cpp -rob */
>> 22 #define ALLOC #alloc
>> 23 #define EXECINSTR #execinstr
>> 24
>> 25 #define EX_ENTRY(l1, l2) \
>> 26 .section __ex_table,ALLOC; \
>> 27 .align 4; \
>> 28 .word l1, l2; \
>> 29 .text;
>> 30
>> 31 #define EX(x,y,a,b) \
>> 32 98: x,y; \
>> 33 .section .fixup,ALLOC,EXECINSTR; \
>> 34 .align 4; \
>> 35 99: retl; \
>> 36 a, b, %o0; \
>> 37 EX_ENTRY(98b, 99b)
>> 38
>> 39 #define EX2(x,y,c,d,e,a,b) \
>> 40 98: x,y; \
>> 41 .section .fixup,ALLOC,EXECINSTR; \
>> 42 .align 4; \
>> 43 99: c, d, e; \
>> 44 retl; \
>> 45 a, b, %o0; \
>> 46 EX_ENTRY(98b, 99b)
>> 47
>> 48 #define EXO2(x,y) \
>> 49 98: x, y; \
>> 50 EX_ENTRY(98b, 97f)
>> 51
>> 52 #define LD(insn, src, offset, reg, label) \
>> 53 98: insn [%src + (offset)], %reg; \
>> 54 .section .fixup,ALLOC,EXECINSTR; \
>> 55 99: ba label; \
>> 56 mov offset, %g5; \
>> 57 EX_ENTRY(98b, 99b)
>> 58
>> 59 #define ST(insn, dst, offset, reg, label) \
>> 60 98: insn %reg, [%dst + (offset)]; \
>> 61 .section .fixup,ALLOC,EXECINSTR; \
>> 62 99: ba label; \
>> 63 mov offset, %g5; \
>> 64 EX_ENTRY(98b, 99b)
>> 65
>> 66 /* Both these macros have to start with exactly the same insn */
>> 67 /* left: g7 + (g1 % 128) - offset */
>> 68 #define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
>> 69 LD(ldd, src, offset + 0x00, t0, bigchunk_fault) \
>> 70 LD(ldd, src, offset + 0x08, t2, bigchunk_fault) \
>> 71 LD(ldd, src, offset + 0x10, t4, bigchunk_fault) \
>> 72 LD(ldd, src, offset + 0x18, t6, bigchunk_fault) \
>> 73 ST(st, dst, offset + 0x00, t0, bigchunk_fault) \
>> 74 ST(st, dst, offset + 0x04, t1, bigchunk_fault) \
>> 75 ST(st, dst, offset + 0x08, t2, bigchunk_fault) \
>> 76 ST(st, dst, offset + 0x0c, t3, bigchunk_fault) \
>> 77 ST(st, dst, offset + 0x10, t4, bigchunk_fault) \
>> 78 ST(st, dst, offset + 0x14, t5, bigchunk_fault) \
>> 79 ST(st, dst, offset + 0x18, t6, bigchunk_fault) \
>> 80 ST(st, dst, offset + 0x1c, t7, bigchunk_fault)
>> 81
>> 82 /* left: g7 + (g1 % 128) - offset */
>> 83 #define MOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
>> 84 LD(ldd, src, offset + 0x00, t0, bigchunk_fault) \
>> 85 LD(ldd, src, offset + 0x08, t2, bigchunk_fault) \
>> 86 LD(ldd, src, offset + 0x10, t4, bigchunk_fault) \
>> 87 LD(ldd, src, offset + 0x18, t6, bigchunk_fault) \
>> 88 ST(std, dst, offset + 0x00, t0, bigchunk_fault) \
>> 89 ST(std, dst, offset + 0x08, t2, bigchunk_fault) \
>> 90 ST(std, dst, offset + 0x10, t4, bigchunk_fault) \
>> 91 ST(std, dst, offset + 0x18, t6, bigchunk_fault)
>> 92
>> 93 .section .fixup,#alloc,#execinstr
>> 94 bigchunk_fault:
>> 95 sub %g7, %g5, %o0
>> 96 and %g1, 127, %g1
>> 97 retl
>> 98 add %o0, %g1, %o0
>> 99
>> 100 /* left: offset + 16 + (g1 % 16) */
>> 101 #define MOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \
>> 102 LD(ldd, src, -(offset + 0x10), t0, lastchunk_fault) \
>> 103 LD(ldd, src, -(offset + 0x08), t2, lastchunk_fault) \
>> 104 ST(st, dst, -(offset + 0x10), t0, lastchunk_fault) \
>> 105 ST(st, dst, -(offset + 0x0c), t1, lastchunk_fault) \
>> 106 ST(st, dst, -(offset + 0x08), t2, lastchunk_fault) \
>> 107 ST(st, dst, -(offset + 0x04), t3, lastchunk_fault)
>> 108
>> 109 .section .fixup,#alloc,#execinstr
>> 110 lastchunk_fault:
>> 111 and %g1, 15, %g1
>> 112 retl
>> 113 sub %g1, %g5, %o0
>> 114
>> 115 /* left: o3 + (o2 % 16) - offset */
>> 116 #define MOVE_HALFCHUNK(src, dst, offset, t0, t1, t2, t3) \
>> 117 LD(lduh, src, offset + 0x00, t0, halfchunk_fault) \
>> 118 LD(lduh, src, offset + 0x02, t1, halfchunk_fault) \
>> 119 LD(lduh, src, offset + 0x04, t2, halfchunk_fault) \
>> 120 LD(lduh, src, offset + 0x06, t3, halfchunk_fault) \
>> 121 ST(sth, dst, offset + 0x00, t0, halfchunk_fault) \
>> 122 ST(sth, dst, offset + 0x02, t1, halfchunk_fault) \
>> 123 ST(sth, dst, offset + 0x04, t2, halfchunk_fault) \
>> 124 ST(sth, dst, offset + 0x06, t3, halfchunk_fault)
>> 125
>> 126 /* left: o3 + (o2 % 16) + offset + 2 */
>> 127 #define MOVE_SHORTCHUNK(src, dst, offset, t0, t1) \
>> 128 LD(ldub, src, -(offset + 0x02), t0, halfchunk_fault) \
>> 129 LD(ldub, src, -(offset + 0x01), t1, halfchunk_fault) \
>> 130 ST(stb, dst, -(offset + 0x02), t0, halfchunk_fault) \
>> 131 ST(stb, dst, -(offset + 0x01), t1, halfchunk_fault)
>> 132
>> 133 .section .fixup,#alloc,#execinstr
>> 134 halfchunk_fault:
>> 135 and %o2, 15, %o2
>> 136 sub %o3, %g5, %o3
>> 137 retl
>> 138 add %o2, %o3, %o0
>> 139
>> 140 /* left: offset + 2 + (o2 % 2) */
>> 141 #define MOVE_LAST_SHORTCHUNK(src, dst, offset, t0, t1) \
>> 142 LD(ldub, src, -(offset + 0x02), t0, last_shortchunk_fault) \
>> 143 LD(ldub, src, -(offset + 0x01), t1, last_shortchunk_fault) \
>> 144 ST(stb, dst, -(offset + 0x02), t0, last_shortchunk_fault) \
>> 145 ST(stb, dst, -(offset + 0x01), t1, last_shortchunk_fault)
>> 146
>> 147 .section .fixup,#alloc,#execinstr
>> 148 last_shortchunk_fault:
>> 149 and %o2, 1, %o2
>> 150 retl
>> 151 sub %o2, %g5, %o0
>> 152
>> 153 .text
>> 154 .align 4
>> 155
>> 156 .globl __copy_user_begin
>> 157 __copy_user_begin:
>> 158
>> 159 .globl __copy_user
>> 160 EXPORT_SYMBOL(__copy_user)
>> 161 dword_align:
>> 162 andcc %o1, 1, %g0
>> 163 be 4f
>> 164 andcc %o1, 2, %g0
>> 165
>> 166 EXO2(ldub [%o1], %g2)
>> 167 add %o1, 1, %o1
>> 168 EXO2(stb %g2, [%o0])
>> 169 sub %o2, 1, %o2
>> 170 bne 3f
>> 171 add %o0, 1, %o0
>> 172
>> 173 EXO2(lduh [%o1], %g2)
>> 174 add %o1, 2, %o1
>> 175 EXO2(sth %g2, [%o0])
>> 176 sub %o2, 2, %o2
>> 177 b 3f
>> 178 add %o0, 2, %o0
>> 179 4:
>> 180 EXO2(lduh [%o1], %g2)
>> 181 add %o1, 2, %o1
>> 182 EXO2(sth %g2, [%o0])
>> 183 sub %o2, 2, %o2
>> 184 b 3f
>> 185 add %o0, 2, %o0
>> 186
>> 187 __copy_user: /* %o0=dst %o1=src %o2=len */
>> 188 xor %o0, %o1, %o4
>> 189 1:
>> 190 andcc %o4, 3, %o5
>> 191 2:
>> 192 bne cannot_optimize
>> 193 cmp %o2, 15
>> 194
>> 195 bleu short_aligned_end
>> 196 andcc %o1, 3, %g0
>> 197
>> 198 bne dword_align
>> 199 3:
>> 200 andcc %o1, 4, %g0
>> 201
>> 202 be 2f
>> 203 mov %o2, %g1
>> 204
>> 205 EXO2(ld [%o1], %o4)
>> 206 sub %g1, 4, %g1
>> 207 EXO2(st %o4, [%o0])
>> 208 add %o1, 4, %o1
>> 209 add %o0, 4, %o0
>> 210 2:
>> 211 andcc %g1, 0xffffff80, %g7
>> 212 be 3f
>> 213 andcc %o0, 4, %g0
>> 214
>> 215 be ldd_std + 4
>> 216 5:
>> 217 MOVE_BIGCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5)
>> 218 MOVE_BIGCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
>> 219 MOVE_BIGCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
>> 220 MOVE_BIGCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
>> 221 subcc %g7, 128, %g7
>> 222 add %o1, 128, %o1
>> 223 bne 5b
>> 224 add %o0, 128, %o0
>> 225 3:
>> 226 andcc %g1, 0x70, %g7
>> 227 be copy_user_table_end
>> 228 andcc %g1, 8, %g0
>> 229
>> 230 sethi %hi(copy_user_table_end), %o5
>> 231 srl %g7, 1, %o4
>> 232 add %g7, %o4, %o4
>> 233 add %o1, %g7, %o1
>> 234 sub %o5, %o4, %o5
>> 235 jmpl %o5 + %lo(copy_user_table_end), %g0
>> 236 add %o0, %g7, %o0
>> 237
>> 238 MOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g4, g5)
>> 239 MOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g4, g5)
>> 240 MOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g4, g5)
>> 241 MOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g4, g5)
>> 242 MOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g4, g5)
>> 243 MOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g4, g5)
>> 244 MOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g4, g5)
>> 245 copy_user_table_end:
>> 246 be copy_user_last7
>> 247 andcc %g1, 4, %g0
>> 248
>> 249 EX(ldd [%o1], %g2, and %g1, 0xf)
>> 250 add %o0, 8, %o0
>> 251 add %o1, 8, %o1
>> 252 EX(st %g2, [%o0 - 0x08], and %g1, 0xf)
>> 253 EX2(st %g3, [%o0 - 0x04], and %g1, 0xf, %g1, sub %g1, 4)
>> 254 copy_user_last7:
>> 255 be 1f
>> 256 andcc %g1, 2, %g0
>> 257
>> 258 EX(ld [%o1], %g2, and %g1, 7)
>> 259 add %o1, 4, %o1
>> 260 EX(st %g2, [%o0], and %g1, 7)
>> 261 add %o0, 4, %o0
>> 262 1:
>> 263 be 1f
>> 264 andcc %g1, 1, %g0
>> 265
>> 266 EX(lduh [%o1], %g2, and %g1, 3)
>> 267 add %o1, 2, %o1
>> 268 EX(sth %g2, [%o0], and %g1, 3)
>> 269 add %o0, 2, %o0
>> 270 1:
>> 271 be 1f
>> 272 nop
>> 273
>> 274 EX(ldub [%o1], %g2, add %g0, 1)
>> 275 EX(stb %g2, [%o0], add %g0, 1)
>> 276 1:
>> 277 retl
>> 278 clr %o0
>> 279
>> 280 ldd_std:
>> 281 MOVE_BIGALIGNCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5)
>> 282 MOVE_BIGALIGNCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
>> 283 MOVE_BIGALIGNCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
>> 284 MOVE_BIGALIGNCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
>> 285 subcc %g7, 128, %g7
>> 286 add %o1, 128, %o1
>> 287 bne ldd_std
>> 288 add %o0, 128, %o0
>> 289
>> 290 andcc %g1, 0x70, %g7
>> 291 be copy_user_table_end
>> 292 andcc %g1, 8, %g0
>> 293
>> 294 sethi %hi(copy_user_table_end), %o5
>> 295 srl %g7, 1, %o4
>> 296 add %g7, %o4, %o4
>> 297 add %o1, %g7, %o1
>> 298 sub %o5, %o4, %o5
>> 299 jmpl %o5 + %lo(copy_user_table_end), %g0
>> 300 add %o0, %g7, %o0
>> 301
>> 302 cannot_optimize:
>> 303 bleu short_end
>> 304 cmp %o5, 2
>> 305
>> 306 bne byte_chunk
>> 307 and %o2, 0xfffffff0, %o3
>> 308
>> 309 andcc %o1, 1, %g0
>> 310 be 10f
>> 311 nop
>> 312
>> 313 EXO2(ldub [%o1], %g2)
>> 314 add %o1, 1, %o1
>> 315 EXO2(stb %g2, [%o0])
>> 316 sub %o2, 1, %o2
>> 317 andcc %o2, 0xfffffff0, %o3
>> 318 be short_end
>> 319 add %o0, 1, %o0
>> 320 10:
>> 321 MOVE_HALFCHUNK(o1, o0, 0x00, g2, g3, g4, g5)
>> 322 MOVE_HALFCHUNK(o1, o0, 0x08, g2, g3, g4, g5)
>> 323 subcc %o3, 0x10, %o3
>> 324 add %o1, 0x10, %o1
>> 325 bne 10b
>> 326 add %o0, 0x10, %o0
>> 327 b 2f
>> 328 and %o2, 0xe, %o3
>> 329
>> 330 byte_chunk:
>> 331 MOVE_SHORTCHUNK(o1, o0, -0x02, g2, g3)
>> 332 MOVE_SHORTCHUNK(o1, o0, -0x04, g2, g3)
>> 333 MOVE_SHORTCHUNK(o1, o0, -0x06, g2, g3)
>> 334 MOVE_SHORTCHUNK(o1, o0, -0x08, g2, g3)
>> 335 MOVE_SHORTCHUNK(o1, o0, -0x0a, g2, g3)
>> 336 MOVE_SHORTCHUNK(o1, o0, -0x0c, g2, g3)
>> 337 MOVE_SHORTCHUNK(o1, o0, -0x0e, g2, g3)
>> 338 MOVE_SHORTCHUNK(o1, o0, -0x10, g2, g3)
>> 339 subcc %o3, 0x10, %o3
>> 340 add %o1, 0x10, %o1
>> 341 bne byte_chunk
>> 342 add %o0, 0x10, %o0
>> 343
>> 344 short_end:
>> 345 and %o2, 0xe, %o3
>> 346 2:
>> 347 sethi %hi(short_table_end), %o5
>> 348 sll %o3, 3, %o4
>> 349 add %o0, %o3, %o0
>> 350 sub %o5, %o4, %o5
>> 351 add %o1, %o3, %o1
>> 352 jmpl %o5 + %lo(short_table_end), %g0
>> 353 andcc %o2, 1, %g0
>> 354 MOVE_LAST_SHORTCHUNK(o1, o0, 0x0c, g2, g3)
>> 355 MOVE_LAST_SHORTCHUNK(o1, o0, 0x0a, g2, g3)
>> 356 MOVE_LAST_SHORTCHUNK(o1, o0, 0x08, g2, g3)
>> 357 MOVE_LAST_SHORTCHUNK(o1, o0, 0x06, g2, g3)
>> 358 MOVE_LAST_SHORTCHUNK(o1, o0, 0x04, g2, g3)
>> 359 MOVE_LAST_SHORTCHUNK(o1, o0, 0x02, g2, g3)
>> 360 MOVE_LAST_SHORTCHUNK(o1, o0, 0x00, g2, g3)
>> 361 short_table_end:
>> 362 be 1f
>> 363 nop
>> 364 EX(ldub [%o1], %g2, add %g0, 1)
>> 365 EX(stb %g2, [%o0], add %g0, 1)
>> 366 1:
>> 367 retl
>> 368 clr %o0
>> 369
>> 370 short_aligned_end:
>> 371 bne short_end
>> 372 andcc %o2, 8, %g0
>> 373
>> 374 be 1f
>> 375 andcc %o2, 4, %g0
>> 376
>> 377 EXO2(ld [%o1 + 0x00], %g2)
>> 378 EXO2(ld [%o1 + 0x04], %g3)
>> 379 add %o1, 8, %o1
>> 380 EXO2(st %g2, [%o0 + 0x00])
>> 381 EX(st %g3, [%o0 + 0x04], sub %o2, 4)
>> 382 add %o0, 8, %o0
>> 383 1:
>> 384 b copy_user_last7
>> 385 mov %o2, %g1
16 386
17 /* Allow an exception for an insn; exit if we !! 387 .section .fixup,#alloc,#execinstr
18 #define EXI(x,y...) \ !! 388 .align 4
19 99: x,##y; \ !! 389 97:
20 .section __ex_table,"a"; \ !! 390 retl
21 .long 99b - .; \ !! 391 mov %o2, %o0
22 lda $31, $exitin-99b($31); \ <<
23 .previous <<
24 <<
25 #define EXO(x,y...) \ <<
26 99: x,##y; \ <<
27 .section __ex_table,"a"; \ <<
28 .long 99b - .; \ <<
29 lda $31, $exitout-99b($31); \ <<
30 .previous <<
31 <<
32 .set noat <<
33 .align 4 <<
34 .globl __copy_user <<
35 .ent __copy_user <<
36 __copy_user: <<
37 .prologue 0 <<
38 mov $18,$0 <<
39 and $16,7,$3 <<
40 beq $0,$35 <<
41 beq $3,$36 <<
42 subq $3,8,$3 <<
43 .align 4 <<
44 $37: <<
45 EXI( ldq_u $1,0($17) ) <<
46 EXO( ldq_u $2,0($16) ) <<
47 extbl $1,$17,$1 <<
48 mskbl $2,$16,$2 <<
49 insbl $1,$16,$1 <<
50 addq $3,1,$3 <<
51 bis $1,$2,$1 <<
52 EXO( stq_u $1,0($16) ) <<
53 subq $0,1,$0 <<
54 addq $16,1,$16 <<
55 addq $17,1,$17 <<
56 beq $0,$41 <<
57 bne $3,$37 <<
58 $36: <<
59 and $17,7,$1 <<
60 bic $0,7,$4 <<
61 beq $1,$43 <<
62 beq $4,$48 <<
63 EXI( ldq_u $3,0($17) ) <<
64 .align 4 <<
65 $50: <<
66 EXI( ldq_u $2,8($17) ) <<
67 subq $4,8,$4 <<
68 extql $3,$17,$3 <<
69 extqh $2,$17,$1 <<
70 bis $3,$1,$1 <<
71 EXO( stq $1,0($16) ) <<
72 addq $17,8,$17 <<
73 subq $0,8,$0 <<
74 addq $16,8,$16 <<
75 bis $2,$2,$3 <<
76 bne $4,$50 <<
77 $48: <<
78 beq $0,$41 <<
79 .align 4 <<
80 $57: <<
81 EXI( ldq_u $1,0($17) ) <<
82 EXO( ldq_u $2,0($16) ) <<
83 extbl $1,$17,$1 <<
84 mskbl $2,$16,$2 <<
85 insbl $1,$16,$1 <<
86 bis $1,$2,$1 <<
87 EXO( stq_u $1,0($16) ) <<
88 subq $0,1,$0 <<
89 addq $16,1,$16 <<
90 addq $17,1,$17 <<
91 bne $0,$57 <<
92 br $31,$41 <<
93 .align 4 <<
94 $43: <<
95 beq $4,$65 <<
96 .align 4 <<
97 $66: <<
98 EXI( ldq $1,0($17) ) <<
99 subq $4,8,$4 <<
100 EXO( stq $1,0($16) ) <<
101 addq $17,8,$17 <<
102 subq $0,8,$0 <<
103 addq $16,8,$16 <<
104 bne $4,$66 <<
105 $65: <<
106 beq $0,$41 <<
107 EXI( ldq $2,0($17) ) <<
108 EXO( ldq $1,0($16) ) <<
109 mskql $2,$0,$2 <<
110 mskqh $1,$0,$1 <<
111 bis $2,$1,$2 <<
112 EXO( stq $2,0($16) ) <<
113 bis $31,$31,$0 <<
114 $41: <<
115 $35: <<
116 $exitin: <<
117 $exitout: <<
118 ret $31,($26),1 <<
119 392
120 .end __copy_user !! 393 .globl __copy_user_end
121 EXPORT_SYMBOL(__copy_user) !! 394 __copy_user_end:
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.