Linux/arch/sparc/lib/copy

Version: ~ [ linux-6.11-rc3 ] ~ [ linux-6.10.4 ] ~ [ linux-6.9.12 ] ~ [ linux-6.8.12 ] ~ [ linux-6.7.12 ] ~ [ linux-6.6.45 ] ~ [ linux-6.5.13 ] ~ [ linux-6.4.16 ] ~ [ linux-6.3.13 ] ~ [ linux-6.2.16 ] ~ [ linux-6.1.104 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.164 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.223 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.281 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.319 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.336 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.4.302 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

Diff markup

1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* copy_user.S: Sparc optimized copy_from_user !! 2 /* >> 3 * arch/alpha/lib/copy_user.S 3 * 4 * 4 * Copyright(C) 1995 Linus Torvalds !! 5 * Copy to/from user space, handling exceptions as we go.. This 5 * Copyright(C) 1996 David S. Miller !! 6 * isn't exactly pretty. 6 * Copyright(C) 1996 Eddie C. Dost << 7 * Copyright(C) 1996,1998 Jakub Jelinek << 8 * 7 * 9 * derived from: !! 8 * This is essentially the same as "memcpy()", but with a few twists. 10 * e-mail between David and Eddie. !! 9 * Notably, we have to make sure that $0 is always up-to-date and 11 * !! 10 * contains the right "bytes left to copy" value (and that it is updated 12 * Returns 0 if successful, otherwise count of !! 11 * only _after_ a successful copy). There is also some rather minor >> 12 * exception setup stuff.. 13 */ 13 */ 14 14 15 #include <linux/export.h> 15 #include <linux/export.h> 16 #include <asm/ptrace.h> << 17 #include <asm/asmmacro.h> << 18 #include <asm/page.h> << 19 #include <asm/thread_info.h> << 20 << 21 /* Work around cpp -rob */ << 22 #define ALLOC #alloc << 23 #define EXECINSTR #execinstr << 24 << 25 #define EX_ENTRY(l1, l2) << 26 .section __ex_table,ALLOC; << 27 .align 4; << 28 .word l1, l2; << 29 .text; << 30 << 31 #define EX(x,y,a,b) << 32 98: x,y; << 33 .section .fixup,ALLOC,EXECINSTR; << 34 .align 4; << 35 99: retl; << 36 a, b, %o0; << 37 EX_ENTRY(98b, 99b) << 38 << 39 #define EX2(x,y,c,d,e,a,b) << 40 98: x,y; << 41 .section .fixup,ALLOC,EXECINSTR; << 42 .align 4; << 43 99: c, d, e; << 44 retl; << 45 a, b, %o0; << 46 EX_ENTRY(98b, 99b) << 47 << 48 #define EXO2(x,y) << 49 98: x, y; << 50 EX_ENTRY(98b, 97f) << 51 << 52 #define LD(insn, src, offset, reg, label) << 53 98: insn [%src + (offset)], %reg; << 54 .section .fixup,ALLOC,EXECINSTR; << 55 99: ba label; << 56 mov offset, %g5; << 57 EX_ENTRY(98b, 99b) << 58 << 59 #define ST(insn, dst, offset, reg, label) << 60 98: insn %reg, [%dst + (offset)]; << 61 .section .fixup,ALLOC,EXECINSTR; << 62 99: ba label; << 63 mov offset, %g5; << 64 EX_ENTRY(98b, 99b) << 65 << 66 /* Both these macros have to start with exactl << 67 /* left: g7 + (g1 % 128) - offset */ << 68 #define MOVE_BIGCHUNK(src, dst, offset, t0, t1 << 69 LD(ldd, src, offset + 0x00, t0, bigchu << 70 LD(ldd, src, offset + 0x08, t2, bigchu << 71 LD(ldd, src, offset + 0x10, t4, bigchu << 72 LD(ldd, src, offset + 0x18, t6, bigchu << 73 ST(st, dst, offset + 0x00, t0, bigchun << 74 ST(st, dst, offset + 0x04, t1, bigchun << 75 ST(st, dst, offset + 0x08, t2, bigchun << 76 ST(st, dst, offset + 0x0c, t3, bigchun << 77 ST(st, dst, offset + 0x10, t4, bigchun << 78 ST(st, dst, offset + 0x14, t5, bigchun << 79 ST(st, dst, offset + 0x18, t6, bigchun << 80 ST(st, dst, offset + 0x1c, t7, bigchun << 81 << 82 /* left: g7 + (g1 % 128) - offset */ << 83 #define MOVE_BIGALIGNCHUNK(src, dst, offset, t << 84 LD(ldd, src, offset + 0x00, t0, bigchu << 85 LD(ldd, src, offset + 0x08, t2, bigchu << 86 LD(ldd, src, offset + 0x10, t4, bigchu << 87 LD(ldd, src, offset + 0x18, t6, bigchu << 88 ST(std, dst, offset + 0x00, t0, bigchu << 89 ST(std, dst, offset + 0x08, t2, bigchu << 90 ST(std, dst, offset + 0x10, t4, bigchu << 91 ST(std, dst, offset + 0x18, t6, bigchu << 92 << 93 .section .fixup,#alloc,#execinstr << 94 bigchunk_fault: << 95 sub %g7, %g5, %o0 << 96 and %g1, 127, %g1 << 97 retl << 98 add %o0, %g1, %o0 << 99 << 100 /* left: offset + 16 + (g1 % 16) */ << 101 #define MOVE_LASTCHUNK(src, dst, offset, t0, t << 102 LD(ldd, src, -(offset + 0x10), t0, las << 103 LD(ldd, src, -(offset + 0x08), t2, las << 104 ST(st, dst, -(offset + 0x10), t0, last << 105 ST(st, dst, -(offset + 0x0c), t1, last << 106 ST(st, dst, -(offset + 0x08), t2, last << 107 ST(st, dst, -(offset + 0x04), t3, last << 108 << 109 .section .fixup,#alloc,#execinstr << 110 lastchunk_fault: << 111 and %g1, 15, %g1 << 112 retl << 113 sub %g1, %g5, %o0 << 114 << 115 /* left: o3 + (o2 % 16) - offset */ << 116 #define MOVE_HALFCHUNK(src, dst, offset, t0, t << 117 LD(lduh, src, offset + 0x00, t0, halfc << 118 LD(lduh, src, offset + 0x02, t1, halfc << 119 LD(lduh, src, offset + 0x04, t2, halfc << 120 LD(lduh, src, offset + 0x06, t3, halfc << 121 ST(sth, dst, offset + 0x00, t0, halfch << 122 ST(sth, dst, offset + 0x02, t1, halfch << 123 ST(sth, dst, offset + 0x04, t2, halfch << 124 ST(sth, dst, offset + 0x06, t3, halfch << 125 << 126 /* left: o3 + (o2 % 16) + offset + 2 */ << 127 #define MOVE_SHORTCHUNK(src, dst, offset, t0, << 128 LD(ldub, src, -(offset + 0x02), t0, ha << 129 LD(ldub, src, -(offset + 0x01), t1, ha << 130 ST(stb, dst, -(offset + 0x02), t0, hal << 131 ST(stb, dst, -(offset + 0x01), t1, hal << 132 << 133 .section .fixup,#alloc,#execinstr << 134 halfchunk_fault: << 135 and %o2, 15, %o2 << 136 sub %o3, %g5, %o3 << 137 retl << 138 add %o2, %o3, %o0 << 139 << 140 /* left: offset + 2 + (o2 % 2) */ << 141 #define MOVE_LAST_SHORTCHUNK(src, dst, offset, << 142 LD(ldub, src, -(offset + 0x02), t0, la << 143 LD(ldub, src, -(offset + 0x01), t1, la << 144 ST(stb, dst, -(offset + 0x02), t0, las << 145 ST(stb, dst, -(offset + 0x01), t1, las << 146 << 147 .section .fixup,#alloc,#execinstr << 148 last_shortchunk_fault: << 149 and %o2, 1, %o2 << 150 retl << 151 sub %o2, %g5, %o0 << 152 << 153 .text << 154 .align 4 << 155 << 156 .globl __copy_user_begin << 157 __copy_user_begin: << 158 << 159 .globl __copy_user << 160 EXPORT_SYMBOL(__copy_user) << 161 dword_align: << 162 andcc %o1, 1, %g0 << 163 be 4f << 164 andcc %o1, 2, %g0 << 165 << 166 EXO2(ldub [%o1], %g2) << 167 add %o1, 1, %o1 << 168 EXO2(stb %g2, [%o0]) << 169 sub %o2, 1, %o2 << 170 bne 3f << 171 add %o0, 1, %o0 << 172 << 173 EXO2(lduh [%o1], %g2) << 174 add %o1, 2, %o1 << 175 EXO2(sth %g2, [%o0]) << 176 sub %o2, 2, %o2 << 177 b 3f << 178 add %o0, 2, %o0 << 179 4: << 180 EXO2(lduh [%o1], %g2) << 181 add %o1, 2, %o1 << 182 EXO2(sth %g2, [%o0]) << 183 sub %o2, 2, %o2 << 184 b 3f << 185 add %o0, 2, %o0 << 186 << 187 __copy_user: /* %o0=dst %o1=src %o2=len */ << 188 xor %o0, %o1, %o4 << 189 1: << 190 andcc %o4, 3, %o5 << 191 2: << 192 bne cannot_optimize << 193 cmp %o2, 15 << 194 << 195 bleu short_aligned_end << 196 andcc %o1, 3, %g0 << 197 << 198 bne dword_align << 199 3: << 200 andcc %o1, 4, %g0 << 201 << 202 be 2f << 203 mov %o2, %g1 << 204 << 205 EXO2(ld [%o1], %o4) << 206 sub %g1, 4, %g1 << 207 EXO2(st %o4, [%o0]) << 208 add %o1, 4, %o1 << 209 add %o0, 4, %o0 << 210 2: << 211 andcc %g1, 0xffffff80, %g7 << 212 be 3f << 213 andcc %o0, 4, %g0 << 214 << 215 be ldd_std + 4 << 216 5: << 217 MOVE_BIGCHUNK(o1, o0, 0x00, o2, o3, o4 << 218 MOVE_BIGCHUNK(o1, o0, 0x20, o2, o3, o4 << 219 MOVE_BIGCHUNK(o1, o0, 0x40, o2, o3, o4 << 220 MOVE_BIGCHUNK(o1, o0, 0x60, o2, o3, o4 << 221 subcc %g7, 128, %g7 << 222 add %o1, 128, %o1 << 223 bne 5b << 224 add %o0, 128, %o0 << 225 3: << 226 andcc %g1, 0x70, %g7 << 227 be copy_user_table_end << 228 andcc %g1, 8, %g0 << 229 << 230 sethi %hi(copy_user_table_end), %o5 << 231 srl %g7, 1, %o4 << 232 add %g7, %o4, %o4 << 233 add %o1, %g7, %o1 << 234 sub %o5, %o4, %o5 << 235 jmpl %o5 + %lo(copy_user_table_end) << 236 add %o0, %g7, %o0 << 237 << 238 MOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g << 239 MOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g << 240 MOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g << 241 MOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g << 242 MOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g << 243 MOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g << 244 MOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g << 245 copy_user_table_end: << 246 be copy_user_last7 << 247 andcc %g1, 4, %g0 << 248 << 249 EX(ldd [%o1], %g2, and %g1, 0xf) << 250 add %o0, 8, %o0 << 251 add %o1, 8, %o1 << 252 EX(st %g2, [%o0 - 0x08], and %g1, 0x << 253 EX2(st %g3, [%o0 - 0x04], and %g1, 0x << 254 copy_user_last7: << 255 be 1f << 256 andcc %g1, 2, %g0 << 257 << 258 EX(ld [%o1], %g2, and %g1, 7) << 259 add %o1, 4, %o1 << 260 EX(st %g2, [%o0], and %g1, 7) << 261 add %o0, 4, %o0 << 262 1: << 263 be 1f << 264 andcc %g1, 1, %g0 << 265 << 266 EX(lduh [%o1], %g2, and %g1, 3) << 267 add %o1, 2, %o1 << 268 EX(sth %g2, [%o0], and %g1, 3) << 269 add %o0, 2, %o0 << 270 1: << 271 be 1f << 272 nop << 273 << 274 EX(ldub [%o1], %g2, add %g0, 1) << 275 EX(stb %g2, [%o0], add %g0, 1) << 276 1: << 277 retl << 278 clr %o0 << 279 << 280 ldd_std: << 281 MOVE_BIGALIGNCHUNK(o1, o0, 0x00, o2, o << 282 MOVE_BIGALIGNCHUNK(o1, o0, 0x20, o2, o << 283 MOVE_BIGALIGNCHUNK(o1, o0, 0x40, o2, o << 284 MOVE_BIGALIGNCHUNK(o1, o0, 0x60, o2, o << 285 subcc %g7, 128, %g7 << 286 add %o1, 128, %o1 << 287 bne ldd_std << 288 add %o0, 128, %o0 << 289 << 290 andcc %g1, 0x70, %g7 << 291 be copy_user_table_end << 292 andcc %g1, 8, %g0 << 293 << 294 sethi %hi(copy_user_table_end), %o5 << 295 srl %g7, 1, %o4 << 296 add %g7, %o4, %o4 << 297 add %o1, %g7, %o1 << 298 sub %o5, %o4, %o5 << 299 jmpl %o5 + %lo(copy_user_table_end) << 300 add %o0, %g7, %o0 << 301 << 302 cannot_optimize: << 303 bleu short_end << 304 cmp %o5, 2 << 305 << 306 bne byte_chunk << 307 and %o2, 0xfffffff0, %o3 << 308 << 309 andcc %o1, 1, %g0 << 310 be 10f << 311 nop << 312 << 313 EXO2(ldub [%o1], %g2) << 314 add %o1, 1, %o1 << 315 EXO2(stb %g2, [%o0]) << 316 sub %o2, 1, %o2 << 317 andcc %o2, 0xfffffff0, %o3 << 318 be short_end << 319 add %o0, 1, %o0 << 320 10: << 321 MOVE_HALFCHUNK(o1, o0, 0x00, g2, g3, g << 322 MOVE_HALFCHUNK(o1, o0, 0x08, g2, g3, g << 323 subcc %o3, 0x10, %o3 << 324 add %o1, 0x10, %o1 << 325 bne 10b << 326 add %o0, 0x10, %o0 << 327 b 2f << 328 and %o2, 0xe, %o3 << 329 << 330 byte_chunk: << 331 MOVE_SHORTCHUNK(o1, o0, -0x02, g2, g3) << 332 MOVE_SHORTCHUNK(o1, o0, -0x04, g2, g3) << 333 MOVE_SHORTCHUNK(o1, o0, -0x06, g2, g3) << 334 MOVE_SHORTCHUNK(o1, o0, -0x08, g2, g3) << 335 MOVE_SHORTCHUNK(o1, o0, -0x0a, g2, g3) << 336 MOVE_SHORTCHUNK(o1, o0, -0x0c, g2, g3) << 337 MOVE_SHORTCHUNK(o1, o0, -0x0e, g2, g3) << 338 MOVE_SHORTCHUNK(o1, o0, -0x10, g2, g3) << 339 subcc %o3, 0x10, %o3 << 340 add %o1, 0x10, %o1 << 341 bne byte_chunk << 342 add %o0, 0x10, %o0 << 343 << 344 short_end: << 345 and %o2, 0xe, %o3 << 346 2: << 347 sethi %hi(short_table_end), %o5 << 348 sll %o3, 3, %o4 << 349 add %o0, %o3, %o0 << 350 sub %o5, %o4, %o5 << 351 add %o1, %o3, %o1 << 352 jmpl %o5 + %lo(short_table_end), %g << 353 andcc %o2, 1, %g0 << 354 MOVE_LAST_SHORTCHUNK(o1, o0, 0x0c, g2, << 355 MOVE_LAST_SHORTCHUNK(o1, o0, 0x0a, g2, << 356 MOVE_LAST_SHORTCHUNK(o1, o0, 0x08, g2, << 357 MOVE_LAST_SHORTCHUNK(o1, o0, 0x06, g2, << 358 MOVE_LAST_SHORTCHUNK(o1, o0, 0x04, g2, << 359 MOVE_LAST_SHORTCHUNK(o1, o0, 0x02, g2, << 360 MOVE_LAST_SHORTCHUNK(o1, o0, 0x00, g2, << 361 short_table_end: << 362 be 1f << 363 nop << 364 EX(ldub [%o1], %g2, add %g0, 1) << 365 EX(stb %g2, [%o0], add %g0, 1) << 366 1: << 367 retl << 368 clr %o0 << 369 << 370 short_aligned_end: << 371 bne short_end << 372 andcc %o2, 8, %g0 << 373 << 374 be 1f << 375 andcc %o2, 4, %g0 << 376 << 377 EXO2(ld [%o1 + 0x00], %g2) << 378 EXO2(ld [%o1 + 0x04], %g3) << 379 add %o1, 8, %o1 << 380 EXO2(st %g2, [%o0 + 0x00]) << 381 EX(st %g3, [%o0 + 0x04], sub %o2, 4) << 382 add %o0, 8, %o0 << 383 1: << 384 b copy_user_last7 << 385 mov %o2, %g1 << 386 16 387 .section .fixup,#alloc,#execinstr !! 17 /* Allow an exception for an insn; exit if we get one. */ 388 .align 4 !! 18 #define EXI(x,y...) \ 389 97: !! 19 99: x,##y; \ 390 retl !! 20 .section __ex_table,"a"; \ 391 mov %o2, %o0 !! 21 .long 99b - .; \ >> 22 lda $31, $exitin-99b($31); \ >> 23 .previous >> 24 >> 25 #define EXO(x,y...) \ >> 26 99: x,##y; \ >> 27 .section __ex_table,"a"; \ >> 28 .long 99b - .; \ >> 29 lda $31, $exitout-99b($31); \ >> 30 .previous >> 31 >> 32 .set noat >> 33 .align 4 >> 34 .globl __copy_user >> 35 .ent __copy_user >> 36 __copy_user: >> 37 .prologue 0 >> 38 mov $18,$0 >> 39 and $16,7,$3 >> 40 beq $0,$35 >> 41 beq $3,$36 >> 42 subq $3,8,$3 >> 43 .align 4 >> 44 $37: >> 45 EXI( ldq_u $1,0($17) ) >> 46 EXO( ldq_u $2,0($16) ) >> 47 extbl $1,$17,$1 >> 48 mskbl $2,$16,$2 >> 49 insbl $1,$16,$1 >> 50 addq $3,1,$3 >> 51 bis $1,$2,$1 >> 52 EXO( stq_u $1,0($16) ) >> 53 subq $0,1,$0 >> 54 addq $16,1,$16 >> 55 addq $17,1,$17 >> 56 beq $0,$41 >> 57 bne $3,$37 >> 58 $36: >> 59 and $17,7,$1 >> 60 bic $0,7,$4 >> 61 beq $1,$43 >> 62 beq $4,$48 >> 63 EXI( ldq_u $3,0($17) ) >> 64 .align 4 >> 65 $50: >> 66 EXI( ldq_u $2,8($17) ) >> 67 subq $4,8,$4 >> 68 extql $3,$17,$3 >> 69 extqh $2,$17,$1 >> 70 bis $3,$1,$1 >> 71 EXO( stq $1,0($16) ) >> 72 addq $17,8,$17 >> 73 subq $0,8,$0 >> 74 addq $16,8,$16 >> 75 bis $2,$2,$3 >> 76 bne $4,$50 >> 77 $48: >> 78 beq $0,$41 >> 79 .align 4 >> 80 $57: >> 81 EXI( ldq_u $1,0($17) ) >> 82 EXO( ldq_u $2,0($16) ) >> 83 extbl $1,$17,$1 >> 84 mskbl $2,$16,$2 >> 85 insbl $1,$16,$1 >> 86 bis $1,$2,$1 >> 87 EXO( stq_u $1,0($16) ) >> 88 subq $0,1,$0 >> 89 addq $16,1,$16 >> 90 addq $17,1,$17 >> 91 bne $0,$57 >> 92 br $31,$41 >> 93 .align 4 >> 94 $43: >> 95 beq $4,$65 >> 96 .align 4 >> 97 $66: >> 98 EXI( ldq $1,0($17) ) >> 99 subq $4,8,$4 >> 100 EXO( stq $1,0($16) ) >> 101 addq $17,8,$17 >> 102 subq $0,8,$0 >> 103 addq $16,8,$16 >> 104 bne $4,$66 >> 105 $65: >> 106 beq $0,$41 >> 107 EXI( ldq $2,0($17) ) >> 108 EXO( ldq $1,0($16) ) >> 109 mskql $2,$0,$2 >> 110 mskqh $1,$0,$1 >> 111 bis $2,$1,$2 >> 112 EXO( stq $2,0($16) ) >> 113 bis $31,$31,$0 >> 114 $41: >> 115 $35: >> 116 $exitin: >> 117 $exitout: >> 118 ret $31,($26),1 392 119 393 .globl __copy_user_end !! 120 .end __copy_user 394 __copy_user_end: !! 121 EXPORT_SYMBOL(__copy_user)

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

TOMOYO Linux Cross Reference
Linux/arch/sparc/lib/copy_user.S

Diff markup

Differences between /arch/sparc/lib/copy_user.S (Architecture sparc) and /arch/alpha/lib/copy_user.S (Architecture alpha)

TOMOYO Linux Cross Reference Linux/arch/sparc/lib/copy_user.S

Diff markup

Differences between /arch/sparc/lib/copy_user.S (Architecture sparc) and /arch/alpha/lib/copy_user.S (Architecture alpha)

TOMOYO Linux Cross Reference
Linux/arch/sparc/lib/copy_user.S