1 /* SPDX-License-Identifier: GPL-2.0 */ <<
2 /* bzero.S: Simple prefetching memset, bzero, 1 /* bzero.S: Simple prefetching memset, bzero, and clear_user
3 * implementations. 2 * implementations.
4 * 3 *
5 * Copyright (C) 2005 David S. Miller <davem@da 4 * Copyright (C) 2005 David S. Miller <davem@davemloft.net>
6 */ 5 */
7 6
8 #include <linux/export.h> <<
9 #include <linux/linkage.h> <<
10 <<
11 .text 7 .text
12 8
13 ENTRY(memset) /* %o0=buf, %o1=pat, %o2=len */ !! 9 .globl __memset
>> 10 .type __memset, #function
>> 11 __memset: /* %o0=buf, %o1=pat, %o2=len */
>> 12
>> 13 .globl memset
>> 14 .type memset, #function
>> 15 memset: /* %o0=buf, %o1=pat, %o2=len */
14 and %o1, 0xff, %o3 16 and %o1, 0xff, %o3
15 mov %o2, %o1 17 mov %o2, %o1
16 sllx %o3, 8, %g1 18 sllx %o3, 8, %g1
17 or %g1, %o3, %o2 19 or %g1, %o3, %o2
18 sllx %o2, 16, %g1 20 sllx %o2, 16, %g1
19 or %g1, %o2, %o2 21 or %g1, %o2, %o2
20 sllx %o2, 32, %g1 22 sllx %o2, 32, %g1
21 ba,pt %xcc, 1f 23 ba,pt %xcc, 1f
22 or %g1, %o2, %o2 24 or %g1, %o2, %o2
23 25
24 ENTRY(__bzero) /* %o0=buf, %o1=len */ !! 26 .globl __bzero
>> 27 .type __bzero, #function
>> 28 __bzero: /* %o0=buf, %o1=len */
25 clr %o2 29 clr %o2
26 1: mov %o0, %o3 30 1: mov %o0, %o3
27 brz,pn %o1, __bzero_done 31 brz,pn %o1, __bzero_done
28 cmp %o1, 16 32 cmp %o1, 16
29 bl,pn %icc, __bzero_tiny 33 bl,pn %icc, __bzero_tiny
30 prefetch [%o0 + 0x000], #n_writ 34 prefetch [%o0 + 0x000], #n_writes
31 andcc %o0, 0x3, %g0 35 andcc %o0, 0x3, %g0
32 be,pt %icc, 2f 36 be,pt %icc, 2f
33 1: stb %o2, [%o0 + 0x00] 37 1: stb %o2, [%o0 + 0x00]
34 add %o0, 1, %o0 38 add %o0, 1, %o0
35 andcc %o0, 0x3, %g0 39 andcc %o0, 0x3, %g0
36 bne,pn %icc, 1b 40 bne,pn %icc, 1b
37 sub %o1, 1, %o1 41 sub %o1, 1, %o1
38 2: andcc %o0, 0x7, %g0 42 2: andcc %o0, 0x7, %g0
39 be,pt %icc, 3f 43 be,pt %icc, 3f
40 stw %o2, [%o0 + 0x00] 44 stw %o2, [%o0 + 0x00]
41 sub %o1, 4, %o1 45 sub %o1, 4, %o1
42 add %o0, 4, %o0 46 add %o0, 4, %o0
43 3: and %o1, 0x38, %g1 47 3: and %o1, 0x38, %g1
44 cmp %o1, 0x40 48 cmp %o1, 0x40
45 andn %o1, 0x3f, %o4 49 andn %o1, 0x3f, %o4
46 bl,pn %icc, 5f 50 bl,pn %icc, 5f
47 and %o1, 0x7, %o1 51 and %o1, 0x7, %o1
48 prefetch [%o0 + 0x040], #n_writ 52 prefetch [%o0 + 0x040], #n_writes
49 prefetch [%o0 + 0x080], #n_writ 53 prefetch [%o0 + 0x080], #n_writes
50 prefetch [%o0 + 0x0c0], #n_writ 54 prefetch [%o0 + 0x0c0], #n_writes
51 prefetch [%o0 + 0x100], #n_writ 55 prefetch [%o0 + 0x100], #n_writes
52 prefetch [%o0 + 0x140], #n_writ 56 prefetch [%o0 + 0x140], #n_writes
53 4: prefetch [%o0 + 0x180], #n_writ 57 4: prefetch [%o0 + 0x180], #n_writes
54 stx %o2, [%o0 + 0x00] 58 stx %o2, [%o0 + 0x00]
55 stx %o2, [%o0 + 0x08] 59 stx %o2, [%o0 + 0x08]
56 stx %o2, [%o0 + 0x10] 60 stx %o2, [%o0 + 0x10]
57 stx %o2, [%o0 + 0x18] 61 stx %o2, [%o0 + 0x18]
58 stx %o2, [%o0 + 0x20] 62 stx %o2, [%o0 + 0x20]
59 stx %o2, [%o0 + 0x28] 63 stx %o2, [%o0 + 0x28]
60 stx %o2, [%o0 + 0x30] 64 stx %o2, [%o0 + 0x30]
61 stx %o2, [%o0 + 0x38] 65 stx %o2, [%o0 + 0x38]
62 subcc %o4, 0x40, %o4 66 subcc %o4, 0x40, %o4
63 bne,pt %icc, 4b 67 bne,pt %icc, 4b
64 add %o0, 0x40, %o0 68 add %o0, 0x40, %o0
65 brz,pn %g1, 6f 69 brz,pn %g1, 6f
66 nop 70 nop
67 5: stx %o2, [%o0 + 0x00] 71 5: stx %o2, [%o0 + 0x00]
68 subcc %g1, 8, %g1 72 subcc %g1, 8, %g1
69 bne,pt %icc, 5b 73 bne,pt %icc, 5b
70 add %o0, 0x8, %o0 74 add %o0, 0x8, %o0
71 6: brz,pt %o1, __bzero_done 75 6: brz,pt %o1, __bzero_done
72 nop 76 nop
73 __bzero_tiny: 77 __bzero_tiny:
74 1: stb %o2, [%o0 + 0x00] 78 1: stb %o2, [%o0 + 0x00]
75 subcc %o1, 1, %o1 79 subcc %o1, 1, %o1
76 bne,pt %icc, 1b 80 bne,pt %icc, 1b
77 add %o0, 1, %o0 81 add %o0, 1, %o0
78 __bzero_done: 82 __bzero_done:
79 retl 83 retl
80 mov %o3, %o0 84 mov %o3, %o0
81 ENDPROC(__bzero) !! 85 .size __bzero, .-__bzero
82 ENDPROC(memset) !! 86 .size __memset, .-__memset
83 EXPORT_SYMBOL(__bzero) !! 87 .size memset, .-memset
84 EXPORT_SYMBOL(memset) <<
85 88
86 #define EX_ST(x,y) \ 89 #define EX_ST(x,y) \
87 98: x,y; \ 90 98: x,y; \
88 .section __ex_table,"a";\ 91 .section __ex_table,"a";\
89 .align 4; \ 92 .align 4; \
90 .word 98b, __retl_o1; \ 93 .word 98b, __retl_o1; \
91 .text; \ 94 .text; \
92 .align 4; 95 .align 4;
93 96
94 ENTRY(__clear_user) /* %o0=buf, %o1=len */ !! 97 .globl __clear_user
>> 98 .type __clear_user, #function
>> 99 __clear_user: /* %o0=buf, %o1=len */
95 brz,pn %o1, __clear_user_done 100 brz,pn %o1, __clear_user_done
96 cmp %o1, 16 101 cmp %o1, 16
97 bl,pn %icc, __clear_user_tin 102 bl,pn %icc, __clear_user_tiny
98 EX_ST(prefetcha [%o0 + 0x00] %asi, #n 103 EX_ST(prefetcha [%o0 + 0x00] %asi, #n_writes)
99 andcc %o0, 0x3, %g0 104 andcc %o0, 0x3, %g0
100 be,pt %icc, 2f 105 be,pt %icc, 2f
101 1: EX_ST(stba %g0, [%o0 + 0x00] %asi 106 1: EX_ST(stba %g0, [%o0 + 0x00] %asi)
102 add %o0, 1, %o0 107 add %o0, 1, %o0
103 andcc %o0, 0x3, %g0 108 andcc %o0, 0x3, %g0
104 bne,pn %icc, 1b 109 bne,pn %icc, 1b
105 sub %o1, 1, %o1 110 sub %o1, 1, %o1
106 2: andcc %o0, 0x7, %g0 111 2: andcc %o0, 0x7, %g0
107 be,pt %icc, 3f 112 be,pt %icc, 3f
108 EX_ST(stwa %g0, [%o0 + 0x00] %asi 113 EX_ST(stwa %g0, [%o0 + 0x00] %asi)
109 sub %o1, 4, %o1 114 sub %o1, 4, %o1
110 add %o0, 4, %o0 115 add %o0, 4, %o0
111 3: and %o1, 0x38, %g1 116 3: and %o1, 0x38, %g1
112 cmp %o1, 0x40 117 cmp %o1, 0x40
113 andn %o1, 0x3f, %o4 118 andn %o1, 0x3f, %o4
114 bl,pn %icc, 5f 119 bl,pn %icc, 5f
115 and %o1, 0x7, %o1 120 and %o1, 0x7, %o1
116 EX_ST(prefetcha [%o0 + 0x040] %asi, #n 121 EX_ST(prefetcha [%o0 + 0x040] %asi, #n_writes)
117 EX_ST(prefetcha [%o0 + 0x080] %asi, #n 122 EX_ST(prefetcha [%o0 + 0x080] %asi, #n_writes)
118 EX_ST(prefetcha [%o0 + 0x0c0] %asi, #n 123 EX_ST(prefetcha [%o0 + 0x0c0] %asi, #n_writes)
119 EX_ST(prefetcha [%o0 + 0x100] %asi, #n 124 EX_ST(prefetcha [%o0 + 0x100] %asi, #n_writes)
120 EX_ST(prefetcha [%o0 + 0x140] %asi, #n 125 EX_ST(prefetcha [%o0 + 0x140] %asi, #n_writes)
121 4: EX_ST(prefetcha [%o0 + 0x180] %asi, #n 126 4: EX_ST(prefetcha [%o0 + 0x180] %asi, #n_writes)
122 EX_ST(stxa %g0, [%o0 + 0x00] %asi 127 EX_ST(stxa %g0, [%o0 + 0x00] %asi)
123 EX_ST(stxa %g0, [%o0 + 0x08] %asi 128 EX_ST(stxa %g0, [%o0 + 0x08] %asi)
124 EX_ST(stxa %g0, [%o0 + 0x10] %asi 129 EX_ST(stxa %g0, [%o0 + 0x10] %asi)
125 EX_ST(stxa %g0, [%o0 + 0x18] %asi 130 EX_ST(stxa %g0, [%o0 + 0x18] %asi)
126 EX_ST(stxa %g0, [%o0 + 0x20] %asi 131 EX_ST(stxa %g0, [%o0 + 0x20] %asi)
127 EX_ST(stxa %g0, [%o0 + 0x28] %asi 132 EX_ST(stxa %g0, [%o0 + 0x28] %asi)
128 EX_ST(stxa %g0, [%o0 + 0x30] %asi 133 EX_ST(stxa %g0, [%o0 + 0x30] %asi)
129 EX_ST(stxa %g0, [%o0 + 0x38] %asi 134 EX_ST(stxa %g0, [%o0 + 0x38] %asi)
130 subcc %o4, 0x40, %o4 135 subcc %o4, 0x40, %o4
131 bne,pt %icc, 4b 136 bne,pt %icc, 4b
132 add %o0, 0x40, %o0 137 add %o0, 0x40, %o0
133 brz,pn %g1, 6f 138 brz,pn %g1, 6f
134 nop 139 nop
135 5: EX_ST(stxa %g0, [%o0 + 0x00] %asi 140 5: EX_ST(stxa %g0, [%o0 + 0x00] %asi)
136 subcc %g1, 8, %g1 141 subcc %g1, 8, %g1
137 bne,pt %icc, 5b 142 bne,pt %icc, 5b
138 add %o0, 0x8, %o0 143 add %o0, 0x8, %o0
139 6: brz,pt %o1, __clear_user_done 144 6: brz,pt %o1, __clear_user_done
140 nop 145 nop
141 __clear_user_tiny: 146 __clear_user_tiny:
142 1: EX_ST(stba %g0, [%o0 + 0x00] %asi 147 1: EX_ST(stba %g0, [%o0 + 0x00] %asi)
143 subcc %o1, 1, %o1 148 subcc %o1, 1, %o1
144 bne,pt %icc, 1b 149 bne,pt %icc, 1b
145 add %o0, 1, %o0 150 add %o0, 1, %o0
146 __clear_user_done: 151 __clear_user_done:
147 retl 152 retl
148 clr %o0 153 clr %o0
149 ENDPROC(__clear_user) !! 154 .size __clear_user, .-__clear_user
150 EXPORT_SYMBOL(__clear_user) <<
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.