1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* 2 /*
3 * arch/alpha/lib/ev6-clear_user.S 3 * arch/alpha/lib/ev6-clear_user.S
4 * 21264 version contributed by Rick Gorton <ri 4 * 21264 version contributed by Rick Gorton <rick.gorton@alpha-processor.com>
5 * 5 *
6 * Zero user space, handling exceptions as we 6 * Zero user space, handling exceptions as we go.
7 * 7 *
8 * We have to make sure that $0 is always up-t 8 * We have to make sure that $0 is always up-to-date and contains the
9 * right "bytes left to zero" value (and that 9 * right "bytes left to zero" value (and that it is updated only _after_
10 * a successful copy). There is also some rat 10 * a successful copy). There is also some rather minor exception setup
11 * stuff. 11 * stuff.
12 * 12 *
13 * Much of the information about 21264 schedul 13 * Much of the information about 21264 scheduling/coding comes from:
14 * Compiler Writer's Guide for the Alpha 14 * Compiler Writer's Guide for the Alpha 21264
15 * abbreviated as 'CWG' in other comments 15 * abbreviated as 'CWG' in other comments here
16 * ftp.digital.com/pub/Digital/info/semic 16 * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
17 * Scheduling notation: 17 * Scheduling notation:
18 * E - either cluster 18 * E - either cluster
19 * U - upper subcluster; U0 - subcl 19 * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1
20 * L - lower subcluster; L0 - subcl 20 * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1
21 * Try not to change the actual algorithm if p 21 * Try not to change the actual algorithm if possible for consistency.
22 * Determining actual stalls (other than slott 22 * Determining actual stalls (other than slotting) doesn't appear to be easy to do.
23 * From perusing the source code context where 23 * From perusing the source code context where this routine is called, it is
24 * a fair assumption that significant fraction 24 * a fair assumption that significant fractions of entire pages are zeroed, so
25 * it's going to be worth the effort to hand-u 25 * it's going to be worth the effort to hand-unroll a big loop, and use wh64.
26 * ASSUMPTION: 26 * ASSUMPTION:
27 * The believed purpose of only updating 27 * The believed purpose of only updating $0 after a store is that a signal
28 * may come along during the execution of 28 * may come along during the execution of this chunk of code, and we don't
29 * want to leave a hole (and we also want 29 * want to leave a hole (and we also want to avoid repeating lots of work)
30 */ 30 */
31 31
32 #include <linux/export.h> 32 #include <linux/export.h>
33 /* Allow an exception for an insn; exit if we 33 /* Allow an exception for an insn; exit if we get one. */
34 #define EX(x,y...) \ 34 #define EX(x,y...) \
35 99: x,##y; \ 35 99: x,##y; \
36 .section __ex_table,"a"; \ 36 .section __ex_table,"a"; \
37 .long 99b - .; \ 37 .long 99b - .; \
38 lda $31, $exception-99b($31); \ 38 lda $31, $exception-99b($31); \
39 .previous 39 .previous
40 40
41 .set noat 41 .set noat
42 .set noreorder 42 .set noreorder
43 .align 4 43 .align 4
44 44
45 .globl __clear_user 45 .globl __clear_user
46 .ent __clear_user 46 .ent __clear_user
47 .frame $30, 0, $26 47 .frame $30, 0, $26
48 .prologue 0 48 .prologue 0
49 49
50 # Pipeline inf 50 # Pipeline info : Slotting & Comments
51 __clear_user: 51 __clear_user:
52 and $17, $17, $0 52 and $17, $17, $0
53 and $16, 7, $4 # .. E .. .. 53 and $16, 7, $4 # .. E .. .. : find dest head misalignment
54 beq $0, $zerolength # U .. .. .. 54 beq $0, $zerolength # U .. .. .. : U L U L
55 55
56 addq $0, $4, $1 # .. .. .. E 56 addq $0, $4, $1 # .. .. .. E : bias counter
57 and $1, 7, $2 # .. .. E .. 57 and $1, 7, $2 # .. .. E .. : number of misaligned bytes in tail
58 # Note - we never actually use $2, so this is 58 # Note - we never actually use $2, so this is a moot computation
59 # and we can rewrite this later... 59 # and we can rewrite this later...
60 srl $1, 3, $1 # .. E .. .. 60 srl $1, 3, $1 # .. E .. .. : number of quadwords to clear
61 beq $4, $headalign # U .. .. .. 61 beq $4, $headalign # U .. .. .. : U L U L
62 62
63 /* 63 /*
64 * Head is not aligned. Write (8 - $4) bytes 64 * Head is not aligned. Write (8 - $4) bytes to head of destination
65 * This means $16 is known to be misaligned 65 * This means $16 is known to be misaligned
66 */ 66 */
67 EX( ldq_u $5, 0($16) ) # .. .. .. L 67 EX( ldq_u $5, 0($16) ) # .. .. .. L : load dst word to mask back in
68 beq $1, $onebyte # .. .. U .. 68 beq $1, $onebyte # .. .. U .. : sub-word store?
69 mskql $5, $16, $5 # .. U .. .. 69 mskql $5, $16, $5 # .. U .. .. : take care of misaligned head
70 addq $16, 8, $16 # E .. .. .. 70 addq $16, 8, $16 # E .. .. .. : L U U L
71 71
72 EX( stq_u $5, -8($16) ) # .. .. .. L 72 EX( stq_u $5, -8($16) ) # .. .. .. L :
73 subq $1, 1, $1 # .. .. E .. 73 subq $1, 1, $1 # .. .. E .. :
74 addq $0, $4, $0 # .. E .. .. 74 addq $0, $4, $0 # .. E .. .. : bytes left -= 8 - misalignment
75 subq $0, 8, $0 # E .. .. .. 75 subq $0, 8, $0 # E .. .. .. : U L U L
76 76
77 .align 4 77 .align 4
78 /* 78 /*
79 * (The .align directive ought to be a moot po 79 * (The .align directive ought to be a moot point)
80 * values upon initial entry to the loop 80 * values upon initial entry to the loop
81 * $1 is number of quadwords to clear (zero is 81 * $1 is number of quadwords to clear (zero is a valid value)
82 * $2 is number of trailing bytes (0..7) ($2 n 82 * $2 is number of trailing bytes (0..7) ($2 never used...)
83 * $16 is known to be aligned 0mod8 83 * $16 is known to be aligned 0mod8
84 */ 84 */
85 $headalign: 85 $headalign:
86 subq $1, 16, $4 # .. .. .. E 86 subq $1, 16, $4 # .. .. .. E : If < 16, we can not use the huge loop
87 and $16, 0x3f, $2 # .. .. E .. 87 and $16, 0x3f, $2 # .. .. E .. : Forward work for huge loop
88 subq $2, 0x40, $3 # .. E .. .. 88 subq $2, 0x40, $3 # .. E .. .. : bias counter (huge loop)
89 blt $4, $trailquad # U .. .. .. 89 blt $4, $trailquad # U .. .. .. : U L U L
90 90
91 /* 91 /*
92 * We know that we're going to do at least 16 92 * We know that we're going to do at least 16 quads, which means we are
93 * going to be able to use the large block cle 93 * going to be able to use the large block clear loop at least once.
94 * Figure out how many quads we need to clear 94 * Figure out how many quads we need to clear before we are 0mod64 aligned
95 * so we can use the wh64 instruction. 95 * so we can use the wh64 instruction.
96 */ 96 */
97 97
98 nop # .. .. .. E 98 nop # .. .. .. E
99 nop # .. .. E .. 99 nop # .. .. E ..
100 nop # .. E .. .. 100 nop # .. E .. ..
101 beq $3, $bigalign # U .. .. .. 101 beq $3, $bigalign # U .. .. .. : U L U L : Aligned 0mod64
102 102
103 $alignmod64: 103 $alignmod64:
104 EX( stq_u $31, 0($16) ) # .. .. .. L 104 EX( stq_u $31, 0($16) ) # .. .. .. L
105 addq $3, 8, $3 # .. .. E .. 105 addq $3, 8, $3 # .. .. E ..
106 subq $0, 8, $0 # .. E .. .. 106 subq $0, 8, $0 # .. E .. ..
107 nop # E .. .. .. 107 nop # E .. .. .. : U L U L
108 108
109 nop # .. .. .. E 109 nop # .. .. .. E
110 subq $1, 1, $1 # .. .. E .. 110 subq $1, 1, $1 # .. .. E ..
111 addq $16, 8, $16 # .. E .. .. 111 addq $16, 8, $16 # .. E .. ..
112 blt $3, $alignmod64 # U .. .. .. 112 blt $3, $alignmod64 # U .. .. .. : U L U L
113 113
114 $bigalign: 114 $bigalign:
115 /* 115 /*
116 * $0 is the number of bytes left 116 * $0 is the number of bytes left
117 * $1 is the number of quads left 117 * $1 is the number of quads left
118 * $16 is aligned 0mod64 118 * $16 is aligned 0mod64
119 * we know that we'll be taking a minimum of o 119 * we know that we'll be taking a minimum of one trip through
120 * CWG Section 3.7.6: do not expect a sustaine 120 * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle
121 * We are _not_ going to update $0 after every 121 * We are _not_ going to update $0 after every single store. That
122 * would be silly, because there will be cross 122 * would be silly, because there will be cross-cluster dependencies
123 * no matter how the code is scheduled. By do 123 * no matter how the code is scheduled. By doing it in slightly
124 * staggered fashion, we can still do this loo 124 * staggered fashion, we can still do this loop in 5 fetches
125 * The worse case will be doing two extra quad 125 * The worse case will be doing two extra quads in some future execution,
126 * in the event of an interrupted clear. 126 * in the event of an interrupted clear.
127 * Assumes the wh64 needs to be for 2 trips th 127 * Assumes the wh64 needs to be for 2 trips through the loop in the future
128 * The wh64 is issued on for the starting dest 128 * The wh64 is issued on for the starting destination address for trip +2
129 * through the loop, and if there are less tha 129 * through the loop, and if there are less than two trips left, the target
130 * address will be for the current trip. 130 * address will be for the current trip.
131 */ 131 */
132 nop # E : 132 nop # E :
133 nop # E : 133 nop # E :
134 nop # E : 134 nop # E :
135 bis $16,$16,$3 # E : U L U L 135 bis $16,$16,$3 # E : U L U L : Initial wh64 address is dest
136 /* This might actually help for the cu 136 /* This might actually help for the current trip... */
137 137
138 $do_wh64: 138 $do_wh64:
139 wh64 ($3) # .. .. .. L1 139 wh64 ($3) # .. .. .. L1 : memory subsystem hint
140 subq $1, 16, $4 # .. .. E .. 140 subq $1, 16, $4 # .. .. E .. : Forward calculation - repeat the loop?
141 EX( stq_u $31, 0($16) ) # .. L .. .. 141 EX( stq_u $31, 0($16) ) # .. L .. ..
142 subq $0, 8, $0 # E .. .. .. 142 subq $0, 8, $0 # E .. .. .. : U L U L
143 143
144 addq $16, 128, $3 # E : Target a 144 addq $16, 128, $3 # E : Target address of wh64
145 EX( stq_u $31, 8($16) ) # L : 145 EX( stq_u $31, 8($16) ) # L :
146 EX( stq_u $31, 16($16) ) # L : 146 EX( stq_u $31, 16($16) ) # L :
147 subq $0, 16, $0 # E : U L L U 147 subq $0, 16, $0 # E : U L L U
148 148
149 nop # E : 149 nop # E :
150 EX( stq_u $31, 24($16) ) # L : 150 EX( stq_u $31, 24($16) ) # L :
151 EX( stq_u $31, 32($16) ) # L : 151 EX( stq_u $31, 32($16) ) # L :
152 subq $0, 168, $5 # E : U L L U 152 subq $0, 168, $5 # E : U L L U : two trips through the loop left?
153 /* 168 = 192 - 24, since we've already 153 /* 168 = 192 - 24, since we've already completed some stores */
154 154
155 subq $0, 16, $0 # E : 155 subq $0, 16, $0 # E :
156 EX( stq_u $31, 40($16) ) # L : 156 EX( stq_u $31, 40($16) ) # L :
157 EX( stq_u $31, 48($16) ) # L : 157 EX( stq_u $31, 48($16) ) # L :
158 cmovlt $5, $16, $3 # E : U L L U 158 cmovlt $5, $16, $3 # E : U L L U : Latency 2, extra mapping cycle
159 159
160 subq $1, 8, $1 # E : 160 subq $1, 8, $1 # E :
161 subq $0, 16, $0 # E : 161 subq $0, 16, $0 # E :
162 EX( stq_u $31, 56($16) ) # L : 162 EX( stq_u $31, 56($16) ) # L :
163 nop # E : U L U L 163 nop # E : U L U L
164 164
165 nop # E : 165 nop # E :
166 subq $0, 8, $0 # E : 166 subq $0, 8, $0 # E :
167 addq $16, 64, $16 # E : 167 addq $16, 64, $16 # E :
168 bge $4, $do_wh64 # U : U L U L 168 bge $4, $do_wh64 # U : U L U L
169 169
170 $trailquad: 170 $trailquad:
171 # zero to 16 quadwords left to store, 171 # zero to 16 quadwords left to store, plus any trailing bytes
172 # $1 is the number of quadwords left t 172 # $1 is the number of quadwords left to go.
173 # 173 #
174 nop # .. .. .. E 174 nop # .. .. .. E
175 nop # .. .. E .. 175 nop # .. .. E ..
176 nop # .. E .. .. 176 nop # .. E .. ..
177 beq $1, $trailbytes # U .. .. .. 177 beq $1, $trailbytes # U .. .. .. : U L U L : Only 0..7 bytes to go
178 178
179 $onequad: 179 $onequad:
180 EX( stq_u $31, 0($16) ) # .. .. .. L 180 EX( stq_u $31, 0($16) ) # .. .. .. L
181 subq $1, 1, $1 # .. .. E .. 181 subq $1, 1, $1 # .. .. E ..
182 subq $0, 8, $0 # .. E .. .. 182 subq $0, 8, $0 # .. E .. ..
183 nop # E .. .. .. 183 nop # E .. .. .. : U L U L
184 184
185 nop # .. .. .. E 185 nop # .. .. .. E
186 nop # .. .. E .. 186 nop # .. .. E ..
187 addq $16, 8, $16 # .. E .. .. 187 addq $16, 8, $16 # .. E .. ..
188 bgt $1, $onequad # U .. .. .. 188 bgt $1, $onequad # U .. .. .. : U L U L
189 189
190 # We have an unknown number of bytes l 190 # We have an unknown number of bytes left to go.
191 $trailbytes: 191 $trailbytes:
192 nop # .. .. .. E 192 nop # .. .. .. E
193 nop # .. .. E .. 193 nop # .. .. E ..
194 nop # .. E .. .. 194 nop # .. E .. ..
195 beq $0, $zerolength # U .. .. .. 195 beq $0, $zerolength # U .. .. .. : U L U L
196 196
197 # $0 contains the number of bytes left 197 # $0 contains the number of bytes left to copy (0..31)
198 # so we will use $0 as the loop counte 198 # so we will use $0 as the loop counter
199 # We know for a fact that $0 > 0 zero 199 # We know for a fact that $0 > 0 zero due to previous context
200 $onebyte: 200 $onebyte:
201 EX( stb $31, 0($16) ) # .. .. .. L 201 EX( stb $31, 0($16) ) # .. .. .. L
202 subq $0, 1, $0 # .. .. E .. 202 subq $0, 1, $0 # .. .. E .. :
203 addq $16, 1, $16 # .. E .. .. 203 addq $16, 1, $16 # .. E .. .. :
204 bgt $0, $onebyte # U .. .. .. 204 bgt $0, $onebyte # U .. .. .. : U L U L
205 205
206 $zerolength: 206 $zerolength:
207 $exception: # Destination 207 $exception: # Destination for exception recovery(?)
208 nop # .. .. .. E 208 nop # .. .. .. E :
209 nop # .. .. E .. 209 nop # .. .. E .. :
210 nop # .. E .. .. 210 nop # .. E .. .. :
211 ret $31, ($26), 1 # L0 .. .. .. 211 ret $31, ($26), 1 # L0 .. .. .. : L U L U
212 .end __clear_user 212 .end __clear_user
213 EXPORT_SYMBOL(__clear_user) 213 EXPORT_SYMBOL(__clear_user)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.