1 /* SPDX-License-Identifier: GPL-2.0-or-later *
2 /*
3 *
4 * Copyright (C) IBM Corporation, 2012
5 *
6 * Author: Anton Blanchard <anton@au.ibm.com>
7 */
8
9 #include <linux/export.h>
10 #include <asm/ppc_asm.h>
11 #include <asm/linkage.h>
12 #include <asm/asm-offsets.h>
13
14 /**
15 * __arch_clear_user: - Zero a block of memory
16 * @to: Destination address, in user space.
17 * @n: Number of bytes to zero.
18 *
19 * Zero a block of memory in user space. Call
20 * the specified block with access_ok() before
21 *
22 * Returns number of bytes that could not be c
23 * On success, this will be zero.
24 */
25
26 .macro err1
27 100:
28 EX_TABLE(100b,.Ldo_err1)
29 .endm
30
31 .macro err2
32 200:
33 EX_TABLE(200b,.Ldo_err2)
34 .endm
35
36 .macro err3
37 300:
38 EX_TABLE(300b,.Ldo_err3)
39 .endm
40
41 .Ldo_err1:
42 mr r3,r8
43
44 .Ldo_err2:
45 mtctr r4
46 1:
47 err3; stb r0,0(r3)
48 addi r3,r3,1
49 addi r4,r4,-1
50 bdnz 1b
51
52 .Ldo_err3:
53 mr r3,r4
54 blr
55
56 _GLOBAL_TOC(__arch_clear_user)
57 cmpdi r4,32
58 neg r6,r3
59 li r0,0
60 blt .Lshort_clear
61 mr r8,r3
62 mtocrf 0x01,r6
63 clrldi r6,r6,(64-3)
64
65 /* Get the destination 8 byte aligned
66 bf cr7*4+3,1f
67 err1; stb r0,0(r3)
68 addi r3,r3,1
69
70 1: bf cr7*4+2,2f
71 err1; sth r0,0(r3)
72 addi r3,r3,2
73
74 2: bf cr7*4+1,3f
75 err1; stw r0,0(r3)
76 addi r3,r3,4
77
78 3: sub r4,r4,r6
79
80 cmpdi r4,32
81 cmpdi cr1,r4,512
82 blt .Lshort_clear
83 bgt cr1,.Llong_clear
84
85 .Lmedium_clear:
86 srdi r6,r4,5
87 mtctr r6
88
89 /* Do 32 byte chunks */
90 4:
91 err2; std r0,0(r3)
92 err2; std r0,8(r3)
93 err2; std r0,16(r3)
94 err2; std r0,24(r3)
95 addi r3,r3,32
96 addi r4,r4,-32
97 bdnz 4b
98
99 .Lshort_clear:
100 /* up to 31 bytes to go */
101 cmpdi r4,16
102 blt 6f
103 err2; std r0,0(r3)
104 err2; std r0,8(r3)
105 addi r3,r3,16
106 addi r4,r4,-16
107
108 /* Up to 15 bytes to go */
109 6: mr r8,r3
110 clrldi r4,r4,(64-4)
111 mtocrf 0x01,r4
112 bf cr7*4+0,7f
113 err1; std r0,0(r3)
114 addi r3,r3,8
115
116 7: bf cr7*4+1,8f
117 err1; stw r0,0(r3)
118 addi r3,r3,4
119
120 8: bf cr7*4+2,9f
121 err1; sth r0,0(r3)
122 addi r3,r3,2
123
124 9: bf cr7*4+3,10f
125 err1; stb r0,0(r3)
126
127 10: li r3,0
128 blr
129
130 .Llong_clear:
131 LOAD_REG_ADDR(r5, ppc64_caches)
132
133 bf cr7*4+0,11f
134 err2; std r0,0(r3)
135 addi r3,r3,8
136 addi r4,r4,-8
137
138 /* Destination is 16 byte aligned, nee
139 11: lwz r7,DCACHEL1LOGBLOCKSIZE(r5)
140 lwz r9,DCACHEL1BLOCKSIZE(r5)
141
142 /*
143 * With worst case alignment the long
144 * of 1 byte less than 2 cachelines.
145 */
146 sldi r10,r9,2
147 cmpd r4,r10
148 blt .Lmedium_clear
149
150 neg r6,r3
151 addi r10,r9,-1
152 and. r5,r6,r10
153 beq 13f
154
155 srdi r6,r5,4
156 mtctr r6
157 mr r8,r3
158 12:
159 err1; std r0,0(r3)
160 err1; std r0,8(r3)
161 addi r3,r3,16
162 bdnz 12b
163
164 sub r4,r4,r5
165
166 13: srd r6,r4,r7
167 mtctr r6
168 mr r8,r3
169 14:
170 err1; dcbz 0,r3
171 add r3,r3,r9
172 bdnz 14b
173
174 and r4,r4,r10
175
176 cmpdi r4,32
177 blt .Lshort_clear
178 b .Lmedium_clear
179 EXPORT_SYMBOL(__arch_clear_user)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.