1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Cache maintenance
4 *
5 * Copyright (C) 2001 Deep Blue Solutions Ltd.
6 * Copyright (C) 2012 ARM Ltd.
7 */
8
9 #include <linux/errno.h>
10 #include <linux/linkage.h>
11 #include <linux/init.h>
12 #include <asm/assembler.h>
13 #include <asm/cpufeature.h>
14 #include <asm/alternative.h>
15 #include <asm/asm-uaccess.h>
16
17 /*
18 * caches_clean_inval_pou_macro(start,end
19 *
20 * Ensure that the I and D caches are coh
21 * This is typically used when code has b
22 * and will be executed.
23 *
24 * - start - virtual start address of r
25 * - end - virtual end address of reg
26 * - fixup - optional label to branch t
27 */
28 .macro caches_clean_inval_pou_macro, fixup
29 alternative_if ARM64_HAS_CACHE_IDC
30 dsb ishst
31 b .Ldc_skip_\@
32 alternative_else_nop_endif
33 mov x2, x0
34 mov x3, x1
35 dcache_by_line_op cvau, ish, x2, x3, x
36 .Ldc_skip_\@:
37 alternative_if ARM64_HAS_CACHE_DIC
38 isb
39 b .Lic_skip_\@
40 alternative_else_nop_endif
41 invalidate_icache_by_line x0, x1, x2,
42 .Lic_skip_\@:
43 .endm
44
45 /*
46 * caches_clean_inval_pou(start,end)
47 *
48 * Ensure that the I and D caches are coh
49 * This is typically used when code has b
50 * and will be executed.
51 *
52 * - start - virtual start address of r
53 * - end - virtual end address of reg
54 */
55 SYM_FUNC_START(caches_clean_inval_pou)
56 caches_clean_inval_pou_macro
57 ret
58 SYM_FUNC_END(caches_clean_inval_pou)
59 SYM_FUNC_ALIAS(__pi_caches_clean_inval_pou, ca
60
61 /*
62 * caches_clean_inval_user_pou(start,end)
63 *
64 * Ensure that the I and D caches are coh
65 * This is typically used when code has b
66 * and will be executed.
67 *
68 * - start - virtual start address of r
69 * - end - virtual end address of reg
70 */
71 SYM_FUNC_START(caches_clean_inval_user_pou)
72 uaccess_ttbr0_enable x2, x3, x4
73
74 caches_clean_inval_pou_macro 2f
75 mov x0, xzr
76 1:
77 uaccess_ttbr0_disable x1, x2
78 ret
79 2:
80 mov x0, #-EFAULT
81 b 1b
82 SYM_FUNC_END(caches_clean_inval_user_pou)
83
84 /*
85 * icache_inval_pou(start,end)
86 *
87 * Ensure that the I cache is invalid wit
88 *
89 * - start - virtual start address of r
90 * - end - virtual end address of reg
91 */
92 SYM_FUNC_START(icache_inval_pou)
93 alternative_if ARM64_HAS_CACHE_DIC
94 isb
95 ret
96 alternative_else_nop_endif
97
98 invalidate_icache_by_line x0, x1, x2,
99 ret
100 SYM_FUNC_END(icache_inval_pou)
101
102 /*
103 * dcache_clean_inval_poc(start, end)
104 *
105 * Ensure that any D-cache lines for the
106 * are cleaned and invalidated to the PoC
107 *
108 * - start - virtual start address of r
109 * - end - virtual end address of reg
110 */
111 SYM_FUNC_START(__pi_dcache_clean_inval_poc)
112 dcache_by_line_op civac, sy, x0, x1, x
113 ret
114 SYM_FUNC_END(__pi_dcache_clean_inval_poc)
115 SYM_FUNC_ALIAS(dcache_clean_inval_poc, __pi_dc
116
117 /*
118 * dcache_clean_pou(start, end)
119 *
120 * Ensure that any D-cache lines for the
121 * are cleaned to the PoU.
122 *
123 * - start - virtual start address of r
124 * - end - virtual end address of reg
125 */
126 SYM_FUNC_START(dcache_clean_pou)
127 alternative_if ARM64_HAS_CACHE_IDC
128 dsb ishst
129 ret
130 alternative_else_nop_endif
131 dcache_by_line_op cvau, ish, x0, x1, x
132 ret
133 SYM_FUNC_END(dcache_clean_pou)
134
135 /*
136 * dcache_inval_poc(start, end)
137 *
138 * Ensure that any D-cache lines for the
139 * are invalidated. Any partial lines at
140 * also cleaned to PoC to prevent data lo
141 *
142 * - start - kernel start address of re
143 * - end - kernel end address of regi
144 */
145 SYM_FUNC_START(__pi_dcache_inval_poc)
146 dcache_line_size x2, x3
147 sub x3, x2, #1
148 tst x1, x3
149 bic x1, x1, x3
150 b.eq 1f
151 dc civac, x1
152 1: tst x0, x3
153 bic x0, x0, x3
154 b.eq 2f
155 dc civac, x0
156 b 3f
157 2: dc ivac, x0
158 3: add x0, x0, x2
159 cmp x0, x1
160 b.lo 2b
161 dsb sy
162 ret
163 SYM_FUNC_END(__pi_dcache_inval_poc)
164 SYM_FUNC_ALIAS(dcache_inval_poc, __pi_dcache_i
165
166 /*
167 * dcache_clean_poc(start, end)
168 *
169 * Ensure that any D-cache lines for the
170 * are cleaned to the PoC.
171 *
172 * - start - virtual start address of r
173 * - end - virtual end address of reg
174 */
175 SYM_FUNC_START(__pi_dcache_clean_poc)
176 dcache_by_line_op cvac, sy, x0, x1, x2
177 ret
178 SYM_FUNC_END(__pi_dcache_clean_poc)
179 SYM_FUNC_ALIAS(dcache_clean_poc, __pi_dcache_c
180
181 /*
182 * dcache_clean_pop(start, end)
183 *
184 * Ensure that any D-cache lines for the
185 * are cleaned to the PoP.
186 *
187 * - start - virtual start address of r
188 * - end - virtual end address of reg
189 */
190 SYM_FUNC_START(__pi_dcache_clean_pop)
191 alternative_if_not ARM64_HAS_DCPOP
192 b dcache_clean_poc
193 alternative_else_nop_endif
194 dcache_by_line_op cvap, sy, x0, x1, x2
195 ret
196 SYM_FUNC_END(__pi_dcache_clean_pop)
197 SYM_FUNC_ALIAS(dcache_clean_pop, __pi_dcache_c
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.