1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) 2012 ARM Ltd.
4 * Author: Catalin Marinas <catalin.marinas@arm.com>
5 * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
6 * Copyright (C) 2021 SiFive
7 */
8 #include <linux/compiler.h>
9 #include <linux/irqflags.h>
10 #include <linux/percpu.h>
11 #include <linux/preempt.h>
12 #include <linux/types.h>
13
14 #include <asm/vector.h>
15 #include <asm/switch_to.h>
16 #include <asm/simd.h>
17 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
18 #include <asm/asm-prototypes.h>
19 #endif
20
21 static inline void riscv_v_flags_set(u32 flags)
22 {
23 WRITE_ONCE(current->thread.riscv_v_flags, flags);
24 }
25
26 static inline void riscv_v_start(u32 flags)
27 {
28 int orig;
29
30 orig = riscv_v_flags();
31 BUG_ON((orig & flags) != 0);
32 riscv_v_flags_set(orig | flags);
33 barrier();
34 }
35
36 static inline void riscv_v_stop(u32 flags)
37 {
38 int orig;
39
40 barrier();
41 orig = riscv_v_flags();
42 BUG_ON((orig & flags) == 0);
43 riscv_v_flags_set(orig & ~flags);
44 }
45
46 /*
47 * Claim ownership of the CPU vector context for use by the calling context.
48 *
49 * The caller may freely manipulate the vector context metadata until
50 * put_cpu_vector_context() is called.
51 */
52 void get_cpu_vector_context(void)
53 {
54 /*
55 * disable softirqs so it is impossible for softirqs to nest
56 * get_cpu_vector_context() when kernel is actively using Vector.
57 */
58 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
59 local_bh_disable();
60 else
61 preempt_disable();
62
63 riscv_v_start(RISCV_KERNEL_MODE_V);
64 }
65
66 /*
67 * Release the CPU vector context.
68 *
69 * Must be called from a context in which get_cpu_vector_context() was
70 * previously called, with no call to put_cpu_vector_context() in the
71 * meantime.
72 */
73 void put_cpu_vector_context(void)
74 {
75 riscv_v_stop(RISCV_KERNEL_MODE_V);
76
77 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
78 local_bh_enable();
79 else
80 preempt_enable();
81 }
82
83 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
84 static __always_inline u32 *riscv_v_flags_ptr(void)
85 {
86 return ¤t->thread.riscv_v_flags;
87 }
88
89 static inline void riscv_preempt_v_set_dirty(void)
90 {
91 *riscv_v_flags_ptr() |= RISCV_PREEMPT_V_DIRTY;
92 }
93
94 static inline void riscv_preempt_v_reset_flags(void)
95 {
96 *riscv_v_flags_ptr() &= ~(RISCV_PREEMPT_V_DIRTY | RISCV_PREEMPT_V_NEED_RESTORE);
97 }
98
99 static inline void riscv_v_ctx_depth_inc(void)
100 {
101 *riscv_v_flags_ptr() += RISCV_V_CTX_UNIT_DEPTH;
102 }
103
104 static inline void riscv_v_ctx_depth_dec(void)
105 {
106 *riscv_v_flags_ptr() -= RISCV_V_CTX_UNIT_DEPTH;
107 }
108
109 static inline u32 riscv_v_ctx_get_depth(void)
110 {
111 return *riscv_v_flags_ptr() & RISCV_V_CTX_DEPTH_MASK;
112 }
113
114 static int riscv_v_stop_kernel_context(void)
115 {
116 if (riscv_v_ctx_get_depth() != 0 || !riscv_preempt_v_started(current))
117 return 1;
118
119 riscv_preempt_v_clear_dirty(current);
120 riscv_v_stop(RISCV_PREEMPT_V);
121 return 0;
122 }
123
124 static int riscv_v_start_kernel_context(bool *is_nested)
125 {
126 struct __riscv_v_ext_state *kvstate, *uvstate;
127
128 kvstate = ¤t->thread.kernel_vstate;
129 if (!kvstate->datap)
130 return -ENOENT;
131
132 if (riscv_preempt_v_started(current)) {
133 WARN_ON(riscv_v_ctx_get_depth() == 0);
134 *is_nested = true;
135 get_cpu_vector_context();
136 if (riscv_preempt_v_dirty(current)) {
137 __riscv_v_vstate_save(kvstate, kvstate->datap);
138 riscv_preempt_v_clear_dirty(current);
139 }
140 riscv_preempt_v_set_restore(current);
141 return 0;
142 }
143
144 /* Transfer the ownership of V from user to kernel, then save */
145 riscv_v_start(RISCV_PREEMPT_V | RISCV_PREEMPT_V_DIRTY);
146 if ((task_pt_regs(current)->status & SR_VS) == SR_VS_DIRTY) {
147 uvstate = ¤t->thread.vstate;
148 __riscv_v_vstate_save(uvstate, uvstate->datap);
149 }
150 riscv_preempt_v_clear_dirty(current);
151 return 0;
152 }
153
154 /* low-level V context handling code, called with irq disabled */
155 asmlinkage void riscv_v_context_nesting_start(struct pt_regs *regs)
156 {
157 int depth;
158
159 if (!riscv_preempt_v_started(current))
160 return;
161
162 depth = riscv_v_ctx_get_depth();
163 if (depth == 0 && (regs->status & SR_VS) == SR_VS_DIRTY)
164 riscv_preempt_v_set_dirty();
165
166 riscv_v_ctx_depth_inc();
167 }
168
169 asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs)
170 {
171 struct __riscv_v_ext_state *vstate = ¤t->thread.kernel_vstate;
172 u32 depth;
173
174 WARN_ON(!irqs_disabled());
175
176 if (!riscv_preempt_v_started(current))
177 return;
178
179 riscv_v_ctx_depth_dec();
180 depth = riscv_v_ctx_get_depth();
181 if (depth == 0) {
182 if (riscv_preempt_v_restore(current)) {
183 __riscv_v_vstate_restore(vstate, vstate->datap);
184 __riscv_v_vstate_clean(regs);
185 riscv_preempt_v_reset_flags();
186 }
187 }
188 }
189 #else
190 #define riscv_v_start_kernel_context(nested) (-ENOENT)
191 #define riscv_v_stop_kernel_context() (-ENOENT)
192 #endif /* CONFIG_RISCV_ISA_V_PREEMPTIVE */
193
194 /*
195 * kernel_vector_begin(): obtain the CPU vector registers for use by the calling
196 * context
197 *
198 * Must not be called unless may_use_simd() returns true.
199 * Task context in the vector registers is saved back to memory as necessary.
200 *
201 * A matching call to kernel_vector_end() must be made before returning from the
202 * calling context.
203 *
204 * The caller may freely use the vector registers until kernel_vector_end() is
205 * called.
206 */
207 void kernel_vector_begin(void)
208 {
209 bool nested = false;
210
211 if (WARN_ON(!has_vector()))
212 return;
213
214 BUG_ON(!may_use_simd());
215
216 if (riscv_v_start_kernel_context(&nested)) {
217 get_cpu_vector_context();
218 riscv_v_vstate_save(¤t->thread.vstate, task_pt_regs(current));
219 }
220
221 if (!nested)
222 riscv_v_vstate_set_restore(current, task_pt_regs(current));
223
224 riscv_v_enable();
225 }
226 EXPORT_SYMBOL_GPL(kernel_vector_begin);
227
228 /*
229 * kernel_vector_end(): give the CPU vector registers back to the current task
230 *
231 * Must be called from a context in which kernel_vector_begin() was previously
232 * called, with no call to kernel_vector_end() in the meantime.
233 *
234 * The caller must not use the vector registers after this function is called,
235 * unless kernel_vector_begin() is called again in the meantime.
236 */
237 void kernel_vector_end(void)
238 {
239 if (WARN_ON(!has_vector()))
240 return;
241
242 riscv_v_disable();
243
244 if (riscv_v_stop_kernel_context())
245 put_cpu_vector_context();
246 }
247 EXPORT_SYMBOL_GPL(kernel_vector_end);
248
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.