1 /* SPDX-License-Identifier: GPL-2.0-only */ 1
2 /*
3 * Copyright (C) 2012 Regents of the Universit
4 */
5
6 #ifndef _ASM_RISCV_BITOPS_H
7 #define _ASM_RISCV_BITOPS_H
8
9 #ifndef _LINUX_BITOPS_H
10 #error "Only <linux/bitops.h> can be included
11 #endif /* _LINUX_BITOPS_H */
12
13 #include <linux/compiler.h>
14 #include <linux/irqflags.h>
15 #include <asm/barrier.h>
16 #include <asm/bitsperlong.h>
17
18 #if !defined(CONFIG_RISCV_ISA_ZBB) || defined(
19 #include <asm-generic/bitops/__ffs.h>
20 #include <asm-generic/bitops/__fls.h>
21 #include <asm-generic/bitops/ffs.h>
22 #include <asm-generic/bitops/fls.h>
23
24 #else
25 #define __HAVE_ARCH___FFS
26 #define __HAVE_ARCH___FLS
27 #define __HAVE_ARCH_FFS
28 #define __HAVE_ARCH_FLS
29
30 #include <asm-generic/bitops/__ffs.h>
31 #include <asm-generic/bitops/__fls.h>
32 #include <asm-generic/bitops/ffs.h>
33 #include <asm-generic/bitops/fls.h>
34
35 #include <asm/alternative-macros.h>
36 #include <asm/hwcap.h>
37
38 #if (BITS_PER_LONG == 64)
39 #define CTZW "ctzw "
40 #define CLZW "clzw "
41 #elif (BITS_PER_LONG == 32)
42 #define CTZW "ctz "
43 #define CLZW "clz "
44 #else
45 #error "Unexpected BITS_PER_LONG"
46 #endif
47
48 static __always_inline unsigned long variable_
49 {
50 asm goto(ALTERNATIVE("j %l[legacy]", "
51 RISCV_IS
52 : : : : legacy);
53
54 asm volatile (".option push\n"
55 ".option arch,+zbb\n"
56 "ctz %0, %1\n"
57 ".option pop\n"
58 : "=r" (word) : "r" (wor
59
60 return word;
61
62 legacy:
63 return generic___ffs(word);
64 }
65
66 /**
67 * __ffs - find first set bit in a long word
68 * @word: The word to search
69 *
70 * Undefined if no set bit exists, so code sho
71 */
72 #define __ffs(word)
73 (__builtin_constant_p(word) ?
74 (unsigned long)__builtin_ctzl(word) :
75 variable__ffs(word))
76
77 static __always_inline unsigned long variable_
78 {
79 asm goto(ALTERNATIVE("j %l[legacy]", "
80 RISCV_IS
81 : : : : legacy);
82
83 asm volatile (".option push\n"
84 ".option arch,+zbb\n"
85 "clz %0, %1\n"
86 ".option pop\n"
87 : "=r" (word) : "r" (wor
88
89 return BITS_PER_LONG - 1 - word;
90
91 legacy:
92 return generic___fls(word);
93 }
94
95 /**
96 * __fls - find last set bit in a long word
97 * @word: the word to search
98 *
99 * Undefined if no set bit exists, so code sho
100 */
101 #define __fls(word)
102 (__builtin_constant_p(word) ?
103 (unsigned long)(BITS_PER_LONG - 1 - _
104 variable__fls(word))
105
106 static __always_inline int variable_ffs(int x)
107 {
108 asm goto(ALTERNATIVE("j %l[legacy]", "
109 RISCV_IS
110 : : : : legacy);
111
112 if (!x)
113 return 0;
114
115 asm volatile (".option push\n"
116 ".option arch,+zbb\n"
117 CTZW "%0, %1\n"
118 ".option pop\n"
119 : "=r" (x) : "r" (x) :);
120
121 return x + 1;
122
123 legacy:
124 return generic_ffs(x);
125 }
126
127 /**
128 * ffs - find first set bit in a word
129 * @x: the word to search
130 *
131 * This is defined the same way as the libc an
132 *
133 * ffs(value) returns 0 if value is 0 or the p
134 * value is nonzero. The first (least signific
135 */
136 #define ffs(x) (__builtin_constant_p(x) ? __bu
137
138 static __always_inline int variable_fls(unsign
139 {
140 asm goto(ALTERNATIVE("j %l[legacy]", "
141 RISCV_IS
142 : : : : legacy);
143
144 if (!x)
145 return 0;
146
147 asm volatile (".option push\n"
148 ".option arch,+zbb\n"
149 CLZW "%0, %1\n"
150 ".option pop\n"
151 : "=r" (x) : "r" (x) :);
152
153 return 32 - x;
154
155 legacy:
156 return generic_fls(x);
157 }
158
159 /**
160 * fls - find last set bit in a word
161 * @x: the word to search
162 *
163 * This is defined in a similar way as ffs, bu
164 * significant set bit.
165 *
166 * fls(value) returns 0 if value is 0 or the p
167 * value is nonzero. The last (most significan
168 */
169 #define fls(x)
170 ({
171 typeof(x) x_ = (x);
172 __builtin_constant_p(x_) ?
173 ((x_ != 0) ? (32 - __builtin_clz(x_))
174 :
175 variable_fls(x_);
176 })
177
178 #endif /* !defined(CONFIG_RISCV_ISA_ZBB) || de
179
180 #include <asm-generic/bitops/ffz.h>
181 #include <asm-generic/bitops/fls64.h>
182 #include <asm-generic/bitops/sched.h>
183
184 #include <asm/arch_hweight.h>
185
186 #include <asm-generic/bitops/const_hweight.h>
187
188 #if (BITS_PER_LONG == 64)
189 #define __AMO(op) "amo" #op ".d"
190 #elif (BITS_PER_LONG == 32)
191 #define __AMO(op) "amo" #op ".w"
192 #else
193 #error "Unexpected BITS_PER_LONG"
194 #endif
195
196 #define __test_and_op_bit_ord(op, mod, nr, add
197 ({
198 unsigned long __res, __mask;
199 __mask = BIT_MASK(nr);
200 __asm__ __volatile__ (
201 __AMO(op) #ord " %0, %2, %1"
202 : "=r" (__res), "+A" (addr[BIT
203 : "r" (mod(__mask))
204 : "memory");
205 ((__res & __mask) != 0);
206 })
207
208 #define __op_bit_ord(op, mod, nr, addr, ord)
209 __asm__ __volatile__ (
210 __AMO(op) #ord " zero, %1, %0"
211 : "+A" (addr[BIT_WORD(nr)])
212 : "r" (mod(BIT_MASK(nr)))
213 : "memory");
214
215 #define __test_and_op_bit(op, mod, nr, addr)
216 __test_and_op_bit_ord(op, mod, nr, add
217 #define __op_bit(op, mod, nr, addr)
218 __op_bit_ord(op, mod, nr, addr, )
219
220 /* Bitmask modifiers */
221 #define __NOP(x) (x)
222 #define __NOT(x) (~(x))
223
224 /**
225 * arch_test_and_set_bit - Set a bit and retur
226 * @nr: Bit to set
227 * @addr: Address to count from
228 *
229 * This operation may be reordered on other ar
230 */
231 static inline int arch_test_and_set_bit(int nr
232 {
233 return __test_and_op_bit(or, __NOP, nr
234 }
235
236 /**
237 * arch_test_and_clear_bit - Clear a bit and r
238 * @nr: Bit to clear
239 * @addr: Address to count from
240 *
241 * This operation can be reordered on other ar
242 */
243 static inline int arch_test_and_clear_bit(int
244 {
245 return __test_and_op_bit(and, __NOT, n
246 }
247
248 /**
249 * arch_test_and_change_bit - Change a bit and
250 * @nr: Bit to change
251 * @addr: Address to count from
252 *
253 * This operation is atomic and cannot be reor
254 * It also implies a memory barrier.
255 */
256 static inline int arch_test_and_change_bit(int
257 {
258 return __test_and_op_bit(xor, __NOP, n
259 }
260
261 /**
262 * arch_set_bit - Atomically set a bit in memo
263 * @nr: the bit to set
264 * @addr: the address to start counting from
265 *
266 * Note: there are no guarantees that this fun
267 * on non x86 architectures, so if you are wri
268 * make sure not to rely on its reordering gua
269 *
270 * Note that @nr may be almost arbitrarily lar
271 * restricted to acting on a single-word quant
272 */
273 static inline void arch_set_bit(int nr, volati
274 {
275 __op_bit(or, __NOP, nr, addr);
276 }
277
278 /**
279 * arch_clear_bit - Clears a bit in memory
280 * @nr: Bit to clear
281 * @addr: Address to start counting from
282 *
283 * Note: there are no guarantees that this fun
284 * on non x86 architectures, so if you are wri
285 * make sure not to rely on its reordering gua
286 */
287 static inline void arch_clear_bit(int nr, vola
288 {
289 __op_bit(and, __NOT, nr, addr);
290 }
291
292 /**
293 * arch_change_bit - Toggle a bit in memory
294 * @nr: Bit to change
295 * @addr: Address to start counting from
296 *
297 * change_bit() may be reordered on other arc
298 * Note that @nr may be almost arbitrarily lar
299 * restricted to acting on a single-word quant
300 */
301 static inline void arch_change_bit(int nr, vol
302 {
303 __op_bit(xor, __NOP, nr, addr);
304 }
305
306 /**
307 * arch_test_and_set_bit_lock - Set a bit and
308 * @nr: Bit to set
309 * @addr: Address to count from
310 *
311 * This operation is atomic and provides acqui
312 * It can be used to implement bit locks.
313 */
314 static inline int arch_test_and_set_bit_lock(
315 unsigned long nr, volatile unsigned lo
316 {
317 return __test_and_op_bit_ord(or, __NOP
318 }
319
320 /**
321 * arch_clear_bit_unlock - Clear a bit in memo
322 * @nr: the bit to set
323 * @addr: the address to start counting from
324 *
325 * This operation is atomic and provides relea
326 */
327 static inline void arch_clear_bit_unlock(
328 unsigned long nr, volatile unsigned lo
329 {
330 __op_bit_ord(and, __NOT, nr, addr, .rl
331 }
332
333 /**
334 * arch___clear_bit_unlock - Clear a bit in me
335 * @nr: the bit to set
336 * @addr: the address to start counting from
337 *
338 * This operation is like clear_bit_unlock, ho
339 * It does provide release barrier semantics s
340 * a bit lock, however it would only be used i
341 * any bits in the memory until the lock is re
342 * if the bit lock itself protects access to t
343 *
344 * On RISC-V systems there seems to be no bene
345 * non-atomic property here: it's a lot more i
346 * provide release semantics anyway.
347 */
348 static inline void arch___clear_bit_unlock(
349 unsigned long nr, volatile unsigned lo
350 {
351 arch_clear_bit_unlock(nr, addr);
352 }
353
354 static inline bool arch_xor_unlock_is_negative
355 volatile unsigned long *addr)
356 {
357 unsigned long res;
358 __asm__ __volatile__ (
359 __AMO(xor) ".rl %0, %2, %1"
360 : "=r" (res), "+A" (*addr)
361 : "r" (__NOP(mask))
362 : "memory");
363 return (res & BIT(7)) != 0;
364 }
365
366 #undef __test_and_op_bit
367 #undef __op_bit
368 #undef __NOP
369 #undef __NOT
370 #undef __AMO
371
372 #include <asm-generic/bitops/instrumented-atom
373 #include <asm-generic/bitops/instrumented-lock
374
375 #include <asm-generic/bitops/non-atomic.h>
376 #include <asm-generic/bitops/le.h>
377 #include <asm-generic/bitops/ext2-atomic.h>
378
379 #endif /* _ASM_RISCV_BITOPS_H */
380
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.