1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2012 Regents of the University of California
4 */
5
6 #ifndef _ASM_RISCV_BITOPS_H
7 #define _ASM_RISCV_BITOPS_H
8
9 #ifndef _LINUX_BITOPS_H
10 #error "Only <linux/bitops.h> can be included directly"
11 #endif /* _LINUX_BITOPS_H */
12
13 #include <linux/compiler.h>
14 #include <linux/irqflags.h>
15 #include <asm/barrier.h>
16 #include <asm/bitsperlong.h>
17
18 #if !defined(CONFIG_RISCV_ISA_ZBB) || defined(NO_ALTERNATIVE)
19 #include <asm-generic/bitops/__ffs.h>
20 #include <asm-generic/bitops/__fls.h>
21 #include <asm-generic/bitops/ffs.h>
22 #include <asm-generic/bitops/fls.h>
23
24 #else
25 #define __HAVE_ARCH___FFS
26 #define __HAVE_ARCH___FLS
27 #define __HAVE_ARCH_FFS
28 #define __HAVE_ARCH_FLS
29
30 #include <asm-generic/bitops/__ffs.h>
31 #include <asm-generic/bitops/__fls.h>
32 #include <asm-generic/bitops/ffs.h>
33 #include <asm-generic/bitops/fls.h>
34
35 #include <asm/alternative-macros.h>
36 #include <asm/hwcap.h>
37
38 #if (BITS_PER_LONG == 64)
39 #define CTZW "ctzw "
40 #define CLZW "clzw "
41 #elif (BITS_PER_LONG == 32)
42 #define CTZW "ctz "
43 #define CLZW "clz "
44 #else
45 #error "Unexpected BITS_PER_LONG"
46 #endif
47
48 static __always_inline unsigned long variable__ffs(unsigned long word)
49 {
50 asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0,
51 RISCV_ISA_EXT_ZBB, 1)
52 : : : : legacy);
53
54 asm volatile (".option push\n"
55 ".option arch,+zbb\n"
56 "ctz %0, %1\n"
57 ".option pop\n"
58 : "=r" (word) : "r" (word) :);
59
60 return word;
61
62 legacy:
63 return generic___ffs(word);
64 }
65
66 /**
67 * __ffs - find first set bit in a long word
68 * @word: The word to search
69 *
70 * Undefined if no set bit exists, so code should check against 0 first.
71 */
72 #define __ffs(word) \
73 (__builtin_constant_p(word) ? \
74 (unsigned long)__builtin_ctzl(word) : \
75 variable__ffs(word))
76
77 static __always_inline unsigned long variable__fls(unsigned long word)
78 {
79 asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0,
80 RISCV_ISA_EXT_ZBB, 1)
81 : : : : legacy);
82
83 asm volatile (".option push\n"
84 ".option arch,+zbb\n"
85 "clz %0, %1\n"
86 ".option pop\n"
87 : "=r" (word) : "r" (word) :);
88
89 return BITS_PER_LONG - 1 - word;
90
91 legacy:
92 return generic___fls(word);
93 }
94
95 /**
96 * __fls - find last set bit in a long word
97 * @word: the word to search
98 *
99 * Undefined if no set bit exists, so code should check against 0 first.
100 */
101 #define __fls(word) \
102 (__builtin_constant_p(word) ? \
103 (unsigned long)(BITS_PER_LONG - 1 - __builtin_clzl(word)) : \
104 variable__fls(word))
105
106 static __always_inline int variable_ffs(int x)
107 {
108 asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0,
109 RISCV_ISA_EXT_ZBB, 1)
110 : : : : legacy);
111
112 if (!x)
113 return 0;
114
115 asm volatile (".option push\n"
116 ".option arch,+zbb\n"
117 CTZW "%0, %1\n"
118 ".option pop\n"
119 : "=r" (x) : "r" (x) :);
120
121 return x + 1;
122
123 legacy:
124 return generic_ffs(x);
125 }
126
127 /**
128 * ffs - find first set bit in a word
129 * @x: the word to search
130 *
131 * This is defined the same way as the libc and compiler builtin ffs routines.
132 *
133 * ffs(value) returns 0 if value is 0 or the position of the first set bit if
134 * value is nonzero. The first (least significant) bit is at position 1.
135 */
136 #define ffs(x) (__builtin_constant_p(x) ? __builtin_ffs(x) : variable_ffs(x))
137
138 static __always_inline int variable_fls(unsigned int x)
139 {
140 asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0,
141 RISCV_ISA_EXT_ZBB, 1)
142 : : : : legacy);
143
144 if (!x)
145 return 0;
146
147 asm volatile (".option push\n"
148 ".option arch,+zbb\n"
149 CLZW "%0, %1\n"
150 ".option pop\n"
151 : "=r" (x) : "r" (x) :);
152
153 return 32 - x;
154
155 legacy:
156 return generic_fls(x);
157 }
158
159 /**
160 * fls - find last set bit in a word
161 * @x: the word to search
162 *
163 * This is defined in a similar way as ffs, but returns the position of the most
164 * significant set bit.
165 *
166 * fls(value) returns 0 if value is 0 or the position of the last set bit if
167 * value is nonzero. The last (most significant) bit is at position 32.
168 */
169 #define fls(x) \
170 ({ \
171 typeof(x) x_ = (x); \
172 __builtin_constant_p(x_) ? \
173 ((x_ != 0) ? (32 - __builtin_clz(x_)) : 0) \
174 : \
175 variable_fls(x_); \
176 })
177
178 #endif /* !defined(CONFIG_RISCV_ISA_ZBB) || defined(NO_ALTERNATIVE) */
179
180 #include <asm-generic/bitops/ffz.h>
181 #include <asm-generic/bitops/fls64.h>
182 #include <asm-generic/bitops/sched.h>
183
184 #include <asm/arch_hweight.h>
185
186 #include <asm-generic/bitops/const_hweight.h>
187
188 #if (BITS_PER_LONG == 64)
189 #define __AMO(op) "amo" #op ".d"
190 #elif (BITS_PER_LONG == 32)
191 #define __AMO(op) "amo" #op ".w"
192 #else
193 #error "Unexpected BITS_PER_LONG"
194 #endif
195
196 #define __test_and_op_bit_ord(op, mod, nr, addr, ord) \
197 ({ \
198 unsigned long __res, __mask; \
199 __mask = BIT_MASK(nr); \
200 __asm__ __volatile__ ( \
201 __AMO(op) #ord " %0, %2, %1" \
202 : "=r" (__res), "+A" (addr[BIT_WORD(nr)]) \
203 : "r" (mod(__mask)) \
204 : "memory"); \
205 ((__res & __mask) != 0); \
206 })
207
208 #define __op_bit_ord(op, mod, nr, addr, ord) \
209 __asm__ __volatile__ ( \
210 __AMO(op) #ord " zero, %1, %0" \
211 : "+A" (addr[BIT_WORD(nr)]) \
212 : "r" (mod(BIT_MASK(nr))) \
213 : "memory");
214
215 #define __test_and_op_bit(op, mod, nr, addr) \
216 __test_and_op_bit_ord(op, mod, nr, addr, .aqrl)
217 #define __op_bit(op, mod, nr, addr) \
218 __op_bit_ord(op, mod, nr, addr, )
219
220 /* Bitmask modifiers */
221 #define __NOP(x) (x)
222 #define __NOT(x) (~(x))
223
224 /**
225 * test_and_set_bit - Set a bit and return its old value
226 * @nr: Bit to set
227 * @addr: Address to count from
228 *
229 * This operation may be reordered on other architectures than x86.
230 */
231 static inline int test_and_set_bit(int nr, volatile unsigned long *addr)
232 {
233 return __test_and_op_bit(or, __NOP, nr, addr);
234 }
235
236 /**
237 * test_and_clear_bit - Clear a bit and return its old value
238 * @nr: Bit to clear
239 * @addr: Address to count from
240 *
241 * This operation can be reordered on other architectures other than x86.
242 */
243 static inline int test_and_clear_bit(int nr, volatile unsigned long *addr)
244 {
245 return __test_and_op_bit(and, __NOT, nr, addr);
246 }
247
248 /**
249 * test_and_change_bit - Change a bit and return its old value
250 * @nr: Bit to change
251 * @addr: Address to count from
252 *
253 * This operation is atomic and cannot be reordered.
254 * It also implies a memory barrier.
255 */
256 static inline int test_and_change_bit(int nr, volatile unsigned long *addr)
257 {
258 return __test_and_op_bit(xor, __NOP, nr, addr);
259 }
260
261 /**
262 * set_bit - Atomically set a bit in memory
263 * @nr: the bit to set
264 * @addr: the address to start counting from
265 *
266 * Note: there are no guarantees that this function will not be reordered
267 * on non x86 architectures, so if you are writing portable code,
268 * make sure not to rely on its reordering guarantees.
269 *
270 * Note that @nr may be almost arbitrarily large; this function is not
271 * restricted to acting on a single-word quantity.
272 */
273 static inline void set_bit(int nr, volatile unsigned long *addr)
274 {
275 __op_bit(or, __NOP, nr, addr);
276 }
277
278 /**
279 * clear_bit - Clears a bit in memory
280 * @nr: Bit to clear
281 * @addr: Address to start counting from
282 *
283 * Note: there are no guarantees that this function will not be reordered
284 * on non x86 architectures, so if you are writing portable code,
285 * make sure not to rely on its reordering guarantees.
286 */
287 static inline void clear_bit(int nr, volatile unsigned long *addr)
288 {
289 __op_bit(and, __NOT, nr, addr);
290 }
291
292 /**
293 * change_bit - Toggle a bit in memory
294 * @nr: Bit to change
295 * @addr: Address to start counting from
296 *
297 * change_bit() may be reordered on other architectures than x86.
298 * Note that @nr may be almost arbitrarily large; this function is not
299 * restricted to acting on a single-word quantity.
300 */
301 static inline void change_bit(int nr, volatile unsigned long *addr)
302 {
303 __op_bit(xor, __NOP, nr, addr);
304 }
305
306 /**
307 * test_and_set_bit_lock - Set a bit and return its old value, for lock
308 * @nr: Bit to set
309 * @addr: Address to count from
310 *
311 * This operation is atomic and provides acquire barrier semantics.
312 * It can be used to implement bit locks.
313 */
314 static inline int test_and_set_bit_lock(
315 unsigned long nr, volatile unsigned long *addr)
316 {
317 return __test_and_op_bit_ord(or, __NOP, nr, addr, .aq);
318 }
319
320 /**
321 * clear_bit_unlock - Clear a bit in memory, for unlock
322 * @nr: the bit to set
323 * @addr: the address to start counting from
324 *
325 * This operation is atomic and provides release barrier semantics.
326 */
327 static inline void clear_bit_unlock(
328 unsigned long nr, volatile unsigned long *addr)
329 {
330 __op_bit_ord(and, __NOT, nr, addr, .rl);
331 }
332
333 /**
334 * __clear_bit_unlock - Clear a bit in memory, for unlock
335 * @nr: the bit to set
336 * @addr: the address to start counting from
337 *
338 * This operation is like clear_bit_unlock, however it is not atomic.
339 * It does provide release barrier semantics so it can be used to unlock
340 * a bit lock, however it would only be used if no other CPU can modify
341 * any bits in the memory until the lock is released (a good example is
342 * if the bit lock itself protects access to the other bits in the word).
343 *
344 * On RISC-V systems there seems to be no benefit to taking advantage of the
345 * non-atomic property here: it's a lot more instructions and we still have to
346 * provide release semantics anyway.
347 */
348 static inline void __clear_bit_unlock(
349 unsigned long nr, volatile unsigned long *addr)
350 {
351 clear_bit_unlock(nr, addr);
352 }
353
354 static inline bool xor_unlock_is_negative_byte(unsigned long mask,
355 volatile unsigned long *addr)
356 {
357 unsigned long res;
358 __asm__ __volatile__ (
359 __AMO(xor) ".rl %0, %2, %1"
360 : "=r" (res), "+A" (*addr)
361 : "r" (__NOP(mask))
362 : "memory");
363 return (res & BIT(7)) != 0;
364 }
365
366 #undef __test_and_op_bit
367 #undef __op_bit
368 #undef __NOP
369 #undef __NOT
370 #undef __AMO
371
372 #include <asm-generic/bitops/non-atomic.h>
373 #include <asm-generic/bitops/le.h>
374 #include <asm-generic/bitops/ext2-atomic.h>
375
376 #endif /* _ASM_RISCV_BITOPS_H */
377
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.