1 /* SPDX-License-Identifier: GPL-2.0-only */
2 #ifndef __ASM_UNROLL_H__
3 #define __ASM_UNROLL_H__
4
5 /*
6 * Explicitly unroll a loop, for use in cases where doing so is performance
7 * critical.
8 *
9 * Ideally we'd rely upon the compiler to provide this but there's no commonly
10 * available means to do so. For example GCC's "#pragma GCC unroll"
11 * functionality would be ideal but is only available from GCC 8 onwards. Using
12 * -funroll-loops is an option but GCC tends to make poor choices when
13 * compiling our string functions. -funroll-all-loops leads to massive code
14 * bloat, even if only applied to the string functions.
15 */
16 #define unroll(times, fn, ...) do { \
17 extern void bad_unroll(void) \
18 __compiletime_error("Unsupported unroll"); \
19 \
20 /* \
21 * We can't unroll if the number of iterations isn't \
22 * compile-time constant. Unfortunately clang versions \
23 * up until 8.0 tend to miss obvious constants & cause \
24 * this check to fail, even though they go on to \
25 * generate reasonable code for the switch statement, \
26 * so we skip the sanity check for those compilers. \
27 */ \
28 BUILD_BUG_ON(!__builtin_constant_p(times)); \
29 \
30 switch (times) { \
31 case 32: fn(__VA_ARGS__); fallthrough; \
32 case 31: fn(__VA_ARGS__); fallthrough; \
33 case 30: fn(__VA_ARGS__); fallthrough; \
34 case 29: fn(__VA_ARGS__); fallthrough; \
35 case 28: fn(__VA_ARGS__); fallthrough; \
36 case 27: fn(__VA_ARGS__); fallthrough; \
37 case 26: fn(__VA_ARGS__); fallthrough; \
38 case 25: fn(__VA_ARGS__); fallthrough; \
39 case 24: fn(__VA_ARGS__); fallthrough; \
40 case 23: fn(__VA_ARGS__); fallthrough; \
41 case 22: fn(__VA_ARGS__); fallthrough; \
42 case 21: fn(__VA_ARGS__); fallthrough; \
43 case 20: fn(__VA_ARGS__); fallthrough; \
44 case 19: fn(__VA_ARGS__); fallthrough; \
45 case 18: fn(__VA_ARGS__); fallthrough; \
46 case 17: fn(__VA_ARGS__); fallthrough; \
47 case 16: fn(__VA_ARGS__); fallthrough; \
48 case 15: fn(__VA_ARGS__); fallthrough; \
49 case 14: fn(__VA_ARGS__); fallthrough; \
50 case 13: fn(__VA_ARGS__); fallthrough; \
51 case 12: fn(__VA_ARGS__); fallthrough; \
52 case 11: fn(__VA_ARGS__); fallthrough; \
53 case 10: fn(__VA_ARGS__); fallthrough; \
54 case 9: fn(__VA_ARGS__); fallthrough; \
55 case 8: fn(__VA_ARGS__); fallthrough; \
56 case 7: fn(__VA_ARGS__); fallthrough; \
57 case 6: fn(__VA_ARGS__); fallthrough; \
58 case 5: fn(__VA_ARGS__); fallthrough; \
59 case 4: fn(__VA_ARGS__); fallthrough; \
60 case 3: fn(__VA_ARGS__); fallthrough; \
61 case 2: fn(__VA_ARGS__); fallthrough; \
62 case 1: fn(__VA_ARGS__); fallthrough; \
63 case 0: break; \
64 \
65 default: \
66 /* \
67 * Either the iteration count is unreasonable \
68 * or we need to add more cases above. \
69 */ \
70 bad_unroll(); \
71 break; \
72 } \
73 } while (0)
74
75 #endif /* __ASM_UNROLL_H__ */
76
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.