1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_SCHED_IDLE_H 2 #ifndef _LINUX_SCHED_IDLE_H
3 #define _LINUX_SCHED_IDLE_H 3 #define _LINUX_SCHED_IDLE_H
4 4
5 #include <linux/sched.h> 5 #include <linux/sched.h>
6 6
7 enum cpu_idle_type { 7 enum cpu_idle_type {
8 __CPU_NOT_IDLE = 0, <<
9 CPU_IDLE, 8 CPU_IDLE,
>> 9 CPU_NOT_IDLE,
10 CPU_NEWLY_IDLE, 10 CPU_NEWLY_IDLE,
11 CPU_MAX_IDLE_TYPES 11 CPU_MAX_IDLE_TYPES
12 }; 12 };
13 13
14 #ifdef CONFIG_SMP <<
15 extern void wake_up_if_idle(int cpu); 14 extern void wake_up_if_idle(int cpu);
16 #else <<
17 static inline void wake_up_if_idle(int cpu) { <<
18 #endif <<
19 15
20 /* 16 /*
21 * Idle thread specific functions to determine 17 * Idle thread specific functions to determine the need_resched
22 * polling state. 18 * polling state.
23 */ 19 */
24 #ifdef TIF_POLLING_NRFLAG 20 #ifdef TIF_POLLING_NRFLAG
25 21
26 #ifdef _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC !! 22 static inline void __current_set_polling(void)
27 <<
28 static __always_inline void __current_set_poll <<
29 { <<
30 arch_set_bit(TIF_POLLING_NRFLAG, <<
31 (unsigned long *)(¤ <<
32 } <<
33 <<
34 static __always_inline void __current_clr_poll <<
35 { <<
36 arch_clear_bit(TIF_POLLING_NRFLAG, <<
37 (unsigned long *)(&curr <<
38 } <<
39 <<
40 #else <<
41 <<
42 static __always_inline void __current_set_poll <<
43 { 23 {
44 set_bit(TIF_POLLING_NRFLAG, !! 24 set_thread_flag(TIF_POLLING_NRFLAG);
45 (unsigned long *)(¤t_thr <<
46 } 25 }
47 26
48 static __always_inline void __current_clr_poll !! 27 static inline bool __must_check current_set_polling_and_test(void)
49 { <<
50 clear_bit(TIF_POLLING_NRFLAG, <<
51 (unsigned long *)(¤t_t <<
52 } <<
53 <<
54 #endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_ATO <<
55 <<
56 static __always_inline bool __must_check curre <<
57 { 28 {
58 __current_set_polling(); 29 __current_set_polling();
59 30
60 /* 31 /*
61 * Polling state must be visible befor 32 * Polling state must be visible before we test NEED_RESCHED,
62 * paired by resched_curr() 33 * paired by resched_curr()
63 */ 34 */
64 smp_mb__after_atomic(); 35 smp_mb__after_atomic();
65 36
66 return unlikely(tif_need_resched()); 37 return unlikely(tif_need_resched());
67 } 38 }
68 39
69 static __always_inline bool __must_check curre !! 40 static inline void __current_clr_polling(void)
>> 41 {
>> 42 clear_thread_flag(TIF_POLLING_NRFLAG);
>> 43 }
>> 44
>> 45 static inline bool __must_check current_clr_polling_and_test(void)
70 { 46 {
71 __current_clr_polling(); 47 __current_clr_polling();
72 48
73 /* 49 /*
74 * Polling state must be visible befor 50 * Polling state must be visible before we test NEED_RESCHED,
75 * paired by resched_curr() 51 * paired by resched_curr()
76 */ 52 */
77 smp_mb__after_atomic(); 53 smp_mb__after_atomic();
78 54
79 return unlikely(tif_need_resched()); 55 return unlikely(tif_need_resched());
80 } 56 }
81 57
82 #else 58 #else
83 static inline void __current_set_polling(void) 59 static inline void __current_set_polling(void) { }
84 static inline void __current_clr_polling(void) 60 static inline void __current_clr_polling(void) { }
85 61
86 static inline bool __must_check current_set_po 62 static inline bool __must_check current_set_polling_and_test(void)
87 { 63 {
88 return unlikely(tif_need_resched()); 64 return unlikely(tif_need_resched());
89 } 65 }
90 static inline bool __must_check current_clr_po 66 static inline bool __must_check current_clr_polling_and_test(void)
91 { 67 {
92 return unlikely(tif_need_resched()); 68 return unlikely(tif_need_resched());
93 } 69 }
94 #endif 70 #endif
95 71
96 static __always_inline void current_clr_pollin !! 72 static inline void current_clr_polling(void)
97 { 73 {
98 __current_clr_polling(); 74 __current_clr_polling();
99 75
100 /* 76 /*
101 * Ensure we check TIF_NEED_RESCHED af 77 * Ensure we check TIF_NEED_RESCHED after we clear the polling bit.
102 * Once the bit is cleared, we'll get 78 * Once the bit is cleared, we'll get IPIs with every new
103 * TIF_NEED_RESCHED and the IPI handle 79 * TIF_NEED_RESCHED and the IPI handler, scheduler_ipi(), will also
104 * fold. 80 * fold.
105 */ 81 */
106 smp_mb(); /* paired with resched_curr( 82 smp_mb(); /* paired with resched_curr() */
107 83
108 preempt_fold_need_resched(); 84 preempt_fold_need_resched();
109 } 85 }
110 86
111 #endif /* _LINUX_SCHED_IDLE_H */ 87 #endif /* _LINUX_SCHED_IDLE_H */
112 88
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.