1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_SCHED_IDLE_H 2 #ifndef _LINUX_SCHED_IDLE_H
3 #define _LINUX_SCHED_IDLE_H 3 #define _LINUX_SCHED_IDLE_H
4 4
5 #include <linux/sched.h> 5 #include <linux/sched.h>
6 6
7 enum cpu_idle_type { 7 enum cpu_idle_type {
8 __CPU_NOT_IDLE = 0, <<
9 CPU_IDLE, 8 CPU_IDLE,
>> 9 CPU_NOT_IDLE,
10 CPU_NEWLY_IDLE, 10 CPU_NEWLY_IDLE,
11 CPU_MAX_IDLE_TYPES 11 CPU_MAX_IDLE_TYPES
12 }; 12 };
13 13
14 #ifdef CONFIG_SMP 14 #ifdef CONFIG_SMP
15 extern void wake_up_if_idle(int cpu); 15 extern void wake_up_if_idle(int cpu);
16 #else 16 #else
17 static inline void wake_up_if_idle(int cpu) { 17 static inline void wake_up_if_idle(int cpu) { }
18 #endif 18 #endif
19 19
20 /* 20 /*
21 * Idle thread specific functions to determine 21 * Idle thread specific functions to determine the need_resched
22 * polling state. 22 * polling state.
23 */ 23 */
24 #ifdef TIF_POLLING_NRFLAG 24 #ifdef TIF_POLLING_NRFLAG
25 25
26 #ifdef _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC !! 26 static inline void __current_set_polling(void)
27 <<
28 static __always_inline void __current_set_poll <<
29 { <<
30 arch_set_bit(TIF_POLLING_NRFLAG, <<
31 (unsigned long *)(¤ <<
32 } <<
33 <<
34 static __always_inline void __current_clr_poll <<
35 { <<
36 arch_clear_bit(TIF_POLLING_NRFLAG, <<
37 (unsigned long *)(&curr <<
38 } <<
39 <<
40 #else <<
41 <<
42 static __always_inline void __current_set_poll <<
43 { 27 {
44 set_bit(TIF_POLLING_NRFLAG, !! 28 set_thread_flag(TIF_POLLING_NRFLAG);
45 (unsigned long *)(¤t_thr <<
46 } 29 }
47 30
48 static __always_inline void __current_clr_poll !! 31 static inline bool __must_check current_set_polling_and_test(void)
49 { <<
50 clear_bit(TIF_POLLING_NRFLAG, <<
51 (unsigned long *)(¤t_t <<
52 } <<
53 <<
54 #endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_ATO <<
55 <<
56 static __always_inline bool __must_check curre <<
57 { 32 {
58 __current_set_polling(); 33 __current_set_polling();
59 34
60 /* 35 /*
61 * Polling state must be visible befor 36 * Polling state must be visible before we test NEED_RESCHED,
62 * paired by resched_curr() 37 * paired by resched_curr()
63 */ 38 */
64 smp_mb__after_atomic(); 39 smp_mb__after_atomic();
65 40
66 return unlikely(tif_need_resched()); 41 return unlikely(tif_need_resched());
67 } 42 }
68 43
69 static __always_inline bool __must_check curre !! 44 static inline void __current_clr_polling(void)
>> 45 {
>> 46 clear_thread_flag(TIF_POLLING_NRFLAG);
>> 47 }
>> 48
>> 49 static inline bool __must_check current_clr_polling_and_test(void)
70 { 50 {
71 __current_clr_polling(); 51 __current_clr_polling();
72 52
73 /* 53 /*
74 * Polling state must be visible befor 54 * Polling state must be visible before we test NEED_RESCHED,
75 * paired by resched_curr() 55 * paired by resched_curr()
76 */ 56 */
77 smp_mb__after_atomic(); 57 smp_mb__after_atomic();
78 58
79 return unlikely(tif_need_resched()); 59 return unlikely(tif_need_resched());
80 } 60 }
81 61
82 #else 62 #else
83 static inline void __current_set_polling(void) 63 static inline void __current_set_polling(void) { }
84 static inline void __current_clr_polling(void) 64 static inline void __current_clr_polling(void) { }
85 65
86 static inline bool __must_check current_set_po 66 static inline bool __must_check current_set_polling_and_test(void)
87 { 67 {
88 return unlikely(tif_need_resched()); 68 return unlikely(tif_need_resched());
89 } 69 }
90 static inline bool __must_check current_clr_po 70 static inline bool __must_check current_clr_polling_and_test(void)
91 { 71 {
92 return unlikely(tif_need_resched()); 72 return unlikely(tif_need_resched());
93 } 73 }
94 #endif 74 #endif
95 75
96 static __always_inline void current_clr_pollin !! 76 static inline void current_clr_polling(void)
97 { 77 {
98 __current_clr_polling(); 78 __current_clr_polling();
99 79
100 /* 80 /*
101 * Ensure we check TIF_NEED_RESCHED af 81 * Ensure we check TIF_NEED_RESCHED after we clear the polling bit.
102 * Once the bit is cleared, we'll get 82 * Once the bit is cleared, we'll get IPIs with every new
103 * TIF_NEED_RESCHED and the IPI handle 83 * TIF_NEED_RESCHED and the IPI handler, scheduler_ipi(), will also
104 * fold. 84 * fold.
105 */ 85 */
106 smp_mb(); /* paired with resched_curr( 86 smp_mb(); /* paired with resched_curr() */
107 87
108 preempt_fold_need_resched(); 88 preempt_fold_need_resched();
109 } 89 }
110 90
111 #endif /* _LINUX_SCHED_IDLE_H */ 91 #endif /* _LINUX_SCHED_IDLE_H */
112 92
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.