1 /* SPDX-License-Identifier: GPL-2.0 */ <<
2 #ifndef _LINUX_SCHED_IDLE_H 1 #ifndef _LINUX_SCHED_IDLE_H
3 #define _LINUX_SCHED_IDLE_H 2 #define _LINUX_SCHED_IDLE_H
4 3
5 #include <linux/sched.h> 4 #include <linux/sched.h>
6 5
7 enum cpu_idle_type { 6 enum cpu_idle_type {
8 __CPU_NOT_IDLE = 0, <<
9 CPU_IDLE, 7 CPU_IDLE,
>> 8 CPU_NOT_IDLE,
10 CPU_NEWLY_IDLE, 9 CPU_NEWLY_IDLE,
11 CPU_MAX_IDLE_TYPES 10 CPU_MAX_IDLE_TYPES
12 }; 11 };
13 12
14 #ifdef CONFIG_SMP <<
15 extern void wake_up_if_idle(int cpu); 13 extern void wake_up_if_idle(int cpu);
16 #else <<
17 static inline void wake_up_if_idle(int cpu) { <<
18 #endif <<
19 14
20 /* 15 /*
21 * Idle thread specific functions to determine 16 * Idle thread specific functions to determine the need_resched
22 * polling state. 17 * polling state.
23 */ 18 */
24 #ifdef TIF_POLLING_NRFLAG 19 #ifdef TIF_POLLING_NRFLAG
25 20
26 #ifdef _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC !! 21 static inline void __current_set_polling(void)
27 <<
28 static __always_inline void __current_set_poll <<
29 { <<
30 arch_set_bit(TIF_POLLING_NRFLAG, <<
31 (unsigned long *)(¤ <<
32 } <<
33 <<
34 static __always_inline void __current_clr_poll <<
35 { <<
36 arch_clear_bit(TIF_POLLING_NRFLAG, <<
37 (unsigned long *)(&curr <<
38 } <<
39 <<
40 #else <<
41 <<
42 static __always_inline void __current_set_poll <<
43 { 22 {
44 set_bit(TIF_POLLING_NRFLAG, !! 23 set_thread_flag(TIF_POLLING_NRFLAG);
45 (unsigned long *)(¤t_thr <<
46 } 24 }
47 25
48 static __always_inline void __current_clr_poll !! 26 static inline bool __must_check current_set_polling_and_test(void)
49 { <<
50 clear_bit(TIF_POLLING_NRFLAG, <<
51 (unsigned long *)(¤t_t <<
52 } <<
53 <<
54 #endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_ATO <<
55 <<
56 static __always_inline bool __must_check curre <<
57 { 27 {
58 __current_set_polling(); 28 __current_set_polling();
59 29
60 /* 30 /*
61 * Polling state must be visible befor 31 * Polling state must be visible before we test NEED_RESCHED,
62 * paired by resched_curr() 32 * paired by resched_curr()
63 */ 33 */
64 smp_mb__after_atomic(); 34 smp_mb__after_atomic();
65 35
66 return unlikely(tif_need_resched()); 36 return unlikely(tif_need_resched());
67 } 37 }
68 38
69 static __always_inline bool __must_check curre !! 39 static inline void __current_clr_polling(void)
>> 40 {
>> 41 clear_thread_flag(TIF_POLLING_NRFLAG);
>> 42 }
>> 43
>> 44 static inline bool __must_check current_clr_polling_and_test(void)
70 { 45 {
71 __current_clr_polling(); 46 __current_clr_polling();
72 47
73 /* 48 /*
74 * Polling state must be visible befor 49 * Polling state must be visible before we test NEED_RESCHED,
75 * paired by resched_curr() 50 * paired by resched_curr()
76 */ 51 */
77 smp_mb__after_atomic(); 52 smp_mb__after_atomic();
78 53
79 return unlikely(tif_need_resched()); 54 return unlikely(tif_need_resched());
80 } 55 }
81 56
82 #else 57 #else
83 static inline void __current_set_polling(void) 58 static inline void __current_set_polling(void) { }
84 static inline void __current_clr_polling(void) 59 static inline void __current_clr_polling(void) { }
85 60
86 static inline bool __must_check current_set_po 61 static inline bool __must_check current_set_polling_and_test(void)
87 { 62 {
88 return unlikely(tif_need_resched()); 63 return unlikely(tif_need_resched());
89 } 64 }
90 static inline bool __must_check current_clr_po 65 static inline bool __must_check current_clr_polling_and_test(void)
91 { 66 {
92 return unlikely(tif_need_resched()); 67 return unlikely(tif_need_resched());
93 } 68 }
94 #endif 69 #endif
95 70
96 static __always_inline void current_clr_pollin !! 71 static inline void current_clr_polling(void)
97 { 72 {
98 __current_clr_polling(); 73 __current_clr_polling();
99 74
100 /* 75 /*
101 * Ensure we check TIF_NEED_RESCHED af 76 * Ensure we check TIF_NEED_RESCHED after we clear the polling bit.
102 * Once the bit is cleared, we'll get 77 * Once the bit is cleared, we'll get IPIs with every new
103 * TIF_NEED_RESCHED and the IPI handle 78 * TIF_NEED_RESCHED and the IPI handler, scheduler_ipi(), will also
104 * fold. 79 * fold.
105 */ 80 */
106 smp_mb(); /* paired with resched_curr( 81 smp_mb(); /* paired with resched_curr() */
107 82
108 preempt_fold_need_resched(); 83 preempt_fold_need_resched();
109 } 84 }
110 85
111 #endif /* _LINUX_SCHED_IDLE_H */ 86 #endif /* _LINUX_SCHED_IDLE_H */
112 87
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.