1 #ifdef CONFIG_SMP
2 #include "sched-pelt.h"
3
4 int __update_load_avg_blocked_se(u64 now, struct sched_entity *se);
5 int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se);
6 int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq);
7 int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
8 int update_dl_rq_load_avg(u64 now, struct rq *rq, int running);
9
10 #ifdef CONFIG_SCHED_HW_PRESSURE
11 int update_hw_load_avg(u64 now, struct rq *rq, u64 capacity);
12
13 static inline u64 hw_load_avg(struct rq *rq)
14 {
15 return READ_ONCE(rq->avg_hw.load_avg);
16 }
17 #else
18 static inline int
19 update_hw_load_avg(u64 now, struct rq *rq, u64 capacity)
20 {
21 return 0;
22 }
23
24 static inline u64 hw_load_avg(struct rq *rq)
25 {
26 return 0;
27 }
28 #endif
29
30 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
31 int update_irq_load_avg(struct rq *rq, u64 running);
32 #else
33 static inline int
34 update_irq_load_avg(struct rq *rq, u64 running)
35 {
36 return 0;
37 }
38 #endif
39
40 #define PELT_MIN_DIVIDER (LOAD_AVG_MAX - 1024)
41
42 static inline u32 get_pelt_divider(struct sched_avg *avg)
43 {
44 return PELT_MIN_DIVIDER + avg->period_contrib;
45 }
46
47 static inline void cfs_se_util_change(struct sched_avg *avg)
48 {
49 unsigned int enqueued;
50
51 if (!sched_feat(UTIL_EST))
52 return;
53
54 /* Avoid store if the flag has been already reset */
55 enqueued = avg->util_est;
56 if (!(enqueued & UTIL_AVG_UNCHANGED))
57 return;
58
59 /* Reset flag to report util_avg has been updated */
60 enqueued &= ~UTIL_AVG_UNCHANGED;
61 WRITE_ONCE(avg->util_est, enqueued);
62 }
63
64 static inline u64 rq_clock_pelt(struct rq *rq)
65 {
66 lockdep_assert_rq_held(rq);
67 assert_clock_updated(rq);
68
69 return rq->clock_pelt - rq->lost_idle_time;
70 }
71
72 /* The rq is idle, we can sync to clock_task */
73 static inline void _update_idle_rq_clock_pelt(struct rq *rq)
74 {
75 rq->clock_pelt = rq_clock_task(rq);
76
77 u64_u32_store(rq->clock_idle, rq_clock(rq));
78 /* Paired with smp_rmb in migrate_se_pelt_lag() */
79 smp_wmb();
80 u64_u32_store(rq->clock_pelt_idle, rq_clock_pelt(rq));
81 }
82
83 /*
84 * The clock_pelt scales the time to reflect the effective amount of
85 * computation done during the running delta time but then sync back to
86 * clock_task when rq is idle.
87 *
88 *
89 * absolute time | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16
90 * @ max capacity ------******---------------******---------------
91 * @ half capacity ------************---------************---------
92 * clock pelt | 1| 2| 3| 4| 7| 8| 9| 10| 11|14|15|16
93 *
94 */
95 static inline void update_rq_clock_pelt(struct rq *rq, s64 delta)
96 {
97 if (unlikely(is_idle_task(rq->curr))) {
98 _update_idle_rq_clock_pelt(rq);
99 return;
100 }
101
102 /*
103 * When a rq runs at a lower compute capacity, it will need
104 * more time to do the same amount of work than at max
105 * capacity. In order to be invariant, we scale the delta to
106 * reflect how much work has been really done.
107 * Running longer results in stealing idle time that will
108 * disturb the load signal compared to max capacity. This
109 * stolen idle time will be automatically reflected when the
110 * rq will be idle and the clock will be synced with
111 * rq_clock_task.
112 */
113
114 /*
115 * Scale the elapsed time to reflect the real amount of
116 * computation
117 */
118 delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq)));
119 delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq)));
120
121 rq->clock_pelt += delta;
122 }
123
124 /*
125 * When rq becomes idle, we have to check if it has lost idle time
126 * because it was fully busy. A rq is fully used when the /Sum util_sum
127 * is greater or equal to:
128 * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT;
129 * For optimization and computing rounding purpose, we don't take into account
130 * the position in the current window (period_contrib) and we use the higher
131 * bound of util_sum to decide.
132 */
133 static inline void update_idle_rq_clock_pelt(struct rq *rq)
134 {
135 u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX;
136 u32 util_sum = rq->cfs.avg.util_sum;
137 util_sum += rq->avg_rt.util_sum;
138 util_sum += rq->avg_dl.util_sum;
139
140 /*
141 * Reflecting stolen time makes sense only if the idle
142 * phase would be present at max capacity. As soon as the
143 * utilization of a rq has reached the maximum value, it is
144 * considered as an always running rq without idle time to
145 * steal. This potential idle time is considered as lost in
146 * this case. We keep track of this lost idle time compare to
147 * rq's clock_task.
148 */
149 if (util_sum >= divider)
150 rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt;
151
152 _update_idle_rq_clock_pelt(rq);
153 }
154
155 #ifdef CONFIG_CFS_BANDWIDTH
156 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
157 {
158 u64 throttled;
159
160 if (unlikely(cfs_rq->throttle_count))
161 throttled = U64_MAX;
162 else
163 throttled = cfs_rq->throttled_clock_pelt_time;
164
165 u64_u32_store(cfs_rq->throttled_pelt_idle, throttled);
166 }
167
168 /* rq->task_clock normalized against any time this cfs_rq has spent throttled */
169 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
170 {
171 if (unlikely(cfs_rq->throttle_count))
172 return cfs_rq->throttled_clock_pelt - cfs_rq->throttled_clock_pelt_time;
173
174 return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time;
175 }
176 #else
177 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { }
178 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
179 {
180 return rq_clock_pelt(rq_of(cfs_rq));
181 }
182 #endif
183
184 #else
185
186 static inline int
187 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
188 {
189 return 0;
190 }
191
192 static inline int
193 update_rt_rq_load_avg(u64 now, struct rq *rq, int running)
194 {
195 return 0;
196 }
197
198 static inline int
199 update_dl_rq_load_avg(u64 now, struct rq *rq, int running)
200 {
201 return 0;
202 }
203
204 static inline int
205 update_hw_load_avg(u64 now, struct rq *rq, u64 capacity)
206 {
207 return 0;
208 }
209
210 static inline u64 hw_load_avg(struct rq *rq)
211 {
212 return 0;
213 }
214
215 static inline int
216 update_irq_load_avg(struct rq *rq, u64 running)
217 {
218 return 0;
219 }
220
221 static inline u64 rq_clock_pelt(struct rq *rq)
222 {
223 return rq_clock_task(rq);
224 }
225
226 static inline void
227 update_rq_clock_pelt(struct rq *rq, s64 delta) { }
228
229 static inline void
230 update_idle_rq_clock_pelt(struct rq *rq) { }
231
232 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { }
233 #endif
234
235
236
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.