1 // SPDX-License-Identifier: GPL-2.0-only <<
2 /* 1 /*
3 * linux/mm/vmstat.c 2 * linux/mm/vmstat.c
4 * 3 *
5 * Manages VM statistics 4 * Manages VM statistics
6 * Copyright (C) 1991, 1992, 1993, 1994 Linu 5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
7 * 6 *
8 * zoned VM statistics 7 * zoned VM statistics
9 * Copyright (C) 2006 Silicon Graphics, Inc., 8 * Copyright (C) 2006 Silicon Graphics, Inc.,
10 * Christoph Lameter <christoph@l 9 * Christoph Lameter <christoph@lameter.com>
11 * Copyright (C) 2008-2014 Christoph Lameter 10 * Copyright (C) 2008-2014 Christoph Lameter
12 */ 11 */
13 #include <linux/fs.h> 12 #include <linux/fs.h>
14 #include <linux/mm.h> 13 #include <linux/mm.h>
15 #include <linux/err.h> 14 #include <linux/err.h>
16 #include <linux/module.h> 15 #include <linux/module.h>
17 #include <linux/slab.h> 16 #include <linux/slab.h>
18 #include <linux/cpu.h> 17 #include <linux/cpu.h>
19 #include <linux/cpumask.h> 18 #include <linux/cpumask.h>
20 #include <linux/vmstat.h> 19 #include <linux/vmstat.h>
21 #include <linux/proc_fs.h> 20 #include <linux/proc_fs.h>
22 #include <linux/seq_file.h> 21 #include <linux/seq_file.h>
23 #include <linux/debugfs.h> 22 #include <linux/debugfs.h>
24 #include <linux/sched.h> 23 #include <linux/sched.h>
25 #include <linux/math64.h> 24 #include <linux/math64.h>
26 #include <linux/writeback.h> 25 #include <linux/writeback.h>
27 #include <linux/compaction.h> 26 #include <linux/compaction.h>
28 #include <linux/mm_inline.h> 27 #include <linux/mm_inline.h>
>> 28 #include <linux/page_ext.h>
29 #include <linux/page_owner.h> 29 #include <linux/page_owner.h>
30 #include <linux/sched/isolation.h> <<
31 30
32 #include "internal.h" 31 #include "internal.h"
33 32
34 #ifdef CONFIG_NUMA <<
35 int sysctl_vm_numa_stat = ENABLE_NUMA_STAT; <<
36 <<
37 /* zero numa counters within a zone */ <<
38 static void zero_zone_numa_counters(struct zon <<
39 { <<
40 int item, cpu; <<
41 <<
42 for (item = 0; item < NR_VM_NUMA_EVENT <<
43 atomic_long_set(&zone->vm_numa <<
44 for_each_online_cpu(cpu) { <<
45 per_cpu_ptr(zone->per_ <<
46 <<
47 } <<
48 } <<
49 } <<
50 <<
51 /* zero numa counters of all the populated zon <<
52 static void zero_zones_numa_counters(void) <<
53 { <<
54 struct zone *zone; <<
55 <<
56 for_each_populated_zone(zone) <<
57 zero_zone_numa_counters(zone); <<
58 } <<
59 <<
60 /* zero global numa counters */ <<
61 static void zero_global_numa_counters(void) <<
62 { <<
63 int item; <<
64 <<
65 for (item = 0; item < NR_VM_NUMA_EVENT <<
66 atomic_long_set(&vm_numa_event <<
67 } <<
68 <<
69 static void invalid_numa_statistics(void) <<
70 { <<
71 zero_zones_numa_counters(); <<
72 zero_global_numa_counters(); <<
73 } <<
74 <<
75 static DEFINE_MUTEX(vm_numa_stat_lock); <<
76 <<
77 int sysctl_vm_numa_stat_handler(const struct c <<
78 void *buffer, size_t *length, <<
79 { <<
80 int ret, oldval; <<
81 <<
82 mutex_lock(&vm_numa_stat_lock); <<
83 if (write) <<
84 oldval = sysctl_vm_numa_stat; <<
85 ret = proc_dointvec_minmax(table, writ <<
86 if (ret || !write) <<
87 goto out; <<
88 <<
89 if (oldval == sysctl_vm_numa_stat) <<
90 goto out; <<
91 else if (sysctl_vm_numa_stat == ENABLE <<
92 static_branch_enable(&vm_numa_ <<
93 pr_info("enable numa statistic <<
94 } else { <<
95 static_branch_disable(&vm_numa <<
96 invalid_numa_statistics(); <<
97 pr_info("disable numa statisti <<
98 } <<
99 <<
100 out: <<
101 mutex_unlock(&vm_numa_stat_lock); <<
102 return ret; <<
103 } <<
104 #endif <<
105 <<
106 #ifdef CONFIG_VM_EVENT_COUNTERS 33 #ifdef CONFIG_VM_EVENT_COUNTERS
107 DEFINE_PER_CPU(struct vm_event_state, vm_event 34 DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}};
108 EXPORT_PER_CPU_SYMBOL(vm_event_states); 35 EXPORT_PER_CPU_SYMBOL(vm_event_states);
109 36
110 static void sum_vm_events(unsigned long *ret) 37 static void sum_vm_events(unsigned long *ret)
111 { 38 {
112 int cpu; 39 int cpu;
113 int i; 40 int i;
114 41
115 memset(ret, 0, NR_VM_EVENT_ITEMS * siz 42 memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long));
116 43
117 for_each_online_cpu(cpu) { 44 for_each_online_cpu(cpu) {
118 struct vm_event_state *this = 45 struct vm_event_state *this = &per_cpu(vm_event_states, cpu);
119 46
120 for (i = 0; i < NR_VM_EVENT_IT 47 for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
121 ret[i] += this->event[ 48 ret[i] += this->event[i];
122 } 49 }
123 } 50 }
124 51
125 /* 52 /*
126 * Accumulate the vm event counters across all 53 * Accumulate the vm event counters across all CPUs.
127 * The result is unavoidably approximate - it 54 * The result is unavoidably approximate - it can change
128 * during and after execution of this function 55 * during and after execution of this function.
129 */ 56 */
130 void all_vm_events(unsigned long *ret) 57 void all_vm_events(unsigned long *ret)
131 { 58 {
132 cpus_read_lock(); !! 59 get_online_cpus();
133 sum_vm_events(ret); 60 sum_vm_events(ret);
134 cpus_read_unlock(); !! 61 put_online_cpus();
135 } 62 }
136 EXPORT_SYMBOL_GPL(all_vm_events); 63 EXPORT_SYMBOL_GPL(all_vm_events);
137 64
138 /* 65 /*
139 * Fold the foreign cpu events into our own. 66 * Fold the foreign cpu events into our own.
140 * 67 *
141 * This is adding to the events on one process 68 * This is adding to the events on one processor
142 * but keeps the global counts constant. 69 * but keeps the global counts constant.
143 */ 70 */
144 void vm_events_fold_cpu(int cpu) 71 void vm_events_fold_cpu(int cpu)
145 { 72 {
146 struct vm_event_state *fold_state = &p 73 struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu);
147 int i; 74 int i;
148 75
149 for (i = 0; i < NR_VM_EVENT_ITEMS; i++ 76 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
150 count_vm_events(i, fold_state- 77 count_vm_events(i, fold_state->event[i]);
151 fold_state->event[i] = 0; 78 fold_state->event[i] = 0;
152 } 79 }
153 } 80 }
154 81
155 #endif /* CONFIG_VM_EVENT_COUNTERS */ 82 #endif /* CONFIG_VM_EVENT_COUNTERS */
156 83
157 /* 84 /*
158 * Manage combined zone based / global counter 85 * Manage combined zone based / global counters
159 * 86 *
160 * vm_stat contains the global counters 87 * vm_stat contains the global counters
161 */ 88 */
162 atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITE 89 atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp;
163 atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITE 90 atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS] __cacheline_aligned_in_smp;
164 atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_I <<
165 EXPORT_SYMBOL(vm_zone_stat); 91 EXPORT_SYMBOL(vm_zone_stat);
166 EXPORT_SYMBOL(vm_node_stat); 92 EXPORT_SYMBOL(vm_node_stat);
167 93
168 #ifdef CONFIG_NUMA <<
169 static void fold_vm_zone_numa_events(struct zo <<
170 { <<
171 unsigned long zone_numa_events[NR_VM_N <<
172 int cpu; <<
173 enum numa_stat_item item; <<
174 <<
175 for_each_online_cpu(cpu) { <<
176 struct per_cpu_zonestat *pzsta <<
177 <<
178 pzstats = per_cpu_ptr(zone->pe <<
179 for (item = 0; item < NR_VM_NU <<
180 zone_numa_events[item] <<
181 } <<
182 <<
183 for (item = 0; item < NR_VM_NUMA_EVENT <<
184 zone_numa_event_add(zone_numa_ <<
185 } <<
186 <<
187 void fold_vm_numa_events(void) <<
188 { <<
189 struct zone *zone; <<
190 <<
191 for_each_populated_zone(zone) <<
192 fold_vm_zone_numa_events(zone) <<
193 } <<
194 #endif <<
195 <<
196 #ifdef CONFIG_SMP 94 #ifdef CONFIG_SMP
197 95
198 int calculate_pressure_threshold(struct zone * 96 int calculate_pressure_threshold(struct zone *zone)
199 { 97 {
200 int threshold; 98 int threshold;
201 int watermark_distance; 99 int watermark_distance;
202 100
203 /* 101 /*
204 * As vmstats are not up to date, ther 102 * As vmstats are not up to date, there is drift between the estimated
205 * and real values. For high threshold 103 * and real values. For high thresholds and a high number of CPUs, it
206 * is possible for the min watermark t 104 * is possible for the min watermark to be breached while the estimated
207 * value looks fine. The pressure thre 105 * value looks fine. The pressure threshold is a reduced value such
208 * that even the maximum amount of dri 106 * that even the maximum amount of drift will not accidentally breach
209 * the min watermark 107 * the min watermark
210 */ 108 */
211 watermark_distance = low_wmark_pages(z 109 watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone);
212 threshold = max(1, (int)(watermark_dis 110 threshold = max(1, (int)(watermark_distance / num_online_cpus()));
213 111
214 /* 112 /*
215 * Maximum threshold is 125 113 * Maximum threshold is 125
216 */ 114 */
217 threshold = min(125, threshold); 115 threshold = min(125, threshold);
218 116
219 return threshold; 117 return threshold;
220 } 118 }
221 119
222 int calculate_normal_threshold(struct zone *zo 120 int calculate_normal_threshold(struct zone *zone)
223 { 121 {
224 int threshold; 122 int threshold;
225 int mem; /* memory in 128 MB un 123 int mem; /* memory in 128 MB units */
226 124
227 /* 125 /*
228 * The threshold scales with the numbe 126 * The threshold scales with the number of processors and the amount
229 * of memory per zone. More memory mea 127 * of memory per zone. More memory means that we can defer updates for
230 * longer, more processors could lead 128 * longer, more processors could lead to more contention.
231 * fls() is used to have a cheap way o 129 * fls() is used to have a cheap way of logarithmic scaling.
232 * 130 *
233 * Some sample thresholds: 131 * Some sample thresholds:
234 * 132 *
235 * Threshold Processors (fls) !! 133 * Threshold Processors (fls) Zonesize fls(mem+1)
236 * ----------------------------------- 134 * ------------------------------------------------------------------
237 * 8 1 1 135 * 8 1 1 0.9-1 GB 4
238 * 16 2 2 136 * 16 2 2 0.9-1 GB 4
239 * 20 2 2 137 * 20 2 2 1-2 GB 5
240 * 24 2 2 138 * 24 2 2 2-4 GB 6
241 * 28 2 2 139 * 28 2 2 4-8 GB 7
242 * 32 2 2 140 * 32 2 2 8-16 GB 8
243 * 4 2 2 141 * 4 2 2 <128M 1
244 * 30 4 3 142 * 30 4 3 2-4 GB 5
245 * 48 4 3 143 * 48 4 3 8-16 GB 8
246 * 32 8 4 144 * 32 8 4 1-2 GB 4
247 * 32 8 4 145 * 32 8 4 0.9-1GB 4
248 * 10 16 5 146 * 10 16 5 <128M 1
249 * 40 16 5 147 * 40 16 5 900M 4
250 * 70 64 7 148 * 70 64 7 2-4 GB 5
251 * 84 64 7 149 * 84 64 7 4-8 GB 6
252 * 108 512 9 150 * 108 512 9 4-8 GB 6
253 * 125 1024 10 151 * 125 1024 10 8-16 GB 8
254 * 125 1024 10 152 * 125 1024 10 16-32 GB 9
255 */ 153 */
256 154
257 mem = zone_managed_pages(zone) >> (27 !! 155 mem = zone->managed_pages >> (27 - PAGE_SHIFT);
258 156
259 threshold = 2 * fls(num_online_cpus()) 157 threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem));
260 158
261 /* 159 /*
262 * Maximum threshold is 125 160 * Maximum threshold is 125
263 */ 161 */
264 threshold = min(125, threshold); 162 threshold = min(125, threshold);
265 163
266 return threshold; 164 return threshold;
267 } 165 }
268 166
269 /* 167 /*
270 * Refresh the thresholds for each zone. 168 * Refresh the thresholds for each zone.
271 */ 169 */
272 void refresh_zone_stat_thresholds(void) 170 void refresh_zone_stat_thresholds(void)
273 { 171 {
274 struct pglist_data *pgdat; 172 struct pglist_data *pgdat;
275 struct zone *zone; 173 struct zone *zone;
276 int cpu; 174 int cpu;
277 int threshold; 175 int threshold;
278 176
279 /* Zero current pgdat thresholds */ 177 /* Zero current pgdat thresholds */
280 for_each_online_pgdat(pgdat) { 178 for_each_online_pgdat(pgdat) {
281 for_each_online_cpu(cpu) { 179 for_each_online_cpu(cpu) {
282 per_cpu_ptr(pgdat->per 180 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = 0;
283 } 181 }
284 } 182 }
285 183
286 for_each_populated_zone(zone) { 184 for_each_populated_zone(zone) {
287 struct pglist_data *pgdat = zo 185 struct pglist_data *pgdat = zone->zone_pgdat;
288 unsigned long max_drift, toler 186 unsigned long max_drift, tolerate_drift;
289 187
290 threshold = calculate_normal_t 188 threshold = calculate_normal_threshold(zone);
291 189
292 for_each_online_cpu(cpu) { 190 for_each_online_cpu(cpu) {
293 int pgdat_threshold; 191 int pgdat_threshold;
294 192
295 per_cpu_ptr(zone->per_ !! 193 per_cpu_ptr(zone->pageset, cpu)->stat_threshold
296 194 = threshold;
297 195
298 /* Base nodestat thres 196 /* Base nodestat threshold on the largest populated zone. */
299 pgdat_threshold = per_ 197 pgdat_threshold = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold;
300 per_cpu_ptr(pgdat->per 198 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold
301 = max(threshol 199 = max(threshold, pgdat_threshold);
302 } 200 }
303 201
304 /* 202 /*
305 * Only set percpu_drift_mark 203 * Only set percpu_drift_mark if there is a danger that
306 * NR_FREE_PAGES reports the l 204 * NR_FREE_PAGES reports the low watermark is ok when in fact
307 * the min watermark could be 205 * the min watermark could be breached by an allocation
308 */ 206 */
309 tolerate_drift = low_wmark_pag 207 tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone);
310 max_drift = num_online_cpus() 208 max_drift = num_online_cpus() * threshold;
311 if (max_drift > tolerate_drift 209 if (max_drift > tolerate_drift)
312 zone->percpu_drift_mar 210 zone->percpu_drift_mark = high_wmark_pages(zone) +
313 max_dr 211 max_drift;
314 } 212 }
315 } 213 }
316 214
317 void set_pgdat_percpu_threshold(pg_data_t *pgd 215 void set_pgdat_percpu_threshold(pg_data_t *pgdat,
318 int (*calculat 216 int (*calculate_pressure)(struct zone *))
319 { 217 {
320 struct zone *zone; 218 struct zone *zone;
321 int cpu; 219 int cpu;
322 int threshold; 220 int threshold;
323 int i; 221 int i;
324 222
325 for (i = 0; i < pgdat->nr_zones; i++) 223 for (i = 0; i < pgdat->nr_zones; i++) {
326 zone = &pgdat->node_zones[i]; 224 zone = &pgdat->node_zones[i];
327 if (!zone->percpu_drift_mark) 225 if (!zone->percpu_drift_mark)
328 continue; 226 continue;
329 227
330 threshold = (*calculate_pressu 228 threshold = (*calculate_pressure)(zone);
331 for_each_online_cpu(cpu) 229 for_each_online_cpu(cpu)
332 per_cpu_ptr(zone->per_ !! 230 per_cpu_ptr(zone->pageset, cpu)->stat_threshold
333 231 = threshold;
334 } 232 }
335 } 233 }
336 234
337 /* 235 /*
338 * For use when we know that interrupts are di 236 * For use when we know that interrupts are disabled,
339 * or when we know that preemption is disabled 237 * or when we know that preemption is disabled and that
340 * particular counter cannot be updated from i 238 * particular counter cannot be updated from interrupt context.
341 */ 239 */
342 void __mod_zone_page_state(struct zone *zone, 240 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
343 long delta) 241 long delta)
344 { 242 {
345 struct per_cpu_zonestat __percpu *pcp !! 243 struct per_cpu_pageset __percpu *pcp = zone->pageset;
346 s8 __percpu *p = pcp->vm_stat_diff + i 244 s8 __percpu *p = pcp->vm_stat_diff + item;
347 long x; 245 long x;
348 long t; 246 long t;
349 247
350 /* <<
351 * Accurate vmstat updates require a R <<
352 * atomicity is provided by IRQs being <<
353 * or via local_lock_irq. On PREEMPT_R <<
354 * CPU migrations and preemption poten <<
355 * disable preemption. <<
356 */ <<
357 preempt_disable_nested(); <<
358 <<
359 x = delta + __this_cpu_read(*p); 248 x = delta + __this_cpu_read(*p);
360 249
361 t = __this_cpu_read(pcp->stat_threshol 250 t = __this_cpu_read(pcp->stat_threshold);
362 251
363 if (unlikely(abs(x) > t)) { !! 252 if (unlikely(x > t || x < -t)) {
364 zone_page_state_add(x, zone, i 253 zone_page_state_add(x, zone, item);
365 x = 0; 254 x = 0;
366 } 255 }
367 __this_cpu_write(*p, x); 256 __this_cpu_write(*p, x);
368 <<
369 preempt_enable_nested(); <<
370 } 257 }
371 EXPORT_SYMBOL(__mod_zone_page_state); 258 EXPORT_SYMBOL(__mod_zone_page_state);
372 259
373 void __mod_node_page_state(struct pglist_data 260 void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
374 long delta) 261 long delta)
375 { 262 {
376 struct per_cpu_nodestat __percpu *pcp 263 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
377 s8 __percpu *p = pcp->vm_node_stat_dif 264 s8 __percpu *p = pcp->vm_node_stat_diff + item;
378 long x; 265 long x;
379 long t; 266 long t;
380 267
381 if (vmstat_item_in_bytes(item)) { <<
382 /* <<
383 * Only cgroups use subpage ac <<
384 * the global level, these ite <<
385 * multiples of whole pages. S <<
386 * internally to keep the per- <<
387 */ <<
388 VM_WARN_ON_ONCE(delta & (PAGE_ <<
389 delta >>= PAGE_SHIFT; <<
390 } <<
391 <<
392 /* See __mod_node_page_state */ <<
393 preempt_disable_nested(); <<
394 <<
395 x = delta + __this_cpu_read(*p); 268 x = delta + __this_cpu_read(*p);
396 269
397 t = __this_cpu_read(pcp->stat_threshol 270 t = __this_cpu_read(pcp->stat_threshold);
398 271
399 if (unlikely(abs(x) > t)) { !! 272 if (unlikely(x > t || x < -t)) {
400 node_page_state_add(x, pgdat, 273 node_page_state_add(x, pgdat, item);
401 x = 0; 274 x = 0;
402 } 275 }
403 __this_cpu_write(*p, x); 276 __this_cpu_write(*p, x);
404 <<
405 preempt_enable_nested(); <<
406 } 277 }
407 EXPORT_SYMBOL(__mod_node_page_state); 278 EXPORT_SYMBOL(__mod_node_page_state);
408 279
409 /* 280 /*
410 * Optimized increment and decrement functions 281 * Optimized increment and decrement functions.
411 * 282 *
412 * These are only for a single page and theref 283 * These are only for a single page and therefore can take a struct page *
413 * argument instead of struct zone *. This all 284 * argument instead of struct zone *. This allows the inclusion of the code
414 * generated for page_zone(page) into the opti 285 * generated for page_zone(page) into the optimized functions.
415 * 286 *
416 * No overflow check is necessary and therefor 287 * No overflow check is necessary and therefore the differential can be
417 * incremented or decremented in place which m 288 * incremented or decremented in place which may allow the compilers to
418 * generate better code. 289 * generate better code.
419 * The increment or decrement is known and the 290 * The increment or decrement is known and therefore one boundary check can
420 * be omitted. 291 * be omitted.
421 * 292 *
422 * NOTE: These functions are very performance 293 * NOTE: These functions are very performance sensitive. Change only
423 * with care. 294 * with care.
424 * 295 *
425 * Some processors have inc/dec instructions t 296 * Some processors have inc/dec instructions that are atomic vs an interrupt.
426 * However, the code must first determine the 297 * However, the code must first determine the differential location in a zone
427 * based on the processor number and then inc/ 298 * based on the processor number and then inc/dec the counter. There is no
428 * guarantee without disabling preemption that 299 * guarantee without disabling preemption that the processor will not change
429 * in between and therefore the atomicity vs. 300 * in between and therefore the atomicity vs. interrupt cannot be exploited
430 * in a useful way here. 301 * in a useful way here.
431 */ 302 */
432 void __inc_zone_state(struct zone *zone, enum 303 void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
433 { 304 {
434 struct per_cpu_zonestat __percpu *pcp !! 305 struct per_cpu_pageset __percpu *pcp = zone->pageset;
435 s8 __percpu *p = pcp->vm_stat_diff + i 306 s8 __percpu *p = pcp->vm_stat_diff + item;
436 s8 v, t; 307 s8 v, t;
437 308
438 /* See __mod_node_page_state */ <<
439 preempt_disable_nested(); <<
440 <<
441 v = __this_cpu_inc_return(*p); 309 v = __this_cpu_inc_return(*p);
442 t = __this_cpu_read(pcp->stat_threshol 310 t = __this_cpu_read(pcp->stat_threshold);
443 if (unlikely(v > t)) { 311 if (unlikely(v > t)) {
444 s8 overstep = t >> 1; 312 s8 overstep = t >> 1;
445 313
446 zone_page_state_add(v + overst 314 zone_page_state_add(v + overstep, zone, item);
447 __this_cpu_write(*p, -overstep 315 __this_cpu_write(*p, -overstep);
448 } 316 }
449 <<
450 preempt_enable_nested(); <<
451 } 317 }
452 318
453 void __inc_node_state(struct pglist_data *pgda 319 void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
454 { 320 {
455 struct per_cpu_nodestat __percpu *pcp 321 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
456 s8 __percpu *p = pcp->vm_node_stat_dif 322 s8 __percpu *p = pcp->vm_node_stat_diff + item;
457 s8 v, t; 323 s8 v, t;
458 324
459 VM_WARN_ON_ONCE(vmstat_item_in_bytes(i <<
460 <<
461 /* See __mod_node_page_state */ <<
462 preempt_disable_nested(); <<
463 <<
464 v = __this_cpu_inc_return(*p); 325 v = __this_cpu_inc_return(*p);
465 t = __this_cpu_read(pcp->stat_threshol 326 t = __this_cpu_read(pcp->stat_threshold);
466 if (unlikely(v > t)) { 327 if (unlikely(v > t)) {
467 s8 overstep = t >> 1; 328 s8 overstep = t >> 1;
468 329
469 node_page_state_add(v + overst 330 node_page_state_add(v + overstep, pgdat, item);
470 __this_cpu_write(*p, -overstep 331 __this_cpu_write(*p, -overstep);
471 } 332 }
472 <<
473 preempt_enable_nested(); <<
474 } 333 }
475 334
476 void __inc_zone_page_state(struct page *page, 335 void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
477 { 336 {
478 __inc_zone_state(page_zone(page), item 337 __inc_zone_state(page_zone(page), item);
479 } 338 }
480 EXPORT_SYMBOL(__inc_zone_page_state); 339 EXPORT_SYMBOL(__inc_zone_page_state);
481 340
482 void __inc_node_page_state(struct page *page, 341 void __inc_node_page_state(struct page *page, enum node_stat_item item)
483 { 342 {
484 __inc_node_state(page_pgdat(page), ite 343 __inc_node_state(page_pgdat(page), item);
485 } 344 }
486 EXPORT_SYMBOL(__inc_node_page_state); 345 EXPORT_SYMBOL(__inc_node_page_state);
487 346
488 void __dec_zone_state(struct zone *zone, enum 347 void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
489 { 348 {
490 struct per_cpu_zonestat __percpu *pcp !! 349 struct per_cpu_pageset __percpu *pcp = zone->pageset;
491 s8 __percpu *p = pcp->vm_stat_diff + i 350 s8 __percpu *p = pcp->vm_stat_diff + item;
492 s8 v, t; 351 s8 v, t;
493 352
494 /* See __mod_node_page_state */ <<
495 preempt_disable_nested(); <<
496 <<
497 v = __this_cpu_dec_return(*p); 353 v = __this_cpu_dec_return(*p);
498 t = __this_cpu_read(pcp->stat_threshol 354 t = __this_cpu_read(pcp->stat_threshold);
499 if (unlikely(v < - t)) { 355 if (unlikely(v < - t)) {
500 s8 overstep = t >> 1; 356 s8 overstep = t >> 1;
501 357
502 zone_page_state_add(v - overst 358 zone_page_state_add(v - overstep, zone, item);
503 __this_cpu_write(*p, overstep) 359 __this_cpu_write(*p, overstep);
504 } 360 }
505 <<
506 preempt_enable_nested(); <<
507 } 361 }
508 362
509 void __dec_node_state(struct pglist_data *pgda 363 void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
510 { 364 {
511 struct per_cpu_nodestat __percpu *pcp 365 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
512 s8 __percpu *p = pcp->vm_node_stat_dif 366 s8 __percpu *p = pcp->vm_node_stat_diff + item;
513 s8 v, t; 367 s8 v, t;
514 368
515 VM_WARN_ON_ONCE(vmstat_item_in_bytes(i <<
516 <<
517 /* See __mod_node_page_state */ <<
518 preempt_disable_nested(); <<
519 <<
520 v = __this_cpu_dec_return(*p); 369 v = __this_cpu_dec_return(*p);
521 t = __this_cpu_read(pcp->stat_threshol 370 t = __this_cpu_read(pcp->stat_threshold);
522 if (unlikely(v < - t)) { 371 if (unlikely(v < - t)) {
523 s8 overstep = t >> 1; 372 s8 overstep = t >> 1;
524 373
525 node_page_state_add(v - overst 374 node_page_state_add(v - overstep, pgdat, item);
526 __this_cpu_write(*p, overstep) 375 __this_cpu_write(*p, overstep);
527 } 376 }
528 <<
529 preempt_enable_nested(); <<
530 } 377 }
531 378
532 void __dec_zone_page_state(struct page *page, 379 void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
533 { 380 {
534 __dec_zone_state(page_zone(page), item 381 __dec_zone_state(page_zone(page), item);
535 } 382 }
536 EXPORT_SYMBOL(__dec_zone_page_state); 383 EXPORT_SYMBOL(__dec_zone_page_state);
537 384
538 void __dec_node_page_state(struct page *page, 385 void __dec_node_page_state(struct page *page, enum node_stat_item item)
539 { 386 {
540 __dec_node_state(page_pgdat(page), ite 387 __dec_node_state(page_pgdat(page), item);
541 } 388 }
542 EXPORT_SYMBOL(__dec_node_page_state); 389 EXPORT_SYMBOL(__dec_node_page_state);
543 390
544 #ifdef CONFIG_HAVE_CMPXCHG_LOCAL 391 #ifdef CONFIG_HAVE_CMPXCHG_LOCAL
545 /* 392 /*
546 * If we have cmpxchg_local support then we do 393 * If we have cmpxchg_local support then we do not need to incur the overhead
547 * that comes with local_irq_save/restore if w 394 * that comes with local_irq_save/restore if we use this_cpu_cmpxchg.
548 * 395 *
549 * mod_state() modifies the zone counter state 396 * mod_state() modifies the zone counter state through atomic per cpu
550 * operations. 397 * operations.
551 * 398 *
552 * Overstep mode specifies how overstep should 399 * Overstep mode specifies how overstep should handled:
553 * 0 No overstepping 400 * 0 No overstepping
554 * 1 Overstepping half of threshold 401 * 1 Overstepping half of threshold
555 * -1 Overstepping minus half of thre 402 * -1 Overstepping minus half of threshold
556 */ 403 */
557 static inline void mod_zone_state(struct zone 404 static inline void mod_zone_state(struct zone *zone,
558 enum zone_stat_item item, long delta, i 405 enum zone_stat_item item, long delta, int overstep_mode)
559 { 406 {
560 struct per_cpu_zonestat __percpu *pcp !! 407 struct per_cpu_pageset __percpu *pcp = zone->pageset;
561 s8 __percpu *p = pcp->vm_stat_diff + i 408 s8 __percpu *p = pcp->vm_stat_diff + item;
562 long n, t, z; !! 409 long o, n, t, z;
563 s8 o; <<
564 410
565 o = this_cpu_read(*p); <<
566 do { 411 do {
567 z = 0; /* overflow to zone co 412 z = 0; /* overflow to zone counters */
568 413
569 /* 414 /*
570 * The fetching of the stat_th 415 * The fetching of the stat_threshold is racy. We may apply
571 * a counter threshold to the 416 * a counter threshold to the wrong the cpu if we get
572 * rescheduled while executing 417 * rescheduled while executing here. However, the next
573 * counter update will apply t 418 * counter update will apply the threshold again and
574 * therefore bring the counter 419 * therefore bring the counter under the threshold again.
575 * 420 *
576 * Most of the time the thresh 421 * Most of the time the thresholds are the same anyways
577 * for all cpus in a zone. 422 * for all cpus in a zone.
578 */ 423 */
579 t = this_cpu_read(pcp->stat_th 424 t = this_cpu_read(pcp->stat_threshold);
580 425
581 n = delta + (long)o; !! 426 o = this_cpu_read(*p);
>> 427 n = delta + o;
582 428
583 if (abs(n) > t) { !! 429 if (n > t || n < -t) {
584 int os = overstep_mode 430 int os = overstep_mode * (t >> 1) ;
585 431
586 /* Overflow must be ad 432 /* Overflow must be added to zone counters */
587 z = n + os; 433 z = n + os;
588 n = -os; 434 n = -os;
589 } 435 }
590 } while (!this_cpu_try_cmpxchg(*p, &o, !! 436 } while (this_cpu_cmpxchg(*p, o, n) != o);
591 437
592 if (z) 438 if (z)
593 zone_page_state_add(z, zone, i 439 zone_page_state_add(z, zone, item);
594 } 440 }
595 441
596 void mod_zone_page_state(struct zone *zone, en 442 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
597 long delta) 443 long delta)
598 { 444 {
599 mod_zone_state(zone, item, delta, 0); 445 mod_zone_state(zone, item, delta, 0);
600 } 446 }
601 EXPORT_SYMBOL(mod_zone_page_state); 447 EXPORT_SYMBOL(mod_zone_page_state);
602 448
603 void inc_zone_page_state(struct page *page, en 449 void inc_zone_page_state(struct page *page, enum zone_stat_item item)
604 { 450 {
605 mod_zone_state(page_zone(page), item, 451 mod_zone_state(page_zone(page), item, 1, 1);
606 } 452 }
607 EXPORT_SYMBOL(inc_zone_page_state); 453 EXPORT_SYMBOL(inc_zone_page_state);
608 454
609 void dec_zone_page_state(struct page *page, en 455 void dec_zone_page_state(struct page *page, enum zone_stat_item item)
610 { 456 {
611 mod_zone_state(page_zone(page), item, 457 mod_zone_state(page_zone(page), item, -1, -1);
612 } 458 }
613 EXPORT_SYMBOL(dec_zone_page_state); 459 EXPORT_SYMBOL(dec_zone_page_state);
614 460
615 static inline void mod_node_state(struct pglis 461 static inline void mod_node_state(struct pglist_data *pgdat,
616 enum node_stat_item item, int delta, in 462 enum node_stat_item item, int delta, int overstep_mode)
617 { 463 {
618 struct per_cpu_nodestat __percpu *pcp 464 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
619 s8 __percpu *p = pcp->vm_node_stat_dif 465 s8 __percpu *p = pcp->vm_node_stat_diff + item;
620 long n, t, z; !! 466 long o, n, t, z;
621 s8 o; <<
622 <<
623 if (vmstat_item_in_bytes(item)) { <<
624 /* <<
625 * Only cgroups use subpage ac <<
626 * the global level, these ite <<
627 * multiples of whole pages. S <<
628 * internally to keep the per- <<
629 */ <<
630 VM_WARN_ON_ONCE(delta & (PAGE_ <<
631 delta >>= PAGE_SHIFT; <<
632 } <<
633 467
634 o = this_cpu_read(*p); <<
635 do { 468 do {
636 z = 0; /* overflow to node co 469 z = 0; /* overflow to node counters */
637 470
638 /* 471 /*
639 * The fetching of the stat_th 472 * The fetching of the stat_threshold is racy. We may apply
640 * a counter threshold to the 473 * a counter threshold to the wrong the cpu if we get
641 * rescheduled while executing 474 * rescheduled while executing here. However, the next
642 * counter update will apply t 475 * counter update will apply the threshold again and
643 * therefore bring the counter 476 * therefore bring the counter under the threshold again.
644 * 477 *
645 * Most of the time the thresh 478 * Most of the time the thresholds are the same anyways
646 * for all cpus in a node. 479 * for all cpus in a node.
647 */ 480 */
648 t = this_cpu_read(pcp->stat_th 481 t = this_cpu_read(pcp->stat_threshold);
649 482
650 n = delta + (long)o; !! 483 o = this_cpu_read(*p);
>> 484 n = delta + o;
651 485
652 if (abs(n) > t) { !! 486 if (n > t || n < -t) {
653 int os = overstep_mode 487 int os = overstep_mode * (t >> 1) ;
654 488
655 /* Overflow must be ad 489 /* Overflow must be added to node counters */
656 z = n + os; 490 z = n + os;
657 n = -os; 491 n = -os;
658 } 492 }
659 } while (!this_cpu_try_cmpxchg(*p, &o, !! 493 } while (this_cpu_cmpxchg(*p, o, n) != o);
660 494
661 if (z) 495 if (z)
662 node_page_state_add(z, pgdat, 496 node_page_state_add(z, pgdat, item);
663 } 497 }
664 498
665 void mod_node_page_state(struct pglist_data *p 499 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
666 long d 500 long delta)
667 { 501 {
668 mod_node_state(pgdat, item, delta, 0); 502 mod_node_state(pgdat, item, delta, 0);
669 } 503 }
670 EXPORT_SYMBOL(mod_node_page_state); 504 EXPORT_SYMBOL(mod_node_page_state);
671 505
672 void inc_node_state(struct pglist_data *pgdat, 506 void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
673 { 507 {
674 mod_node_state(pgdat, item, 1, 1); 508 mod_node_state(pgdat, item, 1, 1);
675 } 509 }
676 510
677 void inc_node_page_state(struct page *page, en 511 void inc_node_page_state(struct page *page, enum node_stat_item item)
678 { 512 {
679 mod_node_state(page_pgdat(page), item, 513 mod_node_state(page_pgdat(page), item, 1, 1);
680 } 514 }
681 EXPORT_SYMBOL(inc_node_page_state); 515 EXPORT_SYMBOL(inc_node_page_state);
682 516
683 void dec_node_page_state(struct page *page, en 517 void dec_node_page_state(struct page *page, enum node_stat_item item)
684 { 518 {
685 mod_node_state(page_pgdat(page), item, 519 mod_node_state(page_pgdat(page), item, -1, -1);
686 } 520 }
687 EXPORT_SYMBOL(dec_node_page_state); 521 EXPORT_SYMBOL(dec_node_page_state);
688 #else 522 #else
689 /* 523 /*
690 * Use interrupt disable to serialize counter 524 * Use interrupt disable to serialize counter updates
691 */ 525 */
692 void mod_zone_page_state(struct zone *zone, en 526 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
693 long delta) 527 long delta)
694 { 528 {
695 unsigned long flags; 529 unsigned long flags;
696 530
697 local_irq_save(flags); 531 local_irq_save(flags);
698 __mod_zone_page_state(zone, item, delt 532 __mod_zone_page_state(zone, item, delta);
699 local_irq_restore(flags); 533 local_irq_restore(flags);
700 } 534 }
701 EXPORT_SYMBOL(mod_zone_page_state); 535 EXPORT_SYMBOL(mod_zone_page_state);
702 536
703 void inc_zone_page_state(struct page *page, en 537 void inc_zone_page_state(struct page *page, enum zone_stat_item item)
704 { 538 {
705 unsigned long flags; 539 unsigned long flags;
706 struct zone *zone; 540 struct zone *zone;
707 541
708 zone = page_zone(page); 542 zone = page_zone(page);
709 local_irq_save(flags); 543 local_irq_save(flags);
710 __inc_zone_state(zone, item); 544 __inc_zone_state(zone, item);
711 local_irq_restore(flags); 545 local_irq_restore(flags);
712 } 546 }
713 EXPORT_SYMBOL(inc_zone_page_state); 547 EXPORT_SYMBOL(inc_zone_page_state);
714 548
715 void dec_zone_page_state(struct page *page, en 549 void dec_zone_page_state(struct page *page, enum zone_stat_item item)
716 { 550 {
717 unsigned long flags; 551 unsigned long flags;
718 552
719 local_irq_save(flags); 553 local_irq_save(flags);
720 __dec_zone_page_state(page, item); 554 __dec_zone_page_state(page, item);
721 local_irq_restore(flags); 555 local_irq_restore(flags);
722 } 556 }
723 EXPORT_SYMBOL(dec_zone_page_state); 557 EXPORT_SYMBOL(dec_zone_page_state);
724 558
725 void inc_node_state(struct pglist_data *pgdat, 559 void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
726 { 560 {
727 unsigned long flags; 561 unsigned long flags;
728 562
729 local_irq_save(flags); 563 local_irq_save(flags);
730 __inc_node_state(pgdat, item); 564 __inc_node_state(pgdat, item);
731 local_irq_restore(flags); 565 local_irq_restore(flags);
732 } 566 }
733 EXPORT_SYMBOL(inc_node_state); 567 EXPORT_SYMBOL(inc_node_state);
734 568
735 void mod_node_page_state(struct pglist_data *p 569 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
736 long d 570 long delta)
737 { 571 {
738 unsigned long flags; 572 unsigned long flags;
739 573
740 local_irq_save(flags); 574 local_irq_save(flags);
741 __mod_node_page_state(pgdat, item, del 575 __mod_node_page_state(pgdat, item, delta);
742 local_irq_restore(flags); 576 local_irq_restore(flags);
743 } 577 }
744 EXPORT_SYMBOL(mod_node_page_state); 578 EXPORT_SYMBOL(mod_node_page_state);
745 579
746 void inc_node_page_state(struct page *page, en 580 void inc_node_page_state(struct page *page, enum node_stat_item item)
747 { 581 {
748 unsigned long flags; 582 unsigned long flags;
749 struct pglist_data *pgdat; 583 struct pglist_data *pgdat;
750 584
751 pgdat = page_pgdat(page); 585 pgdat = page_pgdat(page);
752 local_irq_save(flags); 586 local_irq_save(flags);
753 __inc_node_state(pgdat, item); 587 __inc_node_state(pgdat, item);
754 local_irq_restore(flags); 588 local_irq_restore(flags);
755 } 589 }
756 EXPORT_SYMBOL(inc_node_page_state); 590 EXPORT_SYMBOL(inc_node_page_state);
757 591
758 void dec_node_page_state(struct page *page, en 592 void dec_node_page_state(struct page *page, enum node_stat_item item)
759 { 593 {
760 unsigned long flags; 594 unsigned long flags;
761 595
762 local_irq_save(flags); 596 local_irq_save(flags);
763 __dec_node_page_state(page, item); 597 __dec_node_page_state(page, item);
764 local_irq_restore(flags); 598 local_irq_restore(flags);
765 } 599 }
766 EXPORT_SYMBOL(dec_node_page_state); 600 EXPORT_SYMBOL(dec_node_page_state);
767 #endif 601 #endif
768 602
769 /* 603 /*
770 * Fold a differential into the global counter 604 * Fold a differential into the global counters.
771 * Returns the number of counters updated. 605 * Returns the number of counters updated.
772 */ 606 */
773 static int fold_diff(int *zone_diff, int *node 607 static int fold_diff(int *zone_diff, int *node_diff)
774 { 608 {
775 int i; 609 int i;
776 int changes = 0; 610 int changes = 0;
777 611
778 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; 612 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
779 if (zone_diff[i]) { 613 if (zone_diff[i]) {
780 atomic_long_add(zone_d 614 atomic_long_add(zone_diff[i], &vm_zone_stat[i]);
781 changes++; 615 changes++;
782 } 616 }
783 617
784 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; 618 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
785 if (node_diff[i]) { 619 if (node_diff[i]) {
786 atomic_long_add(node_d 620 atomic_long_add(node_diff[i], &vm_node_stat[i]);
787 changes++; 621 changes++;
788 } 622 }
789 return changes; 623 return changes;
790 } 624 }
791 625
792 /* 626 /*
793 * Update the zone counters for the current cp 627 * Update the zone counters for the current cpu.
794 * 628 *
795 * Note that refresh_cpu_vm_stats strives to o 629 * Note that refresh_cpu_vm_stats strives to only access
796 * node local memory. The per cpu pagesets on 630 * node local memory. The per cpu pagesets on remote zones are placed
797 * in the memory local to the processor using 631 * in the memory local to the processor using that pageset. So the
798 * loop over all zones will access a series of 632 * loop over all zones will access a series of cachelines local to
799 * the processor. 633 * the processor.
800 * 634 *
801 * The call to zone_page_state_add updates the 635 * The call to zone_page_state_add updates the cachelines with the
802 * statistics in the remote zone struct as wel 636 * statistics in the remote zone struct as well as the global cachelines
803 * with the global counters. These could cause 637 * with the global counters. These could cause remote node cache line
804 * bouncing and will have to be only done when 638 * bouncing and will have to be only done when necessary.
805 * 639 *
806 * The function returns the number of global c 640 * The function returns the number of global counters updated.
807 */ 641 */
808 static int refresh_cpu_vm_stats(bool do_pagese 642 static int refresh_cpu_vm_stats(bool do_pagesets)
809 { 643 {
810 struct pglist_data *pgdat; 644 struct pglist_data *pgdat;
811 struct zone *zone; 645 struct zone *zone;
812 int i; 646 int i;
813 int global_zone_diff[NR_VM_ZONE_STAT_I 647 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, };
814 int global_node_diff[NR_VM_NODE_STAT_I 648 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, };
815 int changes = 0; 649 int changes = 0;
816 650
817 for_each_populated_zone(zone) { 651 for_each_populated_zone(zone) {
818 struct per_cpu_zonestat __perc !! 652 struct per_cpu_pageset __percpu *p = zone->pageset;
819 struct per_cpu_pages __percpu <<
820 653
821 for (i = 0; i < NR_VM_ZONE_STA 654 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
822 int v; 655 int v;
823 656
824 v = this_cpu_xchg(pzst !! 657 v = this_cpu_xchg(p->vm_stat_diff[i], 0);
825 if (v) { 658 if (v) {
826 659
827 atomic_long_ad 660 atomic_long_add(v, &zone->vm_stat[i]);
828 global_zone_di 661 global_zone_diff[i] += v;
829 #ifdef CONFIG_NUMA 662 #ifdef CONFIG_NUMA
830 /* 3 seconds i 663 /* 3 seconds idle till flush */
831 __this_cpu_wri !! 664 __this_cpu_write(p->expire, 3);
832 #endif 665 #endif
833 } 666 }
834 } 667 }
835 !! 668 #ifdef CONFIG_NUMA
836 if (do_pagesets) { 669 if (do_pagesets) {
837 cond_resched(); 670 cond_resched();
838 <<
839 changes += decay_pcp_h <<
840 #ifdef CONFIG_NUMA <<
841 /* 671 /*
842 * Deal with draining 672 * Deal with draining the remote pageset of this
843 * processor 673 * processor
844 * 674 *
845 * Check if there are 675 * Check if there are pages remaining in this pageset
846 * if not then there i 676 * if not then there is nothing to expire.
847 */ 677 */
848 if (!__this_cpu_read(p !! 678 if (!__this_cpu_read(p->expire) ||
849 !__this_cpu_rea !! 679 !__this_cpu_read(p->pcp.count))
850 continue; 680 continue;
851 681
852 /* 682 /*
853 * We never drain zone 683 * We never drain zones local to this processor.
854 */ 684 */
855 if (zone_to_nid(zone) 685 if (zone_to_nid(zone) == numa_node_id()) {
856 __this_cpu_wri !! 686 __this_cpu_write(p->expire, 0);
857 continue; 687 continue;
858 } 688 }
859 689
860 if (__this_cpu_dec_ret !! 690 if (__this_cpu_dec_return(p->expire))
861 changes++; <<
862 continue; 691 continue;
863 } <<
864 692
865 if (__this_cpu_read(pc !! 693 if (__this_cpu_read(p->pcp.count)) {
866 drain_zone_pag !! 694 drain_zone_pages(zone, this_cpu_ptr(&p->pcp));
867 changes++; 695 changes++;
868 } 696 }
869 #endif <<
870 } 697 }
>> 698 #endif
871 } 699 }
872 700
873 for_each_online_pgdat(pgdat) { 701 for_each_online_pgdat(pgdat) {
874 struct per_cpu_nodestat __perc 702 struct per_cpu_nodestat __percpu *p = pgdat->per_cpu_nodestats;
875 703
876 for (i = 0; i < NR_VM_NODE_STA 704 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
877 int v; 705 int v;
878 706
879 v = this_cpu_xchg(p->v 707 v = this_cpu_xchg(p->vm_node_stat_diff[i], 0);
880 if (v) { 708 if (v) {
881 atomic_long_ad 709 atomic_long_add(v, &pgdat->vm_stat[i]);
882 global_node_di 710 global_node_diff[i] += v;
883 } 711 }
884 } 712 }
885 } 713 }
886 714
887 changes += fold_diff(global_zone_diff, 715 changes += fold_diff(global_zone_diff, global_node_diff);
888 return changes; 716 return changes;
889 } 717 }
890 718
891 /* 719 /*
892 * Fold the data for an offline cpu into the g 720 * Fold the data for an offline cpu into the global array.
893 * There cannot be any access by the offline c 721 * There cannot be any access by the offline cpu and therefore
894 * synchronization is simplified. 722 * synchronization is simplified.
895 */ 723 */
896 void cpu_vm_stats_fold(int cpu) 724 void cpu_vm_stats_fold(int cpu)
897 { 725 {
898 struct pglist_data *pgdat; 726 struct pglist_data *pgdat;
899 struct zone *zone; 727 struct zone *zone;
900 int i; 728 int i;
901 int global_zone_diff[NR_VM_ZONE_STAT_I 729 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, };
902 int global_node_diff[NR_VM_NODE_STAT_I 730 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, };
903 731
904 for_each_populated_zone(zone) { 732 for_each_populated_zone(zone) {
905 struct per_cpu_zonestat *pzsta !! 733 struct per_cpu_pageset *p;
906 734
907 pzstats = per_cpu_ptr(zone->pe !! 735 p = per_cpu_ptr(zone->pageset, cpu);
908 736
909 for (i = 0; i < NR_VM_ZONE_STA !! 737 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
910 if (pzstats->vm_stat_d !! 738 if (p->vm_stat_diff[i]) {
911 int v; 739 int v;
912 740
913 v = pzstats->v !! 741 v = p->vm_stat_diff[i];
914 pzstats->vm_st !! 742 p->vm_stat_diff[i] = 0;
915 atomic_long_ad 743 atomic_long_add(v, &zone->vm_stat[i]);
916 global_zone_di 744 global_zone_diff[i] += v;
917 } 745 }
918 } <<
919 #ifdef CONFIG_NUMA <<
920 for (i = 0; i < NR_VM_NUMA_EVE <<
921 if (pzstats->vm_numa_e <<
922 unsigned long <<
923 <<
924 v = pzstats->v <<
925 pzstats->vm_nu <<
926 zone_numa_even <<
927 } <<
928 } <<
929 #endif <<
930 } 746 }
931 747
932 for_each_online_pgdat(pgdat) { 748 for_each_online_pgdat(pgdat) {
933 struct per_cpu_nodestat *p; 749 struct per_cpu_nodestat *p;
934 750
935 p = per_cpu_ptr(pgdat->per_cpu 751 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu);
936 752
937 for (i = 0; i < NR_VM_NODE_STA 753 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
938 if (p->vm_node_stat_di 754 if (p->vm_node_stat_diff[i]) {
939 int v; 755 int v;
940 756
941 v = p->vm_node 757 v = p->vm_node_stat_diff[i];
942 p->vm_node_sta 758 p->vm_node_stat_diff[i] = 0;
943 atomic_long_ad 759 atomic_long_add(v, &pgdat->vm_stat[i]);
944 global_node_di 760 global_node_diff[i] += v;
945 } 761 }
946 } 762 }
947 763
948 fold_diff(global_zone_diff, global_nod 764 fold_diff(global_zone_diff, global_node_diff);
949 } 765 }
950 766
951 /* 767 /*
952 * this is only called if !populated_zone(zone 768 * this is only called if !populated_zone(zone), which implies no other users of
953 * pset->vm_stat_diff[] exist. !! 769 * pset->vm_stat_diff[] exsist.
954 */ 770 */
955 void drain_zonestat(struct zone *zone, struct !! 771 void drain_zonestat(struct zone *zone, struct per_cpu_pageset *pset)
956 { 772 {
957 unsigned long v; <<
958 int i; 773 int i;
959 774
960 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; !! 775 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
961 if (pzstats->vm_stat_diff[i]) !! 776 if (pset->vm_stat_diff[i]) {
962 v = pzstats->vm_stat_d !! 777 int v = pset->vm_stat_diff[i];
963 pzstats->vm_stat_diff[ !! 778 pset->vm_stat_diff[i] = 0;
964 zone_page_state_add(v, !! 779 atomic_long_add(v, &zone->vm_stat[i]);
965 } !! 780 atomic_long_add(v, &vm_zone_stat[i]);
966 } <<
967 <<
968 #ifdef CONFIG_NUMA <<
969 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS <<
970 if (pzstats->vm_numa_event[i]) <<
971 v = pzstats->vm_numa_e <<
972 pzstats->vm_numa_event <<
973 zone_numa_event_add(v, <<
974 } 781 }
975 } <<
976 #endif <<
977 } 782 }
978 #endif 783 #endif
979 784
980 #ifdef CONFIG_NUMA 785 #ifdef CONFIG_NUMA
981 /* 786 /*
982 * Determine the per node value of a stat item 787 * Determine the per node value of a stat item. This function
983 * is called frequently in a NUMA machine, so 788 * is called frequently in a NUMA machine, so try to be as
984 * frugal as possible. 789 * frugal as possible.
985 */ 790 */
986 unsigned long sum_zone_node_page_state(int nod 791 unsigned long sum_zone_node_page_state(int node,
987 enum zone_sta 792 enum zone_stat_item item)
988 { 793 {
989 struct zone *zones = NODE_DATA(node)-> 794 struct zone *zones = NODE_DATA(node)->node_zones;
990 int i; 795 int i;
991 unsigned long count = 0; 796 unsigned long count = 0;
992 797
993 for (i = 0; i < MAX_NR_ZONES; i++) 798 for (i = 0; i < MAX_NR_ZONES; i++)
994 count += zone_page_state(zones 799 count += zone_page_state(zones + i, item);
995 800
996 return count; 801 return count;
997 } 802 }
998 803
999 /* Determine the per node value of a numa stat <<
1000 unsigned long sum_zone_numa_event_state(int n <<
1001 enum numa_st <<
1002 { <<
1003 struct zone *zones = NODE_DATA(node)- <<
1004 unsigned long count = 0; <<
1005 int i; <<
1006 <<
1007 for (i = 0; i < MAX_NR_ZONES; i++) <<
1008 count += zone_numa_event_stat <<
1009 <<
1010 return count; <<
1011 } <<
1012 <<
1013 /* 804 /*
1014 * Determine the per node value of a stat ite 805 * Determine the per node value of a stat item.
1015 */ 806 */
1016 unsigned long node_page_state_pages(struct pg !! 807 unsigned long node_page_state(struct pglist_data *pgdat,
1017 enum node !! 808 enum node_stat_item item)
1018 { 809 {
1019 long x = atomic_long_read(&pgdat->vm_ 810 long x = atomic_long_read(&pgdat->vm_stat[item]);
1020 #ifdef CONFIG_SMP 811 #ifdef CONFIG_SMP
1021 if (x < 0) 812 if (x < 0)
1022 x = 0; 813 x = 0;
1023 #endif 814 #endif
1024 return x; 815 return x;
1025 } 816 }
1026 <<
1027 unsigned long node_page_state(struct pglist_d <<
1028 enum node_stat_ <<
1029 { <<
1030 VM_WARN_ON_ONCE(vmstat_item_in_bytes( <<
1031 <<
1032 return node_page_state_pages(pgdat, i <<
1033 } <<
1034 #endif 817 #endif
1035 818
1036 /* <<
1037 * Count number of pages "struct page" and "s <<
1038 * nr_memmap_boot_pages: # of pages allocated <<
1039 * nr_memmap_pages: # of pages that were allo <<
1040 */ <<
1041 static atomic_long_t nr_memmap_boot_pages = A <<
1042 static atomic_long_t nr_memmap_pages = ATOMIC <<
1043 <<
1044 void memmap_boot_pages_add(long delta) <<
1045 { <<
1046 atomic_long_add(delta, &nr_memmap_boo <<
1047 } <<
1048 <<
1049 void memmap_pages_add(long delta) <<
1050 { <<
1051 atomic_long_add(delta, &nr_memmap_pag <<
1052 } <<
1053 <<
1054 #ifdef CONFIG_COMPACTION 819 #ifdef CONFIG_COMPACTION
1055 820
1056 struct contig_page_info { 821 struct contig_page_info {
1057 unsigned long free_pages; 822 unsigned long free_pages;
1058 unsigned long free_blocks_total; 823 unsigned long free_blocks_total;
1059 unsigned long free_blocks_suitable; 824 unsigned long free_blocks_suitable;
1060 }; 825 };
1061 826
1062 /* 827 /*
1063 * Calculate the number of free pages in a zo 828 * Calculate the number of free pages in a zone, how many contiguous
1064 * pages are free and how many are large enou 829 * pages are free and how many are large enough to satisfy an allocation of
1065 * the target size. Note that this function m 830 * the target size. Note that this function makes no attempt to estimate
1066 * how many suitable free blocks there *might 831 * how many suitable free blocks there *might* be if MOVABLE pages were
1067 * migrated. Calculating that is possible, bu 832 * migrated. Calculating that is possible, but expensive and can be
1068 * figured out from userspace 833 * figured out from userspace
1069 */ 834 */
1070 static void fill_contig_page_info(struct zone 835 static void fill_contig_page_info(struct zone *zone,
1071 unsigned int 836 unsigned int suitable_order,
1072 struct contig 837 struct contig_page_info *info)
1073 { 838 {
1074 unsigned int order; 839 unsigned int order;
1075 840
1076 info->free_pages = 0; 841 info->free_pages = 0;
1077 info->free_blocks_total = 0; 842 info->free_blocks_total = 0;
1078 info->free_blocks_suitable = 0; 843 info->free_blocks_suitable = 0;
1079 844
1080 for (order = 0; order < NR_PAGE_ORDER !! 845 for (order = 0; order < MAX_ORDER; order++) {
1081 unsigned long blocks; 846 unsigned long blocks;
1082 847
1083 /* !! 848 /* Count number of free blocks */
1084 * Count number of free block !! 849 blocks = zone->free_area[order].nr_free;
1085 * <<
1086 * Access to nr_free is lockl <<
1087 * diagnostic purposes. Use d <<
1088 */ <<
1089 blocks = data_race(zone->free <<
1090 info->free_blocks_total += bl 850 info->free_blocks_total += blocks;
1091 851
1092 /* Count free base pages */ 852 /* Count free base pages */
1093 info->free_pages += blocks << 853 info->free_pages += blocks << order;
1094 854
1095 /* Count the suitable free bl 855 /* Count the suitable free blocks */
1096 if (order >= suitable_order) 856 if (order >= suitable_order)
1097 info->free_blocks_sui 857 info->free_blocks_suitable += blocks <<
1098 858 (order - suitable_order);
1099 } 859 }
1100 } 860 }
1101 861
1102 /* 862 /*
1103 * A fragmentation index only makes sense if 863 * A fragmentation index only makes sense if an allocation of a requested
1104 * size would fail. If that is true, the frag 864 * size would fail. If that is true, the fragmentation index indicates
1105 * whether external fragmentation or a lack o 865 * whether external fragmentation or a lack of memory was the problem.
1106 * The value can be used to determine if page 866 * The value can be used to determine if page reclaim or compaction
1107 * should be used 867 * should be used
1108 */ 868 */
1109 static int __fragmentation_index(unsigned int 869 static int __fragmentation_index(unsigned int order, struct contig_page_info *info)
1110 { 870 {
1111 unsigned long requested = 1UL << orde 871 unsigned long requested = 1UL << order;
1112 872
1113 if (WARN_ON_ONCE(order > MAX_PAGE_ORD <<
1114 return 0; <<
1115 <<
1116 if (!info->free_blocks_total) 873 if (!info->free_blocks_total)
1117 return 0; 874 return 0;
1118 875
1119 /* Fragmentation index only makes sen 876 /* Fragmentation index only makes sense when a request would fail */
1120 if (info->free_blocks_suitable) 877 if (info->free_blocks_suitable)
1121 return -1000; 878 return -1000;
1122 879
1123 /* 880 /*
1124 * Index is between 0 and 1 so return 881 * Index is between 0 and 1 so return within 3 decimal places
1125 * 882 *
1126 * 0 => allocation would fail due to 883 * 0 => allocation would fail due to lack of memory
1127 * 1 => allocation would fail due to 884 * 1 => allocation would fail due to fragmentation
1128 */ 885 */
1129 return 1000 - div_u64( (1000+(div_u64 886 return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total);
1130 } 887 }
1131 888
1132 /* <<
1133 * Calculates external fragmentation within a <<
1134 * It is defined as the percentage of pages f <<
1135 * less than 1 << order. It returns values in <<
1136 */ <<
1137 unsigned int extfrag_for_order(struct zone *z <<
1138 { <<
1139 struct contig_page_info info; <<
1140 <<
1141 fill_contig_page_info(zone, order, &i <<
1142 if (info.free_pages == 0) <<
1143 return 0; <<
1144 <<
1145 return div_u64((info.free_pages - <<
1146 (info.free_blocks_sui <<
1147 info.free_pages); <<
1148 } <<
1149 <<
1150 /* Same as __fragmentation index but allocs c 889 /* Same as __fragmentation index but allocs contig_page_info on stack */
1151 int fragmentation_index(struct zone *zone, un 890 int fragmentation_index(struct zone *zone, unsigned int order)
1152 { 891 {
1153 struct contig_page_info info; 892 struct contig_page_info info;
1154 893
1155 fill_contig_page_info(zone, order, &i 894 fill_contig_page_info(zone, order, &info);
1156 return __fragmentation_index(order, & 895 return __fragmentation_index(order, &info);
1157 } 896 }
1158 #endif 897 #endif
1159 898
1160 #if defined(CONFIG_PROC_FS) || defined(CONFIG !! 899 #if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || defined(CONFIG_NUMA)
1161 defined(CONFIG_NUMA) || defined(CONFIG_ME <<
1162 #ifdef CONFIG_ZONE_DMA 900 #ifdef CONFIG_ZONE_DMA
1163 #define TEXT_FOR_DMA(xx) xx "_dma", 901 #define TEXT_FOR_DMA(xx) xx "_dma",
1164 #else 902 #else
1165 #define TEXT_FOR_DMA(xx) 903 #define TEXT_FOR_DMA(xx)
1166 #endif 904 #endif
1167 905
1168 #ifdef CONFIG_ZONE_DMA32 906 #ifdef CONFIG_ZONE_DMA32
1169 #define TEXT_FOR_DMA32(xx) xx "_dma32", 907 #define TEXT_FOR_DMA32(xx) xx "_dma32",
1170 #else 908 #else
1171 #define TEXT_FOR_DMA32(xx) 909 #define TEXT_FOR_DMA32(xx)
1172 #endif 910 #endif
1173 911
1174 #ifdef CONFIG_HIGHMEM 912 #ifdef CONFIG_HIGHMEM
1175 #define TEXT_FOR_HIGHMEM(xx) xx "_high", 913 #define TEXT_FOR_HIGHMEM(xx) xx "_high",
1176 #else 914 #else
1177 #define TEXT_FOR_HIGHMEM(xx) 915 #define TEXT_FOR_HIGHMEM(xx)
1178 #endif 916 #endif
1179 917
1180 #ifdef CONFIG_ZONE_DEVICE <<
1181 #define TEXT_FOR_DEVICE(xx) xx "_device", <<
1182 #else <<
1183 #define TEXT_FOR_DEVICE(xx) <<
1184 #endif <<
1185 <<
1186 #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) 918 #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) TEXT_FOR_DMA32(xx) xx "_normal", \
1187 TEXT_ !! 919 TEXT_FOR_HIGHMEM(xx) xx "_movable",
1188 TEXT_ <<
1189 920
1190 const char * const vmstat_text[] = { 921 const char * const vmstat_text[] = {
1191 /* enum zone_stat_item counters */ !! 922 /* enum zone_stat_item countes */
1192 "nr_free_pages", 923 "nr_free_pages",
1193 "nr_zone_inactive_anon", 924 "nr_zone_inactive_anon",
1194 "nr_zone_active_anon", 925 "nr_zone_active_anon",
1195 "nr_zone_inactive_file", 926 "nr_zone_inactive_file",
1196 "nr_zone_active_file", 927 "nr_zone_active_file",
1197 "nr_zone_unevictable", 928 "nr_zone_unevictable",
1198 "nr_zone_write_pending", 929 "nr_zone_write_pending",
1199 "nr_mlock", 930 "nr_mlock",
>> 931 "nr_slab_reclaimable",
>> 932 "nr_slab_unreclaimable",
>> 933 "nr_page_table_pages",
>> 934 "nr_kernel_stack",
>> 935 "nr_overhead",
1200 "nr_bounce", 936 "nr_bounce",
1201 #if IS_ENABLED(CONFIG_ZSMALLOC) 937 #if IS_ENABLED(CONFIG_ZSMALLOC)
1202 "nr_zspages", 938 "nr_zspages",
1203 #endif 939 #endif
1204 "nr_free_cma", <<
1205 #ifdef CONFIG_UNACCEPTED_MEMORY <<
1206 "nr_unaccepted", <<
1207 #endif <<
1208 <<
1209 /* enum numa_stat_item counters */ <<
1210 #ifdef CONFIG_NUMA 940 #ifdef CONFIG_NUMA
1211 "numa_hit", 941 "numa_hit",
1212 "numa_miss", 942 "numa_miss",
1213 "numa_foreign", 943 "numa_foreign",
1214 "numa_interleave", 944 "numa_interleave",
1215 "numa_local", 945 "numa_local",
1216 "numa_other", 946 "numa_other",
1217 #endif 947 #endif
>> 948 "nr_free_cma",
1218 949
1219 /* enum node_stat_item counters */ !! 950 /* Node-based counters */
1220 "nr_inactive_anon", 951 "nr_inactive_anon",
1221 "nr_active_anon", 952 "nr_active_anon",
1222 "nr_inactive_file", 953 "nr_inactive_file",
1223 "nr_active_file", 954 "nr_active_file",
1224 "nr_unevictable", 955 "nr_unevictable",
1225 "nr_slab_reclaimable", <<
1226 "nr_slab_unreclaimable", <<
1227 "nr_isolated_anon", 956 "nr_isolated_anon",
1228 "nr_isolated_file", 957 "nr_isolated_file",
1229 "workingset_nodes", !! 958 "nr_pages_scanned",
1230 "workingset_refault_anon", !! 959 "workingset_refault",
1231 "workingset_refault_file", !! 960 "workingset_activate",
1232 "workingset_activate_anon", <<
1233 "workingset_activate_file", <<
1234 "workingset_restore_anon", <<
1235 "workingset_restore_file", <<
1236 "workingset_nodereclaim", 961 "workingset_nodereclaim",
1237 "nr_anon_pages", 962 "nr_anon_pages",
1238 "nr_mapped", 963 "nr_mapped",
1239 "nr_file_pages", 964 "nr_file_pages",
1240 "nr_dirty", 965 "nr_dirty",
1241 "nr_writeback", 966 "nr_writeback",
1242 "nr_writeback_temp", 967 "nr_writeback_temp",
1243 "nr_shmem", 968 "nr_shmem",
1244 "nr_shmem_hugepages", 969 "nr_shmem_hugepages",
1245 "nr_shmem_pmdmapped", 970 "nr_shmem_pmdmapped",
1246 "nr_file_hugepages", <<
1247 "nr_file_pmdmapped", <<
1248 "nr_anon_transparent_hugepages", 971 "nr_anon_transparent_hugepages",
>> 972 "nr_unstable",
1249 "nr_vmscan_write", 973 "nr_vmscan_write",
1250 "nr_vmscan_immediate_reclaim", 974 "nr_vmscan_immediate_reclaim",
1251 "nr_dirtied", 975 "nr_dirtied",
1252 "nr_written", 976 "nr_written",
1253 "nr_throttled_written", !! 977
1254 "nr_kernel_misc_reclaimable", !! 978 /* enum writeback_stat_item counters */
1255 "nr_foll_pin_acquired", <<
1256 "nr_foll_pin_released", <<
1257 "nr_kernel_stack", <<
1258 #if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) <<
1259 "nr_shadow_call_stack", <<
1260 #endif <<
1261 "nr_page_table_pages", <<
1262 "nr_sec_page_table_pages", <<
1263 #ifdef CONFIG_IOMMU_SUPPORT <<
1264 "nr_iommu_pages", <<
1265 #endif <<
1266 #ifdef CONFIG_SWAP <<
1267 "nr_swapcached", <<
1268 #endif <<
1269 #ifdef CONFIG_NUMA_BALANCING <<
1270 "pgpromote_success", <<
1271 "pgpromote_candidate", <<
1272 #endif <<
1273 "pgdemote_kswapd", <<
1274 "pgdemote_direct", <<
1275 "pgdemote_khugepaged", <<
1276 /* system-wide enum vm_stat_item coun <<
1277 "nr_dirty_threshold", 979 "nr_dirty_threshold",
1278 "nr_dirty_background_threshold", 980 "nr_dirty_background_threshold",
1279 "nr_memmap_pages", <<
1280 "nr_memmap_boot_pages", <<
1281 981
1282 #if defined(CONFIG_VM_EVENT_COUNTERS) || defi !! 982 #ifdef CONFIG_VM_EVENT_COUNTERS
1283 /* enum vm_event_item counters */ 983 /* enum vm_event_item counters */
1284 "pgpgin", 984 "pgpgin",
1285 "pgpgout", 985 "pgpgout",
1286 "pswpin", 986 "pswpin",
1287 "pswpout", 987 "pswpout",
1288 988
1289 TEXTS_FOR_ZONES("pgalloc") 989 TEXTS_FOR_ZONES("pgalloc")
1290 TEXTS_FOR_ZONES("allocstall") 990 TEXTS_FOR_ZONES("allocstall")
1291 TEXTS_FOR_ZONES("pgskip") 991 TEXTS_FOR_ZONES("pgskip")
1292 992
1293 "pgfree", 993 "pgfree",
1294 "pgactivate", 994 "pgactivate",
1295 "pgdeactivate", 995 "pgdeactivate",
1296 "pglazyfree", <<
1297 996
1298 "pgfault", 997 "pgfault",
1299 "pgmajfault", 998 "pgmajfault",
1300 "pglazyfreed", 999 "pglazyfreed",
1301 1000
1302 "pgrefill", 1001 "pgrefill",
1303 "pgreuse", <<
1304 "pgsteal_kswapd", 1002 "pgsteal_kswapd",
1305 "pgsteal_direct", 1003 "pgsteal_direct",
1306 "pgsteal_khugepaged", <<
1307 "pgscan_kswapd", 1004 "pgscan_kswapd",
1308 "pgscan_direct", 1005 "pgscan_direct",
1309 "pgscan_khugepaged", <<
1310 "pgscan_direct_throttle", 1006 "pgscan_direct_throttle",
1311 "pgscan_anon", <<
1312 "pgscan_file", <<
1313 "pgsteal_anon", <<
1314 "pgsteal_file", <<
1315 1007
1316 #ifdef CONFIG_NUMA 1008 #ifdef CONFIG_NUMA
1317 "zone_reclaim_success", <<
1318 "zone_reclaim_failed", 1009 "zone_reclaim_failed",
1319 #endif 1010 #endif
1320 "pginodesteal", 1011 "pginodesteal",
1321 "slabs_scanned", 1012 "slabs_scanned",
1322 "kswapd_inodesteal", 1013 "kswapd_inodesteal",
1323 "kswapd_low_wmark_hit_quickly", 1014 "kswapd_low_wmark_hit_quickly",
1324 "kswapd_high_wmark_hit_quickly", 1015 "kswapd_high_wmark_hit_quickly",
1325 "pageoutrun", 1016 "pageoutrun",
1326 1017
1327 "pgrotated", 1018 "pgrotated",
1328 1019
1329 "drop_pagecache", 1020 "drop_pagecache",
1330 "drop_slab", 1021 "drop_slab",
1331 "oom_kill", <<
1332 1022
1333 #ifdef CONFIG_NUMA_BALANCING 1023 #ifdef CONFIG_NUMA_BALANCING
1334 "numa_pte_updates", 1024 "numa_pte_updates",
1335 "numa_huge_pte_updates", 1025 "numa_huge_pte_updates",
1336 "numa_hint_faults", 1026 "numa_hint_faults",
1337 "numa_hint_faults_local", 1027 "numa_hint_faults_local",
1338 "numa_pages_migrated", 1028 "numa_pages_migrated",
1339 #endif 1029 #endif
1340 #ifdef CONFIG_MIGRATION 1030 #ifdef CONFIG_MIGRATION
1341 "pgmigrate_success", 1031 "pgmigrate_success",
1342 "pgmigrate_fail", 1032 "pgmigrate_fail",
1343 "thp_migration_success", <<
1344 "thp_migration_fail", <<
1345 "thp_migration_split", <<
1346 #endif 1033 #endif
1347 #ifdef CONFIG_COMPACTION 1034 #ifdef CONFIG_COMPACTION
1348 "compact_migrate_scanned", 1035 "compact_migrate_scanned",
1349 "compact_free_scanned", 1036 "compact_free_scanned",
1350 "compact_isolated", 1037 "compact_isolated",
1351 "compact_stall", 1038 "compact_stall",
1352 "compact_fail", 1039 "compact_fail",
1353 "compact_success", 1040 "compact_success",
1354 "compact_daemon_wake", 1041 "compact_daemon_wake",
1355 "compact_daemon_migrate_scanned", <<
1356 "compact_daemon_free_scanned", <<
1357 #endif 1042 #endif
1358 1043
1359 #ifdef CONFIG_HUGETLB_PAGE 1044 #ifdef CONFIG_HUGETLB_PAGE
1360 "htlb_buddy_alloc_success", 1045 "htlb_buddy_alloc_success",
1361 "htlb_buddy_alloc_fail", 1046 "htlb_buddy_alloc_fail",
1362 #endif 1047 #endif
1363 #ifdef CONFIG_CMA <<
1364 "cma_alloc_success", <<
1365 "cma_alloc_fail", <<
1366 #endif <<
1367 "unevictable_pgs_culled", 1048 "unevictable_pgs_culled",
1368 "unevictable_pgs_scanned", 1049 "unevictable_pgs_scanned",
1369 "unevictable_pgs_rescued", 1050 "unevictable_pgs_rescued",
1370 "unevictable_pgs_mlocked", 1051 "unevictable_pgs_mlocked",
1371 "unevictable_pgs_munlocked", 1052 "unevictable_pgs_munlocked",
1372 "unevictable_pgs_cleared", 1053 "unevictable_pgs_cleared",
1373 "unevictable_pgs_stranded", 1054 "unevictable_pgs_stranded",
1374 1055
1375 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1056 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1376 "thp_fault_alloc", 1057 "thp_fault_alloc",
1377 "thp_fault_fallback", 1058 "thp_fault_fallback",
1378 "thp_fault_fallback_charge", <<
1379 "thp_collapse_alloc", 1059 "thp_collapse_alloc",
1380 "thp_collapse_alloc_failed", 1060 "thp_collapse_alloc_failed",
1381 "thp_file_alloc", 1061 "thp_file_alloc",
1382 "thp_file_fallback", <<
1383 "thp_file_fallback_charge", <<
1384 "thp_file_mapped", 1062 "thp_file_mapped",
1385 "thp_split_page", 1063 "thp_split_page",
1386 "thp_split_page_failed", 1064 "thp_split_page_failed",
1387 "thp_deferred_split_page", 1065 "thp_deferred_split_page",
1388 "thp_underused_split_page", <<
1389 "thp_split_pmd", 1066 "thp_split_pmd",
1390 "thp_scan_exceed_none_pte", <<
1391 "thp_scan_exceed_swap_pte", <<
1392 "thp_scan_exceed_share_pte", <<
1393 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_ <<
1394 "thp_split_pud", <<
1395 #endif <<
1396 "thp_zero_page_alloc", 1067 "thp_zero_page_alloc",
1397 "thp_zero_page_alloc_failed", 1068 "thp_zero_page_alloc_failed",
1398 "thp_swpout", <<
1399 "thp_swpout_fallback", <<
1400 #endif 1069 #endif
1401 #ifdef CONFIG_MEMORY_BALLOON 1070 #ifdef CONFIG_MEMORY_BALLOON
1402 "balloon_inflate", 1071 "balloon_inflate",
1403 "balloon_deflate", 1072 "balloon_deflate",
1404 #ifdef CONFIG_BALLOON_COMPACTION 1073 #ifdef CONFIG_BALLOON_COMPACTION
1405 "balloon_migrate", 1074 "balloon_migrate",
1406 #endif 1075 #endif
1407 #endif /* CONFIG_MEMORY_BALLOON */ 1076 #endif /* CONFIG_MEMORY_BALLOON */
1408 #ifdef CONFIG_DEBUG_TLBFLUSH 1077 #ifdef CONFIG_DEBUG_TLBFLUSH
1409 "nr_tlb_remote_flush", 1078 "nr_tlb_remote_flush",
1410 "nr_tlb_remote_flush_received", 1079 "nr_tlb_remote_flush_received",
1411 "nr_tlb_local_flush_all", 1080 "nr_tlb_local_flush_all",
1412 "nr_tlb_local_flush_one", 1081 "nr_tlb_local_flush_one",
1413 #endif /* CONFIG_DEBUG_TLBFLUSH */ 1082 #endif /* CONFIG_DEBUG_TLBFLUSH */
1414 1083
1415 #ifdef CONFIG_SWAP !! 1084 #ifdef CONFIG_DEBUG_VM_VMACACHE
1416 "swap_ra", !! 1085 "vmacache_find_calls",
1417 "swap_ra_hit", !! 1086 "vmacache_find_hits",
1418 #ifdef CONFIG_KSM <<
1419 "ksm_swpin_copy", <<
1420 #endif <<
1421 #endif <<
1422 #ifdef CONFIG_KSM <<
1423 "cow_ksm", <<
1424 #endif <<
1425 #ifdef CONFIG_ZSWAP <<
1426 "zswpin", <<
1427 "zswpout", <<
1428 "zswpwb", <<
1429 #endif <<
1430 #ifdef CONFIG_X86 <<
1431 "direct_map_level2_splits", <<
1432 "direct_map_level3_splits", <<
1433 #endif <<
1434 #ifdef CONFIG_PER_VMA_LOCK_STATS <<
1435 "vma_lock_success", <<
1436 "vma_lock_abort", <<
1437 "vma_lock_retry", <<
1438 "vma_lock_miss", <<
1439 #endif <<
1440 #ifdef CONFIG_DEBUG_STACK_USAGE <<
1441 "kstack_1k", <<
1442 #if THREAD_SIZE > 1024 <<
1443 "kstack_2k", <<
1444 #endif <<
1445 #if THREAD_SIZE > 2048 <<
1446 "kstack_4k", <<
1447 #endif <<
1448 #if THREAD_SIZE > 4096 <<
1449 "kstack_8k", <<
1450 #endif 1087 #endif
1451 #if THREAD_SIZE > 8192 !! 1088 #endif /* CONFIG_VM_EVENTS_COUNTERS */
1452 "kstack_16k", <<
1453 #endif <<
1454 #if THREAD_SIZE > 16384 <<
1455 "kstack_32k", <<
1456 #endif <<
1457 #if THREAD_SIZE > 32768 <<
1458 "kstack_64k", <<
1459 #endif <<
1460 #if THREAD_SIZE > 65536 <<
1461 "kstack_rest", <<
1462 #endif <<
1463 #endif <<
1464 #endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_ <<
1465 }; 1089 };
1466 #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || C !! 1090 #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA */
>> 1091
1467 1092
1468 #if (defined(CONFIG_DEBUG_FS) && defined(CONF 1093 #if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \
1469 defined(CONFIG_PROC_FS) 1094 defined(CONFIG_PROC_FS)
1470 static void *frag_start(struct seq_file *m, l 1095 static void *frag_start(struct seq_file *m, loff_t *pos)
1471 { 1096 {
1472 pg_data_t *pgdat; 1097 pg_data_t *pgdat;
1473 loff_t node = *pos; 1098 loff_t node = *pos;
1474 1099
1475 for (pgdat = first_online_pgdat(); 1100 for (pgdat = first_online_pgdat();
1476 pgdat && node; 1101 pgdat && node;
1477 pgdat = next_online_pgdat(pgdat) 1102 pgdat = next_online_pgdat(pgdat))
1478 --node; 1103 --node;
1479 1104
1480 return pgdat; 1105 return pgdat;
1481 } 1106 }
1482 1107
1483 static void *frag_next(struct seq_file *m, vo 1108 static void *frag_next(struct seq_file *m, void *arg, loff_t *pos)
1484 { 1109 {
1485 pg_data_t *pgdat = (pg_data_t *)arg; 1110 pg_data_t *pgdat = (pg_data_t *)arg;
1486 1111
1487 (*pos)++; 1112 (*pos)++;
1488 return next_online_pgdat(pgdat); 1113 return next_online_pgdat(pgdat);
1489 } 1114 }
1490 1115
1491 static void frag_stop(struct seq_file *m, voi 1116 static void frag_stop(struct seq_file *m, void *arg)
1492 { 1117 {
1493 } 1118 }
1494 1119
1495 /* !! 1120 /* Walk all the zones in a node and print using a callback */
1496 * Walk zones in a node and print using a cal <<
1497 * If @assert_populated is true, only use cal <<
1498 */ <<
1499 static void walk_zones_in_node(struct seq_fil 1121 static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat,
1500 bool assert_populated, bool n <<
1501 void (*print)(struct seq_file 1122 void (*print)(struct seq_file *m, pg_data_t *, struct zone *))
1502 { 1123 {
1503 struct zone *zone; 1124 struct zone *zone;
1504 struct zone *node_zones = pgdat->node 1125 struct zone *node_zones = pgdat->node_zones;
1505 unsigned long flags; 1126 unsigned long flags;
1506 1127
1507 for (zone = node_zones; zone - node_z 1128 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
1508 if (assert_populated && !popu !! 1129 if (!populated_zone(zone))
1509 continue; 1130 continue;
1510 1131
1511 if (!nolock) !! 1132 spin_lock_irqsave(&zone->lock, flags);
1512 spin_lock_irqsave(&zo <<
1513 print(m, pgdat, zone); 1133 print(m, pgdat, zone);
1514 if (!nolock) !! 1134 spin_unlock_irqrestore(&zone->lock, flags);
1515 spin_unlock_irqrestor <<
1516 } 1135 }
1517 } 1136 }
1518 #endif 1137 #endif
1519 1138
1520 #ifdef CONFIG_PROC_FS 1139 #ifdef CONFIG_PROC_FS
1521 static void frag_show_print(struct seq_file * 1140 static void frag_show_print(struct seq_file *m, pg_data_t *pgdat,
1522 1141 struct zone *zone)
1523 { 1142 {
1524 int order; 1143 int order;
1525 1144
1526 seq_printf(m, "Node %d, zone %8s ", p 1145 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
1527 for (order = 0; order < NR_PAGE_ORDER !! 1146 for (order = 0; order < MAX_ORDER; ++order)
1528 /* !! 1147 seq_printf(m, "%6lu ", zone->free_area[order].nr_free);
1529 * Access to nr_free is lockl <<
1530 * printing purposes. Use dat <<
1531 */ <<
1532 seq_printf(m, "%6lu ", data_r <<
1533 seq_putc(m, '\n'); 1148 seq_putc(m, '\n');
1534 } 1149 }
1535 1150
1536 /* 1151 /*
1537 * This walks the free areas for each zone. 1152 * This walks the free areas for each zone.
1538 */ 1153 */
1539 static int frag_show(struct seq_file *m, void 1154 static int frag_show(struct seq_file *m, void *arg)
1540 { 1155 {
1541 pg_data_t *pgdat = (pg_data_t *)arg; 1156 pg_data_t *pgdat = (pg_data_t *)arg;
1542 walk_zones_in_node(m, pgdat, true, fa !! 1157 walk_zones_in_node(m, pgdat, frag_show_print);
1543 return 0; 1158 return 0;
1544 } 1159 }
1545 1160
1546 static void pagetypeinfo_showfree_print(struc 1161 static void pagetypeinfo_showfree_print(struct seq_file *m,
1547 pg_da 1162 pg_data_t *pgdat, struct zone *zone)
1548 { 1163 {
1549 int order, mtype; 1164 int order, mtype;
1550 1165
1551 for (mtype = 0; mtype < MIGRATE_TYPES 1166 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) {
1552 seq_printf(m, "Node %4d, zone 1167 seq_printf(m, "Node %4d, zone %8s, type %12s ",
1553 pgdat 1168 pgdat->node_id,
1554 zone- 1169 zone->name,
1555 migra 1170 migratetype_names[mtype]);
1556 for (order = 0; order < NR_PA !! 1171 for (order = 0; order < MAX_ORDER; ++order) {
1557 unsigned long freecou 1172 unsigned long freecount = 0;
1558 struct free_area *are 1173 struct free_area *area;
1559 struct list_head *cur 1174 struct list_head *curr;
1560 bool overflow = false <<
1561 1175
1562 area = &(zone->free_a 1176 area = &(zone->free_area[order]);
1563 1177
1564 list_for_each(curr, & !! 1178 list_for_each(curr, &area->free_list[mtype])
1565 /* !! 1179 freecount++;
1566 * Cap the fr !! 1180 seq_printf(m, "%6lu ", freecount);
1567 * be really <<
1568 * so a long <<
1569 * hard locku <<
1570 * debugging <<
1571 * of pages o <<
1572 * sufficient <<
1573 */ <<
1574 if (++freecou <<
1575 overf <<
1576 break <<
1577 } <<
1578 } <<
1579 seq_printf(m, "%s%6lu <<
1580 spin_unlock_irq(&zone 1181 spin_unlock_irq(&zone->lock);
1581 cond_resched(); 1182 cond_resched();
1582 spin_lock_irq(&zone-> 1183 spin_lock_irq(&zone->lock);
1583 } 1184 }
1584 seq_putc(m, '\n'); 1185 seq_putc(m, '\n');
1585 } 1186 }
1586 } 1187 }
1587 1188
1588 /* Print out the free pages at each order for 1189 /* Print out the free pages at each order for each migatetype */
1589 static void pagetypeinfo_showfree(struct seq_ !! 1190 static int pagetypeinfo_showfree(struct seq_file *m, void *arg)
1590 { 1191 {
1591 int order; 1192 int order;
1592 pg_data_t *pgdat = (pg_data_t *)arg; 1193 pg_data_t *pgdat = (pg_data_t *)arg;
1593 1194
1594 /* Print header */ 1195 /* Print header */
1595 seq_printf(m, "%-43s ", "Free pages c 1196 seq_printf(m, "%-43s ", "Free pages count per migrate type at order");
1596 for (order = 0; order < NR_PAGE_ORDER !! 1197 for (order = 0; order < MAX_ORDER; ++order)
1597 seq_printf(m, "%6d ", order); 1198 seq_printf(m, "%6d ", order);
1598 seq_putc(m, '\n'); 1199 seq_putc(m, '\n');
1599 1200
1600 walk_zones_in_node(m, pgdat, true, fa !! 1201 walk_zones_in_node(m, pgdat, pagetypeinfo_showfree_print);
>> 1202
>> 1203 return 0;
1601 } 1204 }
1602 1205
1603 static void pagetypeinfo_showblockcount_print 1206 static void pagetypeinfo_showblockcount_print(struct seq_file *m,
1604 pg_da 1207 pg_data_t *pgdat, struct zone *zone)
1605 { 1208 {
1606 int mtype; 1209 int mtype;
1607 unsigned long pfn; 1210 unsigned long pfn;
1608 unsigned long start_pfn = zone->zone_ 1211 unsigned long start_pfn = zone->zone_start_pfn;
1609 unsigned long end_pfn = zone_end_pfn( 1212 unsigned long end_pfn = zone_end_pfn(zone);
1610 unsigned long count[MIGRATE_TYPES] = 1213 unsigned long count[MIGRATE_TYPES] = { 0, };
1611 1214
1612 for (pfn = start_pfn; pfn < end_pfn; 1215 for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
1613 struct page *page; 1216 struct page *page;
1614 1217
1615 page = pfn_to_online_page(pfn !! 1218 if (!pfn_valid(pfn))
1616 if (!page) !! 1219 continue;
>> 1220
>> 1221 page = pfn_to_page(pfn);
>> 1222
>> 1223 /* Watch for unexpected holes punched in the memmap */
>> 1224 if (!memmap_valid_within(pfn, page, zone))
1617 continue; 1225 continue;
1618 1226
1619 if (page_zone(page) != zone) 1227 if (page_zone(page) != zone)
1620 continue; 1228 continue;
1621 1229
1622 mtype = get_pageblock_migrate 1230 mtype = get_pageblock_migratetype(page);
1623 1231
1624 if (mtype < MIGRATE_TYPES) 1232 if (mtype < MIGRATE_TYPES)
1625 count[mtype]++; 1233 count[mtype]++;
1626 } 1234 }
1627 1235
1628 /* Print counts */ 1236 /* Print counts */
1629 seq_printf(m, "Node %d, zone %8s ", p 1237 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
1630 for (mtype = 0; mtype < MIGRATE_TYPES 1238 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1631 seq_printf(m, "%12lu ", count 1239 seq_printf(m, "%12lu ", count[mtype]);
1632 seq_putc(m, '\n'); 1240 seq_putc(m, '\n');
1633 } 1241 }
1634 1242
1635 /* Print out the number of pageblocks for eac !! 1243 /* Print out the free pages at each order for each migratetype */
1636 static void pagetypeinfo_showblockcount(struc !! 1244 static int pagetypeinfo_showblockcount(struct seq_file *m, void *arg)
1637 { 1245 {
1638 int mtype; 1246 int mtype;
1639 pg_data_t *pgdat = (pg_data_t *)arg; 1247 pg_data_t *pgdat = (pg_data_t *)arg;
1640 1248
1641 seq_printf(m, "\n%-23s", "Number of b 1249 seq_printf(m, "\n%-23s", "Number of blocks type ");
1642 for (mtype = 0; mtype < MIGRATE_TYPES 1250 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1643 seq_printf(m, "%12s ", migrat 1251 seq_printf(m, "%12s ", migratetype_names[mtype]);
1644 seq_putc(m, '\n'); 1252 seq_putc(m, '\n');
1645 walk_zones_in_node(m, pgdat, true, fa !! 1253 walk_zones_in_node(m, pgdat, pagetypeinfo_showblockcount_print);
1646 pagetypeinfo_showblockcount_p !! 1254
>> 1255 return 0;
1647 } 1256 }
1648 1257
1649 /* 1258 /*
1650 * Print out the number of pageblocks for eac 1259 * Print out the number of pageblocks for each migratetype that contain pages
1651 * of other types. This gives an indication o 1260 * of other types. This gives an indication of how well fallbacks are being
1652 * contained by rmqueue_fallback(). It requir 1261 * contained by rmqueue_fallback(). It requires information from PAGE_OWNER
1653 * to determine what is going on 1262 * to determine what is going on
1654 */ 1263 */
1655 static void pagetypeinfo_showmixedcount(struc 1264 static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat)
1656 { 1265 {
1657 #ifdef CONFIG_PAGE_OWNER 1266 #ifdef CONFIG_PAGE_OWNER
1658 int mtype; 1267 int mtype;
1659 1268
1660 if (!static_branch_unlikely(&page_own 1269 if (!static_branch_unlikely(&page_owner_inited))
1661 return; 1270 return;
1662 1271
1663 drain_all_pages(NULL); 1272 drain_all_pages(NULL);
1664 1273
1665 seq_printf(m, "\n%-23s", "Number of m 1274 seq_printf(m, "\n%-23s", "Number of mixed blocks ");
1666 for (mtype = 0; mtype < MIGRATE_TYPES 1275 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1667 seq_printf(m, "%12s ", migrat 1276 seq_printf(m, "%12s ", migratetype_names[mtype]);
1668 seq_putc(m, '\n'); 1277 seq_putc(m, '\n');
1669 1278
1670 walk_zones_in_node(m, pgdat, true, tr !! 1279 walk_zones_in_node(m, pgdat, pagetypeinfo_showmixedcount_print);
1671 pagetypeinfo_showmixedcount_p <<
1672 #endif /* CONFIG_PAGE_OWNER */ 1280 #endif /* CONFIG_PAGE_OWNER */
1673 } 1281 }
1674 1282
1675 /* 1283 /*
1676 * This prints out statistics in relation to 1284 * This prints out statistics in relation to grouping pages by mobility.
1677 * It is expensive to collect so do not const 1285 * It is expensive to collect so do not constantly read the file.
1678 */ 1286 */
1679 static int pagetypeinfo_show(struct seq_file 1287 static int pagetypeinfo_show(struct seq_file *m, void *arg)
1680 { 1288 {
1681 pg_data_t *pgdat = (pg_data_t *)arg; 1289 pg_data_t *pgdat = (pg_data_t *)arg;
1682 1290
1683 /* check memoryless node */ 1291 /* check memoryless node */
1684 if (!node_state(pgdat->node_id, N_MEM 1292 if (!node_state(pgdat->node_id, N_MEMORY))
1685 return 0; 1293 return 0;
1686 1294
1687 seq_printf(m, "Page block order: %d\n 1295 seq_printf(m, "Page block order: %d\n", pageblock_order);
1688 seq_printf(m, "Pages per block: %lu\ 1296 seq_printf(m, "Pages per block: %lu\n", pageblock_nr_pages);
1689 seq_putc(m, '\n'); 1297 seq_putc(m, '\n');
1690 pagetypeinfo_showfree(m, pgdat); 1298 pagetypeinfo_showfree(m, pgdat);
1691 pagetypeinfo_showblockcount(m, pgdat) 1299 pagetypeinfo_showblockcount(m, pgdat);
1692 pagetypeinfo_showmixedcount(m, pgdat) 1300 pagetypeinfo_showmixedcount(m, pgdat);
1693 1301
1694 return 0; 1302 return 0;
1695 } 1303 }
1696 1304
1697 static const struct seq_operations fragmentat 1305 static const struct seq_operations fragmentation_op = {
1698 .start = frag_start, 1306 .start = frag_start,
1699 .next = frag_next, 1307 .next = frag_next,
1700 .stop = frag_stop, 1308 .stop = frag_stop,
1701 .show = frag_show, 1309 .show = frag_show,
1702 }; 1310 };
1703 1311
>> 1312 static int fragmentation_open(struct inode *inode, struct file *file)
>> 1313 {
>> 1314 return seq_open(file, &fragmentation_op);
>> 1315 }
>> 1316
>> 1317 static const struct file_operations fragmentation_file_operations = {
>> 1318 .open = fragmentation_open,
>> 1319 .read = seq_read,
>> 1320 .llseek = seq_lseek,
>> 1321 .release = seq_release,
>> 1322 };
>> 1323
1704 static const struct seq_operations pagetypein 1324 static const struct seq_operations pagetypeinfo_op = {
1705 .start = frag_start, 1325 .start = frag_start,
1706 .next = frag_next, 1326 .next = frag_next,
1707 .stop = frag_stop, 1327 .stop = frag_stop,
1708 .show = pagetypeinfo_show, 1328 .show = pagetypeinfo_show,
1709 }; 1329 };
1710 1330
>> 1331 static int pagetypeinfo_open(struct inode *inode, struct file *file)
>> 1332 {
>> 1333 return seq_open(file, &pagetypeinfo_op);
>> 1334 }
>> 1335
>> 1336 static const struct file_operations pagetypeinfo_file_ops = {
>> 1337 .open = pagetypeinfo_open,
>> 1338 .read = seq_read,
>> 1339 .llseek = seq_lseek,
>> 1340 .release = seq_release,
>> 1341 };
>> 1342
1711 static bool is_zone_first_populated(pg_data_t 1343 static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone)
1712 { 1344 {
1713 int zid; 1345 int zid;
1714 1346
1715 for (zid = 0; zid < MAX_NR_ZONES; zid 1347 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1716 struct zone *compare = &pgdat 1348 struct zone *compare = &pgdat->node_zones[zid];
1717 1349
1718 if (populated_zone(compare)) 1350 if (populated_zone(compare))
1719 return zone == compar 1351 return zone == compare;
1720 } 1352 }
1721 1353
1722 return false; 1354 return false;
1723 } 1355 }
1724 1356
1725 static void zoneinfo_show_print(struct seq_fi 1357 static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
1726 1358 struct zone *zone)
1727 { 1359 {
1728 int i; 1360 int i;
1729 seq_printf(m, "Node %d, zone %8s", pg 1361 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);
1730 if (is_zone_first_populated(pgdat, zo 1362 if (is_zone_first_populated(pgdat, zone)) {
1731 seq_printf(m, "\n per-node s 1363 seq_printf(m, "\n per-node stats");
1732 for (i = 0; i < NR_VM_NODE_ST 1364 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
1733 unsigned long pages = !! 1365 seq_printf(m, "\n %-12s %lu",
1734 !! 1366 vmstat_text[i + NR_VM_ZONE_STAT_ITEMS],
1735 if (vmstat_item_print !! 1367 node_page_state(pgdat, i));
1736 pages /= HPAG <<
1737 seq_printf(m, "\n <<
1738 pages); <<
1739 } 1368 }
1740 } 1369 }
1741 seq_printf(m, 1370 seq_printf(m,
1742 "\n pages free %lu" 1371 "\n pages free %lu"
1743 "\n boost %lu" <<
1744 "\n min %lu" 1372 "\n min %lu"
1745 "\n low %lu" 1373 "\n low %lu"
1746 "\n high %lu" 1374 "\n high %lu"
1747 "\n promo %lu" !! 1375 "\n node_scanned %lu"
1748 "\n spanned %lu" 1376 "\n spanned %lu"
1749 "\n present %lu" 1377 "\n present %lu"
1750 "\n managed %lu" !! 1378 "\n managed %lu",
1751 "\n cma %lu", <<
1752 zone_page_state(zone, NR_F 1379 zone_page_state(zone, NR_FREE_PAGES),
1753 zone->watermark_boost, <<
1754 min_wmark_pages(zone), 1380 min_wmark_pages(zone),
1755 low_wmark_pages(zone), 1381 low_wmark_pages(zone),
1756 high_wmark_pages(zone), 1382 high_wmark_pages(zone),
1757 promo_wmark_pages(zone), !! 1383 node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED),
1758 zone->spanned_pages, 1384 zone->spanned_pages,
1759 zone->present_pages, 1385 zone->present_pages,
1760 zone_managed_pages(zone), !! 1386 zone->managed_pages);
1761 zone_cma_pages(zone)); <<
1762 1387
1763 seq_printf(m, 1388 seq_printf(m,
1764 "\n protection: (%l 1389 "\n protection: (%ld",
1765 zone->lowmem_reserve[0]); 1390 zone->lowmem_reserve[0]);
1766 for (i = 1; i < ARRAY_SIZE(zone->lowm 1391 for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++)
1767 seq_printf(m, ", %ld", zone-> 1392 seq_printf(m, ", %ld", zone->lowmem_reserve[i]);
1768 seq_putc(m, ')'); 1393 seq_putc(m, ')');
1769 1394
1770 /* If unpopulated, no other informati 1395 /* If unpopulated, no other information is useful */
1771 if (!populated_zone(zone)) { 1396 if (!populated_zone(zone)) {
1772 seq_putc(m, '\n'); 1397 seq_putc(m, '\n');
1773 return; 1398 return;
1774 } 1399 }
1775 1400
1776 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS 1401 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
1777 seq_printf(m, "\n %-12s !! 1402 seq_printf(m, "\n %-12s %lu", vmstat_text[i],
1778 zone_page_state(zo !! 1403 zone_page_state(zone, i));
1779 <<
1780 #ifdef CONFIG_NUMA <<
1781 for (i = 0; i < NR_VM_NUMA_EVENT_ITEM <<
1782 seq_printf(m, "\n %-12s <<
1783 zone_numa_event_st <<
1784 #endif <<
1785 1404
1786 seq_printf(m, "\n pagesets"); 1405 seq_printf(m, "\n pagesets");
1787 for_each_online_cpu(i) { 1406 for_each_online_cpu(i) {
1788 struct per_cpu_pages *pcp; !! 1407 struct per_cpu_pageset *pageset;
1789 struct per_cpu_zonestat __may <<
1790 1408
1791 pcp = per_cpu_ptr(zone->per_c !! 1409 pageset = per_cpu_ptr(zone->pageset, i);
1792 seq_printf(m, 1410 seq_printf(m,
1793 "\n cpu: %i" 1411 "\n cpu: %i"
1794 "\n c 1412 "\n count: %i"
1795 "\n h 1413 "\n high: %i"
1796 "\n b 1414 "\n batch: %i",
1797 i, 1415 i,
1798 pcp->count, !! 1416 pageset->pcp.count,
1799 pcp->high, !! 1417 pageset->pcp.high,
1800 pcp->batch); !! 1418 pageset->pcp.batch);
1801 #ifdef CONFIG_SMP 1419 #ifdef CONFIG_SMP
1802 pzstats = per_cpu_ptr(zone->p <<
1803 seq_printf(m, "\n vm stats t 1420 seq_printf(m, "\n vm stats threshold: %d",
1804 pzstats->stat !! 1421 pageset->stat_threshold);
1805 #endif 1422 #endif
1806 } 1423 }
1807 seq_printf(m, 1424 seq_printf(m,
1808 "\n node_unreclaimable: 1425 "\n node_unreclaimable: %u"
1809 "\n start_pfn: !! 1426 "\n start_pfn: %lu"
>> 1427 "\n node_inactive_ratio: %u",
1810 pgdat->kswapd_failures >= 1428 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES,
1811 zone->zone_start_pfn); !! 1429 zone->zone_start_pfn,
>> 1430 zone->zone_pgdat->inactive_ratio);
1812 seq_putc(m, '\n'); 1431 seq_putc(m, '\n');
1813 } 1432 }
1814 1433
1815 /* 1434 /*
1816 * Output information about zones in @pgdat. !! 1435 * Output information about zones in @pgdat.
1817 * of whether they are populated or not: lowm <<
1818 * set of all zones and userspace would not b <<
1819 * suppressed here (zoneinfo displays the eff <<
1820 */ 1436 */
1821 static int zoneinfo_show(struct seq_file *m, 1437 static int zoneinfo_show(struct seq_file *m, void *arg)
1822 { 1438 {
1823 pg_data_t *pgdat = (pg_data_t *)arg; 1439 pg_data_t *pgdat = (pg_data_t *)arg;
1824 walk_zones_in_node(m, pgdat, false, f !! 1440 walk_zones_in_node(m, pgdat, zoneinfo_show_print);
1825 return 0; 1441 return 0;
1826 } 1442 }
1827 1443
1828 static const struct seq_operations zoneinfo_o 1444 static const struct seq_operations zoneinfo_op = {
1829 .start = frag_start, /* iterate over 1445 .start = frag_start, /* iterate over all zones. The same as in
1830 * fragmentatio 1446 * fragmentation. */
1831 .next = frag_next, 1447 .next = frag_next,
1832 .stop = frag_stop, 1448 .stop = frag_stop,
1833 .show = zoneinfo_show, 1449 .show = zoneinfo_show,
1834 }; 1450 };
1835 1451
1836 #define NR_VMSTAT_ITEMS (NR_VM_ZONE_STAT_ITEM !! 1452 static int zoneinfo_open(struct inode *inode, struct file *file)
1837 NR_VM_NUMA_EVENT_ITE !! 1453 {
1838 NR_VM_NODE_STAT_ITEM !! 1454 return seq_open(file, &zoneinfo_op);
1839 NR_VM_STAT_ITEMS + \ !! 1455 }
1840 (IS_ENABLED(CONFIG_V !! 1456
1841 NR_VM_EVENT_ITEMS : !! 1457 static const struct file_operations proc_zoneinfo_file_operations = {
>> 1458 .open = zoneinfo_open,
>> 1459 .read = seq_read,
>> 1460 .llseek = seq_lseek,
>> 1461 .release = seq_release,
>> 1462 };
>> 1463
>> 1464 enum writeback_stat_item {
>> 1465 NR_DIRTY_THRESHOLD,
>> 1466 NR_DIRTY_BG_THRESHOLD,
>> 1467 NR_VM_WRITEBACK_STAT_ITEMS,
>> 1468 };
1842 1469
1843 static void *vmstat_start(struct seq_file *m, 1470 static void *vmstat_start(struct seq_file *m, loff_t *pos)
1844 { 1471 {
1845 unsigned long *v; 1472 unsigned long *v;
1846 int i; !! 1473 int i, stat_items_size;
1847 1474
1848 if (*pos >= NR_VMSTAT_ITEMS) !! 1475 if (*pos >= ARRAY_SIZE(vmstat_text))
1849 return NULL; 1476 return NULL;
>> 1477 stat_items_size = NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long) +
>> 1478 NR_VM_NODE_STAT_ITEMS * sizeof(unsigned long) +
>> 1479 NR_VM_WRITEBACK_STAT_ITEMS * sizeof(unsigned long);
1850 1480
1851 BUILD_BUG_ON(ARRAY_SIZE(vmstat_text) !! 1481 #ifdef CONFIG_VM_EVENT_COUNTERS
1852 fold_vm_numa_events(); !! 1482 stat_items_size += sizeof(struct vm_event_state);
1853 v = kmalloc_array(NR_VMSTAT_ITEMS, si !! 1483 #endif
>> 1484
>> 1485 v = kmalloc(stat_items_size, GFP_KERNEL);
1854 m->private = v; 1486 m->private = v;
1855 if (!v) 1487 if (!v)
1856 return ERR_PTR(-ENOMEM); 1488 return ERR_PTR(-ENOMEM);
1857 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS 1489 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
1858 v[i] = global_zone_page_state !! 1490 v[i] = global_page_state(i);
1859 v += NR_VM_ZONE_STAT_ITEMS; 1491 v += NR_VM_ZONE_STAT_ITEMS;
1860 1492
1861 #ifdef CONFIG_NUMA !! 1493 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
1862 for (i = 0; i < NR_VM_NUMA_EVENT_ITEM !! 1494 v[i] = global_node_page_state(i);
1863 v[i] = global_numa_event_stat <<
1864 v += NR_VM_NUMA_EVENT_ITEMS; <<
1865 #endif <<
1866 <<
1867 for (i = 0; i < NR_VM_NODE_STAT_ITEMS <<
1868 v[i] = global_node_page_state <<
1869 if (vmstat_item_print_in_thp( <<
1870 v[i] /= HPAGE_PMD_NR; <<
1871 } <<
1872 v += NR_VM_NODE_STAT_ITEMS; 1495 v += NR_VM_NODE_STAT_ITEMS;
1873 1496
1874 global_dirty_limits(v + NR_DIRTY_BG_T 1497 global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD,
1875 v + NR_DIRTY_THRE 1498 v + NR_DIRTY_THRESHOLD);
1876 v[NR_MEMMAP_PAGES] = atomic_long_read !! 1499 v += NR_VM_WRITEBACK_STAT_ITEMS;
1877 v[NR_MEMMAP_BOOT_PAGES] = atomic_long <<
1878 v += NR_VM_STAT_ITEMS; <<
1879 1500
1880 #ifdef CONFIG_VM_EVENT_COUNTERS 1501 #ifdef CONFIG_VM_EVENT_COUNTERS
1881 all_vm_events(v); 1502 all_vm_events(v);
1882 v[PGPGIN] /= 2; /* sectors -> 1503 v[PGPGIN] /= 2; /* sectors -> kbytes */
1883 v[PGPGOUT] /= 2; 1504 v[PGPGOUT] /= 2;
1884 #endif 1505 #endif
1885 return (unsigned long *)m->private + 1506 return (unsigned long *)m->private + *pos;
1886 } 1507 }
1887 1508
1888 static void *vmstat_next(struct seq_file *m, 1509 static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos)
1889 { 1510 {
1890 (*pos)++; 1511 (*pos)++;
1891 if (*pos >= NR_VMSTAT_ITEMS) !! 1512 if (*pos >= ARRAY_SIZE(vmstat_text))
1892 return NULL; 1513 return NULL;
1893 return (unsigned long *)m->private + 1514 return (unsigned long *)m->private + *pos;
1894 } 1515 }
1895 1516
1896 static int vmstat_show(struct seq_file *m, vo 1517 static int vmstat_show(struct seq_file *m, void *arg)
1897 { 1518 {
1898 unsigned long *l = arg; 1519 unsigned long *l = arg;
1899 unsigned long off = l - (unsigned lon 1520 unsigned long off = l - (unsigned long *)m->private;
1900 1521
1901 seq_puts(m, vmstat_text[off]); 1522 seq_puts(m, vmstat_text[off]);
1902 seq_put_decimal_ull(m, " ", *l); 1523 seq_put_decimal_ull(m, " ", *l);
1903 seq_putc(m, '\n'); 1524 seq_putc(m, '\n');
1904 <<
1905 if (off == NR_VMSTAT_ITEMS - 1) { <<
1906 /* <<
1907 * We've come to the end - ad <<
1908 * breaking userspace which m <<
1909 */ <<
1910 seq_puts(m, "nr_unstable 0\n" <<
1911 } <<
1912 return 0; 1525 return 0;
1913 } 1526 }
1914 1527
1915 static void vmstat_stop(struct seq_file *m, v 1528 static void vmstat_stop(struct seq_file *m, void *arg)
1916 { 1529 {
1917 kfree(m->private); 1530 kfree(m->private);
1918 m->private = NULL; 1531 m->private = NULL;
1919 } 1532 }
1920 1533
1921 static const struct seq_operations vmstat_op 1534 static const struct seq_operations vmstat_op = {
1922 .start = vmstat_start, 1535 .start = vmstat_start,
1923 .next = vmstat_next, 1536 .next = vmstat_next,
1924 .stop = vmstat_stop, 1537 .stop = vmstat_stop,
1925 .show = vmstat_show, 1538 .show = vmstat_show,
1926 }; 1539 };
>> 1540
>> 1541 static int vmstat_open(struct inode *inode, struct file *file)
>> 1542 {
>> 1543 return seq_open(file, &vmstat_op);
>> 1544 }
>> 1545
>> 1546 static const struct file_operations proc_vmstat_file_operations = {
>> 1547 .open = vmstat_open,
>> 1548 .read = seq_read,
>> 1549 .llseek = seq_lseek,
>> 1550 .release = seq_release,
>> 1551 };
1927 #endif /* CONFIG_PROC_FS */ 1552 #endif /* CONFIG_PROC_FS */
1928 1553
1929 #ifdef CONFIG_SMP 1554 #ifdef CONFIG_SMP
>> 1555 static struct workqueue_struct *vmstat_wq;
1930 static DEFINE_PER_CPU(struct delayed_work, vm 1556 static DEFINE_PER_CPU(struct delayed_work, vmstat_work);
1931 int sysctl_stat_interval __read_mostly = HZ; 1557 int sysctl_stat_interval __read_mostly = HZ;
1932 1558
1933 #ifdef CONFIG_PROC_FS 1559 #ifdef CONFIG_PROC_FS
1934 static void refresh_vm_stats(struct work_stru 1560 static void refresh_vm_stats(struct work_struct *work)
1935 { 1561 {
1936 refresh_cpu_vm_stats(true); 1562 refresh_cpu_vm_stats(true);
1937 } 1563 }
1938 1564
1939 int vmstat_refresh(const struct ctl_table *ta !! 1565 int vmstat_refresh(struct ctl_table *table, int write,
1940 void *buffer, size_t *lenp !! 1566 void __user *buffer, size_t *lenp, loff_t *ppos)
1941 { 1567 {
1942 long val; 1568 long val;
1943 int err; 1569 int err;
1944 int i; 1570 int i;
1945 1571
1946 /* 1572 /*
1947 * The regular update, every sysctl_s 1573 * The regular update, every sysctl_stat_interval, may come later
1948 * than expected: leaving a significa 1574 * than expected: leaving a significant amount in per_cpu buckets.
1949 * This is particularly misleading wh 1575 * This is particularly misleading when checking a quantity of HUGE
1950 * pages, immediately after running a 1576 * pages, immediately after running a test. /proc/sys/vm/stat_refresh,
1951 * which can equally be echo'ed to or 1577 * which can equally be echo'ed to or cat'ted from (by root),
1952 * can be used to update the stats ju 1578 * can be used to update the stats just before reading them.
1953 * 1579 *
1954 * Oh, and since global_zone_page_sta !! 1580 * Oh, and since global_page_state() etc. are so careful to hide
1955 * transiently negative values, repor 1581 * transiently negative values, report an error here if any of
1956 * the stats is negative, so we know 1582 * the stats is negative, so we know to go looking for imbalance.
1957 */ 1583 */
1958 err = schedule_on_each_cpu(refresh_vm 1584 err = schedule_on_each_cpu(refresh_vm_stats);
1959 if (err) 1585 if (err)
1960 return err; 1586 return err;
1961 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS 1587 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
1962 /* <<
1963 * Skip checking stats known <<
1964 */ <<
1965 switch (i) { <<
1966 case NR_ZONE_WRITE_PENDING: <<
1967 case NR_FREE_CMA_PAGES: <<
1968 continue; <<
1969 } <<
1970 val = atomic_long_read(&vm_zo 1588 val = atomic_long_read(&vm_zone_stat[i]);
1971 if (val < 0) { 1589 if (val < 0) {
1972 pr_warn("%s: %s %ld\n !! 1590 switch (i) {
1973 __func__, zon !! 1591 case NR_PAGES_SCANNED:
1974 } !! 1592 /*
1975 } !! 1593 * This is often seen to go negative in
1976 for (i = 0; i < NR_VM_NODE_STAT_ITEMS !! 1594 * recent kernels, but not to go permanently
1977 /* !! 1595 * negative. Whilst it would be nicer not to
1978 * Skip checking stats known !! 1596 * have exceptions, rooting them out would be
1979 */ !! 1597 * another task, of rather low priority.
1980 switch (i) { !! 1598 */
1981 case NR_WRITEBACK: !! 1599 break;
1982 continue; !! 1600 default:
1983 } !! 1601 pr_warn("%s: %s %ld\n",
1984 val = atomic_long_read(&vm_no !! 1602 __func__, vmstat_text[i], val);
1985 if (val < 0) { !! 1603 err = -EINVAL;
1986 pr_warn("%s: %s %ld\n !! 1604 break;
1987 __func__, nod !! 1605 }
1988 } 1606 }
1989 } 1607 }
>> 1608 if (err)
>> 1609 return err;
1990 if (write) 1610 if (write)
1991 *ppos += *lenp; 1611 *ppos += *lenp;
1992 else 1612 else
1993 *lenp = 0; 1613 *lenp = 0;
1994 return 0; 1614 return 0;
1995 } 1615 }
1996 #endif /* CONFIG_PROC_FS */ 1616 #endif /* CONFIG_PROC_FS */
1997 1617
1998 static void vmstat_update(struct work_struct 1618 static void vmstat_update(struct work_struct *w)
1999 { 1619 {
2000 if (refresh_cpu_vm_stats(true)) { 1620 if (refresh_cpu_vm_stats(true)) {
2001 /* 1621 /*
2002 * Counters were updated so w 1622 * Counters were updated so we expect more updates
2003 * to occur in the future. Ke 1623 * to occur in the future. Keep on running the
2004 * update worker thread. 1624 * update worker thread.
2005 */ 1625 */
2006 queue_delayed_work_on(smp_pro !! 1626 queue_delayed_work_on(smp_processor_id(), vmstat_wq,
2007 this_cpu_ptr( 1627 this_cpu_ptr(&vmstat_work),
2008 round_jiffies 1628 round_jiffies_relative(sysctl_stat_interval));
2009 } 1629 }
2010 } 1630 }
2011 1631
2012 /* 1632 /*
>> 1633 * Switch off vmstat processing and then fold all the remaining differentials
>> 1634 * until the diffs stay at zero. The function is used by NOHZ and can only be
>> 1635 * invoked when tick processing is not active.
>> 1636 */
>> 1637 /*
2013 * Check if the diffs for a certain cpu indic 1638 * Check if the diffs for a certain cpu indicate that
2014 * an update is needed. 1639 * an update is needed.
2015 */ 1640 */
2016 static bool need_update(int cpu) 1641 static bool need_update(int cpu)
2017 { 1642 {
2018 pg_data_t *last_pgdat = NULL; <<
2019 struct zone *zone; 1643 struct zone *zone;
2020 1644
2021 for_each_populated_zone(zone) { 1645 for_each_populated_zone(zone) {
2022 struct per_cpu_zonestat *pzst !! 1646 struct per_cpu_pageset *p = per_cpu_ptr(zone->pageset, cpu);
2023 struct per_cpu_nodestat *n; <<
2024 1647
>> 1648 BUILD_BUG_ON(sizeof(p->vm_stat_diff[0]) != 1);
2025 /* 1649 /*
2026 * The fast way of checking i 1650 * The fast way of checking if there are any vmstat diffs.
>> 1651 * This works because the diffs are byte sized items.
2027 */ 1652 */
2028 if (memchr_inv(pzstats->vm_st !! 1653 if (memchr_inv(p->vm_stat_diff, 0, NR_VM_ZONE_STAT_ITEMS))
2029 return true; 1654 return true;
2030 1655
2031 if (last_pgdat == zone->zone_ <<
2032 continue; <<
2033 last_pgdat = zone->zone_pgdat <<
2034 n = per_cpu_ptr(zone->zone_pg <<
2035 if (memchr_inv(n->vm_node_sta <<
2036 return true; <<
2037 } 1656 }
2038 return false; 1657 return false;
2039 } 1658 }
2040 1659
2041 /* 1660 /*
2042 * Switch off vmstat processing and then fold 1661 * Switch off vmstat processing and then fold all the remaining differentials
2043 * until the diffs stay at zero. The function 1662 * until the diffs stay at zero. The function is used by NOHZ and can only be
2044 * invoked when tick processing is not active 1663 * invoked when tick processing is not active.
2045 */ 1664 */
2046 void quiet_vmstat(void) 1665 void quiet_vmstat(void)
2047 { 1666 {
2048 if (system_state != SYSTEM_RUNNING) 1667 if (system_state != SYSTEM_RUNNING)
2049 return; 1668 return;
2050 1669
2051 if (!delayed_work_pending(this_cpu_pt 1670 if (!delayed_work_pending(this_cpu_ptr(&vmstat_work)))
2052 return; 1671 return;
2053 1672
2054 if (!need_update(smp_processor_id())) 1673 if (!need_update(smp_processor_id()))
2055 return; 1674 return;
2056 1675
2057 /* 1676 /*
2058 * Just refresh counters and do not c 1677 * Just refresh counters and do not care about the pending delayed
2059 * vmstat_update. It doesn't fire tha 1678 * vmstat_update. It doesn't fire that often to matter and canceling
2060 * it would be too expensive from thi 1679 * it would be too expensive from this path.
2061 * vmstat_shepherd will take care abo 1680 * vmstat_shepherd will take care about that for us.
2062 */ 1681 */
2063 refresh_cpu_vm_stats(false); 1682 refresh_cpu_vm_stats(false);
2064 } 1683 }
2065 1684
2066 /* 1685 /*
2067 * Shepherd worker thread that checks the 1686 * Shepherd worker thread that checks the
2068 * differentials of processors that have thei 1687 * differentials of processors that have their worker
2069 * threads for vm statistics updates disabled 1688 * threads for vm statistics updates disabled because of
2070 * inactivity. 1689 * inactivity.
2071 */ 1690 */
2072 static void vmstat_shepherd(struct work_struc 1691 static void vmstat_shepherd(struct work_struct *w);
2073 1692
2074 static DECLARE_DEFERRABLE_WORK(shepherd, vmst 1693 static DECLARE_DEFERRABLE_WORK(shepherd, vmstat_shepherd);
2075 1694
2076 static void vmstat_shepherd(struct work_struc 1695 static void vmstat_shepherd(struct work_struct *w)
2077 { 1696 {
2078 int cpu; 1697 int cpu;
2079 1698
2080 cpus_read_lock(); !! 1699 get_online_cpus();
2081 /* Check processors whose vmstat work 1700 /* Check processors whose vmstat worker threads have been disabled */
2082 for_each_online_cpu(cpu) { 1701 for_each_online_cpu(cpu) {
2083 struct delayed_work *dw = &pe 1702 struct delayed_work *dw = &per_cpu(vmstat_work, cpu);
2084 1703
2085 /* <<
2086 * In kernel users of vmstat <<
2087 * they are using zone_page_s <<
2088 * an imprecision as the regu <<
2089 * cumulative error can grow <<
2090 * <<
2091 * From that POV the regular <<
2092 * been isolated from the ker <<
2093 * infrastructure ever notici <<
2094 * for all isolated CPUs to a <<
2095 */ <<
2096 if (cpu_is_isolated(cpu)) <<
2097 continue; <<
2098 <<
2099 if (!delayed_work_pending(dw) 1704 if (!delayed_work_pending(dw) && need_update(cpu))
2100 queue_delayed_work_on !! 1705 queue_delayed_work_on(cpu, vmstat_wq, dw, 0);
2101 <<
2102 cond_resched(); <<
2103 } 1706 }
2104 cpus_read_unlock(); !! 1707 put_online_cpus();
2105 1708
2106 schedule_delayed_work(&shepherd, 1709 schedule_delayed_work(&shepherd,
2107 round_jiffies_relative(sysctl 1710 round_jiffies_relative(sysctl_stat_interval));
2108 } 1711 }
2109 1712
2110 static void __init start_shepherd_timer(void) 1713 static void __init start_shepherd_timer(void)
2111 { 1714 {
2112 int cpu; 1715 int cpu;
2113 1716
2114 for_each_possible_cpu(cpu) 1717 for_each_possible_cpu(cpu)
2115 INIT_DEFERRABLE_WORK(per_cpu_ 1718 INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work, cpu),
2116 vmstat_update); 1719 vmstat_update);
2117 1720
>> 1721 vmstat_wq = alloc_workqueue("vmstat", WQ_FREEZABLE|WQ_MEM_RECLAIM, 0);
2118 schedule_delayed_work(&shepherd, 1722 schedule_delayed_work(&shepherd,
2119 round_jiffies_relative(sysctl 1723 round_jiffies_relative(sysctl_stat_interval));
2120 } 1724 }
2121 1725
2122 static void __init init_cpu_node_state(void) 1726 static void __init init_cpu_node_state(void)
2123 { 1727 {
2124 int node; !! 1728 int cpu;
2125 1729
2126 for_each_online_node(node) { !! 1730 get_online_cpus();
2127 if (!cpumask_empty(cpumask_of !! 1731 for_each_online_cpu(cpu)
2128 node_set_state(node, !! 1732 node_set_state(cpu_to_node(cpu), N_CPU);
2129 } !! 1733 put_online_cpus();
2130 } 1734 }
2131 1735
2132 static int vmstat_cpu_online(unsigned int cpu !! 1736 static void vmstat_cpu_dead(int node)
2133 { 1737 {
2134 refresh_zone_stat_thresholds(); !! 1738 int cpu;
2135 1739
2136 if (!node_state(cpu_to_node(cpu), N_C !! 1740 get_online_cpus();
2137 node_set_state(cpu_to_node(cp !! 1741 for_each_online_cpu(cpu)
2138 } !! 1742 if (cpu_to_node(cpu) == node)
>> 1743 goto end;
2139 1744
2140 return 0; !! 1745 node_clear_state(node, N_CPU);
2141 } !! 1746 end:
2142 !! 1747 put_online_cpus();
2143 static int vmstat_cpu_down_prep(unsigned int <<
2144 { <<
2145 cancel_delayed_work_sync(&per_cpu(vms <<
2146 return 0; <<
2147 } 1748 }
2148 1749
2149 static int vmstat_cpu_dead(unsigned int cpu) !! 1750 /*
>> 1751 * Use the cpu notifier to insure that the thresholds are recalculated
>> 1752 * when necessary.
>> 1753 */
>> 1754 static int vmstat_cpuup_callback(struct notifier_block *nfb,
>> 1755 unsigned long action,
>> 1756 void *hcpu)
2150 { 1757 {
2151 const struct cpumask *node_cpus; !! 1758 long cpu = (long)hcpu;
2152 int node; <<
2153 <<
2154 node = cpu_to_node(cpu); <<
2155 <<
2156 refresh_zone_stat_thresholds(); <<
2157 node_cpus = cpumask_of_node(node); <<
2158 if (!cpumask_empty(node_cpus)) <<
2159 return 0; <<
2160 1759
2161 node_clear_state(node, N_CPU); !! 1760 switch (action) {
2162 !! 1761 case CPU_ONLINE:
2163 return 0; !! 1762 case CPU_ONLINE_FROZEN:
>> 1763 refresh_zone_stat_thresholds();
>> 1764 node_set_state(cpu_to_node(cpu), N_CPU);
>> 1765 break;
>> 1766 case CPU_DOWN_PREPARE:
>> 1767 case CPU_DOWN_PREPARE_FROZEN:
>> 1768 cancel_delayed_work_sync(&per_cpu(vmstat_work, cpu));
>> 1769 break;
>> 1770 case CPU_DOWN_FAILED:
>> 1771 case CPU_DOWN_FAILED_FROZEN:
>> 1772 break;
>> 1773 case CPU_DEAD:
>> 1774 case CPU_DEAD_FROZEN:
>> 1775 refresh_zone_stat_thresholds();
>> 1776 vmstat_cpu_dead(cpu_to_node(cpu));
>> 1777 break;
>> 1778 default:
>> 1779 break;
>> 1780 }
>> 1781 return NOTIFY_OK;
2164 } 1782 }
2165 1783
>> 1784 static struct notifier_block vmstat_notifier =
>> 1785 { &vmstat_cpuup_callback, NULL, 0 };
2166 #endif 1786 #endif
2167 1787
2168 struct workqueue_struct *mm_percpu_wq; !! 1788 static int __init setup_vmstat(void)
2169 <<
2170 void __init init_mm_internals(void) <<
2171 { 1789 {
2172 int ret __maybe_unused; <<
2173 <<
2174 mm_percpu_wq = alloc_workqueue("mm_pe <<
2175 <<
2176 #ifdef CONFIG_SMP 1790 #ifdef CONFIG_SMP
2177 ret = cpuhp_setup_state_nocalls(CPUHP !! 1791 cpu_notifier_register_begin();
2178 NULL, !! 1792 __register_cpu_notifier(&vmstat_notifier);
2179 if (ret < 0) <<
2180 pr_err("vmstat: failed to reg <<
2181 <<
2182 ret = cpuhp_setup_state_nocalls(CPUHP <<
2183 vmsta <<
2184 vmsta <<
2185 if (ret < 0) <<
2186 pr_err("vmstat: failed to reg <<
2187 <<
2188 cpus_read_lock(); <<
2189 init_cpu_node_state(); 1793 init_cpu_node_state();
2190 cpus_read_unlock(); <<
2191 1794
2192 start_shepherd_timer(); 1795 start_shepherd_timer();
>> 1796 cpu_notifier_register_done();
2193 #endif 1797 #endif
2194 #ifdef CONFIG_PROC_FS 1798 #ifdef CONFIG_PROC_FS
2195 proc_create_seq("buddyinfo", 0444, NU !! 1799 proc_create("buddyinfo", S_IRUGO, NULL, &fragmentation_file_operations);
2196 proc_create_seq("pagetypeinfo", 0400, !! 1800 proc_create("pagetypeinfo", 0400, NULL, &pagetypeinfo_file_ops);
2197 proc_create_seq("vmstat", 0444, NULL, !! 1801 proc_create("vmstat", S_IRUGO, NULL, &proc_vmstat_file_operations);
2198 proc_create_seq("zoneinfo", 0444, NUL !! 1802 proc_create("zoneinfo", S_IRUGO, NULL, &proc_zoneinfo_file_operations);
2199 #endif 1803 #endif
>> 1804 return 0;
2200 } 1805 }
>> 1806 module_init(setup_vmstat)
2201 1807
2202 #if defined(CONFIG_DEBUG_FS) && defined(CONFI 1808 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)
2203 1809
2204 /* 1810 /*
2205 * Return an index indicating how much of the 1811 * Return an index indicating how much of the available free memory is
2206 * unusable for an allocation of the requeste 1812 * unusable for an allocation of the requested size.
2207 */ 1813 */
2208 static int unusable_free_index(unsigned int o 1814 static int unusable_free_index(unsigned int order,
2209 struct contig 1815 struct contig_page_info *info)
2210 { 1816 {
2211 /* No free memory is interpreted as a 1817 /* No free memory is interpreted as all free memory is unusable */
2212 if (info->free_pages == 0) 1818 if (info->free_pages == 0)
2213 return 1000; 1819 return 1000;
2214 1820
2215 /* 1821 /*
2216 * Index should be a value between 0 1822 * Index should be a value between 0 and 1. Return a value to 3
2217 * decimal places. 1823 * decimal places.
2218 * 1824 *
2219 * 0 => no fragmentation 1825 * 0 => no fragmentation
2220 * 1 => high fragmentation 1826 * 1 => high fragmentation
2221 */ 1827 */
2222 return div_u64((info->free_pages - (i 1828 return div_u64((info->free_pages - (info->free_blocks_suitable << order)) * 1000ULL, info->free_pages);
2223 1829
2224 } 1830 }
2225 1831
2226 static void unusable_show_print(struct seq_fi 1832 static void unusable_show_print(struct seq_file *m,
2227 pg_da 1833 pg_data_t *pgdat, struct zone *zone)
2228 { 1834 {
2229 unsigned int order; 1835 unsigned int order;
2230 int index; 1836 int index;
2231 struct contig_page_info info; 1837 struct contig_page_info info;
2232 1838
2233 seq_printf(m, "Node %d, zone %8s ", 1839 seq_printf(m, "Node %d, zone %8s ",
2234 pgdat->node_i 1840 pgdat->node_id,
2235 zone->name); 1841 zone->name);
2236 for (order = 0; order < NR_PAGE_ORDER !! 1842 for (order = 0; order < MAX_ORDER; ++order) {
2237 fill_contig_page_info(zone, o 1843 fill_contig_page_info(zone, order, &info);
2238 index = unusable_free_index(o 1844 index = unusable_free_index(order, &info);
2239 seq_printf(m, "%d.%03d ", ind 1845 seq_printf(m, "%d.%03d ", index / 1000, index % 1000);
2240 } 1846 }
2241 1847
2242 seq_putc(m, '\n'); 1848 seq_putc(m, '\n');
2243 } 1849 }
2244 1850
2245 /* 1851 /*
2246 * Display unusable free space index 1852 * Display unusable free space index
2247 * 1853 *
2248 * The unusable free space index measures how 1854 * The unusable free space index measures how much of the available free
2249 * memory cannot be used to satisfy an alloca 1855 * memory cannot be used to satisfy an allocation of a given size and is a
2250 * value between 0 and 1. The higher the valu 1856 * value between 0 and 1. The higher the value, the more of free memory is
2251 * unusable and by implication, the worse the 1857 * unusable and by implication, the worse the external fragmentation is. This
2252 * can be expressed as a percentage by multip 1858 * can be expressed as a percentage by multiplying by 100.
2253 */ 1859 */
2254 static int unusable_show(struct seq_file *m, 1860 static int unusable_show(struct seq_file *m, void *arg)
2255 { 1861 {
2256 pg_data_t *pgdat = (pg_data_t *)arg; 1862 pg_data_t *pgdat = (pg_data_t *)arg;
2257 1863
2258 /* check memoryless node */ 1864 /* check memoryless node */
2259 if (!node_state(pgdat->node_id, N_MEM 1865 if (!node_state(pgdat->node_id, N_MEMORY))
2260 return 0; 1866 return 0;
2261 1867
2262 walk_zones_in_node(m, pgdat, true, fa !! 1868 walk_zones_in_node(m, pgdat, unusable_show_print);
2263 1869
2264 return 0; 1870 return 0;
2265 } 1871 }
2266 1872
2267 static const struct seq_operations unusable_s !! 1873 static const struct seq_operations unusable_op = {
2268 .start = frag_start, 1874 .start = frag_start,
2269 .next = frag_next, 1875 .next = frag_next,
2270 .stop = frag_stop, 1876 .stop = frag_stop,
2271 .show = unusable_show, 1877 .show = unusable_show,
2272 }; 1878 };
2273 1879
2274 DEFINE_SEQ_ATTRIBUTE(unusable); !! 1880 static int unusable_open(struct inode *inode, struct file *file)
>> 1881 {
>> 1882 return seq_open(file, &unusable_op);
>> 1883 }
>> 1884
>> 1885 static const struct file_operations unusable_file_ops = {
>> 1886 .open = unusable_open,
>> 1887 .read = seq_read,
>> 1888 .llseek = seq_lseek,
>> 1889 .release = seq_release,
>> 1890 };
2275 1891
2276 static void extfrag_show_print(struct seq_fil 1892 static void extfrag_show_print(struct seq_file *m,
2277 pg_da 1893 pg_data_t *pgdat, struct zone *zone)
2278 { 1894 {
2279 unsigned int order; 1895 unsigned int order;
2280 int index; 1896 int index;
2281 1897
2282 /* Alloc on stack as interrupts are d 1898 /* Alloc on stack as interrupts are disabled for zone walk */
2283 struct contig_page_info info; 1899 struct contig_page_info info;
2284 1900
2285 seq_printf(m, "Node %d, zone %8s ", 1901 seq_printf(m, "Node %d, zone %8s ",
2286 pgdat->node_i 1902 pgdat->node_id,
2287 zone->name); 1903 zone->name);
2288 for (order = 0; order < NR_PAGE_ORDER !! 1904 for (order = 0; order < MAX_ORDER; ++order) {
2289 fill_contig_page_info(zone, o 1905 fill_contig_page_info(zone, order, &info);
2290 index = __fragmentation_index 1906 index = __fragmentation_index(order, &info);
2291 seq_printf(m, "%2d.%03d ", in !! 1907 seq_printf(m, "%d.%03d ", index / 1000, index % 1000);
2292 } 1908 }
2293 1909
2294 seq_putc(m, '\n'); 1910 seq_putc(m, '\n');
2295 } 1911 }
2296 1912
2297 /* 1913 /*
2298 * Display fragmentation index for orders tha 1914 * Display fragmentation index for orders that allocations would fail for
2299 */ 1915 */
2300 static int extfrag_show(struct seq_file *m, v 1916 static int extfrag_show(struct seq_file *m, void *arg)
2301 { 1917 {
2302 pg_data_t *pgdat = (pg_data_t *)arg; 1918 pg_data_t *pgdat = (pg_data_t *)arg;
2303 1919
2304 walk_zones_in_node(m, pgdat, true, fa !! 1920 walk_zones_in_node(m, pgdat, extfrag_show_print);
2305 1921
2306 return 0; 1922 return 0;
2307 } 1923 }
2308 1924
2309 static const struct seq_operations extfrag_so !! 1925 static const struct seq_operations extfrag_op = {
2310 .start = frag_start, 1926 .start = frag_start,
2311 .next = frag_next, 1927 .next = frag_next,
2312 .stop = frag_stop, 1928 .stop = frag_stop,
2313 .show = extfrag_show, 1929 .show = extfrag_show,
2314 }; 1930 };
2315 1931
2316 DEFINE_SEQ_ATTRIBUTE(extfrag); !! 1932 static int extfrag_open(struct inode *inode, struct file *file)
>> 1933 {
>> 1934 return seq_open(file, &extfrag_op);
>> 1935 }
>> 1936
>> 1937 static const struct file_operations extfrag_file_ops = {
>> 1938 .open = extfrag_open,
>> 1939 .read = seq_read,
>> 1940 .llseek = seq_lseek,
>> 1941 .release = seq_release,
>> 1942 };
2317 1943
2318 static int __init extfrag_debug_init(void) 1944 static int __init extfrag_debug_init(void)
2319 { 1945 {
2320 struct dentry *extfrag_debug_root; 1946 struct dentry *extfrag_debug_root;
2321 1947
2322 extfrag_debug_root = debugfs_create_d 1948 extfrag_debug_root = debugfs_create_dir("extfrag", NULL);
>> 1949 if (!extfrag_debug_root)
>> 1950 return -ENOMEM;
2323 1951
2324 debugfs_create_file("unusable_index", !! 1952 if (!debugfs_create_file("unusable_index", 0444,
2325 &unusable_fops); !! 1953 extfrag_debug_root, NULL, &unusable_file_ops))
2326 !! 1954 goto fail;
2327 debugfs_create_file("extfrag_index", !! 1955
2328 &extfrag_fops); !! 1956 if (!debugfs_create_file("extfrag_index", 0444,
>> 1957 extfrag_debug_root, NULL, &extfrag_file_ops))
>> 1958 goto fail;
2329 1959
2330 return 0; 1960 return 0;
>> 1961 fail:
>> 1962 debugfs_remove_recursive(extfrag_debug_root);
>> 1963 return -ENOMEM;
2331 } 1964 }
2332 1965
2333 module_init(extfrag_debug_init); 1966 module_init(extfrag_debug_init);
2334 <<
2335 #endif 1967 #endif
2336 1968
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.