1 // SPDX-License-Identifier: GPL-2.0-only <<
2 /* 1 /*
3 * linux/mm/vmstat.c 2 * linux/mm/vmstat.c
4 * 3 *
5 * Manages VM statistics 4 * Manages VM statistics
6 * Copyright (C) 1991, 1992, 1993, 1994 Linu 5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
7 * 6 *
8 * zoned VM statistics 7 * zoned VM statistics
9 * Copyright (C) 2006 Silicon Graphics, Inc., 8 * Copyright (C) 2006 Silicon Graphics, Inc.,
10 * Christoph Lameter <christoph@l 9 * Christoph Lameter <christoph@lameter.com>
11 * Copyright (C) 2008-2014 Christoph Lameter 10 * Copyright (C) 2008-2014 Christoph Lameter
12 */ 11 */
13 #include <linux/fs.h> 12 #include <linux/fs.h>
14 #include <linux/mm.h> 13 #include <linux/mm.h>
15 #include <linux/err.h> 14 #include <linux/err.h>
16 #include <linux/module.h> 15 #include <linux/module.h>
17 #include <linux/slab.h> 16 #include <linux/slab.h>
18 #include <linux/cpu.h> 17 #include <linux/cpu.h>
19 #include <linux/cpumask.h> 18 #include <linux/cpumask.h>
20 #include <linux/vmstat.h> 19 #include <linux/vmstat.h>
21 #include <linux/proc_fs.h> 20 #include <linux/proc_fs.h>
22 #include <linux/seq_file.h> 21 #include <linux/seq_file.h>
23 #include <linux/debugfs.h> 22 #include <linux/debugfs.h>
24 #include <linux/sched.h> 23 #include <linux/sched.h>
25 #include <linux/math64.h> 24 #include <linux/math64.h>
26 #include <linux/writeback.h> 25 #include <linux/writeback.h>
27 #include <linux/compaction.h> 26 #include <linux/compaction.h>
28 #include <linux/mm_inline.h> 27 #include <linux/mm_inline.h>
>> 28 #include <linux/page_ext.h>
29 #include <linux/page_owner.h> 29 #include <linux/page_owner.h>
30 #include <linux/sched/isolation.h> <<
31 30
32 #include "internal.h" 31 #include "internal.h"
33 32
34 #ifdef CONFIG_NUMA <<
35 int sysctl_vm_numa_stat = ENABLE_NUMA_STAT; <<
36 <<
37 /* zero numa counters within a zone */ <<
38 static void zero_zone_numa_counters(struct zon <<
39 { <<
40 int item, cpu; <<
41 <<
42 for (item = 0; item < NR_VM_NUMA_EVENT <<
43 atomic_long_set(&zone->vm_numa <<
44 for_each_online_cpu(cpu) { <<
45 per_cpu_ptr(zone->per_ <<
46 <<
47 } <<
48 } <<
49 } <<
50 <<
51 /* zero numa counters of all the populated zon <<
52 static void zero_zones_numa_counters(void) <<
53 { <<
54 struct zone *zone; <<
55 <<
56 for_each_populated_zone(zone) <<
57 zero_zone_numa_counters(zone); <<
58 } <<
59 <<
60 /* zero global numa counters */ <<
61 static void zero_global_numa_counters(void) <<
62 { <<
63 int item; <<
64 <<
65 for (item = 0; item < NR_VM_NUMA_EVENT <<
66 atomic_long_set(&vm_numa_event <<
67 } <<
68 <<
69 static void invalid_numa_statistics(void) <<
70 { <<
71 zero_zones_numa_counters(); <<
72 zero_global_numa_counters(); <<
73 } <<
74 <<
75 static DEFINE_MUTEX(vm_numa_stat_lock); <<
76 <<
77 int sysctl_vm_numa_stat_handler(const struct c <<
78 void *buffer, size_t *length, <<
79 { <<
80 int ret, oldval; <<
81 <<
82 mutex_lock(&vm_numa_stat_lock); <<
83 if (write) <<
84 oldval = sysctl_vm_numa_stat; <<
85 ret = proc_dointvec_minmax(table, writ <<
86 if (ret || !write) <<
87 goto out; <<
88 <<
89 if (oldval == sysctl_vm_numa_stat) <<
90 goto out; <<
91 else if (sysctl_vm_numa_stat == ENABLE <<
92 static_branch_enable(&vm_numa_ <<
93 pr_info("enable numa statistic <<
94 } else { <<
95 static_branch_disable(&vm_numa <<
96 invalid_numa_statistics(); <<
97 pr_info("disable numa statisti <<
98 } <<
99 <<
100 out: <<
101 mutex_unlock(&vm_numa_stat_lock); <<
102 return ret; <<
103 } <<
104 #endif <<
105 <<
106 #ifdef CONFIG_VM_EVENT_COUNTERS 33 #ifdef CONFIG_VM_EVENT_COUNTERS
107 DEFINE_PER_CPU(struct vm_event_state, vm_event 34 DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}};
108 EXPORT_PER_CPU_SYMBOL(vm_event_states); 35 EXPORT_PER_CPU_SYMBOL(vm_event_states);
109 36
110 static void sum_vm_events(unsigned long *ret) 37 static void sum_vm_events(unsigned long *ret)
111 { 38 {
112 int cpu; 39 int cpu;
113 int i; 40 int i;
114 41
115 memset(ret, 0, NR_VM_EVENT_ITEMS * siz 42 memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long));
116 43
117 for_each_online_cpu(cpu) { 44 for_each_online_cpu(cpu) {
118 struct vm_event_state *this = 45 struct vm_event_state *this = &per_cpu(vm_event_states, cpu);
119 46
120 for (i = 0; i < NR_VM_EVENT_IT 47 for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
121 ret[i] += this->event[ 48 ret[i] += this->event[i];
122 } 49 }
123 } 50 }
124 51
125 /* 52 /*
126 * Accumulate the vm event counters across all 53 * Accumulate the vm event counters across all CPUs.
127 * The result is unavoidably approximate - it 54 * The result is unavoidably approximate - it can change
128 * during and after execution of this function 55 * during and after execution of this function.
129 */ 56 */
130 void all_vm_events(unsigned long *ret) 57 void all_vm_events(unsigned long *ret)
131 { 58 {
132 cpus_read_lock(); !! 59 get_online_cpus();
133 sum_vm_events(ret); 60 sum_vm_events(ret);
134 cpus_read_unlock(); !! 61 put_online_cpus();
135 } 62 }
136 EXPORT_SYMBOL_GPL(all_vm_events); 63 EXPORT_SYMBOL_GPL(all_vm_events);
137 64
138 /* 65 /*
139 * Fold the foreign cpu events into our own. 66 * Fold the foreign cpu events into our own.
140 * 67 *
141 * This is adding to the events on one process 68 * This is adding to the events on one processor
142 * but keeps the global counts constant. 69 * but keeps the global counts constant.
143 */ 70 */
144 void vm_events_fold_cpu(int cpu) 71 void vm_events_fold_cpu(int cpu)
145 { 72 {
146 struct vm_event_state *fold_state = &p 73 struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu);
147 int i; 74 int i;
148 75
149 for (i = 0; i < NR_VM_EVENT_ITEMS; i++ 76 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
150 count_vm_events(i, fold_state- 77 count_vm_events(i, fold_state->event[i]);
151 fold_state->event[i] = 0; 78 fold_state->event[i] = 0;
152 } 79 }
153 } 80 }
154 81
155 #endif /* CONFIG_VM_EVENT_COUNTERS */ 82 #endif /* CONFIG_VM_EVENT_COUNTERS */
156 83
157 /* 84 /*
158 * Manage combined zone based / global counter 85 * Manage combined zone based / global counters
159 * 86 *
160 * vm_stat contains the global counters 87 * vm_stat contains the global counters
161 */ 88 */
162 atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITE 89 atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp;
163 atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITE 90 atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS] __cacheline_aligned_in_smp;
164 atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_I <<
165 EXPORT_SYMBOL(vm_zone_stat); 91 EXPORT_SYMBOL(vm_zone_stat);
166 EXPORT_SYMBOL(vm_node_stat); 92 EXPORT_SYMBOL(vm_node_stat);
167 93
168 #ifdef CONFIG_NUMA <<
169 static void fold_vm_zone_numa_events(struct zo <<
170 { <<
171 unsigned long zone_numa_events[NR_VM_N <<
172 int cpu; <<
173 enum numa_stat_item item; <<
174 <<
175 for_each_online_cpu(cpu) { <<
176 struct per_cpu_zonestat *pzsta <<
177 <<
178 pzstats = per_cpu_ptr(zone->pe <<
179 for (item = 0; item < NR_VM_NU <<
180 zone_numa_events[item] <<
181 } <<
182 <<
183 for (item = 0; item < NR_VM_NUMA_EVENT <<
184 zone_numa_event_add(zone_numa_ <<
185 } <<
186 <<
187 void fold_vm_numa_events(void) <<
188 { <<
189 struct zone *zone; <<
190 <<
191 for_each_populated_zone(zone) <<
192 fold_vm_zone_numa_events(zone) <<
193 } <<
194 #endif <<
195 <<
196 #ifdef CONFIG_SMP 94 #ifdef CONFIG_SMP
197 95
198 int calculate_pressure_threshold(struct zone * 96 int calculate_pressure_threshold(struct zone *zone)
199 { 97 {
200 int threshold; 98 int threshold;
201 int watermark_distance; 99 int watermark_distance;
202 100
203 /* 101 /*
204 * As vmstats are not up to date, ther 102 * As vmstats are not up to date, there is drift between the estimated
205 * and real values. For high threshold 103 * and real values. For high thresholds and a high number of CPUs, it
206 * is possible for the min watermark t 104 * is possible for the min watermark to be breached while the estimated
207 * value looks fine. The pressure thre 105 * value looks fine. The pressure threshold is a reduced value such
208 * that even the maximum amount of dri 106 * that even the maximum amount of drift will not accidentally breach
209 * the min watermark 107 * the min watermark
210 */ 108 */
211 watermark_distance = low_wmark_pages(z 109 watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone);
212 threshold = max(1, (int)(watermark_dis 110 threshold = max(1, (int)(watermark_distance / num_online_cpus()));
213 111
214 /* 112 /*
215 * Maximum threshold is 125 113 * Maximum threshold is 125
216 */ 114 */
217 threshold = min(125, threshold); 115 threshold = min(125, threshold);
218 116
219 return threshold; 117 return threshold;
220 } 118 }
221 119
222 int calculate_normal_threshold(struct zone *zo 120 int calculate_normal_threshold(struct zone *zone)
223 { 121 {
224 int threshold; 122 int threshold;
225 int mem; /* memory in 128 MB un 123 int mem; /* memory in 128 MB units */
226 124
227 /* 125 /*
228 * The threshold scales with the numbe 126 * The threshold scales with the number of processors and the amount
229 * of memory per zone. More memory mea 127 * of memory per zone. More memory means that we can defer updates for
230 * longer, more processors could lead 128 * longer, more processors could lead to more contention.
231 * fls() is used to have a cheap way o 129 * fls() is used to have a cheap way of logarithmic scaling.
232 * 130 *
233 * Some sample thresholds: 131 * Some sample thresholds:
234 * 132 *
235 * Threshold Processors (fls) !! 133 * Threshold Processors (fls) Zonesize fls(mem+1)
236 * ----------------------------------- 134 * ------------------------------------------------------------------
237 * 8 1 1 135 * 8 1 1 0.9-1 GB 4
238 * 16 2 2 136 * 16 2 2 0.9-1 GB 4
239 * 20 2 2 137 * 20 2 2 1-2 GB 5
240 * 24 2 2 138 * 24 2 2 2-4 GB 6
241 * 28 2 2 139 * 28 2 2 4-8 GB 7
242 * 32 2 2 140 * 32 2 2 8-16 GB 8
243 * 4 2 2 141 * 4 2 2 <128M 1
244 * 30 4 3 142 * 30 4 3 2-4 GB 5
245 * 48 4 3 143 * 48 4 3 8-16 GB 8
246 * 32 8 4 144 * 32 8 4 1-2 GB 4
247 * 32 8 4 145 * 32 8 4 0.9-1GB 4
248 * 10 16 5 146 * 10 16 5 <128M 1
249 * 40 16 5 147 * 40 16 5 900M 4
250 * 70 64 7 148 * 70 64 7 2-4 GB 5
251 * 84 64 7 149 * 84 64 7 4-8 GB 6
252 * 108 512 9 150 * 108 512 9 4-8 GB 6
253 * 125 1024 10 151 * 125 1024 10 8-16 GB 8
254 * 125 1024 10 152 * 125 1024 10 16-32 GB 9
255 */ 153 */
256 154
257 mem = zone_managed_pages(zone) >> (27 !! 155 mem = zone->managed_pages >> (27 - PAGE_SHIFT);
258 156
259 threshold = 2 * fls(num_online_cpus()) 157 threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem));
260 158
261 /* 159 /*
262 * Maximum threshold is 125 160 * Maximum threshold is 125
263 */ 161 */
264 threshold = min(125, threshold); 162 threshold = min(125, threshold);
265 163
266 return threshold; 164 return threshold;
267 } 165 }
268 166
269 /* 167 /*
270 * Refresh the thresholds for each zone. 168 * Refresh the thresholds for each zone.
271 */ 169 */
272 void refresh_zone_stat_thresholds(void) 170 void refresh_zone_stat_thresholds(void)
273 { 171 {
274 struct pglist_data *pgdat; 172 struct pglist_data *pgdat;
275 struct zone *zone; 173 struct zone *zone;
276 int cpu; 174 int cpu;
277 int threshold; 175 int threshold;
278 176
279 /* Zero current pgdat thresholds */ 177 /* Zero current pgdat thresholds */
280 for_each_online_pgdat(pgdat) { 178 for_each_online_pgdat(pgdat) {
281 for_each_online_cpu(cpu) { 179 for_each_online_cpu(cpu) {
282 per_cpu_ptr(pgdat->per 180 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = 0;
283 } 181 }
284 } 182 }
285 183
286 for_each_populated_zone(zone) { 184 for_each_populated_zone(zone) {
287 struct pglist_data *pgdat = zo 185 struct pglist_data *pgdat = zone->zone_pgdat;
288 unsigned long max_drift, toler 186 unsigned long max_drift, tolerate_drift;
289 187
290 threshold = calculate_normal_t 188 threshold = calculate_normal_threshold(zone);
291 189
292 for_each_online_cpu(cpu) { 190 for_each_online_cpu(cpu) {
293 int pgdat_threshold; 191 int pgdat_threshold;
294 192
295 per_cpu_ptr(zone->per_ !! 193 per_cpu_ptr(zone->pageset, cpu)->stat_threshold
296 194 = threshold;
297 195
298 /* Base nodestat thres 196 /* Base nodestat threshold on the largest populated zone. */
299 pgdat_threshold = per_ 197 pgdat_threshold = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold;
300 per_cpu_ptr(pgdat->per 198 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold
301 = max(threshol 199 = max(threshold, pgdat_threshold);
302 } 200 }
303 201
304 /* 202 /*
305 * Only set percpu_drift_mark 203 * Only set percpu_drift_mark if there is a danger that
306 * NR_FREE_PAGES reports the l 204 * NR_FREE_PAGES reports the low watermark is ok when in fact
307 * the min watermark could be 205 * the min watermark could be breached by an allocation
308 */ 206 */
309 tolerate_drift = low_wmark_pag 207 tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone);
310 max_drift = num_online_cpus() 208 max_drift = num_online_cpus() * threshold;
311 if (max_drift > tolerate_drift 209 if (max_drift > tolerate_drift)
312 zone->percpu_drift_mar 210 zone->percpu_drift_mark = high_wmark_pages(zone) +
313 max_dr 211 max_drift;
314 } 212 }
315 } 213 }
316 214
317 void set_pgdat_percpu_threshold(pg_data_t *pgd 215 void set_pgdat_percpu_threshold(pg_data_t *pgdat,
318 int (*calculat 216 int (*calculate_pressure)(struct zone *))
319 { 217 {
320 struct zone *zone; 218 struct zone *zone;
321 int cpu; 219 int cpu;
322 int threshold; 220 int threshold;
323 int i; 221 int i;
324 222
325 for (i = 0; i < pgdat->nr_zones; i++) 223 for (i = 0; i < pgdat->nr_zones; i++) {
326 zone = &pgdat->node_zones[i]; 224 zone = &pgdat->node_zones[i];
327 if (!zone->percpu_drift_mark) 225 if (!zone->percpu_drift_mark)
328 continue; 226 continue;
329 227
330 threshold = (*calculate_pressu 228 threshold = (*calculate_pressure)(zone);
331 for_each_online_cpu(cpu) 229 for_each_online_cpu(cpu)
332 per_cpu_ptr(zone->per_ !! 230 per_cpu_ptr(zone->pageset, cpu)->stat_threshold
333 231 = threshold;
334 } 232 }
335 } 233 }
336 234
337 /* 235 /*
338 * For use when we know that interrupts are di 236 * For use when we know that interrupts are disabled,
339 * or when we know that preemption is disabled 237 * or when we know that preemption is disabled and that
340 * particular counter cannot be updated from i 238 * particular counter cannot be updated from interrupt context.
341 */ 239 */
342 void __mod_zone_page_state(struct zone *zone, 240 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
343 long delta) 241 long delta)
344 { 242 {
345 struct per_cpu_zonestat __percpu *pcp !! 243 struct per_cpu_pageset __percpu *pcp = zone->pageset;
346 s8 __percpu *p = pcp->vm_stat_diff + i 244 s8 __percpu *p = pcp->vm_stat_diff + item;
347 long x; 245 long x;
348 long t; 246 long t;
349 247
350 /* <<
351 * Accurate vmstat updates require a R <<
352 * atomicity is provided by IRQs being <<
353 * or via local_lock_irq. On PREEMPT_R <<
354 * CPU migrations and preemption poten <<
355 * disable preemption. <<
356 */ <<
357 preempt_disable_nested(); <<
358 <<
359 x = delta + __this_cpu_read(*p); 248 x = delta + __this_cpu_read(*p);
360 249
361 t = __this_cpu_read(pcp->stat_threshol 250 t = __this_cpu_read(pcp->stat_threshold);
362 251
363 if (unlikely(abs(x) > t)) { !! 252 if (unlikely(x > t || x < -t)) {
364 zone_page_state_add(x, zone, i 253 zone_page_state_add(x, zone, item);
365 x = 0; 254 x = 0;
366 } 255 }
367 __this_cpu_write(*p, x); 256 __this_cpu_write(*p, x);
368 <<
369 preempt_enable_nested(); <<
370 } 257 }
371 EXPORT_SYMBOL(__mod_zone_page_state); 258 EXPORT_SYMBOL(__mod_zone_page_state);
372 259
373 void __mod_node_page_state(struct pglist_data 260 void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
374 long delta) 261 long delta)
375 { 262 {
376 struct per_cpu_nodestat __percpu *pcp 263 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
377 s8 __percpu *p = pcp->vm_node_stat_dif 264 s8 __percpu *p = pcp->vm_node_stat_diff + item;
378 long x; 265 long x;
379 long t; 266 long t;
380 267
381 if (vmstat_item_in_bytes(item)) { <<
382 /* <<
383 * Only cgroups use subpage ac <<
384 * the global level, these ite <<
385 * multiples of whole pages. S <<
386 * internally to keep the per- <<
387 */ <<
388 VM_WARN_ON_ONCE(delta & (PAGE_ <<
389 delta >>= PAGE_SHIFT; <<
390 } <<
391 <<
392 /* See __mod_node_page_state */ <<
393 preempt_disable_nested(); <<
394 <<
395 x = delta + __this_cpu_read(*p); 268 x = delta + __this_cpu_read(*p);
396 269
397 t = __this_cpu_read(pcp->stat_threshol 270 t = __this_cpu_read(pcp->stat_threshold);
398 271
399 if (unlikely(abs(x) > t)) { !! 272 if (unlikely(x > t || x < -t)) {
400 node_page_state_add(x, pgdat, 273 node_page_state_add(x, pgdat, item);
401 x = 0; 274 x = 0;
402 } 275 }
403 __this_cpu_write(*p, x); 276 __this_cpu_write(*p, x);
404 <<
405 preempt_enable_nested(); <<
406 } 277 }
407 EXPORT_SYMBOL(__mod_node_page_state); 278 EXPORT_SYMBOL(__mod_node_page_state);
408 279
409 /* 280 /*
410 * Optimized increment and decrement functions 281 * Optimized increment and decrement functions.
411 * 282 *
412 * These are only for a single page and theref 283 * These are only for a single page and therefore can take a struct page *
413 * argument instead of struct zone *. This all 284 * argument instead of struct zone *. This allows the inclusion of the code
414 * generated for page_zone(page) into the opti 285 * generated for page_zone(page) into the optimized functions.
415 * 286 *
416 * No overflow check is necessary and therefor 287 * No overflow check is necessary and therefore the differential can be
417 * incremented or decremented in place which m 288 * incremented or decremented in place which may allow the compilers to
418 * generate better code. 289 * generate better code.
419 * The increment or decrement is known and the 290 * The increment or decrement is known and therefore one boundary check can
420 * be omitted. 291 * be omitted.
421 * 292 *
422 * NOTE: These functions are very performance 293 * NOTE: These functions are very performance sensitive. Change only
423 * with care. 294 * with care.
424 * 295 *
425 * Some processors have inc/dec instructions t 296 * Some processors have inc/dec instructions that are atomic vs an interrupt.
426 * However, the code must first determine the 297 * However, the code must first determine the differential location in a zone
427 * based on the processor number and then inc/ 298 * based on the processor number and then inc/dec the counter. There is no
428 * guarantee without disabling preemption that 299 * guarantee without disabling preemption that the processor will not change
429 * in between and therefore the atomicity vs. 300 * in between and therefore the atomicity vs. interrupt cannot be exploited
430 * in a useful way here. 301 * in a useful way here.
431 */ 302 */
432 void __inc_zone_state(struct zone *zone, enum 303 void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
433 { 304 {
434 struct per_cpu_zonestat __percpu *pcp !! 305 struct per_cpu_pageset __percpu *pcp = zone->pageset;
435 s8 __percpu *p = pcp->vm_stat_diff + i 306 s8 __percpu *p = pcp->vm_stat_diff + item;
436 s8 v, t; 307 s8 v, t;
437 308
438 /* See __mod_node_page_state */ <<
439 preempt_disable_nested(); <<
440 <<
441 v = __this_cpu_inc_return(*p); 309 v = __this_cpu_inc_return(*p);
442 t = __this_cpu_read(pcp->stat_threshol 310 t = __this_cpu_read(pcp->stat_threshold);
443 if (unlikely(v > t)) { 311 if (unlikely(v > t)) {
444 s8 overstep = t >> 1; 312 s8 overstep = t >> 1;
445 313
446 zone_page_state_add(v + overst 314 zone_page_state_add(v + overstep, zone, item);
447 __this_cpu_write(*p, -overstep 315 __this_cpu_write(*p, -overstep);
448 } 316 }
449 <<
450 preempt_enable_nested(); <<
451 } 317 }
452 318
453 void __inc_node_state(struct pglist_data *pgda 319 void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
454 { 320 {
455 struct per_cpu_nodestat __percpu *pcp 321 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
456 s8 __percpu *p = pcp->vm_node_stat_dif 322 s8 __percpu *p = pcp->vm_node_stat_diff + item;
457 s8 v, t; 323 s8 v, t;
458 324
459 VM_WARN_ON_ONCE(vmstat_item_in_bytes(i <<
460 <<
461 /* See __mod_node_page_state */ <<
462 preempt_disable_nested(); <<
463 <<
464 v = __this_cpu_inc_return(*p); 325 v = __this_cpu_inc_return(*p);
465 t = __this_cpu_read(pcp->stat_threshol 326 t = __this_cpu_read(pcp->stat_threshold);
466 if (unlikely(v > t)) { 327 if (unlikely(v > t)) {
467 s8 overstep = t >> 1; 328 s8 overstep = t >> 1;
468 329
469 node_page_state_add(v + overst 330 node_page_state_add(v + overstep, pgdat, item);
470 __this_cpu_write(*p, -overstep 331 __this_cpu_write(*p, -overstep);
471 } 332 }
472 <<
473 preempt_enable_nested(); <<
474 } 333 }
475 334
476 void __inc_zone_page_state(struct page *page, 335 void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
477 { 336 {
478 __inc_zone_state(page_zone(page), item 337 __inc_zone_state(page_zone(page), item);
479 } 338 }
480 EXPORT_SYMBOL(__inc_zone_page_state); 339 EXPORT_SYMBOL(__inc_zone_page_state);
481 340
482 void __inc_node_page_state(struct page *page, 341 void __inc_node_page_state(struct page *page, enum node_stat_item item)
483 { 342 {
484 __inc_node_state(page_pgdat(page), ite 343 __inc_node_state(page_pgdat(page), item);
485 } 344 }
486 EXPORT_SYMBOL(__inc_node_page_state); 345 EXPORT_SYMBOL(__inc_node_page_state);
487 346
488 void __dec_zone_state(struct zone *zone, enum 347 void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
489 { 348 {
490 struct per_cpu_zonestat __percpu *pcp !! 349 struct per_cpu_pageset __percpu *pcp = zone->pageset;
491 s8 __percpu *p = pcp->vm_stat_diff + i 350 s8 __percpu *p = pcp->vm_stat_diff + item;
492 s8 v, t; 351 s8 v, t;
493 352
494 /* See __mod_node_page_state */ <<
495 preempt_disable_nested(); <<
496 <<
497 v = __this_cpu_dec_return(*p); 353 v = __this_cpu_dec_return(*p);
498 t = __this_cpu_read(pcp->stat_threshol 354 t = __this_cpu_read(pcp->stat_threshold);
499 if (unlikely(v < - t)) { 355 if (unlikely(v < - t)) {
500 s8 overstep = t >> 1; 356 s8 overstep = t >> 1;
501 357
502 zone_page_state_add(v - overst 358 zone_page_state_add(v - overstep, zone, item);
503 __this_cpu_write(*p, overstep) 359 __this_cpu_write(*p, overstep);
504 } 360 }
505 <<
506 preempt_enable_nested(); <<
507 } 361 }
508 362
509 void __dec_node_state(struct pglist_data *pgda 363 void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
510 { 364 {
511 struct per_cpu_nodestat __percpu *pcp 365 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
512 s8 __percpu *p = pcp->vm_node_stat_dif 366 s8 __percpu *p = pcp->vm_node_stat_diff + item;
513 s8 v, t; 367 s8 v, t;
514 368
515 VM_WARN_ON_ONCE(vmstat_item_in_bytes(i <<
516 <<
517 /* See __mod_node_page_state */ <<
518 preempt_disable_nested(); <<
519 <<
520 v = __this_cpu_dec_return(*p); 369 v = __this_cpu_dec_return(*p);
521 t = __this_cpu_read(pcp->stat_threshol 370 t = __this_cpu_read(pcp->stat_threshold);
522 if (unlikely(v < - t)) { 371 if (unlikely(v < - t)) {
523 s8 overstep = t >> 1; 372 s8 overstep = t >> 1;
524 373
525 node_page_state_add(v - overst 374 node_page_state_add(v - overstep, pgdat, item);
526 __this_cpu_write(*p, overstep) 375 __this_cpu_write(*p, overstep);
527 } 376 }
528 <<
529 preempt_enable_nested(); <<
530 } 377 }
531 378
532 void __dec_zone_page_state(struct page *page, 379 void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
533 { 380 {
534 __dec_zone_state(page_zone(page), item 381 __dec_zone_state(page_zone(page), item);
535 } 382 }
536 EXPORT_SYMBOL(__dec_zone_page_state); 383 EXPORT_SYMBOL(__dec_zone_page_state);
537 384
538 void __dec_node_page_state(struct page *page, 385 void __dec_node_page_state(struct page *page, enum node_stat_item item)
539 { 386 {
540 __dec_node_state(page_pgdat(page), ite 387 __dec_node_state(page_pgdat(page), item);
541 } 388 }
542 EXPORT_SYMBOL(__dec_node_page_state); 389 EXPORT_SYMBOL(__dec_node_page_state);
543 390
544 #ifdef CONFIG_HAVE_CMPXCHG_LOCAL 391 #ifdef CONFIG_HAVE_CMPXCHG_LOCAL
545 /* 392 /*
546 * If we have cmpxchg_local support then we do 393 * If we have cmpxchg_local support then we do not need to incur the overhead
547 * that comes with local_irq_save/restore if w 394 * that comes with local_irq_save/restore if we use this_cpu_cmpxchg.
548 * 395 *
549 * mod_state() modifies the zone counter state 396 * mod_state() modifies the zone counter state through atomic per cpu
550 * operations. 397 * operations.
551 * 398 *
552 * Overstep mode specifies how overstep should 399 * Overstep mode specifies how overstep should handled:
553 * 0 No overstepping 400 * 0 No overstepping
554 * 1 Overstepping half of threshold 401 * 1 Overstepping half of threshold
555 * -1 Overstepping minus half of thre 402 * -1 Overstepping minus half of threshold
556 */ 403 */
557 static inline void mod_zone_state(struct zone 404 static inline void mod_zone_state(struct zone *zone,
558 enum zone_stat_item item, long delta, i 405 enum zone_stat_item item, long delta, int overstep_mode)
559 { 406 {
560 struct per_cpu_zonestat __percpu *pcp !! 407 struct per_cpu_pageset __percpu *pcp = zone->pageset;
561 s8 __percpu *p = pcp->vm_stat_diff + i 408 s8 __percpu *p = pcp->vm_stat_diff + item;
562 long n, t, z; !! 409 long o, n, t, z;
563 s8 o; <<
564 410
565 o = this_cpu_read(*p); <<
566 do { 411 do {
567 z = 0; /* overflow to zone co 412 z = 0; /* overflow to zone counters */
568 413
569 /* 414 /*
570 * The fetching of the stat_th 415 * The fetching of the stat_threshold is racy. We may apply
571 * a counter threshold to the 416 * a counter threshold to the wrong the cpu if we get
572 * rescheduled while executing 417 * rescheduled while executing here. However, the next
573 * counter update will apply t 418 * counter update will apply the threshold again and
574 * therefore bring the counter 419 * therefore bring the counter under the threshold again.
575 * 420 *
576 * Most of the time the thresh 421 * Most of the time the thresholds are the same anyways
577 * for all cpus in a zone. 422 * for all cpus in a zone.
578 */ 423 */
579 t = this_cpu_read(pcp->stat_th 424 t = this_cpu_read(pcp->stat_threshold);
580 425
581 n = delta + (long)o; !! 426 o = this_cpu_read(*p);
>> 427 n = delta + o;
582 428
583 if (abs(n) > t) { !! 429 if (n > t || n < -t) {
584 int os = overstep_mode 430 int os = overstep_mode * (t >> 1) ;
585 431
586 /* Overflow must be ad 432 /* Overflow must be added to zone counters */
587 z = n + os; 433 z = n + os;
588 n = -os; 434 n = -os;
589 } 435 }
590 } while (!this_cpu_try_cmpxchg(*p, &o, !! 436 } while (this_cpu_cmpxchg(*p, o, n) != o);
591 437
592 if (z) 438 if (z)
593 zone_page_state_add(z, zone, i 439 zone_page_state_add(z, zone, item);
594 } 440 }
595 441
596 void mod_zone_page_state(struct zone *zone, en 442 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
597 long delta) 443 long delta)
598 { 444 {
599 mod_zone_state(zone, item, delta, 0); 445 mod_zone_state(zone, item, delta, 0);
600 } 446 }
601 EXPORT_SYMBOL(mod_zone_page_state); 447 EXPORT_SYMBOL(mod_zone_page_state);
602 448
603 void inc_zone_page_state(struct page *page, en 449 void inc_zone_page_state(struct page *page, enum zone_stat_item item)
604 { 450 {
605 mod_zone_state(page_zone(page), item, 451 mod_zone_state(page_zone(page), item, 1, 1);
606 } 452 }
607 EXPORT_SYMBOL(inc_zone_page_state); 453 EXPORT_SYMBOL(inc_zone_page_state);
608 454
609 void dec_zone_page_state(struct page *page, en 455 void dec_zone_page_state(struct page *page, enum zone_stat_item item)
610 { 456 {
611 mod_zone_state(page_zone(page), item, 457 mod_zone_state(page_zone(page), item, -1, -1);
612 } 458 }
613 EXPORT_SYMBOL(dec_zone_page_state); 459 EXPORT_SYMBOL(dec_zone_page_state);
614 460
615 static inline void mod_node_state(struct pglis 461 static inline void mod_node_state(struct pglist_data *pgdat,
616 enum node_stat_item item, int delta, in 462 enum node_stat_item item, int delta, int overstep_mode)
617 { 463 {
618 struct per_cpu_nodestat __percpu *pcp 464 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
619 s8 __percpu *p = pcp->vm_node_stat_dif 465 s8 __percpu *p = pcp->vm_node_stat_diff + item;
620 long n, t, z; !! 466 long o, n, t, z;
621 s8 o; <<
622 <<
623 if (vmstat_item_in_bytes(item)) { <<
624 /* <<
625 * Only cgroups use subpage ac <<
626 * the global level, these ite <<
627 * multiples of whole pages. S <<
628 * internally to keep the per- <<
629 */ <<
630 VM_WARN_ON_ONCE(delta & (PAGE_ <<
631 delta >>= PAGE_SHIFT; <<
632 } <<
633 467
634 o = this_cpu_read(*p); <<
635 do { 468 do {
636 z = 0; /* overflow to node co 469 z = 0; /* overflow to node counters */
637 470
638 /* 471 /*
639 * The fetching of the stat_th 472 * The fetching of the stat_threshold is racy. We may apply
640 * a counter threshold to the 473 * a counter threshold to the wrong the cpu if we get
641 * rescheduled while executing 474 * rescheduled while executing here. However, the next
642 * counter update will apply t 475 * counter update will apply the threshold again and
643 * therefore bring the counter 476 * therefore bring the counter under the threshold again.
644 * 477 *
645 * Most of the time the thresh 478 * Most of the time the thresholds are the same anyways
646 * for all cpus in a node. 479 * for all cpus in a node.
647 */ 480 */
648 t = this_cpu_read(pcp->stat_th 481 t = this_cpu_read(pcp->stat_threshold);
649 482
650 n = delta + (long)o; !! 483 o = this_cpu_read(*p);
>> 484 n = delta + o;
651 485
652 if (abs(n) > t) { !! 486 if (n > t || n < -t) {
653 int os = overstep_mode 487 int os = overstep_mode * (t >> 1) ;
654 488
655 /* Overflow must be ad 489 /* Overflow must be added to node counters */
656 z = n + os; 490 z = n + os;
657 n = -os; 491 n = -os;
658 } 492 }
659 } while (!this_cpu_try_cmpxchg(*p, &o, !! 493 } while (this_cpu_cmpxchg(*p, o, n) != o);
660 494
661 if (z) 495 if (z)
662 node_page_state_add(z, pgdat, 496 node_page_state_add(z, pgdat, item);
663 } 497 }
664 498
665 void mod_node_page_state(struct pglist_data *p 499 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
666 long d 500 long delta)
667 { 501 {
668 mod_node_state(pgdat, item, delta, 0); 502 mod_node_state(pgdat, item, delta, 0);
669 } 503 }
670 EXPORT_SYMBOL(mod_node_page_state); 504 EXPORT_SYMBOL(mod_node_page_state);
671 505
672 void inc_node_state(struct pglist_data *pgdat, 506 void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
673 { 507 {
674 mod_node_state(pgdat, item, 1, 1); 508 mod_node_state(pgdat, item, 1, 1);
675 } 509 }
676 510
677 void inc_node_page_state(struct page *page, en 511 void inc_node_page_state(struct page *page, enum node_stat_item item)
678 { 512 {
679 mod_node_state(page_pgdat(page), item, 513 mod_node_state(page_pgdat(page), item, 1, 1);
680 } 514 }
681 EXPORT_SYMBOL(inc_node_page_state); 515 EXPORT_SYMBOL(inc_node_page_state);
682 516
683 void dec_node_page_state(struct page *page, en 517 void dec_node_page_state(struct page *page, enum node_stat_item item)
684 { 518 {
685 mod_node_state(page_pgdat(page), item, 519 mod_node_state(page_pgdat(page), item, -1, -1);
686 } 520 }
687 EXPORT_SYMBOL(dec_node_page_state); 521 EXPORT_SYMBOL(dec_node_page_state);
688 #else 522 #else
689 /* 523 /*
690 * Use interrupt disable to serialize counter 524 * Use interrupt disable to serialize counter updates
691 */ 525 */
692 void mod_zone_page_state(struct zone *zone, en 526 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
693 long delta) 527 long delta)
694 { 528 {
695 unsigned long flags; 529 unsigned long flags;
696 530
697 local_irq_save(flags); 531 local_irq_save(flags);
698 __mod_zone_page_state(zone, item, delt 532 __mod_zone_page_state(zone, item, delta);
699 local_irq_restore(flags); 533 local_irq_restore(flags);
700 } 534 }
701 EXPORT_SYMBOL(mod_zone_page_state); 535 EXPORT_SYMBOL(mod_zone_page_state);
702 536
703 void inc_zone_page_state(struct page *page, en 537 void inc_zone_page_state(struct page *page, enum zone_stat_item item)
704 { 538 {
705 unsigned long flags; 539 unsigned long flags;
706 struct zone *zone; 540 struct zone *zone;
707 541
708 zone = page_zone(page); 542 zone = page_zone(page);
709 local_irq_save(flags); 543 local_irq_save(flags);
710 __inc_zone_state(zone, item); 544 __inc_zone_state(zone, item);
711 local_irq_restore(flags); 545 local_irq_restore(flags);
712 } 546 }
713 EXPORT_SYMBOL(inc_zone_page_state); 547 EXPORT_SYMBOL(inc_zone_page_state);
714 548
715 void dec_zone_page_state(struct page *page, en 549 void dec_zone_page_state(struct page *page, enum zone_stat_item item)
716 { 550 {
717 unsigned long flags; 551 unsigned long flags;
718 552
719 local_irq_save(flags); 553 local_irq_save(flags);
720 __dec_zone_page_state(page, item); 554 __dec_zone_page_state(page, item);
721 local_irq_restore(flags); 555 local_irq_restore(flags);
722 } 556 }
723 EXPORT_SYMBOL(dec_zone_page_state); 557 EXPORT_SYMBOL(dec_zone_page_state);
724 558
725 void inc_node_state(struct pglist_data *pgdat, 559 void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
726 { 560 {
727 unsigned long flags; 561 unsigned long flags;
728 562
729 local_irq_save(flags); 563 local_irq_save(flags);
730 __inc_node_state(pgdat, item); 564 __inc_node_state(pgdat, item);
731 local_irq_restore(flags); 565 local_irq_restore(flags);
732 } 566 }
733 EXPORT_SYMBOL(inc_node_state); 567 EXPORT_SYMBOL(inc_node_state);
734 568
735 void mod_node_page_state(struct pglist_data *p 569 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
736 long d 570 long delta)
737 { 571 {
738 unsigned long flags; 572 unsigned long flags;
739 573
740 local_irq_save(flags); 574 local_irq_save(flags);
741 __mod_node_page_state(pgdat, item, del 575 __mod_node_page_state(pgdat, item, delta);
742 local_irq_restore(flags); 576 local_irq_restore(flags);
743 } 577 }
744 EXPORT_SYMBOL(mod_node_page_state); 578 EXPORT_SYMBOL(mod_node_page_state);
745 579
746 void inc_node_page_state(struct page *page, en 580 void inc_node_page_state(struct page *page, enum node_stat_item item)
747 { 581 {
748 unsigned long flags; 582 unsigned long flags;
749 struct pglist_data *pgdat; 583 struct pglist_data *pgdat;
750 584
751 pgdat = page_pgdat(page); 585 pgdat = page_pgdat(page);
752 local_irq_save(flags); 586 local_irq_save(flags);
753 __inc_node_state(pgdat, item); 587 __inc_node_state(pgdat, item);
754 local_irq_restore(flags); 588 local_irq_restore(flags);
755 } 589 }
756 EXPORT_SYMBOL(inc_node_page_state); 590 EXPORT_SYMBOL(inc_node_page_state);
757 591
758 void dec_node_page_state(struct page *page, en 592 void dec_node_page_state(struct page *page, enum node_stat_item item)
759 { 593 {
760 unsigned long flags; 594 unsigned long flags;
761 595
762 local_irq_save(flags); 596 local_irq_save(flags);
763 __dec_node_page_state(page, item); 597 __dec_node_page_state(page, item);
764 local_irq_restore(flags); 598 local_irq_restore(flags);
765 } 599 }
766 EXPORT_SYMBOL(dec_node_page_state); 600 EXPORT_SYMBOL(dec_node_page_state);
767 #endif 601 #endif
768 602
769 /* 603 /*
770 * Fold a differential into the global counter 604 * Fold a differential into the global counters.
771 * Returns the number of counters updated. 605 * Returns the number of counters updated.
772 */ 606 */
773 static int fold_diff(int *zone_diff, int *node 607 static int fold_diff(int *zone_diff, int *node_diff)
774 { 608 {
775 int i; 609 int i;
776 int changes = 0; 610 int changes = 0;
777 611
778 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; 612 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
779 if (zone_diff[i]) { 613 if (zone_diff[i]) {
780 atomic_long_add(zone_d 614 atomic_long_add(zone_diff[i], &vm_zone_stat[i]);
781 changes++; 615 changes++;
782 } 616 }
783 617
784 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; 618 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
785 if (node_diff[i]) { 619 if (node_diff[i]) {
786 atomic_long_add(node_d 620 atomic_long_add(node_diff[i], &vm_node_stat[i]);
787 changes++; 621 changes++;
788 } 622 }
789 return changes; 623 return changes;
790 } 624 }
791 625
792 /* 626 /*
793 * Update the zone counters for the current cp 627 * Update the zone counters for the current cpu.
794 * 628 *
795 * Note that refresh_cpu_vm_stats strives to o 629 * Note that refresh_cpu_vm_stats strives to only access
796 * node local memory. The per cpu pagesets on 630 * node local memory. The per cpu pagesets on remote zones are placed
797 * in the memory local to the processor using 631 * in the memory local to the processor using that pageset. So the
798 * loop over all zones will access a series of 632 * loop over all zones will access a series of cachelines local to
799 * the processor. 633 * the processor.
800 * 634 *
801 * The call to zone_page_state_add updates the 635 * The call to zone_page_state_add updates the cachelines with the
802 * statistics in the remote zone struct as wel 636 * statistics in the remote zone struct as well as the global cachelines
803 * with the global counters. These could cause 637 * with the global counters. These could cause remote node cache line
804 * bouncing and will have to be only done when 638 * bouncing and will have to be only done when necessary.
805 * 639 *
806 * The function returns the number of global c 640 * The function returns the number of global counters updated.
807 */ 641 */
808 static int refresh_cpu_vm_stats(bool do_pagese 642 static int refresh_cpu_vm_stats(bool do_pagesets)
809 { 643 {
810 struct pglist_data *pgdat; 644 struct pglist_data *pgdat;
811 struct zone *zone; 645 struct zone *zone;
812 int i; 646 int i;
813 int global_zone_diff[NR_VM_ZONE_STAT_I 647 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, };
814 int global_node_diff[NR_VM_NODE_STAT_I 648 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, };
815 int changes = 0; 649 int changes = 0;
816 650
817 for_each_populated_zone(zone) { 651 for_each_populated_zone(zone) {
818 struct per_cpu_zonestat __perc !! 652 struct per_cpu_pageset __percpu *p = zone->pageset;
819 struct per_cpu_pages __percpu <<
820 653
821 for (i = 0; i < NR_VM_ZONE_STA 654 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
822 int v; 655 int v;
823 656
824 v = this_cpu_xchg(pzst !! 657 v = this_cpu_xchg(p->vm_stat_diff[i], 0);
825 if (v) { 658 if (v) {
826 659
827 atomic_long_ad 660 atomic_long_add(v, &zone->vm_stat[i]);
828 global_zone_di 661 global_zone_diff[i] += v;
829 #ifdef CONFIG_NUMA 662 #ifdef CONFIG_NUMA
830 /* 3 seconds i 663 /* 3 seconds idle till flush */
831 __this_cpu_wri !! 664 __this_cpu_write(p->expire, 3);
832 #endif 665 #endif
833 } 666 }
834 } 667 }
835 !! 668 #ifdef CONFIG_NUMA
836 if (do_pagesets) { 669 if (do_pagesets) {
837 cond_resched(); 670 cond_resched();
838 <<
839 changes += decay_pcp_h <<
840 #ifdef CONFIG_NUMA <<
841 /* 671 /*
842 * Deal with draining 672 * Deal with draining the remote pageset of this
843 * processor 673 * processor
844 * 674 *
845 * Check if there are 675 * Check if there are pages remaining in this pageset
846 * if not then there i 676 * if not then there is nothing to expire.
847 */ 677 */
848 if (!__this_cpu_read(p !! 678 if (!__this_cpu_read(p->expire) ||
849 !__this_cpu_rea !! 679 !__this_cpu_read(p->pcp.count))
850 continue; 680 continue;
851 681
852 /* 682 /*
853 * We never drain zone 683 * We never drain zones local to this processor.
854 */ 684 */
855 if (zone_to_nid(zone) 685 if (zone_to_nid(zone) == numa_node_id()) {
856 __this_cpu_wri !! 686 __this_cpu_write(p->expire, 0);
857 continue; 687 continue;
858 } 688 }
859 689
860 if (__this_cpu_dec_ret !! 690 if (__this_cpu_dec_return(p->expire))
861 changes++; <<
862 continue; 691 continue;
863 } <<
864 692
865 if (__this_cpu_read(pc !! 693 if (__this_cpu_read(p->pcp.count)) {
866 drain_zone_pag !! 694 drain_zone_pages(zone, this_cpu_ptr(&p->pcp));
867 changes++; 695 changes++;
868 } 696 }
869 #endif <<
870 } 697 }
>> 698 #endif
871 } 699 }
872 700
873 for_each_online_pgdat(pgdat) { 701 for_each_online_pgdat(pgdat) {
874 struct per_cpu_nodestat __perc 702 struct per_cpu_nodestat __percpu *p = pgdat->per_cpu_nodestats;
875 703
876 for (i = 0; i < NR_VM_NODE_STA 704 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
877 int v; 705 int v;
878 706
879 v = this_cpu_xchg(p->v 707 v = this_cpu_xchg(p->vm_node_stat_diff[i], 0);
880 if (v) { 708 if (v) {
881 atomic_long_ad 709 atomic_long_add(v, &pgdat->vm_stat[i]);
882 global_node_di 710 global_node_diff[i] += v;
883 } 711 }
884 } 712 }
885 } 713 }
886 714
887 changes += fold_diff(global_zone_diff, 715 changes += fold_diff(global_zone_diff, global_node_diff);
888 return changes; 716 return changes;
889 } 717 }
890 718
891 /* 719 /*
892 * Fold the data for an offline cpu into the g 720 * Fold the data for an offline cpu into the global array.
893 * There cannot be any access by the offline c 721 * There cannot be any access by the offline cpu and therefore
894 * synchronization is simplified. 722 * synchronization is simplified.
895 */ 723 */
896 void cpu_vm_stats_fold(int cpu) 724 void cpu_vm_stats_fold(int cpu)
897 { 725 {
898 struct pglist_data *pgdat; 726 struct pglist_data *pgdat;
899 struct zone *zone; 727 struct zone *zone;
900 int i; 728 int i;
901 int global_zone_diff[NR_VM_ZONE_STAT_I 729 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, };
902 int global_node_diff[NR_VM_NODE_STAT_I 730 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, };
903 731
904 for_each_populated_zone(zone) { 732 for_each_populated_zone(zone) {
905 struct per_cpu_zonestat *pzsta !! 733 struct per_cpu_pageset *p;
906 734
907 pzstats = per_cpu_ptr(zone->pe !! 735 p = per_cpu_ptr(zone->pageset, cpu);
908 736
909 for (i = 0; i < NR_VM_ZONE_STA !! 737 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
910 if (pzstats->vm_stat_d !! 738 if (p->vm_stat_diff[i]) {
911 int v; 739 int v;
912 740
913 v = pzstats->v !! 741 v = p->vm_stat_diff[i];
914 pzstats->vm_st !! 742 p->vm_stat_diff[i] = 0;
915 atomic_long_ad 743 atomic_long_add(v, &zone->vm_stat[i]);
916 global_zone_di 744 global_zone_diff[i] += v;
917 } 745 }
918 } <<
919 #ifdef CONFIG_NUMA <<
920 for (i = 0; i < NR_VM_NUMA_EVE <<
921 if (pzstats->vm_numa_e <<
922 unsigned long <<
923 <<
924 v = pzstats->v <<
925 pzstats->vm_nu <<
926 zone_numa_even <<
927 } <<
928 } <<
929 #endif <<
930 } 746 }
931 747
932 for_each_online_pgdat(pgdat) { 748 for_each_online_pgdat(pgdat) {
933 struct per_cpu_nodestat *p; 749 struct per_cpu_nodestat *p;
934 750
935 p = per_cpu_ptr(pgdat->per_cpu 751 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu);
936 752
937 for (i = 0; i < NR_VM_NODE_STA 753 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
938 if (p->vm_node_stat_di 754 if (p->vm_node_stat_diff[i]) {
939 int v; 755 int v;
940 756
941 v = p->vm_node 757 v = p->vm_node_stat_diff[i];
942 p->vm_node_sta 758 p->vm_node_stat_diff[i] = 0;
943 atomic_long_ad 759 atomic_long_add(v, &pgdat->vm_stat[i]);
944 global_node_di 760 global_node_diff[i] += v;
945 } 761 }
946 } 762 }
947 763
948 fold_diff(global_zone_diff, global_nod 764 fold_diff(global_zone_diff, global_node_diff);
949 } 765 }
950 766
951 /* 767 /*
952 * this is only called if !populated_zone(zone 768 * this is only called if !populated_zone(zone), which implies no other users of
953 * pset->vm_stat_diff[] exist. !! 769 * pset->vm_stat_diff[] exsist.
954 */ 770 */
955 void drain_zonestat(struct zone *zone, struct !! 771 void drain_zonestat(struct zone *zone, struct per_cpu_pageset *pset)
956 { 772 {
957 unsigned long v; <<
958 int i; 773 int i;
959 774
960 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; !! 775 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
961 if (pzstats->vm_stat_diff[i]) !! 776 if (pset->vm_stat_diff[i]) {
962 v = pzstats->vm_stat_d !! 777 int v = pset->vm_stat_diff[i];
963 pzstats->vm_stat_diff[ !! 778 pset->vm_stat_diff[i] = 0;
964 zone_page_state_add(v, !! 779 atomic_long_add(v, &zone->vm_stat[i]);
965 } !! 780 atomic_long_add(v, &vm_zone_stat[i]);
966 } <<
967 <<
968 #ifdef CONFIG_NUMA <<
969 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS <<
970 if (pzstats->vm_numa_event[i]) <<
971 v = pzstats->vm_numa_e <<
972 pzstats->vm_numa_event <<
973 zone_numa_event_add(v, <<
974 } 781 }
975 } <<
976 #endif <<
977 } 782 }
978 #endif 783 #endif
979 784
980 #ifdef CONFIG_NUMA 785 #ifdef CONFIG_NUMA
981 /* 786 /*
982 * Determine the per node value of a stat item 787 * Determine the per node value of a stat item. This function
983 * is called frequently in a NUMA machine, so 788 * is called frequently in a NUMA machine, so try to be as
984 * frugal as possible. 789 * frugal as possible.
985 */ 790 */
986 unsigned long sum_zone_node_page_state(int nod 791 unsigned long sum_zone_node_page_state(int node,
987 enum zone_sta 792 enum zone_stat_item item)
988 { 793 {
989 struct zone *zones = NODE_DATA(node)-> 794 struct zone *zones = NODE_DATA(node)->node_zones;
990 int i; 795 int i;
991 unsigned long count = 0; 796 unsigned long count = 0;
992 797
993 for (i = 0; i < MAX_NR_ZONES; i++) 798 for (i = 0; i < MAX_NR_ZONES; i++)
994 count += zone_page_state(zones 799 count += zone_page_state(zones + i, item);
995 800
996 return count; 801 return count;
997 } 802 }
998 803
999 /* Determine the per node value of a numa stat <<
1000 unsigned long sum_zone_numa_event_state(int n <<
1001 enum numa_st <<
1002 { <<
1003 struct zone *zones = NODE_DATA(node)- <<
1004 unsigned long count = 0; <<
1005 int i; <<
1006 <<
1007 for (i = 0; i < MAX_NR_ZONES; i++) <<
1008 count += zone_numa_event_stat <<
1009 <<
1010 return count; <<
1011 } <<
1012 <<
1013 /* 804 /*
1014 * Determine the per node value of a stat ite 805 * Determine the per node value of a stat item.
1015 */ 806 */
1016 unsigned long node_page_state_pages(struct pg !! 807 unsigned long node_page_state(struct pglist_data *pgdat,
1017 enum node !! 808 enum node_stat_item item)
1018 { 809 {
1019 long x = atomic_long_read(&pgdat->vm_ 810 long x = atomic_long_read(&pgdat->vm_stat[item]);
1020 #ifdef CONFIG_SMP 811 #ifdef CONFIG_SMP
1021 if (x < 0) 812 if (x < 0)
1022 x = 0; 813 x = 0;
1023 #endif 814 #endif
1024 return x; 815 return x;
1025 } 816 }
1026 <<
1027 unsigned long node_page_state(struct pglist_d <<
1028 enum node_stat_ <<
1029 { <<
1030 VM_WARN_ON_ONCE(vmstat_item_in_bytes( <<
1031 <<
1032 return node_page_state_pages(pgdat, i <<
1033 } <<
1034 #endif 817 #endif
1035 818
1036 /* <<
1037 * Count number of pages "struct page" and "s <<
1038 * nr_memmap_boot_pages: # of pages allocated <<
1039 * nr_memmap_pages: # of pages that were allo <<
1040 */ <<
1041 static atomic_long_t nr_memmap_boot_pages = A <<
1042 static atomic_long_t nr_memmap_pages = ATOMIC <<
1043 <<
1044 void memmap_boot_pages_add(long delta) <<
1045 { <<
1046 atomic_long_add(delta, &nr_memmap_boo <<
1047 } <<
1048 <<
1049 void memmap_pages_add(long delta) <<
1050 { <<
1051 atomic_long_add(delta, &nr_memmap_pag <<
1052 } <<
1053 <<
1054 #ifdef CONFIG_COMPACTION 819 #ifdef CONFIG_COMPACTION
1055 820
1056 struct contig_page_info { 821 struct contig_page_info {
1057 unsigned long free_pages; 822 unsigned long free_pages;
1058 unsigned long free_blocks_total; 823 unsigned long free_blocks_total;
1059 unsigned long free_blocks_suitable; 824 unsigned long free_blocks_suitable;
1060 }; 825 };
1061 826
1062 /* 827 /*
1063 * Calculate the number of free pages in a zo 828 * Calculate the number of free pages in a zone, how many contiguous
1064 * pages are free and how many are large enou 829 * pages are free and how many are large enough to satisfy an allocation of
1065 * the target size. Note that this function m 830 * the target size. Note that this function makes no attempt to estimate
1066 * how many suitable free blocks there *might 831 * how many suitable free blocks there *might* be if MOVABLE pages were
1067 * migrated. Calculating that is possible, bu 832 * migrated. Calculating that is possible, but expensive and can be
1068 * figured out from userspace 833 * figured out from userspace
1069 */ 834 */
1070 static void fill_contig_page_info(struct zone 835 static void fill_contig_page_info(struct zone *zone,
1071 unsigned int 836 unsigned int suitable_order,
1072 struct contig 837 struct contig_page_info *info)
1073 { 838 {
1074 unsigned int order; 839 unsigned int order;
1075 840
1076 info->free_pages = 0; 841 info->free_pages = 0;
1077 info->free_blocks_total = 0; 842 info->free_blocks_total = 0;
1078 info->free_blocks_suitable = 0; 843 info->free_blocks_suitable = 0;
1079 844
1080 for (order = 0; order < NR_PAGE_ORDER !! 845 for (order = 0; order < MAX_ORDER; order++) {
1081 unsigned long blocks; 846 unsigned long blocks;
1082 847
1083 /* !! 848 /* Count number of free blocks */
1084 * Count number of free block !! 849 blocks = zone->free_area[order].nr_free;
1085 * <<
1086 * Access to nr_free is lockl <<
1087 * diagnostic purposes. Use d <<
1088 */ <<
1089 blocks = data_race(zone->free <<
1090 info->free_blocks_total += bl 850 info->free_blocks_total += blocks;
1091 851
1092 /* Count free base pages */ 852 /* Count free base pages */
1093 info->free_pages += blocks << 853 info->free_pages += blocks << order;
1094 854
1095 /* Count the suitable free bl 855 /* Count the suitable free blocks */
1096 if (order >= suitable_order) 856 if (order >= suitable_order)
1097 info->free_blocks_sui 857 info->free_blocks_suitable += blocks <<
1098 858 (order - suitable_order);
1099 } 859 }
1100 } 860 }
1101 861
1102 /* 862 /*
1103 * A fragmentation index only makes sense if 863 * A fragmentation index only makes sense if an allocation of a requested
1104 * size would fail. If that is true, the frag 864 * size would fail. If that is true, the fragmentation index indicates
1105 * whether external fragmentation or a lack o 865 * whether external fragmentation or a lack of memory was the problem.
1106 * The value can be used to determine if page 866 * The value can be used to determine if page reclaim or compaction
1107 * should be used 867 * should be used
1108 */ 868 */
1109 static int __fragmentation_index(unsigned int 869 static int __fragmentation_index(unsigned int order, struct contig_page_info *info)
1110 { 870 {
1111 unsigned long requested = 1UL << orde 871 unsigned long requested = 1UL << order;
1112 872
1113 if (WARN_ON_ONCE(order > MAX_PAGE_ORD <<
1114 return 0; <<
1115 <<
1116 if (!info->free_blocks_total) 873 if (!info->free_blocks_total)
1117 return 0; 874 return 0;
1118 875
1119 /* Fragmentation index only makes sen 876 /* Fragmentation index only makes sense when a request would fail */
1120 if (info->free_blocks_suitable) 877 if (info->free_blocks_suitable)
1121 return -1000; 878 return -1000;
1122 879
1123 /* 880 /*
1124 * Index is between 0 and 1 so return 881 * Index is between 0 and 1 so return within 3 decimal places
1125 * 882 *
1126 * 0 => allocation would fail due to 883 * 0 => allocation would fail due to lack of memory
1127 * 1 => allocation would fail due to 884 * 1 => allocation would fail due to fragmentation
1128 */ 885 */
1129 return 1000 - div_u64( (1000+(div_u64 886 return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total);
1130 } 887 }
1131 888
1132 /* <<
1133 * Calculates external fragmentation within a <<
1134 * It is defined as the percentage of pages f <<
1135 * less than 1 << order. It returns values in <<
1136 */ <<
1137 unsigned int extfrag_for_order(struct zone *z <<
1138 { <<
1139 struct contig_page_info info; <<
1140 <<
1141 fill_contig_page_info(zone, order, &i <<
1142 if (info.free_pages == 0) <<
1143 return 0; <<
1144 <<
1145 return div_u64((info.free_pages - <<
1146 (info.free_blocks_sui <<
1147 info.free_pages); <<
1148 } <<
1149 <<
1150 /* Same as __fragmentation index but allocs c 889 /* Same as __fragmentation index but allocs contig_page_info on stack */
1151 int fragmentation_index(struct zone *zone, un 890 int fragmentation_index(struct zone *zone, unsigned int order)
1152 { 891 {
1153 struct contig_page_info info; 892 struct contig_page_info info;
1154 893
1155 fill_contig_page_info(zone, order, &i 894 fill_contig_page_info(zone, order, &info);
1156 return __fragmentation_index(order, & 895 return __fragmentation_index(order, &info);
1157 } 896 }
1158 #endif 897 #endif
1159 898
1160 #if defined(CONFIG_PROC_FS) || defined(CONFIG !! 899 #if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || defined(CONFIG_NUMA)
1161 defined(CONFIG_NUMA) || defined(CONFIG_ME <<
1162 #ifdef CONFIG_ZONE_DMA 900 #ifdef CONFIG_ZONE_DMA
1163 #define TEXT_FOR_DMA(xx) xx "_dma", 901 #define TEXT_FOR_DMA(xx) xx "_dma",
1164 #else 902 #else
1165 #define TEXT_FOR_DMA(xx) 903 #define TEXT_FOR_DMA(xx)
1166 #endif 904 #endif
1167 905
1168 #ifdef CONFIG_ZONE_DMA32 906 #ifdef CONFIG_ZONE_DMA32
1169 #define TEXT_FOR_DMA32(xx) xx "_dma32", 907 #define TEXT_FOR_DMA32(xx) xx "_dma32",
1170 #else 908 #else
1171 #define TEXT_FOR_DMA32(xx) 909 #define TEXT_FOR_DMA32(xx)
1172 #endif 910 #endif
1173 911
1174 #ifdef CONFIG_HIGHMEM 912 #ifdef CONFIG_HIGHMEM
1175 #define TEXT_FOR_HIGHMEM(xx) xx "_high", 913 #define TEXT_FOR_HIGHMEM(xx) xx "_high",
1176 #else 914 #else
1177 #define TEXT_FOR_HIGHMEM(xx) 915 #define TEXT_FOR_HIGHMEM(xx)
1178 #endif 916 #endif
1179 917
1180 #ifdef CONFIG_ZONE_DEVICE <<
1181 #define TEXT_FOR_DEVICE(xx) xx "_device", <<
1182 #else <<
1183 #define TEXT_FOR_DEVICE(xx) <<
1184 #endif <<
1185 <<
1186 #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) 918 #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) TEXT_FOR_DMA32(xx) xx "_normal", \
1187 TEXT_ !! 919 TEXT_FOR_HIGHMEM(xx) xx "_movable",
1188 TEXT_ <<
1189 920
1190 const char * const vmstat_text[] = { 921 const char * const vmstat_text[] = {
1191 /* enum zone_stat_item counters */ !! 922 /* enum zone_stat_item countes */
1192 "nr_free_pages", 923 "nr_free_pages",
1193 "nr_zone_inactive_anon", 924 "nr_zone_inactive_anon",
1194 "nr_zone_active_anon", 925 "nr_zone_active_anon",
1195 "nr_zone_inactive_file", 926 "nr_zone_inactive_file",
1196 "nr_zone_active_file", 927 "nr_zone_active_file",
1197 "nr_zone_unevictable", 928 "nr_zone_unevictable",
1198 "nr_zone_write_pending", 929 "nr_zone_write_pending",
1199 "nr_mlock", 930 "nr_mlock",
>> 931 "nr_slab_reclaimable",
>> 932 "nr_slab_unreclaimable",
>> 933 "nr_page_table_pages",
>> 934 "nr_kernel_stack",
1200 "nr_bounce", 935 "nr_bounce",
1201 #if IS_ENABLED(CONFIG_ZSMALLOC) 936 #if IS_ENABLED(CONFIG_ZSMALLOC)
1202 "nr_zspages", 937 "nr_zspages",
1203 #endif 938 #endif
1204 "nr_free_cma", <<
1205 #ifdef CONFIG_UNACCEPTED_MEMORY <<
1206 "nr_unaccepted", <<
1207 #endif <<
1208 <<
1209 /* enum numa_stat_item counters */ <<
1210 #ifdef CONFIG_NUMA 939 #ifdef CONFIG_NUMA
1211 "numa_hit", 940 "numa_hit",
1212 "numa_miss", 941 "numa_miss",
1213 "numa_foreign", 942 "numa_foreign",
1214 "numa_interleave", 943 "numa_interleave",
1215 "numa_local", 944 "numa_local",
1216 "numa_other", 945 "numa_other",
1217 #endif 946 #endif
>> 947 "nr_free_cma",
1218 948
1219 /* enum node_stat_item counters */ !! 949 /* Node-based counters */
1220 "nr_inactive_anon", 950 "nr_inactive_anon",
1221 "nr_active_anon", 951 "nr_active_anon",
1222 "nr_inactive_file", 952 "nr_inactive_file",
1223 "nr_active_file", 953 "nr_active_file",
1224 "nr_unevictable", 954 "nr_unevictable",
1225 "nr_slab_reclaimable", <<
1226 "nr_slab_unreclaimable", <<
1227 "nr_isolated_anon", 955 "nr_isolated_anon",
1228 "nr_isolated_file", 956 "nr_isolated_file",
1229 "workingset_nodes", !! 957 "nr_pages_scanned",
1230 "workingset_refault_anon", !! 958 "workingset_refault",
1231 "workingset_refault_file", !! 959 "workingset_activate",
1232 "workingset_activate_anon", <<
1233 "workingset_activate_file", <<
1234 "workingset_restore_anon", <<
1235 "workingset_restore_file", <<
1236 "workingset_nodereclaim", 960 "workingset_nodereclaim",
1237 "nr_anon_pages", 961 "nr_anon_pages",
1238 "nr_mapped", 962 "nr_mapped",
1239 "nr_file_pages", 963 "nr_file_pages",
1240 "nr_dirty", 964 "nr_dirty",
1241 "nr_writeback", 965 "nr_writeback",
1242 "nr_writeback_temp", 966 "nr_writeback_temp",
1243 "nr_shmem", 967 "nr_shmem",
1244 "nr_shmem_hugepages", 968 "nr_shmem_hugepages",
1245 "nr_shmem_pmdmapped", 969 "nr_shmem_pmdmapped",
1246 "nr_file_hugepages", <<
1247 "nr_file_pmdmapped", <<
1248 "nr_anon_transparent_hugepages", 970 "nr_anon_transparent_hugepages",
>> 971 "nr_unstable",
1249 "nr_vmscan_write", 972 "nr_vmscan_write",
1250 "nr_vmscan_immediate_reclaim", 973 "nr_vmscan_immediate_reclaim",
1251 "nr_dirtied", 974 "nr_dirtied",
1252 "nr_written", 975 "nr_written",
1253 "nr_throttled_written", !! 976
1254 "nr_kernel_misc_reclaimable", !! 977 /* enum writeback_stat_item counters */
1255 "nr_foll_pin_acquired", <<
1256 "nr_foll_pin_released", <<
1257 "nr_kernel_stack", <<
1258 #if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) <<
1259 "nr_shadow_call_stack", <<
1260 #endif <<
1261 "nr_page_table_pages", <<
1262 "nr_sec_page_table_pages", <<
1263 #ifdef CONFIG_IOMMU_SUPPORT <<
1264 "nr_iommu_pages", <<
1265 #endif <<
1266 #ifdef CONFIG_SWAP <<
1267 "nr_swapcached", <<
1268 #endif <<
1269 #ifdef CONFIG_NUMA_BALANCING <<
1270 "pgpromote_success", <<
1271 "pgpromote_candidate", <<
1272 #endif <<
1273 "pgdemote_kswapd", <<
1274 "pgdemote_direct", <<
1275 "pgdemote_khugepaged", <<
1276 /* system-wide enum vm_stat_item coun <<
1277 "nr_dirty_threshold", 978 "nr_dirty_threshold",
1278 "nr_dirty_background_threshold", 979 "nr_dirty_background_threshold",
1279 "nr_memmap_pages", <<
1280 "nr_memmap_boot_pages", <<
1281 980
1282 #if defined(CONFIG_VM_EVENT_COUNTERS) || defi !! 981 #ifdef CONFIG_VM_EVENT_COUNTERS
1283 /* enum vm_event_item counters */ 982 /* enum vm_event_item counters */
1284 "pgpgin", 983 "pgpgin",
1285 "pgpgout", 984 "pgpgout",
1286 "pswpin", 985 "pswpin",
1287 "pswpout", 986 "pswpout",
1288 987
1289 TEXTS_FOR_ZONES("pgalloc") 988 TEXTS_FOR_ZONES("pgalloc")
1290 TEXTS_FOR_ZONES("allocstall") 989 TEXTS_FOR_ZONES("allocstall")
1291 TEXTS_FOR_ZONES("pgskip") 990 TEXTS_FOR_ZONES("pgskip")
1292 991
1293 "pgfree", 992 "pgfree",
1294 "pgactivate", 993 "pgactivate",
1295 "pgdeactivate", 994 "pgdeactivate",
1296 "pglazyfree", <<
1297 995
1298 "pgfault", 996 "pgfault",
1299 "pgmajfault", 997 "pgmajfault",
1300 "pglazyfreed", 998 "pglazyfreed",
1301 999
1302 "pgrefill", 1000 "pgrefill",
1303 "pgreuse", <<
1304 "pgsteal_kswapd", 1001 "pgsteal_kswapd",
1305 "pgsteal_direct", 1002 "pgsteal_direct",
1306 "pgsteal_khugepaged", <<
1307 "pgscan_kswapd", 1003 "pgscan_kswapd",
1308 "pgscan_direct", 1004 "pgscan_direct",
1309 "pgscan_khugepaged", <<
1310 "pgscan_direct_throttle", 1005 "pgscan_direct_throttle",
1311 "pgscan_anon", <<
1312 "pgscan_file", <<
1313 "pgsteal_anon", <<
1314 "pgsteal_file", <<
1315 1006
1316 #ifdef CONFIG_NUMA 1007 #ifdef CONFIG_NUMA
1317 "zone_reclaim_success", <<
1318 "zone_reclaim_failed", 1008 "zone_reclaim_failed",
1319 #endif 1009 #endif
1320 "pginodesteal", 1010 "pginodesteal",
1321 "slabs_scanned", 1011 "slabs_scanned",
1322 "kswapd_inodesteal", 1012 "kswapd_inodesteal",
1323 "kswapd_low_wmark_hit_quickly", 1013 "kswapd_low_wmark_hit_quickly",
1324 "kswapd_high_wmark_hit_quickly", 1014 "kswapd_high_wmark_hit_quickly",
1325 "pageoutrun", 1015 "pageoutrun",
1326 1016
1327 "pgrotated", 1017 "pgrotated",
1328 1018
1329 "drop_pagecache", 1019 "drop_pagecache",
1330 "drop_slab", 1020 "drop_slab",
1331 "oom_kill", <<
1332 1021
1333 #ifdef CONFIG_NUMA_BALANCING 1022 #ifdef CONFIG_NUMA_BALANCING
1334 "numa_pte_updates", 1023 "numa_pte_updates",
1335 "numa_huge_pte_updates", 1024 "numa_huge_pte_updates",
1336 "numa_hint_faults", 1025 "numa_hint_faults",
1337 "numa_hint_faults_local", 1026 "numa_hint_faults_local",
1338 "numa_pages_migrated", 1027 "numa_pages_migrated",
1339 #endif 1028 #endif
1340 #ifdef CONFIG_MIGRATION 1029 #ifdef CONFIG_MIGRATION
1341 "pgmigrate_success", 1030 "pgmigrate_success",
1342 "pgmigrate_fail", 1031 "pgmigrate_fail",
1343 "thp_migration_success", <<
1344 "thp_migration_fail", <<
1345 "thp_migration_split", <<
1346 #endif 1032 #endif
1347 #ifdef CONFIG_COMPACTION 1033 #ifdef CONFIG_COMPACTION
1348 "compact_migrate_scanned", 1034 "compact_migrate_scanned",
1349 "compact_free_scanned", 1035 "compact_free_scanned",
1350 "compact_isolated", 1036 "compact_isolated",
1351 "compact_stall", 1037 "compact_stall",
1352 "compact_fail", 1038 "compact_fail",
1353 "compact_success", 1039 "compact_success",
1354 "compact_daemon_wake", 1040 "compact_daemon_wake",
1355 "compact_daemon_migrate_scanned", <<
1356 "compact_daemon_free_scanned", <<
1357 #endif 1041 #endif
1358 1042
1359 #ifdef CONFIG_HUGETLB_PAGE 1043 #ifdef CONFIG_HUGETLB_PAGE
1360 "htlb_buddy_alloc_success", 1044 "htlb_buddy_alloc_success",
1361 "htlb_buddy_alloc_fail", 1045 "htlb_buddy_alloc_fail",
1362 #endif 1046 #endif
1363 #ifdef CONFIG_CMA <<
1364 "cma_alloc_success", <<
1365 "cma_alloc_fail", <<
1366 #endif <<
1367 "unevictable_pgs_culled", 1047 "unevictable_pgs_culled",
1368 "unevictable_pgs_scanned", 1048 "unevictable_pgs_scanned",
1369 "unevictable_pgs_rescued", 1049 "unevictable_pgs_rescued",
1370 "unevictable_pgs_mlocked", 1050 "unevictable_pgs_mlocked",
1371 "unevictable_pgs_munlocked", 1051 "unevictable_pgs_munlocked",
1372 "unevictable_pgs_cleared", 1052 "unevictable_pgs_cleared",
1373 "unevictable_pgs_stranded", 1053 "unevictable_pgs_stranded",
1374 1054
1375 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1055 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1376 "thp_fault_alloc", 1056 "thp_fault_alloc",
1377 "thp_fault_fallback", 1057 "thp_fault_fallback",
1378 "thp_fault_fallback_charge", <<
1379 "thp_collapse_alloc", 1058 "thp_collapse_alloc",
1380 "thp_collapse_alloc_failed", 1059 "thp_collapse_alloc_failed",
1381 "thp_file_alloc", 1060 "thp_file_alloc",
1382 "thp_file_fallback", <<
1383 "thp_file_fallback_charge", <<
1384 "thp_file_mapped", 1061 "thp_file_mapped",
1385 "thp_split_page", 1062 "thp_split_page",
1386 "thp_split_page_failed", 1063 "thp_split_page_failed",
1387 "thp_deferred_split_page", 1064 "thp_deferred_split_page",
1388 "thp_underused_split_page", <<
1389 "thp_split_pmd", 1065 "thp_split_pmd",
1390 "thp_scan_exceed_none_pte", <<
1391 "thp_scan_exceed_swap_pte", <<
1392 "thp_scan_exceed_share_pte", <<
1393 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_ <<
1394 "thp_split_pud", <<
1395 #endif <<
1396 "thp_zero_page_alloc", 1066 "thp_zero_page_alloc",
1397 "thp_zero_page_alloc_failed", 1067 "thp_zero_page_alloc_failed",
1398 "thp_swpout", <<
1399 "thp_swpout_fallback", <<
1400 #endif 1068 #endif
1401 #ifdef CONFIG_MEMORY_BALLOON 1069 #ifdef CONFIG_MEMORY_BALLOON
1402 "balloon_inflate", 1070 "balloon_inflate",
1403 "balloon_deflate", 1071 "balloon_deflate",
1404 #ifdef CONFIG_BALLOON_COMPACTION 1072 #ifdef CONFIG_BALLOON_COMPACTION
1405 "balloon_migrate", 1073 "balloon_migrate",
1406 #endif 1074 #endif
1407 #endif /* CONFIG_MEMORY_BALLOON */ 1075 #endif /* CONFIG_MEMORY_BALLOON */
1408 #ifdef CONFIG_DEBUG_TLBFLUSH 1076 #ifdef CONFIG_DEBUG_TLBFLUSH
>> 1077 #ifdef CONFIG_SMP
1409 "nr_tlb_remote_flush", 1078 "nr_tlb_remote_flush",
1410 "nr_tlb_remote_flush_received", 1079 "nr_tlb_remote_flush_received",
>> 1080 #endif /* CONFIG_SMP */
1411 "nr_tlb_local_flush_all", 1081 "nr_tlb_local_flush_all",
1412 "nr_tlb_local_flush_one", 1082 "nr_tlb_local_flush_one",
1413 #endif /* CONFIG_DEBUG_TLBFLUSH */ 1083 #endif /* CONFIG_DEBUG_TLBFLUSH */
1414 1084
1415 #ifdef CONFIG_SWAP !! 1085 #ifdef CONFIG_DEBUG_VM_VMACACHE
1416 "swap_ra", !! 1086 "vmacache_find_calls",
1417 "swap_ra_hit", !! 1087 "vmacache_find_hits",
1418 #ifdef CONFIG_KSM !! 1088 "vmacache_full_flushes",
1419 "ksm_swpin_copy", <<
1420 #endif <<
1421 #endif <<
1422 #ifdef CONFIG_KSM <<
1423 "cow_ksm", <<
1424 #endif <<
1425 #ifdef CONFIG_ZSWAP <<
1426 "zswpin", <<
1427 "zswpout", <<
1428 "zswpwb", <<
1429 #endif <<
1430 #ifdef CONFIG_X86 <<
1431 "direct_map_level2_splits", <<
1432 "direct_map_level3_splits", <<
1433 #endif 1089 #endif
1434 #ifdef CONFIG_PER_VMA_LOCK_STATS !! 1090 #endif /* CONFIG_VM_EVENTS_COUNTERS */
1435 "vma_lock_success", <<
1436 "vma_lock_abort", <<
1437 "vma_lock_retry", <<
1438 "vma_lock_miss", <<
1439 #endif <<
1440 #ifdef CONFIG_DEBUG_STACK_USAGE <<
1441 "kstack_1k", <<
1442 #if THREAD_SIZE > 1024 <<
1443 "kstack_2k", <<
1444 #endif <<
1445 #if THREAD_SIZE > 2048 <<
1446 "kstack_4k", <<
1447 #endif <<
1448 #if THREAD_SIZE > 4096 <<
1449 "kstack_8k", <<
1450 #endif <<
1451 #if THREAD_SIZE > 8192 <<
1452 "kstack_16k", <<
1453 #endif <<
1454 #if THREAD_SIZE > 16384 <<
1455 "kstack_32k", <<
1456 #endif <<
1457 #if THREAD_SIZE > 32768 <<
1458 "kstack_64k", <<
1459 #endif <<
1460 #if THREAD_SIZE > 65536 <<
1461 "kstack_rest", <<
1462 #endif <<
1463 #endif <<
1464 #endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_ <<
1465 }; 1091 };
1466 #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || C !! 1092 #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA */
>> 1093
1467 1094
1468 #if (defined(CONFIG_DEBUG_FS) && defined(CONF 1095 #if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \
1469 defined(CONFIG_PROC_FS) 1096 defined(CONFIG_PROC_FS)
1470 static void *frag_start(struct seq_file *m, l 1097 static void *frag_start(struct seq_file *m, loff_t *pos)
1471 { 1098 {
1472 pg_data_t *pgdat; 1099 pg_data_t *pgdat;
1473 loff_t node = *pos; 1100 loff_t node = *pos;
1474 1101
1475 for (pgdat = first_online_pgdat(); 1102 for (pgdat = first_online_pgdat();
1476 pgdat && node; 1103 pgdat && node;
1477 pgdat = next_online_pgdat(pgdat) 1104 pgdat = next_online_pgdat(pgdat))
1478 --node; 1105 --node;
1479 1106
1480 return pgdat; 1107 return pgdat;
1481 } 1108 }
1482 1109
1483 static void *frag_next(struct seq_file *m, vo 1110 static void *frag_next(struct seq_file *m, void *arg, loff_t *pos)
1484 { 1111 {
1485 pg_data_t *pgdat = (pg_data_t *)arg; 1112 pg_data_t *pgdat = (pg_data_t *)arg;
1486 1113
1487 (*pos)++; 1114 (*pos)++;
1488 return next_online_pgdat(pgdat); 1115 return next_online_pgdat(pgdat);
1489 } 1116 }
1490 1117
1491 static void frag_stop(struct seq_file *m, voi 1118 static void frag_stop(struct seq_file *m, void *arg)
1492 { 1119 {
1493 } 1120 }
1494 1121
1495 /* !! 1122 /* Walk all the zones in a node and print using a callback */
1496 * Walk zones in a node and print using a cal <<
1497 * If @assert_populated is true, only use cal <<
1498 */ <<
1499 static void walk_zones_in_node(struct seq_fil 1123 static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat,
1500 bool assert_populated, bool n <<
1501 void (*print)(struct seq_file 1124 void (*print)(struct seq_file *m, pg_data_t *, struct zone *))
1502 { 1125 {
1503 struct zone *zone; 1126 struct zone *zone;
1504 struct zone *node_zones = pgdat->node 1127 struct zone *node_zones = pgdat->node_zones;
1505 unsigned long flags; 1128 unsigned long flags;
1506 1129
1507 for (zone = node_zones; zone - node_z 1130 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
1508 if (assert_populated && !popu !! 1131 if (!populated_zone(zone))
1509 continue; 1132 continue;
1510 1133
1511 if (!nolock) !! 1134 spin_lock_irqsave(&zone->lock, flags);
1512 spin_lock_irqsave(&zo <<
1513 print(m, pgdat, zone); 1135 print(m, pgdat, zone);
1514 if (!nolock) !! 1136 spin_unlock_irqrestore(&zone->lock, flags);
1515 spin_unlock_irqrestor <<
1516 } 1137 }
1517 } 1138 }
1518 #endif 1139 #endif
1519 1140
1520 #ifdef CONFIG_PROC_FS 1141 #ifdef CONFIG_PROC_FS
1521 static void frag_show_print(struct seq_file * 1142 static void frag_show_print(struct seq_file *m, pg_data_t *pgdat,
1522 1143 struct zone *zone)
1523 { 1144 {
1524 int order; 1145 int order;
1525 1146
1526 seq_printf(m, "Node %d, zone %8s ", p 1147 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
1527 for (order = 0; order < NR_PAGE_ORDER !! 1148 for (order = 0; order < MAX_ORDER; ++order)
1528 /* !! 1149 seq_printf(m, "%6lu ", zone->free_area[order].nr_free);
1529 * Access to nr_free is lockl <<
1530 * printing purposes. Use dat <<
1531 */ <<
1532 seq_printf(m, "%6lu ", data_r <<
1533 seq_putc(m, '\n'); 1150 seq_putc(m, '\n');
1534 } 1151 }
1535 1152
1536 /* 1153 /*
1537 * This walks the free areas for each zone. 1154 * This walks the free areas for each zone.
1538 */ 1155 */
1539 static int frag_show(struct seq_file *m, void 1156 static int frag_show(struct seq_file *m, void *arg)
1540 { 1157 {
1541 pg_data_t *pgdat = (pg_data_t *)arg; 1158 pg_data_t *pgdat = (pg_data_t *)arg;
1542 walk_zones_in_node(m, pgdat, true, fa !! 1159 walk_zones_in_node(m, pgdat, frag_show_print);
1543 return 0; 1160 return 0;
1544 } 1161 }
1545 1162
1546 static void pagetypeinfo_showfree_print(struc 1163 static void pagetypeinfo_showfree_print(struct seq_file *m,
1547 pg_da 1164 pg_data_t *pgdat, struct zone *zone)
1548 { 1165 {
1549 int order, mtype; 1166 int order, mtype;
1550 1167
1551 for (mtype = 0; mtype < MIGRATE_TYPES 1168 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) {
1552 seq_printf(m, "Node %4d, zone 1169 seq_printf(m, "Node %4d, zone %8s, type %12s ",
1553 pgdat 1170 pgdat->node_id,
1554 zone- 1171 zone->name,
1555 migra 1172 migratetype_names[mtype]);
1556 for (order = 0; order < NR_PA !! 1173 for (order = 0; order < MAX_ORDER; ++order) {
1557 unsigned long freecou 1174 unsigned long freecount = 0;
1558 struct free_area *are 1175 struct free_area *area;
1559 struct list_head *cur 1176 struct list_head *curr;
1560 bool overflow = false <<
1561 1177
1562 area = &(zone->free_a 1178 area = &(zone->free_area[order]);
1563 1179
1564 list_for_each(curr, & !! 1180 list_for_each(curr, &area->free_list[mtype])
1565 /* !! 1181 freecount++;
1566 * Cap the fr !! 1182 seq_printf(m, "%6lu ", freecount);
1567 * be really <<
1568 * so a long <<
1569 * hard locku <<
1570 * debugging <<
1571 * of pages o <<
1572 * sufficient <<
1573 */ <<
1574 if (++freecou <<
1575 overf <<
1576 break <<
1577 } <<
1578 } <<
1579 seq_printf(m, "%s%6lu <<
1580 spin_unlock_irq(&zone <<
1581 cond_resched(); <<
1582 spin_lock_irq(&zone-> <<
1583 } 1183 }
1584 seq_putc(m, '\n'); 1184 seq_putc(m, '\n');
1585 } 1185 }
1586 } 1186 }
1587 1187
1588 /* Print out the free pages at each order for 1188 /* Print out the free pages at each order for each migatetype */
1589 static void pagetypeinfo_showfree(struct seq_ !! 1189 static int pagetypeinfo_showfree(struct seq_file *m, void *arg)
1590 { 1190 {
1591 int order; 1191 int order;
1592 pg_data_t *pgdat = (pg_data_t *)arg; 1192 pg_data_t *pgdat = (pg_data_t *)arg;
1593 1193
1594 /* Print header */ 1194 /* Print header */
1595 seq_printf(m, "%-43s ", "Free pages c 1195 seq_printf(m, "%-43s ", "Free pages count per migrate type at order");
1596 for (order = 0; order < NR_PAGE_ORDER !! 1196 for (order = 0; order < MAX_ORDER; ++order)
1597 seq_printf(m, "%6d ", order); 1197 seq_printf(m, "%6d ", order);
1598 seq_putc(m, '\n'); 1198 seq_putc(m, '\n');
1599 1199
1600 walk_zones_in_node(m, pgdat, true, fa !! 1200 walk_zones_in_node(m, pgdat, pagetypeinfo_showfree_print);
>> 1201
>> 1202 return 0;
1601 } 1203 }
1602 1204
1603 static void pagetypeinfo_showblockcount_print 1205 static void pagetypeinfo_showblockcount_print(struct seq_file *m,
1604 pg_da 1206 pg_data_t *pgdat, struct zone *zone)
1605 { 1207 {
1606 int mtype; 1208 int mtype;
1607 unsigned long pfn; 1209 unsigned long pfn;
1608 unsigned long start_pfn = zone->zone_ 1210 unsigned long start_pfn = zone->zone_start_pfn;
1609 unsigned long end_pfn = zone_end_pfn( 1211 unsigned long end_pfn = zone_end_pfn(zone);
1610 unsigned long count[MIGRATE_TYPES] = 1212 unsigned long count[MIGRATE_TYPES] = { 0, };
1611 1213
1612 for (pfn = start_pfn; pfn < end_pfn; 1214 for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
1613 struct page *page; 1215 struct page *page;
1614 1216
1615 page = pfn_to_online_page(pfn !! 1217 if (!pfn_valid(pfn))
1616 if (!page) !! 1218 continue;
>> 1219
>> 1220 page = pfn_to_page(pfn);
>> 1221
>> 1222 /* Watch for unexpected holes punched in the memmap */
>> 1223 if (!memmap_valid_within(pfn, page, zone))
1617 continue; 1224 continue;
1618 1225
1619 if (page_zone(page) != zone) 1226 if (page_zone(page) != zone)
1620 continue; 1227 continue;
1621 1228
1622 mtype = get_pageblock_migrate 1229 mtype = get_pageblock_migratetype(page);
1623 1230
1624 if (mtype < MIGRATE_TYPES) 1231 if (mtype < MIGRATE_TYPES)
1625 count[mtype]++; 1232 count[mtype]++;
1626 } 1233 }
1627 1234
1628 /* Print counts */ 1235 /* Print counts */
1629 seq_printf(m, "Node %d, zone %8s ", p 1236 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
1630 for (mtype = 0; mtype < MIGRATE_TYPES 1237 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1631 seq_printf(m, "%12lu ", count 1238 seq_printf(m, "%12lu ", count[mtype]);
1632 seq_putc(m, '\n'); 1239 seq_putc(m, '\n');
1633 } 1240 }
1634 1241
1635 /* Print out the number of pageblocks for eac !! 1242 /* Print out the free pages at each order for each migratetype */
1636 static void pagetypeinfo_showblockcount(struc !! 1243 static int pagetypeinfo_showblockcount(struct seq_file *m, void *arg)
1637 { 1244 {
1638 int mtype; 1245 int mtype;
1639 pg_data_t *pgdat = (pg_data_t *)arg; 1246 pg_data_t *pgdat = (pg_data_t *)arg;
1640 1247
1641 seq_printf(m, "\n%-23s", "Number of b 1248 seq_printf(m, "\n%-23s", "Number of blocks type ");
1642 for (mtype = 0; mtype < MIGRATE_TYPES 1249 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1643 seq_printf(m, "%12s ", migrat 1250 seq_printf(m, "%12s ", migratetype_names[mtype]);
1644 seq_putc(m, '\n'); 1251 seq_putc(m, '\n');
1645 walk_zones_in_node(m, pgdat, true, fa !! 1252 walk_zones_in_node(m, pgdat, pagetypeinfo_showblockcount_print);
1646 pagetypeinfo_showblockcount_p !! 1253
>> 1254 return 0;
1647 } 1255 }
1648 1256
1649 /* 1257 /*
1650 * Print out the number of pageblocks for eac 1258 * Print out the number of pageblocks for each migratetype that contain pages
1651 * of other types. This gives an indication o 1259 * of other types. This gives an indication of how well fallbacks are being
1652 * contained by rmqueue_fallback(). It requir 1260 * contained by rmqueue_fallback(). It requires information from PAGE_OWNER
1653 * to determine what is going on 1261 * to determine what is going on
1654 */ 1262 */
1655 static void pagetypeinfo_showmixedcount(struc 1263 static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat)
1656 { 1264 {
1657 #ifdef CONFIG_PAGE_OWNER 1265 #ifdef CONFIG_PAGE_OWNER
1658 int mtype; 1266 int mtype;
1659 1267
1660 if (!static_branch_unlikely(&page_own 1268 if (!static_branch_unlikely(&page_owner_inited))
1661 return; 1269 return;
1662 1270
1663 drain_all_pages(NULL); 1271 drain_all_pages(NULL);
1664 1272
1665 seq_printf(m, "\n%-23s", "Number of m 1273 seq_printf(m, "\n%-23s", "Number of mixed blocks ");
1666 for (mtype = 0; mtype < MIGRATE_TYPES 1274 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1667 seq_printf(m, "%12s ", migrat 1275 seq_printf(m, "%12s ", migratetype_names[mtype]);
1668 seq_putc(m, '\n'); 1276 seq_putc(m, '\n');
1669 1277
1670 walk_zones_in_node(m, pgdat, true, tr !! 1278 walk_zones_in_node(m, pgdat, pagetypeinfo_showmixedcount_print);
1671 pagetypeinfo_showmixedcount_p <<
1672 #endif /* CONFIG_PAGE_OWNER */ 1279 #endif /* CONFIG_PAGE_OWNER */
1673 } 1280 }
1674 1281
1675 /* 1282 /*
1676 * This prints out statistics in relation to 1283 * This prints out statistics in relation to grouping pages by mobility.
1677 * It is expensive to collect so do not const 1284 * It is expensive to collect so do not constantly read the file.
1678 */ 1285 */
1679 static int pagetypeinfo_show(struct seq_file 1286 static int pagetypeinfo_show(struct seq_file *m, void *arg)
1680 { 1287 {
1681 pg_data_t *pgdat = (pg_data_t *)arg; 1288 pg_data_t *pgdat = (pg_data_t *)arg;
1682 1289
1683 /* check memoryless node */ 1290 /* check memoryless node */
1684 if (!node_state(pgdat->node_id, N_MEM 1291 if (!node_state(pgdat->node_id, N_MEMORY))
1685 return 0; 1292 return 0;
1686 1293
1687 seq_printf(m, "Page block order: %d\n 1294 seq_printf(m, "Page block order: %d\n", pageblock_order);
1688 seq_printf(m, "Pages per block: %lu\ 1295 seq_printf(m, "Pages per block: %lu\n", pageblock_nr_pages);
1689 seq_putc(m, '\n'); 1296 seq_putc(m, '\n');
1690 pagetypeinfo_showfree(m, pgdat); 1297 pagetypeinfo_showfree(m, pgdat);
1691 pagetypeinfo_showblockcount(m, pgdat) 1298 pagetypeinfo_showblockcount(m, pgdat);
1692 pagetypeinfo_showmixedcount(m, pgdat) 1299 pagetypeinfo_showmixedcount(m, pgdat);
1693 1300
1694 return 0; 1301 return 0;
1695 } 1302 }
1696 1303
1697 static const struct seq_operations fragmentat 1304 static const struct seq_operations fragmentation_op = {
1698 .start = frag_start, 1305 .start = frag_start,
1699 .next = frag_next, 1306 .next = frag_next,
1700 .stop = frag_stop, 1307 .stop = frag_stop,
1701 .show = frag_show, 1308 .show = frag_show,
1702 }; 1309 };
1703 1310
>> 1311 static int fragmentation_open(struct inode *inode, struct file *file)
>> 1312 {
>> 1313 return seq_open(file, &fragmentation_op);
>> 1314 }
>> 1315
>> 1316 static const struct file_operations fragmentation_file_operations = {
>> 1317 .open = fragmentation_open,
>> 1318 .read = seq_read,
>> 1319 .llseek = seq_lseek,
>> 1320 .release = seq_release,
>> 1321 };
>> 1322
1704 static const struct seq_operations pagetypein 1323 static const struct seq_operations pagetypeinfo_op = {
1705 .start = frag_start, 1324 .start = frag_start,
1706 .next = frag_next, 1325 .next = frag_next,
1707 .stop = frag_stop, 1326 .stop = frag_stop,
1708 .show = pagetypeinfo_show, 1327 .show = pagetypeinfo_show,
1709 }; 1328 };
1710 1329
>> 1330 static int pagetypeinfo_open(struct inode *inode, struct file *file)
>> 1331 {
>> 1332 return seq_open(file, &pagetypeinfo_op);
>> 1333 }
>> 1334
>> 1335 static const struct file_operations pagetypeinfo_file_ops = {
>> 1336 .open = pagetypeinfo_open,
>> 1337 .read = seq_read,
>> 1338 .llseek = seq_lseek,
>> 1339 .release = seq_release,
>> 1340 };
>> 1341
1711 static bool is_zone_first_populated(pg_data_t 1342 static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone)
1712 { 1343 {
1713 int zid; 1344 int zid;
1714 1345
1715 for (zid = 0; zid < MAX_NR_ZONES; zid 1346 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1716 struct zone *compare = &pgdat 1347 struct zone *compare = &pgdat->node_zones[zid];
1717 1348
1718 if (populated_zone(compare)) 1349 if (populated_zone(compare))
1719 return zone == compar 1350 return zone == compare;
1720 } 1351 }
1721 1352
>> 1353 /* The zone must be somewhere! */
>> 1354 WARN_ON_ONCE(1);
1722 return false; 1355 return false;
1723 } 1356 }
1724 1357
1725 static void zoneinfo_show_print(struct seq_fi 1358 static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
1726 1359 struct zone *zone)
1727 { 1360 {
1728 int i; 1361 int i;
1729 seq_printf(m, "Node %d, zone %8s", pg 1362 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);
1730 if (is_zone_first_populated(pgdat, zo 1363 if (is_zone_first_populated(pgdat, zone)) {
1731 seq_printf(m, "\n per-node s 1364 seq_printf(m, "\n per-node stats");
1732 for (i = 0; i < NR_VM_NODE_ST 1365 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
1733 unsigned long pages = !! 1366 seq_printf(m, "\n %-12s %lu",
1734 !! 1367 vmstat_text[i + NR_VM_ZONE_STAT_ITEMS],
1735 if (vmstat_item_print !! 1368 node_page_state(pgdat, i));
1736 pages /= HPAG <<
1737 seq_printf(m, "\n <<
1738 pages); <<
1739 } 1369 }
1740 } 1370 }
1741 seq_printf(m, 1371 seq_printf(m,
1742 "\n pages free %lu" 1372 "\n pages free %lu"
1743 "\n boost %lu" <<
1744 "\n min %lu" 1373 "\n min %lu"
1745 "\n low %lu" 1374 "\n low %lu"
1746 "\n high %lu" 1375 "\n high %lu"
1747 "\n promo %lu" !! 1376 "\n node_scanned %lu"
1748 "\n spanned %lu" 1377 "\n spanned %lu"
1749 "\n present %lu" 1378 "\n present %lu"
1750 "\n managed %lu" !! 1379 "\n managed %lu",
1751 "\n cma %lu", <<
1752 zone_page_state(zone, NR_F 1380 zone_page_state(zone, NR_FREE_PAGES),
1753 zone->watermark_boost, <<
1754 min_wmark_pages(zone), 1381 min_wmark_pages(zone),
1755 low_wmark_pages(zone), 1382 low_wmark_pages(zone),
1756 high_wmark_pages(zone), 1383 high_wmark_pages(zone),
1757 promo_wmark_pages(zone), !! 1384 node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED),
1758 zone->spanned_pages, 1385 zone->spanned_pages,
1759 zone->present_pages, 1386 zone->present_pages,
1760 zone_managed_pages(zone), !! 1387 zone->managed_pages);
1761 zone_cma_pages(zone)); !! 1388
>> 1389 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
>> 1390 seq_printf(m, "\n %-12s %lu", vmstat_text[i],
>> 1391 zone_page_state(zone, i));
1762 1392
1763 seq_printf(m, 1393 seq_printf(m,
1764 "\n protection: (%l 1394 "\n protection: (%ld",
1765 zone->lowmem_reserve[0]); 1395 zone->lowmem_reserve[0]);
1766 for (i = 1; i < ARRAY_SIZE(zone->lowm 1396 for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++)
1767 seq_printf(m, ", %ld", zone-> 1397 seq_printf(m, ", %ld", zone->lowmem_reserve[i]);
1768 seq_putc(m, ')'); !! 1398 seq_printf(m,
1769 !! 1399 ")"
1770 /* If unpopulated, no other informati !! 1400 "\n pagesets");
1771 if (!populated_zone(zone)) { <<
1772 seq_putc(m, '\n'); <<
1773 return; <<
1774 } <<
1775 <<
1776 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS <<
1777 seq_printf(m, "\n %-12s <<
1778 zone_page_state(zo <<
1779 <<
1780 #ifdef CONFIG_NUMA <<
1781 for (i = 0; i < NR_VM_NUMA_EVENT_ITEM <<
1782 seq_printf(m, "\n %-12s <<
1783 zone_numa_event_st <<
1784 #endif <<
1785 <<
1786 seq_printf(m, "\n pagesets"); <<
1787 for_each_online_cpu(i) { 1401 for_each_online_cpu(i) {
1788 struct per_cpu_pages *pcp; !! 1402 struct per_cpu_pageset *pageset;
1789 struct per_cpu_zonestat __may <<
1790 1403
1791 pcp = per_cpu_ptr(zone->per_c !! 1404 pageset = per_cpu_ptr(zone->pageset, i);
1792 seq_printf(m, 1405 seq_printf(m,
1793 "\n cpu: %i" 1406 "\n cpu: %i"
1794 "\n c 1407 "\n count: %i"
1795 "\n h 1408 "\n high: %i"
1796 "\n b 1409 "\n batch: %i",
1797 i, 1410 i,
1798 pcp->count, !! 1411 pageset->pcp.count,
1799 pcp->high, !! 1412 pageset->pcp.high,
1800 pcp->batch); !! 1413 pageset->pcp.batch);
1801 #ifdef CONFIG_SMP 1414 #ifdef CONFIG_SMP
1802 pzstats = per_cpu_ptr(zone->p <<
1803 seq_printf(m, "\n vm stats t 1415 seq_printf(m, "\n vm stats threshold: %d",
1804 pzstats->stat !! 1416 pageset->stat_threshold);
1805 #endif 1417 #endif
1806 } 1418 }
1807 seq_printf(m, 1419 seq_printf(m,
1808 "\n node_unreclaimable: 1420 "\n node_unreclaimable: %u"
1809 "\n start_pfn: !! 1421 "\n start_pfn: %lu"
1810 pgdat->kswapd_failures >= !! 1422 "\n node_inactive_ratio: %u",
1811 zone->zone_start_pfn); !! 1423 !pgdat_reclaimable(zone->zone_pgdat),
>> 1424 zone->zone_start_pfn,
>> 1425 zone->zone_pgdat->inactive_ratio);
1812 seq_putc(m, '\n'); 1426 seq_putc(m, '\n');
1813 } 1427 }
1814 1428
1815 /* 1429 /*
1816 * Output information about zones in @pgdat. !! 1430 * Output information about zones in @pgdat.
1817 * of whether they are populated or not: lowm <<
1818 * set of all zones and userspace would not b <<
1819 * suppressed here (zoneinfo displays the eff <<
1820 */ 1431 */
1821 static int zoneinfo_show(struct seq_file *m, 1432 static int zoneinfo_show(struct seq_file *m, void *arg)
1822 { 1433 {
1823 pg_data_t *pgdat = (pg_data_t *)arg; 1434 pg_data_t *pgdat = (pg_data_t *)arg;
1824 walk_zones_in_node(m, pgdat, false, f !! 1435 walk_zones_in_node(m, pgdat, zoneinfo_show_print);
1825 return 0; 1436 return 0;
1826 } 1437 }
1827 1438
1828 static const struct seq_operations zoneinfo_o 1439 static const struct seq_operations zoneinfo_op = {
1829 .start = frag_start, /* iterate over 1440 .start = frag_start, /* iterate over all zones. The same as in
1830 * fragmentatio 1441 * fragmentation. */
1831 .next = frag_next, 1442 .next = frag_next,
1832 .stop = frag_stop, 1443 .stop = frag_stop,
1833 .show = zoneinfo_show, 1444 .show = zoneinfo_show,
1834 }; 1445 };
1835 1446
1836 #define NR_VMSTAT_ITEMS (NR_VM_ZONE_STAT_ITEM !! 1447 static int zoneinfo_open(struct inode *inode, struct file *file)
1837 NR_VM_NUMA_EVENT_ITE !! 1448 {
1838 NR_VM_NODE_STAT_ITEM !! 1449 return seq_open(file, &zoneinfo_op);
1839 NR_VM_STAT_ITEMS + \ !! 1450 }
1840 (IS_ENABLED(CONFIG_V !! 1451
1841 NR_VM_EVENT_ITEMS : !! 1452 static const struct file_operations proc_zoneinfo_file_operations = {
>> 1453 .open = zoneinfo_open,
>> 1454 .read = seq_read,
>> 1455 .llseek = seq_lseek,
>> 1456 .release = seq_release,
>> 1457 };
>> 1458
>> 1459 enum writeback_stat_item {
>> 1460 NR_DIRTY_THRESHOLD,
>> 1461 NR_DIRTY_BG_THRESHOLD,
>> 1462 NR_VM_WRITEBACK_STAT_ITEMS,
>> 1463 };
1842 1464
1843 static void *vmstat_start(struct seq_file *m, 1465 static void *vmstat_start(struct seq_file *m, loff_t *pos)
1844 { 1466 {
1845 unsigned long *v; 1467 unsigned long *v;
1846 int i; !! 1468 int i, stat_items_size;
1847 1469
1848 if (*pos >= NR_VMSTAT_ITEMS) !! 1470 if (*pos >= ARRAY_SIZE(vmstat_text))
1849 return NULL; 1471 return NULL;
>> 1472 stat_items_size = NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long) +
>> 1473 NR_VM_NODE_STAT_ITEMS * sizeof(unsigned long) +
>> 1474 NR_VM_WRITEBACK_STAT_ITEMS * sizeof(unsigned long);
1850 1475
1851 BUILD_BUG_ON(ARRAY_SIZE(vmstat_text) !! 1476 #ifdef CONFIG_VM_EVENT_COUNTERS
1852 fold_vm_numa_events(); !! 1477 stat_items_size += sizeof(struct vm_event_state);
1853 v = kmalloc_array(NR_VMSTAT_ITEMS, si !! 1478 #endif
>> 1479
>> 1480 v = kmalloc(stat_items_size, GFP_KERNEL);
1854 m->private = v; 1481 m->private = v;
1855 if (!v) 1482 if (!v)
1856 return ERR_PTR(-ENOMEM); 1483 return ERR_PTR(-ENOMEM);
1857 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS 1484 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
1858 v[i] = global_zone_page_state !! 1485 v[i] = global_page_state(i);
1859 v += NR_VM_ZONE_STAT_ITEMS; 1486 v += NR_VM_ZONE_STAT_ITEMS;
1860 1487
1861 #ifdef CONFIG_NUMA !! 1488 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
1862 for (i = 0; i < NR_VM_NUMA_EVENT_ITEM !! 1489 v[i] = global_node_page_state(i);
1863 v[i] = global_numa_event_stat <<
1864 v += NR_VM_NUMA_EVENT_ITEMS; <<
1865 #endif <<
1866 <<
1867 for (i = 0; i < NR_VM_NODE_STAT_ITEMS <<
1868 v[i] = global_node_page_state <<
1869 if (vmstat_item_print_in_thp( <<
1870 v[i] /= HPAGE_PMD_NR; <<
1871 } <<
1872 v += NR_VM_NODE_STAT_ITEMS; 1490 v += NR_VM_NODE_STAT_ITEMS;
1873 1491
1874 global_dirty_limits(v + NR_DIRTY_BG_T 1492 global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD,
1875 v + NR_DIRTY_THRE 1493 v + NR_DIRTY_THRESHOLD);
1876 v[NR_MEMMAP_PAGES] = atomic_long_read !! 1494 v += NR_VM_WRITEBACK_STAT_ITEMS;
1877 v[NR_MEMMAP_BOOT_PAGES] = atomic_long <<
1878 v += NR_VM_STAT_ITEMS; <<
1879 1495
1880 #ifdef CONFIG_VM_EVENT_COUNTERS 1496 #ifdef CONFIG_VM_EVENT_COUNTERS
1881 all_vm_events(v); 1497 all_vm_events(v);
1882 v[PGPGIN] /= 2; /* sectors -> 1498 v[PGPGIN] /= 2; /* sectors -> kbytes */
1883 v[PGPGOUT] /= 2; 1499 v[PGPGOUT] /= 2;
1884 #endif 1500 #endif
1885 return (unsigned long *)m->private + 1501 return (unsigned long *)m->private + *pos;
1886 } 1502 }
1887 1503
1888 static void *vmstat_next(struct seq_file *m, 1504 static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos)
1889 { 1505 {
1890 (*pos)++; 1506 (*pos)++;
1891 if (*pos >= NR_VMSTAT_ITEMS) !! 1507 if (*pos >= ARRAY_SIZE(vmstat_text))
1892 return NULL; 1508 return NULL;
1893 return (unsigned long *)m->private + 1509 return (unsigned long *)m->private + *pos;
1894 } 1510 }
1895 1511
1896 static int vmstat_show(struct seq_file *m, vo 1512 static int vmstat_show(struct seq_file *m, void *arg)
1897 { 1513 {
1898 unsigned long *l = arg; 1514 unsigned long *l = arg;
1899 unsigned long off = l - (unsigned lon 1515 unsigned long off = l - (unsigned long *)m->private;
1900 1516
1901 seq_puts(m, vmstat_text[off]); 1517 seq_puts(m, vmstat_text[off]);
1902 seq_put_decimal_ull(m, " ", *l); 1518 seq_put_decimal_ull(m, " ", *l);
1903 seq_putc(m, '\n'); 1519 seq_putc(m, '\n');
1904 <<
1905 if (off == NR_VMSTAT_ITEMS - 1) { <<
1906 /* <<
1907 * We've come to the end - ad <<
1908 * breaking userspace which m <<
1909 */ <<
1910 seq_puts(m, "nr_unstable 0\n" <<
1911 } <<
1912 return 0; 1520 return 0;
1913 } 1521 }
1914 1522
1915 static void vmstat_stop(struct seq_file *m, v 1523 static void vmstat_stop(struct seq_file *m, void *arg)
1916 { 1524 {
1917 kfree(m->private); 1525 kfree(m->private);
1918 m->private = NULL; 1526 m->private = NULL;
1919 } 1527 }
1920 1528
1921 static const struct seq_operations vmstat_op 1529 static const struct seq_operations vmstat_op = {
1922 .start = vmstat_start, 1530 .start = vmstat_start,
1923 .next = vmstat_next, 1531 .next = vmstat_next,
1924 .stop = vmstat_stop, 1532 .stop = vmstat_stop,
1925 .show = vmstat_show, 1533 .show = vmstat_show,
1926 }; 1534 };
>> 1535
>> 1536 static int vmstat_open(struct inode *inode, struct file *file)
>> 1537 {
>> 1538 return seq_open(file, &vmstat_op);
>> 1539 }
>> 1540
>> 1541 static const struct file_operations proc_vmstat_file_operations = {
>> 1542 .open = vmstat_open,
>> 1543 .read = seq_read,
>> 1544 .llseek = seq_lseek,
>> 1545 .release = seq_release,
>> 1546 };
1927 #endif /* CONFIG_PROC_FS */ 1547 #endif /* CONFIG_PROC_FS */
1928 1548
1929 #ifdef CONFIG_SMP 1549 #ifdef CONFIG_SMP
>> 1550 static struct workqueue_struct *vmstat_wq;
1930 static DEFINE_PER_CPU(struct delayed_work, vm 1551 static DEFINE_PER_CPU(struct delayed_work, vmstat_work);
1931 int sysctl_stat_interval __read_mostly = HZ; 1552 int sysctl_stat_interval __read_mostly = HZ;
1932 1553
1933 #ifdef CONFIG_PROC_FS 1554 #ifdef CONFIG_PROC_FS
1934 static void refresh_vm_stats(struct work_stru 1555 static void refresh_vm_stats(struct work_struct *work)
1935 { 1556 {
1936 refresh_cpu_vm_stats(true); 1557 refresh_cpu_vm_stats(true);
1937 } 1558 }
1938 1559
1939 int vmstat_refresh(const struct ctl_table *ta !! 1560 int vmstat_refresh(struct ctl_table *table, int write,
1940 void *buffer, size_t *lenp !! 1561 void __user *buffer, size_t *lenp, loff_t *ppos)
1941 { 1562 {
1942 long val; 1563 long val;
1943 int err; 1564 int err;
1944 int i; 1565 int i;
1945 1566
1946 /* 1567 /*
1947 * The regular update, every sysctl_s 1568 * The regular update, every sysctl_stat_interval, may come later
1948 * than expected: leaving a significa 1569 * than expected: leaving a significant amount in per_cpu buckets.
1949 * This is particularly misleading wh 1570 * This is particularly misleading when checking a quantity of HUGE
1950 * pages, immediately after running a 1571 * pages, immediately after running a test. /proc/sys/vm/stat_refresh,
1951 * which can equally be echo'ed to or 1572 * which can equally be echo'ed to or cat'ted from (by root),
1952 * can be used to update the stats ju 1573 * can be used to update the stats just before reading them.
1953 * 1574 *
1954 * Oh, and since global_zone_page_sta !! 1575 * Oh, and since global_page_state() etc. are so careful to hide
1955 * transiently negative values, repor 1576 * transiently negative values, report an error here if any of
1956 * the stats is negative, so we know 1577 * the stats is negative, so we know to go looking for imbalance.
1957 */ 1578 */
1958 err = schedule_on_each_cpu(refresh_vm 1579 err = schedule_on_each_cpu(refresh_vm_stats);
1959 if (err) 1580 if (err)
1960 return err; 1581 return err;
1961 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS 1582 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
1962 /* <<
1963 * Skip checking stats known <<
1964 */ <<
1965 switch (i) { <<
1966 case NR_ZONE_WRITE_PENDING: <<
1967 case NR_FREE_CMA_PAGES: <<
1968 continue; <<
1969 } <<
1970 val = atomic_long_read(&vm_zo 1583 val = atomic_long_read(&vm_zone_stat[i]);
1971 if (val < 0) { 1584 if (val < 0) {
1972 pr_warn("%s: %s %ld\n !! 1585 switch (i) {
1973 __func__, zon !! 1586 case NR_PAGES_SCANNED:
1974 } !! 1587 /*
1975 } !! 1588 * This is often seen to go negative in
1976 for (i = 0; i < NR_VM_NODE_STAT_ITEMS !! 1589 * recent kernels, but not to go permanently
1977 /* !! 1590 * negative. Whilst it would be nicer not to
1978 * Skip checking stats known !! 1591 * have exceptions, rooting them out would be
1979 */ !! 1592 * another task, of rather low priority.
1980 switch (i) { !! 1593 */
1981 case NR_WRITEBACK: !! 1594 break;
1982 continue; !! 1595 default:
1983 } !! 1596 pr_warn("%s: %s %ld\n",
1984 val = atomic_long_read(&vm_no !! 1597 __func__, vmstat_text[i], val);
1985 if (val < 0) { !! 1598 err = -EINVAL;
1986 pr_warn("%s: %s %ld\n !! 1599 break;
1987 __func__, nod !! 1600 }
1988 } 1601 }
1989 } 1602 }
>> 1603 if (err)
>> 1604 return err;
1990 if (write) 1605 if (write)
1991 *ppos += *lenp; 1606 *ppos += *lenp;
1992 else 1607 else
1993 *lenp = 0; 1608 *lenp = 0;
1994 return 0; 1609 return 0;
1995 } 1610 }
1996 #endif /* CONFIG_PROC_FS */ 1611 #endif /* CONFIG_PROC_FS */
1997 1612
1998 static void vmstat_update(struct work_struct 1613 static void vmstat_update(struct work_struct *w)
1999 { 1614 {
2000 if (refresh_cpu_vm_stats(true)) { 1615 if (refresh_cpu_vm_stats(true)) {
2001 /* 1616 /*
2002 * Counters were updated so w 1617 * Counters were updated so we expect more updates
2003 * to occur in the future. Ke 1618 * to occur in the future. Keep on running the
2004 * update worker thread. 1619 * update worker thread.
2005 */ 1620 */
2006 queue_delayed_work_on(smp_pro !! 1621 queue_delayed_work_on(smp_processor_id(), vmstat_wq,
2007 this_cpu_ptr( 1622 this_cpu_ptr(&vmstat_work),
2008 round_jiffies 1623 round_jiffies_relative(sysctl_stat_interval));
2009 } 1624 }
2010 } 1625 }
2011 1626
2012 /* 1627 /*
>> 1628 * Switch off vmstat processing and then fold all the remaining differentials
>> 1629 * until the diffs stay at zero. The function is used by NOHZ and can only be
>> 1630 * invoked when tick processing is not active.
>> 1631 */
>> 1632 /*
2013 * Check if the diffs for a certain cpu indic 1633 * Check if the diffs for a certain cpu indicate that
2014 * an update is needed. 1634 * an update is needed.
2015 */ 1635 */
2016 static bool need_update(int cpu) 1636 static bool need_update(int cpu)
2017 { 1637 {
2018 pg_data_t *last_pgdat = NULL; <<
2019 struct zone *zone; 1638 struct zone *zone;
2020 1639
2021 for_each_populated_zone(zone) { 1640 for_each_populated_zone(zone) {
2022 struct per_cpu_zonestat *pzst !! 1641 struct per_cpu_pageset *p = per_cpu_ptr(zone->pageset, cpu);
2023 struct per_cpu_nodestat *n; <<
2024 1642
>> 1643 BUILD_BUG_ON(sizeof(p->vm_stat_diff[0]) != 1);
2025 /* 1644 /*
2026 * The fast way of checking i 1645 * The fast way of checking if there are any vmstat diffs.
>> 1646 * This works because the diffs are byte sized items.
2027 */ 1647 */
2028 if (memchr_inv(pzstats->vm_st !! 1648 if (memchr_inv(p->vm_stat_diff, 0, NR_VM_ZONE_STAT_ITEMS))
2029 return true; 1649 return true;
2030 1650
2031 if (last_pgdat == zone->zone_ <<
2032 continue; <<
2033 last_pgdat = zone->zone_pgdat <<
2034 n = per_cpu_ptr(zone->zone_pg <<
2035 if (memchr_inv(n->vm_node_sta <<
2036 return true; <<
2037 } 1651 }
2038 return false; 1652 return false;
2039 } 1653 }
2040 1654
2041 /* 1655 /*
2042 * Switch off vmstat processing and then fold 1656 * Switch off vmstat processing and then fold all the remaining differentials
2043 * until the diffs stay at zero. The function 1657 * until the diffs stay at zero. The function is used by NOHZ and can only be
2044 * invoked when tick processing is not active 1658 * invoked when tick processing is not active.
2045 */ 1659 */
2046 void quiet_vmstat(void) 1660 void quiet_vmstat(void)
2047 { 1661 {
2048 if (system_state != SYSTEM_RUNNING) 1662 if (system_state != SYSTEM_RUNNING)
2049 return; 1663 return;
2050 1664
2051 if (!delayed_work_pending(this_cpu_pt 1665 if (!delayed_work_pending(this_cpu_ptr(&vmstat_work)))
2052 return; 1666 return;
2053 1667
2054 if (!need_update(smp_processor_id())) 1668 if (!need_update(smp_processor_id()))
2055 return; 1669 return;
2056 1670
2057 /* 1671 /*
2058 * Just refresh counters and do not c 1672 * Just refresh counters and do not care about the pending delayed
2059 * vmstat_update. It doesn't fire tha 1673 * vmstat_update. It doesn't fire that often to matter and canceling
2060 * it would be too expensive from thi 1674 * it would be too expensive from this path.
2061 * vmstat_shepherd will take care abo 1675 * vmstat_shepherd will take care about that for us.
2062 */ 1676 */
2063 refresh_cpu_vm_stats(false); 1677 refresh_cpu_vm_stats(false);
2064 } 1678 }
2065 1679
2066 /* 1680 /*
2067 * Shepherd worker thread that checks the 1681 * Shepherd worker thread that checks the
2068 * differentials of processors that have thei 1682 * differentials of processors that have their worker
2069 * threads for vm statistics updates disabled 1683 * threads for vm statistics updates disabled because of
2070 * inactivity. 1684 * inactivity.
2071 */ 1685 */
2072 static void vmstat_shepherd(struct work_struc 1686 static void vmstat_shepherd(struct work_struct *w);
2073 1687
2074 static DECLARE_DEFERRABLE_WORK(shepherd, vmst 1688 static DECLARE_DEFERRABLE_WORK(shepherd, vmstat_shepherd);
2075 1689
2076 static void vmstat_shepherd(struct work_struc 1690 static void vmstat_shepherd(struct work_struct *w)
2077 { 1691 {
2078 int cpu; 1692 int cpu;
2079 1693
2080 cpus_read_lock(); !! 1694 get_online_cpus();
2081 /* Check processors whose vmstat work 1695 /* Check processors whose vmstat worker threads have been disabled */
2082 for_each_online_cpu(cpu) { 1696 for_each_online_cpu(cpu) {
2083 struct delayed_work *dw = &pe 1697 struct delayed_work *dw = &per_cpu(vmstat_work, cpu);
2084 1698
2085 /* <<
2086 * In kernel users of vmstat <<
2087 * they are using zone_page_s <<
2088 * an imprecision as the regu <<
2089 * cumulative error can grow <<
2090 * <<
2091 * From that POV the regular <<
2092 * been isolated from the ker <<
2093 * infrastructure ever notici <<
2094 * for all isolated CPUs to a <<
2095 */ <<
2096 if (cpu_is_isolated(cpu)) <<
2097 continue; <<
2098 <<
2099 if (!delayed_work_pending(dw) 1699 if (!delayed_work_pending(dw) && need_update(cpu))
2100 queue_delayed_work_on !! 1700 queue_delayed_work_on(cpu, vmstat_wq, dw, 0);
2101 <<
2102 cond_resched(); <<
2103 } 1701 }
2104 cpus_read_unlock(); !! 1702 put_online_cpus();
2105 1703
2106 schedule_delayed_work(&shepherd, 1704 schedule_delayed_work(&shepherd,
2107 round_jiffies_relative(sysctl 1705 round_jiffies_relative(sysctl_stat_interval));
2108 } 1706 }
2109 1707
2110 static void __init start_shepherd_timer(void) 1708 static void __init start_shepherd_timer(void)
2111 { 1709 {
2112 int cpu; 1710 int cpu;
2113 1711
2114 for_each_possible_cpu(cpu) 1712 for_each_possible_cpu(cpu)
2115 INIT_DEFERRABLE_WORK(per_cpu_ 1713 INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work, cpu),
2116 vmstat_update); 1714 vmstat_update);
2117 1715
>> 1716 vmstat_wq = alloc_workqueue("vmstat", WQ_FREEZABLE|WQ_MEM_RECLAIM, 0);
2118 schedule_delayed_work(&shepherd, 1717 schedule_delayed_work(&shepherd,
2119 round_jiffies_relative(sysctl 1718 round_jiffies_relative(sysctl_stat_interval));
2120 } 1719 }
2121 1720
2122 static void __init init_cpu_node_state(void) 1721 static void __init init_cpu_node_state(void)
2123 { 1722 {
2124 int node; 1723 int node;
2125 1724
2126 for_each_online_node(node) { 1725 for_each_online_node(node) {
2127 if (!cpumask_empty(cpumask_of !! 1726 if (cpumask_weight(cpumask_of_node(node)) > 0)
2128 node_set_state(node, 1727 node_set_state(node, N_CPU);
2129 } 1728 }
2130 } 1729 }
2131 1730
2132 static int vmstat_cpu_online(unsigned int cpu 1731 static int vmstat_cpu_online(unsigned int cpu)
2133 { 1732 {
2134 refresh_zone_stat_thresholds(); 1733 refresh_zone_stat_thresholds();
2135 !! 1734 node_set_state(cpu_to_node(cpu), N_CPU);
2136 if (!node_state(cpu_to_node(cpu), N_C <<
2137 node_set_state(cpu_to_node(cp <<
2138 } <<
2139 <<
2140 return 0; 1735 return 0;
2141 } 1736 }
2142 1737
2143 static int vmstat_cpu_down_prep(unsigned int 1738 static int vmstat_cpu_down_prep(unsigned int cpu)
2144 { 1739 {
2145 cancel_delayed_work_sync(&per_cpu(vms 1740 cancel_delayed_work_sync(&per_cpu(vmstat_work, cpu));
2146 return 0; 1741 return 0;
2147 } 1742 }
2148 1743
2149 static int vmstat_cpu_dead(unsigned int cpu) 1744 static int vmstat_cpu_dead(unsigned int cpu)
2150 { 1745 {
2151 const struct cpumask *node_cpus; 1746 const struct cpumask *node_cpus;
2152 int node; 1747 int node;
2153 1748
2154 node = cpu_to_node(cpu); 1749 node = cpu_to_node(cpu);
2155 1750
2156 refresh_zone_stat_thresholds(); 1751 refresh_zone_stat_thresholds();
2157 node_cpus = cpumask_of_node(node); 1752 node_cpus = cpumask_of_node(node);
2158 if (!cpumask_empty(node_cpus)) !! 1753 if (cpumask_weight(node_cpus) > 0)
2159 return 0; 1754 return 0;
2160 1755
2161 node_clear_state(node, N_CPU); 1756 node_clear_state(node, N_CPU);
2162 <<
2163 return 0; 1757 return 0;
2164 } 1758 }
2165 1759
2166 #endif 1760 #endif
2167 1761
2168 struct workqueue_struct *mm_percpu_wq; !! 1762 static int __init setup_vmstat(void)
2169 <<
2170 void __init init_mm_internals(void) <<
2171 { 1763 {
2172 int ret __maybe_unused; <<
2173 <<
2174 mm_percpu_wq = alloc_workqueue("mm_pe <<
2175 <<
2176 #ifdef CONFIG_SMP 1764 #ifdef CONFIG_SMP
>> 1765 int ret;
>> 1766
2177 ret = cpuhp_setup_state_nocalls(CPUHP 1767 ret = cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD, "mm/vmstat:dead",
2178 NULL, 1768 NULL, vmstat_cpu_dead);
2179 if (ret < 0) 1769 if (ret < 0)
2180 pr_err("vmstat: failed to reg 1770 pr_err("vmstat: failed to register 'dead' hotplug state\n");
2181 1771
2182 ret = cpuhp_setup_state_nocalls(CPUHP 1772 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "mm/vmstat:online",
2183 vmsta 1773 vmstat_cpu_online,
2184 vmsta 1774 vmstat_cpu_down_prep);
2185 if (ret < 0) 1775 if (ret < 0)
2186 pr_err("vmstat: failed to reg 1776 pr_err("vmstat: failed to register 'online' hotplug state\n");
2187 1777
2188 cpus_read_lock(); !! 1778 get_online_cpus();
2189 init_cpu_node_state(); 1779 init_cpu_node_state();
2190 cpus_read_unlock(); !! 1780 put_online_cpus();
2191 1781
2192 start_shepherd_timer(); 1782 start_shepherd_timer();
2193 #endif 1783 #endif
2194 #ifdef CONFIG_PROC_FS 1784 #ifdef CONFIG_PROC_FS
2195 proc_create_seq("buddyinfo", 0444, NU !! 1785 proc_create("buddyinfo", S_IRUGO, NULL, &fragmentation_file_operations);
2196 proc_create_seq("pagetypeinfo", 0400, !! 1786 proc_create("pagetypeinfo", S_IRUGO, NULL, &pagetypeinfo_file_ops);
2197 proc_create_seq("vmstat", 0444, NULL, !! 1787 proc_create("vmstat", S_IRUGO, NULL, &proc_vmstat_file_operations);
2198 proc_create_seq("zoneinfo", 0444, NUL !! 1788 proc_create("zoneinfo", S_IRUGO, NULL, &proc_zoneinfo_file_operations);
2199 #endif 1789 #endif
>> 1790 return 0;
2200 } 1791 }
>> 1792 module_init(setup_vmstat)
2201 1793
2202 #if defined(CONFIG_DEBUG_FS) && defined(CONFI 1794 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)
2203 1795
2204 /* 1796 /*
2205 * Return an index indicating how much of the 1797 * Return an index indicating how much of the available free memory is
2206 * unusable for an allocation of the requeste 1798 * unusable for an allocation of the requested size.
2207 */ 1799 */
2208 static int unusable_free_index(unsigned int o 1800 static int unusable_free_index(unsigned int order,
2209 struct contig 1801 struct contig_page_info *info)
2210 { 1802 {
2211 /* No free memory is interpreted as a 1803 /* No free memory is interpreted as all free memory is unusable */
2212 if (info->free_pages == 0) 1804 if (info->free_pages == 0)
2213 return 1000; 1805 return 1000;
2214 1806
2215 /* 1807 /*
2216 * Index should be a value between 0 1808 * Index should be a value between 0 and 1. Return a value to 3
2217 * decimal places. 1809 * decimal places.
2218 * 1810 *
2219 * 0 => no fragmentation 1811 * 0 => no fragmentation
2220 * 1 => high fragmentation 1812 * 1 => high fragmentation
2221 */ 1813 */
2222 return div_u64((info->free_pages - (i 1814 return div_u64((info->free_pages - (info->free_blocks_suitable << order)) * 1000ULL, info->free_pages);
2223 1815
2224 } 1816 }
2225 1817
2226 static void unusable_show_print(struct seq_fi 1818 static void unusable_show_print(struct seq_file *m,
2227 pg_da 1819 pg_data_t *pgdat, struct zone *zone)
2228 { 1820 {
2229 unsigned int order; 1821 unsigned int order;
2230 int index; 1822 int index;
2231 struct contig_page_info info; 1823 struct contig_page_info info;
2232 1824
2233 seq_printf(m, "Node %d, zone %8s ", 1825 seq_printf(m, "Node %d, zone %8s ",
2234 pgdat->node_i 1826 pgdat->node_id,
2235 zone->name); 1827 zone->name);
2236 for (order = 0; order < NR_PAGE_ORDER !! 1828 for (order = 0; order < MAX_ORDER; ++order) {
2237 fill_contig_page_info(zone, o 1829 fill_contig_page_info(zone, order, &info);
2238 index = unusable_free_index(o 1830 index = unusable_free_index(order, &info);
2239 seq_printf(m, "%d.%03d ", ind 1831 seq_printf(m, "%d.%03d ", index / 1000, index % 1000);
2240 } 1832 }
2241 1833
2242 seq_putc(m, '\n'); 1834 seq_putc(m, '\n');
2243 } 1835 }
2244 1836
2245 /* 1837 /*
2246 * Display unusable free space index 1838 * Display unusable free space index
2247 * 1839 *
2248 * The unusable free space index measures how 1840 * The unusable free space index measures how much of the available free
2249 * memory cannot be used to satisfy an alloca 1841 * memory cannot be used to satisfy an allocation of a given size and is a
2250 * value between 0 and 1. The higher the valu 1842 * value between 0 and 1. The higher the value, the more of free memory is
2251 * unusable and by implication, the worse the 1843 * unusable and by implication, the worse the external fragmentation is. This
2252 * can be expressed as a percentage by multip 1844 * can be expressed as a percentage by multiplying by 100.
2253 */ 1845 */
2254 static int unusable_show(struct seq_file *m, 1846 static int unusable_show(struct seq_file *m, void *arg)
2255 { 1847 {
2256 pg_data_t *pgdat = (pg_data_t *)arg; 1848 pg_data_t *pgdat = (pg_data_t *)arg;
2257 1849
2258 /* check memoryless node */ 1850 /* check memoryless node */
2259 if (!node_state(pgdat->node_id, N_MEM 1851 if (!node_state(pgdat->node_id, N_MEMORY))
2260 return 0; 1852 return 0;
2261 1853
2262 walk_zones_in_node(m, pgdat, true, fa !! 1854 walk_zones_in_node(m, pgdat, unusable_show_print);
2263 1855
2264 return 0; 1856 return 0;
2265 } 1857 }
2266 1858
2267 static const struct seq_operations unusable_s !! 1859 static const struct seq_operations unusable_op = {
2268 .start = frag_start, 1860 .start = frag_start,
2269 .next = frag_next, 1861 .next = frag_next,
2270 .stop = frag_stop, 1862 .stop = frag_stop,
2271 .show = unusable_show, 1863 .show = unusable_show,
2272 }; 1864 };
2273 1865
2274 DEFINE_SEQ_ATTRIBUTE(unusable); !! 1866 static int unusable_open(struct inode *inode, struct file *file)
>> 1867 {
>> 1868 return seq_open(file, &unusable_op);
>> 1869 }
>> 1870
>> 1871 static const struct file_operations unusable_file_ops = {
>> 1872 .open = unusable_open,
>> 1873 .read = seq_read,
>> 1874 .llseek = seq_lseek,
>> 1875 .release = seq_release,
>> 1876 };
2275 1877
2276 static void extfrag_show_print(struct seq_fil 1878 static void extfrag_show_print(struct seq_file *m,
2277 pg_da 1879 pg_data_t *pgdat, struct zone *zone)
2278 { 1880 {
2279 unsigned int order; 1881 unsigned int order;
2280 int index; 1882 int index;
2281 1883
2282 /* Alloc on stack as interrupts are d 1884 /* Alloc on stack as interrupts are disabled for zone walk */
2283 struct contig_page_info info; 1885 struct contig_page_info info;
2284 1886
2285 seq_printf(m, "Node %d, zone %8s ", 1887 seq_printf(m, "Node %d, zone %8s ",
2286 pgdat->node_i 1888 pgdat->node_id,
2287 zone->name); 1889 zone->name);
2288 for (order = 0; order < NR_PAGE_ORDER !! 1890 for (order = 0; order < MAX_ORDER; ++order) {
2289 fill_contig_page_info(zone, o 1891 fill_contig_page_info(zone, order, &info);
2290 index = __fragmentation_index 1892 index = __fragmentation_index(order, &info);
2291 seq_printf(m, "%2d.%03d ", in !! 1893 seq_printf(m, "%d.%03d ", index / 1000, index % 1000);
2292 } 1894 }
2293 1895
2294 seq_putc(m, '\n'); 1896 seq_putc(m, '\n');
2295 } 1897 }
2296 1898
2297 /* 1899 /*
2298 * Display fragmentation index for orders tha 1900 * Display fragmentation index for orders that allocations would fail for
2299 */ 1901 */
2300 static int extfrag_show(struct seq_file *m, v 1902 static int extfrag_show(struct seq_file *m, void *arg)
2301 { 1903 {
2302 pg_data_t *pgdat = (pg_data_t *)arg; 1904 pg_data_t *pgdat = (pg_data_t *)arg;
2303 1905
2304 walk_zones_in_node(m, pgdat, true, fa !! 1906 walk_zones_in_node(m, pgdat, extfrag_show_print);
2305 1907
2306 return 0; 1908 return 0;
2307 } 1909 }
2308 1910
2309 static const struct seq_operations extfrag_so !! 1911 static const struct seq_operations extfrag_op = {
2310 .start = frag_start, 1912 .start = frag_start,
2311 .next = frag_next, 1913 .next = frag_next,
2312 .stop = frag_stop, 1914 .stop = frag_stop,
2313 .show = extfrag_show, 1915 .show = extfrag_show,
2314 }; 1916 };
2315 1917
2316 DEFINE_SEQ_ATTRIBUTE(extfrag); !! 1918 static int extfrag_open(struct inode *inode, struct file *file)
>> 1919 {
>> 1920 return seq_open(file, &extfrag_op);
>> 1921 }
>> 1922
>> 1923 static const struct file_operations extfrag_file_ops = {
>> 1924 .open = extfrag_open,
>> 1925 .read = seq_read,
>> 1926 .llseek = seq_lseek,
>> 1927 .release = seq_release,
>> 1928 };
2317 1929
2318 static int __init extfrag_debug_init(void) 1930 static int __init extfrag_debug_init(void)
2319 { 1931 {
2320 struct dentry *extfrag_debug_root; 1932 struct dentry *extfrag_debug_root;
2321 1933
2322 extfrag_debug_root = debugfs_create_d 1934 extfrag_debug_root = debugfs_create_dir("extfrag", NULL);
>> 1935 if (!extfrag_debug_root)
>> 1936 return -ENOMEM;
2323 1937
2324 debugfs_create_file("unusable_index", !! 1938 if (!debugfs_create_file("unusable_index", 0444,
2325 &unusable_fops); !! 1939 extfrag_debug_root, NULL, &unusable_file_ops))
2326 !! 1940 goto fail;
2327 debugfs_create_file("extfrag_index", !! 1941
2328 &extfrag_fops); !! 1942 if (!debugfs_create_file("extfrag_index", 0444,
>> 1943 extfrag_debug_root, NULL, &extfrag_file_ops))
>> 1944 goto fail;
2329 1945
2330 return 0; 1946 return 0;
>> 1947 fail:
>> 1948 debugfs_remove_recursive(extfrag_debug_root);
>> 1949 return -ENOMEM;
2331 } 1950 }
2332 1951
2333 module_init(extfrag_debug_init); 1952 module_init(extfrag_debug_init);
2334 <<
2335 #endif 1953 #endif
2336 1954
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.