1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/debugfs.h> 2 #include <linux/debugfs.h>
3 #include <linux/mm.h> 3 #include <linux/mm.h>
4 #include <linux/slab.h> 4 #include <linux/slab.h>
5 #include <linux/uaccess.h> 5 #include <linux/uaccess.h>
6 #include <linux/memblock.h> 6 #include <linux/memblock.h>
7 #include <linux/stacktrace.h> 7 #include <linux/stacktrace.h>
8 #include <linux/page_owner.h> 8 #include <linux/page_owner.h>
9 #include <linux/jump_label.h> 9 #include <linux/jump_label.h>
10 #include <linux/migrate.h> 10 #include <linux/migrate.h>
11 #include <linux/stackdepot.h> 11 #include <linux/stackdepot.h>
12 #include <linux/seq_file.h> 12 #include <linux/seq_file.h>
13 #include <linux/memcontrol.h> 13 #include <linux/memcontrol.h>
14 #include <linux/sched/clock.h> 14 #include <linux/sched/clock.h>
15 15
16 #include "internal.h" 16 #include "internal.h"
17 17
18 /* 18 /*
19 * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_ 19 * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack)
20 * to use off stack temporal storage 20 * to use off stack temporal storage
21 */ 21 */
22 #define PAGE_OWNER_STACK_DEPTH (16) 22 #define PAGE_OWNER_STACK_DEPTH (16)
23 23
24 struct page_owner { 24 struct page_owner {
25 unsigned short order; 25 unsigned short order;
26 short last_migrate_reason; 26 short last_migrate_reason;
27 gfp_t gfp_mask; 27 gfp_t gfp_mask;
28 depot_stack_handle_t handle; 28 depot_stack_handle_t handle;
29 depot_stack_handle_t free_handle; 29 depot_stack_handle_t free_handle;
30 u64 ts_nsec; 30 u64 ts_nsec;
31 u64 free_ts_nsec; 31 u64 free_ts_nsec;
32 char comm[TASK_COMM_LEN]; 32 char comm[TASK_COMM_LEN];
33 pid_t pid; 33 pid_t pid;
34 pid_t tgid; 34 pid_t tgid;
35 pid_t free_pid; <<
36 pid_t free_tgid; <<
37 }; 35 };
38 36
39 struct stack { <<
40 struct stack_record *stack_record; <<
41 struct stack *next; <<
42 }; <<
43 static struct stack dummy_stack; <<
44 static struct stack failure_stack; <<
45 static struct stack *stack_list; <<
46 static DEFINE_SPINLOCK(stack_list_lock); <<
47 <<
48 static bool page_owner_enabled __initdata; 37 static bool page_owner_enabled __initdata;
49 DEFINE_STATIC_KEY_FALSE(page_owner_inited); 38 DEFINE_STATIC_KEY_FALSE(page_owner_inited);
50 39
51 static depot_stack_handle_t dummy_handle; 40 static depot_stack_handle_t dummy_handle;
52 static depot_stack_handle_t failure_handle; 41 static depot_stack_handle_t failure_handle;
53 static depot_stack_handle_t early_handle; 42 static depot_stack_handle_t early_handle;
54 43
55 static void init_early_allocated_pages(void); 44 static void init_early_allocated_pages(void);
56 45
57 static inline void set_current_in_page_owner(v <<
58 { <<
59 /* <<
60 * Avoid recursion. <<
61 * <<
62 * We might need to allocate more memo <<
63 * sure to signal it in order to avoid <<
64 */ <<
65 current->in_page_owner = 1; <<
66 } <<
67 <<
68 static inline void unset_current_in_page_owner <<
69 { <<
70 current->in_page_owner = 0; <<
71 } <<
72 <<
73 static int __init early_page_owner_param(char 46 static int __init early_page_owner_param(char *buf)
74 { 47 {
75 int ret = kstrtobool(buf, &page_owner_ 48 int ret = kstrtobool(buf, &page_owner_enabled);
76 49
77 if (page_owner_enabled) 50 if (page_owner_enabled)
78 stack_depot_request_early_init !! 51 stack_depot_want_early_init();
79 52
80 return ret; 53 return ret;
81 } 54 }
82 early_param("page_owner", early_page_owner_par 55 early_param("page_owner", early_page_owner_param);
83 56
84 static __init bool need_page_owner(void) 57 static __init bool need_page_owner(void)
85 { 58 {
86 return page_owner_enabled; 59 return page_owner_enabled;
87 } 60 }
88 61
89 static __always_inline depot_stack_handle_t cr 62 static __always_inline depot_stack_handle_t create_dummy_stack(void)
90 { 63 {
91 unsigned long entries[4]; 64 unsigned long entries[4];
92 unsigned int nr_entries; 65 unsigned int nr_entries;
93 66
94 nr_entries = stack_trace_save(entries, 67 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0);
95 return stack_depot_save(entries, nr_en 68 return stack_depot_save(entries, nr_entries, GFP_KERNEL);
96 } 69 }
97 70
98 static noinline void register_dummy_stack(void 71 static noinline void register_dummy_stack(void)
99 { 72 {
100 dummy_handle = create_dummy_stack(); 73 dummy_handle = create_dummy_stack();
101 } 74 }
102 75
103 static noinline void register_failure_stack(vo 76 static noinline void register_failure_stack(void)
104 { 77 {
105 failure_handle = create_dummy_stack(); 78 failure_handle = create_dummy_stack();
106 } 79 }
107 80
108 static noinline void register_early_stack(void 81 static noinline void register_early_stack(void)
109 { 82 {
110 early_handle = create_dummy_stack(); 83 early_handle = create_dummy_stack();
111 } 84 }
112 85
113 static __init void init_page_owner(void) 86 static __init void init_page_owner(void)
114 { 87 {
115 if (!page_owner_enabled) 88 if (!page_owner_enabled)
116 return; 89 return;
117 90
118 register_dummy_stack(); 91 register_dummy_stack();
119 register_failure_stack(); 92 register_failure_stack();
120 register_early_stack(); 93 register_early_stack();
121 init_early_allocated_pages(); <<
122 /* Initialize dummy and failure stacks <<
123 dummy_stack.stack_record = __stack_dep <<
124 failure_stack.stack_record = __stack_d <<
125 if (dummy_stack.stack_record) <<
126 refcount_set(&dummy_stack.stac <<
127 if (failure_stack.stack_record) <<
128 refcount_set(&failure_stack.st <<
129 dummy_stack.next = &failure_stack; <<
130 stack_list = &dummy_stack; <<
131 static_branch_enable(&page_owner_inite 94 static_branch_enable(&page_owner_inited);
>> 95 init_early_allocated_pages();
132 } 96 }
133 97
134 struct page_ext_operations page_owner_ops = { 98 struct page_ext_operations page_owner_ops = {
135 .size = sizeof(struct page_owner), 99 .size = sizeof(struct page_owner),
136 .need = need_page_owner, 100 .need = need_page_owner,
137 .init = init_page_owner, 101 .init = init_page_owner,
138 .need_shared_flags = true, <<
139 }; 102 };
140 103
141 static inline struct page_owner *get_page_owne 104 static inline struct page_owner *get_page_owner(struct page_ext *page_ext)
142 { 105 {
143 return page_ext_data(page_ext, &page_o !! 106 return (void *)page_ext + page_owner_ops.offset;
144 } 107 }
145 108
146 static noinline depot_stack_handle_t save_stac 109 static noinline depot_stack_handle_t save_stack(gfp_t flags)
147 { 110 {
148 unsigned long entries[PAGE_OWNER_STACK 111 unsigned long entries[PAGE_OWNER_STACK_DEPTH];
149 depot_stack_handle_t handle; 112 depot_stack_handle_t handle;
150 unsigned int nr_entries; 113 unsigned int nr_entries;
151 114
>> 115 /*
>> 116 * Avoid recursion.
>> 117 *
>> 118 * Sometimes page metadata allocation tracking requires more
>> 119 * memory to be allocated:
>> 120 * - when new stack trace is saved to stack depot
>> 121 * - when backtrace itself is calculated (ia64)
>> 122 */
152 if (current->in_page_owner) 123 if (current->in_page_owner)
153 return dummy_handle; 124 return dummy_handle;
>> 125 current->in_page_owner = 1;
154 126
155 set_current_in_page_owner(); <<
156 nr_entries = stack_trace_save(entries, 127 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2);
157 handle = stack_depot_save(entries, nr_ 128 handle = stack_depot_save(entries, nr_entries, flags);
158 if (!handle) 129 if (!handle)
159 handle = failure_handle; 130 handle = failure_handle;
160 unset_current_in_page_owner(); <<
161 131
>> 132 current->in_page_owner = 0;
162 return handle; 133 return handle;
163 } 134 }
164 135
165 static void add_stack_record_to_list(struct st !! 136 void __reset_page_owner(struct page *page, unsigned short order)
166 gfp_t gfp <<
167 { <<
168 unsigned long flags; <<
169 struct stack *stack; <<
170 <<
171 set_current_in_page_owner(); <<
172 stack = kmalloc(sizeof(*stack), gfp_ne <<
173 if (!stack) { <<
174 unset_current_in_page_owner(); <<
175 return; <<
176 } <<
177 unset_current_in_page_owner(); <<
178 <<
179 stack->stack_record = stack_record; <<
180 stack->next = NULL; <<
181 <<
182 spin_lock_irqsave(&stack_list_lock, fl <<
183 stack->next = stack_list; <<
184 /* <<
185 * This pairs with smp_load_acquire() <<
186 * stack_start(). This guarantees that <<
187 * will see an updated stack_list befo <<
188 * traverse the list. <<
189 */ <<
190 smp_store_release(&stack_list, stack); <<
191 spin_unlock_irqrestore(&stack_list_loc <<
192 } <<
193 <<
194 static void inc_stack_record_count(depot_stack <<
195 int nr_base <<
196 { 137 {
197 struct stack_record *stack_record = __ !! 138 int i;
>> 139 struct page_ext *page_ext;
>> 140 depot_stack_handle_t handle;
>> 141 struct page_owner *page_owner;
>> 142 u64 free_ts_nsec = local_clock();
198 143
199 if (!stack_record) !! 144 page_ext = page_ext_get(page);
>> 145 if (unlikely(!page_ext))
200 return; 146 return;
201 147
202 /* !! 148 handle = save_stack(GFP_NOWAIT | __GFP_NOWARN);
203 * New stack_record's that do not use !! 149 for (i = 0; i < (1 << order); i++) {
204 * with REFCOUNT_SATURATED to catch sp !! 150 __clear_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags);
205 * refcount. !! 151 page_owner = get_page_owner(page_ext);
206 * Since we do not use STACK_DEPOT_FLA !! 152 page_owner->free_handle = handle;
207 * set a refcount of 1 ourselves. !! 153 page_owner->free_ts_nsec = free_ts_nsec;
208 */ !! 154 page_ext = page_ext_next(page_ext);
209 if (refcount_read(&stack_record->count <<
210 int old = REFCOUNT_SATURATED; <<
211 <<
212 if (atomic_try_cmpxchg_relaxed <<
213 /* Add the new stack_r <<
214 add_stack_record_to_li <<
215 } 155 }
216 refcount_add(nr_base_pages, &stack_rec !! 156 page_ext_put(page_ext);
217 } <<
218 <<
219 static void dec_stack_record_count(depot_stack <<
220 int nr_base <<
221 { <<
222 struct stack_record *stack_record = __ <<
223 <<
224 if (!stack_record) <<
225 return; <<
226 <<
227 if (refcount_sub_and_test(nr_base_page <<
228 pr_warn("%s: refcount went to <<
229 handle); <<
230 } 157 }
231 158
232 static inline void __update_page_owner_handle( !! 159 static inline void __set_page_owner_handle(struct page_ext *page_ext,
233 !! 160 depot_stack_handle_t handle,
234 !! 161 unsigned short order, gfp_t gfp_mask)
235 <<
236 <<
237 <<
238 { 162 {
239 int i; <<
240 struct page_owner *page_owner; 163 struct page_owner *page_owner;
>> 164 int i;
241 165
242 for (i = 0; i < (1 << order); i++) { 166 for (i = 0; i < (1 << order); i++) {
243 page_owner = get_page_owner(pa 167 page_owner = get_page_owner(page_ext);
244 page_owner->handle = handle; 168 page_owner->handle = handle;
245 page_owner->order = order; 169 page_owner->order = order;
246 page_owner->gfp_mask = gfp_mas 170 page_owner->gfp_mask = gfp_mask;
247 page_owner->last_migrate_reaso !! 171 page_owner->last_migrate_reason = -1;
248 page_owner->pid = pid; !! 172 page_owner->pid = current->pid;
249 page_owner->tgid = tgid; !! 173 page_owner->tgid = current->tgid;
250 page_owner->ts_nsec = ts_nsec; !! 174 page_owner->ts_nsec = local_clock();
251 strscpy(page_owner->comm, comm !! 175 strscpy(page_owner->comm, current->comm,
252 sizeof(page_owner->com 176 sizeof(page_owner->comm));
253 __set_bit(PAGE_EXT_OWNER, &pag 177 __set_bit(PAGE_EXT_OWNER, &page_ext->flags);
254 __set_bit(PAGE_EXT_OWNER_ALLOC 178 __set_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags);
255 page_ext = page_ext_next(page_ <<
256 } <<
257 } <<
258 179
259 static inline void __update_page_owner_free_ha <<
260 <<
261 <<
262 <<
263 <<
264 { <<
265 int i; <<
266 struct page_owner *page_owner; <<
267 <<
268 for (i = 0; i < (1 << order); i++) { <<
269 page_owner = get_page_owner(pa <<
270 /* Only __reset_page_owner() w <<
271 if (handle) { <<
272 __clear_bit(PAGE_EXT_O <<
273 page_owner->free_handl <<
274 } <<
275 page_owner->free_ts_nsec = fre <<
276 page_owner->free_pid = current <<
277 page_owner->free_tgid = curren <<
278 page_ext = page_ext_next(page_ 180 page_ext = page_ext_next(page_ext);
279 } 181 }
280 } 182 }
281 183
282 void __reset_page_owner(struct page *page, uns <<
283 { <<
284 struct page_ext *page_ext; <<
285 depot_stack_handle_t handle; <<
286 depot_stack_handle_t alloc_handle; <<
287 struct page_owner *page_owner; <<
288 u64 free_ts_nsec = local_clock(); <<
289 <<
290 page_ext = page_ext_get(page); <<
291 if (unlikely(!page_ext)) <<
292 return; <<
293 <<
294 page_owner = get_page_owner(page_ext); <<
295 alloc_handle = page_owner->handle; <<
296 <<
297 handle = save_stack(GFP_NOWAIT | __GFP <<
298 __update_page_owner_free_handle(page_e <<
299 curren <<
300 page_ext_put(page_ext); <<
301 <<
302 if (alloc_handle != early_handle) <<
303 /* <<
304 * early_handle is being set a <<
305 * early allocated pages. See <<
306 * Since their refcount is not <<
307 * the machinery is not ready <<
308 * their refcount either. <<
309 */ <<
310 dec_stack_record_count(alloc_h <<
311 } <<
312 <<
313 noinline void __set_page_owner(struct page *pa 184 noinline void __set_page_owner(struct page *page, unsigned short order,
314 gfp_t 185 gfp_t gfp_mask)
315 { 186 {
316 struct page_ext *page_ext; 187 struct page_ext *page_ext;
317 u64 ts_nsec = local_clock(); <<
318 depot_stack_handle_t handle; 188 depot_stack_handle_t handle;
319 189
320 handle = save_stack(gfp_mask); 190 handle = save_stack(gfp_mask);
321 191
322 page_ext = page_ext_get(page); 192 page_ext = page_ext_get(page);
323 if (unlikely(!page_ext)) 193 if (unlikely(!page_ext))
324 return; 194 return;
325 __update_page_owner_handle(page_ext, h !! 195 __set_page_owner_handle(page_ext, handle, order, gfp_mask);
326 ts_nsec, cu <<
327 current->co <<
328 page_ext_put(page_ext); 196 page_ext_put(page_ext);
329 inc_stack_record_count(handle, gfp_mas <<
330 } 197 }
331 198
332 void __set_page_owner_migrate_reason(struct pa 199 void __set_page_owner_migrate_reason(struct page *page, int reason)
333 { 200 {
334 struct page_ext *page_ext = page_ext_g 201 struct page_ext *page_ext = page_ext_get(page);
335 struct page_owner *page_owner; 202 struct page_owner *page_owner;
336 203
337 if (unlikely(!page_ext)) 204 if (unlikely(!page_ext))
338 return; 205 return;
339 206
340 page_owner = get_page_owner(page_ext); 207 page_owner = get_page_owner(page_ext);
341 page_owner->last_migrate_reason = reas 208 page_owner->last_migrate_reason = reason;
342 page_ext_put(page_ext); 209 page_ext_put(page_ext);
343 } 210 }
344 211
345 void __split_page_owner(struct page *page, int !! 212 void __split_page_owner(struct page *page, unsigned int nr)
346 { 213 {
347 int i; 214 int i;
348 struct page_ext *page_ext = page_ext_g 215 struct page_ext *page_ext = page_ext_get(page);
349 struct page_owner *page_owner; 216 struct page_owner *page_owner;
350 217
351 if (unlikely(!page_ext)) 218 if (unlikely(!page_ext))
352 return; 219 return;
353 220
354 for (i = 0; i < (1 << old_order); i++) !! 221 for (i = 0; i < nr; i++) {
355 page_owner = get_page_owner(pa 222 page_owner = get_page_owner(page_ext);
356 page_owner->order = new_order; !! 223 page_owner->order = 0;
357 page_ext = page_ext_next(page_ 224 page_ext = page_ext_next(page_ext);
358 } 225 }
359 page_ext_put(page_ext); 226 page_ext_put(page_ext);
360 } 227 }
361 228
362 void __folio_copy_owner(struct folio *newfolio 229 void __folio_copy_owner(struct folio *newfolio, struct folio *old)
363 { 230 {
364 int i; <<
365 struct page_ext *old_ext; 231 struct page_ext *old_ext;
366 struct page_ext *new_ext; 232 struct page_ext *new_ext;
367 struct page_owner *old_page_owner; !! 233 struct page_owner *old_page_owner, *new_page_owner;
368 struct page_owner *new_page_owner; <<
369 depot_stack_handle_t migrate_handle; <<
370 234
371 old_ext = page_ext_get(&old->page); 235 old_ext = page_ext_get(&old->page);
372 if (unlikely(!old_ext)) 236 if (unlikely(!old_ext))
373 return; 237 return;
374 238
375 new_ext = page_ext_get(&newfolio->page 239 new_ext = page_ext_get(&newfolio->page);
376 if (unlikely(!new_ext)) { 240 if (unlikely(!new_ext)) {
377 page_ext_put(old_ext); 241 page_ext_put(old_ext);
378 return; 242 return;
379 } 243 }
380 244
381 old_page_owner = get_page_owner(old_ex 245 old_page_owner = get_page_owner(old_ext);
382 new_page_owner = get_page_owner(new_ex 246 new_page_owner = get_page_owner(new_ext);
383 migrate_handle = new_page_owner->handl !! 247 new_page_owner->order = old_page_owner->order;
384 __update_page_owner_handle(new_ext, ol !! 248 new_page_owner->gfp_mask = old_page_owner->gfp_mask;
385 old_page_ow !! 249 new_page_owner->last_migrate_reason =
386 old_page_ow !! 250 old_page_owner->last_migrate_reason;
387 old_page_ow !! 251 new_page_owner->handle = old_page_owner->handle;
388 old_page_ow !! 252 new_page_owner->pid = old_page_owner->pid;
389 /* !! 253 new_page_owner->tgid = old_page_owner->tgid;
390 * Do not proactively clear PAGE_EXT_O !! 254 new_page_owner->ts_nsec = old_page_owner->ts_nsec;
391 * will be freed after migration. Keep !! 255 new_page_owner->free_ts_nsec = old_page_owner->ts_nsec;
392 * useful. !! 256 strcpy(new_page_owner->comm, old_page_owner->comm);
393 */ !! 257
394 __update_page_owner_free_handle(new_ex <<
395 old_pa <<
396 old_pa <<
397 old_pa <<
398 /* 258 /*
399 * We linked the original stack to the !! 259 * We don't clear the bit on the old folio as it's going to be freed
400 * for the new one and the old folio o !! 260 * after migration. Until then, the info can be useful in case of
401 * when subtracting those pages from t !! 261 * a bug, and the overall stats will be off a bit only temporarily.
>> 262 * Also, migrate_misplaced_transhuge_page() can still fail the
>> 263 * migration and then we want the old folio to retain the info. But
>> 264 * in that case we also don't need to explicitly clear the info from
>> 265 * the new page, which will be freed.
402 */ 266 */
403 for (i = 0; i < (1 << new_page_owner-> !! 267 __set_bit(PAGE_EXT_OWNER, &new_ext->flags);
404 old_page_owner->handle = migra !! 268 __set_bit(PAGE_EXT_OWNER_ALLOCATED, &new_ext->flags);
405 old_ext = page_ext_next(old_ex <<
406 old_page_owner = get_page_owne <<
407 } <<
408 <<
409 page_ext_put(new_ext); 269 page_ext_put(new_ext);
410 page_ext_put(old_ext); 270 page_ext_put(old_ext);
411 } 271 }
412 272
413 void pagetypeinfo_showmixedcount_print(struct 273 void pagetypeinfo_showmixedcount_print(struct seq_file *m,
414 pg_data 274 pg_data_t *pgdat, struct zone *zone)
415 { 275 {
416 struct page *page; 276 struct page *page;
417 struct page_ext *page_ext; 277 struct page_ext *page_ext;
418 struct page_owner *page_owner; 278 struct page_owner *page_owner;
419 unsigned long pfn, block_end_pfn; 279 unsigned long pfn, block_end_pfn;
420 unsigned long end_pfn = zone_end_pfn(z 280 unsigned long end_pfn = zone_end_pfn(zone);
421 unsigned long count[MIGRATE_TYPES] = { 281 unsigned long count[MIGRATE_TYPES] = { 0, };
422 int pageblock_mt, page_mt; 282 int pageblock_mt, page_mt;
423 int i; 283 int i;
424 284
425 /* Scan block by block. First and last 285 /* Scan block by block. First and last block may be incomplete */
426 pfn = zone->zone_start_pfn; 286 pfn = zone->zone_start_pfn;
427 287
428 /* 288 /*
429 * Walk the zone in pageblock_nr_pages 289 * Walk the zone in pageblock_nr_pages steps. If a page block spans
430 * a zone boundary, it will be double 290 * a zone boundary, it will be double counted between zones. This does
431 * not matter as the mixed block count 291 * not matter as the mixed block count will still be correct
432 */ 292 */
433 for (; pfn < end_pfn; ) { 293 for (; pfn < end_pfn; ) {
434 page = pfn_to_online_page(pfn) 294 page = pfn_to_online_page(pfn);
435 if (!page) { 295 if (!page) {
436 pfn = ALIGN(pfn + 1, M 296 pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES);
437 continue; 297 continue;
438 } 298 }
439 299
440 block_end_pfn = pageblock_end_ 300 block_end_pfn = pageblock_end_pfn(pfn);
441 block_end_pfn = min(block_end_ 301 block_end_pfn = min(block_end_pfn, end_pfn);
442 302
443 pageblock_mt = get_pageblock_m 303 pageblock_mt = get_pageblock_migratetype(page);
444 304
445 for (; pfn < block_end_pfn; pf 305 for (; pfn < block_end_pfn; pfn++) {
446 /* The pageblock is on 306 /* The pageblock is online, no need to recheck. */
447 page = pfn_to_page(pfn 307 page = pfn_to_page(pfn);
448 308
449 if (page_zone(page) != 309 if (page_zone(page) != zone)
450 continue; 310 continue;
451 311
452 if (PageBuddy(page)) { 312 if (PageBuddy(page)) {
453 unsigned long 313 unsigned long freepage_order;
454 314
455 freepage_order 315 freepage_order = buddy_order_unsafe(page);
456 if (freepage_o !! 316 if (freepage_order < MAX_ORDER)
457 pfn += 317 pfn += (1UL << freepage_order) - 1;
458 continue; 318 continue;
459 } 319 }
460 320
461 if (PageReserved(page) 321 if (PageReserved(page))
462 continue; 322 continue;
463 323
464 page_ext = page_ext_ge 324 page_ext = page_ext_get(page);
465 if (unlikely(!page_ext 325 if (unlikely(!page_ext))
466 continue; 326 continue;
467 327
468 if (!test_bit(PAGE_EXT 328 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))
469 goto ext_put_c 329 goto ext_put_continue;
470 330
471 page_owner = get_page_ 331 page_owner = get_page_owner(page_ext);
472 page_mt = gfp_migratet 332 page_mt = gfp_migratetype(page_owner->gfp_mask);
473 if (pageblock_mt != pa 333 if (pageblock_mt != page_mt) {
474 if (is_migrate 334 if (is_migrate_cma(pageblock_mt))
475 count[ 335 count[MIGRATE_MOVABLE]++;
476 else 336 else
477 count[ 337 count[pageblock_mt]++;
478 338
479 pfn = block_en 339 pfn = block_end_pfn;
480 page_ext_put(p 340 page_ext_put(page_ext);
481 break; 341 break;
482 } 342 }
483 pfn += (1UL << page_ow 343 pfn += (1UL << page_owner->order) - 1;
484 ext_put_continue: 344 ext_put_continue:
485 page_ext_put(page_ext) 345 page_ext_put(page_ext);
486 } 346 }
487 } 347 }
488 348
489 /* Print counts */ 349 /* Print counts */
490 seq_printf(m, "Node %d, zone %8s ", pg 350 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
491 for (i = 0; i < MIGRATE_TYPES; i++) 351 for (i = 0; i < MIGRATE_TYPES; i++)
492 seq_printf(m, "%12lu ", count[ 352 seq_printf(m, "%12lu ", count[i]);
493 seq_putc(m, '\n'); 353 seq_putc(m, '\n');
494 } 354 }
495 355
496 /* 356 /*
497 * Looking for memcg information and print it 357 * Looking for memcg information and print it out
498 */ 358 */
499 static inline int print_page_owner_memcg(char 359 static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret,
500 struc 360 struct page *page)
501 { 361 {
502 #ifdef CONFIG_MEMCG 362 #ifdef CONFIG_MEMCG
503 unsigned long memcg_data; 363 unsigned long memcg_data;
504 struct mem_cgroup *memcg; 364 struct mem_cgroup *memcg;
505 bool online; 365 bool online;
506 char name[80]; 366 char name[80];
507 367
508 rcu_read_lock(); 368 rcu_read_lock();
509 memcg_data = READ_ONCE(page->memcg_dat 369 memcg_data = READ_ONCE(page->memcg_data);
510 if (!memcg_data) 370 if (!memcg_data)
511 goto out_unlock; 371 goto out_unlock;
512 372
513 if (memcg_data & MEMCG_DATA_OBJEXTS) !! 373 if (memcg_data & MEMCG_DATA_OBJCGS)
514 ret += scnprintf(kbuf + ret, c 374 ret += scnprintf(kbuf + ret, count - ret,
515 "Slab cache pa 375 "Slab cache page\n");
516 376
517 memcg = page_memcg_check(page); 377 memcg = page_memcg_check(page);
518 if (!memcg) 378 if (!memcg)
519 goto out_unlock; 379 goto out_unlock;
520 380
521 online = (memcg->css.flags & CSS_ONLIN 381 online = (memcg->css.flags & CSS_ONLINE);
522 cgroup_name(memcg->css.cgroup, name, s 382 cgroup_name(memcg->css.cgroup, name, sizeof(name));
523 ret += scnprintf(kbuf + ret, count - r 383 ret += scnprintf(kbuf + ret, count - ret,
524 "Charged %sto %smemcg 384 "Charged %sto %smemcg %s\n",
525 PageMemcgKmem(page) ? 385 PageMemcgKmem(page) ? "(via objcg) " : "",
526 online ? "" : "offline 386 online ? "" : "offline ",
527 name); 387 name);
528 out_unlock: 388 out_unlock:
529 rcu_read_unlock(); 389 rcu_read_unlock();
530 #endif /* CONFIG_MEMCG */ 390 #endif /* CONFIG_MEMCG */
531 391
532 return ret; 392 return ret;
533 } 393 }
534 394
535 static ssize_t 395 static ssize_t
536 print_page_owner(char __user *buf, size_t coun 396 print_page_owner(char __user *buf, size_t count, unsigned long pfn,
537 struct page *page, struct page 397 struct page *page, struct page_owner *page_owner,
538 depot_stack_handle_t handle) 398 depot_stack_handle_t handle)
539 { 399 {
540 int ret, pageblock_mt, page_mt; 400 int ret, pageblock_mt, page_mt;
541 char *kbuf; 401 char *kbuf;
542 402
543 count = min_t(size_t, count, PAGE_SIZE 403 count = min_t(size_t, count, PAGE_SIZE);
544 kbuf = kmalloc(count, GFP_KERNEL); 404 kbuf = kmalloc(count, GFP_KERNEL);
545 if (!kbuf) 405 if (!kbuf)
546 return -ENOMEM; 406 return -ENOMEM;
547 407
548 ret = scnprintf(kbuf, count, 408 ret = scnprintf(kbuf, count,
549 "Page allocated via or !! 409 "Page allocated via order %u, mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu ns, free_ts %llu ns\n",
550 page_owner->order, pag 410 page_owner->order, page_owner->gfp_mask,
551 &page_owner->gfp_mask, 411 &page_owner->gfp_mask, page_owner->pid,
552 page_owner->tgid, page 412 page_owner->tgid, page_owner->comm,
553 page_owner->ts_nsec); !! 413 page_owner->ts_nsec, page_owner->free_ts_nsec);
554 414
555 /* Print information relevant to group 415 /* Print information relevant to grouping pages by mobility */
556 pageblock_mt = get_pageblock_migratety 416 pageblock_mt = get_pageblock_migratetype(page);
557 page_mt = gfp_migratetype(page_owner- 417 page_mt = gfp_migratetype(page_owner->gfp_mask);
558 ret += scnprintf(kbuf + ret, count - r 418 ret += scnprintf(kbuf + ret, count - ret,
559 "PFN 0x%lx type %s Blo !! 419 "PFN %lu type %s Block %lu type %s Flags %pGp\n",
560 pfn, 420 pfn,
561 migratetype_names[page 421 migratetype_names[page_mt],
562 pfn >> pageblock_order 422 pfn >> pageblock_order,
563 migratetype_names[page 423 migratetype_names[pageblock_mt],
564 &page->flags); 424 &page->flags);
565 425
566 ret += stack_depot_snprint(handle, kbu 426 ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0);
567 if (ret >= count) 427 if (ret >= count)
568 goto err; 428 goto err;
569 429
570 if (page_owner->last_migrate_reason != 430 if (page_owner->last_migrate_reason != -1) {
571 ret += scnprintf(kbuf + ret, c 431 ret += scnprintf(kbuf + ret, count - ret,
572 "Page has been migrate 432 "Page has been migrated, last migrate reason: %s\n",
573 migrate_reason_names[p 433 migrate_reason_names[page_owner->last_migrate_reason]);
574 } 434 }
575 435
576 ret = print_page_owner_memcg(kbuf, cou 436 ret = print_page_owner_memcg(kbuf, count, ret, page);
577 437
578 ret += snprintf(kbuf + ret, count - re 438 ret += snprintf(kbuf + ret, count - ret, "\n");
579 if (ret >= count) 439 if (ret >= count)
580 goto err; 440 goto err;
581 441
582 if (copy_to_user(buf, kbuf, ret)) 442 if (copy_to_user(buf, kbuf, ret))
583 ret = -EFAULT; 443 ret = -EFAULT;
584 444
585 kfree(kbuf); 445 kfree(kbuf);
586 return ret; 446 return ret;
587 447
588 err: 448 err:
589 kfree(kbuf); 449 kfree(kbuf);
590 return -ENOMEM; 450 return -ENOMEM;
591 } 451 }
592 452
593 void __dump_page_owner(const struct page *page 453 void __dump_page_owner(const struct page *page)
594 { 454 {
595 struct page_ext *page_ext = page_ext_g 455 struct page_ext *page_ext = page_ext_get((void *)page);
596 struct page_owner *page_owner; 456 struct page_owner *page_owner;
597 depot_stack_handle_t handle; 457 depot_stack_handle_t handle;
598 gfp_t gfp_mask; 458 gfp_t gfp_mask;
599 int mt; 459 int mt;
600 460
601 if (unlikely(!page_ext)) { 461 if (unlikely(!page_ext)) {
602 pr_alert("There is not page ex 462 pr_alert("There is not page extension available.\n");
603 return; 463 return;
604 } 464 }
605 465
606 page_owner = get_page_owner(page_ext); 466 page_owner = get_page_owner(page_ext);
607 gfp_mask = page_owner->gfp_mask; 467 gfp_mask = page_owner->gfp_mask;
608 mt = gfp_migratetype(gfp_mask); 468 mt = gfp_migratetype(gfp_mask);
609 469
610 if (!test_bit(PAGE_EXT_OWNER, &page_ex 470 if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) {
611 pr_alert("page_owner info is n 471 pr_alert("page_owner info is not present (never set?)\n");
612 page_ext_put(page_ext); 472 page_ext_put(page_ext);
613 return; 473 return;
614 } 474 }
615 475
616 if (test_bit(PAGE_EXT_OWNER_ALLOCATED, 476 if (test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))
617 pr_alert("page_owner tracks th 477 pr_alert("page_owner tracks the page as allocated\n");
618 else 478 else
619 pr_alert("page_owner tracks th 479 pr_alert("page_owner tracks the page as freed\n");
620 480
621 pr_alert("page last allocated via orde 481 pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu, free_ts %llu\n",
622 page_owner->order, migratetyp 482 page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask,
623 page_owner->pid, page_owner-> 483 page_owner->pid, page_owner->tgid, page_owner->comm,
624 page_owner->ts_nsec, page_own 484 page_owner->ts_nsec, page_owner->free_ts_nsec);
625 485
626 handle = READ_ONCE(page_owner->handle) 486 handle = READ_ONCE(page_owner->handle);
627 if (!handle) 487 if (!handle)
628 pr_alert("page_owner allocatio 488 pr_alert("page_owner allocation stack trace missing\n");
629 else 489 else
630 stack_depot_print(handle); 490 stack_depot_print(handle);
631 491
632 handle = READ_ONCE(page_owner->free_ha 492 handle = READ_ONCE(page_owner->free_handle);
633 if (!handle) { 493 if (!handle) {
634 pr_alert("page_owner free stac 494 pr_alert("page_owner free stack trace missing\n");
635 } else { 495 } else {
636 pr_alert("page last free pid % !! 496 pr_alert("page last free stack trace:\n");
637 page_owner->free_pid <<
638 stack_depot_print(handle); 497 stack_depot_print(handle);
639 } 498 }
640 499
641 if (page_owner->last_migrate_reason != 500 if (page_owner->last_migrate_reason != -1)
642 pr_alert("page has been migrat 501 pr_alert("page has been migrated, last migrate reason: %s\n",
643 migrate_reason_names[p 502 migrate_reason_names[page_owner->last_migrate_reason]);
644 page_ext_put(page_ext); 503 page_ext_put(page_ext);
645 } 504 }
646 505
647 static ssize_t 506 static ssize_t
648 read_page_owner(struct file *file, char __user 507 read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos)
649 { 508 {
650 unsigned long pfn; 509 unsigned long pfn;
651 struct page *page; 510 struct page *page;
652 struct page_ext *page_ext; 511 struct page_ext *page_ext;
653 struct page_owner *page_owner; 512 struct page_owner *page_owner;
654 depot_stack_handle_t handle; 513 depot_stack_handle_t handle;
655 514
656 if (!static_branch_unlikely(&page_owne 515 if (!static_branch_unlikely(&page_owner_inited))
657 return -EINVAL; 516 return -EINVAL;
658 517
659 page = NULL; 518 page = NULL;
660 if (*ppos == 0) 519 if (*ppos == 0)
661 pfn = min_low_pfn; 520 pfn = min_low_pfn;
662 else 521 else
663 pfn = *ppos; 522 pfn = *ppos;
664 /* Find a valid PFN or the start of a 523 /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */
665 while (!pfn_valid(pfn) && (pfn & (MAX_ 524 while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0)
666 pfn++; 525 pfn++;
667 526
668 /* Find an allocated page */ 527 /* Find an allocated page */
669 for (; pfn < max_pfn; pfn++) { 528 for (; pfn < max_pfn; pfn++) {
670 /* 529 /*
671 * This temporary page_owner i 530 * This temporary page_owner is required so
672 * that we can avoid the conte 531 * that we can avoid the context switches while holding
673 * the rcu lock and copying th 532 * the rcu lock and copying the page owner information to
674 * user through copy_to_user() 533 * user through copy_to_user() or GFP_KERNEL allocations.
675 */ 534 */
676 struct page_owner page_owner_t 535 struct page_owner page_owner_tmp;
677 536
678 /* 537 /*
679 * If the new page is in a new 538 * If the new page is in a new MAX_ORDER_NR_PAGES area,
680 * validate the area as existi 539 * validate the area as existing, skip it if not
681 */ 540 */
682 if ((pfn & (MAX_ORDER_NR_PAGES 541 if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) {
683 pfn += MAX_ORDER_NR_PA 542 pfn += MAX_ORDER_NR_PAGES - 1;
684 continue; 543 continue;
685 } 544 }
686 545
687 page = pfn_to_page(pfn); 546 page = pfn_to_page(pfn);
688 if (PageBuddy(page)) { 547 if (PageBuddy(page)) {
689 unsigned long freepage 548 unsigned long freepage_order = buddy_order_unsafe(page);
690 549
691 if (freepage_order <= !! 550 if (freepage_order < MAX_ORDER)
692 pfn += (1UL << 551 pfn += (1UL << freepage_order) - 1;
693 continue; 552 continue;
694 } 553 }
695 554
696 page_ext = page_ext_get(page); 555 page_ext = page_ext_get(page);
697 if (unlikely(!page_ext)) 556 if (unlikely(!page_ext))
698 continue; 557 continue;
699 558
700 /* 559 /*
701 * Some pages could be missed 560 * Some pages could be missed by concurrent allocation or free,
702 * because we don't hold the z 561 * because we don't hold the zone lock.
703 */ 562 */
704 if (!test_bit(PAGE_EXT_OWNER, 563 if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags))
705 goto ext_put_continue; 564 goto ext_put_continue;
706 565
707 /* 566 /*
708 * Although we do have the inf 567 * Although we do have the info about past allocation of free
709 * pages, it's not relevant fo 568 * pages, it's not relevant for current memory usage.
710 */ 569 */
711 if (!test_bit(PAGE_EXT_OWNER_A 570 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))
712 goto ext_put_continue; 571 goto ext_put_continue;
713 572
714 page_owner = get_page_owner(pa 573 page_owner = get_page_owner(page_ext);
715 574
716 /* 575 /*
717 * Don't print "tail" pages of 576 * Don't print "tail" pages of high-order allocations as that
718 * would inflate the stats. 577 * would inflate the stats.
719 */ 578 */
720 if (!IS_ALIGNED(pfn, 1 << page 579 if (!IS_ALIGNED(pfn, 1 << page_owner->order))
721 goto ext_put_continue; 580 goto ext_put_continue;
722 581
723 /* 582 /*
724 * Access to page_ext->handle 583 * Access to page_ext->handle isn't synchronous so we should
725 * be careful to access it. 584 * be careful to access it.
726 */ 585 */
727 handle = READ_ONCE(page_owner- 586 handle = READ_ONCE(page_owner->handle);
728 if (!handle) 587 if (!handle)
729 goto ext_put_continue; 588 goto ext_put_continue;
730 589
731 /* Record the next PFN to read 590 /* Record the next PFN to read in the file offset */
732 *ppos = pfn + 1; 591 *ppos = pfn + 1;
733 592
734 page_owner_tmp = *page_owner; 593 page_owner_tmp = *page_owner;
735 page_ext_put(page_ext); 594 page_ext_put(page_ext);
736 return print_page_owner(buf, c 595 return print_page_owner(buf, count, pfn, page,
737 &page_owner_tm 596 &page_owner_tmp, handle);
738 ext_put_continue: 597 ext_put_continue:
739 page_ext_put(page_ext); 598 page_ext_put(page_ext);
740 } 599 }
741 600
742 return 0; 601 return 0;
743 } 602 }
744 603
745 static loff_t lseek_page_owner(struct file *fi 604 static loff_t lseek_page_owner(struct file *file, loff_t offset, int orig)
746 { 605 {
747 switch (orig) { 606 switch (orig) {
748 case SEEK_SET: 607 case SEEK_SET:
749 file->f_pos = offset; 608 file->f_pos = offset;
750 break; 609 break;
751 case SEEK_CUR: 610 case SEEK_CUR:
752 file->f_pos += offset; 611 file->f_pos += offset;
753 break; 612 break;
754 default: 613 default:
755 return -EINVAL; 614 return -EINVAL;
756 } 615 }
757 return file->f_pos; 616 return file->f_pos;
758 } 617 }
759 618
760 static void init_pages_in_zone(pg_data_t *pgda 619 static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone)
761 { 620 {
762 unsigned long pfn = zone->zone_start_p 621 unsigned long pfn = zone->zone_start_pfn;
763 unsigned long end_pfn = zone_end_pfn(z 622 unsigned long end_pfn = zone_end_pfn(zone);
764 unsigned long count = 0; 623 unsigned long count = 0;
765 624
766 /* 625 /*
767 * Walk the zone in pageblock_nr_pages 626 * Walk the zone in pageblock_nr_pages steps. If a page block spans
768 * a zone boundary, it will be double 627 * a zone boundary, it will be double counted between zones. This does
769 * not matter as the mixed block count 628 * not matter as the mixed block count will still be correct
770 */ 629 */
771 for (; pfn < end_pfn; ) { 630 for (; pfn < end_pfn; ) {
772 unsigned long block_end_pfn; 631 unsigned long block_end_pfn;
773 632
774 if (!pfn_valid(pfn)) { 633 if (!pfn_valid(pfn)) {
775 pfn = ALIGN(pfn + 1, M 634 pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES);
776 continue; 635 continue;
777 } 636 }
778 637
779 block_end_pfn = pageblock_end_ 638 block_end_pfn = pageblock_end_pfn(pfn);
780 block_end_pfn = min(block_end_ 639 block_end_pfn = min(block_end_pfn, end_pfn);
781 640
782 for (; pfn < block_end_pfn; pf 641 for (; pfn < block_end_pfn; pfn++) {
783 struct page *page = pf 642 struct page *page = pfn_to_page(pfn);
784 struct page_ext *page_ 643 struct page_ext *page_ext;
785 644
786 if (page_zone(page) != 645 if (page_zone(page) != zone)
787 continue; 646 continue;
788 647
789 /* 648 /*
790 * To avoid having to 649 * To avoid having to grab zone->lock, be a little
791 * careful when readin 650 * careful when reading buddy page order. The only
792 * danger is that we s 651 * danger is that we skip too much and potentially miss
793 * some early allocate 652 * some early allocated pages, which is better than
794 * heavy lock contenti 653 * heavy lock contention.
795 */ 654 */
796 if (PageBuddy(page)) { 655 if (PageBuddy(page)) {
797 unsigned long 656 unsigned long order = buddy_order_unsafe(page);
798 657
799 if (order > 0 !! 658 if (order > 0 && order < MAX_ORDER)
800 pfn += 659 pfn += (1UL << order) - 1;
801 continue; 660 continue;
802 } 661 }
803 662
804 if (PageReserved(page) 663 if (PageReserved(page))
805 continue; 664 continue;
806 665
807 page_ext = page_ext_ge 666 page_ext = page_ext_get(page);
808 if (unlikely(!page_ext 667 if (unlikely(!page_ext))
809 continue; 668 continue;
810 669
811 /* Maybe overlapping z 670 /* Maybe overlapping zone */
812 if (test_bit(PAGE_EXT_ 671 if (test_bit(PAGE_EXT_OWNER, &page_ext->flags))
813 goto ext_put_c 672 goto ext_put_continue;
814 673
815 /* Found early allocat 674 /* Found early allocated page */
816 __update_page_owner_ha !! 675 __set_page_owner_handle(page_ext, early_handle,
817 !! 676 0, 0);
818 <<
819 count++; 677 count++;
820 ext_put_continue: 678 ext_put_continue:
821 page_ext_put(page_ext) 679 page_ext_put(page_ext);
822 } 680 }
823 cond_resched(); 681 cond_resched();
824 } 682 }
825 683
826 pr_info("Node %d, zone %8s: page owner 684 pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n",
827 pgdat->node_id, zone->name, co 685 pgdat->node_id, zone->name, count);
828 } 686 }
829 687
830 static void init_zones_in_node(pg_data_t *pgda 688 static void init_zones_in_node(pg_data_t *pgdat)
831 { 689 {
832 struct zone *zone; 690 struct zone *zone;
833 struct zone *node_zones = pgdat->node_ 691 struct zone *node_zones = pgdat->node_zones;
834 692
835 for (zone = node_zones; zone - node_zo 693 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
836 if (!populated_zone(zone)) 694 if (!populated_zone(zone))
837 continue; 695 continue;
838 696
839 init_pages_in_zone(pgdat, zone 697 init_pages_in_zone(pgdat, zone);
840 } 698 }
841 } 699 }
842 700
843 static void init_early_allocated_pages(void) 701 static void init_early_allocated_pages(void)
844 { 702 {
845 pg_data_t *pgdat; 703 pg_data_t *pgdat;
846 704
847 for_each_online_pgdat(pgdat) 705 for_each_online_pgdat(pgdat)
848 init_zones_in_node(pgdat); 706 init_zones_in_node(pgdat);
849 } 707 }
850 708
851 static const struct file_operations proc_page_ 709 static const struct file_operations proc_page_owner_operations = {
852 .read = read_page_owner, 710 .read = read_page_owner,
853 .llseek = lseek_page_owner, 711 .llseek = lseek_page_owner,
854 }; 712 };
855 713
856 static void *stack_start(struct seq_file *m, l <<
857 { <<
858 struct stack *stack; <<
859 <<
860 if (*ppos == -1UL) <<
861 return NULL; <<
862 <<
863 if (!*ppos) { <<
864 /* <<
865 * This pairs with smp_store_r <<
866 * add_stack_record_to_list(), <<
867 * value of stack_list. <<
868 */ <<
869 stack = smp_load_acquire(&stac <<
870 m->private = stack; <<
871 } else { <<
872 stack = m->private; <<
873 } <<
874 <<
875 return stack; <<
876 } <<
877 <<
878 static void *stack_next(struct seq_file *m, vo <<
879 { <<
880 struct stack *stack = v; <<
881 <<
882 stack = stack->next; <<
883 *ppos = stack ? *ppos + 1 : -1UL; <<
884 m->private = stack; <<
885 <<
886 return stack; <<
887 } <<
888 <<
889 static unsigned long page_owner_pages_threshol <<
890 <<
891 static int stack_print(struct seq_file *m, voi <<
892 { <<
893 int i, nr_base_pages; <<
894 struct stack *stack = v; <<
895 unsigned long *entries; <<
896 unsigned long nr_entries; <<
897 struct stack_record *stack_record = st <<
898 <<
899 if (!stack->stack_record) <<
900 return 0; <<
901 <<
902 nr_entries = stack_record->size; <<
903 entries = stack_record->entries; <<
904 nr_base_pages = refcount_read(&stack_r <<
905 <<
906 if (nr_base_pages < 1 || nr_base_pages <<
907 return 0; <<
908 <<
909 for (i = 0; i < nr_entries; i++) <<
910 seq_printf(m, " %pS\n", (void <<
911 seq_printf(m, "nr_base_pages: %d\n\n", <<
912 <<
913 return 0; <<
914 } <<
915 <<
916 static void stack_stop(struct seq_file *m, voi <<
917 { <<
918 } <<
919 <<
920 static const struct seq_operations page_owner_ <<
921 .start = stack_start, <<
922 .next = stack_next, <<
923 .stop = stack_stop, <<
924 .show = stack_print <<
925 }; <<
926 <<
927 static int page_owner_stack_open(struct inode <<
928 { <<
929 return seq_open_private(file, &page_ow <<
930 } <<
931 <<
932 static const struct file_operations page_owner <<
933 .open = page_owner_stack_ope <<
934 .read = seq_read, <<
935 .llseek = seq_lseek, <<
936 .release = seq_release, <<
937 }; <<
938 <<
939 static int page_owner_threshold_get(void *data <<
940 { <<
941 *val = READ_ONCE(page_owner_pages_thre <<
942 return 0; <<
943 } <<
944 <<
945 static int page_owner_threshold_set(void *data <<
946 { <<
947 WRITE_ONCE(page_owner_pages_threshold, <<
948 return 0; <<
949 } <<
950 <<
951 DEFINE_SIMPLE_ATTRIBUTE(proc_page_owner_thresh <<
952 &page_owner_threshold_ <<
953 <<
954 <<
955 static int __init pageowner_init(void) 714 static int __init pageowner_init(void)
956 { 715 {
957 struct dentry *dir; <<
958 <<
959 if (!static_branch_unlikely(&page_owne 716 if (!static_branch_unlikely(&page_owner_inited)) {
960 pr_info("page_owner is disable 717 pr_info("page_owner is disabled\n");
961 return 0; 718 return 0;
962 } 719 }
963 720
964 debugfs_create_file("page_owner", 0400 721 debugfs_create_file("page_owner", 0400, NULL, NULL,
965 &proc_page_owner_o 722 &proc_page_owner_operations);
966 dir = debugfs_create_dir("page_owner_s <<
967 debugfs_create_file("show_stacks", 040 <<
968 &page_owner_stack_ <<
969 debugfs_create_file("count_threshold", <<
970 &proc_page_owner_t <<
971 723
972 return 0; 724 return 0;
973 } 725 }
974 late_initcall(pageowner_init) 726 late_initcall(pageowner_init)
975 727
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.