1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/debugfs.h> 2 #include <linux/debugfs.h>
3 #include <linux/mm.h> 3 #include <linux/mm.h>
4 #include <linux/slab.h> 4 #include <linux/slab.h>
5 #include <linux/uaccess.h> 5 #include <linux/uaccess.h>
6 #include <linux/memblock.h> 6 #include <linux/memblock.h>
7 #include <linux/stacktrace.h> 7 #include <linux/stacktrace.h>
8 #include <linux/page_owner.h> 8 #include <linux/page_owner.h>
9 #include <linux/jump_label.h> 9 #include <linux/jump_label.h>
10 #include <linux/migrate.h> 10 #include <linux/migrate.h>
11 #include <linux/stackdepot.h> 11 #include <linux/stackdepot.h>
12 #include <linux/seq_file.h> 12 #include <linux/seq_file.h>
13 #include <linux/memcontrol.h> <<
14 #include <linux/sched/clock.h> <<
15 13
16 #include "internal.h" 14 #include "internal.h"
17 15
18 /* 16 /*
19 * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_ 17 * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack)
20 * to use off stack temporal storage 18 * to use off stack temporal storage
21 */ 19 */
22 #define PAGE_OWNER_STACK_DEPTH (16) 20 #define PAGE_OWNER_STACK_DEPTH (16)
23 21
24 struct page_owner { 22 struct page_owner {
25 unsigned short order; 23 unsigned short order;
26 short last_migrate_reason; 24 short last_migrate_reason;
27 gfp_t gfp_mask; 25 gfp_t gfp_mask;
28 depot_stack_handle_t handle; 26 depot_stack_handle_t handle;
29 depot_stack_handle_t free_handle; 27 depot_stack_handle_t free_handle;
30 u64 ts_nsec; <<
31 u64 free_ts_nsec; <<
32 char comm[TASK_COMM_LEN]; <<
33 pid_t pid; <<
34 pid_t tgid; <<
35 pid_t free_pid; <<
36 pid_t free_tgid; <<
37 }; 28 };
38 29
39 struct stack { !! 30 static bool page_owner_enabled = false;
40 struct stack_record *stack_record; <<
41 struct stack *next; <<
42 }; <<
43 static struct stack dummy_stack; <<
44 static struct stack failure_stack; <<
45 static struct stack *stack_list; <<
46 static DEFINE_SPINLOCK(stack_list_lock); <<
47 <<
48 static bool page_owner_enabled __initdata; <<
49 DEFINE_STATIC_KEY_FALSE(page_owner_inited); 31 DEFINE_STATIC_KEY_FALSE(page_owner_inited);
50 32
51 static depot_stack_handle_t dummy_handle; 33 static depot_stack_handle_t dummy_handle;
52 static depot_stack_handle_t failure_handle; 34 static depot_stack_handle_t failure_handle;
53 static depot_stack_handle_t early_handle; 35 static depot_stack_handle_t early_handle;
54 36
55 static void init_early_allocated_pages(void); 37 static void init_early_allocated_pages(void);
56 38
57 static inline void set_current_in_page_owner(v <<
58 { <<
59 /* <<
60 * Avoid recursion. <<
61 * <<
62 * We might need to allocate more memo <<
63 * sure to signal it in order to avoid <<
64 */ <<
65 current->in_page_owner = 1; <<
66 } <<
67 <<
68 static inline void unset_current_in_page_owner <<
69 { <<
70 current->in_page_owner = 0; <<
71 } <<
72 <<
73 static int __init early_page_owner_param(char 39 static int __init early_page_owner_param(char *buf)
74 { 40 {
75 int ret = kstrtobool(buf, &page_owner_ !! 41 if (!buf)
>> 42 return -EINVAL;
76 43
77 if (page_owner_enabled) !! 44 if (strcmp(buf, "on") == 0)
78 stack_depot_request_early_init !! 45 page_owner_enabled = true;
79 46
80 return ret; !! 47 return 0;
81 } 48 }
82 early_param("page_owner", early_page_owner_par 49 early_param("page_owner", early_page_owner_param);
83 50
84 static __init bool need_page_owner(void) !! 51 static bool need_page_owner(void)
85 { 52 {
86 return page_owner_enabled; 53 return page_owner_enabled;
87 } 54 }
88 55
89 static __always_inline depot_stack_handle_t cr 56 static __always_inline depot_stack_handle_t create_dummy_stack(void)
90 { 57 {
91 unsigned long entries[4]; 58 unsigned long entries[4];
92 unsigned int nr_entries; 59 unsigned int nr_entries;
93 60
94 nr_entries = stack_trace_save(entries, 61 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0);
95 return stack_depot_save(entries, nr_en 62 return stack_depot_save(entries, nr_entries, GFP_KERNEL);
96 } 63 }
97 64
98 static noinline void register_dummy_stack(void 65 static noinline void register_dummy_stack(void)
99 { 66 {
100 dummy_handle = create_dummy_stack(); 67 dummy_handle = create_dummy_stack();
101 } 68 }
102 69
103 static noinline void register_failure_stack(vo 70 static noinline void register_failure_stack(void)
104 { 71 {
105 failure_handle = create_dummy_stack(); 72 failure_handle = create_dummy_stack();
106 } 73 }
107 74
108 static noinline void register_early_stack(void 75 static noinline void register_early_stack(void)
109 { 76 {
110 early_handle = create_dummy_stack(); 77 early_handle = create_dummy_stack();
111 } 78 }
112 79
113 static __init void init_page_owner(void) !! 80 static void init_page_owner(void)
114 { 81 {
115 if (!page_owner_enabled) 82 if (!page_owner_enabled)
116 return; 83 return;
117 84
118 register_dummy_stack(); 85 register_dummy_stack();
119 register_failure_stack(); 86 register_failure_stack();
120 register_early_stack(); 87 register_early_stack();
121 init_early_allocated_pages(); <<
122 /* Initialize dummy and failure stacks <<
123 dummy_stack.stack_record = __stack_dep <<
124 failure_stack.stack_record = __stack_d <<
125 if (dummy_stack.stack_record) <<
126 refcount_set(&dummy_stack.stac <<
127 if (failure_stack.stack_record) <<
128 refcount_set(&failure_stack.st <<
129 dummy_stack.next = &failure_stack; <<
130 stack_list = &dummy_stack; <<
131 static_branch_enable(&page_owner_inite 88 static_branch_enable(&page_owner_inited);
>> 89 init_early_allocated_pages();
132 } 90 }
133 91
134 struct page_ext_operations page_owner_ops = { 92 struct page_ext_operations page_owner_ops = {
135 .size = sizeof(struct page_owner), 93 .size = sizeof(struct page_owner),
136 .need = need_page_owner, 94 .need = need_page_owner,
137 .init = init_page_owner, 95 .init = init_page_owner,
138 .need_shared_flags = true, <<
139 }; 96 };
140 97
141 static inline struct page_owner *get_page_owne 98 static inline struct page_owner *get_page_owner(struct page_ext *page_ext)
142 { 99 {
143 return page_ext_data(page_ext, &page_o !! 100 return (void *)page_ext + page_owner_ops.offset;
>> 101 }
>> 102
>> 103 static inline bool check_recursive_alloc(unsigned long *entries,
>> 104 unsigned int nr_entries,
>> 105 unsigned long ip)
>> 106 {
>> 107 unsigned int i;
>> 108
>> 109 for (i = 0; i < nr_entries; i++) {
>> 110 if (entries[i] == ip)
>> 111 return true;
>> 112 }
>> 113 return false;
144 } 114 }
145 115
146 static noinline depot_stack_handle_t save_stac 116 static noinline depot_stack_handle_t save_stack(gfp_t flags)
147 { 117 {
148 unsigned long entries[PAGE_OWNER_STACK 118 unsigned long entries[PAGE_OWNER_STACK_DEPTH];
149 depot_stack_handle_t handle; 119 depot_stack_handle_t handle;
150 unsigned int nr_entries; 120 unsigned int nr_entries;
151 121
152 if (current->in_page_owner) !! 122 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2);
>> 123
>> 124 /*
>> 125 * We need to check recursion here because our request to
>> 126 * stackdepot could trigger memory allocation to save new
>> 127 * entry. New memory allocation would reach here and call
>> 128 * stack_depot_save_entries() again if we don't catch it. There is
>> 129 * still not enough memory in stackdepot so it would try to
>> 130 * allocate memory again and loop forever.
>> 131 */
>> 132 if (check_recursive_alloc(entries, nr_entries, _RET_IP_))
153 return dummy_handle; 133 return dummy_handle;
154 134
155 set_current_in_page_owner(); <<
156 nr_entries = stack_trace_save(entries, <<
157 handle = stack_depot_save(entries, nr_ 135 handle = stack_depot_save(entries, nr_entries, flags);
158 if (!handle) 136 if (!handle)
159 handle = failure_handle; 137 handle = failure_handle;
160 unset_current_in_page_owner(); <<
161 138
162 return handle; 139 return handle;
163 } 140 }
164 141
165 static void add_stack_record_to_list(struct st !! 142 void __reset_page_owner(struct page *page, unsigned int order)
166 gfp_t gfp <<
167 { 143 {
168 unsigned long flags; !! 144 int i;
169 struct stack *stack; !! 145 struct page_ext *page_ext;
170 !! 146 depot_stack_handle_t handle = 0;
171 set_current_in_page_owner(); !! 147 struct page_owner *page_owner;
172 stack = kmalloc(sizeof(*stack), gfp_ne <<
173 if (!stack) { <<
174 unset_current_in_page_owner(); <<
175 return; <<
176 } <<
177 unset_current_in_page_owner(); <<
178 <<
179 stack->stack_record = stack_record; <<
180 stack->next = NULL; <<
181 <<
182 spin_lock_irqsave(&stack_list_lock, fl <<
183 stack->next = stack_list; <<
184 /* <<
185 * This pairs with smp_load_acquire() <<
186 * stack_start(). This guarantees that <<
187 * will see an updated stack_list befo <<
188 * traverse the list. <<
189 */ <<
190 smp_store_release(&stack_list, stack); <<
191 spin_unlock_irqrestore(&stack_list_loc <<
192 } <<
193 148
194 static void inc_stack_record_count(depot_stack !! 149 handle = save_stack(GFP_NOWAIT | __GFP_NOWARN);
195 int nr_base <<
196 { <<
197 struct stack_record *stack_record = __ <<
198 150
199 if (!stack_record) !! 151 page_ext = lookup_page_ext(page);
>> 152 if (unlikely(!page_ext))
200 return; 153 return;
201 !! 154 for (i = 0; i < (1 << order); i++) {
202 /* !! 155 __clear_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags);
203 * New stack_record's that do not use !! 156 page_owner = get_page_owner(page_ext);
204 * with REFCOUNT_SATURATED to catch sp !! 157 page_owner->free_handle = handle;
205 * refcount. !! 158 page_ext = page_ext_next(page_ext);
206 * Since we do not use STACK_DEPOT_FLA <<
207 * set a refcount of 1 ourselves. <<
208 */ <<
209 if (refcount_read(&stack_record->count <<
210 int old = REFCOUNT_SATURATED; <<
211 <<
212 if (atomic_try_cmpxchg_relaxed <<
213 /* Add the new stack_r <<
214 add_stack_record_to_li <<
215 } 159 }
216 refcount_add(nr_base_pages, &stack_rec <<
217 } 160 }
218 161
219 static void dec_stack_record_count(depot_stack !! 162 static inline void __set_page_owner_handle(struct page *page,
220 int nr_base !! 163 struct page_ext *page_ext, depot_stack_handle_t handle,
>> 164 unsigned int order, gfp_t gfp_mask)
221 { 165 {
222 struct stack_record *stack_record = __ <<
223 <<
224 if (!stack_record) <<
225 return; <<
226 <<
227 if (refcount_sub_and_test(nr_base_page <<
228 pr_warn("%s: refcount went to <<
229 handle); <<
230 } <<
231 <<
232 static inline void __update_page_owner_handle( <<
233 <<
234 <<
235 <<
236 <<
237 <<
238 { <<
239 int i; <<
240 struct page_owner *page_owner; 166 struct page_owner *page_owner;
>> 167 int i;
241 168
242 for (i = 0; i < (1 << order); i++) { 169 for (i = 0; i < (1 << order); i++) {
243 page_owner = get_page_owner(pa 170 page_owner = get_page_owner(page_ext);
244 page_owner->handle = handle; 171 page_owner->handle = handle;
245 page_owner->order = order; 172 page_owner->order = order;
246 page_owner->gfp_mask = gfp_mas 173 page_owner->gfp_mask = gfp_mask;
247 page_owner->last_migrate_reaso !! 174 page_owner->last_migrate_reason = -1;
248 page_owner->pid = pid; <<
249 page_owner->tgid = tgid; <<
250 page_owner->ts_nsec = ts_nsec; <<
251 strscpy(page_owner->comm, comm <<
252 sizeof(page_owner->com <<
253 __set_bit(PAGE_EXT_OWNER, &pag 175 __set_bit(PAGE_EXT_OWNER, &page_ext->flags);
254 __set_bit(PAGE_EXT_OWNER_ALLOC 176 __set_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags);
255 page_ext = page_ext_next(page_ <<
256 } <<
257 } <<
258 <<
259 static inline void __update_page_owner_free_ha <<
260 <<
261 <<
262 <<
263 <<
264 { <<
265 int i; <<
266 struct page_owner *page_owner; <<
267 177
268 for (i = 0; i < (1 << order); i++) { <<
269 page_owner = get_page_owner(pa <<
270 /* Only __reset_page_owner() w <<
271 if (handle) { <<
272 __clear_bit(PAGE_EXT_O <<
273 page_owner->free_handl <<
274 } <<
275 page_owner->free_ts_nsec = fre <<
276 page_owner->free_pid = current <<
277 page_owner->free_tgid = curren <<
278 page_ext = page_ext_next(page_ 178 page_ext = page_ext_next(page_ext);
279 } 179 }
280 } 180 }
281 181
282 void __reset_page_owner(struct page *page, uns !! 182 noinline void __set_page_owner(struct page *page, unsigned int order,
>> 183 gfp_t gfp_mask)
283 { 184 {
284 struct page_ext *page_ext; !! 185 struct page_ext *page_ext = lookup_page_ext(page);
285 depot_stack_handle_t handle; 186 depot_stack_handle_t handle;
286 depot_stack_handle_t alloc_handle; <<
287 struct page_owner *page_owner; <<
288 u64 free_ts_nsec = local_clock(); <<
289 187
290 page_ext = page_ext_get(page); <<
291 if (unlikely(!page_ext)) 188 if (unlikely(!page_ext))
292 return; 189 return;
293 190
294 page_owner = get_page_owner(page_ext); <<
295 alloc_handle = page_owner->handle; <<
296 <<
297 handle = save_stack(GFP_NOWAIT | __GFP <<
298 __update_page_owner_free_handle(page_e <<
299 curren <<
300 page_ext_put(page_ext); <<
301 <<
302 if (alloc_handle != early_handle) <<
303 /* <<
304 * early_handle is being set a <<
305 * early allocated pages. See <<
306 * Since their refcount is not <<
307 * the machinery is not ready <<
308 * their refcount either. <<
309 */ <<
310 dec_stack_record_count(alloc_h <<
311 } <<
312 <<
313 noinline void __set_page_owner(struct page *pa <<
314 gfp_t <<
315 { <<
316 struct page_ext *page_ext; <<
317 u64 ts_nsec = local_clock(); <<
318 depot_stack_handle_t handle; <<
319 <<
320 handle = save_stack(gfp_mask); 191 handle = save_stack(gfp_mask);
321 !! 192 __set_page_owner_handle(page, page_ext, handle, order, gfp_mask);
322 page_ext = page_ext_get(page); <<
323 if (unlikely(!page_ext)) <<
324 return; <<
325 __update_page_owner_handle(page_ext, h <<
326 ts_nsec, cu <<
327 current->co <<
328 page_ext_put(page_ext); <<
329 inc_stack_record_count(handle, gfp_mas <<
330 } 193 }
331 194
332 void __set_page_owner_migrate_reason(struct pa 195 void __set_page_owner_migrate_reason(struct page *page, int reason)
333 { 196 {
334 struct page_ext *page_ext = page_ext_g !! 197 struct page_ext *page_ext = lookup_page_ext(page);
335 struct page_owner *page_owner; 198 struct page_owner *page_owner;
336 199
337 if (unlikely(!page_ext)) 200 if (unlikely(!page_ext))
338 return; 201 return;
339 202
340 page_owner = get_page_owner(page_ext); 203 page_owner = get_page_owner(page_ext);
341 page_owner->last_migrate_reason = reas 204 page_owner->last_migrate_reason = reason;
342 page_ext_put(page_ext); <<
343 } 205 }
344 206
345 void __split_page_owner(struct page *page, int !! 207 void __split_page_owner(struct page *page, unsigned int nr)
346 { 208 {
347 int i; 209 int i;
348 struct page_ext *page_ext = page_ext_g !! 210 struct page_ext *page_ext = lookup_page_ext(page);
349 struct page_owner *page_owner; 211 struct page_owner *page_owner;
350 212
351 if (unlikely(!page_ext)) 213 if (unlikely(!page_ext))
352 return; 214 return;
353 215
354 for (i = 0; i < (1 << old_order); i++) !! 216 for (i = 0; i < nr; i++) {
355 page_owner = get_page_owner(pa 217 page_owner = get_page_owner(page_ext);
356 page_owner->order = new_order; !! 218 page_owner->order = 0;
357 page_ext = page_ext_next(page_ 219 page_ext = page_ext_next(page_ext);
358 } 220 }
359 page_ext_put(page_ext); <<
360 } 221 }
361 222
362 void __folio_copy_owner(struct folio *newfolio !! 223 void __copy_page_owner(struct page *oldpage, struct page *newpage)
363 { 224 {
364 int i; !! 225 struct page_ext *old_ext = lookup_page_ext(oldpage);
365 struct page_ext *old_ext; !! 226 struct page_ext *new_ext = lookup_page_ext(newpage);
366 struct page_ext *new_ext; !! 227 struct page_owner *old_page_owner, *new_page_owner;
367 struct page_owner *old_page_owner; <<
368 struct page_owner *new_page_owner; <<
369 depot_stack_handle_t migrate_handle; <<
370 228
371 old_ext = page_ext_get(&old->page); !! 229 if (unlikely(!old_ext || !new_ext))
372 if (unlikely(!old_ext)) <<
373 return; 230 return;
374 231
375 new_ext = page_ext_get(&newfolio->page <<
376 if (unlikely(!new_ext)) { <<
377 page_ext_put(old_ext); <<
378 return; <<
379 } <<
380 <<
381 old_page_owner = get_page_owner(old_ex 232 old_page_owner = get_page_owner(old_ext);
382 new_page_owner = get_page_owner(new_ex 233 new_page_owner = get_page_owner(new_ext);
383 migrate_handle = new_page_owner->handl !! 234 new_page_owner->order = old_page_owner->order;
384 __update_page_owner_handle(new_ext, ol !! 235 new_page_owner->gfp_mask = old_page_owner->gfp_mask;
385 old_page_ow !! 236 new_page_owner->last_migrate_reason =
386 old_page_ow !! 237 old_page_owner->last_migrate_reason;
387 old_page_ow !! 238 new_page_owner->handle = old_page_owner->handle;
388 old_page_ow !! 239
389 /* <<
390 * Do not proactively clear PAGE_EXT_O <<
391 * will be freed after migration. Keep <<
392 * useful. <<
393 */ <<
394 __update_page_owner_free_handle(new_ex <<
395 old_pa <<
396 old_pa <<
397 old_pa <<
398 /* 240 /*
399 * We linked the original stack to the !! 241 * We don't clear the bit on the oldpage as it's going to be freed
400 * for the new one and the old folio o !! 242 * after migration. Until then, the info can be useful in case of
401 * when subtracting those pages from t !! 243 * a bug, and the overal stats will be off a bit only temporarily.
>> 244 * Also, migrate_misplaced_transhuge_page() can still fail the
>> 245 * migration and then we want the oldpage to retain the info. But
>> 246 * in that case we also don't need to explicitly clear the info from
>> 247 * the new page, which will be freed.
402 */ 248 */
403 for (i = 0; i < (1 << new_page_owner-> !! 249 __set_bit(PAGE_EXT_OWNER, &new_ext->flags);
404 old_page_owner->handle = migra !! 250 __set_bit(PAGE_EXT_OWNER_ALLOCATED, &new_ext->flags);
405 old_ext = page_ext_next(old_ex <<
406 old_page_owner = get_page_owne <<
407 } <<
408 <<
409 page_ext_put(new_ext); <<
410 page_ext_put(old_ext); <<
411 } 251 }
412 252
413 void pagetypeinfo_showmixedcount_print(struct 253 void pagetypeinfo_showmixedcount_print(struct seq_file *m,
414 pg_data 254 pg_data_t *pgdat, struct zone *zone)
415 { 255 {
416 struct page *page; 256 struct page *page;
417 struct page_ext *page_ext; 257 struct page_ext *page_ext;
418 struct page_owner *page_owner; 258 struct page_owner *page_owner;
419 unsigned long pfn, block_end_pfn; !! 259 unsigned long pfn = zone->zone_start_pfn, block_end_pfn;
420 unsigned long end_pfn = zone_end_pfn(z !! 260 unsigned long end_pfn = pfn + zone->spanned_pages;
421 unsigned long count[MIGRATE_TYPES] = { 261 unsigned long count[MIGRATE_TYPES] = { 0, };
422 int pageblock_mt, page_mt; 262 int pageblock_mt, page_mt;
423 int i; 263 int i;
424 264
425 /* Scan block by block. First and last 265 /* Scan block by block. First and last block may be incomplete */
426 pfn = zone->zone_start_pfn; 266 pfn = zone->zone_start_pfn;
427 267
428 /* 268 /*
429 * Walk the zone in pageblock_nr_pages 269 * Walk the zone in pageblock_nr_pages steps. If a page block spans
430 * a zone boundary, it will be double 270 * a zone boundary, it will be double counted between zones. This does
431 * not matter as the mixed block count 271 * not matter as the mixed block count will still be correct
432 */ 272 */
433 for (; pfn < end_pfn; ) { 273 for (; pfn < end_pfn; ) {
434 page = pfn_to_online_page(pfn) 274 page = pfn_to_online_page(pfn);
435 if (!page) { 275 if (!page) {
436 pfn = ALIGN(pfn + 1, M 276 pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES);
437 continue; 277 continue;
438 } 278 }
439 279
440 block_end_pfn = pageblock_end_ !! 280 block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
441 block_end_pfn = min(block_end_ 281 block_end_pfn = min(block_end_pfn, end_pfn);
442 282
443 pageblock_mt = get_pageblock_m 283 pageblock_mt = get_pageblock_migratetype(page);
444 284
445 for (; pfn < block_end_pfn; pf 285 for (; pfn < block_end_pfn; pfn++) {
>> 286 if (!pfn_valid_within(pfn))
>> 287 continue;
>> 288
446 /* The pageblock is on 289 /* The pageblock is online, no need to recheck. */
447 page = pfn_to_page(pfn 290 page = pfn_to_page(pfn);
448 291
449 if (page_zone(page) != 292 if (page_zone(page) != zone)
450 continue; 293 continue;
451 294
452 if (PageBuddy(page)) { 295 if (PageBuddy(page)) {
453 unsigned long 296 unsigned long freepage_order;
454 297
455 freepage_order !! 298 freepage_order = page_order_unsafe(page);
456 if (freepage_o !! 299 if (freepage_order < MAX_ORDER)
457 pfn += 300 pfn += (1UL << freepage_order) - 1;
458 continue; 301 continue;
459 } 302 }
460 303
461 if (PageReserved(page) 304 if (PageReserved(page))
462 continue; 305 continue;
463 306
464 page_ext = page_ext_ge !! 307 page_ext = lookup_page_ext(page);
465 if (unlikely(!page_ext 308 if (unlikely(!page_ext))
466 continue; 309 continue;
467 310
468 if (!test_bit(PAGE_EXT 311 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))
469 goto ext_put_c !! 312 continue;
470 313
471 page_owner = get_page_ 314 page_owner = get_page_owner(page_ext);
472 page_mt = gfp_migratet !! 315 page_mt = gfpflags_to_migratetype(
>> 316 page_owner->gfp_mask);
473 if (pageblock_mt != pa 317 if (pageblock_mt != page_mt) {
474 if (is_migrate 318 if (is_migrate_cma(pageblock_mt))
475 count[ 319 count[MIGRATE_MOVABLE]++;
476 else 320 else
477 count[ 321 count[pageblock_mt]++;
478 322
479 pfn = block_en 323 pfn = block_end_pfn;
480 page_ext_put(p <<
481 break; 324 break;
482 } 325 }
483 pfn += (1UL << page_ow 326 pfn += (1UL << page_owner->order) - 1;
484 ext_put_continue: <<
485 page_ext_put(page_ext) <<
486 } 327 }
487 } 328 }
488 329
489 /* Print counts */ 330 /* Print counts */
490 seq_printf(m, "Node %d, zone %8s ", pg 331 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
491 for (i = 0; i < MIGRATE_TYPES; i++) 332 for (i = 0; i < MIGRATE_TYPES; i++)
492 seq_printf(m, "%12lu ", count[ 333 seq_printf(m, "%12lu ", count[i]);
493 seq_putc(m, '\n'); 334 seq_putc(m, '\n');
494 } 335 }
495 336
496 /* <<
497 * Looking for memcg information and print it <<
498 */ <<
499 static inline int print_page_owner_memcg(char <<
500 struc <<
501 { <<
502 #ifdef CONFIG_MEMCG <<
503 unsigned long memcg_data; <<
504 struct mem_cgroup *memcg; <<
505 bool online; <<
506 char name[80]; <<
507 <<
508 rcu_read_lock(); <<
509 memcg_data = READ_ONCE(page->memcg_dat <<
510 if (!memcg_data) <<
511 goto out_unlock; <<
512 <<
513 if (memcg_data & MEMCG_DATA_OBJEXTS) <<
514 ret += scnprintf(kbuf + ret, c <<
515 "Slab cache pa <<
516 <<
517 memcg = page_memcg_check(page); <<
518 if (!memcg) <<
519 goto out_unlock; <<
520 <<
521 online = (memcg->css.flags & CSS_ONLIN <<
522 cgroup_name(memcg->css.cgroup, name, s <<
523 ret += scnprintf(kbuf + ret, count - r <<
524 "Charged %sto %smemcg <<
525 PageMemcgKmem(page) ? <<
526 online ? "" : "offline <<
527 name); <<
528 out_unlock: <<
529 rcu_read_unlock(); <<
530 #endif /* CONFIG_MEMCG */ <<
531 <<
532 return ret; <<
533 } <<
534 <<
535 static ssize_t 337 static ssize_t
536 print_page_owner(char __user *buf, size_t coun 338 print_page_owner(char __user *buf, size_t count, unsigned long pfn,
537 struct page *page, struct page 339 struct page *page, struct page_owner *page_owner,
538 depot_stack_handle_t handle) 340 depot_stack_handle_t handle)
539 { 341 {
540 int ret, pageblock_mt, page_mt; 342 int ret, pageblock_mt, page_mt;
>> 343 unsigned long *entries;
>> 344 unsigned int nr_entries;
541 char *kbuf; 345 char *kbuf;
542 346
543 count = min_t(size_t, count, PAGE_SIZE 347 count = min_t(size_t, count, PAGE_SIZE);
544 kbuf = kmalloc(count, GFP_KERNEL); 348 kbuf = kmalloc(count, GFP_KERNEL);
545 if (!kbuf) 349 if (!kbuf)
546 return -ENOMEM; 350 return -ENOMEM;
547 351
548 ret = scnprintf(kbuf, count, !! 352 ret = snprintf(kbuf, count,
549 "Page allocated via or !! 353 "Page allocated via order %u, mask %#x(%pGg)\n",
550 page_owner->order, pag 354 page_owner->order, page_owner->gfp_mask,
551 &page_owner->gfp_mask, !! 355 &page_owner->gfp_mask);
552 page_owner->tgid, page !! 356
553 page_owner->ts_nsec); !! 357 if (ret >= count)
>> 358 goto err;
554 359
555 /* Print information relevant to group 360 /* Print information relevant to grouping pages by mobility */
556 pageblock_mt = get_pageblock_migratety 361 pageblock_mt = get_pageblock_migratetype(page);
557 page_mt = gfp_migratetype(page_owner- !! 362 page_mt = gfpflags_to_migratetype(page_owner->gfp_mask);
558 ret += scnprintf(kbuf + ret, count - r !! 363 ret += snprintf(kbuf + ret, count - ret,
559 "PFN 0x%lx type %s Blo !! 364 "PFN %lu type %s Block %lu type %s Flags %#lx(%pGp)\n",
560 pfn, 365 pfn,
561 migratetype_names[page 366 migratetype_names[page_mt],
562 pfn >> pageblock_order 367 pfn >> pageblock_order,
563 migratetype_names[page 368 migratetype_names[pageblock_mt],
564 &page->flags); !! 369 page->flags, &page->flags);
>> 370
>> 371 if (ret >= count)
>> 372 goto err;
565 373
566 ret += stack_depot_snprint(handle, kbu !! 374 nr_entries = stack_depot_fetch(handle, &entries);
>> 375 ret += stack_trace_snprint(kbuf + ret, count - ret, entries, nr_entries, 0);
567 if (ret >= count) 376 if (ret >= count)
568 goto err; 377 goto err;
569 378
570 if (page_owner->last_migrate_reason != 379 if (page_owner->last_migrate_reason != -1) {
571 ret += scnprintf(kbuf + ret, c !! 380 ret += snprintf(kbuf + ret, count - ret,
572 "Page has been migrate 381 "Page has been migrated, last migrate reason: %s\n",
573 migrate_reason_names[p 382 migrate_reason_names[page_owner->last_migrate_reason]);
>> 383 if (ret >= count)
>> 384 goto err;
574 } 385 }
575 386
576 ret = print_page_owner_memcg(kbuf, cou <<
577 <<
578 ret += snprintf(kbuf + ret, count - re 387 ret += snprintf(kbuf + ret, count - ret, "\n");
579 if (ret >= count) 388 if (ret >= count)
580 goto err; 389 goto err;
581 390
582 if (copy_to_user(buf, kbuf, ret)) 391 if (copy_to_user(buf, kbuf, ret))
583 ret = -EFAULT; 392 ret = -EFAULT;
584 393
585 kfree(kbuf); 394 kfree(kbuf);
586 return ret; 395 return ret;
587 396
588 err: 397 err:
589 kfree(kbuf); 398 kfree(kbuf);
590 return -ENOMEM; 399 return -ENOMEM;
591 } 400 }
592 401
593 void __dump_page_owner(const struct page *page !! 402 void __dump_page_owner(struct page *page)
594 { 403 {
595 struct page_ext *page_ext = page_ext_g !! 404 struct page_ext *page_ext = lookup_page_ext(page);
596 struct page_owner *page_owner; 405 struct page_owner *page_owner;
597 depot_stack_handle_t handle; 406 depot_stack_handle_t handle;
>> 407 unsigned long *entries;
>> 408 unsigned int nr_entries;
598 gfp_t gfp_mask; 409 gfp_t gfp_mask;
599 int mt; 410 int mt;
600 411
601 if (unlikely(!page_ext)) { 412 if (unlikely(!page_ext)) {
602 pr_alert("There is not page ex 413 pr_alert("There is not page extension available.\n");
603 return; 414 return;
604 } 415 }
605 416
606 page_owner = get_page_owner(page_ext); 417 page_owner = get_page_owner(page_ext);
607 gfp_mask = page_owner->gfp_mask; 418 gfp_mask = page_owner->gfp_mask;
608 mt = gfp_migratetype(gfp_mask); !! 419 mt = gfpflags_to_migratetype(gfp_mask);
609 420
610 if (!test_bit(PAGE_EXT_OWNER, &page_ex 421 if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) {
611 pr_alert("page_owner info is n 422 pr_alert("page_owner info is not present (never set?)\n");
612 page_ext_put(page_ext); <<
613 return; 423 return;
614 } 424 }
615 425
616 if (test_bit(PAGE_EXT_OWNER_ALLOCATED, 426 if (test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))
617 pr_alert("page_owner tracks th 427 pr_alert("page_owner tracks the page as allocated\n");
618 else 428 else
619 pr_alert("page_owner tracks th 429 pr_alert("page_owner tracks the page as freed\n");
620 430
621 pr_alert("page last allocated via orde !! 431 pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg)\n",
622 page_owner->order, migratetyp !! 432 page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask);
623 page_owner->pid, page_owner-> <<
624 page_owner->ts_nsec, page_own <<
625 433
626 handle = READ_ONCE(page_owner->handle) 434 handle = READ_ONCE(page_owner->handle);
627 if (!handle) !! 435 if (!handle) {
628 pr_alert("page_owner allocatio 436 pr_alert("page_owner allocation stack trace missing\n");
629 else !! 437 } else {
630 stack_depot_print(handle); !! 438 nr_entries = stack_depot_fetch(handle, &entries);
>> 439 stack_trace_print(entries, nr_entries, 0);
>> 440 }
631 441
632 handle = READ_ONCE(page_owner->free_ha 442 handle = READ_ONCE(page_owner->free_handle);
633 if (!handle) { 443 if (!handle) {
634 pr_alert("page_owner free stac 444 pr_alert("page_owner free stack trace missing\n");
635 } else { 445 } else {
636 pr_alert("page last free pid % !! 446 nr_entries = stack_depot_fetch(handle, &entries);
637 page_owner->free_pid !! 447 pr_alert("page last free stack trace:\n");
638 stack_depot_print(handle); !! 448 stack_trace_print(entries, nr_entries, 0);
639 } 449 }
640 450
641 if (page_owner->last_migrate_reason != 451 if (page_owner->last_migrate_reason != -1)
642 pr_alert("page has been migrat 452 pr_alert("page has been migrated, last migrate reason: %s\n",
643 migrate_reason_names[p 453 migrate_reason_names[page_owner->last_migrate_reason]);
644 page_ext_put(page_ext); <<
645 } 454 }
646 455
647 static ssize_t 456 static ssize_t
648 read_page_owner(struct file *file, char __user 457 read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos)
649 { 458 {
650 unsigned long pfn; 459 unsigned long pfn;
651 struct page *page; 460 struct page *page;
652 struct page_ext *page_ext; 461 struct page_ext *page_ext;
653 struct page_owner *page_owner; 462 struct page_owner *page_owner;
654 depot_stack_handle_t handle; 463 depot_stack_handle_t handle;
655 464
656 if (!static_branch_unlikely(&page_owne 465 if (!static_branch_unlikely(&page_owner_inited))
657 return -EINVAL; 466 return -EINVAL;
658 467
659 page = NULL; 468 page = NULL;
660 if (*ppos == 0) !! 469 pfn = min_low_pfn + *ppos;
661 pfn = min_low_pfn; !! 470
662 else <<
663 pfn = *ppos; <<
664 /* Find a valid PFN or the start of a 471 /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */
665 while (!pfn_valid(pfn) && (pfn & (MAX_ 472 while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0)
666 pfn++; 473 pfn++;
667 474
>> 475 drain_all_pages(NULL);
>> 476
668 /* Find an allocated page */ 477 /* Find an allocated page */
669 for (; pfn < max_pfn; pfn++) { 478 for (; pfn < max_pfn; pfn++) {
670 /* 479 /*
671 * This temporary page_owner i <<
672 * that we can avoid the conte <<
673 * the rcu lock and copying th <<
674 * user through copy_to_user() <<
675 */ <<
676 struct page_owner page_owner_t <<
677 <<
678 /* <<
679 * If the new page is in a new 480 * If the new page is in a new MAX_ORDER_NR_PAGES area,
680 * validate the area as existi 481 * validate the area as existing, skip it if not
681 */ 482 */
682 if ((pfn & (MAX_ORDER_NR_PAGES 483 if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) {
683 pfn += MAX_ORDER_NR_PA 484 pfn += MAX_ORDER_NR_PAGES - 1;
684 continue; 485 continue;
685 } 486 }
686 487
>> 488 /* Check for holes within a MAX_ORDER area */
>> 489 if (!pfn_valid_within(pfn))
>> 490 continue;
>> 491
687 page = pfn_to_page(pfn); 492 page = pfn_to_page(pfn);
688 if (PageBuddy(page)) { 493 if (PageBuddy(page)) {
689 unsigned long freepage !! 494 unsigned long freepage_order = page_order_unsafe(page);
690 495
691 if (freepage_order <= !! 496 if (freepage_order < MAX_ORDER)
692 pfn += (1UL << 497 pfn += (1UL << freepage_order) - 1;
693 continue; 498 continue;
694 } 499 }
695 500
696 page_ext = page_ext_get(page); !! 501 page_ext = lookup_page_ext(page);
697 if (unlikely(!page_ext)) 502 if (unlikely(!page_ext))
698 continue; 503 continue;
699 504
700 /* 505 /*
701 * Some pages could be missed 506 * Some pages could be missed by concurrent allocation or free,
702 * because we don't hold the z 507 * because we don't hold the zone lock.
703 */ 508 */
704 if (!test_bit(PAGE_EXT_OWNER, 509 if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags))
705 goto ext_put_continue; !! 510 continue;
706 511
707 /* 512 /*
708 * Although we do have the inf 513 * Although we do have the info about past allocation of free
709 * pages, it's not relevant fo 514 * pages, it's not relevant for current memory usage.
710 */ 515 */
711 if (!test_bit(PAGE_EXT_OWNER_A 516 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))
712 goto ext_put_continue; !! 517 continue;
713 518
714 page_owner = get_page_owner(pa 519 page_owner = get_page_owner(page_ext);
715 520
716 /* 521 /*
717 * Don't print "tail" pages of 522 * Don't print "tail" pages of high-order allocations as that
718 * would inflate the stats. 523 * would inflate the stats.
719 */ 524 */
720 if (!IS_ALIGNED(pfn, 1 << page 525 if (!IS_ALIGNED(pfn, 1 << page_owner->order))
721 goto ext_put_continue; !! 526 continue;
722 527
723 /* 528 /*
724 * Access to page_ext->handle 529 * Access to page_ext->handle isn't synchronous so we should
725 * be careful to access it. 530 * be careful to access it.
726 */ 531 */
727 handle = READ_ONCE(page_owner- 532 handle = READ_ONCE(page_owner->handle);
728 if (!handle) 533 if (!handle)
729 goto ext_put_continue; !! 534 continue;
730 535
731 /* Record the next PFN to read 536 /* Record the next PFN to read in the file offset */
732 *ppos = pfn + 1; !! 537 *ppos = (pfn - min_low_pfn) + 1;
733 538
734 page_owner_tmp = *page_owner; <<
735 page_ext_put(page_ext); <<
736 return print_page_owner(buf, c 539 return print_page_owner(buf, count, pfn, page,
737 &page_owner_tm !! 540 page_owner, handle);
738 ext_put_continue: <<
739 page_ext_put(page_ext); <<
740 } 541 }
741 542
742 return 0; 543 return 0;
743 } 544 }
744 545
745 static loff_t lseek_page_owner(struct file *fi <<
746 { <<
747 switch (orig) { <<
748 case SEEK_SET: <<
749 file->f_pos = offset; <<
750 break; <<
751 case SEEK_CUR: <<
752 file->f_pos += offset; <<
753 break; <<
754 default: <<
755 return -EINVAL; <<
756 } <<
757 return file->f_pos; <<
758 } <<
759 <<
760 static void init_pages_in_zone(pg_data_t *pgda 546 static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone)
761 { 547 {
762 unsigned long pfn = zone->zone_start_p 548 unsigned long pfn = zone->zone_start_pfn;
763 unsigned long end_pfn = zone_end_pfn(z 549 unsigned long end_pfn = zone_end_pfn(zone);
764 unsigned long count = 0; 550 unsigned long count = 0;
765 551
766 /* 552 /*
767 * Walk the zone in pageblock_nr_pages 553 * Walk the zone in pageblock_nr_pages steps. If a page block spans
768 * a zone boundary, it will be double 554 * a zone boundary, it will be double counted between zones. This does
769 * not matter as the mixed block count 555 * not matter as the mixed block count will still be correct
770 */ 556 */
771 for (; pfn < end_pfn; ) { 557 for (; pfn < end_pfn; ) {
772 unsigned long block_end_pfn; 558 unsigned long block_end_pfn;
773 559
774 if (!pfn_valid(pfn)) { 560 if (!pfn_valid(pfn)) {
775 pfn = ALIGN(pfn + 1, M 561 pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES);
776 continue; 562 continue;
777 } 563 }
778 564
779 block_end_pfn = pageblock_end_ !! 565 block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
780 block_end_pfn = min(block_end_ 566 block_end_pfn = min(block_end_pfn, end_pfn);
781 567
782 for (; pfn < block_end_pfn; pf 568 for (; pfn < block_end_pfn; pfn++) {
783 struct page *page = pf !! 569 struct page *page;
784 struct page_ext *page_ 570 struct page_ext *page_ext;
785 571
>> 572 if (!pfn_valid_within(pfn))
>> 573 continue;
>> 574
>> 575 page = pfn_to_page(pfn);
>> 576
786 if (page_zone(page) != 577 if (page_zone(page) != zone)
787 continue; 578 continue;
788 579
789 /* 580 /*
790 * To avoid having to 581 * To avoid having to grab zone->lock, be a little
791 * careful when readin 582 * careful when reading buddy page order. The only
792 * danger is that we s 583 * danger is that we skip too much and potentially miss
793 * some early allocate 584 * some early allocated pages, which is better than
794 * heavy lock contenti 585 * heavy lock contention.
795 */ 586 */
796 if (PageBuddy(page)) { 587 if (PageBuddy(page)) {
797 unsigned long !! 588 unsigned long order = page_order_unsafe(page);
798 589
799 if (order > 0 !! 590 if (order > 0 && order < MAX_ORDER)
800 pfn += 591 pfn += (1UL << order) - 1;
801 continue; 592 continue;
802 } 593 }
803 594
804 if (PageReserved(page) 595 if (PageReserved(page))
805 continue; 596 continue;
806 597
807 page_ext = page_ext_ge !! 598 page_ext = lookup_page_ext(page);
808 if (unlikely(!page_ext 599 if (unlikely(!page_ext))
809 continue; 600 continue;
810 601
811 /* Maybe overlapping z 602 /* Maybe overlapping zone */
812 if (test_bit(PAGE_EXT_ 603 if (test_bit(PAGE_EXT_OWNER, &page_ext->flags))
813 goto ext_put_c !! 604 continue;
814 605
815 /* Found early allocat 606 /* Found early allocated page */
816 __update_page_owner_ha !! 607 __set_page_owner_handle(page, page_ext, early_handle,
817 !! 608 0, 0);
818 <<
819 count++; 609 count++;
820 ext_put_continue: <<
821 page_ext_put(page_ext) <<
822 } 610 }
823 cond_resched(); 611 cond_resched();
824 } 612 }
825 613
826 pr_info("Node %d, zone %8s: page owner 614 pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n",
827 pgdat->node_id, zone->name, co 615 pgdat->node_id, zone->name, count);
828 } 616 }
829 617
830 static void init_zones_in_node(pg_data_t *pgda 618 static void init_zones_in_node(pg_data_t *pgdat)
831 { 619 {
832 struct zone *zone; 620 struct zone *zone;
833 struct zone *node_zones = pgdat->node_ 621 struct zone *node_zones = pgdat->node_zones;
834 622
835 for (zone = node_zones; zone - node_zo 623 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
836 if (!populated_zone(zone)) 624 if (!populated_zone(zone))
837 continue; 625 continue;
838 626
839 init_pages_in_zone(pgdat, zone 627 init_pages_in_zone(pgdat, zone);
840 } 628 }
841 } 629 }
842 630
843 static void init_early_allocated_pages(void) 631 static void init_early_allocated_pages(void)
844 { 632 {
845 pg_data_t *pgdat; 633 pg_data_t *pgdat;
846 634
847 for_each_online_pgdat(pgdat) 635 for_each_online_pgdat(pgdat)
848 init_zones_in_node(pgdat); 636 init_zones_in_node(pgdat);
849 } 637 }
850 638
851 static const struct file_operations proc_page_ 639 static const struct file_operations proc_page_owner_operations = {
852 .read = read_page_owner, 640 .read = read_page_owner,
853 .llseek = lseek_page_owner, <<
854 }; 641 };
855 642
856 static void *stack_start(struct seq_file *m, l <<
857 { <<
858 struct stack *stack; <<
859 <<
860 if (*ppos == -1UL) <<
861 return NULL; <<
862 <<
863 if (!*ppos) { <<
864 /* <<
865 * This pairs with smp_store_r <<
866 * add_stack_record_to_list(), <<
867 * value of stack_list. <<
868 */ <<
869 stack = smp_load_acquire(&stac <<
870 m->private = stack; <<
871 } else { <<
872 stack = m->private; <<
873 } <<
874 <<
875 return stack; <<
876 } <<
877 <<
878 static void *stack_next(struct seq_file *m, vo <<
879 { <<
880 struct stack *stack = v; <<
881 <<
882 stack = stack->next; <<
883 *ppos = stack ? *ppos + 1 : -1UL; <<
884 m->private = stack; <<
885 <<
886 return stack; <<
887 } <<
888 <<
889 static unsigned long page_owner_pages_threshol <<
890 <<
891 static int stack_print(struct seq_file *m, voi <<
892 { <<
893 int i, nr_base_pages; <<
894 struct stack *stack = v; <<
895 unsigned long *entries; <<
896 unsigned long nr_entries; <<
897 struct stack_record *stack_record = st <<
898 <<
899 if (!stack->stack_record) <<
900 return 0; <<
901 <<
902 nr_entries = stack_record->size; <<
903 entries = stack_record->entries; <<
904 nr_base_pages = refcount_read(&stack_r <<
905 <<
906 if (nr_base_pages < 1 || nr_base_pages <<
907 return 0; <<
908 <<
909 for (i = 0; i < nr_entries; i++) <<
910 seq_printf(m, " %pS\n", (void <<
911 seq_printf(m, "nr_base_pages: %d\n\n", <<
912 <<
913 return 0; <<
914 } <<
915 <<
916 static void stack_stop(struct seq_file *m, voi <<
917 { <<
918 } <<
919 <<
920 static const struct seq_operations page_owner_ <<
921 .start = stack_start, <<
922 .next = stack_next, <<
923 .stop = stack_stop, <<
924 .show = stack_print <<
925 }; <<
926 <<
927 static int page_owner_stack_open(struct inode <<
928 { <<
929 return seq_open_private(file, &page_ow <<
930 } <<
931 <<
932 static const struct file_operations page_owner <<
933 .open = page_owner_stack_ope <<
934 .read = seq_read, <<
935 .llseek = seq_lseek, <<
936 .release = seq_release, <<
937 }; <<
938 <<
939 static int page_owner_threshold_get(void *data <<
940 { <<
941 *val = READ_ONCE(page_owner_pages_thre <<
942 return 0; <<
943 } <<
944 <<
945 static int page_owner_threshold_set(void *data <<
946 { <<
947 WRITE_ONCE(page_owner_pages_threshold, <<
948 return 0; <<
949 } <<
950 <<
951 DEFINE_SIMPLE_ATTRIBUTE(proc_page_owner_thresh <<
952 &page_owner_threshold_ <<
953 <<
954 <<
955 static int __init pageowner_init(void) 643 static int __init pageowner_init(void)
956 { 644 {
957 struct dentry *dir; <<
958 <<
959 if (!static_branch_unlikely(&page_owne 645 if (!static_branch_unlikely(&page_owner_inited)) {
960 pr_info("page_owner is disable 646 pr_info("page_owner is disabled\n");
961 return 0; 647 return 0;
962 } 648 }
963 649
964 debugfs_create_file("page_owner", 0400 650 debugfs_create_file("page_owner", 0400, NULL, NULL,
965 &proc_page_owner_o 651 &proc_page_owner_operations);
966 dir = debugfs_create_dir("page_owner_s <<
967 debugfs_create_file("show_stacks", 040 <<
968 &page_owner_stack_ <<
969 debugfs_create_file("count_threshold", <<
970 &proc_page_owner_t <<
971 652
972 return 0; 653 return 0;
973 } 654 }
974 late_initcall(pageowner_init) 655 late_initcall(pageowner_init)
975 656
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.