1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/debugfs.h> 2 #include <linux/debugfs.h>
3 #include <linux/mm.h> 3 #include <linux/mm.h>
4 #include <linux/slab.h> 4 #include <linux/slab.h>
5 #include <linux/uaccess.h> 5 #include <linux/uaccess.h>
6 #include <linux/memblock.h> 6 #include <linux/memblock.h>
7 #include <linux/stacktrace.h> 7 #include <linux/stacktrace.h>
8 #include <linux/page_owner.h> 8 #include <linux/page_owner.h>
9 #include <linux/jump_label.h> 9 #include <linux/jump_label.h>
10 #include <linux/migrate.h> 10 #include <linux/migrate.h>
11 #include <linux/stackdepot.h> 11 #include <linux/stackdepot.h>
12 #include <linux/seq_file.h> 12 #include <linux/seq_file.h>
13 #include <linux/memcontrol.h> <<
14 #include <linux/sched/clock.h> 13 #include <linux/sched/clock.h>
15 14
16 #include "internal.h" 15 #include "internal.h"
17 16
18 /* 17 /*
19 * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_ 18 * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack)
20 * to use off stack temporal storage 19 * to use off stack temporal storage
21 */ 20 */
22 #define PAGE_OWNER_STACK_DEPTH (16) 21 #define PAGE_OWNER_STACK_DEPTH (16)
23 22
24 struct page_owner { 23 struct page_owner {
25 unsigned short order; 24 unsigned short order;
26 short last_migrate_reason; 25 short last_migrate_reason;
27 gfp_t gfp_mask; 26 gfp_t gfp_mask;
28 depot_stack_handle_t handle; 27 depot_stack_handle_t handle;
29 depot_stack_handle_t free_handle; 28 depot_stack_handle_t free_handle;
30 u64 ts_nsec; 29 u64 ts_nsec;
31 u64 free_ts_nsec; 30 u64 free_ts_nsec;
32 char comm[TASK_COMM_LEN]; <<
33 pid_t pid; 31 pid_t pid;
34 pid_t tgid; <<
35 pid_t free_pid; <<
36 pid_t free_tgid; <<
37 }; 32 };
38 33
39 struct stack { !! 34 static bool page_owner_enabled = false;
40 struct stack_record *stack_record; <<
41 struct stack *next; <<
42 }; <<
43 static struct stack dummy_stack; <<
44 static struct stack failure_stack; <<
45 static struct stack *stack_list; <<
46 static DEFINE_SPINLOCK(stack_list_lock); <<
47 <<
48 static bool page_owner_enabled __initdata; <<
49 DEFINE_STATIC_KEY_FALSE(page_owner_inited); 35 DEFINE_STATIC_KEY_FALSE(page_owner_inited);
50 36
51 static depot_stack_handle_t dummy_handle; 37 static depot_stack_handle_t dummy_handle;
52 static depot_stack_handle_t failure_handle; 38 static depot_stack_handle_t failure_handle;
53 static depot_stack_handle_t early_handle; 39 static depot_stack_handle_t early_handle;
54 40
55 static void init_early_allocated_pages(void); 41 static void init_early_allocated_pages(void);
56 42
57 static inline void set_current_in_page_owner(v <<
58 { <<
59 /* <<
60 * Avoid recursion. <<
61 * <<
62 * We might need to allocate more memo <<
63 * sure to signal it in order to avoid <<
64 */ <<
65 current->in_page_owner = 1; <<
66 } <<
67 <<
68 static inline void unset_current_in_page_owner <<
69 { <<
70 current->in_page_owner = 0; <<
71 } <<
72 <<
73 static int __init early_page_owner_param(char 43 static int __init early_page_owner_param(char *buf)
74 { 44 {
75 int ret = kstrtobool(buf, &page_owner_ !! 45 return kstrtobool(buf, &page_owner_enabled);
76 <<
77 if (page_owner_enabled) <<
78 stack_depot_request_early_init <<
79 <<
80 return ret; <<
81 } 46 }
82 early_param("page_owner", early_page_owner_par 47 early_param("page_owner", early_page_owner_param);
83 48
84 static __init bool need_page_owner(void) 49 static __init bool need_page_owner(void)
85 { 50 {
86 return page_owner_enabled; 51 return page_owner_enabled;
87 } 52 }
88 53
89 static __always_inline depot_stack_handle_t cr 54 static __always_inline depot_stack_handle_t create_dummy_stack(void)
90 { 55 {
91 unsigned long entries[4]; 56 unsigned long entries[4];
92 unsigned int nr_entries; 57 unsigned int nr_entries;
93 58
94 nr_entries = stack_trace_save(entries, 59 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0);
95 return stack_depot_save(entries, nr_en 60 return stack_depot_save(entries, nr_entries, GFP_KERNEL);
96 } 61 }
97 62
98 static noinline void register_dummy_stack(void 63 static noinline void register_dummy_stack(void)
99 { 64 {
100 dummy_handle = create_dummy_stack(); 65 dummy_handle = create_dummy_stack();
101 } 66 }
102 67
103 static noinline void register_failure_stack(vo 68 static noinline void register_failure_stack(void)
104 { 69 {
105 failure_handle = create_dummy_stack(); 70 failure_handle = create_dummy_stack();
106 } 71 }
107 72
108 static noinline void register_early_stack(void 73 static noinline void register_early_stack(void)
109 { 74 {
110 early_handle = create_dummy_stack(); 75 early_handle = create_dummy_stack();
111 } 76 }
112 77
113 static __init void init_page_owner(void) 78 static __init void init_page_owner(void)
114 { 79 {
115 if (!page_owner_enabled) 80 if (!page_owner_enabled)
116 return; 81 return;
117 82
>> 83 stack_depot_init();
>> 84
118 register_dummy_stack(); 85 register_dummy_stack();
119 register_failure_stack(); 86 register_failure_stack();
120 register_early_stack(); 87 register_early_stack();
121 init_early_allocated_pages(); <<
122 /* Initialize dummy and failure stacks <<
123 dummy_stack.stack_record = __stack_dep <<
124 failure_stack.stack_record = __stack_d <<
125 if (dummy_stack.stack_record) <<
126 refcount_set(&dummy_stack.stac <<
127 if (failure_stack.stack_record) <<
128 refcount_set(&failure_stack.st <<
129 dummy_stack.next = &failure_stack; <<
130 stack_list = &dummy_stack; <<
131 static_branch_enable(&page_owner_inite 88 static_branch_enable(&page_owner_inited);
>> 89 init_early_allocated_pages();
132 } 90 }
133 91
134 struct page_ext_operations page_owner_ops = { 92 struct page_ext_operations page_owner_ops = {
135 .size = sizeof(struct page_owner), 93 .size = sizeof(struct page_owner),
136 .need = need_page_owner, 94 .need = need_page_owner,
137 .init = init_page_owner, 95 .init = init_page_owner,
138 .need_shared_flags = true, <<
139 }; 96 };
140 97
141 static inline struct page_owner *get_page_owne 98 static inline struct page_owner *get_page_owner(struct page_ext *page_ext)
142 { 99 {
143 return page_ext_data(page_ext, &page_o !! 100 return (void *)page_ext + page_owner_ops.offset;
144 } 101 }
145 102
146 static noinline depot_stack_handle_t save_stac 103 static noinline depot_stack_handle_t save_stack(gfp_t flags)
147 { 104 {
148 unsigned long entries[PAGE_OWNER_STACK 105 unsigned long entries[PAGE_OWNER_STACK_DEPTH];
149 depot_stack_handle_t handle; 106 depot_stack_handle_t handle;
150 unsigned int nr_entries; 107 unsigned int nr_entries;
151 108
>> 109 /*
>> 110 * Avoid recursion.
>> 111 *
>> 112 * Sometimes page metadata allocation tracking requires more
>> 113 * memory to be allocated:
>> 114 * - when new stack trace is saved to stack depot
>> 115 * - when backtrace itself is calculated (ia64)
>> 116 */
152 if (current->in_page_owner) 117 if (current->in_page_owner)
153 return dummy_handle; 118 return dummy_handle;
>> 119 current->in_page_owner = 1;
154 120
155 set_current_in_page_owner(); <<
156 nr_entries = stack_trace_save(entries, 121 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2);
157 handle = stack_depot_save(entries, nr_ 122 handle = stack_depot_save(entries, nr_entries, flags);
158 if (!handle) 123 if (!handle)
159 handle = failure_handle; 124 handle = failure_handle;
160 unset_current_in_page_owner(); <<
161 125
>> 126 current->in_page_owner = 0;
162 return handle; 127 return handle;
163 } 128 }
164 129
165 static void add_stack_record_to_list(struct st !! 130 void __reset_page_owner(struct page *page, unsigned short order)
166 gfp_t gfp <<
167 { <<
168 unsigned long flags; <<
169 struct stack *stack; <<
170 <<
171 set_current_in_page_owner(); <<
172 stack = kmalloc(sizeof(*stack), gfp_ne <<
173 if (!stack) { <<
174 unset_current_in_page_owner(); <<
175 return; <<
176 } <<
177 unset_current_in_page_owner(); <<
178 <<
179 stack->stack_record = stack_record; <<
180 stack->next = NULL; <<
181 <<
182 spin_lock_irqsave(&stack_list_lock, fl <<
183 stack->next = stack_list; <<
184 /* <<
185 * This pairs with smp_load_acquire() <<
186 * stack_start(). This guarantees that <<
187 * will see an updated stack_list befo <<
188 * traverse the list. <<
189 */ <<
190 smp_store_release(&stack_list, stack); <<
191 spin_unlock_irqrestore(&stack_list_loc <<
192 } <<
193 <<
194 static void inc_stack_record_count(depot_stack <<
195 int nr_base <<
196 { 131 {
197 struct stack_record *stack_record = __ !! 132 int i;
>> 133 struct page_ext *page_ext;
>> 134 depot_stack_handle_t handle;
>> 135 struct page_owner *page_owner;
>> 136 u64 free_ts_nsec = local_clock();
198 137
199 if (!stack_record) !! 138 page_ext = lookup_page_ext(page);
>> 139 if (unlikely(!page_ext))
200 return; 140 return;
201 141
202 /* !! 142 handle = save_stack(GFP_NOWAIT | __GFP_NOWARN);
203 * New stack_record's that do not use !! 143 for (i = 0; i < (1 << order); i++) {
204 * with REFCOUNT_SATURATED to catch sp !! 144 __clear_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags);
205 * refcount. !! 145 page_owner = get_page_owner(page_ext);
206 * Since we do not use STACK_DEPOT_FLA !! 146 page_owner->free_handle = handle;
207 * set a refcount of 1 ourselves. !! 147 page_owner->free_ts_nsec = free_ts_nsec;
208 */ !! 148 page_ext = page_ext_next(page_ext);
209 if (refcount_read(&stack_record->count <<
210 int old = REFCOUNT_SATURATED; <<
211 <<
212 if (atomic_try_cmpxchg_relaxed <<
213 /* Add the new stack_r <<
214 add_stack_record_to_li <<
215 } 149 }
216 refcount_add(nr_base_pages, &stack_rec <<
217 } <<
218 <<
219 static void dec_stack_record_count(depot_stack <<
220 int nr_base <<
221 { <<
222 struct stack_record *stack_record = __ <<
223 <<
224 if (!stack_record) <<
225 return; <<
226 <<
227 if (refcount_sub_and_test(nr_base_page <<
228 pr_warn("%s: refcount went to <<
229 handle); <<
230 } 150 }
231 151
232 static inline void __update_page_owner_handle( !! 152 static inline void __set_page_owner_handle(struct page_ext *page_ext,
233 !! 153 depot_stack_handle_t handle,
234 !! 154 unsigned short order, gfp_t gfp_mask)
235 <<
236 <<
237 <<
238 { 155 {
239 int i; <<
240 struct page_owner *page_owner; 156 struct page_owner *page_owner;
>> 157 int i;
241 158
242 for (i = 0; i < (1 << order); i++) { 159 for (i = 0; i < (1 << order); i++) {
243 page_owner = get_page_owner(pa 160 page_owner = get_page_owner(page_ext);
244 page_owner->handle = handle; 161 page_owner->handle = handle;
245 page_owner->order = order; 162 page_owner->order = order;
246 page_owner->gfp_mask = gfp_mas 163 page_owner->gfp_mask = gfp_mask;
247 page_owner->last_migrate_reaso !! 164 page_owner->last_migrate_reason = -1;
248 page_owner->pid = pid; !! 165 page_owner->pid = current->pid;
249 page_owner->tgid = tgid; !! 166 page_owner->ts_nsec = local_clock();
250 page_owner->ts_nsec = ts_nsec; <<
251 strscpy(page_owner->comm, comm <<
252 sizeof(page_owner->com <<
253 __set_bit(PAGE_EXT_OWNER, &pag 167 __set_bit(PAGE_EXT_OWNER, &page_ext->flags);
254 __set_bit(PAGE_EXT_OWNER_ALLOC 168 __set_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags);
255 page_ext = page_ext_next(page_ <<
256 } <<
257 } <<
258 169
259 static inline void __update_page_owner_free_ha <<
260 <<
261 <<
262 <<
263 <<
264 { <<
265 int i; <<
266 struct page_owner *page_owner; <<
267 <<
268 for (i = 0; i < (1 << order); i++) { <<
269 page_owner = get_page_owner(pa <<
270 /* Only __reset_page_owner() w <<
271 if (handle) { <<
272 __clear_bit(PAGE_EXT_O <<
273 page_owner->free_handl <<
274 } <<
275 page_owner->free_ts_nsec = fre <<
276 page_owner->free_pid = current <<
277 page_owner->free_tgid = curren <<
278 page_ext = page_ext_next(page_ 170 page_ext = page_ext_next(page_ext);
279 } 171 }
280 } 172 }
281 173
282 void __reset_page_owner(struct page *page, uns <<
283 { <<
284 struct page_ext *page_ext; <<
285 depot_stack_handle_t handle; <<
286 depot_stack_handle_t alloc_handle; <<
287 struct page_owner *page_owner; <<
288 u64 free_ts_nsec = local_clock(); <<
289 <<
290 page_ext = page_ext_get(page); <<
291 if (unlikely(!page_ext)) <<
292 return; <<
293 <<
294 page_owner = get_page_owner(page_ext); <<
295 alloc_handle = page_owner->handle; <<
296 <<
297 handle = save_stack(GFP_NOWAIT | __GFP <<
298 __update_page_owner_free_handle(page_e <<
299 curren <<
300 page_ext_put(page_ext); <<
301 <<
302 if (alloc_handle != early_handle) <<
303 /* <<
304 * early_handle is being set a <<
305 * early allocated pages. See <<
306 * Since their refcount is not <<
307 * the machinery is not ready <<
308 * their refcount either. <<
309 */ <<
310 dec_stack_record_count(alloc_h <<
311 } <<
312 <<
313 noinline void __set_page_owner(struct page *pa 174 noinline void __set_page_owner(struct page *page, unsigned short order,
314 gfp_t 175 gfp_t gfp_mask)
315 { 176 {
316 struct page_ext *page_ext; !! 177 struct page_ext *page_ext = lookup_page_ext(page);
317 u64 ts_nsec = local_clock(); <<
318 depot_stack_handle_t handle; 178 depot_stack_handle_t handle;
319 179
320 handle = save_stack(gfp_mask); <<
321 <<
322 page_ext = page_ext_get(page); <<
323 if (unlikely(!page_ext)) 180 if (unlikely(!page_ext))
324 return; 181 return;
325 __update_page_owner_handle(page_ext, h !! 182
326 ts_nsec, cu !! 183 handle = save_stack(gfp_mask);
327 current->co !! 184 __set_page_owner_handle(page_ext, handle, order, gfp_mask);
328 page_ext_put(page_ext); <<
329 inc_stack_record_count(handle, gfp_mas <<
330 } 185 }
331 186
332 void __set_page_owner_migrate_reason(struct pa 187 void __set_page_owner_migrate_reason(struct page *page, int reason)
333 { 188 {
334 struct page_ext *page_ext = page_ext_g !! 189 struct page_ext *page_ext = lookup_page_ext(page);
335 struct page_owner *page_owner; 190 struct page_owner *page_owner;
336 191
337 if (unlikely(!page_ext)) 192 if (unlikely(!page_ext))
338 return; 193 return;
339 194
340 page_owner = get_page_owner(page_ext); 195 page_owner = get_page_owner(page_ext);
341 page_owner->last_migrate_reason = reas 196 page_owner->last_migrate_reason = reason;
342 page_ext_put(page_ext); <<
343 } 197 }
344 198
345 void __split_page_owner(struct page *page, int !! 199 void __split_page_owner(struct page *page, unsigned int nr)
346 { 200 {
347 int i; 201 int i;
348 struct page_ext *page_ext = page_ext_g !! 202 struct page_ext *page_ext = lookup_page_ext(page);
349 struct page_owner *page_owner; 203 struct page_owner *page_owner;
350 204
351 if (unlikely(!page_ext)) 205 if (unlikely(!page_ext))
352 return; 206 return;
353 207
354 for (i = 0; i < (1 << old_order); i++) !! 208 for (i = 0; i < nr; i++) {
355 page_owner = get_page_owner(pa 209 page_owner = get_page_owner(page_ext);
356 page_owner->order = new_order; !! 210 page_owner->order = 0;
357 page_ext = page_ext_next(page_ 211 page_ext = page_ext_next(page_ext);
358 } 212 }
359 page_ext_put(page_ext); <<
360 } 213 }
361 214
362 void __folio_copy_owner(struct folio *newfolio 215 void __folio_copy_owner(struct folio *newfolio, struct folio *old)
363 { 216 {
364 int i; !! 217 struct page_ext *old_ext = lookup_page_ext(&old->page);
365 struct page_ext *old_ext; !! 218 struct page_ext *new_ext = lookup_page_ext(&newfolio->page);
366 struct page_ext *new_ext; !! 219 struct page_owner *old_page_owner, *new_page_owner;
367 struct page_owner *old_page_owner; <<
368 struct page_owner *new_page_owner; <<
369 depot_stack_handle_t migrate_handle; <<
370 220
371 old_ext = page_ext_get(&old->page); !! 221 if (unlikely(!old_ext || !new_ext))
372 if (unlikely(!old_ext)) <<
373 return; 222 return;
374 223
375 new_ext = page_ext_get(&newfolio->page <<
376 if (unlikely(!new_ext)) { <<
377 page_ext_put(old_ext); <<
378 return; <<
379 } <<
380 <<
381 old_page_owner = get_page_owner(old_ex 224 old_page_owner = get_page_owner(old_ext);
382 new_page_owner = get_page_owner(new_ex 225 new_page_owner = get_page_owner(new_ext);
383 migrate_handle = new_page_owner->handl !! 226 new_page_owner->order = old_page_owner->order;
384 __update_page_owner_handle(new_ext, ol !! 227 new_page_owner->gfp_mask = old_page_owner->gfp_mask;
385 old_page_ow !! 228 new_page_owner->last_migrate_reason =
386 old_page_ow !! 229 old_page_owner->last_migrate_reason;
387 old_page_ow !! 230 new_page_owner->handle = old_page_owner->handle;
388 old_page_ow !! 231 new_page_owner->pid = old_page_owner->pid;
389 /* !! 232 new_page_owner->ts_nsec = old_page_owner->ts_nsec;
390 * Do not proactively clear PAGE_EXT_O !! 233 new_page_owner->free_ts_nsec = old_page_owner->ts_nsec;
391 * will be freed after migration. Keep !! 234
392 * useful. <<
393 */ <<
394 __update_page_owner_free_handle(new_ex <<
395 old_pa <<
396 old_pa <<
397 old_pa <<
398 /* 235 /*
399 * We linked the original stack to the !! 236 * We don't clear the bit on the old folio as it's going to be freed
400 * for the new one and the old folio o !! 237 * after migration. Until then, the info can be useful in case of
401 * when subtracting those pages from t !! 238 * a bug, and the overall stats will be off a bit only temporarily.
>> 239 * Also, migrate_misplaced_transhuge_page() can still fail the
>> 240 * migration and then we want the old folio to retain the info. But
>> 241 * in that case we also don't need to explicitly clear the info from
>> 242 * the new page, which will be freed.
402 */ 243 */
403 for (i = 0; i < (1 << new_page_owner-> !! 244 __set_bit(PAGE_EXT_OWNER, &new_ext->flags);
404 old_page_owner->handle = migra !! 245 __set_bit(PAGE_EXT_OWNER_ALLOCATED, &new_ext->flags);
405 old_ext = page_ext_next(old_ex <<
406 old_page_owner = get_page_owne <<
407 } <<
408 <<
409 page_ext_put(new_ext); <<
410 page_ext_put(old_ext); <<
411 } 246 }
412 247
413 void pagetypeinfo_showmixedcount_print(struct 248 void pagetypeinfo_showmixedcount_print(struct seq_file *m,
414 pg_data 249 pg_data_t *pgdat, struct zone *zone)
415 { 250 {
416 struct page *page; 251 struct page *page;
417 struct page_ext *page_ext; 252 struct page_ext *page_ext;
418 struct page_owner *page_owner; 253 struct page_owner *page_owner;
419 unsigned long pfn, block_end_pfn; 254 unsigned long pfn, block_end_pfn;
420 unsigned long end_pfn = zone_end_pfn(z 255 unsigned long end_pfn = zone_end_pfn(zone);
421 unsigned long count[MIGRATE_TYPES] = { 256 unsigned long count[MIGRATE_TYPES] = { 0, };
422 int pageblock_mt, page_mt; 257 int pageblock_mt, page_mt;
423 int i; 258 int i;
424 259
425 /* Scan block by block. First and last 260 /* Scan block by block. First and last block may be incomplete */
426 pfn = zone->zone_start_pfn; 261 pfn = zone->zone_start_pfn;
427 262
428 /* 263 /*
429 * Walk the zone in pageblock_nr_pages 264 * Walk the zone in pageblock_nr_pages steps. If a page block spans
430 * a zone boundary, it will be double 265 * a zone boundary, it will be double counted between zones. This does
431 * not matter as the mixed block count 266 * not matter as the mixed block count will still be correct
432 */ 267 */
433 for (; pfn < end_pfn; ) { 268 for (; pfn < end_pfn; ) {
434 page = pfn_to_online_page(pfn) 269 page = pfn_to_online_page(pfn);
435 if (!page) { 270 if (!page) {
436 pfn = ALIGN(pfn + 1, M 271 pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES);
437 continue; 272 continue;
438 } 273 }
439 274
440 block_end_pfn = pageblock_end_ !! 275 block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
441 block_end_pfn = min(block_end_ 276 block_end_pfn = min(block_end_pfn, end_pfn);
442 277
443 pageblock_mt = get_pageblock_m 278 pageblock_mt = get_pageblock_migratetype(page);
444 279
445 for (; pfn < block_end_pfn; pf 280 for (; pfn < block_end_pfn; pfn++) {
446 /* The pageblock is on 281 /* The pageblock is online, no need to recheck. */
447 page = pfn_to_page(pfn 282 page = pfn_to_page(pfn);
448 283
449 if (page_zone(page) != 284 if (page_zone(page) != zone)
450 continue; 285 continue;
451 286
452 if (PageBuddy(page)) { 287 if (PageBuddy(page)) {
453 unsigned long 288 unsigned long freepage_order;
454 289
455 freepage_order 290 freepage_order = buddy_order_unsafe(page);
456 if (freepage_o !! 291 if (freepage_order < MAX_ORDER)
457 pfn += 292 pfn += (1UL << freepage_order) - 1;
458 continue; 293 continue;
459 } 294 }
460 295
461 if (PageReserved(page) 296 if (PageReserved(page))
462 continue; 297 continue;
463 298
464 page_ext = page_ext_ge !! 299 page_ext = lookup_page_ext(page);
465 if (unlikely(!page_ext 300 if (unlikely(!page_ext))
466 continue; 301 continue;
467 302
468 if (!test_bit(PAGE_EXT 303 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))
469 goto ext_put_c !! 304 continue;
470 305
471 page_owner = get_page_ 306 page_owner = get_page_owner(page_ext);
472 page_mt = gfp_migratet 307 page_mt = gfp_migratetype(page_owner->gfp_mask);
473 if (pageblock_mt != pa 308 if (pageblock_mt != page_mt) {
474 if (is_migrate 309 if (is_migrate_cma(pageblock_mt))
475 count[ 310 count[MIGRATE_MOVABLE]++;
476 else 311 else
477 count[ 312 count[pageblock_mt]++;
478 313
479 pfn = block_en 314 pfn = block_end_pfn;
480 page_ext_put(p <<
481 break; 315 break;
482 } 316 }
483 pfn += (1UL << page_ow 317 pfn += (1UL << page_owner->order) - 1;
484 ext_put_continue: <<
485 page_ext_put(page_ext) <<
486 } 318 }
487 } 319 }
488 320
489 /* Print counts */ 321 /* Print counts */
490 seq_printf(m, "Node %d, zone %8s ", pg 322 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
491 for (i = 0; i < MIGRATE_TYPES; i++) 323 for (i = 0; i < MIGRATE_TYPES; i++)
492 seq_printf(m, "%12lu ", count[ 324 seq_printf(m, "%12lu ", count[i]);
493 seq_putc(m, '\n'); 325 seq_putc(m, '\n');
494 } 326 }
495 327
496 /* <<
497 * Looking for memcg information and print it <<
498 */ <<
499 static inline int print_page_owner_memcg(char <<
500 struc <<
501 { <<
502 #ifdef CONFIG_MEMCG <<
503 unsigned long memcg_data; <<
504 struct mem_cgroup *memcg; <<
505 bool online; <<
506 char name[80]; <<
507 <<
508 rcu_read_lock(); <<
509 memcg_data = READ_ONCE(page->memcg_dat <<
510 if (!memcg_data) <<
511 goto out_unlock; <<
512 <<
513 if (memcg_data & MEMCG_DATA_OBJEXTS) <<
514 ret += scnprintf(kbuf + ret, c <<
515 "Slab cache pa <<
516 <<
517 memcg = page_memcg_check(page); <<
518 if (!memcg) <<
519 goto out_unlock; <<
520 <<
521 online = (memcg->css.flags & CSS_ONLIN <<
522 cgroup_name(memcg->css.cgroup, name, s <<
523 ret += scnprintf(kbuf + ret, count - r <<
524 "Charged %sto %smemcg <<
525 PageMemcgKmem(page) ? <<
526 online ? "" : "offline <<
527 name); <<
528 out_unlock: <<
529 rcu_read_unlock(); <<
530 #endif /* CONFIG_MEMCG */ <<
531 <<
532 return ret; <<
533 } <<
534 <<
535 static ssize_t 328 static ssize_t
536 print_page_owner(char __user *buf, size_t coun 329 print_page_owner(char __user *buf, size_t count, unsigned long pfn,
537 struct page *page, struct page 330 struct page *page, struct page_owner *page_owner,
538 depot_stack_handle_t handle) 331 depot_stack_handle_t handle)
539 { 332 {
540 int ret, pageblock_mt, page_mt; 333 int ret, pageblock_mt, page_mt;
541 char *kbuf; 334 char *kbuf;
542 335
543 count = min_t(size_t, count, PAGE_SIZE 336 count = min_t(size_t, count, PAGE_SIZE);
544 kbuf = kmalloc(count, GFP_KERNEL); 337 kbuf = kmalloc(count, GFP_KERNEL);
545 if (!kbuf) 338 if (!kbuf)
546 return -ENOMEM; 339 return -ENOMEM;
547 340
548 ret = scnprintf(kbuf, count, !! 341 ret = snprintf(kbuf, count,
549 "Page allocated via or !! 342 "Page allocated via order %u, mask %#x(%pGg), pid %d, ts %llu ns, free_ts %llu ns\n",
550 page_owner->order, pag 343 page_owner->order, page_owner->gfp_mask,
551 &page_owner->gfp_mask, 344 &page_owner->gfp_mask, page_owner->pid,
552 page_owner->tgid, page !! 345 page_owner->ts_nsec, page_owner->free_ts_nsec);
553 page_owner->ts_nsec); !! 346
>> 347 if (ret >= count)
>> 348 goto err;
554 349
555 /* Print information relevant to group 350 /* Print information relevant to grouping pages by mobility */
556 pageblock_mt = get_pageblock_migratety 351 pageblock_mt = get_pageblock_migratetype(page);
557 page_mt = gfp_migratetype(page_owner- 352 page_mt = gfp_migratetype(page_owner->gfp_mask);
558 ret += scnprintf(kbuf + ret, count - r !! 353 ret += snprintf(kbuf + ret, count - ret,
559 "PFN 0x%lx type %s Blo !! 354 "PFN %lu type %s Block %lu type %s Flags %pGp\n",
560 pfn, 355 pfn,
561 migratetype_names[page 356 migratetype_names[page_mt],
562 pfn >> pageblock_order 357 pfn >> pageblock_order,
563 migratetype_names[page 358 migratetype_names[pageblock_mt],
564 &page->flags); 359 &page->flags);
565 360
>> 361 if (ret >= count)
>> 362 goto err;
>> 363
566 ret += stack_depot_snprint(handle, kbu 364 ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0);
567 if (ret >= count) 365 if (ret >= count)
568 goto err; 366 goto err;
569 367
570 if (page_owner->last_migrate_reason != 368 if (page_owner->last_migrate_reason != -1) {
571 ret += scnprintf(kbuf + ret, c !! 369 ret += snprintf(kbuf + ret, count - ret,
572 "Page has been migrate 370 "Page has been migrated, last migrate reason: %s\n",
573 migrate_reason_names[p 371 migrate_reason_names[page_owner->last_migrate_reason]);
>> 372 if (ret >= count)
>> 373 goto err;
574 } 374 }
575 375
576 ret = print_page_owner_memcg(kbuf, cou <<
577 <<
578 ret += snprintf(kbuf + ret, count - re 376 ret += snprintf(kbuf + ret, count - ret, "\n");
579 if (ret >= count) 377 if (ret >= count)
580 goto err; 378 goto err;
581 379
582 if (copy_to_user(buf, kbuf, ret)) 380 if (copy_to_user(buf, kbuf, ret))
583 ret = -EFAULT; 381 ret = -EFAULT;
584 382
585 kfree(kbuf); 383 kfree(kbuf);
586 return ret; 384 return ret;
587 385
588 err: 386 err:
589 kfree(kbuf); 387 kfree(kbuf);
590 return -ENOMEM; 388 return -ENOMEM;
591 } 389 }
592 390
593 void __dump_page_owner(const struct page *page 391 void __dump_page_owner(const struct page *page)
594 { 392 {
595 struct page_ext *page_ext = page_ext_g !! 393 struct page_ext *page_ext = lookup_page_ext(page);
596 struct page_owner *page_owner; 394 struct page_owner *page_owner;
597 depot_stack_handle_t handle; 395 depot_stack_handle_t handle;
598 gfp_t gfp_mask; 396 gfp_t gfp_mask;
599 int mt; 397 int mt;
600 398
601 if (unlikely(!page_ext)) { 399 if (unlikely(!page_ext)) {
602 pr_alert("There is not page ex 400 pr_alert("There is not page extension available.\n");
603 return; 401 return;
604 } 402 }
605 403
606 page_owner = get_page_owner(page_ext); 404 page_owner = get_page_owner(page_ext);
607 gfp_mask = page_owner->gfp_mask; 405 gfp_mask = page_owner->gfp_mask;
608 mt = gfp_migratetype(gfp_mask); 406 mt = gfp_migratetype(gfp_mask);
609 407
610 if (!test_bit(PAGE_EXT_OWNER, &page_ex 408 if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) {
611 pr_alert("page_owner info is n 409 pr_alert("page_owner info is not present (never set?)\n");
612 page_ext_put(page_ext); <<
613 return; 410 return;
614 } 411 }
615 412
616 if (test_bit(PAGE_EXT_OWNER_ALLOCATED, 413 if (test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))
617 pr_alert("page_owner tracks th 414 pr_alert("page_owner tracks the page as allocated\n");
618 else 415 else
619 pr_alert("page_owner tracks th 416 pr_alert("page_owner tracks the page as freed\n");
620 417
621 pr_alert("page last allocated via orde !! 418 pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, ts %llu, free_ts %llu\n",
622 page_owner->order, migratetyp 419 page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask,
623 page_owner->pid, page_owner-> !! 420 page_owner->pid, page_owner->ts_nsec, page_owner->free_ts_nsec);
624 page_owner->ts_nsec, page_own <<
625 421
626 handle = READ_ONCE(page_owner->handle) 422 handle = READ_ONCE(page_owner->handle);
627 if (!handle) 423 if (!handle)
628 pr_alert("page_owner allocatio 424 pr_alert("page_owner allocation stack trace missing\n");
629 else 425 else
630 stack_depot_print(handle); 426 stack_depot_print(handle);
631 427
632 handle = READ_ONCE(page_owner->free_ha 428 handle = READ_ONCE(page_owner->free_handle);
633 if (!handle) { 429 if (!handle) {
634 pr_alert("page_owner free stac 430 pr_alert("page_owner free stack trace missing\n");
635 } else { 431 } else {
636 pr_alert("page last free pid % !! 432 pr_alert("page last free stack trace:\n");
637 page_owner->free_pid <<
638 stack_depot_print(handle); 433 stack_depot_print(handle);
639 } 434 }
640 435
641 if (page_owner->last_migrate_reason != 436 if (page_owner->last_migrate_reason != -1)
642 pr_alert("page has been migrat 437 pr_alert("page has been migrated, last migrate reason: %s\n",
643 migrate_reason_names[p 438 migrate_reason_names[page_owner->last_migrate_reason]);
644 page_ext_put(page_ext); <<
645 } 439 }
646 440
647 static ssize_t 441 static ssize_t
648 read_page_owner(struct file *file, char __user 442 read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos)
649 { 443 {
650 unsigned long pfn; 444 unsigned long pfn;
651 struct page *page; 445 struct page *page;
652 struct page_ext *page_ext; 446 struct page_ext *page_ext;
653 struct page_owner *page_owner; 447 struct page_owner *page_owner;
654 depot_stack_handle_t handle; 448 depot_stack_handle_t handle;
655 449
656 if (!static_branch_unlikely(&page_owne 450 if (!static_branch_unlikely(&page_owner_inited))
657 return -EINVAL; 451 return -EINVAL;
658 452
659 page = NULL; 453 page = NULL;
660 if (*ppos == 0) !! 454 pfn = min_low_pfn + *ppos;
661 pfn = min_low_pfn; !! 455
662 else <<
663 pfn = *ppos; <<
664 /* Find a valid PFN or the start of a 456 /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */
665 while (!pfn_valid(pfn) && (pfn & (MAX_ 457 while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0)
666 pfn++; 458 pfn++;
667 459
>> 460 drain_all_pages(NULL);
>> 461
668 /* Find an allocated page */ 462 /* Find an allocated page */
669 for (; pfn < max_pfn; pfn++) { 463 for (; pfn < max_pfn; pfn++) {
670 /* 464 /*
671 * This temporary page_owner i <<
672 * that we can avoid the conte <<
673 * the rcu lock and copying th <<
674 * user through copy_to_user() <<
675 */ <<
676 struct page_owner page_owner_t <<
677 <<
678 /* <<
679 * If the new page is in a new 465 * If the new page is in a new MAX_ORDER_NR_PAGES area,
680 * validate the area as existi 466 * validate the area as existing, skip it if not
681 */ 467 */
682 if ((pfn & (MAX_ORDER_NR_PAGES 468 if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) {
683 pfn += MAX_ORDER_NR_PA 469 pfn += MAX_ORDER_NR_PAGES - 1;
684 continue; 470 continue;
685 } 471 }
686 472
687 page = pfn_to_page(pfn); 473 page = pfn_to_page(pfn);
688 if (PageBuddy(page)) { 474 if (PageBuddy(page)) {
689 unsigned long freepage 475 unsigned long freepage_order = buddy_order_unsafe(page);
690 476
691 if (freepage_order <= !! 477 if (freepage_order < MAX_ORDER)
692 pfn += (1UL << 478 pfn += (1UL << freepage_order) - 1;
693 continue; 479 continue;
694 } 480 }
695 481
696 page_ext = page_ext_get(page); !! 482 page_ext = lookup_page_ext(page);
697 if (unlikely(!page_ext)) 483 if (unlikely(!page_ext))
698 continue; 484 continue;
699 485
700 /* 486 /*
701 * Some pages could be missed 487 * Some pages could be missed by concurrent allocation or free,
702 * because we don't hold the z 488 * because we don't hold the zone lock.
703 */ 489 */
704 if (!test_bit(PAGE_EXT_OWNER, 490 if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags))
705 goto ext_put_continue; !! 491 continue;
706 492
707 /* 493 /*
708 * Although we do have the inf 494 * Although we do have the info about past allocation of free
709 * pages, it's not relevant fo 495 * pages, it's not relevant for current memory usage.
710 */ 496 */
711 if (!test_bit(PAGE_EXT_OWNER_A 497 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))
712 goto ext_put_continue; !! 498 continue;
713 499
714 page_owner = get_page_owner(pa 500 page_owner = get_page_owner(page_ext);
715 501
716 /* 502 /*
717 * Don't print "tail" pages of 503 * Don't print "tail" pages of high-order allocations as that
718 * would inflate the stats. 504 * would inflate the stats.
719 */ 505 */
720 if (!IS_ALIGNED(pfn, 1 << page 506 if (!IS_ALIGNED(pfn, 1 << page_owner->order))
721 goto ext_put_continue; !! 507 continue;
722 508
723 /* 509 /*
724 * Access to page_ext->handle 510 * Access to page_ext->handle isn't synchronous so we should
725 * be careful to access it. 511 * be careful to access it.
726 */ 512 */
727 handle = READ_ONCE(page_owner- 513 handle = READ_ONCE(page_owner->handle);
728 if (!handle) 514 if (!handle)
729 goto ext_put_continue; !! 515 continue;
730 516
731 /* Record the next PFN to read 517 /* Record the next PFN to read in the file offset */
732 *ppos = pfn + 1; !! 518 *ppos = (pfn - min_low_pfn) + 1;
733 519
734 page_owner_tmp = *page_owner; <<
735 page_ext_put(page_ext); <<
736 return print_page_owner(buf, c 520 return print_page_owner(buf, count, pfn, page,
737 &page_owner_tm !! 521 page_owner, handle);
738 ext_put_continue: <<
739 page_ext_put(page_ext); <<
740 } 522 }
741 523
742 return 0; 524 return 0;
743 } 525 }
744 526
745 static loff_t lseek_page_owner(struct file *fi <<
746 { <<
747 switch (orig) { <<
748 case SEEK_SET: <<
749 file->f_pos = offset; <<
750 break; <<
751 case SEEK_CUR: <<
752 file->f_pos += offset; <<
753 break; <<
754 default: <<
755 return -EINVAL; <<
756 } <<
757 return file->f_pos; <<
758 } <<
759 <<
760 static void init_pages_in_zone(pg_data_t *pgda 527 static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone)
761 { 528 {
762 unsigned long pfn = zone->zone_start_p 529 unsigned long pfn = zone->zone_start_pfn;
763 unsigned long end_pfn = zone_end_pfn(z 530 unsigned long end_pfn = zone_end_pfn(zone);
764 unsigned long count = 0; 531 unsigned long count = 0;
765 532
766 /* 533 /*
767 * Walk the zone in pageblock_nr_pages 534 * Walk the zone in pageblock_nr_pages steps. If a page block spans
768 * a zone boundary, it will be double 535 * a zone boundary, it will be double counted between zones. This does
769 * not matter as the mixed block count 536 * not matter as the mixed block count will still be correct
770 */ 537 */
771 for (; pfn < end_pfn; ) { 538 for (; pfn < end_pfn; ) {
772 unsigned long block_end_pfn; 539 unsigned long block_end_pfn;
773 540
774 if (!pfn_valid(pfn)) { 541 if (!pfn_valid(pfn)) {
775 pfn = ALIGN(pfn + 1, M 542 pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES);
776 continue; 543 continue;
777 } 544 }
778 545
779 block_end_pfn = pageblock_end_ !! 546 block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
780 block_end_pfn = min(block_end_ 547 block_end_pfn = min(block_end_pfn, end_pfn);
781 548
782 for (; pfn < block_end_pfn; pf 549 for (; pfn < block_end_pfn; pfn++) {
783 struct page *page = pf 550 struct page *page = pfn_to_page(pfn);
784 struct page_ext *page_ 551 struct page_ext *page_ext;
785 552
786 if (page_zone(page) != 553 if (page_zone(page) != zone)
787 continue; 554 continue;
788 555
789 /* 556 /*
790 * To avoid having to 557 * To avoid having to grab zone->lock, be a little
791 * careful when readin 558 * careful when reading buddy page order. The only
792 * danger is that we s 559 * danger is that we skip too much and potentially miss
793 * some early allocate 560 * some early allocated pages, which is better than
794 * heavy lock contenti 561 * heavy lock contention.
795 */ 562 */
796 if (PageBuddy(page)) { 563 if (PageBuddy(page)) {
797 unsigned long 564 unsigned long order = buddy_order_unsafe(page);
798 565
799 if (order > 0 !! 566 if (order > 0 && order < MAX_ORDER)
800 pfn += 567 pfn += (1UL << order) - 1;
801 continue; 568 continue;
802 } 569 }
803 570
804 if (PageReserved(page) 571 if (PageReserved(page))
805 continue; 572 continue;
806 573
807 page_ext = page_ext_ge !! 574 page_ext = lookup_page_ext(page);
808 if (unlikely(!page_ext 575 if (unlikely(!page_ext))
809 continue; 576 continue;
810 577
811 /* Maybe overlapping z 578 /* Maybe overlapping zone */
812 if (test_bit(PAGE_EXT_ 579 if (test_bit(PAGE_EXT_OWNER, &page_ext->flags))
813 goto ext_put_c !! 580 continue;
814 581
815 /* Found early allocat 582 /* Found early allocated page */
816 __update_page_owner_ha !! 583 __set_page_owner_handle(page_ext, early_handle,
817 !! 584 0, 0);
818 <<
819 count++; 585 count++;
820 ext_put_continue: <<
821 page_ext_put(page_ext) <<
822 } 586 }
823 cond_resched(); 587 cond_resched();
824 } 588 }
825 589
826 pr_info("Node %d, zone %8s: page owner 590 pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n",
827 pgdat->node_id, zone->name, co 591 pgdat->node_id, zone->name, count);
828 } 592 }
829 593
830 static void init_zones_in_node(pg_data_t *pgda 594 static void init_zones_in_node(pg_data_t *pgdat)
831 { 595 {
832 struct zone *zone; 596 struct zone *zone;
833 struct zone *node_zones = pgdat->node_ 597 struct zone *node_zones = pgdat->node_zones;
834 598
835 for (zone = node_zones; zone - node_zo 599 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
836 if (!populated_zone(zone)) 600 if (!populated_zone(zone))
837 continue; 601 continue;
838 602
839 init_pages_in_zone(pgdat, zone 603 init_pages_in_zone(pgdat, zone);
840 } 604 }
841 } 605 }
842 606
843 static void init_early_allocated_pages(void) 607 static void init_early_allocated_pages(void)
844 { 608 {
845 pg_data_t *pgdat; 609 pg_data_t *pgdat;
846 610
847 for_each_online_pgdat(pgdat) 611 for_each_online_pgdat(pgdat)
848 init_zones_in_node(pgdat); 612 init_zones_in_node(pgdat);
849 } 613 }
850 614
851 static const struct file_operations proc_page_ 615 static const struct file_operations proc_page_owner_operations = {
852 .read = read_page_owner, 616 .read = read_page_owner,
853 .llseek = lseek_page_owner, <<
854 }; <<
855 <<
856 static void *stack_start(struct seq_file *m, l <<
857 { <<
858 struct stack *stack; <<
859 <<
860 if (*ppos == -1UL) <<
861 return NULL; <<
862 <<
863 if (!*ppos) { <<
864 /* <<
865 * This pairs with smp_store_r <<
866 * add_stack_record_to_list(), <<
867 * value of stack_list. <<
868 */ <<
869 stack = smp_load_acquire(&stac <<
870 m->private = stack; <<
871 } else { <<
872 stack = m->private; <<
873 } <<
874 <<
875 return stack; <<
876 } <<
877 <<
878 static void *stack_next(struct seq_file *m, vo <<
879 { <<
880 struct stack *stack = v; <<
881 <<
882 stack = stack->next; <<
883 *ppos = stack ? *ppos + 1 : -1UL; <<
884 m->private = stack; <<
885 <<
886 return stack; <<
887 } <<
888 <<
889 static unsigned long page_owner_pages_threshol <<
890 <<
891 static int stack_print(struct seq_file *m, voi <<
892 { <<
893 int i, nr_base_pages; <<
894 struct stack *stack = v; <<
895 unsigned long *entries; <<
896 unsigned long nr_entries; <<
897 struct stack_record *stack_record = st <<
898 <<
899 if (!stack->stack_record) <<
900 return 0; <<
901 <<
902 nr_entries = stack_record->size; <<
903 entries = stack_record->entries; <<
904 nr_base_pages = refcount_read(&stack_r <<
905 <<
906 if (nr_base_pages < 1 || nr_base_pages <<
907 return 0; <<
908 <<
909 for (i = 0; i < nr_entries; i++) <<
910 seq_printf(m, " %pS\n", (void <<
911 seq_printf(m, "nr_base_pages: %d\n\n", <<
912 <<
913 return 0; <<
914 } <<
915 <<
916 static void stack_stop(struct seq_file *m, voi <<
917 { <<
918 } <<
919 <<
920 static const struct seq_operations page_owner_ <<
921 .start = stack_start, <<
922 .next = stack_next, <<
923 .stop = stack_stop, <<
924 .show = stack_print <<
925 }; <<
926 <<
927 static int page_owner_stack_open(struct inode <<
928 { <<
929 return seq_open_private(file, &page_ow <<
930 } <<
931 <<
932 static const struct file_operations page_owner <<
933 .open = page_owner_stack_ope <<
934 .read = seq_read, <<
935 .llseek = seq_lseek, <<
936 .release = seq_release, <<
937 }; 617 };
938 618
939 static int page_owner_threshold_get(void *data <<
940 { <<
941 *val = READ_ONCE(page_owner_pages_thre <<
942 return 0; <<
943 } <<
944 <<
945 static int page_owner_threshold_set(void *data <<
946 { <<
947 WRITE_ONCE(page_owner_pages_threshold, <<
948 return 0; <<
949 } <<
950 <<
951 DEFINE_SIMPLE_ATTRIBUTE(proc_page_owner_thresh <<
952 &page_owner_threshold_ <<
953 <<
954 <<
955 static int __init pageowner_init(void) 619 static int __init pageowner_init(void)
956 { 620 {
957 struct dentry *dir; <<
958 <<
959 if (!static_branch_unlikely(&page_owne 621 if (!static_branch_unlikely(&page_owner_inited)) {
960 pr_info("page_owner is disable 622 pr_info("page_owner is disabled\n");
961 return 0; 623 return 0;
962 } 624 }
963 625
964 debugfs_create_file("page_owner", 0400 626 debugfs_create_file("page_owner", 0400, NULL, NULL,
965 &proc_page_owner_o 627 &proc_page_owner_operations);
966 dir = debugfs_create_dir("page_owner_s <<
967 debugfs_create_file("show_stacks", 040 <<
968 &page_owner_stack_ <<
969 debugfs_create_file("count_threshold", <<
970 &proc_page_owner_t <<
971 628
972 return 0; 629 return 0;
973 } 630 }
974 late_initcall(pageowner_init) 631 late_initcall(pageowner_init)
975 632
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.