1 // SPDX-License-Identifier: GPL-2.0 1
2
3 /*
4 * Copyright (c) 2021, Google LLC.
5 * Pasha Tatashin <pasha.tatashin@soleen.com>
6 */
7 #include <linux/kstrtox.h>
8 #include <linux/mm.h>
9 #include <linux/page_table_check.h>
10 #include <linux/swap.h>
11 #include <linux/swapops.h>
12
13 #undef pr_fmt
14 #define pr_fmt(fmt) "page_table_check: " f
15
16 struct page_table_check {
17 atomic_t anon_map_count;
18 atomic_t file_map_count;
19 };
20
21 static bool __page_table_check_enabled __initd
22 IS_ENABLED(CON
23
24 DEFINE_STATIC_KEY_TRUE(page_table_check_disabl
25 EXPORT_SYMBOL(page_table_check_disabled);
26
27 static int __init early_page_table_check_param
28 {
29 return kstrtobool(buf, &__page_table_c
30 }
31
32 early_param("page_table_check", early_page_tab
33
34 static bool __init need_page_table_check(void)
35 {
36 return __page_table_check_enabled;
37 }
38
39 static void __init init_page_table_check(void)
40 {
41 if (!__page_table_check_enabled)
42 return;
43 static_branch_disable(&page_table_chec
44 }
45
46 struct page_ext_operations page_table_check_op
47 .size = sizeof(struct page_table_check
48 .need = need_page_table_check,
49 .init = init_page_table_check,
50 .need_shared_flags = false,
51 };
52
53 static struct page_table_check *get_page_table
54 {
55 BUG_ON(!page_ext);
56 return page_ext_data(page_ext, &page_t
57 }
58
59 /*
60 * An entry is removed from the page table, de
61 * verify that it is of correct type and count
62 */
63 static void page_table_check_clear(unsigned lo
64 {
65 struct page_ext *page_ext;
66 struct page *page;
67 unsigned long i;
68 bool anon;
69
70 if (!pfn_valid(pfn))
71 return;
72
73 page = pfn_to_page(pfn);
74 page_ext = page_ext_get(page);
75
76 if (!page_ext)
77 return;
78
79 BUG_ON(PageSlab(page));
80 anon = PageAnon(page);
81
82 for (i = 0; i < pgcnt; i++) {
83 struct page_table_check *ptc =
84
85 if (anon) {
86 BUG_ON(atomic_read(&pt
87 BUG_ON(atomic_dec_retu
88 } else {
89 BUG_ON(atomic_read(&pt
90 BUG_ON(atomic_dec_retu
91 }
92 page_ext = page_ext_next(page_
93 }
94 page_ext_put(page_ext);
95 }
96
97 /*
98 * A new entry is added to the page table, inc
99 * verify that it is of correct type and is no
100 * type to a different process.
101 */
102 static void page_table_check_set(unsigned long
103 bool rw)
104 {
105 struct page_ext *page_ext;
106 struct page *page;
107 unsigned long i;
108 bool anon;
109
110 if (!pfn_valid(pfn))
111 return;
112
113 page = pfn_to_page(pfn);
114 page_ext = page_ext_get(page);
115
116 if (!page_ext)
117 return;
118
119 BUG_ON(PageSlab(page));
120 anon = PageAnon(page);
121
122 for (i = 0; i < pgcnt; i++) {
123 struct page_table_check *ptc =
124
125 if (anon) {
126 BUG_ON(atomic_read(&pt
127 BUG_ON(atomic_inc_retu
128 } else {
129 BUG_ON(atomic_read(&pt
130 BUG_ON(atomic_inc_retu
131 }
132 page_ext = page_ext_next(page_
133 }
134 page_ext_put(page_ext);
135 }
136
137 /*
138 * page is on free list, or is being allocated
139 * crash if they are not.
140 */
141 void __page_table_check_zero(struct page *page
142 {
143 struct page_ext *page_ext;
144 unsigned long i;
145
146 BUG_ON(PageSlab(page));
147
148 page_ext = page_ext_get(page);
149
150 if (!page_ext)
151 return;
152
153 for (i = 0; i < (1ul << order); i++) {
154 struct page_table_check *ptc =
155
156 BUG_ON(atomic_read(&ptc->anon_
157 BUG_ON(atomic_read(&ptc->file_
158 page_ext = page_ext_next(page_
159 }
160 page_ext_put(page_ext);
161 }
162
163 void __page_table_check_pte_clear(struct mm_st
164 {
165 if (&init_mm == mm)
166 return;
167
168 if (pte_user_accessible_page(pte)) {
169 page_table_check_clear(pte_pfn
170 }
171 }
172 EXPORT_SYMBOL(__page_table_check_pte_clear);
173
174 void __page_table_check_pmd_clear(struct mm_st
175 {
176 if (&init_mm == mm)
177 return;
178
179 if (pmd_user_accessible_page(pmd)) {
180 page_table_check_clear(pmd_pfn
181 }
182 }
183 EXPORT_SYMBOL(__page_table_check_pmd_clear);
184
185 void __page_table_check_pud_clear(struct mm_st
186 {
187 if (&init_mm == mm)
188 return;
189
190 if (pud_user_accessible_page(pud)) {
191 page_table_check_clear(pud_pfn
192 }
193 }
194 EXPORT_SYMBOL(__page_table_check_pud_clear);
195
196 /* Whether the swap entry cached writable info
197 static inline bool swap_cached_writable(swp_en
198 {
199 return is_writable_device_exclusive_en
200 is_writable_device_private_entry(e
201 is_writable_migration_entry(entry)
202 }
203
204 static inline void page_table_check_pte_flags(
205 {
206 if (pte_present(pte) && pte_uffd_wp(pt
207 WARN_ON_ONCE(pte_write(pte));
208 else if (is_swap_pte(pte) && pte_swp_u
209 WARN_ON_ONCE(swap_cached_writa
210 }
211
212 void __page_table_check_ptes_set(struct mm_str
213 unsigned int nr)
214 {
215 unsigned int i;
216
217 if (&init_mm == mm)
218 return;
219
220 page_table_check_pte_flags(pte);
221
222 for (i = 0; i < nr; i++)
223 __page_table_check_pte_clear(m
224 if (pte_user_accessible_page(pte))
225 page_table_check_set(pte_pfn(p
226 }
227 EXPORT_SYMBOL(__page_table_check_ptes_set);
228
229 static inline void page_table_check_pmd_flags(
230 {
231 if (pmd_present(pmd) && pmd_uffd_wp(pm
232 WARN_ON_ONCE(pmd_write(pmd));
233 else if (is_swap_pmd(pmd) && pmd_swp_u
234 WARN_ON_ONCE(swap_cached_writa
235 }
236
237 void __page_table_check_pmd_set(struct mm_stru
238 {
239 if (&init_mm == mm)
240 return;
241
242 page_table_check_pmd_flags(pmd);
243
244 __page_table_check_pmd_clear(mm, *pmdp
245 if (pmd_user_accessible_page(pmd)) {
246 page_table_check_set(pmd_pfn(p
247 pmd_write
248 }
249 }
250 EXPORT_SYMBOL(__page_table_check_pmd_set);
251
252 void __page_table_check_pud_set(struct mm_stru
253 {
254 if (&init_mm == mm)
255 return;
256
257 __page_table_check_pud_clear(mm, *pudp
258 if (pud_user_accessible_page(pud)) {
259 page_table_check_set(pud_pfn(p
260 pud_write
261 }
262 }
263 EXPORT_SYMBOL(__page_table_check_pud_set);
264
265 void __page_table_check_pte_clear_range(struct
266 unsign
267 pmd_t
268 {
269 if (&init_mm == mm)
270 return;
271
272 if (!pmd_bad(pmd) && !pmd_leaf(pmd)) {
273 pte_t *ptep = pte_offset_map(&
274 unsigned long i;
275
276 if (WARN_ON(!ptep))
277 return;
278 for (i = 0; i < PTRS_PER_PTE;
279 __page_table_check_pte
280 addr += PAGE_SIZE;
281 ptep++;
282 }
283 pte_unmap(ptep - PTRS_PER_PTE)
284 }
285 }
286
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.