1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 2
3 /* 3 /*
4 * Copyright (c) 2021, Google LLC. 4 * Copyright (c) 2021, Google LLC.
5 * Pasha Tatashin <pasha.tatashin@soleen.com> 5 * Pasha Tatashin <pasha.tatashin@soleen.com>
6 */ 6 */
7 #include <linux/kstrtox.h> <<
8 #include <linux/mm.h> 7 #include <linux/mm.h>
9 #include <linux/page_table_check.h> 8 #include <linux/page_table_check.h>
10 #include <linux/swap.h> <<
11 #include <linux/swapops.h> <<
12 9
13 #undef pr_fmt 10 #undef pr_fmt
14 #define pr_fmt(fmt) "page_table_check: " f 11 #define pr_fmt(fmt) "page_table_check: " fmt
15 12
16 struct page_table_check { 13 struct page_table_check {
17 atomic_t anon_map_count; 14 atomic_t anon_map_count;
18 atomic_t file_map_count; 15 atomic_t file_map_count;
19 }; 16 };
20 17
21 static bool __page_table_check_enabled __initd 18 static bool __page_table_check_enabled __initdata =
22 IS_ENABLED(CON 19 IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED);
23 20
24 DEFINE_STATIC_KEY_TRUE(page_table_check_disabl 21 DEFINE_STATIC_KEY_TRUE(page_table_check_disabled);
25 EXPORT_SYMBOL(page_table_check_disabled); 22 EXPORT_SYMBOL(page_table_check_disabled);
26 23
27 static int __init early_page_table_check_param 24 static int __init early_page_table_check_param(char *buf)
28 { 25 {
29 return kstrtobool(buf, &__page_table_c !! 26 return strtobool(buf, &__page_table_check_enabled);
30 } 27 }
31 28
32 early_param("page_table_check", early_page_tab 29 early_param("page_table_check", early_page_table_check_param);
33 30
34 static bool __init need_page_table_check(void) 31 static bool __init need_page_table_check(void)
35 { 32 {
36 return __page_table_check_enabled; 33 return __page_table_check_enabled;
37 } 34 }
38 35
39 static void __init init_page_table_check(void) 36 static void __init init_page_table_check(void)
40 { 37 {
41 if (!__page_table_check_enabled) 38 if (!__page_table_check_enabled)
42 return; 39 return;
43 static_branch_disable(&page_table_chec 40 static_branch_disable(&page_table_check_disabled);
44 } 41 }
45 42
46 struct page_ext_operations page_table_check_op 43 struct page_ext_operations page_table_check_ops = {
47 .size = sizeof(struct page_table_check 44 .size = sizeof(struct page_table_check),
48 .need = need_page_table_check, 45 .need = need_page_table_check,
49 .init = init_page_table_check, 46 .init = init_page_table_check,
50 .need_shared_flags = false, <<
51 }; 47 };
52 48
53 static struct page_table_check *get_page_table 49 static struct page_table_check *get_page_table_check(struct page_ext *page_ext)
54 { 50 {
55 BUG_ON(!page_ext); 51 BUG_ON(!page_ext);
56 return page_ext_data(page_ext, &page_t !! 52 return (void *)(page_ext) + page_table_check_ops.offset;
>> 53 }
>> 54
>> 55 static inline bool pte_user_accessible_page(pte_t pte)
>> 56 {
>> 57 return (pte_val(pte) & _PAGE_PRESENT) && (pte_val(pte) & _PAGE_USER);
>> 58 }
>> 59
>> 60 static inline bool pmd_user_accessible_page(pmd_t pmd)
>> 61 {
>> 62 return pmd_leaf(pmd) && (pmd_val(pmd) & _PAGE_PRESENT) &&
>> 63 (pmd_val(pmd) & _PAGE_USER);
>> 64 }
>> 65
>> 66 static inline bool pud_user_accessible_page(pud_t pud)
>> 67 {
>> 68 return pud_leaf(pud) && (pud_val(pud) & _PAGE_PRESENT) &&
>> 69 (pud_val(pud) & _PAGE_USER);
57 } 70 }
58 71
59 /* 72 /*
60 * An entry is removed from the page table, de !! 73 * An enty is removed from the page table, decrement the counters for that page
61 * verify that it is of correct type and count 74 * verify that it is of correct type and counters do not become negative.
62 */ 75 */
63 static void page_table_check_clear(unsigned lo !! 76 static void page_table_check_clear(struct mm_struct *mm, unsigned long addr,
>> 77 unsigned long pfn, unsigned long pgcnt)
64 { 78 {
65 struct page_ext *page_ext; 79 struct page_ext *page_ext;
66 struct page *page; 80 struct page *page;
67 unsigned long i; 81 unsigned long i;
68 bool anon; 82 bool anon;
69 83
70 if (!pfn_valid(pfn)) 84 if (!pfn_valid(pfn))
71 return; 85 return;
72 86
73 page = pfn_to_page(pfn); 87 page = pfn_to_page(pfn);
74 page_ext = page_ext_get(page); !! 88 page_ext = lookup_page_ext(page);
75 <<
76 if (!page_ext) <<
77 return; <<
78 <<
79 BUG_ON(PageSlab(page)); <<
80 anon = PageAnon(page); 89 anon = PageAnon(page);
81 90
82 for (i = 0; i < pgcnt; i++) { 91 for (i = 0; i < pgcnt; i++) {
83 struct page_table_check *ptc = 92 struct page_table_check *ptc = get_page_table_check(page_ext);
84 93
85 if (anon) { 94 if (anon) {
86 BUG_ON(atomic_read(&pt 95 BUG_ON(atomic_read(&ptc->file_map_count));
87 BUG_ON(atomic_dec_retu 96 BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0);
88 } else { 97 } else {
89 BUG_ON(atomic_read(&pt 98 BUG_ON(atomic_read(&ptc->anon_map_count));
90 BUG_ON(atomic_dec_retu 99 BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0);
91 } 100 }
92 page_ext = page_ext_next(page_ 101 page_ext = page_ext_next(page_ext);
93 } 102 }
94 page_ext_put(page_ext); <<
95 } 103 }
96 104
97 /* 105 /*
98 * A new entry is added to the page table, inc !! 106 * A new enty is added to the page table, increment the counters for that page
99 * verify that it is of correct type and is no 107 * verify that it is of correct type and is not being mapped with a different
100 * type to a different process. 108 * type to a different process.
101 */ 109 */
102 static void page_table_check_set(unsigned long !! 110 static void page_table_check_set(struct mm_struct *mm, unsigned long addr,
>> 111 unsigned long pfn, unsigned long pgcnt,
103 bool rw) 112 bool rw)
104 { 113 {
105 struct page_ext *page_ext; 114 struct page_ext *page_ext;
106 struct page *page; 115 struct page *page;
107 unsigned long i; 116 unsigned long i;
108 bool anon; 117 bool anon;
109 118
110 if (!pfn_valid(pfn)) 119 if (!pfn_valid(pfn))
111 return; 120 return;
112 121
113 page = pfn_to_page(pfn); 122 page = pfn_to_page(pfn);
114 page_ext = page_ext_get(page); !! 123 page_ext = lookup_page_ext(page);
115 <<
116 if (!page_ext) <<
117 return; <<
118 <<
119 BUG_ON(PageSlab(page)); <<
120 anon = PageAnon(page); 124 anon = PageAnon(page);
121 125
122 for (i = 0; i < pgcnt; i++) { 126 for (i = 0; i < pgcnt; i++) {
123 struct page_table_check *ptc = 127 struct page_table_check *ptc = get_page_table_check(page_ext);
124 128
125 if (anon) { 129 if (anon) {
126 BUG_ON(atomic_read(&pt 130 BUG_ON(atomic_read(&ptc->file_map_count));
127 BUG_ON(atomic_inc_retu 131 BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw);
128 } else { 132 } else {
129 BUG_ON(atomic_read(&pt 133 BUG_ON(atomic_read(&ptc->anon_map_count));
130 BUG_ON(atomic_inc_retu 134 BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0);
131 } 135 }
132 page_ext = page_ext_next(page_ 136 page_ext = page_ext_next(page_ext);
133 } 137 }
134 page_ext_put(page_ext); <<
135 } 138 }
136 139
137 /* 140 /*
138 * page is on free list, or is being allocated 141 * page is on free list, or is being allocated, verify that counters are zeroes
139 * crash if they are not. 142 * crash if they are not.
140 */ 143 */
141 void __page_table_check_zero(struct page *page 144 void __page_table_check_zero(struct page *page, unsigned int order)
142 { 145 {
143 struct page_ext *page_ext; !! 146 struct page_ext *page_ext = lookup_page_ext(page);
144 unsigned long i; 147 unsigned long i;
145 148
146 BUG_ON(PageSlab(page)); !! 149 BUG_ON(!page_ext);
147 <<
148 page_ext = page_ext_get(page); <<
149 <<
150 if (!page_ext) <<
151 return; <<
152 <<
153 for (i = 0; i < (1ul << order); i++) { 150 for (i = 0; i < (1ul << order); i++) {
154 struct page_table_check *ptc = 151 struct page_table_check *ptc = get_page_table_check(page_ext);
155 152
156 BUG_ON(atomic_read(&ptc->anon_ 153 BUG_ON(atomic_read(&ptc->anon_map_count));
157 BUG_ON(atomic_read(&ptc->file_ 154 BUG_ON(atomic_read(&ptc->file_map_count));
158 page_ext = page_ext_next(page_ 155 page_ext = page_ext_next(page_ext);
159 } 156 }
160 page_ext_put(page_ext); <<
161 } 157 }
162 158
163 void __page_table_check_pte_clear(struct mm_st !! 159 void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr,
>> 160 pte_t pte)
164 { 161 {
165 if (&init_mm == mm) 162 if (&init_mm == mm)
166 return; 163 return;
167 164
168 if (pte_user_accessible_page(pte)) { 165 if (pte_user_accessible_page(pte)) {
169 page_table_check_clear(pte_pfn !! 166 page_table_check_clear(mm, addr, pte_pfn(pte),
>> 167 PAGE_SIZE >> PAGE_SHIFT);
170 } 168 }
171 } 169 }
172 EXPORT_SYMBOL(__page_table_check_pte_clear); 170 EXPORT_SYMBOL(__page_table_check_pte_clear);
173 171
174 void __page_table_check_pmd_clear(struct mm_st !! 172 void __page_table_check_pmd_clear(struct mm_struct *mm, unsigned long addr,
>> 173 pmd_t pmd)
175 { 174 {
176 if (&init_mm == mm) 175 if (&init_mm == mm)
177 return; 176 return;
178 177
179 if (pmd_user_accessible_page(pmd)) { 178 if (pmd_user_accessible_page(pmd)) {
180 page_table_check_clear(pmd_pfn !! 179 page_table_check_clear(mm, addr, pmd_pfn(pmd),
>> 180 PMD_PAGE_SIZE >> PAGE_SHIFT);
181 } 181 }
182 } 182 }
183 EXPORT_SYMBOL(__page_table_check_pmd_clear); 183 EXPORT_SYMBOL(__page_table_check_pmd_clear);
184 184
185 void __page_table_check_pud_clear(struct mm_st !! 185 void __page_table_check_pud_clear(struct mm_struct *mm, unsigned long addr,
>> 186 pud_t pud)
186 { 187 {
187 if (&init_mm == mm) 188 if (&init_mm == mm)
188 return; 189 return;
189 190
190 if (pud_user_accessible_page(pud)) { 191 if (pud_user_accessible_page(pud)) {
191 page_table_check_clear(pud_pfn !! 192 page_table_check_clear(mm, addr, pud_pfn(pud),
>> 193 PUD_PAGE_SIZE >> PAGE_SHIFT);
192 } 194 }
193 } 195 }
194 EXPORT_SYMBOL(__page_table_check_pud_clear); 196 EXPORT_SYMBOL(__page_table_check_pud_clear);
195 197
196 /* Whether the swap entry cached writable info !! 198 void __page_table_check_pte_set(struct mm_struct *mm, unsigned long addr,
197 static inline bool swap_cached_writable(swp_en !! 199 pte_t *ptep, pte_t pte)
198 { <<
199 return is_writable_device_exclusive_en <<
200 is_writable_device_private_entry(e <<
201 is_writable_migration_entry(entry) <<
202 } <<
203 <<
204 static inline void page_table_check_pte_flags( <<
205 { <<
206 if (pte_present(pte) && pte_uffd_wp(pt <<
207 WARN_ON_ONCE(pte_write(pte)); <<
208 else if (is_swap_pte(pte) && pte_swp_u <<
209 WARN_ON_ONCE(swap_cached_writa <<
210 } <<
211 <<
212 void __page_table_check_ptes_set(struct mm_str <<
213 unsigned int nr) <<
214 { 200 {
215 unsigned int i; <<
216 <<
217 if (&init_mm == mm) 201 if (&init_mm == mm)
218 return; 202 return;
219 203
220 page_table_check_pte_flags(pte); !! 204 __page_table_check_pte_clear(mm, addr, *ptep);
221 !! 205 if (pte_user_accessible_page(pte)) {
222 for (i = 0; i < nr; i++) !! 206 page_table_check_set(mm, addr, pte_pfn(pte),
223 __page_table_check_pte_clear(m !! 207 PAGE_SIZE >> PAGE_SHIFT,
224 if (pte_user_accessible_page(pte)) !! 208 pte_write(pte));
225 page_table_check_set(pte_pfn(p !! 209 }
226 } <<
227 EXPORT_SYMBOL(__page_table_check_ptes_set); <<
228 <<
229 static inline void page_table_check_pmd_flags( <<
230 { <<
231 if (pmd_present(pmd) && pmd_uffd_wp(pm <<
232 WARN_ON_ONCE(pmd_write(pmd)); <<
233 else if (is_swap_pmd(pmd) && pmd_swp_u <<
234 WARN_ON_ONCE(swap_cached_writa <<
235 } 210 }
>> 211 EXPORT_SYMBOL(__page_table_check_pte_set);
236 212
237 void __page_table_check_pmd_set(struct mm_stru !! 213 void __page_table_check_pmd_set(struct mm_struct *mm, unsigned long addr,
>> 214 pmd_t *pmdp, pmd_t pmd)
238 { 215 {
239 if (&init_mm == mm) 216 if (&init_mm == mm)
240 return; 217 return;
241 218
242 page_table_check_pmd_flags(pmd); !! 219 __page_table_check_pmd_clear(mm, addr, *pmdp);
243 <<
244 __page_table_check_pmd_clear(mm, *pmdp <<
245 if (pmd_user_accessible_page(pmd)) { 220 if (pmd_user_accessible_page(pmd)) {
246 page_table_check_set(pmd_pfn(p !! 221 page_table_check_set(mm, addr, pmd_pfn(pmd),
>> 222 PMD_PAGE_SIZE >> PAGE_SHIFT,
247 pmd_write 223 pmd_write(pmd));
248 } 224 }
249 } 225 }
250 EXPORT_SYMBOL(__page_table_check_pmd_set); 226 EXPORT_SYMBOL(__page_table_check_pmd_set);
251 227
252 void __page_table_check_pud_set(struct mm_stru !! 228 void __page_table_check_pud_set(struct mm_struct *mm, unsigned long addr,
>> 229 pud_t *pudp, pud_t pud)
253 { 230 {
254 if (&init_mm == mm) 231 if (&init_mm == mm)
255 return; 232 return;
256 233
257 __page_table_check_pud_clear(mm, *pudp !! 234 __page_table_check_pud_clear(mm, addr, *pudp);
258 if (pud_user_accessible_page(pud)) { 235 if (pud_user_accessible_page(pud)) {
259 page_table_check_set(pud_pfn(p !! 236 page_table_check_set(mm, addr, pud_pfn(pud),
>> 237 PUD_PAGE_SIZE >> PAGE_SHIFT,
260 pud_write 238 pud_write(pud));
261 } 239 }
262 } 240 }
263 EXPORT_SYMBOL(__page_table_check_pud_set); 241 EXPORT_SYMBOL(__page_table_check_pud_set);
264 242
265 void __page_table_check_pte_clear_range(struct 243 void __page_table_check_pte_clear_range(struct mm_struct *mm,
266 unsign 244 unsigned long addr,
267 pmd_t 245 pmd_t pmd)
268 { 246 {
269 if (&init_mm == mm) 247 if (&init_mm == mm)
270 return; 248 return;
271 249
272 if (!pmd_bad(pmd) && !pmd_leaf(pmd)) { 250 if (!pmd_bad(pmd) && !pmd_leaf(pmd)) {
273 pte_t *ptep = pte_offset_map(& 251 pte_t *ptep = pte_offset_map(&pmd, addr);
274 unsigned long i; 252 unsigned long i;
275 253
276 if (WARN_ON(!ptep)) !! 254 pte_unmap(ptep);
277 return; <<
278 for (i = 0; i < PTRS_PER_PTE; 255 for (i = 0; i < PTRS_PER_PTE; i++) {
279 __page_table_check_pte !! 256 __page_table_check_pte_clear(mm, addr, *ptep);
280 addr += PAGE_SIZE; 257 addr += PAGE_SIZE;
281 ptep++; 258 ptep++;
282 } 259 }
283 pte_unmap(ptep - PTRS_PER_PTE) <<
284 } 260 }
285 } 261 }
286 262
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.