1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/set_memory.h>
3 #include <linux/ptdump.h>
4 #include <linux/seq_file.h>
5 #include <linux/debugfs.h>
6 #include <linux/sort.h>
7 #include <linux/mm.h>
8 #include <linux/kfence.h>
9 #include <linux/kasan.h>
10 #include <asm/kasan.h>
11 #include <asm/abs_lowcore.h>
12 #include <asm/nospec-branch.h>
13 #include <asm/sections.h>
14 #include <asm/maccess.h>
15
16 static unsigned long max_addr;
17
18 struct addr_marker {
19 int is_start;
20 unsigned long start_address;
21 const char *name;
22 };
23
24 enum address_markers_idx {
25 KVA_NR = 0,
26 LOWCORE_START_NR,
27 LOWCORE_END_NR,
28 AMODE31_START_NR,
29 AMODE31_END_NR,
30 KERNEL_START_NR,
31 KERNEL_END_NR,
32 #ifdef CONFIG_KFENCE
33 KFENCE_START_NR,
34 KFENCE_END_NR,
35 #endif
36 IDENTITY_START_NR,
37 IDENTITY_END_NR,
38 VMEMMAP_NR,
39 VMEMMAP_END_NR,
40 VMALLOC_NR,
41 VMALLOC_END_NR,
42 #ifdef CONFIG_KMSAN
43 KMSAN_VMALLOC_SHADOW_START_NR,
44 KMSAN_VMALLOC_SHADOW_END_NR,
45 KMSAN_VMALLOC_ORIGIN_START_NR,
46 KMSAN_VMALLOC_ORIGIN_END_NR,
47 KMSAN_MODULES_SHADOW_START_NR,
48 KMSAN_MODULES_SHADOW_END_NR,
49 KMSAN_MODULES_ORIGIN_START_NR,
50 KMSAN_MODULES_ORIGIN_END_NR,
51 #endif
52 MODULES_NR,
53 MODULES_END_NR,
54 ABS_LOWCORE_NR,
55 ABS_LOWCORE_END_NR,
56 MEMCPY_REAL_NR,
57 MEMCPY_REAL_END_NR,
58 #ifdef CONFIG_KASAN
59 KASAN_SHADOW_START_NR,
60 KASAN_SHADOW_END_NR,
61 #endif
62 };
63
64 static struct addr_marker address_markers[] = {
65 [KVA_NR] = {0, 0, "Kernel Virtual Address Space"},
66 [LOWCORE_START_NR] = {1, 0, "Lowcore Start"},
67 [LOWCORE_END_NR] = {0, 0, "Lowcore End"},
68 [IDENTITY_START_NR] = {1, 0, "Identity Mapping Start"},
69 [IDENTITY_END_NR] = {0, 0, "Identity Mapping End"},
70 [AMODE31_START_NR] = {1, 0, "Amode31 Area Start"},
71 [AMODE31_END_NR] = {0, 0, "Amode31 Area End"},
72 [KERNEL_START_NR] = {1, (unsigned long)_stext, "Kernel Image Start"},
73 [KERNEL_END_NR] = {0, (unsigned long)_end, "Kernel Image End"},
74 #ifdef CONFIG_KFENCE
75 [KFENCE_START_NR] = {1, 0, "KFence Pool Start"},
76 [KFENCE_END_NR] = {0, 0, "KFence Pool End"},
77 #endif
78 [VMEMMAP_NR] = {1, 0, "vmemmap Area Start"},
79 [VMEMMAP_END_NR] = {0, 0, "vmemmap Area End"},
80 [VMALLOC_NR] = {1, 0, "vmalloc Area Start"},
81 [VMALLOC_END_NR] = {0, 0, "vmalloc Area End"},
82 #ifdef CONFIG_KMSAN
83 [KMSAN_VMALLOC_SHADOW_START_NR] = {1, 0, "Kmsan vmalloc Shadow Start"},
84 [KMSAN_VMALLOC_SHADOW_END_NR] = {0, 0, "Kmsan vmalloc Shadow End"},
85 [KMSAN_VMALLOC_ORIGIN_START_NR] = {1, 0, "Kmsan vmalloc Origins Start"},
86 [KMSAN_VMALLOC_ORIGIN_END_NR] = {0, 0, "Kmsan vmalloc Origins End"},
87 [KMSAN_MODULES_SHADOW_START_NR] = {1, 0, "Kmsan Modules Shadow Start"},
88 [KMSAN_MODULES_SHADOW_END_NR] = {0, 0, "Kmsan Modules Shadow End"},
89 [KMSAN_MODULES_ORIGIN_START_NR] = {1, 0, "Kmsan Modules Origins Start"},
90 [KMSAN_MODULES_ORIGIN_END_NR] = {0, 0, "Kmsan Modules Origins End"},
91 #endif
92 [MODULES_NR] = {1, 0, "Modules Area Start"},
93 [MODULES_END_NR] = {0, 0, "Modules Area End"},
94 [ABS_LOWCORE_NR] = {1, 0, "Lowcore Area Start"},
95 [ABS_LOWCORE_END_NR] = {0, 0, "Lowcore Area End"},
96 [MEMCPY_REAL_NR] = {1, 0, "Real Memory Copy Area Start"},
97 [MEMCPY_REAL_END_NR] = {0, 0, "Real Memory Copy Area End"},
98 #ifdef CONFIG_KASAN
99 [KASAN_SHADOW_START_NR] = {1, KASAN_SHADOW_START, "Kasan Shadow Start"},
100 [KASAN_SHADOW_END_NR] = {0, KASAN_SHADOW_END, "Kasan Shadow End"},
101 #endif
102 {1, -1UL, NULL}
103 };
104
105 struct pg_state {
106 struct ptdump_state ptdump;
107 struct seq_file *seq;
108 int level;
109 unsigned int current_prot;
110 bool check_wx;
111 unsigned long wx_pages;
112 unsigned long start_address;
113 const struct addr_marker *marker;
114 };
115
116 #define pt_dump_seq_printf(m, fmt, args...) \
117 ({ \
118 struct seq_file *__m = (m); \
119 \
120 if (__m) \
121 seq_printf(__m, fmt, ##args); \
122 })
123
124 #define pt_dump_seq_puts(m, fmt) \
125 ({ \
126 struct seq_file *__m = (m); \
127 \
128 if (__m) \
129 seq_printf(__m, fmt); \
130 })
131
132 static void print_prot(struct seq_file *m, unsigned int pr, int level)
133 {
134 static const char * const level_name[] =
135 { "ASCE", "PGD", "PUD", "PMD", "PTE" };
136
137 pt_dump_seq_printf(m, "%s ", level_name[level]);
138 if (pr & _PAGE_INVALID) {
139 pt_dump_seq_printf(m, "I\n");
140 return;
141 }
142 pt_dump_seq_puts(m, (pr & _PAGE_PROTECT) ? "RO " : "RW ");
143 pt_dump_seq_puts(m, (pr & _PAGE_NOEXEC) ? "NX\n" : "X\n");
144 }
145
146 static void note_prot_wx(struct pg_state *st, unsigned long addr)
147 {
148 if (!st->check_wx)
149 return;
150 if (st->current_prot & _PAGE_INVALID)
151 return;
152 if (st->current_prot & _PAGE_PROTECT)
153 return;
154 if (st->current_prot & _PAGE_NOEXEC)
155 return;
156 /*
157 * The first lowcore page is W+X if spectre mitigations are using
158 * trampolines or the BEAR enhancements facility is not installed,
159 * in which case we have two lpswe instructions in lowcore that need
160 * to be executable.
161 */
162 if (addr == PAGE_SIZE && (nospec_uses_trampoline() || !static_key_enabled(&cpu_has_bear)))
163 return;
164 WARN_ONCE(IS_ENABLED(CONFIG_DEBUG_WX),
165 "s390/mm: Found insecure W+X mapping at address %pS\n",
166 (void *)st->start_address);
167 st->wx_pages += (addr - st->start_address) / PAGE_SIZE;
168 }
169
170 static void note_page_update_state(struct pg_state *st, unsigned long addr, unsigned int prot, int level)
171 {
172 struct seq_file *m = st->seq;
173
174 while (addr >= st->marker[1].start_address) {
175 st->marker++;
176 pt_dump_seq_printf(m, "---[ %s ]---\n", st->marker->name);
177 }
178 st->start_address = addr;
179 st->current_prot = prot;
180 st->level = level;
181 }
182
183 static void note_page(struct ptdump_state *pt_st, unsigned long addr, int level, u64 val)
184 {
185 int width = sizeof(unsigned long) * 2;
186 static const char units[] = "KMGTPE";
187 const char *unit = units;
188 unsigned long delta;
189 struct pg_state *st;
190 struct seq_file *m;
191 unsigned int prot;
192
193 st = container_of(pt_st, struct pg_state, ptdump);
194 m = st->seq;
195 prot = val & (_PAGE_PROTECT | _PAGE_NOEXEC);
196 if (level == 4 && (val & _PAGE_INVALID))
197 prot = _PAGE_INVALID;
198 /* For pmd_none() & friends val gets passed as zero. */
199 if (level != 4 && !val)
200 prot = _PAGE_INVALID;
201 /* Final flush from generic code. */
202 if (level == -1)
203 addr = max_addr;
204 if (st->level == -1) {
205 pt_dump_seq_printf(m, "---[ %s ]---\n", st->marker->name);
206 note_page_update_state(st, addr, prot, level);
207 } else if (prot != st->current_prot || level != st->level ||
208 addr >= st->marker[1].start_address) {
209 note_prot_wx(st, addr);
210 pt_dump_seq_printf(m, "0x%0*lx-0x%0*lx ",
211 width, st->start_address,
212 width, addr);
213 delta = (addr - st->start_address) >> 10;
214 while (!(delta & 0x3ff) && unit[1]) {
215 delta >>= 10;
216 unit++;
217 }
218 pt_dump_seq_printf(m, "%9lu%c ", delta, *unit);
219 print_prot(m, st->current_prot, st->level);
220 note_page_update_state(st, addr, prot, level);
221 }
222 }
223
224 bool ptdump_check_wx(void)
225 {
226 struct pg_state st = {
227 .ptdump = {
228 .note_page = note_page,
229 .range = (struct ptdump_range[]) {
230 {.start = 0, .end = max_addr},
231 {.start = 0, .end = 0},
232 }
233 },
234 .seq = NULL,
235 .level = -1,
236 .current_prot = 0,
237 .check_wx = true,
238 .wx_pages = 0,
239 .start_address = 0,
240 .marker = (struct addr_marker[]) {
241 { .start_address = 0, .name = NULL},
242 { .start_address = -1, .name = NULL},
243 },
244 };
245
246 if (!MACHINE_HAS_NX)
247 return true;
248 ptdump_walk_pgd(&st.ptdump, &init_mm, NULL);
249 if (st.wx_pages) {
250 pr_warn("Checked W+X mappings: FAILED, %lu W+X pages found\n", st.wx_pages);
251
252 return false;
253 } else {
254 pr_info("Checked W+X mappings: passed, no %sW+X pages found\n",
255 (nospec_uses_trampoline() || !static_key_enabled(&cpu_has_bear)) ?
256 "unexpected " : "");
257
258 return true;
259 }
260 }
261
262 #ifdef CONFIG_PTDUMP_DEBUGFS
263 static int ptdump_show(struct seq_file *m, void *v)
264 {
265 struct pg_state st = {
266 .ptdump = {
267 .note_page = note_page,
268 .range = (struct ptdump_range[]) {
269 {.start = 0, .end = max_addr},
270 {.start = 0, .end = 0},
271 }
272 },
273 .seq = m,
274 .level = -1,
275 .current_prot = 0,
276 .check_wx = false,
277 .wx_pages = 0,
278 .start_address = 0,
279 .marker = address_markers,
280 };
281
282 get_online_mems();
283 mutex_lock(&cpa_mutex);
284 ptdump_walk_pgd(&st.ptdump, &init_mm, NULL);
285 mutex_unlock(&cpa_mutex);
286 put_online_mems();
287 return 0;
288 }
289 DEFINE_SHOW_ATTRIBUTE(ptdump);
290 #endif /* CONFIG_PTDUMP_DEBUGFS */
291
292 static int ptdump_cmp(const void *a, const void *b)
293 {
294 const struct addr_marker *ama = a;
295 const struct addr_marker *amb = b;
296
297 if (ama->start_address > amb->start_address)
298 return 1;
299 if (ama->start_address < amb->start_address)
300 return -1;
301 /*
302 * If the start addresses of two markers are identical consider the
303 * marker which defines the start of an area higher than the one which
304 * defines the end of an area. This keeps pairs of markers sorted.
305 */
306 if (ama->is_start)
307 return 1;
308 if (amb->is_start)
309 return -1;
310 return 0;
311 }
312
313 static int pt_dump_init(void)
314 {
315 #ifdef CONFIG_KFENCE
316 unsigned long kfence_start = (unsigned long)__kfence_pool;
317 #endif
318 unsigned long lowcore = (unsigned long)get_lowcore();
319
320 /*
321 * Figure out the maximum virtual address being accessible with the
322 * kernel ASCE. We need this to keep the page table walker functions
323 * from accessing non-existent entries.
324 */
325 max_addr = (get_lowcore()->kernel_asce.val & _REGION_ENTRY_TYPE_MASK) >> 2;
326 max_addr = 1UL << (max_addr * 11 + 31);
327 address_markers[LOWCORE_START_NR].start_address = lowcore;
328 address_markers[LOWCORE_END_NR].start_address = lowcore + sizeof(struct lowcore);
329 address_markers[IDENTITY_START_NR].start_address = __identity_base;
330 address_markers[IDENTITY_END_NR].start_address = __identity_base + ident_map_size;
331 address_markers[AMODE31_START_NR].start_address = (unsigned long)__samode31;
332 address_markers[AMODE31_END_NR].start_address = (unsigned long)__eamode31;
333 address_markers[MODULES_NR].start_address = MODULES_VADDR;
334 address_markers[MODULES_END_NR].start_address = MODULES_END;
335 address_markers[ABS_LOWCORE_NR].start_address = __abs_lowcore;
336 address_markers[ABS_LOWCORE_END_NR].start_address = __abs_lowcore + ABS_LOWCORE_MAP_SIZE;
337 address_markers[MEMCPY_REAL_NR].start_address = __memcpy_real_area;
338 address_markers[MEMCPY_REAL_END_NR].start_address = __memcpy_real_area + MEMCPY_REAL_SIZE;
339 address_markers[VMEMMAP_NR].start_address = (unsigned long) vmemmap;
340 address_markers[VMEMMAP_END_NR].start_address = (unsigned long)vmemmap + vmemmap_size;
341 address_markers[VMALLOC_NR].start_address = VMALLOC_START;
342 address_markers[VMALLOC_END_NR].start_address = VMALLOC_END;
343 #ifdef CONFIG_KFENCE
344 address_markers[KFENCE_START_NR].start_address = kfence_start;
345 address_markers[KFENCE_END_NR].start_address = kfence_start + KFENCE_POOL_SIZE;
346 #endif
347 #ifdef CONFIG_KMSAN
348 address_markers[KMSAN_VMALLOC_SHADOW_START_NR].start_address = KMSAN_VMALLOC_SHADOW_START;
349 address_markers[KMSAN_VMALLOC_SHADOW_END_NR].start_address = KMSAN_VMALLOC_SHADOW_END;
350 address_markers[KMSAN_VMALLOC_ORIGIN_START_NR].start_address = KMSAN_VMALLOC_ORIGIN_START;
351 address_markers[KMSAN_VMALLOC_ORIGIN_END_NR].start_address = KMSAN_VMALLOC_ORIGIN_END;
352 address_markers[KMSAN_MODULES_SHADOW_START_NR].start_address = KMSAN_MODULES_SHADOW_START;
353 address_markers[KMSAN_MODULES_SHADOW_END_NR].start_address = KMSAN_MODULES_SHADOW_END;
354 address_markers[KMSAN_MODULES_ORIGIN_START_NR].start_address = KMSAN_MODULES_ORIGIN_START;
355 address_markers[KMSAN_MODULES_ORIGIN_END_NR].start_address = KMSAN_MODULES_ORIGIN_END;
356 #endif
357 sort(address_markers, ARRAY_SIZE(address_markers) - 1,
358 sizeof(address_markers[0]), ptdump_cmp, NULL);
359 #ifdef CONFIG_PTDUMP_DEBUGFS
360 debugfs_create_file("kernel_page_tables", 0400, NULL, NULL, &ptdump_fops);
361 #endif /* CONFIG_PTDUMP_DEBUGFS */
362 return 0;
363 }
364 device_initcall(pt_dump_init);
365
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.