1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * KMSAN API for subsystems.
4 *
5 * Copyright (C) 2017-2022 Google LLC
6 * Author: Alexander Potapenko <glider@google.com>
7 *
8 */
9 #ifndef _LINUX_KMSAN_H
10 #define _LINUX_KMSAN_H
11
12 #include <linux/dma-direction.h>
13 #include <linux/gfp.h>
14 #include <linux/kmsan-checks.h>
15 #include <linux/types.h>
16
17 struct page;
18 struct kmem_cache;
19 struct task_struct;
20 struct scatterlist;
21 struct urb;
22
23 #ifdef CONFIG_KMSAN
24
25 /**
26 * kmsan_task_create() - Initialize KMSAN state for the task.
27 * @task: task to initialize.
28 */
29 void kmsan_task_create(struct task_struct *task);
30
31 /**
32 * kmsan_task_exit() - Notify KMSAN that a task has exited.
33 * @task: task about to finish.
34 */
35 void kmsan_task_exit(struct task_struct *task);
36
37 /**
38 * kmsan_init_shadow() - Initialize KMSAN shadow at boot time.
39 *
40 * Allocate and initialize KMSAN metadata for early allocations.
41 */
42 void __init kmsan_init_shadow(void);
43
44 /**
45 * kmsan_init_runtime() - Initialize KMSAN state and enable KMSAN.
46 */
47 void __init kmsan_init_runtime(void);
48
49 /**
50 * kmsan_memblock_free_pages() - handle freeing of memblock pages.
51 * @page: struct page to free.
52 * @order: order of @page.
53 *
54 * Freed pages are either returned to buddy allocator or held back to be used
55 * as metadata pages.
56 */
57 bool __init __must_check kmsan_memblock_free_pages(struct page *page,
58 unsigned int order);
59
60 /**
61 * kmsan_alloc_page() - Notify KMSAN about an alloc_pages() call.
62 * @page: struct page pointer returned by alloc_pages().
63 * @order: order of allocated struct page.
64 * @flags: GFP flags used by alloc_pages()
65 *
66 * KMSAN marks 1<<@order pages starting at @page as uninitialized, unless
67 * @flags contain __GFP_ZERO.
68 */
69 void kmsan_alloc_page(struct page *page, unsigned int order, gfp_t flags);
70
71 /**
72 * kmsan_free_page() - Notify KMSAN about a free_pages() call.
73 * @page: struct page pointer passed to free_pages().
74 * @order: order of deallocated struct page.
75 *
76 * KMSAN marks freed memory as uninitialized.
77 */
78 void kmsan_free_page(struct page *page, unsigned int order);
79
80 /**
81 * kmsan_copy_page_meta() - Copy KMSAN metadata between two pages.
82 * @dst: destination page.
83 * @src: source page.
84 *
85 * KMSAN copies the contents of metadata pages for @src into the metadata pages
86 * for @dst. If @dst has no associated metadata pages, nothing happens.
87 * If @src has no associated metadata pages, @dst metadata pages are unpoisoned.
88 */
89 void kmsan_copy_page_meta(struct page *dst, struct page *src);
90
91 /**
92 * kmsan_slab_alloc() - Notify KMSAN about a slab allocation.
93 * @s: slab cache the object belongs to.
94 * @object: object pointer.
95 * @flags: GFP flags passed to the allocator.
96 *
97 * Depending on cache flags and GFP flags, KMSAN sets up the metadata of the
98 * newly created object, marking it as initialized or uninitialized.
99 */
100 void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags);
101
102 /**
103 * kmsan_slab_free() - Notify KMSAN about a slab deallocation.
104 * @s: slab cache the object belongs to.
105 * @object: object pointer.
106 *
107 * KMSAN marks the freed object as uninitialized.
108 */
109 void kmsan_slab_free(struct kmem_cache *s, void *object);
110
111 /**
112 * kmsan_kmalloc_large() - Notify KMSAN about a large slab allocation.
113 * @ptr: object pointer.
114 * @size: object size.
115 * @flags: GFP flags passed to the allocator.
116 *
117 * Similar to kmsan_slab_alloc(), but for large allocations.
118 */
119 void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags);
120
121 /**
122 * kmsan_kfree_large() - Notify KMSAN about a large slab deallocation.
123 * @ptr: object pointer.
124 *
125 * Similar to kmsan_slab_free(), but for large allocations.
126 */
127 void kmsan_kfree_large(const void *ptr);
128
129 /**
130 * kmsan_map_kernel_range_noflush() - Notify KMSAN about a vmap.
131 * @start: start of vmapped range.
132 * @end: end of vmapped range.
133 * @prot: page protection flags used for vmap.
134 * @pages: array of pages.
135 * @page_shift: page_shift passed to vmap_range_noflush().
136 *
137 * KMSAN maps shadow and origin pages of @pages into contiguous ranges in
138 * vmalloc metadata address range. Returns 0 on success, callers must check
139 * for non-zero return value.
140 */
141 int __must_check kmsan_vmap_pages_range_noflush(unsigned long start,
142 unsigned long end,
143 pgprot_t prot,
144 struct page **pages,
145 unsigned int page_shift);
146
147 /**
148 * kmsan_vunmap_kernel_range_noflush() - Notify KMSAN about a vunmap.
149 * @start: start of vunmapped range.
150 * @end: end of vunmapped range.
151 *
152 * KMSAN unmaps the contiguous metadata ranges created by
153 * kmsan_map_kernel_range_noflush().
154 */
155 void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end);
156
157 /**
158 * kmsan_ioremap_page_range() - Notify KMSAN about a ioremap_page_range() call.
159 * @addr: range start.
160 * @end: range end.
161 * @phys_addr: physical range start.
162 * @prot: page protection flags used for ioremap_page_range().
163 * @page_shift: page_shift argument passed to vmap_range_noflush().
164 *
165 * KMSAN creates new metadata pages for the physical pages mapped into the
166 * virtual memory. Returns 0 on success, callers must check for non-zero return
167 * value.
168 */
169 int __must_check kmsan_ioremap_page_range(unsigned long addr, unsigned long end,
170 phys_addr_t phys_addr, pgprot_t prot,
171 unsigned int page_shift);
172
173 /**
174 * kmsan_iounmap_page_range() - Notify KMSAN about a iounmap_page_range() call.
175 * @start: range start.
176 * @end: range end.
177 *
178 * KMSAN unmaps the metadata pages for the given range and, unlike for
179 * vunmap_page_range(), also deallocates them.
180 */
181 void kmsan_iounmap_page_range(unsigned long start, unsigned long end);
182
183 /**
184 * kmsan_handle_dma() - Handle a DMA data transfer.
185 * @page: first page of the buffer.
186 * @offset: offset of the buffer within the first page.
187 * @size: buffer size.
188 * @dir: one of possible dma_data_direction values.
189 *
190 * Depending on @direction, KMSAN:
191 * * checks the buffer, if it is copied to device;
192 * * initializes the buffer, if it is copied from device;
193 * * does both, if this is a DMA_BIDIRECTIONAL transfer.
194 */
195 void kmsan_handle_dma(struct page *page, size_t offset, size_t size,
196 enum dma_data_direction dir);
197
198 /**
199 * kmsan_handle_dma_sg() - Handle a DMA transfer using scatterlist.
200 * @sg: scatterlist holding DMA buffers.
201 * @nents: number of scatterlist entries.
202 * @dir: one of possible dma_data_direction values.
203 *
204 * Depending on @direction, KMSAN:
205 * * checks the buffers in the scatterlist, if they are copied to device;
206 * * initializes the buffers, if they are copied from device;
207 * * does both, if this is a DMA_BIDIRECTIONAL transfer.
208 */
209 void kmsan_handle_dma_sg(struct scatterlist *sg, int nents,
210 enum dma_data_direction dir);
211
212 /**
213 * kmsan_handle_urb() - Handle a USB data transfer.
214 * @urb: struct urb pointer.
215 * @is_out: data transfer direction (true means output to hardware).
216 *
217 * If @is_out is true, KMSAN checks the transfer buffer of @urb. Otherwise,
218 * KMSAN initializes the transfer buffer.
219 */
220 void kmsan_handle_urb(const struct urb *urb, bool is_out);
221
222 /**
223 * kmsan_unpoison_entry_regs() - Handle pt_regs in low-level entry code.
224 * @regs: struct pt_regs pointer received from assembly code.
225 *
226 * KMSAN unpoisons the contents of the passed pt_regs, preventing potential
227 * false positive reports. Unlike kmsan_unpoison_memory(),
228 * kmsan_unpoison_entry_regs() can be called from the regions where
229 * kmsan_in_runtime() returns true, which is the case in early entry code.
230 */
231 void kmsan_unpoison_entry_regs(const struct pt_regs *regs);
232
233 /**
234 * kmsan_get_metadata() - Return a pointer to KMSAN shadow or origins.
235 * @addr: kernel address.
236 * @is_origin: whether to return origins or shadow.
237 *
238 * Return NULL if metadata cannot be found.
239 */
240 void *kmsan_get_metadata(void *addr, bool is_origin);
241
242 /**
243 * kmsan_enable_current(): Enable KMSAN for the current task.
244 *
245 * Each kmsan_enable_current() current call must be preceded by a
246 * kmsan_disable_current() call. These call pairs may be nested.
247 */
248 void kmsan_enable_current(void);
249
250 /**
251 * kmsan_disable_current(): Disable KMSAN for the current task.
252 *
253 * Each kmsan_disable_current() current call must be followed by a
254 * kmsan_enable_current() call. These call pairs may be nested.
255 */
256 void kmsan_disable_current(void);
257
258 /**
259 * memset_no_sanitize_memory(): Fill memory without KMSAN instrumentation.
260 * @s: address of kernel memory to fill.
261 * @c: constant byte to fill the memory with.
262 * @n: number of bytes to fill.
263 *
264 * This is like memset(), but without KMSAN instrumentation.
265 */
266 static inline void *memset_no_sanitize_memory(void *s, int c, size_t n)
267 {
268 return __memset(s, c, n);
269 }
270
271 extern bool kmsan_enabled;
272 extern int panic_on_kmsan;
273
274 /*
275 * KMSAN performs a lot of consistency checks that are currently enabled by
276 * default. BUG_ON is normally discouraged in the kernel, unless used for
277 * debugging, but KMSAN itself is a debugging tool, so it makes little sense to
278 * recover if something goes wrong.
279 */
280 #define KMSAN_WARN_ON(cond) \
281 ({ \
282 const bool __cond = WARN_ON(cond); \
283 if (unlikely(__cond)) { \
284 WRITE_ONCE(kmsan_enabled, false); \
285 if (panic_on_kmsan) { \
286 /* Can't call panic() here because */ \
287 /* of uaccess checks. */ \
288 BUG(); \
289 } \
290 } \
291 __cond; \
292 })
293
294 #else
295
296 static inline void kmsan_init_shadow(void)
297 {
298 }
299
300 static inline void kmsan_init_runtime(void)
301 {
302 }
303
304 static inline bool __must_check kmsan_memblock_free_pages(struct page *page,
305 unsigned int order)
306 {
307 return true;
308 }
309
310 static inline void kmsan_task_create(struct task_struct *task)
311 {
312 }
313
314 static inline void kmsan_task_exit(struct task_struct *task)
315 {
316 }
317
318 static inline void kmsan_alloc_page(struct page *page, unsigned int order,
319 gfp_t flags)
320 {
321 }
322
323 static inline void kmsan_free_page(struct page *page, unsigned int order)
324 {
325 }
326
327 static inline void kmsan_copy_page_meta(struct page *dst, struct page *src)
328 {
329 }
330
331 static inline void kmsan_slab_alloc(struct kmem_cache *s, void *object,
332 gfp_t flags)
333 {
334 }
335
336 static inline void kmsan_slab_free(struct kmem_cache *s, void *object)
337 {
338 }
339
340 static inline void kmsan_kmalloc_large(const void *ptr, size_t size,
341 gfp_t flags)
342 {
343 }
344
345 static inline void kmsan_kfree_large(const void *ptr)
346 {
347 }
348
349 static inline int __must_check kmsan_vmap_pages_range_noflush(
350 unsigned long start, unsigned long end, pgprot_t prot,
351 struct page **pages, unsigned int page_shift)
352 {
353 return 0;
354 }
355
356 static inline void kmsan_vunmap_range_noflush(unsigned long start,
357 unsigned long end)
358 {
359 }
360
361 static inline int __must_check kmsan_ioremap_page_range(unsigned long start,
362 unsigned long end,
363 phys_addr_t phys_addr,
364 pgprot_t prot,
365 unsigned int page_shift)
366 {
367 return 0;
368 }
369
370 static inline void kmsan_iounmap_page_range(unsigned long start,
371 unsigned long end)
372 {
373 }
374
375 static inline void kmsan_handle_dma(struct page *page, size_t offset,
376 size_t size, enum dma_data_direction dir)
377 {
378 }
379
380 static inline void kmsan_handle_dma_sg(struct scatterlist *sg, int nents,
381 enum dma_data_direction dir)
382 {
383 }
384
385 static inline void kmsan_handle_urb(const struct urb *urb, bool is_out)
386 {
387 }
388
389 static inline void kmsan_unpoison_entry_regs(const struct pt_regs *regs)
390 {
391 }
392
393 static inline void kmsan_enable_current(void)
394 {
395 }
396
397 static inline void kmsan_disable_current(void)
398 {
399 }
400
401 static inline void *memset_no_sanitize_memory(void *s, int c, size_t n)
402 {
403 return memset(s, c, n);
404 }
405
406 #define KMSAN_WARN_ON WARN_ON
407
408 #endif
409
410 #endif /* _LINUX_KMSAN_H */
411
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.