1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */
2 2
3 /* 3 /*
4 * This header provides generic wrappers for m 4 * This header provides generic wrappers for memory access instrumentation that
5 * the compiler cannot emit for: KASAN, KCSAN, 5 * the compiler cannot emit for: KASAN, KCSAN, KMSAN.
6 */ 6 */
7 #ifndef _LINUX_INSTRUMENTED_H 7 #ifndef _LINUX_INSTRUMENTED_H
8 #define _LINUX_INSTRUMENTED_H 8 #define _LINUX_INSTRUMENTED_H
9 9
10 #include <linux/compiler.h> 10 #include <linux/compiler.h>
11 #include <linux/kasan-checks.h> 11 #include <linux/kasan-checks.h>
12 #include <linux/kcsan-checks.h> 12 #include <linux/kcsan-checks.h>
13 #include <linux/kmsan-checks.h> 13 #include <linux/kmsan-checks.h>
14 #include <linux/types.h> 14 #include <linux/types.h>
15 15
16 /** 16 /**
17 * instrument_read - instrument regular read a 17 * instrument_read - instrument regular read access
18 * @v: address of access 18 * @v: address of access
19 * @size: size of access 19 * @size: size of access
20 * 20 *
21 * Instrument a regular read access. The instr 21 * Instrument a regular read access. The instrumentation should be inserted
22 * before the actual read happens. 22 * before the actual read happens.
23 */ 23 */
24 static __always_inline void instrument_read(co 24 static __always_inline void instrument_read(const volatile void *v, size_t size)
25 { 25 {
26 kasan_check_read(v, size); 26 kasan_check_read(v, size);
27 kcsan_check_read(v, size); 27 kcsan_check_read(v, size);
28 } 28 }
29 29
30 /** 30 /**
31 * instrument_write - instrument regular write 31 * instrument_write - instrument regular write access
32 * @v: address of access 32 * @v: address of access
33 * @size: size of access 33 * @size: size of access
34 * 34 *
35 * Instrument a regular write access. The inst 35 * Instrument a regular write access. The instrumentation should be inserted
36 * before the actual write happens. 36 * before the actual write happens.
37 */ 37 */
38 static __always_inline void instrument_write(c 38 static __always_inline void instrument_write(const volatile void *v, size_t size)
39 { 39 {
40 kasan_check_write(v, size); 40 kasan_check_write(v, size);
41 kcsan_check_write(v, size); 41 kcsan_check_write(v, size);
42 } 42 }
43 43
44 /** 44 /**
45 * instrument_read_write - instrument regular 45 * instrument_read_write - instrument regular read-write access
46 * @v: address of access 46 * @v: address of access
47 * @size: size of access 47 * @size: size of access
48 * 48 *
49 * Instrument a regular write access. The inst 49 * Instrument a regular write access. The instrumentation should be inserted
50 * before the actual write happens. 50 * before the actual write happens.
51 */ 51 */
52 static __always_inline void instrument_read_wr 52 static __always_inline void instrument_read_write(const volatile void *v, size_t size)
53 { 53 {
54 kasan_check_write(v, size); 54 kasan_check_write(v, size);
55 kcsan_check_read_write(v, size); 55 kcsan_check_read_write(v, size);
56 } 56 }
57 57
58 /** 58 /**
59 * instrument_atomic_read - instrument atomic 59 * instrument_atomic_read - instrument atomic read access
60 * @v: address of access 60 * @v: address of access
61 * @size: size of access 61 * @size: size of access
62 * 62 *
63 * Instrument an atomic read access. The instr 63 * Instrument an atomic read access. The instrumentation should be inserted
64 * before the actual read happens. 64 * before the actual read happens.
65 */ 65 */
66 static __always_inline void instrument_atomic_ 66 static __always_inline void instrument_atomic_read(const volatile void *v, size_t size)
67 { 67 {
68 kasan_check_read(v, size); 68 kasan_check_read(v, size);
69 kcsan_check_atomic_read(v, size); 69 kcsan_check_atomic_read(v, size);
70 } 70 }
71 71
72 /** 72 /**
73 * instrument_atomic_write - instrument atomic 73 * instrument_atomic_write - instrument atomic write access
74 * @v: address of access 74 * @v: address of access
75 * @size: size of access 75 * @size: size of access
76 * 76 *
77 * Instrument an atomic write access. The inst 77 * Instrument an atomic write access. The instrumentation should be inserted
78 * before the actual write happens. 78 * before the actual write happens.
79 */ 79 */
80 static __always_inline void instrument_atomic_ 80 static __always_inline void instrument_atomic_write(const volatile void *v, size_t size)
81 { 81 {
82 kasan_check_write(v, size); 82 kasan_check_write(v, size);
83 kcsan_check_atomic_write(v, size); 83 kcsan_check_atomic_write(v, size);
84 } 84 }
85 85
86 /** 86 /**
87 * instrument_atomic_read_write - instrument a 87 * instrument_atomic_read_write - instrument atomic read-write access
88 * @v: address of access 88 * @v: address of access
89 * @size: size of access 89 * @size: size of access
90 * 90 *
91 * Instrument an atomic read-write access. The 91 * Instrument an atomic read-write access. The instrumentation should be
92 * inserted before the actual write happens. 92 * inserted before the actual write happens.
93 */ 93 */
94 static __always_inline void instrument_atomic_ 94 static __always_inline void instrument_atomic_read_write(const volatile void *v, size_t size)
95 { 95 {
96 kasan_check_write(v, size); 96 kasan_check_write(v, size);
97 kcsan_check_atomic_read_write(v, size) 97 kcsan_check_atomic_read_write(v, size);
98 } 98 }
99 99
100 /** 100 /**
101 * instrument_copy_to_user - instrument reads 101 * instrument_copy_to_user - instrument reads of copy_to_user
102 * @to: destination address 102 * @to: destination address
103 * @from: source address 103 * @from: source address
104 * @n: number of bytes to copy 104 * @n: number of bytes to copy
105 * 105 *
106 * Instrument reads from kernel memory, that a 106 * Instrument reads from kernel memory, that are due to copy_to_user (and
107 * variants). The instrumentation must be inse 107 * variants). The instrumentation must be inserted before the accesses.
108 */ 108 */
109 static __always_inline void 109 static __always_inline void
110 instrument_copy_to_user(void __user *to, const 110 instrument_copy_to_user(void __user *to, const void *from, unsigned long n)
111 { 111 {
112 kasan_check_read(from, n); 112 kasan_check_read(from, n);
113 kcsan_check_read(from, n); 113 kcsan_check_read(from, n);
114 kmsan_copy_to_user(to, from, n, 0); 114 kmsan_copy_to_user(to, from, n, 0);
115 } 115 }
116 116
117 /** 117 /**
118 * instrument_copy_from_user_before - add inst 118 * instrument_copy_from_user_before - add instrumentation before copy_from_user
119 * @to: destination address 119 * @to: destination address
120 * @from: source address 120 * @from: source address
121 * @n: number of bytes to copy 121 * @n: number of bytes to copy
122 * 122 *
123 * Instrument writes to kernel memory, that ar 123 * Instrument writes to kernel memory, that are due to copy_from_user (and
124 * variants). The instrumentation should be in 124 * variants). The instrumentation should be inserted before the accesses.
125 */ 125 */
126 static __always_inline void 126 static __always_inline void
127 instrument_copy_from_user_before(const void *t 127 instrument_copy_from_user_before(const void *to, const void __user *from, unsigned long n)
128 { 128 {
129 kasan_check_write(to, n); 129 kasan_check_write(to, n);
130 kcsan_check_write(to, n); 130 kcsan_check_write(to, n);
131 } 131 }
132 132
133 /** 133 /**
134 * instrument_copy_from_user_after - add instr 134 * instrument_copy_from_user_after - add instrumentation after copy_from_user
135 * @to: destination address 135 * @to: destination address
136 * @from: source address 136 * @from: source address
137 * @n: number of bytes to copy 137 * @n: number of bytes to copy
138 * @left: number of bytes not copied (as retur 138 * @left: number of bytes not copied (as returned by copy_from_user)
139 * 139 *
140 * Instrument writes to kernel memory, that ar 140 * Instrument writes to kernel memory, that are due to copy_from_user (and
141 * variants). The instrumentation should be in 141 * variants). The instrumentation should be inserted after the accesses.
142 */ 142 */
143 static __always_inline void 143 static __always_inline void
144 instrument_copy_from_user_after(const void *to 144 instrument_copy_from_user_after(const void *to, const void __user *from,
145 unsigned long 145 unsigned long n, unsigned long left)
146 { 146 {
147 kmsan_unpoison_memory(to, n - left); 147 kmsan_unpoison_memory(to, n - left);
148 } 148 }
149 149
150 /** 150 /**
151 * instrument_memcpy_before - add instrumentat <<
152 * @to: destination address <<
153 * @from: source address <<
154 * @n: number of bytes to copy <<
155 * <<
156 * Instrument memory accesses that happen in c <<
157 * instrumentation should be inserted before t <<
158 */ <<
159 static __always_inline void instrument_memcpy_ <<
160 <<
161 { <<
162 kasan_check_write(to, n); <<
163 kasan_check_read(from, n); <<
164 kcsan_check_write(to, n); <<
165 kcsan_check_read(from, n); <<
166 } <<
167 <<
168 /** <<
169 * instrument_memcpy_after - add instrumentati <<
170 * @to: destination address <<
171 * @from: source address <<
172 * @n: number of bytes to copy <<
173 * @left: number of bytes not copied (if known <<
174 * <<
175 * Instrument memory accesses that happen in c <<
176 * instrumentation should be inserted after th <<
177 */ <<
178 static __always_inline void instrument_memcpy_ <<
179 <<
180 <<
181 { <<
182 kmsan_memmove(to, from, n - left); <<
183 } <<
184 <<
185 /** <<
186 * instrument_get_user() - add instrumentation 151 * instrument_get_user() - add instrumentation to get_user()-like macros
187 * @to: destination variable, may not be addre 152 * @to: destination variable, may not be address-taken
188 * 153 *
189 * get_user() and friends are fragile, so it m 154 * get_user() and friends are fragile, so it may depend on the implementation
190 * whether the instrumentation happens before 155 * whether the instrumentation happens before or after the data is copied from
191 * the userspace. 156 * the userspace.
192 */ 157 */
193 #define instrument_get_user(to) 158 #define instrument_get_user(to) \
194 ({ 159 ({ \
195 u64 __tmp = (u64)(to); 160 u64 __tmp = (u64)(to); \
196 kmsan_unpoison_memory(&__tmp, sizeof(_ 161 kmsan_unpoison_memory(&__tmp, sizeof(__tmp)); \
197 to = __tmp; 162 to = __tmp; \
198 }) 163 })
199 164
200 165
201 /** 166 /**
202 * instrument_put_user() - add instrumentation 167 * instrument_put_user() - add instrumentation to put_user()-like macros
203 * @from: source address 168 * @from: source address
204 * @ptr: userspace pointer to copy to 169 * @ptr: userspace pointer to copy to
205 * @size: number of bytes to copy 170 * @size: number of bytes to copy
206 * 171 *
207 * put_user() and friends are fragile, so it m 172 * put_user() and friends are fragile, so it may depend on the implementation
208 * whether the instrumentation happens before 173 * whether the instrumentation happens before or after the data is copied from
209 * the userspace. 174 * the userspace.
210 */ 175 */
211 #define instrument_put_user(from, ptr, size) 176 #define instrument_put_user(from, ptr, size) \
212 ({ 177 ({ \
213 kmsan_copy_to_user(ptr, &from, sizeof( 178 kmsan_copy_to_user(ptr, &from, sizeof(from), 0); \
214 }) 179 })
215 180
216 #endif /* _LINUX_INSTRUMENTED_H */ 181 #endif /* _LINUX_INSTRUMENTED_H */
217 182
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.