1 KCOV: code coverage for fuzzing !! 1 kcov: code coverage for fuzzing
2 =============================== 2 ===============================
3 3
4 KCOV collects and exposes kernel code coverage !! 4 kcov exposes kernel code coverage information in a form suitable for coverage-
5 for coverage-guided fuzzing. Coverage data of !! 5 guided fuzzing (randomized testing). Coverage data of a running kernel is
6 the ``kcov`` debugfs file. Coverage collection !! 6 exported via the "kcov" debugfs file. Coverage collection is enabled on a task
7 thus KCOV can capture precise coverage of a si !! 7 basis, and thus it can capture precise coverage of a single system call.
8 !! 8
9 Note that KCOV does not aim to collect as much !! 9 Note that kcov does not aim to collect as much coverage as possible. It aims
10 to collect more or less stable coverage that i !! 10 to collect more or less stable coverage that is function of syscall inputs.
11 To achieve this goal, it does not collect cove !! 11 To achieve this goal it does not collect coverage in soft/hard interrupts
12 (unless remove coverage collection is enabled, !! 12 and instrumentation of some inherently non-deterministic parts of kernel is
13 inherently non-deterministic parts of the kern !! 13 disabled (e.g. scheduler, locking).
14 !! 14
15 Besides collecting code coverage, KCOV can als !! 15 kcov is also able to collect comparison operands from the instrumented code
16 See the "Comparison operands collection" secti !! 16 (this feature currently requires that the kernel is compiled with clang).
17 <<
18 Besides collecting coverage data from syscall <<
19 coverage for annotated parts of the kernel exe <<
20 tasks or soft interrupts. See the "Remote cove <<
21 details. <<
22 17
23 Prerequisites 18 Prerequisites
24 ------------- 19 -------------
25 20
26 KCOV relies on compiler instrumentation and re !! 21 Configure the kernel with::
27 or any Clang version supported by the kernel. <<
28 <<
29 Collecting comparison operands is supported wi <<
30 <<
31 To enable KCOV, configure the kernel with:: <<
32 22
33 CONFIG_KCOV=y 23 CONFIG_KCOV=y
34 24
35 To enable comparison operands collection, set: !! 25 CONFIG_KCOV requires gcc 6.1.0 or later.
>> 26
>> 27 If the comparison operands need to be collected, set::
36 28
37 CONFIG_KCOV_ENABLE_COMPARISONS=y 29 CONFIG_KCOV_ENABLE_COMPARISONS=y
38 30
39 Coverage data only becomes accessible once deb !! 31 Profiling data will only become accessible once debugfs has been mounted::
40 32
41 mount -t debugfs none /sys/kernel/debu 33 mount -t debugfs none /sys/kernel/debug
42 34
43 Coverage collection 35 Coverage collection
44 ------------------- 36 -------------------
45 !! 37 The following program demonstrates coverage collection from within a test
46 The following program demonstrates how to use !! 38 program using kcov:
47 single syscall from within a test program: <<
48 39
49 .. code-block:: c 40 .. code-block:: c
50 41
51 #include <stdio.h> 42 #include <stdio.h>
52 #include <stddef.h> 43 #include <stddef.h>
53 #include <stdint.h> 44 #include <stdint.h>
54 #include <stdlib.h> 45 #include <stdlib.h>
55 #include <sys/types.h> 46 #include <sys/types.h>
56 #include <sys/stat.h> 47 #include <sys/stat.h>
57 #include <sys/ioctl.h> 48 #include <sys/ioctl.h>
58 #include <sys/mman.h> 49 #include <sys/mman.h>
59 #include <unistd.h> 50 #include <unistd.h>
60 #include <fcntl.h> 51 #include <fcntl.h>
61 #include <linux/types.h> <<
62 52
63 #define KCOV_INIT_TRACE 53 #define KCOV_INIT_TRACE _IOR('c', 1, unsigned long)
64 #define KCOV_ENABLE _IO('c 54 #define KCOV_ENABLE _IO('c', 100)
65 #define KCOV_DISABLE 55 #define KCOV_DISABLE _IO('c', 101)
66 #define COVER_SIZE (64<<1 56 #define COVER_SIZE (64<<10)
67 57
68 #define KCOV_TRACE_PC 0 58 #define KCOV_TRACE_PC 0
69 #define KCOV_TRACE_CMP 1 59 #define KCOV_TRACE_CMP 1
70 60
71 int main(int argc, char **argv) 61 int main(int argc, char **argv)
72 { 62 {
73 int fd; 63 int fd;
74 unsigned long *cover, n, i; 64 unsigned long *cover, n, i;
75 65
76 /* A single fd descriptor allows cover 66 /* A single fd descriptor allows coverage collection on a single
77 * thread. 67 * thread.
78 */ 68 */
79 fd = open("/sys/kernel/debug/kcov", O_ 69 fd = open("/sys/kernel/debug/kcov", O_RDWR);
80 if (fd == -1) 70 if (fd == -1)
81 perror("open"), exit(1); 71 perror("open"), exit(1);
82 /* Setup trace mode and trace size. */ 72 /* Setup trace mode and trace size. */
83 if (ioctl(fd, KCOV_INIT_TRACE, COVER_S 73 if (ioctl(fd, KCOV_INIT_TRACE, COVER_SIZE))
84 perror("ioctl"), exit(1); 74 perror("ioctl"), exit(1);
85 /* Mmap buffer shared between kernel- 75 /* Mmap buffer shared between kernel- and user-space. */
86 cover = (unsigned long*)mmap(NULL, COV 76 cover = (unsigned long*)mmap(NULL, COVER_SIZE * sizeof(unsigned long),
87 PROT_READ 77 PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
88 if ((void*)cover == MAP_FAILED) 78 if ((void*)cover == MAP_FAILED)
89 perror("mmap"), exit(1); 79 perror("mmap"), exit(1);
90 /* Enable coverage collection on the c 80 /* Enable coverage collection on the current thread. */
91 if (ioctl(fd, KCOV_ENABLE, KCOV_TRACE_ 81 if (ioctl(fd, KCOV_ENABLE, KCOV_TRACE_PC))
92 perror("ioctl"), exit(1); 82 perror("ioctl"), exit(1);
93 /* Reset coverage from the tail of the 83 /* Reset coverage from the tail of the ioctl() call. */
94 __atomic_store_n(&cover[0], 0, __ATOMI 84 __atomic_store_n(&cover[0], 0, __ATOMIC_RELAXED);
95 /* Call the target syscall call. */ !! 85 /* That's the target syscal call. */
96 read(-1, NULL, 0); 86 read(-1, NULL, 0);
97 /* Read number of PCs collected. */ 87 /* Read number of PCs collected. */
98 n = __atomic_load_n(&cover[0], __ATOMI 88 n = __atomic_load_n(&cover[0], __ATOMIC_RELAXED);
99 for (i = 0; i < n; i++) 89 for (i = 0; i < n; i++)
100 printf("0x%lx\n", cover[i + 1] 90 printf("0x%lx\n", cover[i + 1]);
101 /* Disable coverage collection for the 91 /* Disable coverage collection for the current thread. After this call
102 * coverage can be enabled for a diffe 92 * coverage can be enabled for a different thread.
103 */ 93 */
104 if (ioctl(fd, KCOV_DISABLE, 0)) 94 if (ioctl(fd, KCOV_DISABLE, 0))
105 perror("ioctl"), exit(1); 95 perror("ioctl"), exit(1);
106 /* Free resources. */ 96 /* Free resources. */
107 if (munmap(cover, COVER_SIZE * sizeof( 97 if (munmap(cover, COVER_SIZE * sizeof(unsigned long)))
108 perror("munmap"), exit(1); 98 perror("munmap"), exit(1);
109 if (close(fd)) 99 if (close(fd))
110 perror("close"), exit(1); 100 perror("close"), exit(1);
111 return 0; 101 return 0;
112 } 102 }
113 103
114 After piping through ``addr2line`` the output !! 104 After piping through addr2line output of the program looks as follows::
115 105
116 SyS_read 106 SyS_read
117 fs/read_write.c:562 107 fs/read_write.c:562
118 __fdget_pos 108 __fdget_pos
119 fs/file.c:774 109 fs/file.c:774
120 __fget_light 110 __fget_light
121 fs/file.c:746 111 fs/file.c:746
122 __fget_light 112 __fget_light
123 fs/file.c:750 113 fs/file.c:750
124 __fget_light 114 __fget_light
125 fs/file.c:760 115 fs/file.c:760
126 __fdget_pos 116 __fdget_pos
127 fs/file.c:784 117 fs/file.c:784
128 SyS_read 118 SyS_read
129 fs/read_write.c:562 119 fs/read_write.c:562
130 120
131 If a program needs to collect coverage from se 121 If a program needs to collect coverage from several threads (independently),
132 it needs to open ``/sys/kernel/debug/kcov`` in !! 122 it needs to open /sys/kernel/debug/kcov in each thread separately.
133 123
134 The interface is fine-grained to allow efficie 124 The interface is fine-grained to allow efficient forking of test processes.
135 That is, a parent process opens ``/sys/kernel/ !! 125 That is, a parent process opens /sys/kernel/debug/kcov, enables trace mode,
136 mmaps coverage buffer, and then forks child pr !! 126 mmaps coverage buffer and then forks child processes in a loop. Child processes
137 processes only need to enable coverage (it get !! 127 only need to enable coverage (disable happens automatically on thread end).
138 a thread exits). <<
139 128
140 Comparison operands collection 129 Comparison operands collection
141 ------------------------------ 130 ------------------------------
142 <<
143 Comparison operands collection is similar to c 131 Comparison operands collection is similar to coverage collection:
144 132
145 .. code-block:: c 133 .. code-block:: c
146 134
147 /* Same includes and defines as above. */ 135 /* Same includes and defines as above. */
148 136
149 /* Number of 64-bit words per record. */ 137 /* Number of 64-bit words per record. */
150 #define KCOV_WORDS_PER_CMP 4 138 #define KCOV_WORDS_PER_CMP 4
151 139
152 /* 140 /*
153 * The format for the types of collected c 141 * The format for the types of collected comparisons.
154 * 142 *
155 * Bit 0 shows whether one of the argument 143 * Bit 0 shows whether one of the arguments is a compile-time constant.
156 * Bits 1 & 2 contain log2 of the argument 144 * Bits 1 & 2 contain log2 of the argument size, up to 8 bytes.
157 */ 145 */
158 146
159 #define KCOV_CMP_CONST (1 << 0) 147 #define KCOV_CMP_CONST (1 << 0)
160 #define KCOV_CMP_SIZE(n) ((n) << 1) 148 #define KCOV_CMP_SIZE(n) ((n) << 1)
161 #define KCOV_CMP_MASK KCOV_CMP_S 149 #define KCOV_CMP_MASK KCOV_CMP_SIZE(3)
162 150
163 int main(int argc, char **argv) 151 int main(int argc, char **argv)
164 { 152 {
165 int fd; 153 int fd;
166 uint64_t *cover, type, arg1, arg2, is_ 154 uint64_t *cover, type, arg1, arg2, is_const, size;
167 unsigned long n, i; 155 unsigned long n, i;
168 156
169 fd = open("/sys/kernel/debug/kcov", O_ 157 fd = open("/sys/kernel/debug/kcov", O_RDWR);
170 if (fd == -1) 158 if (fd == -1)
171 perror("open"), exit(1); 159 perror("open"), exit(1);
172 if (ioctl(fd, KCOV_INIT_TRACE, COVER_S 160 if (ioctl(fd, KCOV_INIT_TRACE, COVER_SIZE))
173 perror("ioctl"), exit(1); 161 perror("ioctl"), exit(1);
174 /* 162 /*
175 * Note that the buffer pointer is of t 163 * Note that the buffer pointer is of type uint64_t*, because all
176 * the comparison operands are promoted 164 * the comparison operands are promoted to uint64_t.
177 */ 165 */
178 cover = (uint64_t *)mmap(NULL, COVER_S 166 cover = (uint64_t *)mmap(NULL, COVER_SIZE * sizeof(unsigned long),
179 PROT_READ 167 PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
180 if ((void*)cover == MAP_FAILED) 168 if ((void*)cover == MAP_FAILED)
181 perror("mmap"), exit(1); 169 perror("mmap"), exit(1);
182 /* Note KCOV_TRACE_CMP instead of KCOV 170 /* Note KCOV_TRACE_CMP instead of KCOV_TRACE_PC. */
183 if (ioctl(fd, KCOV_ENABLE, KCOV_TRACE_ 171 if (ioctl(fd, KCOV_ENABLE, KCOV_TRACE_CMP))
184 perror("ioctl"), exit(1); 172 perror("ioctl"), exit(1);
185 __atomic_store_n(&cover[0], 0, __ATOMI 173 __atomic_store_n(&cover[0], 0, __ATOMIC_RELAXED);
186 read(-1, NULL, 0); 174 read(-1, NULL, 0);
187 /* Read number of comparisons collecte 175 /* Read number of comparisons collected. */
188 n = __atomic_load_n(&cover[0], __ATOMI 176 n = __atomic_load_n(&cover[0], __ATOMIC_RELAXED);
189 for (i = 0; i < n; i++) { 177 for (i = 0; i < n; i++) {
190 uint64_t ip; <<
191 <<
192 type = cover[i * KCOV_WORDS_PE 178 type = cover[i * KCOV_WORDS_PER_CMP + 1];
193 /* arg1 and arg2 - operands of 179 /* arg1 and arg2 - operands of the comparison. */
194 arg1 = cover[i * KCOV_WORDS_PE 180 arg1 = cover[i * KCOV_WORDS_PER_CMP + 2];
195 arg2 = cover[i * KCOV_WORDS_PE 181 arg2 = cover[i * KCOV_WORDS_PER_CMP + 3];
196 /* ip - caller address. */ 182 /* ip - caller address. */
197 ip = cover[i * KCOV_WORDS_PER_ 183 ip = cover[i * KCOV_WORDS_PER_CMP + 4];
198 /* size of the operands. */ 184 /* size of the operands. */
199 size = 1 << ((type & KCOV_CMP_ 185 size = 1 << ((type & KCOV_CMP_MASK) >> 1);
200 /* is_const - true if either o 186 /* is_const - true if either operand is a compile-time constant.*/
201 is_const = type & KCOV_CMP_CON 187 is_const = type & KCOV_CMP_CONST;
202 printf("ip: 0x%lx type: 0x%lx, 188 printf("ip: 0x%lx type: 0x%lx, arg1: 0x%lx, arg2: 0x%lx, "
203 "size: %lu, %s\n", 189 "size: %lu, %s\n",
204 ip, type, arg1, arg2, 190 ip, type, arg1, arg2, size,
205 is_const ? "const" : "non-cons 191 is_const ? "const" : "non-const");
206 } 192 }
207 if (ioctl(fd, KCOV_DISABLE, 0)) 193 if (ioctl(fd, KCOV_DISABLE, 0))
208 perror("ioctl"), exit(1); 194 perror("ioctl"), exit(1);
209 /* Free resources. */ 195 /* Free resources. */
210 if (munmap(cover, COVER_SIZE * sizeof( 196 if (munmap(cover, COVER_SIZE * sizeof(unsigned long)))
211 perror("munmap"), exit(1); 197 perror("munmap"), exit(1);
212 if (close(fd)) 198 if (close(fd))
213 perror("close"), exit(1); 199 perror("close"), exit(1);
214 return 0; 200 return 0;
215 } 201 }
216 202
217 Note that the KCOV modes (collection of code c !! 203 Note that the kcov modes (coverage collection or comparison operands) are
218 are mutually exclusive. !! 204 mutually exclusive.
219 <<
220 Remote coverage collection <<
221 -------------------------- <<
222 <<
223 Besides collecting coverage data from handlers <<
224 userspace process, KCOV can also collect cover <<
225 executing in other contexts - so-called "remot <<
226 <<
227 Using KCOV to collect remote coverage requires <<
228 <<
229 1. Modifying kernel code to annotate the code <<
230 should be collected with ``kcov_remote_star <<
231 <<
232 2. Using ``KCOV_REMOTE_ENABLE`` instead of ``K <<
233 process that collects coverage. <<
234 <<
235 Both ``kcov_remote_start`` and ``kcov_remote_s <<
236 ``KCOV_REMOTE_ENABLE`` ioctl accept handles th <<
237 collection sections. The way a handle is used <<
238 matching code section executes. <<
239 <<
240 KCOV supports collecting remote coverage from <<
241 <<
242 1. Global kernel background tasks. These are t <<
243 kernel boot in a limited number of instance <<
244 worker is spawned per one USB HCD). <<
245 <<
246 2. Local kernel background tasks. These are sp <<
247 interacts with some kernel interface and ar <<
248 exits (e.g. vhost workers). <<
249 <<
250 3. Soft interrupts. <<
251 <<
252 For #1 and #3, a unique global handle must be <<
253 corresponding ``kcov_remote_start`` call. Then <<
254 this handle to ``KCOV_REMOTE_ENABLE`` in the ` <<
255 ``kcov_remote_arg`` struct. This will attach t <<
256 section referenced by this handle. Multiple gl <<
257 different code sections can be passed at once. <<
258 <<
259 For #2, the userspace process instead must pas <<
260 ``common_handle`` field of the ``kcov_remote_a <<
261 gets saved to the ``kcov_handle`` field in the <<
262 needs to be passed to the newly spawned local <<
263 modifications. Those tasks should in turn use <<
264 ``kcov_remote_start`` and ``kcov_remote_stop`` <<
265 <<
266 KCOV follows a predefined format for both glob <<
267 handle is a ``u64`` integer. Currently, only t <<
268 are used. Bytes 4-7 are reserved and must be z <<
269 <<
270 For global handles, the top byte of the handle <<
271 this handle belongs to. For example, KCOV uses <<
272 The lower 4 bytes of a global handle denote th <<
273 that subsystem. For example, each ``hub_event` <<
274 as the task instance id. <<
275 <<
276 For common handles, a reserved value ``0`` is <<
277 handles don't belong to a particular subsystem <<
278 handle identify a collective instance of all l <<
279 userspace process that passed a common handle <<
280 <<
281 In practice, any value can be used for common <<
282 is only collected from a single userspace proc <<
283 common handles are used by multiple processes, <<
284 used for each process. One option is to use th <<
285 handle instance id. <<
286 <<
287 The following program demonstrates using KCOV <<
288 local tasks spawned by the process and the glo <<
289 <<
290 .. code-block:: c <<
291 <<
292 /* Same includes and defines as above. */ <<
293 <<
294 struct kcov_remote_arg { <<
295 __u32 trace_mode; <<
296 __u32 area_size; <<
297 __u32 num_handles; <<
298 __aligned_u64 common_handle; <<
299 __aligned_u64 handles[0]; <<
300 }; <<
301 <<
302 #define KCOV_INIT_TRACE <<
303 #define KCOV_DISABLE <<
304 #define KCOV_REMOTE_ENABLE _IOW(' <<
305 <<
306 #define COVER_SIZE (64 << 10) <<
307 <<
308 #define KCOV_TRACE_PC 0 <<
309 <<
310 #define KCOV_SUBSYSTEM_COMMON (0x00u <<
311 #define KCOV_SUBSYSTEM_USB (0x01ull << 56 <<
312 <<
313 #define KCOV_SUBSYSTEM_MASK (0xffull << 56 <<
314 #define KCOV_INSTANCE_MASK (0xffffffffull <<
315 <<
316 static inline __u64 kcov_remote_handle(__u <<
317 { <<
318 if (subsys & ~KCOV_SUBSYSTEM_MASK || i <<
319 return 0; <<
320 return subsys | inst; <<
321 } <<
322 <<
323 #define KCOV_COMMON_ID 0x42 <<
324 #define KCOV_USB_BUS_NUM 1 <<
325 <<
326 int main(int argc, char **argv) <<
327 { <<
328 int fd; <<
329 unsigned long *cover, n, i; <<
330 struct kcov_remote_arg *arg; <<
331 <<
332 fd = open("/sys/kernel/debug/kcov", O_ <<
333 if (fd == -1) <<
334 perror("open"), exit(1); <<
335 if (ioctl(fd, KCOV_INIT_TRACE, COVER_S <<
336 perror("ioctl"), exit(1); <<
337 cover = (unsigned long*)mmap(NULL, COV <<
338 PROT_READ <<
339 if ((void*)cover == MAP_FAILED) <<
340 perror("mmap"), exit(1); <<
341 <<
342 /* Enable coverage collection via comm <<
343 arg = calloc(1, sizeof(*arg) + sizeof( <<
344 if (!arg) <<
345 perror("calloc"), exit(1); <<
346 arg->trace_mode = KCOV_TRACE_PC; <<
347 arg->area_size = COVER_SIZE; <<
348 arg->num_handles = 1; <<
349 arg->common_handle = kcov_remote_handl <<
350 <<
351 arg->handles[0] = kcov_remote_handle(K <<
352 <<
353 if (ioctl(fd, KCOV_REMOTE_ENABLE, arg) <<
354 perror("ioctl"), free(arg), ex <<
355 free(arg); <<
356 <<
357 /* <<
358 * Here the user needs to trigger exec <<
359 * that is either annotated with the c <<
360 * activity on USB bus #1. <<
361 */ <<
362 sleep(2); <<
363 <<
364 n = __atomic_load_n(&cover[0], __ATOMI <<
365 for (i = 0; i < n; i++) <<
366 printf("0x%lx\n", cover[i + 1] <<
367 if (ioctl(fd, KCOV_DISABLE, 0)) <<
368 perror("ioctl"), exit(1); <<
369 if (munmap(cover, COVER_SIZE * sizeof( <<
370 perror("munmap"), exit(1); <<
371 if (close(fd)) <<
372 perror("close"), exit(1); <<
373 return 0; <<
374 } <<
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.