1 KCOV: code coverage for fuzzing
2 ===============================
3
4 KCOV collects and exposes kernel code coverage information in a form suitable
5 for coverage-guided fuzzing. Coverage data of a running kernel is exported via
6 the ``kcov`` debugfs file. Coverage collection is enabled on a task basis, and
7 thus KCOV can capture precise coverage of a single system call.
8
9 Note that KCOV does not aim to collect as much coverage as possible. It aims
10 to collect more or less stable coverage that is a function of syscall inputs.
11 To achieve this goal, it does not collect coverage in soft/hard interrupts
12 (unless remove coverage collection is enabled, see below) and from some
13 inherently non-deterministic parts of the kernel (e.g. scheduler, locking).
14
15 Besides collecting code coverage, KCOV can also collect comparison operands.
16 See the "Comparison operands collection" section for details.
17
18 Besides collecting coverage data from syscall handlers, KCOV can also collect
19 coverage for annotated parts of the kernel executing in background kernel
20 tasks or soft interrupts. See the "Remote coverage collection" section for
21 details.
22
23 Prerequisites
24 -------------
25
26 KCOV relies on compiler instrumentation and requires GCC 6.1.0 or later
27 or any Clang version supported by the kernel.
28
29 Collecting comparison operands is supported with GCC 8+ or with Clang.
30
31 To enable KCOV, configure the kernel with::
32
33 CONFIG_KCOV=y
34
35 To enable comparison operands collection, set::
36
37 CONFIG_KCOV_ENABLE_COMPARISONS=y
38
39 Coverage data only becomes accessible once debugfs has been mounted::
40
41 mount -t debugfs none /sys/kernel/debug
42
43 Coverage collection
44 -------------------
45
46 The following program demonstrates how to use KCOV to collect coverage for a
47 single syscall from within a test program:
48
49 .. code-block:: c
50
51 #include <stdio.h>
52 #include <stddef.h>
53 #include <stdint.h>
54 #include <stdlib.h>
55 #include <sys/types.h>
56 #include <sys/stat.h>
57 #include <sys/ioctl.h>
58 #include <sys/mman.h>
59 #include <unistd.h>
60 #include <fcntl.h>
61 #include <linux/types.h>
62
63 #define KCOV_INIT_TRACE _IOR('c', 1, unsigned long)
64 #define KCOV_ENABLE _IO('c', 100)
65 #define KCOV_DISABLE _IO('c', 101)
66 #define COVER_SIZE (64<<10)
67
68 #define KCOV_TRACE_PC 0
69 #define KCOV_TRACE_CMP 1
70
71 int main(int argc, char **argv)
72 {
73 int fd;
74 unsigned long *cover, n, i;
75
76 /* A single fd descriptor allows coverage collection on a single
77 * thread.
78 */
79 fd = open("/sys/kernel/debug/kcov", O_RDWR);
80 if (fd == -1)
81 perror("open"), exit(1);
82 /* Setup trace mode and trace size. */
83 if (ioctl(fd, KCOV_INIT_TRACE, COVER_SIZE))
84 perror("ioctl"), exit(1);
85 /* Mmap buffer shared between kernel- and user-space. */
86 cover = (unsigned long*)mmap(NULL, COVER_SIZE * sizeof(unsigned long),
87 PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
88 if ((void*)cover == MAP_FAILED)
89 perror("mmap"), exit(1);
90 /* Enable coverage collection on the current thread. */
91 if (ioctl(fd, KCOV_ENABLE, KCOV_TRACE_PC))
92 perror("ioctl"), exit(1);
93 /* Reset coverage from the tail of the ioctl() call. */
94 __atomic_store_n(&cover[0], 0, __ATOMIC_RELAXED);
95 /* Call the target syscall call. */
96 read(-1, NULL, 0);
97 /* Read number of PCs collected. */
98 n = __atomic_load_n(&cover[0], __ATOMIC_RELAXED);
99 for (i = 0; i < n; i++)
100 printf("0x%lx\n", cover[i + 1]);
101 /* Disable coverage collection for the current thread. After this call
102 * coverage can be enabled for a different thread.
103 */
104 if (ioctl(fd, KCOV_DISABLE, 0))
105 perror("ioctl"), exit(1);
106 /* Free resources. */
107 if (munmap(cover, COVER_SIZE * sizeof(unsigned long)))
108 perror("munmap"), exit(1);
109 if (close(fd))
110 perror("close"), exit(1);
111 return 0;
112 }
113
114 After piping through ``addr2line`` the output of the program looks as follows::
115
116 SyS_read
117 fs/read_write.c:562
118 __fdget_pos
119 fs/file.c:774
120 __fget_light
121 fs/file.c:746
122 __fget_light
123 fs/file.c:750
124 __fget_light
125 fs/file.c:760
126 __fdget_pos
127 fs/file.c:784
128 SyS_read
129 fs/read_write.c:562
130
131 If a program needs to collect coverage from several threads (independently),
132 it needs to open ``/sys/kernel/debug/kcov`` in each thread separately.
133
134 The interface is fine-grained to allow efficient forking of test processes.
135 That is, a parent process opens ``/sys/kernel/debug/kcov``, enables trace mode,
136 mmaps coverage buffer, and then forks child processes in a loop. The child
137 processes only need to enable coverage (it gets disabled automatically when
138 a thread exits).
139
140 Comparison operands collection
141 ------------------------------
142
143 Comparison operands collection is similar to coverage collection:
144
145 .. code-block:: c
146
147 /* Same includes and defines as above. */
148
149 /* Number of 64-bit words per record. */
150 #define KCOV_WORDS_PER_CMP 4
151
152 /*
153 * The format for the types of collected comparisons.
154 *
155 * Bit 0 shows whether one of the arguments is a compile-time constant.
156 * Bits 1 & 2 contain log2 of the argument size, up to 8 bytes.
157 */
158
159 #define KCOV_CMP_CONST (1 << 0)
160 #define KCOV_CMP_SIZE(n) ((n) << 1)
161 #define KCOV_CMP_MASK KCOV_CMP_SIZE(3)
162
163 int main(int argc, char **argv)
164 {
165 int fd;
166 uint64_t *cover, type, arg1, arg2, is_const, size;
167 unsigned long n, i;
168
169 fd = open("/sys/kernel/debug/kcov", O_RDWR);
170 if (fd == -1)
171 perror("open"), exit(1);
172 if (ioctl(fd, KCOV_INIT_TRACE, COVER_SIZE))
173 perror("ioctl"), exit(1);
174 /*
175 * Note that the buffer pointer is of type uint64_t*, because all
176 * the comparison operands are promoted to uint64_t.
177 */
178 cover = (uint64_t *)mmap(NULL, COVER_SIZE * sizeof(unsigned long),
179 PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
180 if ((void*)cover == MAP_FAILED)
181 perror("mmap"), exit(1);
182 /* Note KCOV_TRACE_CMP instead of KCOV_TRACE_PC. */
183 if (ioctl(fd, KCOV_ENABLE, KCOV_TRACE_CMP))
184 perror("ioctl"), exit(1);
185 __atomic_store_n(&cover[0], 0, __ATOMIC_RELAXED);
186 read(-1, NULL, 0);
187 /* Read number of comparisons collected. */
188 n = __atomic_load_n(&cover[0], __ATOMIC_RELAXED);
189 for (i = 0; i < n; i++) {
190 uint64_t ip;
191
192 type = cover[i * KCOV_WORDS_PER_CMP + 1];
193 /* arg1 and arg2 - operands of the comparison. */
194 arg1 = cover[i * KCOV_WORDS_PER_CMP + 2];
195 arg2 = cover[i * KCOV_WORDS_PER_CMP + 3];
196 /* ip - caller address. */
197 ip = cover[i * KCOV_WORDS_PER_CMP + 4];
198 /* size of the operands. */
199 size = 1 << ((type & KCOV_CMP_MASK) >> 1);
200 /* is_const - true if either operand is a compile-time constant.*/
201 is_const = type & KCOV_CMP_CONST;
202 printf("ip: 0x%lx type: 0x%lx, arg1: 0x%lx, arg2: 0x%lx, "
203 "size: %lu, %s\n",
204 ip, type, arg1, arg2, size,
205 is_const ? "const" : "non-const");
206 }
207 if (ioctl(fd, KCOV_DISABLE, 0))
208 perror("ioctl"), exit(1);
209 /* Free resources. */
210 if (munmap(cover, COVER_SIZE * sizeof(unsigned long)))
211 perror("munmap"), exit(1);
212 if (close(fd))
213 perror("close"), exit(1);
214 return 0;
215 }
216
217 Note that the KCOV modes (collection of code coverage or comparison operands)
218 are mutually exclusive.
219
220 Remote coverage collection
221 --------------------------
222
223 Besides collecting coverage data from handlers of syscalls issued from a
224 userspace process, KCOV can also collect coverage for parts of the kernel
225 executing in other contexts - so-called "remote" coverage.
226
227 Using KCOV to collect remote coverage requires:
228
229 1. Modifying kernel code to annotate the code section from where coverage
230 should be collected with ``kcov_remote_start`` and ``kcov_remote_stop``.
231
232 2. Using ``KCOV_REMOTE_ENABLE`` instead of ``KCOV_ENABLE`` in the userspace
233 process that collects coverage.
234
235 Both ``kcov_remote_start`` and ``kcov_remote_stop`` annotations and the
236 ``KCOV_REMOTE_ENABLE`` ioctl accept handles that identify particular coverage
237 collection sections. The way a handle is used depends on the context where the
238 matching code section executes.
239
240 KCOV supports collecting remote coverage from the following contexts:
241
242 1. Global kernel background tasks. These are the tasks that are spawned during
243 kernel boot in a limited number of instances (e.g. one USB ``hub_event``
244 worker is spawned per one USB HCD).
245
246 2. Local kernel background tasks. These are spawned when a userspace process
247 interacts with some kernel interface and are usually killed when the process
248 exits (e.g. vhost workers).
249
250 3. Soft interrupts.
251
252 For #1 and #3, a unique global handle must be chosen and passed to the
253 corresponding ``kcov_remote_start`` call. Then a userspace process must pass
254 this handle to ``KCOV_REMOTE_ENABLE`` in the ``handles`` array field of the
255 ``kcov_remote_arg`` struct. This will attach the used KCOV device to the code
256 section referenced by this handle. Multiple global handles identifying
257 different code sections can be passed at once.
258
259 For #2, the userspace process instead must pass a non-zero handle through the
260 ``common_handle`` field of the ``kcov_remote_arg`` struct. This common handle
261 gets saved to the ``kcov_handle`` field in the current ``task_struct`` and
262 needs to be passed to the newly spawned local tasks via custom kernel code
263 modifications. Those tasks should in turn use the passed handle in their
264 ``kcov_remote_start`` and ``kcov_remote_stop`` annotations.
265
266 KCOV follows a predefined format for both global and common handles. Each
267 handle is a ``u64`` integer. Currently, only the one top and the lower 4 bytes
268 are used. Bytes 4-7 are reserved and must be zero.
269
270 For global handles, the top byte of the handle denotes the id of a subsystem
271 this handle belongs to. For example, KCOV uses ``1`` as the USB subsystem id.
272 The lower 4 bytes of a global handle denote the id of a task instance within
273 that subsystem. For example, each ``hub_event`` worker uses the USB bus number
274 as the task instance id.
275
276 For common handles, a reserved value ``0`` is used as a subsystem id, as such
277 handles don't belong to a particular subsystem. The lower 4 bytes of a common
278 handle identify a collective instance of all local tasks spawned by the
279 userspace process that passed a common handle to ``KCOV_REMOTE_ENABLE``.
280
281 In practice, any value can be used for common handle instance id if coverage
282 is only collected from a single userspace process on the system. However, if
283 common handles are used by multiple processes, unique instance ids must be
284 used for each process. One option is to use the process id as the common
285 handle instance id.
286
287 The following program demonstrates using KCOV to collect coverage from both
288 local tasks spawned by the process and the global task that handles USB bus #1:
289
290 .. code-block:: c
291
292 /* Same includes and defines as above. */
293
294 struct kcov_remote_arg {
295 __u32 trace_mode;
296 __u32 area_size;
297 __u32 num_handles;
298 __aligned_u64 common_handle;
299 __aligned_u64 handles[0];
300 };
301
302 #define KCOV_INIT_TRACE _IOR('c', 1, unsigned long)
303 #define KCOV_DISABLE _IO('c', 101)
304 #define KCOV_REMOTE_ENABLE _IOW('c', 102, struct kcov_remote_arg)
305
306 #define COVER_SIZE (64 << 10)
307
308 #define KCOV_TRACE_PC 0
309
310 #define KCOV_SUBSYSTEM_COMMON (0x00ull << 56)
311 #define KCOV_SUBSYSTEM_USB (0x01ull << 56)
312
313 #define KCOV_SUBSYSTEM_MASK (0xffull << 56)
314 #define KCOV_INSTANCE_MASK (0xffffffffull)
315
316 static inline __u64 kcov_remote_handle(__u64 subsys, __u64 inst)
317 {
318 if (subsys & ~KCOV_SUBSYSTEM_MASK || inst & ~KCOV_INSTANCE_MASK)
319 return 0;
320 return subsys | inst;
321 }
322
323 #define KCOV_COMMON_ID 0x42
324 #define KCOV_USB_BUS_NUM 1
325
326 int main(int argc, char **argv)
327 {
328 int fd;
329 unsigned long *cover, n, i;
330 struct kcov_remote_arg *arg;
331
332 fd = open("/sys/kernel/debug/kcov", O_RDWR);
333 if (fd == -1)
334 perror("open"), exit(1);
335 if (ioctl(fd, KCOV_INIT_TRACE, COVER_SIZE))
336 perror("ioctl"), exit(1);
337 cover = (unsigned long*)mmap(NULL, COVER_SIZE * sizeof(unsigned long),
338 PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
339 if ((void*)cover == MAP_FAILED)
340 perror("mmap"), exit(1);
341
342 /* Enable coverage collection via common handle and from USB bus #1. */
343 arg = calloc(1, sizeof(*arg) + sizeof(uint64_t));
344 if (!arg)
345 perror("calloc"), exit(1);
346 arg->trace_mode = KCOV_TRACE_PC;
347 arg->area_size = COVER_SIZE;
348 arg->num_handles = 1;
349 arg->common_handle = kcov_remote_handle(KCOV_SUBSYSTEM_COMMON,
350 KCOV_COMMON_ID);
351 arg->handles[0] = kcov_remote_handle(KCOV_SUBSYSTEM_USB,
352 KCOV_USB_BUS_NUM);
353 if (ioctl(fd, KCOV_REMOTE_ENABLE, arg))
354 perror("ioctl"), free(arg), exit(1);
355 free(arg);
356
357 /*
358 * Here the user needs to trigger execution of a kernel code section
359 * that is either annotated with the common handle, or to trigger some
360 * activity on USB bus #1.
361 */
362 sleep(2);
363
364 n = __atomic_load_n(&cover[0], __ATOMIC_RELAXED);
365 for (i = 0; i < n; i++)
366 printf("0x%lx\n", cover[i + 1]);
367 if (ioctl(fd, KCOV_DISABLE, 0))
368 perror("ioctl"), exit(1);
369 if (munmap(cover, COVER_SIZE * sizeof(unsigned long)))
370 perror("munmap"), exit(1);
371 if (close(fd))
372 perror("close"), exit(1);
373 return 0;
374 }
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.