Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
5c9a8750 DV |
2 | #define pr_fmt(fmt) "kcov: " fmt |
3 | ||
36f05ae8 | 4 | #define DISABLE_BRANCH_PROFILING |
db862358 | 5 | #include <linux/atomic.h> |
5c9a8750 | 6 | #include <linux/compiler.h> |
db862358 KW |
7 | #include <linux/errno.h> |
8 | #include <linux/export.h> | |
5c9a8750 DV |
9 | #include <linux/types.h> |
10 | #include <linux/file.h> | |
11 | #include <linux/fs.h> | |
db862358 | 12 | #include <linux/init.h> |
5c9a8750 | 13 | #include <linux/mm.h> |
db862358 | 14 | #include <linux/preempt.h> |
5c9a8750 | 15 | #include <linux/printk.h> |
166ad0e1 | 16 | #include <linux/sched.h> |
5c9a8750 DV |
17 | #include <linux/slab.h> |
18 | #include <linux/spinlock.h> | |
19 | #include <linux/vmalloc.h> | |
20 | #include <linux/debugfs.h> | |
21 | #include <linux/uaccess.h> | |
22 | #include <linux/kcov.h> | |
4983f0ab | 23 | #include <asm/setup.h> |
5c9a8750 DV |
24 | |
25 | /* | |
26 | * kcov descriptor (one per opened debugfs file). | |
27 | * State transitions of the descriptor: | |
28 | * - initial state after open() | |
29 | * - then there must be a single ioctl(KCOV_INIT_TRACE) call | |
30 | * - then, mmap() call (several calls are allowed but not useful) | |
31 | * - then, repeated enable/disable for a task (only one task a time allowed) | |
32 | */ | |
33 | struct kcov { | |
34 | /* | |
35 | * Reference counter. We keep one for: | |
36 | * - opened file descriptor | |
37 | * - task with enabled coverage (we can't unwire it from another task) | |
38 | */ | |
39 | atomic_t refcount; | |
40 | /* The lock protects mode, size, area and t. */ | |
41 | spinlock_t lock; | |
42 | enum kcov_mode mode; | |
43 | /* Size of arena (in long's for KCOV_MODE_TRACE). */ | |
44 | unsigned size; | |
45 | /* Coverage buffer shared with user space. */ | |
46 | void *area; | |
47 | /* Task for which we collect coverage, or NULL. */ | |
48 | struct task_struct *t; | |
49 | }; | |
50 | ||
51 | /* | |
52 | * Entry point from instrumented code. | |
53 | * This is called once per basic-block/edge. | |
54 | */ | |
bdab42df | 55 | void notrace __sanitizer_cov_trace_pc(void) |
5c9a8750 DV |
56 | { |
57 | struct task_struct *t; | |
58 | enum kcov_mode mode; | |
59 | ||
60 | t = current; | |
61 | /* | |
62 | * We are interested in code coverage as a function of a syscall inputs, | |
63 | * so we ignore code executed in interrupts. | |
64 | */ | |
f61e869d | 65 | if (!t || !in_task()) |
5c9a8750 DV |
66 | return; |
67 | mode = READ_ONCE(t->kcov_mode); | |
68 | if (mode == KCOV_MODE_TRACE) { | |
69 | unsigned long *area; | |
70 | unsigned long pos; | |
4983f0ab AP |
71 | unsigned long ip = _RET_IP_; |
72 | ||
73 | #ifdef CONFIG_RANDOMIZE_BASE | |
74 | ip -= kaslr_offset(); | |
75 | #endif | |
5c9a8750 DV |
76 | |
77 | /* | |
78 | * There is some code that runs in interrupts but for which | |
79 | * in_interrupt() returns false (e.g. preempt_schedule_irq()). | |
80 | * READ_ONCE()/barrier() effectively provides load-acquire wrt | |
81 | * interrupts, there are paired barrier()/WRITE_ONCE() in | |
82 | * kcov_ioctl_locked(). | |
83 | */ | |
84 | barrier(); | |
85 | area = t->kcov_area; | |
86 | /* The first word is number of subsequent PCs. */ | |
87 | pos = READ_ONCE(area[0]) + 1; | |
88 | if (likely(pos < t->kcov_size)) { | |
4983f0ab | 89 | area[pos] = ip; |
5c9a8750 DV |
90 | WRITE_ONCE(area[0], pos); |
91 | } | |
92 | } | |
93 | } | |
94 | EXPORT_SYMBOL(__sanitizer_cov_trace_pc); | |
95 | ||
96 | static void kcov_get(struct kcov *kcov) | |
97 | { | |
98 | atomic_inc(&kcov->refcount); | |
99 | } | |
100 | ||
101 | static void kcov_put(struct kcov *kcov) | |
102 | { | |
103 | if (atomic_dec_and_test(&kcov->refcount)) { | |
104 | vfree(kcov->area); | |
105 | kfree(kcov); | |
106 | } | |
107 | } | |
108 | ||
109 | void kcov_task_init(struct task_struct *t) | |
110 | { | |
a45f5ee6 MR |
111 | WRITE_ONCE(t->kcov_mode, KCOV_MODE_DISABLED); |
112 | barrier(); | |
5c9a8750 DV |
113 | t->kcov_size = 0; |
114 | t->kcov_area = NULL; | |
115 | t->kcov = NULL; | |
116 | } | |
117 | ||
118 | void kcov_task_exit(struct task_struct *t) | |
119 | { | |
120 | struct kcov *kcov; | |
121 | ||
122 | kcov = t->kcov; | |
123 | if (kcov == NULL) | |
124 | return; | |
125 | spin_lock(&kcov->lock); | |
126 | if (WARN_ON(kcov->t != t)) { | |
127 | spin_unlock(&kcov->lock); | |
128 | return; | |
129 | } | |
130 | /* Just to not leave dangling references behind. */ | |
131 | kcov_task_init(t); | |
132 | kcov->t = NULL; | |
133 | spin_unlock(&kcov->lock); | |
134 | kcov_put(kcov); | |
135 | } | |
136 | ||
137 | static int kcov_mmap(struct file *filep, struct vm_area_struct *vma) | |
138 | { | |
139 | int res = 0; | |
140 | void *area; | |
141 | struct kcov *kcov = vma->vm_file->private_data; | |
142 | unsigned long size, off; | |
143 | struct page *page; | |
144 | ||
145 | area = vmalloc_user(vma->vm_end - vma->vm_start); | |
146 | if (!area) | |
147 | return -ENOMEM; | |
148 | ||
149 | spin_lock(&kcov->lock); | |
150 | size = kcov->size * sizeof(unsigned long); | |
151 | if (kcov->mode == KCOV_MODE_DISABLED || vma->vm_pgoff != 0 || | |
152 | vma->vm_end - vma->vm_start != size) { | |
153 | res = -EINVAL; | |
154 | goto exit; | |
155 | } | |
156 | if (!kcov->area) { | |
157 | kcov->area = area; | |
158 | vma->vm_flags |= VM_DONTEXPAND; | |
159 | spin_unlock(&kcov->lock); | |
160 | for (off = 0; off < size; off += PAGE_SIZE) { | |
161 | page = vmalloc_to_page(kcov->area + off); | |
162 | if (vm_insert_page(vma, vma->vm_start + off, page)) | |
163 | WARN_ONCE(1, "vm_insert_page() failed"); | |
164 | } | |
165 | return 0; | |
166 | } | |
167 | exit: | |
168 | spin_unlock(&kcov->lock); | |
169 | vfree(area); | |
170 | return res; | |
171 | } | |
172 | ||
173 | static int kcov_open(struct inode *inode, struct file *filep) | |
174 | { | |
175 | struct kcov *kcov; | |
176 | ||
177 | kcov = kzalloc(sizeof(*kcov), GFP_KERNEL); | |
178 | if (!kcov) | |
179 | return -ENOMEM; | |
180 | atomic_set(&kcov->refcount, 1); | |
181 | spin_lock_init(&kcov->lock); | |
182 | filep->private_data = kcov; | |
183 | return nonseekable_open(inode, filep); | |
184 | } | |
185 | ||
186 | static int kcov_close(struct inode *inode, struct file *filep) | |
187 | { | |
188 | kcov_put(filep->private_data); | |
189 | return 0; | |
190 | } | |
191 | ||
192 | static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, | |
193 | unsigned long arg) | |
194 | { | |
195 | struct task_struct *t; | |
196 | unsigned long size, unused; | |
197 | ||
198 | switch (cmd) { | |
199 | case KCOV_INIT_TRACE: | |
200 | /* | |
201 | * Enable kcov in trace mode and setup buffer size. | |
202 | * Must happen before anything else. | |
203 | */ | |
204 | if (kcov->mode != KCOV_MODE_DISABLED) | |
205 | return -EBUSY; | |
206 | /* | |
207 | * Size must be at least 2 to hold current position and one PC. | |
208 | * Later we allocate size * sizeof(unsigned long) memory, | |
209 | * that must not overflow. | |
210 | */ | |
211 | size = arg; | |
212 | if (size < 2 || size > INT_MAX / sizeof(unsigned long)) | |
213 | return -EINVAL; | |
214 | kcov->size = size; | |
215 | kcov->mode = KCOV_MODE_TRACE; | |
216 | return 0; | |
217 | case KCOV_ENABLE: | |
218 | /* | |
219 | * Enable coverage for the current task. | |
220 | * At this point user must have been enabled trace mode, | |
221 | * and mmapped the file. Coverage collection is disabled only | |
222 | * at task exit or voluntary by KCOV_DISABLE. After that it can | |
223 | * be enabled for another task. | |
224 | */ | |
225 | unused = arg; | |
226 | if (unused != 0 || kcov->mode == KCOV_MODE_DISABLED || | |
227 | kcov->area == NULL) | |
228 | return -EINVAL; | |
5c9a8750 | 229 | t = current; |
4249e8af DV |
230 | if (kcov->t != NULL || t->kcov != NULL) |
231 | return -EBUSY; | |
5c9a8750 DV |
232 | /* Cache in task struct for performance. */ |
233 | t->kcov_size = kcov->size; | |
234 | t->kcov_area = kcov->area; | |
235 | /* See comment in __sanitizer_cov_trace_pc(). */ | |
236 | barrier(); | |
237 | WRITE_ONCE(t->kcov_mode, kcov->mode); | |
238 | t->kcov = kcov; | |
239 | kcov->t = t; | |
240 | /* This is put either in kcov_task_exit() or in KCOV_DISABLE. */ | |
241 | kcov_get(kcov); | |
242 | return 0; | |
243 | case KCOV_DISABLE: | |
244 | /* Disable coverage for the current task. */ | |
245 | unused = arg; | |
246 | if (unused != 0 || current->kcov != kcov) | |
247 | return -EINVAL; | |
248 | t = current; | |
249 | if (WARN_ON(kcov->t != t)) | |
250 | return -EINVAL; | |
251 | kcov_task_init(t); | |
252 | kcov->t = NULL; | |
253 | kcov_put(kcov); | |
254 | return 0; | |
255 | default: | |
256 | return -ENOTTY; | |
257 | } | |
258 | } | |
259 | ||
260 | static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) | |
261 | { | |
262 | struct kcov *kcov; | |
263 | int res; | |
264 | ||
265 | kcov = filep->private_data; | |
266 | spin_lock(&kcov->lock); | |
267 | res = kcov_ioctl_locked(kcov, cmd, arg); | |
268 | spin_unlock(&kcov->lock); | |
269 | return res; | |
270 | } | |
271 | ||
272 | static const struct file_operations kcov_fops = { | |
273 | .open = kcov_open, | |
274 | .unlocked_ioctl = kcov_ioctl, | |
7483e5d4 | 275 | .compat_ioctl = kcov_ioctl, |
5c9a8750 DV |
276 | .mmap = kcov_mmap, |
277 | .release = kcov_close, | |
278 | }; | |
279 | ||
280 | static int __init kcov_init(void) | |
281 | { | |
df4565f9 NS |
282 | /* |
283 | * The kcov debugfs file won't ever get removed and thus, | |
284 | * there is no need to protect it against removal races. The | |
285 | * use of debugfs_create_file_unsafe() is actually safe here. | |
286 | */ | |
287 | if (!debugfs_create_file_unsafe("kcov", 0600, NULL, NULL, &kcov_fops)) { | |
5c9a8750 DV |
288 | pr_err("failed to create kcov in debugfs\n"); |
289 | return -ENOMEM; | |
290 | } | |
291 | return 0; | |
292 | } | |
293 | ||
294 | device_initcall(kcov_init); |