Commit | Line | Data |
---|---|---|
3d083395 SR |
1 | /* |
2 | * Code for replacing ftrace calls with jumps. | |
3 | * | |
4 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> | |
5 | * | |
6 | * Thanks goes to Ingo Molnar, for suggesting the idea. | |
7 | * Mathieu Desnoyers, for suggesting postponing the modifications. | |
8 | * Arjan van de Ven, for keeping me straight, and explaining to me | |
9 | * the dangers of modifying code on the run. | |
10 | */ | |
11 | ||
12 | #include <linux/spinlock.h> | |
13 | #include <linux/hardirq.h> | |
6f93fc07 | 14 | #include <linux/uaccess.h> |
3d083395 SR |
15 | #include <linux/ftrace.h> |
16 | #include <linux/percpu.h> | |
19b3e967 | 17 | #include <linux/sched.h> |
3d083395 SR |
18 | #include <linux/init.h> |
19 | #include <linux/list.h> | |
20 | ||
395a59d0 | 21 | #include <asm/ftrace.h> |
caf4b323 | 22 | #include <linux/ftrace.h> |
732f3ca7 | 23 | #include <asm/nops.h> |
caf4b323 | 24 | #include <asm/nmi.h> |
3d083395 | 25 | |
3d083395 | 26 | |
caf4b323 FW |
27 | |
28 | #ifdef CONFIG_FUNCTION_RET_TRACER | |
29 | ||
30 | /* | |
31 | * These functions are picked from those used on | |
32 | * this page for dynamic ftrace. They have been | |
33 | * simplified to ignore all traces in NMI context. | |
34 | */ | |
35 | static atomic_t in_nmi; | |
36 | ||
37 | void ftrace_nmi_enter(void) | |
38 | { | |
39 | atomic_inc(&in_nmi); | |
40 | } | |
41 | ||
42 | void ftrace_nmi_exit(void) | |
43 | { | |
44 | atomic_dec(&in_nmi); | |
45 | } | |
46 | ||
caf4b323 FW |
47 | /* Add a function return address to the trace stack on thread info.*/ |
48 | static int push_return_trace(unsigned long ret, unsigned long long time, | |
49 | unsigned long func) | |
50 | { | |
51 | int index; | |
62d59d17 | 52 | struct thread_info *ti = current_thread_info(); |
caf4b323 | 53 | |
caf4b323 | 54 | /* The return trace stack is full */ |
62d59d17 FW |
55 | if (ti->curr_ret_stack == FTRACE_RET_STACK_SIZE - 1) |
56 | return -EBUSY; | |
caf4b323 FW |
57 | |
58 | index = ++ti->curr_ret_stack; | |
59 | ti->ret_stack[index].ret = ret; | |
60 | ti->ret_stack[index].func = func; | |
61 | ti->ret_stack[index].calltime = time; | |
62 | ||
62d59d17 | 63 | return 0; |
caf4b323 FW |
64 | } |
65 | ||
66 | /* Retrieve a function return address to the trace stack on thread info.*/ | |
67 | static void pop_return_trace(unsigned long *ret, unsigned long long *time, | |
68 | unsigned long *func) | |
69 | { | |
caf4b323 | 70 | int index; |
caf4b323 | 71 | |
62d59d17 | 72 | struct thread_info *ti = current_thread_info(); |
caf4b323 FW |
73 | index = ti->curr_ret_stack; |
74 | *ret = ti->ret_stack[index].ret; | |
75 | *func = ti->ret_stack[index].func; | |
76 | *time = ti->ret_stack[index].calltime; | |
77 | ti->curr_ret_stack--; | |
caf4b323 FW |
78 | } |
79 | ||
80 | /* | |
81 | * Send the trace to the ring-buffer. | |
82 | * @return the original return address. | |
83 | */ | |
84 | unsigned long ftrace_return_to_handler(void) | |
85 | { | |
86 | struct ftrace_retfunc trace; | |
87 | pop_return_trace(&trace.ret, &trace.calltime, &trace.func); | |
88 | trace.rettime = cpu_clock(raw_smp_processor_id()); | |
89 | ftrace_function_return(&trace); | |
90 | ||
91 | return trace.ret; | |
92 | } | |
93 | ||
94 | /* | |
95 | * Hook the return address and push it in the stack of return addrs | |
96 | * in current thread info. | |
97 | */ | |
caf4b323 FW |
98 | void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr) |
99 | { | |
100 | unsigned long old; | |
101 | unsigned long long calltime; | |
102 | int faulted; | |
103 | unsigned long return_hooker = (unsigned long) | |
104 | &return_to_handler; | |
105 | ||
106 | /* Nmi's are currently unsupported */ | |
107 | if (atomic_read(&in_nmi)) | |
108 | return; | |
109 | ||
110 | /* | |
111 | * Protect against fault, even if it shouldn't | |
112 | * happen. This tool is too much intrusive to | |
113 | * ignore such a protection. | |
114 | */ | |
115 | asm volatile( | |
116 | "1: movl (%[parent_old]), %[old]\n" | |
117 | "2: movl %[return_hooker], (%[parent_replaced])\n" | |
118 | " movl $0, %[faulted]\n" | |
119 | ||
120 | ".section .fixup, \"ax\"\n" | |
121 | "3: movl $1, %[faulted]\n" | |
122 | ".previous\n" | |
123 | ||
124 | ".section __ex_table, \"a\"\n" | |
125 | " .long 1b, 3b\n" | |
126 | " .long 2b, 3b\n" | |
127 | ".previous\n" | |
128 | ||
867f7fb3 | 129 | : [parent_replaced] "=r" (parent), [old] "=r" (old), |
caf4b323 FW |
130 | [faulted] "=r" (faulted) |
131 | : [parent_old] "0" (parent), [return_hooker] "r" (return_hooker) | |
132 | : "memory" | |
133 | ); | |
134 | ||
135 | if (WARN_ON(faulted)) { | |
136 | unregister_ftrace_return(); | |
137 | return; | |
138 | } | |
139 | ||
140 | if (WARN_ON(!__kernel_text_address(old))) { | |
141 | unregister_ftrace_return(); | |
142 | *parent = old; | |
143 | return; | |
144 | } | |
145 | ||
146 | calltime = cpu_clock(raw_smp_processor_id()); | |
147 | ||
148 | if (push_return_trace(old, calltime, self_addr) == -EBUSY) | |
149 | *parent = old; | |
150 | } | |
151 | ||
caf4b323 FW |
152 | #endif |
153 | ||
154 | #ifdef CONFIG_DYNAMIC_FTRACE | |
3d083395 | 155 | |
3d083395 | 156 | union ftrace_code_union { |
395a59d0 | 157 | char code[MCOUNT_INSN_SIZE]; |
3d083395 SR |
158 | struct { |
159 | char e8; | |
160 | int offset; | |
161 | } __attribute__((packed)); | |
162 | }; | |
163 | ||
15adc048 | 164 | static int ftrace_calc_offset(long ip, long addr) |
3c1720f0 SR |
165 | { |
166 | return (int)(addr - ip); | |
167 | } | |
3d083395 | 168 | |
15adc048 | 169 | unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr) |
3c1720f0 SR |
170 | { |
171 | static union ftrace_code_union calc; | |
3d083395 | 172 | |
3c1720f0 | 173 | calc.e8 = 0xe8; |
395a59d0 | 174 | calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr); |
3c1720f0 SR |
175 | |
176 | /* | |
177 | * No locking needed, this must be called via kstop_machine | |
178 | * which in essence is like running on a uniprocessor machine. | |
179 | */ | |
180 | return calc.code; | |
3d083395 SR |
181 | } |
182 | ||
17666f02 SR |
183 | /* |
184 | * Modifying code must take extra care. On an SMP machine, if | |
185 | * the code being modified is also being executed on another CPU | |
186 | * that CPU will have undefined results and possibly take a GPF. | |
187 | * We use kstop_machine to stop other CPUS from exectuing code. | |
188 | * But this does not stop NMIs from happening. We still need | |
189 | * to protect against that. We separate out the modification of | |
190 | * the code to take care of this. | |
191 | * | |
192 | * Two buffers are added: An IP buffer and a "code" buffer. | |
193 | * | |
a26a2a27 | 194 | * 1) Put the instruction pointer into the IP buffer |
17666f02 SR |
195 | * and the new code into the "code" buffer. |
196 | * 2) Set a flag that says we are modifying code | |
197 | * 3) Wait for any running NMIs to finish. | |
198 | * 4) Write the code | |
199 | * 5) clear the flag. | |
200 | * 6) Wait for any running NMIs to finish. | |
201 | * | |
202 | * If an NMI is executed, the first thing it does is to call | |
203 | * "ftrace_nmi_enter". This will check if the flag is set to write | |
204 | * and if it is, it will write what is in the IP and "code" buffers. | |
205 | * | |
206 | * The trick is, it does not matter if everyone is writing the same | |
207 | * content to the code location. Also, if a CPU is executing code | |
208 | * it is OK to write to that code location if the contents being written | |
209 | * are the same as what exists. | |
210 | */ | |
211 | ||
a26a2a27 SR |
212 | static atomic_t in_nmi = ATOMIC_INIT(0); |
213 | static int mod_code_status; /* holds return value of text write */ | |
214 | static int mod_code_write; /* set when NMI should do the write */ | |
215 | static void *mod_code_ip; /* holds the IP to write to */ | |
216 | static void *mod_code_newcode; /* holds the text to write to the IP */ | |
17666f02 | 217 | |
a26a2a27 SR |
218 | static unsigned nmi_wait_count; |
219 | static atomic_t nmi_update_count = ATOMIC_INIT(0); | |
b807c3d0 SR |
220 | |
221 | int ftrace_arch_read_dyn_info(char *buf, int size) | |
222 | { | |
223 | int r; | |
224 | ||
225 | r = snprintf(buf, size, "%u %u", | |
226 | nmi_wait_count, | |
227 | atomic_read(&nmi_update_count)); | |
228 | return r; | |
229 | } | |
230 | ||
17666f02 SR |
231 | static void ftrace_mod_code(void) |
232 | { | |
233 | /* | |
234 | * Yes, more than one CPU process can be writing to mod_code_status. | |
235 | * (and the code itself) | |
236 | * But if one were to fail, then they all should, and if one were | |
237 | * to succeed, then they all should. | |
238 | */ | |
239 | mod_code_status = probe_kernel_write(mod_code_ip, mod_code_newcode, | |
240 | MCOUNT_INSN_SIZE); | |
241 | ||
242 | } | |
243 | ||
244 | void ftrace_nmi_enter(void) | |
245 | { | |
246 | atomic_inc(&in_nmi); | |
247 | /* Must have in_nmi seen before reading write flag */ | |
248 | smp_mb(); | |
b807c3d0 | 249 | if (mod_code_write) { |
17666f02 | 250 | ftrace_mod_code(); |
b807c3d0 SR |
251 | atomic_inc(&nmi_update_count); |
252 | } | |
17666f02 SR |
253 | } |
254 | ||
255 | void ftrace_nmi_exit(void) | |
256 | { | |
257 | /* Finish all executions before clearing in_nmi */ | |
258 | smp_wmb(); | |
259 | atomic_dec(&in_nmi); | |
260 | } | |
261 | ||
262 | static void wait_for_nmi(void) | |
263 | { | |
b807c3d0 SR |
264 | int waited = 0; |
265 | ||
266 | while (atomic_read(&in_nmi)) { | |
267 | waited = 1; | |
17666f02 | 268 | cpu_relax(); |
b807c3d0 SR |
269 | } |
270 | ||
271 | if (waited) | |
272 | nmi_wait_count++; | |
17666f02 SR |
273 | } |
274 | ||
275 | static int | |
276 | do_ftrace_mod_code(unsigned long ip, void *new_code) | |
277 | { | |
278 | mod_code_ip = (void *)ip; | |
279 | mod_code_newcode = new_code; | |
280 | ||
281 | /* The buffers need to be visible before we let NMIs write them */ | |
282 | smp_wmb(); | |
283 | ||
284 | mod_code_write = 1; | |
285 | ||
286 | /* Make sure write bit is visible before we wait on NMIs */ | |
287 | smp_mb(); | |
288 | ||
289 | wait_for_nmi(); | |
290 | ||
291 | /* Make sure all running NMIs have finished before we write the code */ | |
292 | smp_mb(); | |
293 | ||
294 | ftrace_mod_code(); | |
295 | ||
296 | /* Make sure the write happens before clearing the bit */ | |
297 | smp_wmb(); | |
298 | ||
299 | mod_code_write = 0; | |
300 | ||
301 | /* make sure NMIs see the cleared bit */ | |
302 | smp_mb(); | |
303 | ||
304 | wait_for_nmi(); | |
305 | ||
306 | return mod_code_status; | |
307 | } | |
308 | ||
309 | ||
caf4b323 FW |
310 | |
311 | ||
312 | static unsigned char ftrace_nop[MCOUNT_INSN_SIZE]; | |
313 | ||
314 | unsigned char *ftrace_nop_replace(void) | |
315 | { | |
316 | return ftrace_nop; | |
317 | } | |
318 | ||
15adc048 | 319 | int |
3d083395 SR |
320 | ftrace_modify_code(unsigned long ip, unsigned char *old_code, |
321 | unsigned char *new_code) | |
322 | { | |
6f93fc07 | 323 | unsigned char replaced[MCOUNT_INSN_SIZE]; |
3d083395 SR |
324 | |
325 | /* | |
326 | * Note: Due to modules and __init, code can | |
327 | * disappear and change, we need to protect against faulting | |
76aefee5 | 328 | * as well as code changing. We do this by using the |
ab9a0918 | 329 | * probe_kernel_* functions. |
3d083395 SR |
330 | * |
331 | * No real locking needed, this code is run through | |
6f93fc07 | 332 | * kstop_machine, or before SMP starts. |
3d083395 | 333 | */ |
76aefee5 SR |
334 | |
335 | /* read the text we want to modify */ | |
ab9a0918 | 336 | if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE)) |
593eb8a2 | 337 | return -EFAULT; |
6f93fc07 | 338 | |
76aefee5 | 339 | /* Make sure it is what we expect it to be */ |
6f93fc07 | 340 | if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0) |
593eb8a2 | 341 | return -EINVAL; |
3d083395 | 342 | |
76aefee5 | 343 | /* replace the text with the new text */ |
17666f02 | 344 | if (do_ftrace_mod_code(ip, new_code)) |
593eb8a2 | 345 | return -EPERM; |
6f93fc07 SR |
346 | |
347 | sync_core(); | |
3d083395 | 348 | |
6f93fc07 | 349 | return 0; |
3d083395 SR |
350 | } |
351 | ||
15adc048 | 352 | int ftrace_update_ftrace_func(ftrace_func_t func) |
d61f82d0 SR |
353 | { |
354 | unsigned long ip = (unsigned long)(&ftrace_call); | |
395a59d0 | 355 | unsigned char old[MCOUNT_INSN_SIZE], *new; |
d61f82d0 SR |
356 | int ret; |
357 | ||
395a59d0 | 358 | memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE); |
d61f82d0 SR |
359 | new = ftrace_call_replace(ip, (unsigned long)func); |
360 | ret = ftrace_modify_code(ip, old, new); | |
361 | ||
362 | return ret; | |
363 | } | |
364 | ||
d61f82d0 | 365 | int __init ftrace_dyn_arch_init(void *data) |
3d083395 | 366 | { |
732f3ca7 SR |
367 | extern const unsigned char ftrace_test_p6nop[]; |
368 | extern const unsigned char ftrace_test_nop5[]; | |
369 | extern const unsigned char ftrace_test_jmp[]; | |
370 | int faulted = 0; | |
d61f82d0 | 371 | |
732f3ca7 SR |
372 | /* |
373 | * There is no good nop for all x86 archs. | |
374 | * We will default to using the P6_NOP5, but first we | |
375 | * will test to make sure that the nop will actually | |
376 | * work on this CPU. If it faults, we will then | |
377 | * go to a lesser efficient 5 byte nop. If that fails | |
378 | * we then just use a jmp as our nop. This isn't the most | |
379 | * efficient nop, but we can not use a multi part nop | |
380 | * since we would then risk being preempted in the middle | |
381 | * of that nop, and if we enabled tracing then, it might | |
382 | * cause a system crash. | |
383 | * | |
384 | * TODO: check the cpuid to determine the best nop. | |
385 | */ | |
386 | asm volatile ( | |
732f3ca7 SR |
387 | "ftrace_test_jmp:" |
388 | "jmp ftrace_test_p6nop\n" | |
8b27386a AK |
389 | "nop\n" |
390 | "nop\n" | |
391 | "nop\n" /* 2 byte jmp + 3 bytes */ | |
732f3ca7 SR |
392 | "ftrace_test_p6nop:" |
393 | P6_NOP5 | |
394 | "jmp 1f\n" | |
395 | "ftrace_test_nop5:" | |
396 | ".byte 0x66,0x66,0x66,0x66,0x90\n" | |
732f3ca7 SR |
397 | "1:" |
398 | ".section .fixup, \"ax\"\n" | |
399 | "2: movl $1, %0\n" | |
400 | " jmp ftrace_test_nop5\n" | |
401 | "3: movl $2, %0\n" | |
402 | " jmp 1b\n" | |
403 | ".previous\n" | |
404 | _ASM_EXTABLE(ftrace_test_p6nop, 2b) | |
405 | _ASM_EXTABLE(ftrace_test_nop5, 3b) | |
406 | : "=r"(faulted) : "0" (faulted)); | |
407 | ||
408 | switch (faulted) { | |
409 | case 0: | |
410 | pr_info("ftrace: converting mcount calls to 0f 1f 44 00 00\n"); | |
8115f3f0 | 411 | memcpy(ftrace_nop, ftrace_test_p6nop, MCOUNT_INSN_SIZE); |
732f3ca7 SR |
412 | break; |
413 | case 1: | |
414 | pr_info("ftrace: converting mcount calls to 66 66 66 66 90\n"); | |
8115f3f0 | 415 | memcpy(ftrace_nop, ftrace_test_nop5, MCOUNT_INSN_SIZE); |
732f3ca7 SR |
416 | break; |
417 | case 2: | |
8b27386a | 418 | pr_info("ftrace: converting mcount calls to jmp . + 5\n"); |
8115f3f0 | 419 | memcpy(ftrace_nop, ftrace_test_jmp, MCOUNT_INSN_SIZE); |
732f3ca7 SR |
420 | break; |
421 | } | |
422 | ||
423 | /* The return code is retured via data */ | |
424 | *(unsigned long *)data = 0; | |
dfa60aba | 425 | |
3d083395 SR |
426 | return 0; |
427 | } | |
caf4b323 | 428 | #endif |