Commit | Line | Data |
---|---|---|
1f0d69a9 SR |
1 | /* |
2 | * unlikely profiler | |
3 | * | |
4 | * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> | |
5 | */ | |
6 | #include <linux/kallsyms.h> | |
7 | #include <linux/seq_file.h> | |
8 | #include <linux/spinlock.h> | |
65c6dc6a | 9 | #include <linux/irqflags.h> |
1f0d69a9 SR |
10 | #include <linux/debugfs.h> |
11 | #include <linux/uaccess.h> | |
12 | #include <linux/module.h> | |
13 | #include <linux/ftrace.h> | |
14 | #include <linux/hash.h> | |
15 | #include <linux/fs.h> | |
16 | #include <asm/local.h> | |
f633cef0 | 17 | |
1f0d69a9 | 18 | #include "trace.h" |
002bb86d | 19 | #include "trace_stat.h" |
f633cef0 | 20 | #include "trace_output.h" |
1f0d69a9 | 21 | |
2ed84eeb | 22 | #ifdef CONFIG_BRANCH_TRACER |
52f232cb | 23 | |
002bb86d | 24 | static struct tracer branch_trace; |
9f029e83 SR |
25 | static int branch_tracing_enabled __read_mostly; |
26 | static DEFINE_MUTEX(branch_tracing_mutex); | |
e302cf3f | 27 | |
9f029e83 | 28 | static struct trace_array *branch_tracer; |
52f232cb SR |
29 | |
30 | static void | |
9f029e83 | 31 | probe_likely_condition(struct ftrace_branch_data *f, int val, int expect) |
52f232cb | 32 | { |
e1112b4d | 33 | struct ftrace_event_call *call = &event_branch; |
9f029e83 | 34 | struct trace_array *tr = branch_tracer; |
a7603ff4 | 35 | struct trace_array_cpu *data; |
52f232cb | 36 | struct ring_buffer_event *event; |
9f029e83 | 37 | struct trace_branch *entry; |
8f6e8a31 | 38 | struct ring_buffer *buffer; |
0a987751 | 39 | unsigned long flags; |
f1bb1307 | 40 | int pc; |
52f232cb SR |
41 | const char *p; |
42 | ||
f1bb1307 SRRH |
43 | if (current->trace_recursion & TRACE_BRANCH_BIT) |
44 | return; | |
45 | ||
52f232cb SR |
46 | /* |
47 | * I would love to save just the ftrace_likely_data pointer, but | |
48 | * this code can also be used by modules. Ugly things can happen | |
49 | * if the module is unloaded, and then we go and read the | |
50 | * pointer. This is slower, but much safer. | |
51 | */ | |
52 | ||
53 | if (unlikely(!tr)) | |
54 | return; | |
55 | ||
f1bb1307 SRRH |
56 | raw_local_irq_save(flags); |
57 | current->trace_recursion |= TRACE_BRANCH_BIT; | |
58 | data = this_cpu_ptr(tr->trace_buffer.data); | |
59 | if (atomic_read(&data->disabled)) | |
52f232cb SR |
60 | goto out; |
61 | ||
51a763dd | 62 | pc = preempt_count(); |
153e8ed9 | 63 | buffer = tr->trace_buffer.buffer; |
8f6e8a31 | 64 | event = trace_buffer_lock_reserve(buffer, TRACE_BRANCH, |
51a763dd | 65 | sizeof(*entry), flags, pc); |
52f232cb SR |
66 | if (!event) |
67 | goto out; | |
68 | ||
52f232cb | 69 | entry = ring_buffer_event_data(event); |
52f232cb SR |
70 | |
71 | /* Strip off the path, only save the file */ | |
72 | p = f->file + strlen(f->file); | |
73 | while (p >= f->file && *p != '/') | |
74 | p--; | |
75 | p++; | |
76 | ||
77 | strncpy(entry->func, f->func, TRACE_FUNC_SIZE); | |
78 | strncpy(entry->file, p, TRACE_FILE_SIZE); | |
79 | entry->func[TRACE_FUNC_SIZE] = 0; | |
80 | entry->file[TRACE_FILE_SIZE] = 0; | |
81 | entry->line = f->line; | |
82 | entry->correct = val == expect; | |
83 | ||
8f6e8a31 | 84 | if (!filter_check_discard(call, entry, buffer, event)) |
7ffbd48d | 85 | __buffer_unlock_commit(buffer, event); |
52f232cb SR |
86 | |
87 | out: | |
f1bb1307 SRRH |
88 | current->trace_recursion &= ~TRACE_BRANCH_BIT; |
89 | raw_local_irq_restore(flags); | |
52f232cb SR |
90 | } |
91 | ||
92 | static inline | |
9f029e83 | 93 | void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect) |
52f232cb | 94 | { |
9f029e83 | 95 | if (!branch_tracing_enabled) |
52f232cb SR |
96 | return; |
97 | ||
98 | probe_likely_condition(f, val, expect); | |
99 | } | |
100 | ||
9f029e83 | 101 | int enable_branch_tracing(struct trace_array *tr) |
52f232cb | 102 | { |
9f029e83 SR |
103 | mutex_lock(&branch_tracing_mutex); |
104 | branch_tracer = tr; | |
52f232cb SR |
105 | /* |
106 | * Must be seen before enabling. The reader is a condition | |
107 | * where we do not need a matching rmb() | |
108 | */ | |
109 | smp_wmb(); | |
9f029e83 SR |
110 | branch_tracing_enabled++; |
111 | mutex_unlock(&branch_tracing_mutex); | |
52f232cb | 112 | |
f54fc98a | 113 | return 0; |
52f232cb SR |
114 | } |
115 | ||
9f029e83 | 116 | void disable_branch_tracing(void) |
52f232cb | 117 | { |
9f029e83 | 118 | mutex_lock(&branch_tracing_mutex); |
52f232cb | 119 | |
9f029e83 | 120 | if (!branch_tracing_enabled) |
52f232cb SR |
121 | goto out_unlock; |
122 | ||
9f029e83 | 123 | branch_tracing_enabled--; |
52f232cb SR |
124 | |
125 | out_unlock: | |
9f029e83 | 126 | mutex_unlock(&branch_tracing_mutex); |
52f232cb | 127 | } |
80e5ea45 SR |
128 | |
129 | static void start_branch_trace(struct trace_array *tr) | |
130 | { | |
131 | enable_branch_tracing(tr); | |
132 | } | |
133 | ||
134 | static void stop_branch_trace(struct trace_array *tr) | |
135 | { | |
136 | disable_branch_tracing(); | |
137 | } | |
138 | ||
1c80025a | 139 | static int branch_trace_init(struct trace_array *tr) |
80e5ea45 | 140 | { |
80e5ea45 | 141 | start_branch_trace(tr); |
1c80025a | 142 | return 0; |
80e5ea45 SR |
143 | } |
144 | ||
145 | static void branch_trace_reset(struct trace_array *tr) | |
146 | { | |
147 | stop_branch_trace(tr); | |
148 | } | |
149 | ||
ae7462b4 | 150 | static enum print_line_t trace_branch_print(struct trace_iterator *iter, |
a9a57763 | 151 | int flags, struct trace_event *event) |
f633cef0 SR |
152 | { |
153 | struct trace_branch *field; | |
154 | ||
2c9b238e | 155 | trace_assign_type(field, iter->ent); |
f633cef0 | 156 | |
2c9b238e | 157 | if (trace_seq_printf(&iter->seq, "[%s] %s:%s:%d\n", |
f633cef0 SR |
158 | field->correct ? " ok " : " MISS ", |
159 | field->func, | |
160 | field->file, | |
161 | field->line)) | |
162 | return TRACE_TYPE_PARTIAL_LINE; | |
163 | ||
d9793bd8 | 164 | return TRACE_TYPE_HANDLED; |
f633cef0 SR |
165 | } |
166 | ||
557055be Z |
167 | static void branch_print_header(struct seq_file *s) |
168 | { | |
169 | seq_puts(s, "# TASK-PID CPU# TIMESTAMP CORRECT" | |
170 | " FUNC:FILE:LINE\n"); | |
171 | seq_puts(s, "# | | | | | " | |
172 | " |\n"); | |
173 | } | |
e302cf3f | 174 | |
a9a57763 SR |
175 | static struct trace_event_functions trace_branch_funcs = { |
176 | .trace = trace_branch_print, | |
177 | }; | |
178 | ||
f633cef0 | 179 | static struct trace_event trace_branch_event = { |
ef18012b | 180 | .type = TRACE_BRANCH, |
a9a57763 | 181 | .funcs = &trace_branch_funcs, |
f633cef0 SR |
182 | }; |
183 | ||
002bb86d FW |
184 | static struct tracer branch_trace __read_mostly = |
185 | { | |
186 | .name = "branch", | |
187 | .init = branch_trace_init, | |
188 | .reset = branch_trace_reset, | |
189 | #ifdef CONFIG_FTRACE_SELFTEST | |
190 | .selftest = trace_selftest_startup_branch, | |
191 | #endif /* CONFIG_FTRACE_SELFTEST */ | |
557055be | 192 | .print_header = branch_print_header, |
002bb86d FW |
193 | }; |
194 | ||
195 | __init static int init_branch_tracer(void) | |
196 | { | |
197 | int ret; | |
198 | ||
199 | ret = register_ftrace_event(&trace_branch_event); | |
200 | if (!ret) { | |
201 | printk(KERN_WARNING "Warning: could not register " | |
202 | "branch events\n"); | |
203 | return 1; | |
204 | } | |
205 | return register_tracer(&branch_trace); | |
206 | } | |
6f415672 | 207 | core_initcall(init_branch_tracer); |
002bb86d | 208 | |
52f232cb SR |
209 | #else |
210 | static inline | |
9f029e83 | 211 | void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect) |
52f232cb SR |
212 | { |
213 | } | |
2ed84eeb | 214 | #endif /* CONFIG_BRANCH_TRACER */ |
52f232cb | 215 | |
9f029e83 | 216 | void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect) |
1f0d69a9 | 217 | { |
52f232cb SR |
218 | /* |
219 | * I would love to have a trace point here instead, but the | |
220 | * trace point code is so inundated with unlikely and likely | |
221 | * conditions that the recursive nightmare that exists is too | |
222 | * much to try to get working. At least for now. | |
223 | */ | |
224 | trace_likely_condition(f, val, expect); | |
225 | ||
1f0d69a9 SR |
226 | /* FIXME: Make this atomic! */ |
227 | if (val == expect) | |
228 | f->correct++; | |
229 | else | |
230 | f->incorrect++; | |
231 | } | |
232 | EXPORT_SYMBOL(ftrace_likely_update); | |
233 | ||
e302cf3f FW |
234 | extern unsigned long __start_annotated_branch_profile[]; |
235 | extern unsigned long __stop_annotated_branch_profile[]; | |
1f0d69a9 | 236 | |
e302cf3f | 237 | static int annotated_branch_stat_headers(struct seq_file *m) |
1f0d69a9 | 238 | { |
e302cf3f FW |
239 | seq_printf(m, " correct incorrect %% "); |
240 | seq_printf(m, " Function " | |
241 | " File Line\n" | |
242 | " ------- --------- - " | |
243 | " -------- " | |
244 | " ---- ----\n"); | |
245 | return 0; | |
1f0d69a9 SR |
246 | } |
247 | ||
e302cf3f | 248 | static inline long get_incorrect_percent(struct ftrace_branch_data *p) |
1f0d69a9 | 249 | { |
e302cf3f | 250 | long percent; |
1f0d69a9 | 251 | |
e302cf3f FW |
252 | if (p->correct) { |
253 | percent = p->incorrect * 100; | |
254 | percent /= p->correct + p->incorrect; | |
255 | } else | |
256 | percent = p->incorrect ? 100 : -1; | |
1f0d69a9 | 257 | |
e302cf3f | 258 | return percent; |
1f0d69a9 SR |
259 | } |
260 | ||
e302cf3f | 261 | static int branch_stat_show(struct seq_file *m, void *v) |
1f0d69a9 | 262 | { |
9f029e83 | 263 | struct ftrace_branch_data *p = v; |
1f0d69a9 | 264 | const char *f; |
bac28bfe | 265 | long percent; |
1f0d69a9 | 266 | |
1f0d69a9 SR |
267 | /* Only print the file, not the path */ |
268 | f = p->file + strlen(p->file); | |
269 | while (f >= p->file && *f != '/') | |
270 | f--; | |
271 | f++; | |
272 | ||
2bcd521a SR |
273 | /* |
274 | * The miss is overlayed on correct, and hit on incorrect. | |
275 | */ | |
e302cf3f | 276 | percent = get_incorrect_percent(p); |
1f0d69a9 | 277 | |
bac28bfe SR |
278 | seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect); |
279 | if (percent < 0) | |
280 | seq_printf(m, " X "); | |
281 | else | |
282 | seq_printf(m, "%3ld ", percent); | |
1f0d69a9 SR |
283 | seq_printf(m, "%-30.30s %-20.20s %d\n", p->func, f, p->line); |
284 | return 0; | |
285 | } | |
286 | ||
42548008 | 287 | static void *annotated_branch_stat_start(struct tracer_stat *trace) |
e302cf3f FW |
288 | { |
289 | return __start_annotated_branch_profile; | |
290 | } | |
1f0d69a9 | 291 | |
e302cf3f FW |
292 | static void * |
293 | annotated_branch_stat_next(void *v, int idx) | |
1f0d69a9 | 294 | { |
e302cf3f | 295 | struct ftrace_branch_data *p = v; |
1f0d69a9 | 296 | |
e302cf3f | 297 | ++p; |
1f0d69a9 | 298 | |
e302cf3f FW |
299 | if ((void *)p >= (void *)__stop_annotated_branch_profile) |
300 | return NULL; | |
301 | ||
302 | return p; | |
1f0d69a9 SR |
303 | } |
304 | ||
e302cf3f FW |
305 | static int annotated_branch_stat_cmp(void *p1, void *p2) |
306 | { | |
307 | struct ftrace_branch_data *a = p1; | |
308 | struct ftrace_branch_data *b = p2; | |
309 | ||
310 | long percent_a, percent_b; | |
311 | ||
312 | percent_a = get_incorrect_percent(a); | |
313 | percent_b = get_incorrect_percent(b); | |
314 | ||
315 | if (percent_a < percent_b) | |
316 | return -1; | |
317 | if (percent_a > percent_b) | |
318 | return 1; | |
ede55c9d SR |
319 | |
320 | if (a->incorrect < b->incorrect) | |
321 | return -1; | |
322 | if (a->incorrect > b->incorrect) | |
323 | return 1; | |
324 | ||
325 | /* | |
326 | * Since the above shows worse (incorrect) cases | |
327 | * first, we continue that by showing best (correct) | |
328 | * cases last. | |
329 | */ | |
330 | if (a->correct > b->correct) | |
331 | return -1; | |
332 | if (a->correct < b->correct) | |
333 | return 1; | |
334 | ||
335 | return 0; | |
e302cf3f | 336 | } |
1f0d69a9 | 337 | |
002bb86d FW |
338 | static struct tracer_stat annotated_branch_stats = { |
339 | .name = "branch_annotated", | |
340 | .stat_start = annotated_branch_stat_start, | |
341 | .stat_next = annotated_branch_stat_next, | |
342 | .stat_cmp = annotated_branch_stat_cmp, | |
343 | .stat_headers = annotated_branch_stat_headers, | |
344 | .stat_show = branch_stat_show | |
345 | }; | |
346 | ||
347 | __init static int init_annotated_branch_stats(void) | |
348 | { | |
349 | int ret; | |
350 | ||
351 | ret = register_stat_tracer(&annotated_branch_stats); | |
352 | if (!ret) { | |
353 | printk(KERN_WARNING "Warning: could not register " | |
354 | "annotated branches stats\n"); | |
355 | return 1; | |
356 | } | |
357 | return 0; | |
358 | } | |
359 | fs_initcall(init_annotated_branch_stats); | |
360 | ||
2bcd521a | 361 | #ifdef CONFIG_PROFILE_ALL_BRANCHES |
2bcd521a | 362 | |
e302cf3f FW |
363 | extern unsigned long __start_branch_profile[]; |
364 | extern unsigned long __stop_branch_profile[]; | |
1f0d69a9 | 365 | |
e302cf3f FW |
366 | static int all_branch_stat_headers(struct seq_file *m) |
367 | { | |
368 | seq_printf(m, " miss hit %% "); | |
369 | seq_printf(m, " Function " | |
370 | " File Line\n" | |
371 | " ------- --------- - " | |
372 | " -------- " | |
373 | " ---- ----\n"); | |
374 | return 0; | |
375 | } | |
1f0d69a9 | 376 | |
42548008 | 377 | static void *all_branch_stat_start(struct tracer_stat *trace) |
1f0d69a9 | 378 | { |
e302cf3f FW |
379 | return __start_branch_profile; |
380 | } | |
381 | ||
382 | static void * | |
383 | all_branch_stat_next(void *v, int idx) | |
384 | { | |
385 | struct ftrace_branch_data *p = v; | |
1f0d69a9 | 386 | |
e302cf3f | 387 | ++p; |
1f0d69a9 | 388 | |
e302cf3f FW |
389 | if ((void *)p >= (void *)__stop_branch_profile) |
390 | return NULL; | |
1f0d69a9 | 391 | |
e302cf3f FW |
392 | return p; |
393 | } | |
2bcd521a | 394 | |
002bb86d FW |
395 | static struct tracer_stat all_branch_stats = { |
396 | .name = "branch_all", | |
034939b6 FW |
397 | .stat_start = all_branch_stat_start, |
398 | .stat_next = all_branch_stat_next, | |
399 | .stat_headers = all_branch_stat_headers, | |
002bb86d | 400 | .stat_show = branch_stat_show |
034939b6 | 401 | }; |
e302cf3f | 402 | |
002bb86d | 403 | __init static int all_annotated_branch_stats(void) |
e302cf3f | 404 | { |
e302cf3f | 405 | int ret; |
002bb86d FW |
406 | |
407 | ret = register_stat_tracer(&all_branch_stats); | |
e302cf3f | 408 | if (!ret) { |
002bb86d FW |
409 | printk(KERN_WARNING "Warning: could not register " |
410 | "all branches stats\n"); | |
e302cf3f FW |
411 | return 1; |
412 | } | |
002bb86d | 413 | return 0; |
e302cf3f | 414 | } |
002bb86d FW |
415 | fs_initcall(all_annotated_branch_stats); |
416 | #endif /* CONFIG_PROFILE_ALL_BRANCHES */ |