4 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
6 #include <linux/kallsyms.h>
7 #include <linux/seq_file.h>
8 #include <linux/spinlock.h>
9 #include <linux/irqflags.h>
10 #include <linux/debugfs.h>
11 #include <linux/uaccess.h>
12 #include <linux/module.h>
13 #include <linux/ftrace.h>
14 #include <linux/hash.h>
16 #include <asm/local.h>
19 #include "trace_stat.h"
20 #include "trace_output.h"
22 #ifdef CONFIG_BRANCH_TRACER
24 static struct tracer branch_trace
;
25 static int branch_tracing_enabled __read_mostly
;
26 static DEFINE_MUTEX(branch_tracing_mutex
);
28 static struct trace_array
*branch_tracer
;
31 probe_likely_condition(struct ftrace_branch_data
*f
, int val
, int expect
)
33 struct ftrace_event_call
*call
= &event_branch
;
34 struct trace_array
*tr
= branch_tracer
;
35 struct trace_array_cpu
*data
;
36 struct ring_buffer_event
*event
;
37 struct trace_branch
*entry
;
38 struct ring_buffer
*buffer
;
43 if (current
->trace_recursion
& TRACE_BRANCH_BIT
)
47 * I would love to save just the ftrace_likely_data pointer, but
48 * this code can also be used by modules. Ugly things can happen
49 * if the module is unloaded, and then we go and read the
50 * pointer. This is slower, but much safer.
56 raw_local_irq_save(flags
);
57 current
->trace_recursion
|= TRACE_BRANCH_BIT
;
58 data
= this_cpu_ptr(tr
->trace_buffer
.data
);
59 if (atomic_read(&data
->disabled
))
63 buffer
= tr
->trace_buffer
.buffer
;
64 event
= trace_buffer_lock_reserve(buffer
, TRACE_BRANCH
,
65 sizeof(*entry
), flags
, pc
);
69 entry
= ring_buffer_event_data(event
);
71 /* Strip off the path, only save the file */
72 p
= f
->file
+ strlen(f
->file
);
73 while (p
>= f
->file
&& *p
!= '/')
77 strncpy(entry
->func
, f
->func
, TRACE_FUNC_SIZE
);
78 strncpy(entry
->file
, p
, TRACE_FILE_SIZE
);
79 entry
->func
[TRACE_FUNC_SIZE
] = 0;
80 entry
->file
[TRACE_FILE_SIZE
] = 0;
81 entry
->line
= f
->line
;
82 entry
->correct
= val
== expect
;
84 if (!filter_check_discard(call
, entry
, buffer
, event
))
85 __buffer_unlock_commit(buffer
, event
);
88 current
->trace_recursion
&= ~TRACE_BRANCH_BIT
;
89 raw_local_irq_restore(flags
);
93 void trace_likely_condition(struct ftrace_branch_data
*f
, int val
, int expect
)
95 if (!branch_tracing_enabled
)
98 probe_likely_condition(f
, val
, expect
);
101 int enable_branch_tracing(struct trace_array
*tr
)
103 mutex_lock(&branch_tracing_mutex
);
106 * Must be seen before enabling. The reader is a condition
107 * where we do not need a matching rmb()
110 branch_tracing_enabled
++;
111 mutex_unlock(&branch_tracing_mutex
);
116 void disable_branch_tracing(void)
118 mutex_lock(&branch_tracing_mutex
);
120 if (!branch_tracing_enabled
)
123 branch_tracing_enabled
--;
126 mutex_unlock(&branch_tracing_mutex
);
129 static void start_branch_trace(struct trace_array
*tr
)
131 enable_branch_tracing(tr
);
134 static void stop_branch_trace(struct trace_array
*tr
)
136 disable_branch_tracing();
139 static int branch_trace_init(struct trace_array
*tr
)
141 start_branch_trace(tr
);
145 static void branch_trace_reset(struct trace_array
*tr
)
147 stop_branch_trace(tr
);
150 static enum print_line_t
trace_branch_print(struct trace_iterator
*iter
,
151 int flags
, struct trace_event
*event
)
153 struct trace_branch
*field
;
155 trace_assign_type(field
, iter
->ent
);
157 if (trace_seq_printf(&iter
->seq
, "[%s] %s:%s:%d\n",
158 field
->correct
? " ok " : " MISS ",
162 return TRACE_TYPE_PARTIAL_LINE
;
164 return TRACE_TYPE_HANDLED
;
167 static void branch_print_header(struct seq_file
*s
)
169 seq_puts(s
, "# TASK-PID CPU# TIMESTAMP CORRECT"
170 " FUNC:FILE:LINE\n");
171 seq_puts(s
, "# | | | | | "
175 static struct trace_event_functions trace_branch_funcs
= {
176 .trace
= trace_branch_print
,
179 static struct trace_event trace_branch_event
= {
180 .type
= TRACE_BRANCH
,
181 .funcs
= &trace_branch_funcs
,
184 static struct tracer branch_trace __read_mostly
=
187 .init
= branch_trace_init
,
188 .reset
= branch_trace_reset
,
189 #ifdef CONFIG_FTRACE_SELFTEST
190 .selftest
= trace_selftest_startup_branch
,
191 #endif /* CONFIG_FTRACE_SELFTEST */
192 .print_header
= branch_print_header
,
195 __init
static int init_branch_tracer(void)
199 ret
= register_ftrace_event(&trace_branch_event
);
201 printk(KERN_WARNING
"Warning: could not register "
205 return register_tracer(&branch_trace
);
207 core_initcall(init_branch_tracer
);
211 void trace_likely_condition(struct ftrace_branch_data
*f
, int val
, int expect
)
214 #endif /* CONFIG_BRANCH_TRACER */
216 void ftrace_likely_update(struct ftrace_branch_data
*f
, int val
, int expect
)
219 * I would love to have a trace point here instead, but the
220 * trace point code is so inundated with unlikely and likely
221 * conditions that the recursive nightmare that exists is too
222 * much to try to get working. At least for now.
224 trace_likely_condition(f
, val
, expect
);
226 /* FIXME: Make this atomic! */
232 EXPORT_SYMBOL(ftrace_likely_update
);
234 extern unsigned long __start_annotated_branch_profile
[];
235 extern unsigned long __stop_annotated_branch_profile
[];
237 static int annotated_branch_stat_headers(struct seq_file
*m
)
239 seq_printf(m
, " correct incorrect %% ");
240 seq_printf(m
, " Function "
242 " ------- --------- - "
248 static inline long get_incorrect_percent(struct ftrace_branch_data
*p
)
253 percent
= p
->incorrect
* 100;
254 percent
/= p
->correct
+ p
->incorrect
;
256 percent
= p
->incorrect
? 100 : -1;
261 static int branch_stat_show(struct seq_file
*m
, void *v
)
263 struct ftrace_branch_data
*p
= v
;
267 /* Only print the file, not the path */
268 f
= p
->file
+ strlen(p
->file
);
269 while (f
>= p
->file
&& *f
!= '/')
274 * The miss is overlayed on correct, and hit on incorrect.
276 percent
= get_incorrect_percent(p
);
278 seq_printf(m
, "%8lu %8lu ", p
->correct
, p
->incorrect
);
280 seq_printf(m
, " X ");
282 seq_printf(m
, "%3ld ", percent
);
283 seq_printf(m
, "%-30.30s %-20.20s %d\n", p
->func
, f
, p
->line
);
287 static void *annotated_branch_stat_start(struct tracer_stat
*trace
)
289 return __start_annotated_branch_profile
;
293 annotated_branch_stat_next(void *v
, int idx
)
295 struct ftrace_branch_data
*p
= v
;
299 if ((void *)p
>= (void *)__stop_annotated_branch_profile
)
305 static int annotated_branch_stat_cmp(void *p1
, void *p2
)
307 struct ftrace_branch_data
*a
= p1
;
308 struct ftrace_branch_data
*b
= p2
;
310 long percent_a
, percent_b
;
312 percent_a
= get_incorrect_percent(a
);
313 percent_b
= get_incorrect_percent(b
);
315 if (percent_a
< percent_b
)
317 if (percent_a
> percent_b
)
320 if (a
->incorrect
< b
->incorrect
)
322 if (a
->incorrect
> b
->incorrect
)
326 * Since the above shows worse (incorrect) cases
327 * first, we continue that by showing best (correct)
330 if (a
->correct
> b
->correct
)
332 if (a
->correct
< b
->correct
)
338 static struct tracer_stat annotated_branch_stats
= {
339 .name
= "branch_annotated",
340 .stat_start
= annotated_branch_stat_start
,
341 .stat_next
= annotated_branch_stat_next
,
342 .stat_cmp
= annotated_branch_stat_cmp
,
343 .stat_headers
= annotated_branch_stat_headers
,
344 .stat_show
= branch_stat_show
347 __init
static int init_annotated_branch_stats(void)
351 ret
= register_stat_tracer(&annotated_branch_stats
);
353 printk(KERN_WARNING
"Warning: could not register "
354 "annotated branches stats\n");
359 fs_initcall(init_annotated_branch_stats
);
361 #ifdef CONFIG_PROFILE_ALL_BRANCHES
363 extern unsigned long __start_branch_profile
[];
364 extern unsigned long __stop_branch_profile
[];
366 static int all_branch_stat_headers(struct seq_file
*m
)
368 seq_printf(m
, " miss hit %% ");
369 seq_printf(m
, " Function "
371 " ------- --------- - "
377 static void *all_branch_stat_start(struct tracer_stat
*trace
)
379 return __start_branch_profile
;
383 all_branch_stat_next(void *v
, int idx
)
385 struct ftrace_branch_data
*p
= v
;
389 if ((void *)p
>= (void *)__stop_branch_profile
)
395 static struct tracer_stat all_branch_stats
= {
396 .name
= "branch_all",
397 .stat_start
= all_branch_stat_start
,
398 .stat_next
= all_branch_stat_next
,
399 .stat_headers
= all_branch_stat_headers
,
400 .stat_show
= branch_stat_show
403 __init
static int all_annotated_branch_stats(void)
407 ret
= register_stat_tracer(&all_branch_stats
);
409 printk(KERN_WARNING
"Warning: could not register "
410 "all branches stats\n");
415 fs_initcall(all_annotated_branch_stats
);
416 #endif /* CONFIG_PROFILE_ALL_BRANCHES */