Commit | Line | Data |
---|---|---|
81d68a96 | 1 | /* |
73d8b8bc | 2 | * trace irqs off critical timings |
81d68a96 SR |
3 | * |
4 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> | |
5 | * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> | |
6 | * | |
7 | * From code in the latency_tracer, that is: | |
8 | * | |
9 | * Copyright (C) 2004-2006 Ingo Molnar | |
6d49e352 | 10 | * Copyright (C) 2004 Nadia Yvette Chambers |
81d68a96 SR |
11 | */ |
12 | #include <linux/kallsyms.h> | |
13 | #include <linux/debugfs.h> | |
14 | #include <linux/uaccess.h> | |
15 | #include <linux/module.h> | |
16 | #include <linux/ftrace.h> | |
17 | #include <linux/fs.h> | |
18 | ||
19 | #include "trace.h" | |
20 | ||
21 | static struct trace_array *irqsoff_trace __read_mostly; | |
22 | static int tracer_enabled __read_mostly; | |
23 | ||
6cd8a4bb SR |
24 | static DEFINE_PER_CPU(int, tracing_cpu); |
25 | ||
5389f6fa | 26 | static DEFINE_RAW_SPINLOCK(max_trace_lock); |
89b2f978 | 27 | |
6cd8a4bb SR |
28 | enum { |
29 | TRACER_IRQS_OFF = (1 << 1), | |
30 | TRACER_PREEMPT_OFF = (1 << 2), | |
31 | }; | |
32 | ||
33 | static int trace_type __read_mostly; | |
34 | ||
613f04a0 | 35 | static int save_flags; |
328df475 | 36 | static bool function_enabled; |
e9d25fe6 | 37 | |
62b915f1 JO |
38 | static void stop_irqsoff_tracer(struct trace_array *tr, int graph); |
39 | static int start_irqsoff_tracer(struct trace_array *tr, int graph); | |
40 | ||
6cd8a4bb | 41 | #ifdef CONFIG_PREEMPT_TRACER |
e309b41d | 42 | static inline int |
6cd8a4bb SR |
43 | preempt_trace(void) |
44 | { | |
45 | return ((trace_type & TRACER_PREEMPT_OFF) && preempt_count()); | |
46 | } | |
47 | #else | |
48 | # define preempt_trace() (0) | |
49 | #endif | |
50 | ||
51 | #ifdef CONFIG_IRQSOFF_TRACER | |
e309b41d | 52 | static inline int |
6cd8a4bb SR |
53 | irq_trace(void) |
54 | { | |
55 | return ((trace_type & TRACER_IRQS_OFF) && | |
56 | irqs_disabled()); | |
57 | } | |
58 | #else | |
59 | # define irq_trace() (0) | |
60 | #endif | |
61 | ||
62b915f1 JO |
62 | #define TRACE_DISPLAY_GRAPH 1 |
63 | ||
64 | static struct tracer_opt trace_opts[] = { | |
65 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | |
66 | /* display latency trace as call graph */ | |
67 | { TRACER_OPT(display-graph, TRACE_DISPLAY_GRAPH) }, | |
68 | #endif | |
69 | { } /* Empty entry */ | |
70 | }; | |
71 | ||
72 | static struct tracer_flags tracer_flags = { | |
73 | .val = 0, | |
74 | .opts = trace_opts, | |
75 | }; | |
76 | ||
77 | #define is_graph() (tracer_flags.val & TRACE_DISPLAY_GRAPH) | |
78 | ||
81d68a96 SR |
79 | /* |
80 | * Sequence count - we record it when starting a measurement and | |
81 | * skip the latency if the sequence has changed - some other section | |
82 | * did a maximum and could disturb our measurement with serial console | |
83 | * printouts, etc. Truly coinciding maximum latencies should be rare | |
25985edc | 84 | * and what happens together happens separately as well, so this doesn't |
81d68a96 SR |
85 | * decrease the validity of the maximum found: |
86 | */ | |
87 | static __cacheline_aligned_in_smp unsigned long max_sequence; | |
88 | ||
606576ce | 89 | #ifdef CONFIG_FUNCTION_TRACER |
81d68a96 | 90 | /* |
5e6d2b9c SR |
91 | * Prologue for the preempt and irqs off function tracers. |
92 | * | |
93 | * Returns 1 if it is OK to continue, and data->disabled is | |
94 | * incremented. | |
95 | * 0 if the trace is to be ignored, and data->disabled | |
96 | * is kept the same. | |
97 | * | |
98 | * Note, this function is also used outside this ifdef but | |
99 | * inside the #ifdef of the function graph tracer below. | |
100 | * This is OK, since the function graph tracer is | |
101 | * dependent on the function tracer. | |
81d68a96 | 102 | */ |
5e6d2b9c SR |
103 | static int func_prolog_dec(struct trace_array *tr, |
104 | struct trace_array_cpu **data, | |
105 | unsigned long *flags) | |
81d68a96 | 106 | { |
81d68a96 SR |
107 | long disabled; |
108 | int cpu; | |
109 | ||
361943ad SR |
110 | /* |
111 | * Does not matter if we preempt. We test the flags | |
112 | * afterward, to see if irqs are disabled or not. | |
113 | * If we preempt and get a false positive, the flags | |
114 | * test will fail. | |
115 | */ | |
116 | cpu = raw_smp_processor_id(); | |
117 | if (likely(!per_cpu(tracing_cpu, cpu))) | |
5e6d2b9c | 118 | return 0; |
81d68a96 | 119 | |
5e6d2b9c | 120 | local_save_flags(*flags); |
e5509925 SRRH |
121 | /* |
122 | * Slight chance to get a false positive on tracing_cpu, | |
123 | * although I'm starting to think there isn't a chance. | |
124 | * Leave this for now just to be paranoid. | |
125 | */ | |
126 | if (!irqs_disabled_flags(*flags) && !preempt_count()) | |
5e6d2b9c | 127 | return 0; |
81d68a96 | 128 | |
12883efb | 129 | *data = per_cpu_ptr(tr->trace_buffer.data, cpu); |
5e6d2b9c | 130 | disabled = atomic_inc_return(&(*data)->disabled); |
81d68a96 SR |
131 | |
132 | if (likely(disabled == 1)) | |
5e6d2b9c SR |
133 | return 1; |
134 | ||
135 | atomic_dec(&(*data)->disabled); | |
136 | ||
137 | return 0; | |
138 | } | |
139 | ||
140 | /* | |
141 | * irqsoff uses its own tracer function to keep the overhead down: | |
142 | */ | |
143 | static void | |
2f5f6ad9 | 144 | irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip, |
a1e2e31d | 145 | struct ftrace_ops *op, struct pt_regs *pt_regs) |
5e6d2b9c SR |
146 | { |
147 | struct trace_array *tr = irqsoff_trace; | |
148 | struct trace_array_cpu *data; | |
149 | unsigned long flags; | |
150 | ||
151 | if (!func_prolog_dec(tr, &data, &flags)) | |
152 | return; | |
153 | ||
154 | trace_function(tr, ip, parent_ip, flags, preempt_count()); | |
81d68a96 SR |
155 | |
156 | atomic_dec(&data->disabled); | |
157 | } | |
158 | ||
159 | static struct ftrace_ops trace_ops __read_mostly = | |
160 | { | |
161 | .func = irqsoff_tracer_call, | |
4740974a | 162 | .flags = FTRACE_OPS_FL_GLOBAL | FTRACE_OPS_FL_RECURSION_SAFE, |
81d68a96 | 163 | }; |
606576ce | 164 | #endif /* CONFIG_FUNCTION_TRACER */ |
81d68a96 | 165 | |
62b915f1 JO |
166 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
167 | static int irqsoff_set_flag(u32 old_flags, u32 bit, int set) | |
168 | { | |
169 | int cpu; | |
170 | ||
171 | if (!(bit & TRACE_DISPLAY_GRAPH)) | |
172 | return -EINVAL; | |
173 | ||
174 | if (!(is_graph() ^ set)) | |
175 | return 0; | |
176 | ||
177 | stop_irqsoff_tracer(irqsoff_trace, !set); | |
178 | ||
179 | for_each_possible_cpu(cpu) | |
180 | per_cpu(tracing_cpu, cpu) = 0; | |
181 | ||
182 | tracing_max_latency = 0; | |
12883efb | 183 | tracing_reset_online_cpus(&irqsoff_trace->trace_buffer); |
62b915f1 JO |
184 | |
185 | return start_irqsoff_tracer(irqsoff_trace, set); | |
186 | } | |
187 | ||
188 | static int irqsoff_graph_entry(struct ftrace_graph_ent *trace) | |
189 | { | |
190 | struct trace_array *tr = irqsoff_trace; | |
191 | struct trace_array_cpu *data; | |
192 | unsigned long flags; | |
62b915f1 | 193 | int ret; |
62b915f1 JO |
194 | int pc; |
195 | ||
5e6d2b9c | 196 | if (!func_prolog_dec(tr, &data, &flags)) |
62b915f1 JO |
197 | return 0; |
198 | ||
5e6d2b9c SR |
199 | pc = preempt_count(); |
200 | ret = __trace_graph_entry(tr, trace, flags, pc); | |
62b915f1 | 201 | atomic_dec(&data->disabled); |
5e6d2b9c | 202 | |
62b915f1 JO |
203 | return ret; |
204 | } | |
205 | ||
206 | static void irqsoff_graph_return(struct ftrace_graph_ret *trace) | |
207 | { | |
208 | struct trace_array *tr = irqsoff_trace; | |
209 | struct trace_array_cpu *data; | |
210 | unsigned long flags; | |
62b915f1 JO |
211 | int pc; |
212 | ||
5e6d2b9c | 213 | if (!func_prolog_dec(tr, &data, &flags)) |
62b915f1 JO |
214 | return; |
215 | ||
5e6d2b9c SR |
216 | pc = preempt_count(); |
217 | __trace_graph_return(tr, trace, flags, pc); | |
62b915f1 JO |
218 | atomic_dec(&data->disabled); |
219 | } | |
220 | ||
221 | static void irqsoff_trace_open(struct trace_iterator *iter) | |
222 | { | |
223 | if (is_graph()) | |
224 | graph_trace_open(iter); | |
225 | ||
226 | } | |
227 | ||
228 | static void irqsoff_trace_close(struct trace_iterator *iter) | |
229 | { | |
230 | if (iter->private) | |
231 | graph_trace_close(iter); | |
232 | } | |
233 | ||
234 | #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_CPU | \ | |
321e68b0 JO |
235 | TRACE_GRAPH_PRINT_PROC | \ |
236 | TRACE_GRAPH_PRINT_ABS_TIME | \ | |
237 | TRACE_GRAPH_PRINT_DURATION) | |
62b915f1 JO |
238 | |
239 | static enum print_line_t irqsoff_print_line(struct trace_iterator *iter) | |
240 | { | |
62b915f1 JO |
241 | /* |
242 | * In graph mode call the graph tracer output function, | |
243 | * otherwise go with the TRACE_FN event handler | |
244 | */ | |
245 | if (is_graph()) | |
0a772620 | 246 | return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS); |
62b915f1 JO |
247 | |
248 | return TRACE_TYPE_UNHANDLED; | |
249 | } | |
250 | ||
251 | static void irqsoff_print_header(struct seq_file *s) | |
252 | { | |
0a772620 JO |
253 | if (is_graph()) |
254 | print_graph_headers_flags(s, GRAPH_TRACER_FLAGS); | |
255 | else | |
62b915f1 JO |
256 | trace_default_header(s); |
257 | } | |
258 | ||
62b915f1 JO |
259 | static void |
260 | __trace_function(struct trace_array *tr, | |
261 | unsigned long ip, unsigned long parent_ip, | |
262 | unsigned long flags, int pc) | |
263 | { | |
0a772620 JO |
264 | if (is_graph()) |
265 | trace_graph_function(tr, ip, parent_ip, flags, pc); | |
266 | else | |
62b915f1 | 267 | trace_function(tr, ip, parent_ip, flags, pc); |
62b915f1 JO |
268 | } |
269 | ||
270 | #else | |
271 | #define __trace_function trace_function | |
272 | ||
273 | static int irqsoff_set_flag(u32 old_flags, u32 bit, int set) | |
274 | { | |
275 | return -EINVAL; | |
276 | } | |
277 | ||
278 | static int irqsoff_graph_entry(struct ftrace_graph_ent *trace) | |
279 | { | |
280 | return -1; | |
281 | } | |
282 | ||
283 | static enum print_line_t irqsoff_print_line(struct trace_iterator *iter) | |
284 | { | |
285 | return TRACE_TYPE_UNHANDLED; | |
286 | } | |
287 | ||
288 | static void irqsoff_graph_return(struct ftrace_graph_ret *trace) { } | |
62b915f1 JO |
289 | static void irqsoff_trace_open(struct trace_iterator *iter) { } |
290 | static void irqsoff_trace_close(struct trace_iterator *iter) { } | |
7e9a49ef JO |
291 | |
292 | #ifdef CONFIG_FUNCTION_TRACER | |
293 | static void irqsoff_print_header(struct seq_file *s) | |
294 | { | |
295 | trace_default_header(s); | |
296 | } | |
297 | #else | |
298 | static void irqsoff_print_header(struct seq_file *s) | |
299 | { | |
300 | trace_latency_header(s); | |
301 | } | |
302 | #endif /* CONFIG_FUNCTION_TRACER */ | |
62b915f1 JO |
303 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
304 | ||
81d68a96 SR |
305 | /* |
306 | * Should this new latency be reported/recorded? | |
307 | */ | |
e309b41d | 308 | static int report_latency(cycle_t delta) |
81d68a96 SR |
309 | { |
310 | if (tracing_thresh) { | |
311 | if (delta < tracing_thresh) | |
312 | return 0; | |
313 | } else { | |
314 | if (delta <= tracing_max_latency) | |
315 | return 0; | |
316 | } | |
317 | return 1; | |
318 | } | |
319 | ||
e309b41d | 320 | static void |
81d68a96 SR |
321 | check_critical_timing(struct trace_array *tr, |
322 | struct trace_array_cpu *data, | |
323 | unsigned long parent_ip, | |
324 | int cpu) | |
325 | { | |
89b2f978 | 326 | cycle_t T0, T1, delta; |
81d68a96 | 327 | unsigned long flags; |
38697053 | 328 | int pc; |
81d68a96 | 329 | |
81d68a96 | 330 | T0 = data->preempt_timestamp; |
750ed1a4 | 331 | T1 = ftrace_now(cpu); |
81d68a96 SR |
332 | delta = T1-T0; |
333 | ||
334 | local_save_flags(flags); | |
335 | ||
6450c1d3 SR |
336 | pc = preempt_count(); |
337 | ||
81d68a96 SR |
338 | if (!report_latency(delta)) |
339 | goto out; | |
340 | ||
5389f6fa | 341 | raw_spin_lock_irqsave(&max_trace_lock, flags); |
81d68a96 | 342 | |
89b2f978 SR |
343 | /* check if we are still the max latency */ |
344 | if (!report_latency(delta)) | |
345 | goto out_unlock; | |
346 | ||
62b915f1 | 347 | __trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc); |
cc51a0fc SR |
348 | /* Skip 5 functions to get to the irq/preempt enable function */ |
349 | __trace_stack(tr, flags, 5, pc); | |
81d68a96 | 350 | |
81d68a96 | 351 | if (data->critical_sequence != max_sequence) |
89b2f978 | 352 | goto out_unlock; |
81d68a96 | 353 | |
81d68a96 SR |
354 | data->critical_end = parent_ip; |
355 | ||
b5130b1e CE |
356 | if (likely(!is_tracing_stopped())) { |
357 | tracing_max_latency = delta; | |
358 | update_max_tr_single(tr, current, cpu); | |
359 | } | |
81d68a96 | 360 | |
81d68a96 SR |
361 | max_sequence++; |
362 | ||
89b2f978 | 363 | out_unlock: |
5389f6fa | 364 | raw_spin_unlock_irqrestore(&max_trace_lock, flags); |
89b2f978 | 365 | |
81d68a96 SR |
366 | out: |
367 | data->critical_sequence = max_sequence; | |
750ed1a4 | 368 | data->preempt_timestamp = ftrace_now(cpu); |
62b915f1 | 369 | __trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc); |
81d68a96 SR |
370 | } |
371 | ||
e309b41d | 372 | static inline void |
81d68a96 SR |
373 | start_critical_timing(unsigned long ip, unsigned long parent_ip) |
374 | { | |
375 | int cpu; | |
376 | struct trace_array *tr = irqsoff_trace; | |
377 | struct trace_array_cpu *data; | |
378 | unsigned long flags; | |
379 | ||
2fd821ee | 380 | if (!tracer_enabled || !tracing_is_enabled()) |
81d68a96 SR |
381 | return; |
382 | ||
c5f888ca SR |
383 | cpu = raw_smp_processor_id(); |
384 | ||
385 | if (per_cpu(tracing_cpu, cpu)) | |
6cd8a4bb SR |
386 | return; |
387 | ||
12883efb | 388 | data = per_cpu_ptr(tr->trace_buffer.data, cpu); |
81d68a96 | 389 | |
c5f888ca | 390 | if (unlikely(!data) || atomic_read(&data->disabled)) |
81d68a96 SR |
391 | return; |
392 | ||
393 | atomic_inc(&data->disabled); | |
394 | ||
395 | data->critical_sequence = max_sequence; | |
750ed1a4 | 396 | data->preempt_timestamp = ftrace_now(cpu); |
6cd8a4bb | 397 | data->critical_start = parent_ip ? : ip; |
81d68a96 SR |
398 | |
399 | local_save_flags(flags); | |
6cd8a4bb | 400 | |
62b915f1 | 401 | __trace_function(tr, ip, parent_ip, flags, preempt_count()); |
81d68a96 | 402 | |
c5f888ca | 403 | per_cpu(tracing_cpu, cpu) = 1; |
6cd8a4bb | 404 | |
81d68a96 SR |
405 | atomic_dec(&data->disabled); |
406 | } | |
407 | ||
e309b41d | 408 | static inline void |
81d68a96 SR |
409 | stop_critical_timing(unsigned long ip, unsigned long parent_ip) |
410 | { | |
411 | int cpu; | |
412 | struct trace_array *tr = irqsoff_trace; | |
413 | struct trace_array_cpu *data; | |
414 | unsigned long flags; | |
415 | ||
c5f888ca | 416 | cpu = raw_smp_processor_id(); |
6cd8a4bb | 417 | /* Always clear the tracing cpu on stopping the trace */ |
c5f888ca SR |
418 | if (unlikely(per_cpu(tracing_cpu, cpu))) |
419 | per_cpu(tracing_cpu, cpu) = 0; | |
6cd8a4bb SR |
420 | else |
421 | return; | |
422 | ||
2fd821ee | 423 | if (!tracer_enabled || !tracing_is_enabled()) |
81d68a96 SR |
424 | return; |
425 | ||
12883efb | 426 | data = per_cpu_ptr(tr->trace_buffer.data, cpu); |
81d68a96 | 427 | |
3928a8a2 | 428 | if (unlikely(!data) || |
81d68a96 SR |
429 | !data->critical_start || atomic_read(&data->disabled)) |
430 | return; | |
431 | ||
432 | atomic_inc(&data->disabled); | |
c5f888ca | 433 | |
81d68a96 | 434 | local_save_flags(flags); |
62b915f1 | 435 | __trace_function(tr, ip, parent_ip, flags, preempt_count()); |
6cd8a4bb | 436 | check_critical_timing(tr, data, parent_ip ? : ip, cpu); |
81d68a96 SR |
437 | data->critical_start = 0; |
438 | atomic_dec(&data->disabled); | |
439 | } | |
440 | ||
6cd8a4bb | 441 | /* start and stop critical timings used to for stoppage (in idle) */ |
e309b41d | 442 | void start_critical_timings(void) |
81d68a96 | 443 | { |
6cd8a4bb | 444 | if (preempt_trace() || irq_trace()) |
81d68a96 SR |
445 | start_critical_timing(CALLER_ADDR0, CALLER_ADDR1); |
446 | } | |
1fe37104 | 447 | EXPORT_SYMBOL_GPL(start_critical_timings); |
81d68a96 | 448 | |
e309b41d | 449 | void stop_critical_timings(void) |
81d68a96 | 450 | { |
6cd8a4bb | 451 | if (preempt_trace() || irq_trace()) |
81d68a96 SR |
452 | stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1); |
453 | } | |
1fe37104 | 454 | EXPORT_SYMBOL_GPL(stop_critical_timings); |
81d68a96 | 455 | |
6cd8a4bb | 456 | #ifdef CONFIG_IRQSOFF_TRACER |
81d68a96 | 457 | #ifdef CONFIG_PROVE_LOCKING |
e309b41d | 458 | void time_hardirqs_on(unsigned long a0, unsigned long a1) |
81d68a96 | 459 | { |
6cd8a4bb | 460 | if (!preempt_trace() && irq_trace()) |
81d68a96 SR |
461 | stop_critical_timing(a0, a1); |
462 | } | |
463 | ||
e309b41d | 464 | void time_hardirqs_off(unsigned long a0, unsigned long a1) |
81d68a96 | 465 | { |
6cd8a4bb | 466 | if (!preempt_trace() && irq_trace()) |
81d68a96 SR |
467 | start_critical_timing(a0, a1); |
468 | } | |
469 | ||
470 | #else /* !CONFIG_PROVE_LOCKING */ | |
471 | ||
472 | /* | |
473 | * Stubs: | |
474 | */ | |
475 | ||
81d68a96 SR |
476 | void trace_softirqs_on(unsigned long ip) |
477 | { | |
478 | } | |
479 | ||
480 | void trace_softirqs_off(unsigned long ip) | |
481 | { | |
482 | } | |
483 | ||
e309b41d | 484 | inline void print_irqtrace_events(struct task_struct *curr) |
81d68a96 SR |
485 | { |
486 | } | |
487 | ||
488 | /* | |
489 | * We are only interested in hardirq on/off events: | |
490 | */ | |
e309b41d | 491 | void trace_hardirqs_on(void) |
81d68a96 | 492 | { |
6cd8a4bb | 493 | if (!preempt_trace() && irq_trace()) |
81d68a96 SR |
494 | stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1); |
495 | } | |
496 | EXPORT_SYMBOL(trace_hardirqs_on); | |
497 | ||
e309b41d | 498 | void trace_hardirqs_off(void) |
81d68a96 | 499 | { |
6cd8a4bb | 500 | if (!preempt_trace() && irq_trace()) |
81d68a96 SR |
501 | start_critical_timing(CALLER_ADDR0, CALLER_ADDR1); |
502 | } | |
503 | EXPORT_SYMBOL(trace_hardirqs_off); | |
504 | ||
e309b41d | 505 | void trace_hardirqs_on_caller(unsigned long caller_addr) |
81d68a96 | 506 | { |
6cd8a4bb | 507 | if (!preempt_trace() && irq_trace()) |
81d68a96 SR |
508 | stop_critical_timing(CALLER_ADDR0, caller_addr); |
509 | } | |
510 | EXPORT_SYMBOL(trace_hardirqs_on_caller); | |
511 | ||
e309b41d | 512 | void trace_hardirqs_off_caller(unsigned long caller_addr) |
81d68a96 | 513 | { |
6cd8a4bb | 514 | if (!preempt_trace() && irq_trace()) |
81d68a96 SR |
515 | start_critical_timing(CALLER_ADDR0, caller_addr); |
516 | } | |
517 | EXPORT_SYMBOL(trace_hardirqs_off_caller); | |
518 | ||
519 | #endif /* CONFIG_PROVE_LOCKING */ | |
6cd8a4bb SR |
520 | #endif /* CONFIG_IRQSOFF_TRACER */ |
521 | ||
522 | #ifdef CONFIG_PREEMPT_TRACER | |
e309b41d | 523 | void trace_preempt_on(unsigned long a0, unsigned long a1) |
6cd8a4bb | 524 | { |
e36de1de | 525 | if (preempt_trace() && !irq_trace()) |
1e01cb0c | 526 | stop_critical_timing(a0, a1); |
6cd8a4bb SR |
527 | } |
528 | ||
e309b41d | 529 | void trace_preempt_off(unsigned long a0, unsigned long a1) |
6cd8a4bb | 530 | { |
e36de1de | 531 | if (preempt_trace() && !irq_trace()) |
1e01cb0c | 532 | start_critical_timing(a0, a1); |
6cd8a4bb SR |
533 | } |
534 | #endif /* CONFIG_PREEMPT_TRACER */ | |
81d68a96 | 535 | |
328df475 | 536 | static int register_irqsoff_function(int graph, int set) |
81d68a96 | 537 | { |
328df475 | 538 | int ret; |
62b915f1 | 539 | |
328df475 SRRH |
540 | /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */ |
541 | if (function_enabled || (!set && !(trace_flags & TRACE_ITER_FUNCTION))) | |
542 | return 0; | |
543 | ||
544 | if (graph) | |
62b915f1 JO |
545 | ret = register_ftrace_graph(&irqsoff_graph_return, |
546 | &irqsoff_graph_entry); | |
328df475 SRRH |
547 | else |
548 | ret = register_ftrace_function(&trace_ops); | |
549 | ||
550 | if (!ret) | |
551 | function_enabled = true; | |
552 | ||
553 | return ret; | |
554 | } | |
555 | ||
556 | static void unregister_irqsoff_function(int graph) | |
557 | { | |
558 | if (!function_enabled) | |
559 | return; | |
560 | ||
561 | if (graph) | |
562 | unregister_ftrace_graph(); | |
563 | else | |
564 | unregister_ftrace_function(&trace_ops); | |
565 | ||
566 | function_enabled = false; | |
567 | } | |
568 | ||
569 | static void irqsoff_function_set(int set) | |
570 | { | |
571 | if (set) | |
572 | register_irqsoff_function(is_graph(), 1); | |
573 | else | |
574 | unregister_irqsoff_function(is_graph()); | |
575 | } | |
576 | ||
577 | static int irqsoff_flag_changed(struct tracer *tracer, u32 mask, int set) | |
578 | { | |
579 | if (mask & TRACE_ITER_FUNCTION) | |
580 | irqsoff_function_set(set); | |
581 | ||
582 | return trace_keep_overwrite(tracer, mask, set); | |
583 | } | |
584 | ||
585 | static int start_irqsoff_tracer(struct trace_array *tr, int graph) | |
586 | { | |
587 | int ret; | |
588 | ||
589 | ret = register_irqsoff_function(graph, 0); | |
62b915f1 JO |
590 | |
591 | if (!ret && tracing_is_enabled()) | |
9036990d | 592 | tracer_enabled = 1; |
94523e81 | 593 | else |
9036990d | 594 | tracer_enabled = 0; |
62b915f1 JO |
595 | |
596 | return ret; | |
81d68a96 SR |
597 | } |
598 | ||
62b915f1 | 599 | static void stop_irqsoff_tracer(struct trace_array *tr, int graph) |
81d68a96 | 600 | { |
81d68a96 | 601 | tracer_enabled = 0; |
62b915f1 | 602 | |
328df475 | 603 | unregister_irqsoff_function(graph); |
81d68a96 SR |
604 | } |
605 | ||
6cd8a4bb | 606 | static void __irqsoff_tracer_init(struct trace_array *tr) |
81d68a96 | 607 | { |
613f04a0 SRRH |
608 | save_flags = trace_flags; |
609 | ||
610 | /* non overwrite screws up the latency tracers */ | |
2b6080f2 SR |
611 | set_tracer_flag(tr, TRACE_ITER_OVERWRITE, 1); |
612 | set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, 1); | |
e9d25fe6 | 613 | |
745b1626 | 614 | tracing_max_latency = 0; |
81d68a96 | 615 | irqsoff_trace = tr; |
c5f888ca | 616 | /* make sure that the tracer is visible */ |
81d68a96 | 617 | smp_wmb(); |
12883efb | 618 | tracing_reset_online_cpus(&tr->trace_buffer); |
62b915f1 JO |
619 | |
620 | if (start_irqsoff_tracer(tr, is_graph())) | |
621 | printk(KERN_ERR "failed to start irqsoff tracer\n"); | |
81d68a96 SR |
622 | } |
623 | ||
624 | static void irqsoff_tracer_reset(struct trace_array *tr) | |
625 | { | |
613f04a0 SRRH |
626 | int lat_flag = save_flags & TRACE_ITER_LATENCY_FMT; |
627 | int overwrite_flag = save_flags & TRACE_ITER_OVERWRITE; | |
628 | ||
62b915f1 | 629 | stop_irqsoff_tracer(tr, is_graph()); |
e9d25fe6 | 630 | |
2b6080f2 SR |
631 | set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, lat_flag); |
632 | set_tracer_flag(tr, TRACE_ITER_OVERWRITE, overwrite_flag); | |
81d68a96 SR |
633 | } |
634 | ||
9036990d SR |
635 | static void irqsoff_tracer_start(struct trace_array *tr) |
636 | { | |
9036990d | 637 | tracer_enabled = 1; |
9036990d SR |
638 | } |
639 | ||
640 | static void irqsoff_tracer_stop(struct trace_array *tr) | |
641 | { | |
642 | tracer_enabled = 0; | |
81d68a96 SR |
643 | } |
644 | ||
6cd8a4bb | 645 | #ifdef CONFIG_IRQSOFF_TRACER |
1c80025a | 646 | static int irqsoff_tracer_init(struct trace_array *tr) |
6cd8a4bb SR |
647 | { |
648 | trace_type = TRACER_IRQS_OFF; | |
649 | ||
650 | __irqsoff_tracer_init(tr); | |
1c80025a | 651 | return 0; |
6cd8a4bb | 652 | } |
81d68a96 SR |
653 | static struct tracer irqsoff_tracer __read_mostly = |
654 | { | |
655 | .name = "irqsoff", | |
656 | .init = irqsoff_tracer_init, | |
657 | .reset = irqsoff_tracer_reset, | |
9036990d SR |
658 | .start = irqsoff_tracer_start, |
659 | .stop = irqsoff_tracer_stop, | |
f43c738b | 660 | .print_max = true, |
62b915f1 JO |
661 | .print_header = irqsoff_print_header, |
662 | .print_line = irqsoff_print_line, | |
663 | .flags = &tracer_flags, | |
664 | .set_flag = irqsoff_set_flag, | |
328df475 | 665 | .flag_changed = irqsoff_flag_changed, |
60a11774 SR |
666 | #ifdef CONFIG_FTRACE_SELFTEST |
667 | .selftest = trace_selftest_startup_irqsoff, | |
668 | #endif | |
62b915f1 JO |
669 | .open = irqsoff_trace_open, |
670 | .close = irqsoff_trace_close, | |
f43c738b | 671 | .use_max_tr = true, |
81d68a96 | 672 | }; |
6cd8a4bb SR |
673 | # define register_irqsoff(trace) register_tracer(&trace) |
674 | #else | |
675 | # define register_irqsoff(trace) do { } while (0) | |
676 | #endif | |
677 | ||
678 | #ifdef CONFIG_PREEMPT_TRACER | |
1c80025a | 679 | static int preemptoff_tracer_init(struct trace_array *tr) |
6cd8a4bb SR |
680 | { |
681 | trace_type = TRACER_PREEMPT_OFF; | |
682 | ||
683 | __irqsoff_tracer_init(tr); | |
1c80025a | 684 | return 0; |
6cd8a4bb SR |
685 | } |
686 | ||
687 | static struct tracer preemptoff_tracer __read_mostly = | |
688 | { | |
689 | .name = "preemptoff", | |
690 | .init = preemptoff_tracer_init, | |
691 | .reset = irqsoff_tracer_reset, | |
9036990d SR |
692 | .start = irqsoff_tracer_start, |
693 | .stop = irqsoff_tracer_stop, | |
f43c738b | 694 | .print_max = true, |
62b915f1 JO |
695 | .print_header = irqsoff_print_header, |
696 | .print_line = irqsoff_print_line, | |
697 | .flags = &tracer_flags, | |
698 | .set_flag = irqsoff_set_flag, | |
328df475 | 699 | .flag_changed = irqsoff_flag_changed, |
60a11774 SR |
700 | #ifdef CONFIG_FTRACE_SELFTEST |
701 | .selftest = trace_selftest_startup_preemptoff, | |
702 | #endif | |
62b915f1 JO |
703 | .open = irqsoff_trace_open, |
704 | .close = irqsoff_trace_close, | |
f43c738b | 705 | .use_max_tr = true, |
6cd8a4bb SR |
706 | }; |
707 | # define register_preemptoff(trace) register_tracer(&trace) | |
708 | #else | |
709 | # define register_preemptoff(trace) do { } while (0) | |
710 | #endif | |
711 | ||
712 | #if defined(CONFIG_IRQSOFF_TRACER) && \ | |
713 | defined(CONFIG_PREEMPT_TRACER) | |
714 | ||
1c80025a | 715 | static int preemptirqsoff_tracer_init(struct trace_array *tr) |
6cd8a4bb SR |
716 | { |
717 | trace_type = TRACER_IRQS_OFF | TRACER_PREEMPT_OFF; | |
718 | ||
719 | __irqsoff_tracer_init(tr); | |
1c80025a | 720 | return 0; |
6cd8a4bb SR |
721 | } |
722 | ||
723 | static struct tracer preemptirqsoff_tracer __read_mostly = | |
724 | { | |
725 | .name = "preemptirqsoff", | |
726 | .init = preemptirqsoff_tracer_init, | |
727 | .reset = irqsoff_tracer_reset, | |
9036990d SR |
728 | .start = irqsoff_tracer_start, |
729 | .stop = irqsoff_tracer_stop, | |
f43c738b | 730 | .print_max = true, |
62b915f1 JO |
731 | .print_header = irqsoff_print_header, |
732 | .print_line = irqsoff_print_line, | |
733 | .flags = &tracer_flags, | |
734 | .set_flag = irqsoff_set_flag, | |
328df475 | 735 | .flag_changed = irqsoff_flag_changed, |
60a11774 SR |
736 | #ifdef CONFIG_FTRACE_SELFTEST |
737 | .selftest = trace_selftest_startup_preemptirqsoff, | |
738 | #endif | |
62b915f1 JO |
739 | .open = irqsoff_trace_open, |
740 | .close = irqsoff_trace_close, | |
f43c738b | 741 | .use_max_tr = true, |
6cd8a4bb SR |
742 | }; |
743 | ||
744 | # define register_preemptirqsoff(trace) register_tracer(&trace) | |
745 | #else | |
746 | # define register_preemptirqsoff(trace) do { } while (0) | |
747 | #endif | |
81d68a96 SR |
748 | |
749 | __init static int init_irqsoff_tracer(void) | |
750 | { | |
6cd8a4bb SR |
751 | register_irqsoff(irqsoff_tracer); |
752 | register_preemptoff(preemptoff_tracer); | |
753 | register_preemptirqsoff(preemptirqsoff_tracer); | |
81d68a96 SR |
754 | |
755 | return 0; | |
756 | } | |
6f415672 | 757 | core_initcall(init_irqsoff_tracer); |