perf_events: Undo some recursion damage

Make perf_swevent_get_recursion_context return a context number
and disable preemption.

This could be used to remove the IRQ disable from the trace bit
and index the per-cpu buffer with.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Paul Mackerras <paulus@samba.org>
LKML-Reference: <20091123103819.993226816@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 74e98b1d3391205e33d936eb19ce0b3fdaabbf10..43adbd7f0010c21719f75dc05dfbf5d61465abf7 100644 (file)
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -874,8 +874,8 @@ extern int perf_output_begin(struct perf_output_handle *handle,
 extern void perf_output_end(struct perf_output_handle *handle);
 extern void perf_output_copy(struct perf_output_handle *handle,
                             const void *buf, unsigned int len);
-extern int perf_swevent_get_recursion_context(int **recursion);
-extern void perf_swevent_put_recursion_context(int *recursion);
+extern int perf_swevent_get_recursion_context(void);
+extern void perf_swevent_put_recursion_context(int rctx);
 #else
 static inline void
 perf_event_task_sched_in(struct task_struct *task, int cpu)            { }
@@ -904,8 +904,8 @@ static inline void perf_event_mmap(struct vm_area_struct *vma)              { }
 static inline void perf_event_comm(struct task_struct *tsk)            { }
 static inline void perf_event_fork(struct task_struct *tsk)            { }
 static inline void perf_event_init(void)                               { }
-static int perf_swevent_get_recursion_context(int **recursion) { return -1; }
-static void perf_swevent_put_recursion_context(int *recursion)         { }
+static inline int  perf_swevent_get_recursion_context(void)  { return -1; }
+static inline void perf_swevent_put_recursion_context(int rctx)                { }
 
 #endif
 

diff --git a/include/trace/ftrace.h b/include/trace/ftrace.h
index c222ef5238bf366fc98599939a1c954ac9de6e8e..c3417c13e3edea3607adbf8d22545440ea861381 100644 (file)
--- a/include/trace/ftrace.h
+++ b/include/trace/ftrace.h
@@ -724,8 +724,8 @@ __attribute__((section("_ftrace_events"))) event_##call = {         \
 static void ftrace_profile_##call(proto)                               \
 {                                                                      \
        struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
-       extern int perf_swevent_get_recursion_context(int **recursion); \
-       extern void perf_swevent_put_recursion_context(int *recursion); \
+       extern int perf_swevent_get_recursion_context(void);            \
+       extern void perf_swevent_put_recursion_context(int rctx);       \
        struct ftrace_event_call *event_call = &event_##call;           \
        extern void perf_tp_event(int, u64, u64, void *, int);          \
        struct ftrace_raw_##call *entry;                                \
@@ -736,8 +736,8 @@ static void ftrace_profile_##call(proto)                            \
        int __data_size;                                                \
        char *trace_buf;                                                \
        char *raw_data;                                                 \
-       int *recursion;                                                 \
        int __cpu;                                                      \
+       int rctx;                                                       \
        int pc;                                                         \
                                                                        \
        pc = preempt_count();                                           \
@@ -753,8 +753,9 @@ static void ftrace_profile_##call(proto)                            \
                                                                        \
        local_irq_save(irq_flags);                                      \
                                                                        \
-       if (perf_swevent_get_recursion_context(&recursion))             \
-               goto end_recursion;                                             \
+       rctx = perf_swevent_get_recursion_context();                    \
+       if (rctx < 0)                                                   \
+               goto end_recursion;                                     \
                                                                        \
        __cpu = smp_processor_id();                                     \
                                                                        \
@@ -781,9 +782,9 @@ static void ftrace_profile_##call(proto)                            \
        perf_tp_event(event_call->id, __addr, __count, entry,           \
                             __entry_size);                             \
                                                                        \
-end:                                                           \
-       perf_swevent_put_recursion_context(recursion);                  \
-end_recursion:                                                                 \
+end:                                                                   \
+       perf_swevent_put_recursion_context(rctx);                       \
+end_recursion:                                                         \
        local_irq_restore(irq_flags);                                   \
                                                                        \
 }

diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index 50f11b5f8c3d621f17988274077e4df285e959be..0b0d5f72fe7d83a3c6695ed9d589f090b96a7c3e 100644 (file)
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -3869,45 +3869,50 @@ static void perf_swevent_ctx_event(struct perf_event_context *ctx,
        }
 }
 
-/*
- * Must be called with preemption disabled
- */
-int perf_swevent_get_recursion_context(int **recursion)
+int perf_swevent_get_recursion_context(void)
 {
-       struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
+       struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
+       int rctx;
 
        if (in_nmi())
-               *recursion = &cpuctx->recursion[3];
+               rctx = 3;
        else if (in_irq())
-               *recursion = &cpuctx->recursion[2];
+               rctx = 2;
        else if (in_softirq())
-               *recursion = &cpuctx->recursion[1];
+               rctx = 1;
        else
-               *recursion = &cpuctx->recursion[0];
+               rctx = 0;
 
-       if (**recursion)
+       if (cpuctx->recursion[rctx]) {
+               put_cpu_var(perf_cpu_context);
                return -1;
+       }
 
-       (**recursion)++;
+       cpuctx->recursion[rctx]++;
+       barrier();
 
-       return 0;
+       return rctx;
 }
 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
 
-void perf_swevent_put_recursion_context(int *recursion)
+void perf_swevent_put_recursion_context(int rctx)
 {
-       (*recursion)--;
+       struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
+       barrier();
+       cpuctx->recursion[rctx]++;
+       put_cpu_var(perf_cpu_context);
 }
 EXPORT_SYMBOL_GPL(perf_swevent_put_recursion_context);
 
-static void __do_perf_sw_event(enum perf_type_id type, u32 event_id,
-                              u64 nr, int nmi,
-                              struct perf_sample_data *data,
-                              struct pt_regs *regs)
+static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
+                                   u64 nr, int nmi,
+                                   struct perf_sample_data *data,
+                                   struct pt_regs *regs)
 {
+       struct perf_cpu_context *cpuctx;
        struct perf_event_context *ctx;
-       struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 
+       cpuctx = &__get_cpu_var(perf_cpu_context);
        rcu_read_lock();
        perf_swevent_ctx_event(&cpuctx->ctx, type, event_id,
                                 nr, nmi, data, regs);
@@ -3921,34 +3926,22 @@ static void __do_perf_sw_event(enum perf_type_id type, u32 event_id,
        rcu_read_unlock();
 }
 
-static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
-                                   u64 nr, int nmi,
-                                   struct perf_sample_data *data,
-                                   struct pt_regs *regs)
-{
-       int *recursion;
-
-       preempt_disable();
-
-       if (perf_swevent_get_recursion_context(&recursion))
-               goto out;
-
-       __do_perf_sw_event(type, event_id, nr, nmi, data, regs);
-
-       perf_swevent_put_recursion_context(recursion);
-out:
-       preempt_enable();
-}
-
 void __perf_sw_event(u32 event_id, u64 nr, int nmi,
                            struct pt_regs *regs, u64 addr)
 {
        struct perf_sample_data data;
+       int rctx;
+
+       rctx = perf_swevent_get_recursion_context();
+       if (rctx < 0)
+               return;
 
        data.addr = addr;
        data.raw  = NULL;
 
        do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs);
+
+       perf_swevent_put_recursion_context(rctx);
 }
 
 static void perf_swevent_read(struct perf_event *event)
@@ -4172,7 +4165,7 @@ void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
                regs = task_pt_regs(current);
 
        /* Trace events already protected against recursion */
-       __do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
+       do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
                                &data, regs);
 }
 EXPORT_SYMBOL_GPL(perf_tp_event);

diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index 22e6f68b05b37ce57e6691be89e4102276a46f6f..79ce6a2bd74fada4009c5c061dc7e8c4c2a16023 100644 (file)
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -1213,7 +1213,7 @@ static __kprobes int kprobe_profile_func(struct kprobe *kp,
        unsigned long irq_flags;
        char *trace_buf;
        char *raw_data;
-       int *recursion;
+       int rctx;
 
        pc = preempt_count();
        __size = SIZEOF_KPROBE_TRACE_ENTRY(tp->nr_args);
@@ -1229,7 +1229,8 @@ static __kprobes int kprobe_profile_func(struct kprobe *kp,
         */
        local_irq_save(irq_flags);
 
-       if (perf_swevent_get_recursion_context(&recursion))
+       rctx = perf_swevent_get_recursion_context();
+       if (rctx < 0)
                goto end_recursion;
 
        __cpu = smp_processor_id();
@@ -1258,7 +1259,7 @@ static __kprobes int kprobe_profile_func(struct kprobe *kp,
        perf_tp_event(call->id, entry->ip, 1, entry, size);
 
 end:
-       perf_swevent_put_recursion_context(recursion);
+       perf_swevent_put_recursion_context(rctx);
 end_recursion:
        local_irq_restore(irq_flags);
 
@@ -1276,8 +1277,8 @@ static __kprobes int kretprobe_profile_func(struct kretprobe_instance *ri,
        int size, __size, i, pc, __cpu;
        unsigned long irq_flags;
        char *trace_buf;
-       int *recursion;
        char *raw_data;
+       int rctx;
 
        pc = preempt_count();
        __size = SIZEOF_KRETPROBE_TRACE_ENTRY(tp->nr_args);
@@ -1293,7 +1294,8 @@ static __kprobes int kretprobe_profile_func(struct kretprobe_instance *ri,
         */
        local_irq_save(irq_flags);
 
-       if (perf_swevent_get_recursion_context(&recursion))
+       rctx = perf_swevent_get_recursion_context();
+       if (rctx < 0)
                goto end_recursion;
 
        __cpu = smp_processor_id();
@@ -1323,7 +1325,7 @@ static __kprobes int kretprobe_profile_func(struct kretprobe_instance *ri,
        perf_tp_event(call->id, entry->ret_ip, 1, entry, size);
 
 end:
-       perf_swevent_put_recursion_context(recursion);
+       perf_swevent_put_recursion_context(rctx);
 end_recursion:
        local_irq_restore(irq_flags);
 

diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c
index 41b6dd963daad12efcadc7711d9d73319eb96bfe..9189cbe86079d8bff3bc9f48025bc3789ee6ed21 100644 (file)
--- a/kernel/trace/trace_syscalls.c
+++ b/kernel/trace/trace_syscalls.c
@@ -481,8 +481,8 @@ static void prof_syscall_enter(struct pt_regs *regs, long id)
        unsigned long flags;
        char *trace_buf;
        char *raw_data;
-       int *recursion;
        int syscall_nr;
+       int rctx;
        int size;
        int cpu;
 
@@ -506,7 +506,8 @@ static void prof_syscall_enter(struct pt_regs *regs, long id)
        /* Protect the per cpu buffer, begin the rcu read side */
        local_irq_save(flags);
 
-       if (perf_swevent_get_recursion_context(&recursion))
+       rctx = perf_swevent_get_recursion_context();
+       if (rctx < 0)
                goto end_recursion;
 
        cpu = smp_processor_id();
@@ -530,7 +531,7 @@ static void prof_syscall_enter(struct pt_regs *regs, long id)
        perf_tp_event(sys_data->enter_id, 0, 1, rec, size);
 
 end:
-       perf_swevent_put_recursion_context(recursion);
+       perf_swevent_put_recursion_context(rctx);
 end_recursion:
        local_irq_restore(flags);
 }
@@ -582,7 +583,7 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret)
        int syscall_nr;
        char *trace_buf;
        char *raw_data;
-       int *recursion;
+       int rctx;
        int size;
        int cpu;
 
@@ -609,7 +610,8 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret)
        /* Protect the per cpu buffer, begin the rcu read side */
        local_irq_save(flags);
 
-       if (perf_swevent_get_recursion_context(&recursion))
+       rctx = perf_swevent_get_recursion_context();
+       if (rctx < 0)
                goto end_recursion;
 
        cpu = smp_processor_id();
@@ -634,7 +636,7 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret)
        perf_tp_event(sys_data->exit_id, 0, 1, rec, size);
 
 end:
-       perf_swevent_put_recursion_context(recursion);
+       perf_swevent_put_recursion_context(rctx);
 end_recursion:
        local_irq_restore(flags);
 }