perf_events, trace: Fix probe unregister race

tracepoint_probe_unregister() does not synchronize against the probe
callbacks, so do that explicitly. This properly serializes the callbacks
and the free of the data used therein.

Also, use this_cpu_ptr() where possible.

Acked-by: Frederic Weisbecker <fweisbec@gmail.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <1274438476.1674.1702.camel@laptop>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/include/trace/ftrace.h b/include/trace/ftrace.h
index 3d685d1f2a03f0099f159ac96eb648eed107a0a7..5a64905d7278a47fb683a0aceb63cef029dd467b 100644 (file)
--- a/include/trace/ftrace.h
+++ b/include/trace/ftrace.h
@@ -725,7 +725,7 @@ perf_trace_##call(void *__data, proto)                                      \
                                                                        \
        { assign; }                                                     \
                                                                        \
-       head = per_cpu_ptr(event_call->perf_events, smp_processor_id());\
+       head = this_cpu_ptr(event_call->perf_events);                   \
        perf_trace_buf_submit(entry, __entry_size, rctx, __addr,        \
                __count, &__regs, head);                                \
 }

diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c
index cb6f365016e48f0c7f7848cbaebb39d45fb80c45..49c7abf2ba5c1fafce486293c6f95688e017c2ba 100644 (file)
--- a/kernel/trace/trace_event_perf.c
+++ b/kernel/trace/trace_event_perf.c
@@ -116,7 +116,7 @@ int perf_trace_enable(struct perf_event *p_event)
        if (WARN_ON_ONCE(!list))
                return -EINVAL;
 
-       list = per_cpu_ptr(list, smp_processor_id());
+       list = this_cpu_ptr(list);
        hlist_add_head_rcu(&p_event->hlist_entry, list);
 
        return 0;
@@ -142,6 +142,12 @@ void perf_trace_destroy(struct perf_event *p_event)
                                            tp_event->class->perf_probe,
                                            tp_event);
 
+       /*
+        * Ensure our callback won't be called anymore. See
+        * tracepoint_probe_unregister() and __DO_TRACE().
+        */
+       synchronize_sched();
+
        free_percpu(tp_event->perf_events);
        tp_event->perf_events = NULL;
 
@@ -169,7 +175,7 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
        if (*rctxp < 0)
                return NULL;
 
-       raw_data = per_cpu_ptr(perf_trace_buf[*rctxp], smp_processor_id());
+       raw_data = this_cpu_ptr(perf_trace_buf[*rctxp]);
 
        /* zero the dead bytes from align to not leak stack to user */
        memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));

diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index faf7cefd15daf985afaadacb1d7a4b466993aa30..f52b5f50299dae5cc74dfe807054cd7d0e3f875e 100644 (file)
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -1359,7 +1359,7 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp,
        for (i = 0; i < tp->nr_args; i++)
                call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-       head = per_cpu_ptr(call->perf_events, smp_processor_id());
+       head = this_cpu_ptr(call->perf_events);
        perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head);
 }
 
@@ -1392,7 +1392,7 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri,
        for (i = 0; i < tp->nr_args; i++)
                call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-       head = per_cpu_ptr(call->perf_events, smp_processor_id());
+       head = this_cpu_ptr(call->perf_events);
        perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, regs, head);
 }
 

diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c
index d2c859cec9ea85b6f2e32c4c016937bee450aacb..34e35804304b03a7549b2f3a00064e8ec4675d7f 100644 (file)
--- a/kernel/trace/trace_syscalls.c
+++ b/kernel/trace/trace_syscalls.c
@@ -519,7 +519,7 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
        syscall_get_arguments(current, regs, 0, sys_data->nb_args,
                               (unsigned long *)&rec->args);
 
-       head = per_cpu_ptr(sys_data->enter_event->perf_events, smp_processor_id());
+       head = this_cpu_ptr(sys_data->enter_event->perf_events);
        perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head);
 }
 
@@ -595,7 +595,7 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
        rec->nr = syscall_nr;
        rec->ret = syscall_get_return_value(current, regs);
 
-       head = per_cpu_ptr(sys_data->exit_event->perf_events, smp_processor_id());
+       head = this_cpu_ptr(sys_data->exit_event->perf_events);
        perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head);
 }