perf: Optimize perf_output_*() by avoiding local_xchg()

Since the x86 XCHG ins implies LOCK, avoid the use by
using a sequence count instead.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index ce7667616fcbe8bf40cd85c6ffedb3cbd2e3fe02..fe50347dc645a88687ec4ac59f4fc6d65c6e645d 100644 (file)
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -804,6 +804,7 @@ struct perf_output_handle {
        struct perf_mmap_data           *data;
        unsigned long                   head;
        unsigned long                   offset;
+       unsigned long                   wakeup;
        int                             nmi;
        int                             sample;
 };

diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index 1f98c78c33437d2fba494e38c75e11efca7a2232..7e3bcf1a29f074738cdb971f4d15acbba5cc2d96 100644 (file)
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -2917,6 +2917,7 @@ static void perf_output_get_handle(struct perf_output_handle *handle)
 
        preempt_disable();
        local_inc(&data->nest);
+       handle->wakeup = local_read(&data->wakeup);
 }
 
 static void perf_output_put_handle(struct perf_output_handle *handle)
@@ -2950,7 +2951,7 @@ again:
                goto again;
        }
 
-       if (local_xchg(&data->wakeup, 0))
+       if (handle->wakeup != local_read(&data->wakeup))
                perf_output_wakeup(handle);
 
        preempt_enable();