perf_counter: unify and fix delayed counter wakeup

[GitHub/LineageOS/android_kernel_samsung_universal7580.git] / kernel / perf_counter.c

diff --git a/kernel/perf_counter.c b/kernel/perf_counter.c
index 3b862a7988cda5180f79723073e9e77828a41fe6..f70ff80e79d7a6f99f42793a1d5a286fab2c57e4 100644 (file)
--- a/kernel/perf_counter.c
+++ b/kernel/perf_counter.c
@@ -1197,8 +1197,12 @@ static void free_counter_rcu(struct rcu_head *head)
        kfree(counter);
 }
 
+static void perf_pending_sync(struct perf_counter *counter);
+
 static void free_counter(struct perf_counter *counter)
 {
+       perf_pending_sync(counter);
+
        if (counter->destroy)
                counter->destroy(counter);
 
@@ -1528,6 +1532,118 @@ static const struct file_operations perf_fops = {
        .mmap                   = perf_mmap,
 };
 
+/*
+ * Perf counter wakeup
+ *
+ * If there's data, ensure we set the poll() state and publish everything
+ * to user-space before waking everybody up.
+ */
+
+void perf_counter_wakeup(struct perf_counter *counter)
+{
+       struct perf_mmap_data *data;
+
+       rcu_read_lock();
+       data = rcu_dereference(counter->data);
+       if (data) {
+               (void)atomic_xchg(&data->wakeup, POLL_IN);
+               __perf_counter_update_userpage(counter, data);
+       }
+       rcu_read_unlock();
+
+       wake_up_all(&counter->waitq);
+}
+
+/*
+ * Pending wakeups
+ *
+ * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
+ *
+ * The NMI bit means we cannot possibly take locks. Therefore, maintain a
+ * single linked list and use cmpxchg() to add entries lockless.
+ */
+
+#define PENDING_TAIL ((struct perf_wakeup_entry *)-1UL)
+
+static DEFINE_PER_CPU(struct perf_wakeup_entry *, perf_wakeup_head) = {
+       PENDING_TAIL,
+};
+
+static void perf_pending_queue(struct perf_counter *counter)
+{
+       struct perf_wakeup_entry **head;
+       struct perf_wakeup_entry *prev, *next;
+
+       if (cmpxchg(&counter->wakeup.next, NULL, PENDING_TAIL) != NULL)
+               return;
+
+       head = &get_cpu_var(perf_wakeup_head);
+
+       do {
+               prev = counter->wakeup.next = *head;
+               next = &counter->wakeup;
+       } while (cmpxchg(head, prev, next) != prev);
+
+       set_perf_counter_pending();
+
+       put_cpu_var(perf_wakeup_head);
+}
+
+static int __perf_pending_run(void)
+{
+       struct perf_wakeup_entry *list;
+       int nr = 0;
+
+       list = xchg(&__get_cpu_var(perf_wakeup_head), PENDING_TAIL);
+       while (list != PENDING_TAIL) {
+               struct perf_counter *counter = container_of(list,
+                               struct perf_counter, wakeup);
+
+               list = list->next;
+
+               counter->wakeup.next = NULL;
+               /*
+                * Ensure we observe the unqueue before we issue the wakeup,
+                * so that we won't be waiting forever.
+                * -- see perf_not_pending().
+                */
+               smp_wmb();
+
+               perf_counter_wakeup(counter);
+               nr++;
+       }
+
+       return nr;
+}
+
+static inline int perf_not_pending(struct perf_counter *counter)
+{
+       /*
+        * If we flush on whatever cpu we run, there is a chance we don't
+        * need to wait.
+        */
+       get_cpu();
+       __perf_pending_run();
+       put_cpu();
+
+       /*
+        * Ensure we see the proper queue state before going to sleep
+        * so that we do not miss the wakeup. -- see perf_pending_handle()
+        */
+       smp_rmb();
+       return counter->wakeup.next == NULL;
+}
+
+static void perf_pending_sync(struct perf_counter *counter)
+{
+       wait_event(counter->waitq, perf_not_pending(counter));
+}
+
+void perf_counter_do_pending(void)
+{
+       __perf_pending_run();
+}
+
 /*
  * Output
  */
@@ -1611,13 +1727,10 @@ static void perf_output_copy(struct perf_output_handle *handle,
 static void perf_output_end(struct perf_output_handle *handle, int nmi)
 {
        if (handle->wakeup) {
-               (void)atomic_xchg(&handle->data->wakeup, POLL_IN);
-               __perf_counter_update_userpage(handle->counter, handle->data);
-               if (nmi) {
-                       handle->counter->wakeup_pending = 1;
-                       set_perf_counter_pending();
-               } else
-                       wake_up(&handle->counter->waitq);
+               if (nmi)
+                       perf_pending_queue(handle->counter);
+               else
+                       perf_counter_wakeup(handle->counter);
        }
        rcu_read_unlock();
 }
@@ -2211,7 +2324,6 @@ perf_counter_alloc(struct perf_counter_hw_event *hw_event,
 
        counter->cpu                    = cpu;
        counter->hw_event               = *hw_event;
-       counter->wakeup_pending         = 0;
        counter->group_leader           = group_leader;
        counter->hw_ops                 = NULL;
        counter->ctx                    = ctx;