sched: Convert to struct llist

Use the generic llist primitives.

We had a private lockless list implementation in the scheduler in the wake-list
code, now that we have a generic llist implementation that provides all required
operations, switch to it.

This patch is not expected to change any behavior.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Huang Ying <ying.huang@intel.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Link: http://lkml.kernel.org/r/1315836353.26517.42.camel@twins
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 9fda2888a6ab6f4c2a772115e897fac623154625..fc3e8911818af630d02e37acf150fb241a6f6f09 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -90,6 +90,7 @@ struct sched_param {
 #include <linux/task_io_accounting.h>
 #include <linux/latencytop.h>
 #include <linux/cred.h>
+#include <linux/llist.h>
 
 #include <asm/processor.h>
 
@@ -1225,7 +1226,7 @@ struct task_struct {
        unsigned int ptrace;
 
 #ifdef CONFIG_SMP
-       struct task_struct *wake_entry;
+       struct llist_node wake_entry;
        int on_cpu;
 #endif
        int on_rq;

diff --git a/kernel/sched.c b/kernel/sched.c
index c5cf15e1eb57cd6a722e37e56f70ecb7027e40f2..1874c7418319d302223a9e3abed29fe581af89cc 100644 (file)
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -702,7 +702,7 @@ struct rq {
 #endif
 
 #ifdef CONFIG_SMP
-       struct task_struct *wake_list;
+       struct llist_head wake_list;
 #endif
 };
 
@@ -2698,42 +2698,26 @@ static int ttwu_remote(struct task_struct *p, int wake_flags)
 }
 
 #ifdef CONFIG_SMP
-static void sched_ttwu_do_pending(struct task_struct *list)
+static void sched_ttwu_pending(void)
 {
        struct rq *rq = this_rq();
+       struct llist_node *llist = llist_del_all(&rq->wake_list);
+       struct task_struct *p;
 
        raw_spin_lock(&rq->lock);
 
-       while (list) {
-               struct task_struct *p = list;
-               list = list->wake_entry;
+       while (llist) {
+               p = llist_entry(llist, struct task_struct, wake_entry);
+               llist = llist_next(llist);
                ttwu_do_activate(rq, p, 0);
        }
 
        raw_spin_unlock(&rq->lock);
 }
 
-#ifdef CONFIG_HOTPLUG_CPU
-
-static void sched_ttwu_pending(void)
-{
-       struct rq *rq = this_rq();
-       struct task_struct *list = xchg(&rq->wake_list, NULL);
-
-       if (!list)
-               return;
-
-       sched_ttwu_do_pending(list);
-}
-
-#endif /* CONFIG_HOTPLUG_CPU */
-
 void scheduler_ipi(void)
 {
-       struct rq *rq = this_rq();
-       struct task_struct *list = xchg(&rq->wake_list, NULL);
-
-       if (!list)
+       if (llist_empty(&this_rq()->wake_list))
                return;
 
        /*
@@ -2750,25 +2734,13 @@ void scheduler_ipi(void)
         * somewhat pessimize the simple resched case.
         */
        irq_enter();
-       sched_ttwu_do_pending(list);
+       sched_ttwu_pending();
        irq_exit();
 }
 
 static void ttwu_queue_remote(struct task_struct *p, int cpu)
 {
-       struct rq *rq = cpu_rq(cpu);
-       struct task_struct *next = rq->wake_list;
-
-       for (;;) {
-               struct task_struct *old = next;
-
-               p->wake_entry = next;
-               next = cmpxchg(&rq->wake_list, old, p);
-               if (next == old)
-                       break;
-       }
-
-       if (!next)
+       if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list))
                smp_send_reschedule(cpu);
 }