stop_machine: Avoid a sleep and wakeup in stop_one_cpu()

In case @cpu == smp_proccessor_id(), we can avoid a sleep+wakeup
cycle by doing a preemption.

Callers such as sched_exec() can benefit from this change.

Signed-off-by: Cheng Chao <cs.os.kernel@gmail.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: akpm@linux-foundation.org
Cc: chris@chris-wilson.co.uk
Cc: tj@kernel.org
Link: http://lkml.kernel.org/r/1473818510-6779-1-git-send-email-cs.os.kernel@gmail.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index c5f020c601b28094e8a2f1c5d74f1a01f9710468..ff4e3c066dc2e6aee479aacff30556e50b7d5ec1 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1063,8 +1063,12 @@ static int migration_cpu_stop(void *data)
         * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because
         * we're holding p->pi_lock.
         */
-       if (task_rq(p) == rq && task_on_rq_queued(p))
-               rq = __migrate_task(rq, p, arg->dest_cpu);
+       if (task_rq(p) == rq) {
+               if (task_on_rq_queued(p))
+                       rq = __migrate_task(rq, p, arg->dest_cpu);
+               else
+                       p->wake_cpu = arg->dest_cpu;
+       }
        raw_spin_unlock(&rq->lock);
        raw_spin_unlock(&p->pi_lock);
 

diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index 4a1ca5f6da7e15b7c9efd200eff1a455ab1412ab..082e71f17a58544b5b6c0f983d1c3aaf7bda6c62 100644 (file)
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -126,6 +126,11 @@ int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg)
        cpu_stop_init_done(&done, 1);
        if (!cpu_stop_queue_work(cpu, &work))
                return -ENOENT;
+       /*
+        * In case @cpu == smp_proccessor_id() we can avoid a sleep+wakeup
+        * cycle by doing a preemption:
+        */
+       cond_resched();
        wait_for_completion(&done.completion);
        return done.ret;
 }