cpuidle: powerpc: no memory barrier after break from idle

A memory barrier is not required after the task wakes up,
only if we clear the polling flag before waking. The case
where we have work to do is the important one, so optimise
for it.

Reviewed-by: Vaidyanathan Srinivasan <svaidy@linux.vnet.ibm.com>
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>

diff --git a/drivers/cpuidle/cpuidle-powernv.c b/drivers/cpuidle/cpuidle-powernv.c
index 9d03326ac05e052c4cfc0c531ae0977a2af9ba5a..37b0698b7193e60be4107a8be107285e55c65877 100644 (file)
--- a/drivers/cpuidle/cpuidle-powernv.c
+++ b/drivers/cpuidle/cpuidle-powernv.c
@@ -59,14 +59,21 @@ static int snooze_loop(struct cpuidle_device *dev,
        ppc64_runlatch_off();
        HMT_very_low();
        while (!need_resched()) {
-               if (likely(snooze_timeout_en) && get_tb() > snooze_exit_time)
+               if (likely(snooze_timeout_en) && get_tb() > snooze_exit_time) {
+                       /*
+                        * Task has not woken up but we are exiting the polling
+                        * loop anyway. Require a barrier after polling is
+                        * cleared to order subsequent test of need_resched().
+                        */
+                       clear_thread_flag(TIF_POLLING_NRFLAG);
+                       smp_mb();
                        break;
+               }
        }
 
        HMT_medium();
        ppc64_runlatch_on();
        clear_thread_flag(TIF_POLLING_NRFLAG);
-       smp_mb();
 
        return index;
 }

diff --git a/drivers/cpuidle/cpuidle-pseries.c b/drivers/cpuidle/cpuidle-pseries.c
index a404f352d284f9731b37caf22782e0413276abf9..e9b3853d93ea3754cd1972c765a2bab2c8bc4e83 100644 (file)
--- a/drivers/cpuidle/cpuidle-pseries.c
+++ b/drivers/cpuidle/cpuidle-pseries.c
@@ -71,13 +71,20 @@ static int snooze_loop(struct cpuidle_device *dev,
        while (!need_resched()) {
                HMT_low();
                HMT_very_low();
-               if (snooze_timeout_en && get_tb() > snooze_exit_time)
+               if (likely(snooze_timeout_en) && get_tb() > snooze_exit_time) {
+                       /*
+                        * Task has not woken up but we are exiting the polling
+                        * loop anyway. Require a barrier after polling is
+                        * cleared to order subsequent test of need_resched().
+                        */
+                       clear_thread_flag(TIF_POLLING_NRFLAG);
+                       smp_mb();
                        break;
+               }
        }
 
        HMT_medium();
        clear_thread_flag(TIF_POLLING_NRFLAG);
-       smp_mb();
 
        idle_loop_epilog(in_purr);