nohz: Allow rcu extended quiescent state handling seperately from tick stop

It is assumed that rcu won't be used once we switch to tickless
mode and until we restart the tick. However this is not always
true, as in x86-64 where we dereference the idle notifiers after
the tick is stopped.

To prepare for fixing this, add two new APIs:
tick_nohz_idle_enter_norcu() and tick_nohz_idle_exit_norcu().

If no use of RCU is made in the idle loop between
tick_nohz_enter_idle() and tick_nohz_exit_idle() calls, the arch
must instead call the new *_norcu() version such that the arch doesn't
need to call rcu_idle_enter() and rcu_idle_exit().

Otherwise the arch must call tick_nohz_enter_idle() and
tick_nohz_exit_idle() and also call explicitly:

- rcu_idle_enter() after its last use of RCU before the CPU is put
to sleep.
- rcu_idle_exit() before the first use of RCU after the CPU is woken
up.

Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Mike Frysinger <vapier@gentoo.org>
Cc: Guan Xuetao <gxt@mprc.pku.edu.cn>
Cc: David Miller <davem@davemloft.net>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Cc: Hans-Christian Egtvedt <hans-christian.egtvedt@atmel.com>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Russell King <linux@arm.linux.org.uk>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Paul Mundt <lethal@linux-sh.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c
index 3f1f8daf703cfa864f14199ef042aa9122334d9a..47e34c091276f4ee4c272d8bc696f2a66931a191 100644 (file)
--- a/arch/arm/kernel/process.c
+++ b/arch/arm/kernel/process.c
@@ -183,7 +183,7 @@ void cpu_idle(void)
 
        /* endless idle loop with no priority at all */
        while (1) {
-               tick_nohz_idle_enter();
+               tick_nohz_idle_enter_norcu();
                leds_event(led_idle_start);
                while (!need_resched()) {
 #ifdef CONFIG_HOTPLUG_CPU
@@ -213,7 +213,7 @@ void cpu_idle(void)
                        }
                }
                leds_event(led_idle_end);
-               tick_nohz_idle_exit();
+               tick_nohz_idle_exit_norcu();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/avr32/kernel/process.c b/arch/avr32/kernel/process.c
index 6ee7952248dbe5eef0da81f172025017ea0465ba..34c8c703bb161e3585d3bdc581413549d8dfbc3e 100644 (file)
--- a/arch/avr32/kernel/process.c
+++ b/arch/avr32/kernel/process.c
@@ -34,10 +34,10 @@ void cpu_idle(void)
 {
        /* endless idle loop with no priority at all */
        while (1) {
-               tick_nohz_idle_enter();
+               tick_nohz_idle_enter_norcu();
                while (!need_resched())
                        cpu_idle_sleep();
-               tick_nohz_idle_exit();
+               tick_nohz_idle_exit_norcu();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/blackfin/kernel/process.c b/arch/blackfin/kernel/process.c
index 7b141b5c9e8d629609996716b39e01d6e69cb87a..57e07498a0e76db1045028b801ccd718ed860e0b 100644 (file)
--- a/arch/blackfin/kernel/process.c
+++ b/arch/blackfin/kernel/process.c
@@ -88,10 +88,10 @@ void cpu_idle(void)
 #endif
                if (!idle)
                        idle = default_idle;
-               tick_nohz_idle_enter();
+               tick_nohz_idle_enter_norcu();
                while (!need_resched())
                        idle();
-               tick_nohz_idle_exit();
+               tick_nohz_idle_exit_norcu();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/microblaze/kernel/process.c b/arch/microblaze/kernel/process.c
index 5407f09b4be46cf8e09fcd001abb1607a59dd978..13d59f34b94e151ad5af802b3c88d352384f7b2a 100644 (file)
--- a/arch/microblaze/kernel/process.c
+++ b/arch/microblaze/kernel/process.c
@@ -103,10 +103,10 @@ void cpu_idle(void)
                if (!idle)
                        idle = default_idle;
 
-               tick_nohz_idle_enter();
+               tick_nohz_idle_enter_norcu();
                while (!need_resched())
                        idle();
-               tick_nohz_idle_exit();
+               tick_nohz_idle_exit_norcu();
 
                preempt_enable_no_resched();
                schedule();

diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c
index c11e5ca2a434a4cef23b80aa39a58f974ae97be2..17fb3a270160d3cd2b8903aac8a80bbcca6ce7de 100644 (file)
--- a/arch/mips/kernel/process.c
+++ b/arch/mips/kernel/process.c
@@ -56,7 +56,7 @@ void __noreturn cpu_idle(void)
 
        /* endless idle loop with no priority at all */
        while (1) {
-               tick_nohz_idle_enter();
+               tick_nohz_idle_enter_norcu();
                while (!need_resched() && cpu_online(cpu)) {
 #ifdef CONFIG_MIPS_MT_SMTC
                        extern void smtc_idle_loop_hook(void);
@@ -77,7 +77,7 @@ void __noreturn cpu_idle(void)
                     system_state == SYSTEM_BOOTING))
                        play_dead();
 #endif
-               tick_nohz_idle_exit();
+               tick_nohz_idle_exit_norcu();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/openrisc/kernel/idle.c b/arch/openrisc/kernel/idle.c
index fb6a9bf40006d1c5e98771c03152c1d0684b1346..2e82cd0fa5e1939840f3b7d1508b68a761247863 100644 (file)
--- a/arch/openrisc/kernel/idle.c
+++ b/arch/openrisc/kernel/idle.c
@@ -51,7 +51,7 @@ void cpu_idle(void)
 
        /* endless idle loop with no priority at all */
        while (1) {
-               tick_nohz_idle_enter();
+               tick_nohz_idle_enter_norcu();
 
                while (!need_resched()) {
                        check_pgt_cache();
@@ -69,7 +69,7 @@ void cpu_idle(void)
                        set_thread_flag(TIF_POLLING_NRFLAG);
                }
 
-               tick_nohz_idle_exit();
+               tick_nohz_idle_exit_norcu();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/powerpc/kernel/idle.c b/arch/powerpc/kernel/idle.c
index 878572f70ac5f5bcb1c24e5cab91061cb37ef689..2e782a36d8f2e321a5967e2b0e418b424a234fa9 100644 (file)
--- a/arch/powerpc/kernel/idle.c
+++ b/arch/powerpc/kernel/idle.c
@@ -56,7 +56,7 @@ void cpu_idle(void)
 
        set_thread_flag(TIF_POLLING_NRFLAG);
        while (1) {
-               tick_nohz_idle_enter();
+               tick_nohz_idle_enter_norcu();
                while (!need_resched() && !cpu_should_die()) {
                        ppc64_runlatch_off();
 
@@ -93,7 +93,7 @@ void cpu_idle(void)
 
                HMT_medium();
                ppc64_runlatch_on();
-               tick_nohz_idle_exit();
+               tick_nohz_idle_exit_norcu();
                preempt_enable_no_resched();
                if (cpu_should_die())
                        cpu_die();

diff --git a/arch/powerpc/platforms/iseries/setup.c b/arch/powerpc/platforms/iseries/setup.c
index e83dfaf89f6978cd53a4f4688666d55546d9fb19..d69d3d185e89289be10968fcb6517f47bcedbe5c 100644 (file)
--- a/arch/powerpc/platforms/iseries/setup.c
+++ b/arch/powerpc/platforms/iseries/setup.c
@@ -563,7 +563,7 @@ static void yield_shared_processor(void)
 static void iseries_shared_idle(void)
 {
        while (1) {
-               tick_nohz_idle_enter();
+               tick_nohz_idle_enter_norcu();
                while (!need_resched() && !hvlpevent_is_pending()) {
                        local_irq_disable();
                        ppc64_runlatch_off();
@@ -577,7 +577,7 @@ static void iseries_shared_idle(void)
                }
 
                ppc64_runlatch_on();
-               tick_nohz_idle_exit();
+               tick_nohz_idle_exit_norcu();
 
                if (hvlpevent_is_pending())
                        process_iSeries_events();
@@ -593,7 +593,7 @@ static void iseries_dedicated_idle(void)
        set_thread_flag(TIF_POLLING_NRFLAG);
 
        while (1) {
-               tick_nohz_idle_enter();
+               tick_nohz_idle_enter_norcu();
                if (!need_resched()) {
                        while (!need_resched()) {
                                ppc64_runlatch_off();
@@ -610,7 +610,7 @@ static void iseries_dedicated_idle(void)
                }
 
                ppc64_runlatch_on();
-               tick_nohz_idle_exit();
+               tick_nohz_idle_exit_norcu();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c
index 6224f9dbbc1f8c6bca54a6371568dfdb077106c2..6fa987367ae670ec14634f725751a7ef5cb1b4c9 100644 (file)
--- a/arch/s390/kernel/process.c
+++ b/arch/s390/kernel/process.c
@@ -91,10 +91,10 @@ static void default_idle(void)
 void cpu_idle(void)
 {
        for (;;) {
-               tick_nohz_idle_enter();
+               tick_nohz_idle_enter_norcu();
                while (!need_resched())
                        default_idle();
-               tick_nohz_idle_exit();
+               tick_nohz_idle_exit_norcu();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/sh/kernel/idle.c b/arch/sh/kernel/idle.c
index 6015743020a07af21b727f2dfe11544031d525bc..ad58e7535a7c479ba3236584f0c65bcacd019219 100644 (file)
--- a/arch/sh/kernel/idle.c
+++ b/arch/sh/kernel/idle.c
@@ -89,7 +89,7 @@ void cpu_idle(void)
 
        /* endless idle loop with no priority at all */
        while (1) {
-               tick_nohz_idle_enter();
+               tick_nohz_idle_enter_norcu();
 
                while (!need_resched()) {
                        check_pgt_cache();
@@ -111,7 +111,7 @@ void cpu_idle(void)
                        start_critical_timings();
                }
 
-               tick_nohz_idle_exit();
+               tick_nohz_idle_exit_norcu();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c
index 9c2795ba2cfe54763600544635823ce18735db9a..4a0e7d79cb926ee2a302ce78a2cfca247afbc0f4 100644 (file)
--- a/arch/sparc/kernel/process_64.c
+++ b/arch/sparc/kernel/process_64.c
@@ -95,12 +95,12 @@ void cpu_idle(void)
        set_thread_flag(TIF_POLLING_NRFLAG);
 
        while(1) {
-               tick_nohz_idle_enter();
+               tick_nohz_idle_enter_norcu();
 
                while (!need_resched() && !cpu_is_offline(cpu))
                        sparc64_yield(cpu);
 
-               tick_nohz_idle_exit();
+               tick_nohz_idle_exit_norcu();
 
                preempt_enable_no_resched();
 

diff --git a/arch/tile/kernel/process.c b/arch/tile/kernel/process.c
index 920e674aedb946a7e4cc0f20eab9fb92af896f3e..53ac89595ab153b1172dcdd3582677bf90c661c1 100644 (file)
--- a/arch/tile/kernel/process.c
+++ b/arch/tile/kernel/process.c
@@ -85,7 +85,7 @@ void cpu_idle(void)
 
        /* endless idle loop with no priority at all */
        while (1) {
-               tick_nohz_idle_enter();
+               tick_nohz_idle_enter_norcu();
                while (!need_resched()) {
                        if (cpu_is_offline(cpu))
                                BUG();  /* no HOTPLUG_CPU */
@@ -105,7 +105,7 @@ void cpu_idle(void)
                                local_irq_enable();
                        current_thread_info()->status |= TS_POLLING;
                }
-               tick_nohz_idle_exit();
+               tick_nohz_idle_exit_norcu();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/um/kernel/process.c b/arch/um/kernel/process.c
index cfb657e9284912576ebffc7b8b983e6c369b4075..55d2cf455f630b7f70bfd422bce51e707e7d87ab 100644 (file)
--- a/arch/um/kernel/process.c
+++ b/arch/um/kernel/process.c
@@ -246,10 +246,10 @@ void default_idle(void)
                if (need_resched())
                        schedule();
 
-               tick_nohz_idle_enter();
+               tick_nohz_idle_enter_norcu();
                nsecs = disable_timer();
                idle_sleep(nsecs);
-               tick_nohz_idle_exit();
+               tick_nohz_idle_exit_norcu();
        }
 }
 

diff --git a/arch/unicore32/kernel/process.c b/arch/unicore32/kernel/process.c
index 9999b9a84d4677379529a50b1009a9c3a70f8e93..095ff5a57928fb0f7e7e0b6c64bc8d73f6e41d24 100644 (file)
--- a/arch/unicore32/kernel/process.c
+++ b/arch/unicore32/kernel/process.c
@@ -55,7 +55,7 @@ void cpu_idle(void)
 {
        /* endless idle loop with no priority at all */
        while (1) {
-               tick_nohz_idle_enter();
+               tick_nohz_idle_enter_norcu();
                while (!need_resched()) {
                        local_irq_disable();
                        stop_critical_timings();
@@ -63,7 +63,7 @@ void cpu_idle(void)
                        local_irq_enable();
                        start_critical_timings();
                }
-               tick_nohz_idle_exit();
+               tick_nohz_idle_exit_norcu();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 6d9d4d52cac5af848b41a360c1d6cb32eb5bfe65..f94da3920c36bc1501b395b24ef3cac4ace20d78 100644 (file)
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -99,7 +99,7 @@ void cpu_idle(void)
 
        /* endless idle loop with no priority at all */
        while (1) {
-               tick_nohz_idle_enter();
+               tick_nohz_idle_enter_norcu();
                while (!need_resched()) {
 
                        check_pgt_cache();
@@ -116,7 +116,7 @@ void cpu_idle(void)
                                pm_idle();
                        start_critical_timings();
                }
-               tick_nohz_idle_exit();
+               tick_nohz_idle_exit_norcu();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index b069e9d7875f7c3b4552a9b25e0409dc9c1f87e1..18e8cf3581f660efa6d6b6d93fd3e5eb1bd7b533 100644 (file)
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -122,7 +122,7 @@ void cpu_idle(void)
 
        /* endless idle loop with no priority at all */
        while (1) {
-               tick_nohz_idle_enter();
+               tick_nohz_idle_enter_norcu();
                while (!need_resched()) {
 
                        rmb();
@@ -149,7 +149,7 @@ void cpu_idle(void)
                        __exit_idle();
                }
 
-               tick_nohz_idle_exit();
+               tick_nohz_idle_exit_norcu();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/include/linux/tick.h b/include/linux/tick.h
index 0df1d50a408ae13b7ef6528036dcfe580a887c45..327434a057572548f089443c8108422419923402 100644 (file)
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -7,6 +7,7 @@
 #define _LINUX_TICK_H
 
 #include <linux/clockchips.h>
+#include <linux/irqflags.h>
 
 #ifdef CONFIG_GENERIC_CLOCKEVENTS
 
@@ -121,18 +122,57 @@ static inline int tick_oneshot_mode_active(void) { return 0; }
 #endif /* !CONFIG_GENERIC_CLOCKEVENTS */
 
 # ifdef CONFIG_NO_HZ
-extern void tick_nohz_idle_enter(void);
+extern void __tick_nohz_idle_enter(void);
+static inline void tick_nohz_idle_enter(void)
+{
+       local_irq_disable();
+       __tick_nohz_idle_enter();
+       local_irq_enable();
+}
 extern void tick_nohz_idle_exit(void);
+
+/*
+ * Call this pair of function if the arch doesn't make any use
+ * of RCU in-between. You won't need to call rcu_idle_enter() and
+ * rcu_idle_exit().
+ * Otherwise you need to call tick_nohz_idle_enter() and tick_nohz_idle_exit()
+ * and explicitly tell RCU about the window around the place the CPU enters low
+ * power mode where no RCU use is made. This is done by calling rcu_idle_enter()
+ * after the last use of RCU before the CPU is put to sleep and by calling
+ * rcu_idle_exit() before the first use of RCU after the CPU woke up.
+ */
+static inline void tick_nohz_idle_enter_norcu(void)
+{
+       /*
+        * Also call rcu_idle_enter() in the irq disabled section even
+        * if it disables irq itself.
+        * Just an optimization that prevents from an interrupt happening
+        * between it and __tick_nohz_idle_enter() to lose time to help
+        * completing a grace period while we could be in extended grace
+        * period already.
+        */
+       local_irq_disable();
+       __tick_nohz_idle_enter();
+       rcu_idle_enter();
+       local_irq_enable();
+}
+static inline void tick_nohz_idle_exit_norcu(void)
+{
+       rcu_idle_exit();
+       tick_nohz_idle_exit();
+}
 extern void tick_nohz_irq_exit(void);
 extern ktime_t tick_nohz_get_sleep_length(void);
 extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time);
 extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time);
 # else
-static inline void tick_nohz_idle_enter(void)
+static inline void tick_nohz_idle_enter(void) { }
+static inline void tick_nohz_idle_exit(void) { }
+static inline void tick_nohz_idle_enter_norcu(void)
 {
        rcu_idle_enter();
 }
-static inline void tick_nohz_idle_exit(void)
+static inline void tick_nohz_idle_exit_norcu(void)
 {
        rcu_idle_exit();
 }

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 266c242dc354dda5739066e06e5589453b4bc1b4..c76aefe764b0f19e502da8feaf20bbe7acad726f 100644 (file)
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -453,18 +453,22 @@ out:
  *
  * When the next event is more than a tick into the future, stop the idle tick
  * Called when we start the idle loop.
- * This also enters into RCU extended quiescent state so that this CPU doesn't
- * need anymore to be part of any global grace period completion. This way
- * the tick can be stopped safely as we don't need to report quiescent states.
+ *
+ * If no use of RCU is made in the idle loop between
+ * tick_nohz_idle_enter() and tick_nohz_idle_exit() calls, then
+ * tick_nohz_idle_enter_norcu() should be called instead and the arch
+ * doesn't need to call rcu_idle_enter() and rcu_idle_exit() explicitly.
+ *
+ * Otherwise the arch is responsible of calling:
+ *
+ * - rcu_idle_enter() after its last use of RCU before the CPU is put
+ *  to sleep.
+ * - rcu_idle_exit() before the first use of RCU after the CPU is woken up.
  */
-void tick_nohz_idle_enter(void)
+void __tick_nohz_idle_enter(void)
 {
        struct tick_sched *ts;
 
-       WARN_ON_ONCE(irqs_disabled());
-
-       local_irq_disable();
-
        ts = &__get_cpu_var(tick_cpu_sched);
        /*
         * set ts->inidle unconditionally. even if the system did not
@@ -473,9 +477,6 @@ void tick_nohz_idle_enter(void)
         */
        ts->inidle = 1;
        tick_nohz_stop_sched_tick(ts);
-       rcu_idle_enter();
-
-       local_irq_enable();
 }
 
 /**
@@ -551,7 +552,7 @@ void tick_nohz_idle_exit(void)
        ktime_t now;
 
        local_irq_disable();
-       rcu_idle_exit();
+
        if (ts->idle_active || (ts->inidle && ts->tick_stopped))
                now = ktime_get();