rcu: Conditionally compile RCU's eqs warnings

This commit applies some warning-omission micro-optimizations to RCU's
various extended-quiescent-state functions, which are on the kernel/user
hotpath for CONFIG_NO_HZ_FULL=y.

Reported-by: Rik van Riel <riel@redhat.com>
Reported by: Mike Galbraith <umgwanakikbuti@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 0e9ce1272971bf270f2fd14fe5501e31f53c585e..991fa5c5dc5e3e04353a6d74599383c91e814cec 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -620,7 +620,8 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
        struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
 
        trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
-       if (!user && !is_idle_task(current)) {
+       if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+           !user && !is_idle_task(current)) {
                struct task_struct *idle __maybe_unused =
                        idle_task(smp_processor_id());
 
@@ -639,7 +640,8 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
        smp_mb__before_atomic();  /* See above. */
        atomic_inc(&rdtp->dynticks);
        smp_mb__after_atomic();  /* Force ordering with next sojourn. */
-       WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
+       WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+                    atomic_read(&rdtp->dynticks) & 0x1);
        rcu_dynticks_task_enter();
 
        /*
@@ -665,7 +667,8 @@ static void rcu_eqs_enter(bool user)
 
        rdtp = this_cpu_ptr(&rcu_dynticks);
        oldval = rdtp->dynticks_nesting;
-       WARN_ON_ONCE((oldval & DYNTICK_TASK_NEST_MASK) == 0);
+       WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+                    (oldval & DYNTICK_TASK_NEST_MASK) == 0);
        if ((oldval & DYNTICK_TASK_NEST_MASK) == DYNTICK_TASK_NEST_VALUE) {
                rdtp->dynticks_nesting = 0;
                rcu_eqs_enter_common(oldval, user);
@@ -738,7 +741,8 @@ void rcu_irq_exit(void)
        rdtp = this_cpu_ptr(&rcu_dynticks);
        oldval = rdtp->dynticks_nesting;
        rdtp->dynticks_nesting--;
-       WARN_ON_ONCE(rdtp->dynticks_nesting < 0);
+       WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+                    rdtp->dynticks_nesting < 0);
        if (rdtp->dynticks_nesting)
                trace_rcu_dyntick(TPS("--="), oldval, rdtp->dynticks_nesting);
        else
@@ -763,10 +767,12 @@ static void rcu_eqs_exit_common(long long oldval, int user)
        atomic_inc(&rdtp->dynticks);
        /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
        smp_mb__after_atomic();  /* See above. */
-       WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
+       WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+                    !(atomic_read(&rdtp->dynticks) & 0x1));
        rcu_cleanup_after_idle();
        trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
-       if (!user && !is_idle_task(current)) {
+       if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+           !user && !is_idle_task(current)) {
                struct task_struct *idle __maybe_unused =
                        idle_task(smp_processor_id());
 
@@ -790,7 +796,7 @@ static void rcu_eqs_exit(bool user)
 
        rdtp = this_cpu_ptr(&rcu_dynticks);
        oldval = rdtp->dynticks_nesting;
-       WARN_ON_ONCE(oldval < 0);
+       WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
        if (oldval & DYNTICK_TASK_NEST_MASK) {
                rdtp->dynticks_nesting += DYNTICK_TASK_NEST_VALUE;
        } else {
@@ -863,7 +869,8 @@ void rcu_irq_enter(void)
        rdtp = this_cpu_ptr(&rcu_dynticks);
        oldval = rdtp->dynticks_nesting;
        rdtp->dynticks_nesting++;
-       WARN_ON_ONCE(rdtp->dynticks_nesting == 0);
+       WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+                    rdtp->dynticks_nesting == 0);
        if (oldval)
                trace_rcu_dyntick(TPS("++="), oldval, rdtp->dynticks_nesting);
        else

diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index c4e1cf04cf57a4edbfab2d8e6b2e6c61526deb44..b908048f8d6a8e2b33723b222e1a4a88b2841774 100644 (file)
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1373,6 +1373,17 @@ config RCU_TRACE
          Say Y here if you want to enable RCU tracing
          Say N if you are unsure.
 
+config RCU_EQS_DEBUG
+       bool "Use this when adding any sort of NO_HZ support to your arch"
+       depends on DEBUG_KERNEL
+       help
+         This option provides consistency checks in RCU's handling of
+         NO_HZ.  These checks have proven quite helpful in detecting
+         bugs in arch-specific NO_HZ code.
+
+         Say N here if you need ultimate kernel/user switch latencies
+         Say Y if you are unsure
+
 endmenu # "RCU Debugging"
 
 config DEBUG_BLOCK_EXT_DEVT