rcu: Don't redundantly disable irqs in rcu_irq_{enter,exit}()

This commit replaces a local_irq_save()/local_irq_restore() pair with
a lockdep assertion that interrupts are already disabled.  This should
remove the corresponding overhead from the interrupt entry/exit fastpaths.

This change was inspired by the fact that Iftekhar Ahmed's mutation
testing showed that removing rcu_irq_enter()'s call to local_ird_restore()
had no effect, which might indicate that interrupts were always enabled
anyway.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index a0189ba67fde721a824cc7c8db4b1f015197059e..f2b667df113142cee630742a9384d0c4e0aee86e 100644 (file)
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -379,9 +379,9 @@ static inline void rcu_init_nohz(void)
  */
 #define RCU_NONIDLE(a) \
        do { \
-               rcu_irq_enter(); \
+               rcu_irq_enter_irqson(); \
                do { a; } while (0); \
-               rcu_irq_exit(); \
+               rcu_irq_exit_irqson(); \
        } while (0)
 
 /*

diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h
index 4c1aaf9cce7b3ac767c69d7849715d69089a9b48..64809aea661cee43646c4803247604cc40f16d6c 100644 (file)
--- a/include/linux/rcutiny.h
+++ b/include/linux/rcutiny.h
@@ -181,6 +181,14 @@ static inline void rcu_irq_enter(void)
 {
 }
 
+static inline void rcu_irq_exit_irqson(void)
+{
+}
+
+static inline void rcu_irq_enter_irqson(void)
+{
+}
+
 static inline void rcu_irq_exit(void)
 {
 }

diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
index 9d3eda39bcd25bd1a2775c80f8dd58c7640600df..ad1eda9fa4daea077998d253dc604c18c3d444c6 100644 (file)
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -97,6 +97,8 @@ void rcu_idle_enter(void);
 void rcu_idle_exit(void);
 void rcu_irq_enter(void);
 void rcu_irq_exit(void);
+void rcu_irq_enter_irqson(void);
+void rcu_irq_exit_irqson(void);
 
 void exit_rcu(void);
 

diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index 696a339c592c831942a8fcdc9b2e5ea963599747..7834a8a8bf1e2f14de4cda1cef16833429847057 100644 (file)
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -171,8 +171,8 @@ extern void syscall_unregfunc(void);
                                TP_PROTO(data_proto),                   \
                                TP_ARGS(data_args),                     \
                                TP_CONDITION(cond),                     \
-                               rcu_irq_enter(),                        \
-                               rcu_irq_exit());                        \
+                               rcu_irq_enter_irqson(),                 \
+                               rcu_irq_exit_irqson());                 \
        }
 #else
 #define __DECLARE_TRACE_RCU(name, proto, args, cond, data_proto, data_args)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index d6863bceeb45e992ca236e9b97751dd09593549e..40940b0d03109c44c782fbccfd93b0079a7cea65 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -732,7 +732,7 @@ void rcu_user_enter(void)
  *
  * Exit from an interrupt handler, which might possibly result in entering
  * idle mode, in other words, leaving the mode in which read-side critical
- * sections can occur.
+ * sections can occur.  The caller must have disabled interrupts.
  *
  * This code assumes that the idle loop never does anything that might
  * result in unbalanced calls to irq_enter() and irq_exit().  If your
@@ -745,11 +745,10 @@ void rcu_user_enter(void)
  */
 void rcu_irq_exit(void)
 {
-       unsigned long flags;
        long long oldval;
        struct rcu_dynticks *rdtp;
 
-       local_irq_save(flags);
+       RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_irq_exit() invoked with irqs enabled!!!");
        rdtp = this_cpu_ptr(&rcu_dynticks);
        oldval = rdtp->dynticks_nesting;
        rdtp->dynticks_nesting--;
@@ -760,6 +759,17 @@ void rcu_irq_exit(void)
        else
                rcu_eqs_enter_common(oldval, true);
        rcu_sysidle_enter(1);
+}
+
+/*
+ * Wrapper for rcu_irq_exit() where interrupts are enabled.
+ */
+void rcu_irq_exit_irqson(void)
+{
+       unsigned long flags;
+
+       local_irq_save(flags);
+       rcu_irq_exit();
        local_irq_restore(flags);
 }
 
@@ -857,7 +867,7 @@ void rcu_user_exit(void)
  *
  * Enter an interrupt handler, which might possibly result in exiting
  * idle mode, in other words, entering the mode in which read-side critical
- * sections can occur.
+ * sections can occur.  The caller must have disabled interrupts.
  *
  * Note that the Linux kernel is fully capable of entering an interrupt
  * handler that it never exits, for example when doing upcalls to
@@ -873,11 +883,10 @@ void rcu_user_exit(void)
  */
 void rcu_irq_enter(void)
 {
-       unsigned long flags;
        struct rcu_dynticks *rdtp;
        long long oldval;
 
-       local_irq_save(flags);
+       RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_irq_enter() invoked with irqs enabled!!!");
        rdtp = this_cpu_ptr(&rcu_dynticks);
        oldval = rdtp->dynticks_nesting;
        rdtp->dynticks_nesting++;
@@ -888,6 +897,17 @@ void rcu_irq_enter(void)
        else
                rcu_eqs_exit_common(oldval, true);
        rcu_sysidle_exit(1);
+}
+
+/*
+ * Wrapper for rcu_irq_enter() where interrupts are enabled.
+ */
+void rcu_irq_enter_irqson(void)
+{
+       unsigned long flags;
+
+       local_irq_save(flags);
+       rcu_irq_enter();
        local_irq_restore(flags);
 }