rcu: Make non-preemptive schedule be Tasks RCU quiescent state

Currently, a call to schedule() acts as a Tasks RCU quiescent state
only if a context switch actually takes place.  However, just the
call to schedule() guarantees that the calling task has moved off of
whatever tracing trampoline that it might have been one previously.
This commit therefore plumbs schedule()'s "preempt" parameter into
rcu_note_context_switch(), which then records the Tasks RCU quiescent
state, but only if this call to schedule() was -not- due to a preemption.

To avoid adding overhead to the common-case context-switch path,
this commit hides the rcu_note_context_switch() check under an existing
non-common-case check.

Suggested-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index e6146d0074f8f724f2812e57dc654f5167dbf4a2..f531b29207da0636c366cd7305c2516a9928b8d6 100644 (file)
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -363,15 +363,20 @@ static inline void rcu_init_nohz(void)
 #ifdef CONFIG_TASKS_RCU
 #define TASKS_RCU(x) x
 extern struct srcu_struct tasks_rcu_exit_srcu;
-#define rcu_note_voluntary_context_switch(t) \
+#define rcu_note_voluntary_context_switch_lite(t) \
        do { \
-               rcu_all_qs(); \
                if (READ_ONCE((t)->rcu_tasks_holdout)) \
                        WRITE_ONCE((t)->rcu_tasks_holdout, false); \
        } while (0)
+#define rcu_note_voluntary_context_switch(t) \
+       do { \
+               rcu_all_qs(); \
+               rcu_note_voluntary_context_switch_lite(t); \
+       } while (0)
 #else /* #ifdef CONFIG_TASKS_RCU */
 #define TASKS_RCU(x) do { } while (0)
-#define rcu_note_voluntary_context_switch(t)   rcu_all_qs()
+#define rcu_note_voluntary_context_switch_lite(t)      do { } while (0)
+#define rcu_note_voluntary_context_switch(t)           rcu_all_qs()
 #endif /* #else #ifdef CONFIG_TASKS_RCU */
 
 /**

diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h
index 5219be250f00b53d4c8e3ee17616729fe30e6e5e..74d9c3a1feeec494b693501c6ec976c9dd487186 100644 (file)
--- a/include/linux/rcutiny.h
+++ b/include/linux/rcutiny.h
@@ -92,10 +92,11 @@ static inline void kfree_call_rcu(struct rcu_head *head,
        call_rcu(head, func);
 }
 
-static inline void rcu_note_context_switch(void)
-{
-       rcu_sched_qs();
-}
+#define rcu_note_context_switch(preempt) \
+       do { \
+               rcu_sched_qs(); \
+               rcu_note_voluntary_context_switch_lite(current); \
+       } while (0)
 
 /*
  * Take advantage of the fact that there is only one CPU, which
@@ -242,6 +243,10 @@ static inline bool rcu_is_watching(void)
 
 #endif /* #else defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) */
 
+static inline void rcu_request_urgent_qs_task(struct task_struct *t)
+{
+}
+
 static inline void rcu_all_qs(void)
 {
        barrier(); /* Avoid RCU read-side critical sections leaking across. */

diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
index 63a4e4cf40a54315705cec804c8f065bdcf2cdbb..0bacb6b2af6973681b3724a9e4d4311daf730c30 100644 (file)
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -30,7 +30,7 @@
 #ifndef __LINUX_RCUTREE_H
 #define __LINUX_RCUTREE_H
 
-void rcu_note_context_switch(void);
+void rcu_note_context_switch(bool preempt);
 int rcu_needs_cpu(u64 basem, u64 *nextevt);
 void rcu_cpu_stall_reset(void);
 
@@ -41,7 +41,7 @@ void rcu_cpu_stall_reset(void);
  */
 static inline void rcu_virt_note_context_switch(int cpu)
 {
-       rcu_note_context_switch();
+       rcu_note_context_switch(false);
 }
 
 void synchronize_rcu_bh(void);
@@ -108,6 +108,7 @@ void rcu_scheduler_starting(void);
 extern int rcu_scheduler_active __read_mostly;
 
 bool rcu_is_watching(void);
+void rcu_request_urgent_qs_task(struct task_struct *t);
 
 void rcu_all_qs(void);
 

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 3c23435d2083ea26394751b273e144c5717f2fd4..891d97109e09e79f22081d11fd8b1db29fb72137 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -458,7 +458,7 @@ static void rcu_momentary_dyntick_idle(void)
  * and requires special handling for preemptible RCU.
  * The caller must have disabled interrupts.
  */
-void rcu_note_context_switch(void)
+void rcu_note_context_switch(bool preempt)
 {
        barrier(); /* Avoid RCU read-side critical sections leaking down. */
        trace_rcu_utilization(TPS("Start context switch"));
@@ -471,6 +471,8 @@ void rcu_note_context_switch(void)
        if (unlikely(raw_cpu_read(rcu_dynticks.rcu_need_heavy_qs)))
                rcu_momentary_dyntick_idle();
        this_cpu_inc(rcu_dynticks.rcu_qs_ctr);
+       if (!preempt)
+               rcu_note_voluntary_context_switch_lite(current);
 out:
        trace_rcu_utilization(TPS("End context switch"));
        barrier(); /* Avoid RCU read-side critical sections leaking up. */
@@ -1149,6 +1151,24 @@ bool notrace rcu_is_watching(void)
 }
 EXPORT_SYMBOL_GPL(rcu_is_watching);
 
+/*
+ * If a holdout task is actually running, request an urgent quiescent
+ * state from its CPU.  This is unsynchronized, so migrations can cause
+ * the request to go to the wrong CPU.  Which is OK, all that will happen
+ * is that the CPU's next context switch will be a bit slower and next
+ * time around this task will generate another request.
+ */
+void rcu_request_urgent_qs_task(struct task_struct *t)
+{
+       int cpu;
+
+       barrier();
+       cpu = task_cpu(t);
+       if (!task_curr(t))
+               return; /* This task is not running on that CPU. */
+       smp_store_release(per_cpu_ptr(&rcu_dynticks.rcu_urgent_qs, cpu), true);
+}
+
 #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)
 
 /*

diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
index c5df0d7569009c3e2175a19a546742896390a45e..273e869ca21d546b5457987bd676f70c08bba5f3 100644 (file)
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c
@@ -665,6 +665,7 @@ static void check_holdout_task(struct task_struct *t,
                put_task_struct(t);
                return;
        }
+       rcu_request_urgent_qs_task(t);
        if (!needreport)
                return;
        if (*firstreport) {

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 3b31fc05a0f1e45be5985b860a5fde95ee969832..2adf7b6c04e7495b0e0d2459a83cafe9641e5afc 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -3378,7 +3378,7 @@ static void __sched notrace __schedule(bool preempt)
                hrtick_clear(rq);
 
        local_irq_disable();
-       rcu_note_context_switch();
+       rcu_note_context_switch(preempt);
 
        /*
         * Make sure that signal_pending_state()->signal_pending() below