rcu: Make NOCB CPUs migrate CBs directly from outgoing CPU

RCU's CPU-hotplug callback-migration code first moves the outgoing
CPU's callbacks to ->orphan_done and ->orphan_pend, and only then
moves them to the NOCB callback list.  This commit avoids the
extra step (and simplifies the code) by moving the callbacks directly
from the outgoing CPU's callback list to the NOCB callback list.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index aeea697d6f9f32146742aa77b484ff42973b7b4c..4ea28e820f4aad987bfcc82251e8cc9310bfcfef 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -3899,11 +3899,6 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp, unsigned long flags)
 
        lockdep_assert_held(&rsp->orphan_lock);
 
-       /* No-CBs CPUs are handled specially. */
-       if (!IS_ENABLED(CONFIG_HOTPLUG_CPU) ||
-           rcu_nocb_adopt_orphan_cbs(rsp, rdp, flags))
-               return;
-
        /* Do the accounting first. */
        if (rsp->orphan_done.len_lazy != rsp->orphan_done.len)
                rcu_idle_count_callbacks_posted();
@@ -3928,13 +3923,20 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp, unsigned long flags)
 static void rcu_migrate_callbacks(int cpu, struct rcu_state *rsp)
 {
        unsigned long flags;
+       struct rcu_data *my_rdp;
        struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
        struct rcu_node *rnp = rdp->mynode;  /* Outgoing CPU's rdp & rnp. */
 
        if (rcu_is_nocb_cpu(cpu) || rcu_segcblist_empty(&rdp->cblist))
                return;  /* No callbacks to migrate. */
 
-       raw_spin_lock_irqsave(&rsp->orphan_lock, flags);
+       local_irq_save(flags);
+       my_rdp = this_cpu_ptr(rsp->rda);
+       if (rcu_nocb_adopt_orphan_cbs(my_rdp, rdp, flags)) {
+               local_irq_restore(flags);
+               return;
+       }
+       raw_spin_lock(&rsp->orphan_lock); /* irqs already disabled. */
        rcu_send_cbs_to_orphanage(cpu, rsp, rnp, rdp);
        rcu_adopt_orphan_cbs(rsp, flags);
        raw_spin_unlock_irqrestore(&rsp->orphan_lock, flags);

diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index aec53c4d4aeccfd7b216a1875dde1a21a976ecdd..574513cf49b4980901d9956beb64fd03f10168d7 100644 (file)
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -493,7 +493,7 @@ static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq);
 static void rcu_init_one_nocb(struct rcu_node *rnp);
 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
                            bool lazy, unsigned long flags);
-static bool rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
+static bool rcu_nocb_adopt_orphan_cbs(struct rcu_data *my_rdp,
                                      struct rcu_data *rdp,
                                      unsigned long flags);
 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp);

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 908b309d60d7bc29e82f3a049d928fbd8140dfd2..ff7d5ee4981652785efbff800a58fab0b9ef7650 100644 (file)
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1961,30 +1961,19 @@ static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
  * Adopt orphaned callbacks on a no-CBs CPU, or return 0 if this is
  * not a no-CBs CPU.
  */
-static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
+static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_data *my_rdp,
                                                     struct rcu_data *rdp,
                                                     unsigned long flags)
 {
-       long ql = rsp->orphan_done.len;
-       long qll = rsp->orphan_done.len_lazy;
-
-       /* If this is not a no-CBs CPU, tell the caller to do it the old way. */
+       RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_nocb_adopt_orphan_cbs() invoked with irqs enabled!!!");
        if (!rcu_is_nocb_cpu(smp_processor_id()))
-               return false;
-
-       /* First, enqueue the donelist, if any.  This preserves CB ordering. */
-       if (rsp->orphan_done.head) {
-               __call_rcu_nocb_enqueue(rdp, rcu_cblist_head(&rsp->orphan_done),
-                                       rcu_cblist_tail(&rsp->orphan_done),
-                                       ql, qll, flags);
-       }
-       if (rsp->orphan_pend.head) {
-               __call_rcu_nocb_enqueue(rdp, rcu_cblist_head(&rsp->orphan_pend),
-                                       rcu_cblist_tail(&rsp->orphan_pend),
-                                       ql, qll, flags);
-       }
-       rcu_cblist_init(&rsp->orphan_done);
-       rcu_cblist_init(&rsp->orphan_pend);
+               return false; /* Not NOCBs CPU, caller must migrate CBs. */
+       __call_rcu_nocb_enqueue(my_rdp, rcu_segcblist_head(&rdp->cblist),
+                               rcu_segcblist_tail(&rdp->cblist),
+                               rcu_segcblist_n_cbs(&rdp->cblist),
+                               rcu_segcblist_n_lazy_cbs(&rdp->cblist), flags);
+       rcu_segcblist_init(&rdp->cblist);
+       rcu_segcblist_disable(&rdp->cblist);
        return true;
 }
 
@@ -2459,7 +2448,7 @@ static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
        return false;
 }
 
-static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
+static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_data *my_rdp,
                                                     struct rcu_data *rdp,
                                                     unsigned long flags)
 {