rcu: Awaken grace-period kthread if too long since FQS

Recent kernels can fail to awaken the grace-period kthread for
quiescent-state forcing.  This commit is a crude hack that does
a wakeup if a scheduling-clock interrupt sees that it has been
too long since force-quiescent-state (FQS) processing.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 6116cfad18fff6fa22cbeb36ade5472b9470ecea..a739292be6058aa2f256517a457a5ca8f625eb48 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -385,9 +385,11 @@ module_param(qlowmark, long, 0444);
 
 static ulong jiffies_till_first_fqs = ULONG_MAX;
 static ulong jiffies_till_next_fqs = ULONG_MAX;
+static bool rcu_kick_kthreads;
 
 module_param(jiffies_till_first_fqs, ulong, 0644);
 module_param(jiffies_till_next_fqs, ulong, 0644);
+module_param(rcu_kick_kthreads, bool, 0644);
 
 /*
  * How long the grace period must be before we start recruiting
@@ -1251,6 +1253,24 @@ static void rcu_dump_cpu_stacks(struct rcu_state *rsp)
        }
 }
 
+/*
+ * If too much time has passed in the current grace period, and if
+ * so configured, go kick the relevant kthreads.
+ */
+static void rcu_stall_kick_kthreads(struct rcu_state *rsp)
+{
+       unsigned long j;
+
+       if (!rcu_kick_kthreads)
+               return;
+       j = READ_ONCE(rsp->jiffies_kick_kthreads);
+       if (time_after(jiffies, j) && rsp->gp_kthread) {
+               WARN_ONCE(1, "Kicking %s grace-period kthread\n", rsp->name);
+               wake_up_process(rsp->gp_kthread);
+               WRITE_ONCE(rsp->jiffies_kick_kthreads, j + HZ);
+       }
+}
+
 static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum)
 {
        int cpu;
@@ -1262,6 +1282,11 @@ static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum)
        struct rcu_node *rnp = rcu_get_root(rsp);
        long totqlen = 0;
 
+       /* Kick and suppress, if so configured. */
+       rcu_stall_kick_kthreads(rsp);
+       if (rcu_cpu_stall_suppress)
+               return;
+
        /* Only let one CPU complain about others per time interval. */
 
        raw_spin_lock_irqsave_rcu_node(rnp, flags);
@@ -1335,6 +1360,11 @@ static void print_cpu_stall(struct rcu_state *rsp)
        struct rcu_node *rnp = rcu_get_root(rsp);
        long totqlen = 0;
 
+       /* Kick and suppress, if so configured. */
+       rcu_stall_kick_kthreads(rsp);
+       if (rcu_cpu_stall_suppress)
+               return;
+
        /*
         * OK, time to rat on ourselves...
         * See Documentation/RCU/stallwarn.txt for info on how to debug
@@ -1379,8 +1409,10 @@ static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
        unsigned long js;
        struct rcu_node *rnp;
 
-       if (rcu_cpu_stall_suppress || !rcu_gp_in_progress(rsp))
+       if ((rcu_cpu_stall_suppress && !rcu_kick_kthreads) ||
+           !rcu_gp_in_progress(rsp))
                return;
+       rcu_stall_kick_kthreads(rsp);
        j = jiffies;
 
        /*
@@ -2119,8 +2151,11 @@ static int __noreturn rcu_gp_kthread(void *arg)
                }
                ret = 0;
                for (;;) {
-                       if (!ret)
+                       if (!ret) {
                                rsp->jiffies_force_qs = jiffies + j;
+                               WRITE_ONCE(rsp->jiffies_kick_kthreads,
+                                          jiffies + 3 * j);
+                       }
                        trace_rcu_grace_period(rsp->name,
                                               READ_ONCE(rsp->gpnum),
                                               TPS("fqswait"));

diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index df668c0f9e64991346dd94a872ee67e16e97d735..34d3973f72235d9c8b3efb4c9f3ff281239e2ac2 100644 (file)
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -513,6 +513,8 @@ struct rcu_state {
 
        unsigned long jiffies_force_qs;         /* Time at which to invoke */
                                                /*  force_quiescent_state(). */
+       unsigned long jiffies_kick_kthreads;    /* Time at which to kick */
+                                               /*  kthreads, if configured. */
        unsigned long n_force_qs;               /* Number of calls to */
                                                /*  force_quiescent_state(). */
        unsigned long n_force_qs_lh;            /* ~Number of calls leaving */