srcu: Specify auto-expedite holdoff time

On small systems, in the absence of readers, expedited SRCU grace
periods can complete in less than a microsecond.  This means that an
eight-CPU system can have all CPUs doing synchronize_srcu() in a tight
loop and almost always expedite.  This might actually be desirable in
some situations, but in general it is a good way to needlessly burn
CPU cycles.  And in those situations where it is desirable, your friend
is the function synchronize_srcu_expedited().

For other situations, this commit adds a kernel parameter that specifies
a holdoff between completing the last SRCU grace period and auto-expediting
the next.  If the next grace period starts before the holdoff expires,
auto-expediting is disabled.  The holdoff is 50 microseconds by default,
and can be tuned to the desired number of nanoseconds.  A value of zero
disables auto-expediting.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Tested-by: Mike Galbraith <efault@gmx.de>

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index facc20a3f96280472396ad3f7d2e8f2dba62fecc..4a4b9266c4defded63e99430fb9946c0da0aff49 100644 (file)
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -3779,6 +3779,14 @@
        spia_pedr=
        spia_peddr=
 
+       srcutree.exp_holdoff [KNL]
+                       Specifies how many nanoseconds must elapse
+                       since the end of the last SRCU grace period for
+                       a given srcu_struct until the next normal SRCU
+                       grace period will be considered for automatic
+                       expediting.  Set to zero to disable automatic
+                       expediting.
+
        stacktrace      [FTRACE]
                        Enabled the stack tracer on boot up.
 

diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index 86df48d3e97b30c4491408e815154c1ca154056b..32e86d85fd11f54bd973129a6428c4c09c104ace 100644 (file)
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h
@@ -84,6 +84,7 @@ struct srcu_struct {
        unsigned long srcu_gp_seq;              /* Grace-period seq #. */
        unsigned long srcu_gp_seq_needed;       /* Latest gp_seq needed. */
        unsigned long srcu_gp_seq_needed_exp;   /* Furthest future exp GP. */
+       unsigned long srcu_last_gp_end;         /* Last GP end timestamp (ns) */
        struct srcu_data __percpu *sda;         /* Per-CPU srcu_data array. */
        unsigned long srcu_barrier_seq;         /* srcu_barrier seq #. */
        struct mutex srcu_barrier_mutex;        /* Serialize barrier ops. */

diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index 2286e06fd1598cd1f535ed300ed53ebfcb40871d..74c283f9d15e5c8b8395fccd1b0606abf72c283b 100644 (file)
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -34,10 +34,14 @@
 #include <linux/sched.h>
 #include <linux/smp.h>
 #include <linux/delay.h>
+#include <linux/module.h>
 #include <linux/srcu.h>
 
 #include "rcu.h"
 
+ulong exp_holdoff = 50 * 1000; /* Holdoff (ns) for auto-expediting. */
+module_param(exp_holdoff, ulong, 0444);
+
 static void srcu_invoke_callbacks(struct work_struct *work);
 static void srcu_reschedule(struct srcu_struct *sp, unsigned long delay);
 
@@ -145,6 +149,7 @@ static int init_srcu_struct_fields(struct srcu_struct *sp, bool is_static)
                sp->sda = alloc_percpu(struct srcu_data);
        init_srcu_struct_nodes(sp, is_static);
        sp->srcu_gp_seq_needed_exp = 0;
+       sp->srcu_last_gp_end = ktime_get_mono_fast_ns();
        smp_store_release(&sp->srcu_gp_seq_needed, 0); /* Init done. */
        return sp->sda ? 0 : -ENOMEM;
 }
@@ -498,6 +503,7 @@ static void srcu_gp_end(struct srcu_struct *sp)
        idx = rcu_seq_state(sp->srcu_gp_seq);
        WARN_ON_ONCE(idx != SRCU_STATE_SCAN2);
        cbdelay = srcu_get_delay(sp);
+       sp->srcu_last_gp_end = ktime_get_mono_fast_ns();
        rcu_seq_end(&sp->srcu_gp_seq);
        gpseq = rcu_seq_current(&sp->srcu_gp_seq);
        if (ULONG_CMP_LT(sp->srcu_gp_seq_needed_exp, gpseq))
@@ -700,9 +706,10 @@ static void srcu_flip(struct srcu_struct *sp)
  */
 static bool srcu_might_be_idle(struct srcu_struct *sp)
 {
+       unsigned long curseq;
        unsigned long flags;
        struct srcu_data *sdp;
-       unsigned long curseq;
+       unsigned long t;
 
        /* If the local srcu_data structure has callbacks, not idle.  */
        local_irq_save(flags);
@@ -718,6 +725,15 @@ static bool srcu_might_be_idle(struct srcu_struct *sp)
         * Exact information would require acquiring locks, which would
         * kill scalability, hence the probabalistic nature of the probe.
         */
+
+       /* First, see if enough time has passed since the last GP. */
+       t = ktime_get_mono_fast_ns();
+       if (exp_holdoff == 0 ||
+           time_in_range_open(t, sp->srcu_last_gp_end,
+                              sp->srcu_last_gp_end + exp_holdoff))
+               return false; /* Too soon after last GP. */
+
+       /* Next, check for probable idleness. */
        curseq = rcu_seq_current(&sp->srcu_gp_seq);
        smp_mb(); /* Order ->srcu_gp_seq with ->srcu_gp_seq_needed. */
        if (ULONG_CMP_LT(curseq, READ_ONCE(sp->srcu_gp_seq_needed)))