rcu: Add a module parameter to force use of expedited RCU primitives

There have been some embedded applications that would benefit from
use of expedited grace-period primitives.  In some ways, this is
similar to synchronize_net() doing either a normal or an expedited
grace period depending on lock state, but with control outside of
the kernel.

This commit therefore adds rcu_expedited boot and sysfs parameters
that cause the kernel to substitute expedited primitives for the
normal grace-period primitives.

[ paulmck: Add trace/event/rcu.h to kernel/srcu.c to avoid build error.
	   Get rid of infinite loop through contention path.]

Signed-off-by: Antti P Miettinen <amiettinen@nvidia.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c
index 4e316e1acf584dda201b2b7ee9369a05590229a3..8715a798aa7ce1bb038e0cb2274238fd73356995 100644 (file)
--- a/kernel/ksysfs.c
+++ b/kernel/ksysfs.c
@@ -141,6 +141,23 @@ static ssize_t fscaps_show(struct kobject *kobj,
 }
 KERNEL_ATTR_RO(fscaps);
 
+int rcu_expedited;
+static ssize_t rcu_expedited_show(struct kobject *kobj,
+                                 struct kobj_attribute *attr, char *buf)
+{
+       return sprintf(buf, "%d\n", rcu_expedited);
+}
+static ssize_t rcu_expedited_store(struct kobject *kobj,
+                                  struct kobj_attribute *attr,
+                                  const char *buf, size_t count)
+{
+       if (kstrtoint(buf, 0, &rcu_expedited))
+               return -EINVAL;
+
+       return count;
+}
+KERNEL_ATTR_RW(rcu_expedited);
+
 /*
  * Make /sys/kernel/notes give the raw contents of our kernel .notes section.
  */
@@ -182,6 +199,7 @@ static struct attribute * kernel_attrs[] = {
        &kexec_crash_size_attr.attr,
        &vmcoreinfo_attr.attr,
 #endif
+       &rcu_expedited_attr.attr,
        NULL
 };
 

diff --git a/kernel/rcu.h b/kernel/rcu.h
index 8ba99cdc651556305b5faf82d783e6475d14ec67..20dfba576c2b7efde60815321b5dce1c7e049e37 100644 (file)
--- a/kernel/rcu.h
+++ b/kernel/rcu.h
@@ -109,4 +109,6 @@ static inline bool __rcu_reclaim(char *rn, struct rcu_head *head)
        }
 }
 
+extern int rcu_expedited;
+
 #endif /* __LINUX_RCU_H */

diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index 29ca1c6da59496133947e5a1c3d1a860ef261e3e..a2cf76177b443d33bb30b19052524db385cb4fc9 100644 (file)
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -46,12 +46,15 @@
 #include <linux/export.h>
 #include <linux/hardirq.h>
 #include <linux/delay.h>
+#include <linux/module.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/rcu.h>
 
 #include "rcu.h"
 
+module_param(rcu_expedited, int, 0);
+
 #ifdef CONFIG_PREEMPT_RCU
 
 /*

diff --git a/kernel/rcutiny_plugin.h b/kernel/rcutiny_plugin.h
index 3d0190282204308005f1867abe6d53125787e1b0..f85016a2309b94c08121f0e3346c1c24d81d4c18 100644 (file)
--- a/kernel/rcutiny_plugin.h
+++ b/kernel/rcutiny_plugin.h
@@ -706,7 +706,10 @@ void synchronize_rcu(void)
                return;
 
        /* Once we get past the fastpath checks, same code as rcu_barrier(). */
-       rcu_barrier();
+       if (rcu_expedited)
+               synchronize_rcu_expedited();
+       else
+               rcu_barrier();
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu);
 

diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 74df86bd9204aef5ec9d14ca8b4b0094777bef87..f9c17c3995387557533ddaa465177429fe633f11 100644 (file)
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -2224,7 +2224,10 @@ void synchronize_sched(void)
                           "Illegal synchronize_sched() in RCU-sched read-side critical section");
        if (rcu_blocking_is_gp())
                return;
-       wait_rcu_gp(call_rcu_sched);
+       if (rcu_expedited)
+               synchronize_sched_expedited();
+       else
+               wait_rcu_gp(call_rcu_sched);
 }
 EXPORT_SYMBOL_GPL(synchronize_sched);
 
@@ -2245,7 +2248,10 @@ void synchronize_rcu_bh(void)
                           "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section");
        if (rcu_blocking_is_gp())
                return;
-       wait_rcu_gp(call_rcu_bh);
+       if (rcu_expedited)
+               synchronize_rcu_bh_expedited();
+       else
+               wait_rcu_gp(call_rcu_bh);
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
 
@@ -2328,7 +2334,7 @@ void synchronize_sched_expedited(void)
                if (trycount++ < 10) {
                        udelay(trycount * num_online_cpus());
                } else {
-                       synchronize_sched();
+                       wait_rcu_gp(call_rcu_sched);
                        return;
                }
 

diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index f921154881870b0d4d489b84fdb1f40ccde7495b..c177ba0cce9a3fc753d35de15a7bc487c385c61e 100644 (file)
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -679,7 +679,10 @@ void synchronize_rcu(void)
                           "Illegal synchronize_rcu() in RCU read-side critical section");
        if (!rcu_scheduler_active)
                return;
-       wait_rcu_gp(call_rcu);
+       if (rcu_expedited)
+               synchronize_rcu_expedited();
+       else
+               wait_rcu_gp(call_rcu);
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu);
 
@@ -831,7 +834,7 @@ void synchronize_rcu_expedited(void)
                        udelay(trycount * num_online_cpus());
                } else {
                        put_online_cpus();
-                       synchronize_rcu();
+                       wait_rcu_gp(call_rcu);
                        return;
                }
        }

diff --git a/kernel/srcu.c b/kernel/srcu.c
index 97c465ebd8444cebc0f3a6e44ae0fdb04b552a2e..de9074047c92beffc237be9ecf93ca52b814a31d 100644 (file)
--- a/kernel/srcu.c
+++ b/kernel/srcu.c
@@ -34,6 +34,10 @@
 #include <linux/delay.h>
 #include <linux/srcu.h>
 
+#include <trace/events/rcu.h>
+
+#include "rcu.h"
+
 /*
  * Initialize an rcu_batch structure to empty.
  */
@@ -464,7 +468,9 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount)
  */
 void synchronize_srcu(struct srcu_struct *sp)
 {
-       __synchronize_srcu(sp, SYNCHRONIZE_SRCU_TRYCOUNT);
+       __synchronize_srcu(sp, rcu_expedited
+                          ? SYNCHRONIZE_SRCU_EXP_TRYCOUNT
+                          : SYNCHRONIZE_SRCU_TRYCOUNT);
 }
 EXPORT_SYMBOL_GPL(synchronize_srcu);