srcu: Apply trivial callback lists to shrink Tiny SRCU

The rcu_segcblist structure provides quite a bit of functionality, and
Tiny SRCU needs almost none of it.  So this commit replaces Tiny SRCU's
uses of rcu_segcblist with a simple singly linked list with tail pointer.
This change significantly reduces Tiny SRCU's memory footprint, more
than making up for the growth caused by the creation of rcu_segcblist.c

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h
index 4c53e698c6e475e8fe983b0c6e90580cf1e1f2c2..cfbfc540cafcb44ed564b03c5b37b127b08db10b 100644 (file)
--- a/include/linux/srcutiny.h
+++ b/include/linux/srcutiny.h
@@ -33,9 +33,8 @@ struct srcu_struct {
        u8 srcu_gp_waiting;             /* GP waiting for readers? */
        struct swait_queue_head srcu_wq;
                                        /* Last srcu_read_unlock() wakes GP. */
-       unsigned long srcu_gp_seq;      /* GP seq # for callback tagging. */
-       struct rcu_segcblist srcu_cblist;
-                                       /* Pending SRCU callbacks. */
+       struct rcu_head *srcu_cb_head;  /* Pending callbacks: Head. */
+       struct rcu_head **srcu_cb_tail; /* Pending callbacks: Tail. */
        struct work_struct srcu_work;   /* For driving grace periods. */
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
        struct lockdep_map dep_map;
@@ -47,7 +46,7 @@ void srcu_drive_gp(struct work_struct *wp);
 #define __SRCU_STRUCT_INIT(name)                                       \
 {                                                                      \
        .srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq),        \
-       .srcu_cblist = RCU_SEGCBLIST_INITIALIZER(name.srcu_cblist),     \
+       .srcu_cb_tail = &name.srcu_cb_head,                             \
        .srcu_work = __WORK_INITIALIZER(name.srcu_work, srcu_drive_gp), \
        __SRCU_DEP_MAP_INIT(name)                                       \
 }

diff --git a/init/Kconfig b/init/Kconfig
index d928a3724af9357e6dcae2341a86f9f87ecca8f7..a2cfde19e8b891cd79d930e6ba0d57076a0d7015 100644 (file)
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -573,7 +573,7 @@ config RCU_STALL_COMMON
          making these warnings mandatory for the tree variants.
 
 config RCU_NEED_SEGCBLIST
-       def_bool ( TREE_RCU || PREEMPT_RCU || TINY_SRCU || TREE_SRCU )
+       def_bool ( TREE_RCU || PREEMPT_RCU || TREE_SRCU )
 
 config CONTEXT_TRACKING
        bool

diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index 74d9fc20531324fb302dae772bf88cdf2acd2864..6a1e85bd2eacc273fc683948e14b121dfb33aaa2 100644 (file)
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -398,7 +398,7 @@ static inline void srcutorture_get_gp_data(enum rcutorture_type test_type,
        if (test_type != SRCU_FLAVOR)
                return;
        *flags = 0;
-       *completed = sp->srcu_gp_seq;
+       *completed = sp->srcu_idx;
        *gpnum = *completed;
 }
 

diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c
index 988543721d5d2e273c699430d28bed23dddc8dbb..1a1c1047d2edecaba7691d8c6f9fd3851c3c434a 100644 (file)
--- a/kernel/rcu/srcutiny.c
+++ b/kernel/rcu/srcutiny.c
@@ -38,8 +38,8 @@ static int init_srcu_struct_fields(struct srcu_struct *sp)
        sp->srcu_lock_nesting[0] = 0;
        sp->srcu_lock_nesting[1] = 0;
        init_swait_queue_head(&sp->srcu_wq);
-       sp->srcu_gp_seq = 0;
-       rcu_segcblist_init(&sp->srcu_cblist);
+       sp->srcu_cb_head = NULL;
+       sp->srcu_cb_tail = &sp->srcu_cb_head;
        sp->srcu_gp_running = false;
        sp->srcu_gp_waiting = false;
        sp->srcu_idx = 0;
@@ -88,10 +88,10 @@ void cleanup_srcu_struct(struct srcu_struct *sp)
 {
        WARN_ON(sp->srcu_lock_nesting[0] || sp->srcu_lock_nesting[1]);
        flush_work(&sp->srcu_work);
-       WARN_ON(rcu_seq_state(sp->srcu_gp_seq));
        WARN_ON(sp->srcu_gp_running);
        WARN_ON(sp->srcu_gp_waiting);
-       WARN_ON(!rcu_segcblist_empty(&sp->srcu_cblist));
+       WARN_ON(sp->srcu_cb_head);
+       WARN_ON(&sp->srcu_cb_head != sp->srcu_cb_tail);
 }
 EXPORT_SYMBOL_GPL(cleanup_srcu_struct);
 
@@ -117,52 +117,44 @@ EXPORT_SYMBOL_GPL(__srcu_read_unlock);
 void srcu_drive_gp(struct work_struct *wp)
 {
        int idx;
-       struct rcu_cblist ready_cbs;
-       struct srcu_struct *sp;
+       struct rcu_head *lh;
        struct rcu_head *rhp;
+       struct srcu_struct *sp;
 
        sp = container_of(wp, struct srcu_struct, srcu_work);
-       if (sp->srcu_gp_running || rcu_segcblist_empty(&sp->srcu_cblist))
+       if (sp->srcu_gp_running || !READ_ONCE(sp->srcu_cb_head))
                return; /* Already running or nothing to do. */
 
-       /* Tag recently arrived callbacks and wait for readers. */
+       /* Remove recently arrived callbacks and wait for readers. */
        WRITE_ONCE(sp->srcu_gp_running, true);
-       rcu_segcblist_accelerate(&sp->srcu_cblist,
-                                rcu_seq_snap(&sp->srcu_gp_seq));
-       rcu_seq_start(&sp->srcu_gp_seq);
+       local_irq_disable();
+       lh = sp->srcu_cb_head;
+       sp->srcu_cb_head = NULL;
+       sp->srcu_cb_tail = &sp->srcu_cb_head;
+       local_irq_enable();
        idx = sp->srcu_idx;
        WRITE_ONCE(sp->srcu_idx, !sp->srcu_idx);
        WRITE_ONCE(sp->srcu_gp_waiting, true);  /* srcu_read_unlock() wakes! */
        swait_event(sp->srcu_wq, !READ_ONCE(sp->srcu_lock_nesting[idx]));
        WRITE_ONCE(sp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */
-       rcu_seq_end(&sp->srcu_gp_seq);
-
-       /* Update callback list based on GP, and invoke ready callbacks. */
-       rcu_segcblist_advance(&sp->srcu_cblist,
-                             rcu_seq_current(&sp->srcu_gp_seq));
-       if (rcu_segcblist_ready_cbs(&sp->srcu_cblist)) {
-               rcu_cblist_init(&ready_cbs);
-               local_irq_disable();
-               rcu_segcblist_extract_done_cbs(&sp->srcu_cblist, &ready_cbs);
-               local_irq_enable();
-               rhp = rcu_cblist_dequeue(&ready_cbs);
-               for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) {
-                       local_bh_disable();
-                       rhp->func(rhp);
-                       local_bh_enable();
-               }
-               local_irq_disable();
-               rcu_segcblist_insert_count(&sp->srcu_cblist, &ready_cbs);
-               local_irq_enable();
+
+       /* Invoke the callbacks we removed above. */
+       while (lh) {
+               rhp = lh;
+               lh = lh->next;
+               local_bh_disable();
+               rhp->func(rhp);
+               local_bh_enable();
        }
-       WRITE_ONCE(sp->srcu_gp_running, false);
 
        /*
-        * If more callbacks, reschedule ourselves.  This can race with
-        * a call_srcu() at interrupt level, but the ->srcu_gp_running
-        * checks will straighten that out.
+        * Enable rescheduling, and if there are more callbacks,
+        * reschedule ourselves.  This can race with a call_srcu()
+        * at interrupt level, but the ->srcu_gp_running checks will
+        * straighten that out.
         */
-       if (!rcu_segcblist_empty(&sp->srcu_cblist))
+       WRITE_ONCE(sp->srcu_gp_running, false);
+       if (READ_ONCE(sp->srcu_cb_head))
                schedule_work(&sp->srcu_work);
 }
 EXPORT_SYMBOL_GPL(srcu_drive_gp);
@@ -171,14 +163,16 @@ EXPORT_SYMBOL_GPL(srcu_drive_gp);
  * Enqueue an SRCU callback on the specified srcu_struct structure,
  * initiating grace-period processing if it is not already running.
  */
-void call_srcu(struct srcu_struct *sp, struct rcu_head *head,
+void call_srcu(struct srcu_struct *sp, struct rcu_head *rhp,
               rcu_callback_t func)
 {
        unsigned long flags;
 
-       head->func = func;
+       rhp->func = func;
+       rhp->next = NULL;
        local_irq_save(flags);
-       rcu_segcblist_enqueue(&sp->srcu_cblist, head, false);
+       *sp->srcu_cb_tail = rhp;
+       sp->srcu_cb_tail = &rhp->next;
        local_irq_restore(flags);
        if (!READ_ONCE(sp->srcu_gp_running))
                schedule_work(&sp->srcu_work);