rcu: Make ->cpu_no_qs be a union for aggregate OR

This commit converts the rcu_data structure's ->cpu_no_qs field
to a union.  The bytewise side of this union allows individual access
to indications as to whether this CPU needs to find a quiescent state
for a normal (.norm) and/or expedited (.exp) grace period.  The setwise
side of the union allows testing whether or not a quiescent state is
needed at all, for either type of grace period.

For now, only .norm is used.  A later commit will introduce the expedited
usage.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 31e7021ced4d22bd5cee28b28f0f3384ed11d271..3e2875b38eaeaca6f60a545fd81307524847d674 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -245,21 +245,21 @@ static int rcu_gp_in_progress(struct rcu_state *rsp)
  */
 void rcu_sched_qs(void)
 {
-       if (__this_cpu_read(rcu_sched_data.cpu_no_qs)) {
+       if (__this_cpu_read(rcu_sched_data.cpu_no_qs.s)) {
                trace_rcu_grace_period(TPS("rcu_sched"),
                                       __this_cpu_read(rcu_sched_data.gpnum),
                                       TPS("cpuqs"));
-               __this_cpu_write(rcu_sched_data.cpu_no_qs, false);
+               __this_cpu_write(rcu_sched_data.cpu_no_qs.b.norm, false);
        }
 }
 
 void rcu_bh_qs(void)
 {
-       if (__this_cpu_read(rcu_bh_data.cpu_no_qs)) {
+       if (__this_cpu_read(rcu_bh_data.cpu_no_qs.s)) {
                trace_rcu_grace_period(TPS("rcu_bh"),
                                       __this_cpu_read(rcu_bh_data.gpnum),
                                       TPS("cpuqs"));
-               __this_cpu_write(rcu_bh_data.cpu_no_qs, false);
+               __this_cpu_write(rcu_bh_data.cpu_no_qs.b.norm, false);
        }
 }
 
@@ -1744,7 +1744,7 @@ static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp,
                 */
                rdp->gpnum = rnp->gpnum;
                trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart"));
-               rdp->cpu_no_qs = true;
+               rdp->cpu_no_qs.b.norm = true;
                rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
                rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask);
                zero_cpu_stall_ticks(rdp);
@@ -2337,7 +2337,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
        rnp = rdp->mynode;
        raw_spin_lock_irqsave(&rnp->lock, flags);
        smp_mb__after_unlock_lock();
-       if ((rdp->cpu_no_qs &&
+       if ((rdp->cpu_no_qs.b.norm &&
             rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) ||
            rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum ||
            rdp->gpwrap) {
@@ -2348,7 +2348,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
                 * We will instead need a new quiescent state that lies
                 * within the current grace period.
                 */
-               rdp->cpu_no_qs = true;  /* need qs for new gp. */
+               rdp->cpu_no_qs.b.norm = true;   /* need qs for new gp. */
                rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
                raw_spin_unlock_irqrestore(&rnp->lock, flags);
                return;
@@ -2395,7 +2395,7 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
         * Was there a quiescent state since the beginning of the grace
         * period? If no, then exit and wait for the next call.
         */
-       if (rdp->cpu_no_qs &&
+       if (rdp->cpu_no_qs.b.norm &&
            rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr))
                return;
 
@@ -3828,11 +3828,11 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
 
        /* Is the RCU core waiting for a quiescent state from this CPU? */
        if (rcu_scheduler_fully_active &&
-           rdp->core_needs_qs && rdp->cpu_no_qs &&
+           rdp->core_needs_qs && rdp->cpu_no_qs.b.norm &&
            rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) {
                rdp->n_rp_core_needs_qs++;
        } else if (rdp->core_needs_qs &&
-                  (!rdp->cpu_no_qs ||
+                  (!rdp->cpu_no_qs.b.norm ||
                    rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) {
                rdp->n_rp_report_qs++;
                return 1;
@@ -4155,7 +4155,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
        rdp->beenonline = true;  /* We have now been online. */
        rdp->gpnum = rnp->completed; /* Make CPU later note any new GP. */
        rdp->completed = rnp->completed;
-       rdp->cpu_no_qs = true;
+       rdp->cpu_no_qs.b.norm = true;
        rdp->rcu_qs_ctr_snap = per_cpu(rcu_qs_ctr, cpu);
        rdp->core_needs_qs = false;
        trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));

diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index ded4ceebed765d1e57ee346d724e33e6a5d125a3..3eee48bcf52b891590aa75a12ca038c5d8f478fe 100644 (file)
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -286,6 +286,18 @@ struct rcu_node {
        for ((rnp) = (rsp)->level[rcu_num_lvls - 1]; \
             (rnp) < &(rsp)->node[rcu_num_nodes]; (rnp)++)
 
+/*
+ * Union to allow "aggregate OR" operation on the need for a quiescent
+ * state by the normal and expedited grace periods.
+ */
+union rcu_noqs {
+       struct {
+               u8 norm;
+               u8 exp;
+       } b; /* Bits. */
+       u16 s; /* Set of bits, aggregate OR here. */
+};
+
 /* Index values for nxttail array in struct rcu_data. */
 #define RCU_DONE_TAIL          0       /* Also RCU_WAIT head. */
 #define RCU_WAIT_TAIL          1       /* Also RCU_NEXT_READY head. */
@@ -302,7 +314,7 @@ struct rcu_data {
                                        /*  is aware of having started. */
        unsigned long   rcu_qs_ctr_snap;/* Snapshot of rcu_qs_ctr to check */
                                        /*  for rcu_all_qs() invocations. */
-       bool            cpu_no_qs;      /* No QS yet for this CPU. */
+       union rcu_noqs  cpu_no_qs;      /* No QSes yet for this CPU. */
        bool            core_needs_qs;  /* Core waits for quiesc state. */
        bool            beenonline;     /* CPU online at least once. */
        bool            gpwrap;         /* Possible gpnum/completed wrap. */

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 6977ff0dccb94295843675da7973eb0e364421b8..7880202f1e3829f4415a9af62263ea6c9b53e1b3 100644 (file)
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -265,11 +265,11 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp,
  */
 static void rcu_preempt_qs(void)
 {
-       if (__this_cpu_read(rcu_data_p->cpu_no_qs)) {
+       if (__this_cpu_read(rcu_data_p->cpu_no_qs.s)) {
                trace_rcu_grace_period(TPS("rcu_preempt"),
                                       __this_cpu_read(rcu_data_p->gpnum),
                                       TPS("cpuqs"));
-               __this_cpu_write(rcu_data_p->cpu_no_qs, false);
+               __this_cpu_write(rcu_data_p->cpu_no_qs.b.norm, false);
                barrier(); /* Coordinate with rcu_preempt_check_callbacks(). */
                current->rcu_read_unlock_special.b.need_qs = false;
        }
@@ -620,7 +620,7 @@ static void rcu_preempt_check_callbacks(void)
        }
        if (t->rcu_read_lock_nesting > 0 &&
            __this_cpu_read(rcu_data_p->core_needs_qs) &&
-           __this_cpu_read(rcu_data_p->cpu_no_qs))
+           __this_cpu_read(rcu_data_p->cpu_no_qs.b.norm))
                t->rcu_read_unlock_special.b.need_qs = true;
 }
 

diff --git a/kernel/rcu/tree_trace.c b/kernel/rcu/tree_trace.c
index d373e57109b89043f6641d18b85bb83e249e538d..999c3672f990ff6c81a5f237c7735bdd056ab6d8 100644 (file)
--- a/kernel/rcu/tree_trace.c
+++ b/kernel/rcu/tree_trace.c
@@ -121,7 +121,7 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
                   rdp->cpu,
                   cpu_is_offline(rdp->cpu) ? '!' : ' ',
                   ulong2long(rdp->completed), ulong2long(rdp->gpnum),
-                  rdp->cpu_no_qs,
+                  rdp->cpu_no_qs.b.norm,
                   rdp->rcu_qs_ctr_snap == per_cpu(rcu_qs_ctr, rdp->cpu),
                   rdp->core_needs_qs);
        seq_printf(m, " dt=%d/%llx/%d df=%lu",