libnvdimm: cycle flush hints

When the NFIT provides multiple flush hint addresses per-dimm it is
expressing that the platform is capable of processing multiple flush
requests in parallel.  There is some fixed cost per flush request, let
the cost be shared in parallel on multiple cpus.

Since there may not be enough flush hint addresses for each cpu to have
one, keep a per-cpu index of the last used hint, hash it with current
pid, and assume that access pattern and scheduler randomness will keep
the flush-hint usage somewhat staggered across cpus.

Cc: Ross Zwisler <ross.zwisler@linux.intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>

diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h
index 5912bd6b423459aa7f67964e91fdf09a5e525f69..40476399d22793aece0438da0f5a0976cef063ab 100644 (file)
--- a/drivers/nvdimm/nd.h
+++ b/drivers/nvdimm/nd.h
@@ -52,6 +52,7 @@ struct nvdimm_drvdata {
 struct nd_region_data {
        int ns_count;
        int ns_active;
+       unsigned int flush_mask;
        void __iomem *flush_wpq[0][0];
 };
 

diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c
index 5d97b127b715a28d21aefa56755f9b53100c3244..e8d5ba7b29af98f647b119640e79e581996cfdc0 100644 (file)
--- a/drivers/nvdimm/region_devs.c
+++ b/drivers/nvdimm/region_devs.c
@@ -14,6 +14,7 @@
 #include <linux/highmem.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
+#include <linux/hash.h>
 #include <linux/pmem.h>
 #include <linux/sort.h>
 #include <linux/io.h>
@@ -28,6 +29,7 @@
 #include <linux/io-64-nonatomic-hi-lo.h>
 
 static DEFINE_IDA(region_ida);
+static DEFINE_PER_CPU(int, flush_idx);
 
 static int nvdimm_map_flush(struct device *dev, struct nvdimm *nvdimm, int dimm,
                struct nd_region_data *ndrd)
@@ -67,7 +69,7 @@ static int nvdimm_map_flush(struct device *dev, struct nvdimm *nvdimm, int dimm,
 
 int nd_region_activate(struct nd_region *nd_region)
 {
-       int i;
+       int i, num_flush = 0;
        struct nd_region_data *ndrd;
        struct device *dev = &nd_region->dev;
        size_t flush_data_size = sizeof(void *);
@@ -79,6 +81,7 @@ int nd_region_activate(struct nd_region *nd_region)
 
                /* at least one null hint slot per-dimm for the "no-hint" case */
                flush_data_size += sizeof(void *);
+               num_flush = min_not_zero(num_flush, nvdimm->num_flush);
                if (!nvdimm->num_flush)
                        continue;
                flush_data_size += nvdimm->num_flush * sizeof(void *);
@@ -90,6 +93,7 @@ int nd_region_activate(struct nd_region *nd_region)
                return -ENOMEM;
        dev_set_drvdata(dev, ndrd);
 
+       ndrd->flush_mask = (1 << ilog2(num_flush)) - 1;
        for (i = 0; i < nd_region->ndr_mappings; i++) {
                struct nd_mapping *nd_mapping = &nd_region->mapping[i];
                struct nvdimm *nvdimm = nd_mapping->nvdimm;
@@ -878,7 +882,14 @@ EXPORT_SYMBOL_GPL(nvdimm_volatile_region_create);
 void nvdimm_flush(struct nd_region *nd_region)
 {
        struct nd_region_data *ndrd = dev_get_drvdata(&nd_region->dev);
-       int i;
+       int i, idx;
+
+       /*
+        * Try to encourage some diversity in flush hint addresses
+        * across cpus assuming a limited number of flush hints.
+        */
+       idx = this_cpu_read(flush_idx);
+       idx = this_cpu_add_return(flush_idx, hash_32(current->pid + idx, 8));
 
        /*
         * The first wmb() is needed to 'sfence' all previous writes
@@ -890,7 +901,7 @@ void nvdimm_flush(struct nd_region *nd_region)
        wmb();
        for (i = 0; i < nd_region->ndr_mappings; i++)
                if (ndrd->flush_wpq[i][0])
-                       writeq(1, ndrd->flush_wpq[i][0]);
+                       writeq(1, ndrd->flush_wpq[i][idx & ndrd->flush_mask]);
        wmb();
 }
 EXPORT_SYMBOL_GPL(nvdimm_flush);