mm: numa: Limit NUMA scanning to migrate-on-fault VMAs

There is a 90% regression observed with a large Oracle performance test
on a 4 node system. Profiles indicated that the overhead was due to
contention on sp_lock when looking up shared memory policies. These
policies do not have the appropriate flags to allow them to be
automatically balanced so trapping faults on them is pointless. This
patch skips VMAs that do not have MPOL_F_MOF set.

[riel@redhat.com: Initial patch]

Signed-off-by: Mel Gorman <mgorman@suse.de>
Reported-and-tested-by: Joe Mario <jmario@redhat.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1381141781-10992-32-git-send-email-mgorman@suse.de
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h
index da6716b9e3fea8148f45f57ac556367b547f839f..ea4d2495c6464bb2a5789f2566358461cca31330 100644 (file)
--- a/include/linux/mempolicy.h
+++ b/include/linux/mempolicy.h
@@ -136,6 +136,7 @@ struct mempolicy *mpol_shared_policy_lookup(struct shared_policy *sp,
 
 struct mempolicy *get_vma_policy(struct task_struct *tsk,
                struct vm_area_struct *vma, unsigned long addr);
+bool vma_policy_mof(struct task_struct *task, struct vm_area_struct *vma);
 
 extern void numa_default_policy(void);
 extern void numa_policy_init(void);

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 61ec0d4765b95bbdedf43f77254df9d3099fe3f3..d98175d5c2c6ecd9dc4451f598b5615ddab35ebc 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1130,7 +1130,7 @@ void task_numa_work(struct callback_head *work)
                vma = mm->mmap;
        }
        for (; vma; vma = vma->vm_next) {
-               if (!vma_migratable(vma))
+               if (!vma_migratable(vma) || !vma_policy_mof(p, vma))
                        continue;
 
                do {

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 196d8da2b657f84a662d3caa6dbd85f670e09d60..0e895a2eed5fa47b6a6c7c118c29e3232908e6b9 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -1679,6 +1679,30 @@ struct mempolicy *get_vma_policy(struct task_struct *task,
        return pol;
 }
 
+bool vma_policy_mof(struct task_struct *task, struct vm_area_struct *vma)
+{
+       struct mempolicy *pol = get_task_policy(task);
+       if (vma) {
+               if (vma->vm_ops && vma->vm_ops->get_policy) {
+                       bool ret = false;
+
+                       pol = vma->vm_ops->get_policy(vma, vma->vm_start);
+                       if (pol && (pol->flags & MPOL_F_MOF))
+                               ret = true;
+                       mpol_cond_put(pol);
+
+                       return ret;
+               } else if (vma->vm_policy) {
+                       pol = vma->vm_policy;
+               }
+       }
+
+       if (!pol)
+               return default_policy.flags & MPOL_F_MOF;
+
+       return pol->flags & MPOL_F_MOF;
+}
+
 static int apply_policy_zone(struct mempolicy *policy, enum zone_type zone)
 {
        enum zone_type dynamic_policy_zone = policy_zone;