cpusets: update tasks' page/slab spread flags in time

Fix the bug that the kernel didn't spread page cache/slab object evenly
over all the allowed nodes when spread flags were set by updating tasks'
page/slab spread flags in time.

Signed-off-by: Miao Xie <miaox@cn.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Paul Menage <menage@google.com>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/kernel/cpuset.c b/kernel/cpuset.c
index 66b24d9b663805e674e27b05ea936e979c6cc198..af5a83d52187b59b562d211e8f9c00d49f3ad731 100644 (file)
--- a/kernel/cpuset.c
+++ b/kernel/cpuset.c
@@ -406,7 +406,6 @@ void cpuset_update_task_memory_state(void)
                cs = task_cs(tsk); /* Maybe changed when task not locked */
                guarantee_online_mems(cs, &tsk->mems_allowed);
                tsk->cpuset_mems_generation = cs->mems_generation;
-               cpuset_update_task_spread_flag(cs, tsk);
                task_unlock(tsk);
                mutex_unlock(&callback_mutex);
                mpol_rebind_task(tsk, &tsk->mems_allowed);
@@ -1203,6 +1202,46 @@ static int update_relax_domain_level(struct cpuset *cs, s64 val)
        return 0;
 }
 
+/*
+ * cpuset_change_flag - make a task's spread flags the same as its cpuset's
+ * @tsk: task to be updated
+ * @scan: struct cgroup_scanner containing the cgroup of the task
+ *
+ * Called by cgroup_scan_tasks() for each task in a cgroup.
+ *
+ * We don't need to re-check for the cgroup/cpuset membership, since we're
+ * holding cgroup_lock() at this point.
+ */
+static void cpuset_change_flag(struct task_struct *tsk,
+                               struct cgroup_scanner *scan)
+{
+       cpuset_update_task_spread_flag(cgroup_cs(scan->cg), tsk);
+}
+
+/*
+ * update_tasks_flags - update the spread flags of tasks in the cpuset.
+ * @cs: the cpuset in which each task's spread flags needs to be changed
+ * @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()
+ *
+ * Called with cgroup_mutex held
+ *
+ * The cgroup_scan_tasks() function will scan all the tasks in a cgroup,
+ * calling callback functions for each.
+ *
+ * No return value. It's guaranteed that cgroup_scan_tasks() always returns 0
+ * if @heap != NULL.
+ */
+static void update_tasks_flags(struct cpuset *cs, struct ptr_heap *heap)
+{
+       struct cgroup_scanner scan;
+
+       scan.cg = cs->css.cgroup;
+       scan.test_task = NULL;
+       scan.process_task = cpuset_change_flag;
+       scan.heap = heap;
+       cgroup_scan_tasks(&scan);
+}
+
 /*
  * update_flag - read a 0 or a 1 in a file and update associated flag
  * bit:                the bit to update (see cpuset_flagbits_t)
@@ -1216,8 +1255,10 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
                       int turning_on)
 {
        struct cpuset *trialcs;
-       int err;
        int balance_flag_changed;
+       int spread_flag_changed;
+       struct ptr_heap heap;
+       int err;
 
        trialcs = alloc_trial_cpuset(cs);
        if (!trialcs)
@@ -1232,9 +1273,16 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
        if (err < 0)
                goto out;
 
+       err = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
+       if (err < 0)
+               goto out;
+
        balance_flag_changed = (is_sched_load_balance(cs) !=
                                is_sched_load_balance(trialcs));
 
+       spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs))
+                       || (is_spread_page(cs) != is_spread_page(trialcs)));
+
        mutex_lock(&callback_mutex);
        cs->flags = trialcs->flags;
        mutex_unlock(&callback_mutex);
@@ -1242,6 +1290,9 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
        if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed)
                async_rebuild_sched_domains();
 
+       if (spread_flag_changed)
+               update_tasks_flags(cs, &heap);
+       heap_free(&heap);
 out:
        free_trial_cpuset(trialcs);
        return err;
@@ -1392,6 +1443,8 @@ static void cpuset_attach(struct cgroup_subsys *ss,
        if (err)
                return;
 
+       cpuset_update_task_spread_flag(cs, tsk);
+
        from = oldcs->mems_allowed;
        to = cs->mems_allowed;
        mm = get_task_mm(tsk);
@@ -1453,11 +1506,9 @@ static int cpuset_write_u64(struct cgroup *cgrp, struct cftype *cft, u64 val)
                break;
        case FILE_SPREAD_PAGE:
                retval = update_flag(CS_SPREAD_PAGE, cs, val);
-               cs->mems_generation = cpuset_mems_generation++;
                break;
        case FILE_SPREAD_SLAB:
                retval = update_flag(CS_SPREAD_SLAB, cs, val);
-               cs->mems_generation = cpuset_mems_generation++;
                break;
        default:
                retval = -EINVAL;