[PATCH] sched: balance on fork

Reimplement the balance on exec balancing to be sched-domains aware.  Use this
to also do balance on fork balancing.  Make x86_64 do balance on fork over the
NUMA domain.

The problem that the non sched domains aware blancing became apparent on dual
core, multi socket opterons.  What we want is for the new tasks to be sent to
a different socket, but more often than not, we would first load up our
sibling core, or fill two cores of a single remote socket before selecting a
new one.

This gives large improvements to STREAM on such systems.

Signed-off-by: Nick Piggin <nickpiggin@yahoo.com.au>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/include/asm-x86_64/topology.h b/include/asm-x86_64/topology.h
index 9cb7459ce722775a2907b5b47c1bba0906dd8843..802d09b9c99ff9308dfa72ba77c6495448e9c23e 100644 (file)
--- a/include/asm-x86_64/topology.h
+++ b/include/asm-x86_64/topology.h
@@ -44,9 +44,11 @@ extern int __node_distance(int, int);
        .idle_idx               = 2,                    \
        .newidle_idx            = 1,                    \
        .wake_idx               = 1,                    \
+       .forkexec_idx           = 1,                    \
        .per_cpu_gain           = 100,                  \
        .flags                  = SD_LOAD_BALANCE       \
                                | SD_BALANCE_NEWIDLE    \
+                               | SD_BALANCE_FORK       \
                                | SD_BALANCE_EXEC       \
                                | SD_WAKE_BALANCE,      \
        .last_balance           = jiffies,              \

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 664981ac1fb60a3d9fe2aaee58d2546b60bf3f8e..613491d3a875cf6424d01ffb83cee41bc3cc248b 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -460,10 +460,11 @@ enum idle_type
 #define SD_LOAD_BALANCE                1       /* Do load balancing on this domain. */
 #define SD_BALANCE_NEWIDLE     2       /* Balance when about to become idle */
 #define SD_BALANCE_EXEC                4       /* Balance on exec */
-#define SD_WAKE_IDLE           8       /* Wake to idle CPU on task wakeup */
-#define SD_WAKE_AFFINE         16      /* Wake task to waking CPU */
-#define SD_WAKE_BALANCE                32      /* Perform balancing at task wakeup */
-#define SD_SHARE_CPUPOWER      64      /* Domain members share cpu power */
+#define SD_BALANCE_FORK                8       /* Balance on fork, clone */
+#define SD_WAKE_IDLE           16      /* Wake to idle CPU on task wakeup */
+#define SD_WAKE_AFFINE         32      /* Wake task to waking CPU */
+#define SD_WAKE_BALANCE                64      /* Perform balancing at task wakeup */
+#define SD_SHARE_CPUPOWER      128     /* Domain members share cpu power */
 
 struct sched_group {
        struct sched_group *next;       /* Must be a circular list */
@@ -492,6 +493,7 @@ struct sched_domain {
        unsigned int idle_idx;
        unsigned int newidle_idx;
        unsigned int wake_idx;
+       unsigned int forkexec_idx;
        int flags;                      /* See SD_* */
 
        /* Runtime fields. */

diff --git a/include/linux/topology.h b/include/linux/topology.h
index b23ec64df7f19d4f033c0afbce7158c16d35e822..665597207def695392ad3079ee9f8619c6589c5e 100644 (file)
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -93,6 +93,7 @@
        .idle_idx               = 0,                    \
        .newidle_idx            = 0,                    \
        .wake_idx               = 0,                    \
+       .forkexec_idx           = 0,                    \
        .flags                  = SD_LOAD_BALANCE       \
                                | SD_BALANCE_NEWIDLE    \
                                | SD_BALANCE_EXEC       \
@@ -123,6 +124,7 @@
        .idle_idx               = 0,                    \
        .newidle_idx            = 1,                    \
        .wake_idx               = 1,                    \
+       .forkexec_idx           = 0,                    \
        .flags                  = SD_LOAD_BALANCE       \
                                | SD_BALANCE_NEWIDLE    \
                                | SD_BALANCE_EXEC       \

diff --git a/kernel/sched.c b/kernel/sched.c
index 396724a2519f0c5e17aeef1ceeb65c32c5a0b5bf..7ecc237e2aaba8b5efce8ea2b47170d3a268ae39 100644 (file)
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -893,6 +893,79 @@ static inline unsigned long target_load(int cpu, int type)
        return max(rq->cpu_load[type-1], load_now);
 }
 
+/*
+ * find_idlest_group finds and returns the least busy CPU group within the
+ * domain.
+ */
+static struct sched_group *
+find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
+{
+       struct sched_group *idlest = NULL, *this = NULL, *group = sd->groups;
+       unsigned long min_load = ULONG_MAX, this_load = 0;
+       int load_idx = sd->forkexec_idx;
+       int imbalance = 100 + (sd->imbalance_pct-100)/2;
+
+       do {
+               unsigned long load, avg_load;
+               int local_group;
+               int i;
+
+               local_group = cpu_isset(this_cpu, group->cpumask);
+               /* XXX: put a cpus allowed check */
+
+               /* Tally up the load of all CPUs in the group */
+               avg_load = 0;
+
+               for_each_cpu_mask(i, group->cpumask) {
+                       /* Bias balancing toward cpus of our domain */
+                       if (local_group)
+                               load = source_load(i, load_idx);
+                       else
+                               load = target_load(i, load_idx);
+
+                       avg_load += load;
+               }
+
+               /* Adjust by relative CPU power of the group */
+               avg_load = (avg_load * SCHED_LOAD_SCALE) / group->cpu_power;
+
+               if (local_group) {
+                       this_load = avg_load;
+                       this = group;
+               } else if (avg_load < min_load) {
+                       min_load = avg_load;
+                       idlest = group;
+               }
+               group = group->next;
+       } while (group != sd->groups);
+
+       if (!idlest || 100*this_load < imbalance*min_load)
+               return NULL;
+       return idlest;
+}
+
+/*
+ * find_idlest_queue - find the idlest runqueue among the cpus in group.
+ */
+static int find_idlest_cpu(struct sched_group *group, int this_cpu)
+{
+       unsigned long load, min_load = ULONG_MAX;
+       int idlest = -1;
+       int i;
+
+       for_each_cpu_mask(i, group->cpumask) {
+               load = source_load(i, 0);
+
+               if (load < min_load || (load == min_load && i == this_cpu)) {
+                       min_load = load;
+                       idlest = i;
+               }
+       }
+
+       return idlest;
+}
+
+
 #endif
 
 /*
@@ -1107,11 +1180,6 @@ int fastcall wake_up_state(task_t *p, unsigned int state)
        return try_to_wake_up(p, state, 0);
 }
 
-#ifdef CONFIG_SMP
-static int find_idlest_cpu(struct task_struct *p, int this_cpu,
-                          struct sched_domain *sd);
-#endif
-
 /*
  * Perform scheduler related setup for a newly forked process p.
  * p is forked by current.
@@ -1181,12 +1249,38 @@ void fastcall wake_up_new_task(task_t * p, unsigned long clone_flags)
        unsigned long flags;
        int this_cpu, cpu;
        runqueue_t *rq, *this_rq;
+#ifdef CONFIG_SMP
+       struct sched_domain *tmp, *sd = NULL;
+#endif
 
        rq = task_rq_lock(p, &flags);
-       cpu = task_cpu(p);
+       BUG_ON(p->state != TASK_RUNNING);
        this_cpu = smp_processor_id();
+       cpu = task_cpu(p);
 
-       BUG_ON(p->state != TASK_RUNNING);
+#ifdef CONFIG_SMP
+       for_each_domain(cpu, tmp)
+               if (tmp->flags & SD_BALANCE_FORK)
+                       sd = tmp;
+
+       if (sd) {
+               struct sched_group *group;
+
+               cpu = task_cpu(p);
+               group = find_idlest_group(sd, p, cpu);
+               if (group) {
+                       int new_cpu;
+                       new_cpu = find_idlest_cpu(group, cpu);
+                       if (new_cpu != -1 && new_cpu != cpu &&
+                                       cpu_isset(new_cpu, p->cpus_allowed)) {
+                               set_task_cpu(p, new_cpu);
+                               task_rq_unlock(rq, &flags);
+                               rq = task_rq_lock(p, &flags);
+                               cpu = task_cpu(p);
+                       }
+               }
+       }
+#endif
 
        /*
         * We decrease the sleep average of forking parents
@@ -1480,51 +1574,6 @@ static void double_lock_balance(runqueue_t *this_rq, runqueue_t *busiest)
        }
 }
 
-/*
- * find_idlest_cpu - find the least busy runqueue.
- */
-static int find_idlest_cpu(struct task_struct *p, int this_cpu,
-                          struct sched_domain *sd)
-{
-       unsigned long load, min_load, this_load;
-       int i, min_cpu;
-       cpumask_t mask;
-
-       min_cpu = UINT_MAX;
-       min_load = ULONG_MAX;
-
-       cpus_and(mask, sd->span, p->cpus_allowed);
-
-       for_each_cpu_mask(i, mask) {
-               load = target_load(i, sd->wake_idx);
-
-               if (load < min_load) {
-                       min_cpu = i;
-                       min_load = load;
-
-                       /* break out early on an idle CPU: */
-                       if (!min_load)
-                               break;
-               }
-       }
-
-       /* add +1 to account for the new task */
-       this_load = source_load(this_cpu, sd->wake_idx) + SCHED_LOAD_SCALE;
-
-       /*
-        * Would with the addition of the new task to the
-        * current CPU there be an imbalance between this
-        * CPU and the idlest CPU?
-        *
-        * Use half of the balancing threshold - new-context is
-        * a good opportunity to balance.
-        */
-       if (min_load*(100 + (sd->imbalance_pct-100)/2) < this_load*100)
-               return min_cpu;
-
-       return this_cpu;
-}
-
 /*
  * If dest_cpu is allowed for this process, migrate the task to it.
  * This is accomplished by forcing the cpu_allowed mask to only
@@ -1578,8 +1627,15 @@ void sched_exec(void)
                        sd = tmp;
 
        if (sd) {
+               struct sched_group *group;
                schedstat_inc(sd, sbe_attempts);
-               new_cpu = find_idlest_cpu(current, this_cpu, sd);
+               group = find_idlest_group(sd, current, this_cpu);
+               if (!group)
+                       goto out;
+               new_cpu = find_idlest_cpu(group, this_cpu);
+               if (new_cpu == -1)
+                       goto out;
+
                if (new_cpu != this_cpu) {
                        schedstat_inc(sd, sbe_pushed);
                        put_cpu();
@@ -1792,12 +1848,10 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
                if (local_group) {
                        this_load = avg_load;
                        this = group;
-                       goto nextgroup;
                } else if (avg_load > max_load) {
                        max_load = avg_load;
                        busiest = group;
                }
-nextgroup:
                group = group->next;
        } while (group != sd->groups);