sched: Tweak wake_idx

When merging select_task_rq_fair() and sched_balance_self() we lost
the use of wake_idx, restore that and set them to 0 to make wake
balancing more aggressive.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/ia64/include/asm/topology.h b/arch/ia64/include/asm/topology.h
index cf6053b226c38383de57774346fdb1ed4b4d8e39..47f3c51d5e27aae240796a5904f662a2403d075b 100644 (file)
--- a/arch/ia64/include/asm/topology.h
+++ b/arch/ia64/include/asm/topology.h
@@ -62,11 +62,12 @@ void build_cpu_to_node_map(void);
        .busy_idx               = 2,                    \
        .idle_idx               = 1,                    \
        .newidle_idx            = 2,                    \
-       .wake_idx               = 1,                    \
+       .wake_idx               = 0,                    \
        .forkexec_idx           = 1,                    \
        .flags                  = SD_LOAD_BALANCE       \
                                | SD_BALANCE_NEWIDLE    \
                                | SD_BALANCE_EXEC       \
+                               | SD_BALANCE_FORK       \
                                | SD_BALANCE_WAKE       \
                                | SD_WAKE_AFFINE,       \
        .last_balance           = jiffies,              \
@@ -87,7 +88,7 @@ void build_cpu_to_node_map(void);
        .busy_idx               = 3,                    \
        .idle_idx               = 2,                    \
        .newidle_idx            = 2,                    \
-       .wake_idx               = 1,                    \
+       .wake_idx               = 0,                    \
        .forkexec_idx           = 1,                    \
        .flags                  = SD_LOAD_BALANCE       \
                                | SD_BALANCE_EXEC       \

diff --git a/arch/powerpc/include/asm/topology.h b/arch/powerpc/include/asm/topology.h
index c6343313ff592f7d8ccd9ffd8913f3a2e0dcc771..a6b220ab56db1f97af4924d05d047b28dafeb55b 100644 (file)
--- a/arch/powerpc/include/asm/topology.h
+++ b/arch/powerpc/include/asm/topology.h
@@ -58,9 +58,10 @@ static inline int pcibus_to_node(struct pci_bus *bus)
        .busy_idx               = 3,                    \
        .idle_idx               = 1,                    \
        .newidle_idx            = 2,                    \
-       .wake_idx               = 1,                    \
+       .wake_idx               = 0,                    \
        .flags                  = SD_LOAD_BALANCE       \
                                | SD_BALANCE_EXEC       \
+                               | SD_BALANCE_FORK       \
                                | SD_BALANCE_NEWIDLE    \
                                | SD_BALANCE_WAKE       \
                                | SD_SERIALIZE,         \

diff --git a/arch/sh/include/asm/topology.h b/arch/sh/include/asm/topology.h
index dc1531e2f25f52cfe235038a0579b094d80c858d..9054e5c0ad54959f7bcc199f6a965d847bef0ce4 100644 (file)
--- a/arch/sh/include/asm/topology.h
+++ b/arch/sh/include/asm/topology.h
@@ -16,7 +16,7 @@
        .busy_idx               = 3,                    \
        .idle_idx               = 2,                    \
        .newidle_idx            = 2,                    \
-       .wake_idx               = 1,                    \
+       .wake_idx               = 0,                    \
        .forkexec_idx           = 1,                    \
        .flags                  = SD_LOAD_BALANCE       \
                                | SD_BALANCE_FORK       \

diff --git a/arch/sparc/include/asm/topology_64.h b/arch/sparc/include/asm/topology_64.h
index 1d091abd2d132d4ddf1208d81f52c9742f046cf9..bc3a0930ed643f57f999c9595c692dc9d5313aed 100644 (file)
--- a/arch/sparc/include/asm/topology_64.h
+++ b/arch/sparc/include/asm/topology_64.h
@@ -52,7 +52,7 @@ static inline int pcibus_to_node(struct pci_bus *pbus)
        .busy_idx               = 3,                    \
        .idle_idx               = 2,                    \
        .newidle_idx            = 0,                    \
-       .wake_idx               = 1,                    \
+       .wake_idx               = 0,                    \
        .forkexec_idx           = 1,                    \
        .flags                  = SD_LOAD_BALANCE       \
                                | SD_BALANCE_FORK       \

diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h
index 966d58dc62742ca1802c5f6b582ff9dcec65a214..4b1b335097b597d622f20f9975c260a1a6673b4e 100644 (file)
--- a/arch/x86/include/asm/topology.h
+++ b/arch/x86/include/asm/topology.h
@@ -138,7 +138,7 @@ extern unsigned long node_remap_size[];
        .busy_idx               = 3,                                    \
        .idle_idx               = SD_IDLE_IDX,                          \
        .newidle_idx            = SD_NEWIDLE_IDX,                       \
-       .wake_idx               = 1,                                    \
+       .wake_idx               = 0,                                    \
        .forkexec_idx           = SD_FORKEXEC_IDX,                      \
                                                                        \
        .flags                  = 1*SD_LOAD_BALANCE                     \

diff --git a/include/linux/topology.h b/include/linux/topology.h
index 6a8cd15555bb2ef3154dcab175d743e4ad08ad1b..fef57040a4e25a0de9af560f8a6eab8a68f52527 100644 (file)
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -120,7 +120,7 @@ int arch_update_cpu_topology(void);
        .imbalance_pct          = 125,                                  \
        .cache_nice_tries       = 1,                                    \
        .busy_idx               = 2,                                    \
-       .wake_idx               = 1,                                    \
+       .wake_idx               = 0,                                    \
        .forkexec_idx           = 1,                                    \
                                                                        \
        .flags                  = 1*SD_LOAD_BALANCE                     \
@@ -152,7 +152,7 @@ int arch_update_cpu_topology(void);
        .busy_idx               = 2,                                    \
        .idle_idx               = 1,                                    \
        .newidle_idx            = 2,                                    \
-       .wake_idx               = 1,                                    \
+       .wake_idx               = 0,                                    \
        .forkexec_idx           = 1,                                    \
                                                                        \
        .flags                  = 1*SD_LOAD_BALANCE                     \

diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 8b3eddbcf9a48909192700647f9d78949ef247e2..19593568031a693f9f212c350e95492d425eb9ac 100644 (file)
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -1232,12 +1232,27 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
  * domain.
  */
 static struct sched_group *
-find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
+find_idlest_group(struct sched_domain *sd, struct task_struct *p,
+                 int this_cpu, int flag)
 {
        struct sched_group *idlest = NULL, *this = NULL, *group = sd->groups;
        unsigned long min_load = ULONG_MAX, this_load = 0;
-       int load_idx = sd->forkexec_idx;
        int imbalance = 100 + (sd->imbalance_pct-100)/2;
+       int load_idx = 0;
+
+       switch (flag) {
+       case SD_BALANCE_FORK:
+       case SD_BALANCE_EXEC:
+               load_idx = sd->forkexec_idx;
+               break;
+
+       case SD_BALANCE_WAKE:
+               load_idx = sd->wake_idx;
+               break;
+
+       default:
+               break;
+       }
 
        do {
                unsigned long load, avg_load;
@@ -1392,7 +1407,7 @@ static int select_task_rq_fair(struct task_struct *p, int flag, int sync)
                        continue;
                }
 
-               group = find_idlest_group(sd, p, cpu);
+               group = find_idlest_group(sd, p, cpu, flag);
                if (!group) {
                        sd = sd->child;
                        continue;