page allocator: use a pre-calculated value instead of num_online_nodes() in fast...

num_online_nodes() is called in a number of places but most often by the
page allocator when deciding whether the zonelist needs to be filtered
based on cpusets or the zonelist cache.  This is actually a heavy function
and touches a number of cache lines.

This patch stores the number of online nodes at boot time and updates the
value when nodes get onlined and offlined.  The value is then used in a
number of important paths in place of num_online_nodes().

[rientjes@google.com: do not override definition of node_set_online() with macro]
Signed-off-by: Christoph Lameter <cl@linux-foundation.org>
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Dave Hansen <dave@linux.vnet.ibm.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Signed-off-by: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h
index 848025cd708783795c96a35c08a6176d0fe84a9f..829b94b156f28ec0ed7c0d8dbc2dfeab4ea6825b 100644 (file)
--- a/include/linux/nodemask.h
+++ b/include/linux/nodemask.h
@@ -408,6 +408,19 @@ static inline int num_node_state(enum node_states state)
 #define next_online_node(nid)  next_node((nid), node_states[N_ONLINE])
 
 extern int nr_node_ids;
+extern int nr_online_nodes;
+
+static inline void node_set_online(int nid)
+{
+       node_set_state(nid, N_ONLINE);
+       nr_online_nodes = num_node_state(N_ONLINE);
+}
+
+static inline void node_set_offline(int nid)
+{
+       node_clear_state(nid, N_ONLINE);
+       nr_online_nodes = num_node_state(N_ONLINE);
+}
 #else
 
 static inline int node_state(int node, enum node_states state)
@@ -434,7 +447,10 @@ static inline int num_node_state(enum node_states state)
 #define first_online_node      0
 #define next_online_node(nid)  (MAX_NUMNODES)
 #define nr_node_ids            1
+#define nr_online_nodes                1
 
+#define node_set_online(node)     node_set_state((node), N_ONLINE)
+#define node_set_offline(node)    node_clear_state((node), N_ONLINE)
 #endif
 
 #define node_online_map        node_states[N_ONLINE]
@@ -454,9 +470,6 @@ static inline int num_node_state(enum node_states state)
 #define node_online(node)      node_state((node), N_ONLINE)
 #define node_possible(node)    node_state((node), N_POSSIBLE)
 
-#define node_set_online(node)     node_set_state((node), N_ONLINE)
-#define node_set_offline(node)    node_clear_state((node), N_ONLINE)
-
 #define for_each_node(node)       for_each_node_state(node, N_POSSIBLE)
 #define for_each_online_node(node) for_each_node_state(node, N_ONLINE)
 

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 2f8241f300f5422dac4d82f8b859c66c7b0bbf15..7b9b6015b2ecaee62be67c380f09b3dc471d5c3e 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -875,7 +875,7 @@ static void return_unused_surplus_pages(struct hstate *h,
         * can no longer free unreserved surplus pages. This occurs when
         * the nodes with surplus pages have no free pages.
         */
-       unsigned long remaining_iterations = num_online_nodes();
+       unsigned long remaining_iterations = nr_online_nodes;
 
        /* Uncommit the reservation */
        h->resv_huge_pages -= unused_resv_pages;
@@ -904,7 +904,7 @@ static void return_unused_surplus_pages(struct hstate *h,
                        h->surplus_huge_pages--;
                        h->surplus_huge_pages_node[nid]--;
                        nr_pages--;
-                       remaining_iterations = num_online_nodes();
+                       remaining_iterations = nr_online_nodes;
                }
        }
 }

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index e60e41474332301f2f67426375a47b0f01d9f65a..0c9f406e3c44ad143713c0dfdc7724736dcf8e35 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -161,7 +161,9 @@ static unsigned long __meminitdata dma_reserve;
 
 #if MAX_NUMNODES > 1
 int nr_node_ids __read_mostly = MAX_NUMNODES;
+int nr_online_nodes __read_mostly = 1;
 EXPORT_SYMBOL(nr_node_ids);
+EXPORT_SYMBOL(nr_online_nodes);
 #endif
 
 int page_group_by_mobility_disabled __read_mostly;
@@ -1466,7 +1468,7 @@ this_zone_full:
                if (NUMA_BUILD)
                        zlc_mark_zone_full(zonelist, z);
 try_next_zone:
-               if (NUMA_BUILD && !did_zlc_setup && num_online_nodes() > 1) {
+               if (NUMA_BUILD && !did_zlc_setup && nr_online_nodes > 1) {
                        /*
                         * we do zlc_setup after the first zone is tried but only
                         * if there are multiple nodes make it worthwhile
@@ -2265,7 +2267,7 @@ int numa_zonelist_order_handler(ctl_table *table, int write,
 }
 
 
-#define MAX_NODE_LOAD (num_online_nodes())
+#define MAX_NODE_LOAD (nr_online_nodes)
 static int node_load[MAX_NUMNODES];
 
 /**
@@ -2474,7 +2476,7 @@ static void build_zonelists(pg_data_t *pgdat)
 
        /* NUMA-aware ordering of nodes */
        local_node = pgdat->node_id;
-       load = num_online_nodes();
+       load = nr_online_nodes;
        prev_node = local_node;
        nodes_clear(used_mask);
 
@@ -2625,7 +2627,7 @@ void build_all_zonelists(void)
 
        printk("Built %i zonelists in %s order, mobility grouping %s.  "
                "Total pages: %ld\n",
-                       num_online_nodes(),
+                       nr_online_nodes,
                        zonelist_order_name[current_zonelist_order],
                        page_group_by_mobility_disabled ? "off" : "on",
                        vm_total_pages);

diff --git a/mm/slub.c b/mm/slub.c
index 30354bfeb43d5b093669fb0ef62b213e0c3991f2..1c950775d6bec7827a3375da5ca1fc2abe47ac14 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3737,7 +3737,7 @@ static int list_locations(struct kmem_cache *s, char *buf,
                                                 to_cpumask(l->cpus));
                }
 
-               if (num_online_nodes() > 1 && !nodes_empty(l->nodes) &&
+               if (nr_online_nodes > 1 && !nodes_empty(l->nodes) &&
                                len < PAGE_SIZE - 60) {
                        len += sprintf(buf + len, " nodes=");
                        len += nodelist_scnprintf(buf + len, PAGE_SIZE - len - 50,

diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
index 8847add6ca164a7aea722b96978c44abb5ba7158..5ed8931dfe98f8af810a28a046f9c6729f07e6a3 100644 (file)
--- a/net/sunrpc/svc.c
+++ b/net/sunrpc/svc.c
@@ -124,7 +124,7 @@ svc_pool_map_choose_mode(void)
 {
        unsigned int node;
 
-       if (num_online_nodes() > 1) {
+       if (nr_online_nodes > 1) {
                /*
                 * Actually have multiple NUMA nodes,
                 * so split pools on NUMA node boundaries