slub: No need for per node slab counters if !SLUB_DEBUG

The per node counters are used mainly for showing data through the sysfs API.
If that API is not compiled in then there is no point in keeping track of this
data. Disable counters for the number of slabs and the number of total slabs
if !SLUB_DEBUG. Incrementing the per node counters is also accessing a
potentially contended cacheline so this could actually be a performance
benefit to embedded systems.

SLABINFO support is also affected. It now must depends on SLUB_DEBUG (which
is on by default).

Patch also avoids a check for a NULL kmem_cache_node pointer in new_slab()
if the system is not compiled with NUMA support.

[penberg@cs.helsinki.fi: fix oops and move ->nr_slabs into CONFIG_SLUB_DEBUG]
Signed-off-by: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Pekka Enberg <penberg@cs.helsinki.fi>

diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h
index b00c1c73eb0a7a3108f2fcc76d4c6fd4ee54fbb8..79d59c937fac4ca62d0ca98c0b3249d1842d78c1 100644 (file)
--- a/include/linux/slub_def.h
+++ b/include/linux/slub_def.h
@@ -45,9 +45,9 @@ struct kmem_cache_cpu {
 struct kmem_cache_node {
        spinlock_t list_lock;   /* Protect partial list and nr_partial */
        unsigned long nr_partial;
-       atomic_long_t nr_slabs;
        struct list_head partial;
 #ifdef CONFIG_SLUB_DEBUG
+       atomic_long_t nr_slabs;
        struct list_head full;
 #endif
 };

diff --git a/init/Kconfig b/init/Kconfig
index a97924bc5b8dcfa87079ee2f9f70ce9988d4bf40..7fccf09bb95ad8adeb929829dc1bb624c84ffc82 100644 (file)
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -763,7 +763,7 @@ endmenu             # General setup
 config SLABINFO
        bool
        depends on PROC_FS
-       depends on SLAB || SLUB
+       depends on SLAB || SLUB_DEBUG
        default y
 
 config RT_MUTEXES

diff --git a/mm/slub.c b/mm/slub.c
index f924cffb29e793e8f8713d2071d473df1658cb01..7f8aaa291a4e2b15083a0b5ad8e9beb6978a4346 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -837,6 +837,35 @@ static void remove_full(struct kmem_cache *s, struct page *page)
        spin_unlock(&n->list_lock);
 }
 
+/* Tracking of the number of slabs for debugging purposes */
+static inline unsigned long slabs_node(struct kmem_cache *s, int node)
+{
+       struct kmem_cache_node *n = get_node(s, node);
+
+       return atomic_long_read(&n->nr_slabs);
+}
+
+static inline void inc_slabs_node(struct kmem_cache *s, int node)
+{
+       struct kmem_cache_node *n = get_node(s, node);
+
+       /*
+        * May be called early in order to allocate a slab for the
+        * kmem_cache_node structure. Solve the chicken-egg
+        * dilemma by deferring the increment of the count during
+        * bootstrap (see early_kmem_cache_node_alloc).
+        */
+       if (!NUMA_BUILD || n)
+               atomic_long_inc(&n->nr_slabs);
+}
+static inline void dec_slabs_node(struct kmem_cache *s, int node)
+{
+       struct kmem_cache_node *n = get_node(s, node);
+
+       atomic_long_dec(&n->nr_slabs);
+}
+
+/* Object debug checks for alloc/free paths */
 static void setup_object_debug(struct kmem_cache *s, struct page *page,
                                                                void *object)
 {
@@ -1028,6 +1057,11 @@ static inline unsigned long kmem_cache_flags(unsigned long objsize,
        return flags;
 }
 #define slub_debug 0
+
+static inline unsigned long slabs_node(struct kmem_cache *s, int node)
+                                                       { return 0; }
+static inline void inc_slabs_node(struct kmem_cache *s, int node) {}
+static inline void dec_slabs_node(struct kmem_cache *s, int node) {}
 #endif
 /*
  * Slab allocation and freeing
@@ -1066,7 +1100,6 @@ static void setup_object(struct kmem_cache *s, struct page *page,
 static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
 {
        struct page *page;
-       struct kmem_cache_node *n;
        void *start;
        void *last;
        void *p;
@@ -1078,9 +1111,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
        if (!page)
                goto out;
 
-       n = get_node(s, page_to_nid(page));
-       if (n)
-               atomic_long_inc(&n->nr_slabs);
+       inc_slabs_node(s, page_to_nid(page));
        page->slab = s;
        page->flags |= 1 << PG_slab;
        if (s->flags & (SLAB_DEBUG_FREE | SLAB_RED_ZONE | SLAB_POISON |
@@ -1153,9 +1184,7 @@ static void free_slab(struct kmem_cache *s, struct page *page)
 
 static void discard_slab(struct kmem_cache *s, struct page *page)
 {
-       struct kmem_cache_node *n = get_node(s, page_to_nid(page));
-
-       atomic_long_dec(&n->nr_slabs);
+       dec_slabs_node(s, page_to_nid(page));
        free_slab(s, page);
 }
 
@@ -1894,10 +1923,10 @@ static void init_kmem_cache_cpu(struct kmem_cache *s,
 static void init_kmem_cache_node(struct kmem_cache_node *n)
 {
        n->nr_partial = 0;
-       atomic_long_set(&n->nr_slabs, 0);
        spin_lock_init(&n->list_lock);
        INIT_LIST_HEAD(&n->partial);
 #ifdef CONFIG_SLUB_DEBUG
+       atomic_long_set(&n->nr_slabs, 0);
        INIT_LIST_HEAD(&n->full);
 #endif
 }
@@ -2066,7 +2095,7 @@ static struct kmem_cache_node *early_kmem_cache_node_alloc(gfp_t gfpflags,
        init_tracking(kmalloc_caches, n);
 #endif
        init_kmem_cache_node(n);
-       atomic_long_inc(&n->nr_slabs);
+       inc_slabs_node(kmalloc_caches, node);
 
        /*
         * lockdep requires consistent irq usage for each lock
@@ -2379,7 +2408,7 @@ static inline int kmem_cache_close(struct kmem_cache *s)
                struct kmem_cache_node *n = get_node(s, node);
 
                n->nr_partial -= free_list(s, n, &n->partial);
-               if (atomic_long_read(&n->nr_slabs))
+               if (slabs_node(s, node))
                        return 1;
        }
        free_kmem_cache_nodes(s);
@@ -2801,7 +2830,7 @@ static void slab_mem_offline_callback(void *arg)
                         * and offline_pages() function shoudn't call this
                         * callback. So, we must fail.
                         */
-                       BUG_ON(atomic_long_read(&n->nr_slabs));
+                       BUG_ON(slabs_node(s, offline_node));
 
                        s->node[offline_node] = NULL;
                        kmem_cache_free(kmalloc_caches, n);