SLUB: Optimize cacheline use for zeroing

We touch a cacheline in the kmem_cache structure for zeroing to get the
size. However, the hot paths in slab_alloc and slab_free do not reference
any other fields in kmem_cache, so we may have to just bring in the
cacheline for this one access.

Add a new field to kmem_cache_cpu that contains the object size. That
cacheline must already be used in the hotpaths. So we save one cacheline
on every slab_alloc if we zero.

We need to update the kmem_cache_cpu object size if an aliasing operation
changes the objsize of an non debug slab.

Signed-off-by: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h
index f74716b59ce233fc44a9816d567e32fe1bba495d..d65159d1d4f5cd6437ee7abda73a10584c7e1adb 100644 (file)
--- a/include/linux/slub_def.h
+++ b/include/linux/slub_def.h
@@ -16,6 +16,7 @@ struct kmem_cache_cpu {
        struct page *page;
        int node;
        unsigned int offset;
+       unsigned int objsize;
 };
 
 struct kmem_cache_node {

diff --git a/mm/slub.c b/mm/slub.c
index 6d4346ba0c29652b02f80f68e16cdba66e5b3ba7..1d48f383e97d61fb25dd0cb07e3b4e2bb3d7895d 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1576,7 +1576,7 @@ static void __always_inline *slab_alloc(struct kmem_cache *s,
        local_irq_restore(flags);
 
        if (unlikely((gfpflags & __GFP_ZERO) && object))
-               memset(object, 0, s->objsize);
+               memset(object, 0, c->objsize);
 
        return object;
 }
@@ -1858,8 +1858,9 @@ static void init_kmem_cache_cpu(struct kmem_cache *s,
 {
        c->page = NULL;
        c->freelist = NULL;
-       c->offset = s->offset / sizeof(void *);
        c->node = 0;
+       c->offset = s->offset / sizeof(void *);
+       c->objsize = s->objsize;
 }
 
 static void init_kmem_cache_node(struct kmem_cache_node *n)
@@ -2852,12 +2853,21 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size,
        down_write(&slub_lock);
        s = find_mergeable(size, align, flags, name, ctor);
        if (s) {
+               int cpu;
+
                s->refcount++;
                /*
                 * Adjust the object sizes so that we clear
                 * the complete object on kzalloc.
                 */
                s->objsize = max(s->objsize, (int)size);
+
+               /*
+                * And then we need to update the object size in the
+                * per cpu structures
+                */
+               for_each_online_cpu(cpu)
+                       get_cpu_slab(s, cpu)->objsize = s->objsize;
                s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
                up_write(&slub_lock);
                if (sysfs_slab_alias(s, name))