mm/sl[aou]b: Common alignment code

Extract the code to do object alignment from the allocators.
Do the alignment calculations in slab_common so that the
__kmem_cache_create functions of the allocators do not have
to deal with alignment.

Signed-off-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>

diff --git a/mm/slab.c b/mm/slab.c
index e1790e56fd86b500333017b741123d15640da540..2c3a2e0394db6737887cb0309c27988ca0edb621 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -2337,22 +2337,6 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
                size &= ~(BYTES_PER_WORD - 1);
        }
 
-       /* calculate the final buffer alignment: */
-
-       /* 1) arch recommendation: can be overridden for debug */
-       if (flags & SLAB_HWCACHE_ALIGN) {
-               /*
-                * Default alignment: as specified by the arch code.  Except if
-                * an object is really small, then squeeze multiple objects into
-                * one cacheline.
-                */
-               ralign = cache_line_size();
-               while (size <= ralign / 2)
-                       ralign /= 2;
-       } else {
-               ralign = BYTES_PER_WORD;
-       }
-
        /*
         * Redzoning and user store require word alignment or possibly larger.
         * Note this will be overridden by architecture or caller mandated
@@ -2369,10 +2353,6 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
                size &= ~(REDZONE_ALIGN - 1);
        }
 
-       /* 2) arch mandated alignment */
-       if (ralign < ARCH_SLAB_MINALIGN) {
-               ralign = ARCH_SLAB_MINALIGN;
-       }
        /* 3) caller mandated alignment */
        if (ralign < cachep->align) {
                ralign = cachep->align;

diff --git a/mm/slab.h b/mm/slab.h
index 492eafa0b538c5e30ce05b2c88fca6c69026e369..1cb9c9ee0e6f3875854b6f51b206925031faaf55 100644 (file)
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -32,6 +32,9 @@ extern struct list_head slab_caches;
 /* The slab cache that manages slab cache information */
 extern struct kmem_cache *kmem_cache;
 
+unsigned long calculate_alignment(unsigned long flags,
+               unsigned long align, unsigned long size);
+
 /* Functions provided by the slab allocators */
 extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags);
 

diff --git a/mm/slab_common.c b/mm/slab_common.c
index 497b45c25bae09b6ffe089cb3931acf1ef568057..a8e76d79ee65628cb5071b00935202536262c490 100644 (file)
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -72,6 +72,34 @@ static inline int kmem_cache_sanity_check(const char *name, size_t size)
 }
 #endif
 
+/*
+ * Figure out what the alignment of the objects will be given a set of
+ * flags, a user specified alignment and the size of the objects.
+ */
+unsigned long calculate_alignment(unsigned long flags,
+               unsigned long align, unsigned long size)
+{
+       /*
+        * If the user wants hardware cache aligned objects then follow that
+        * suggestion if the object is sufficiently large.
+        *
+        * The hardware cache alignment cannot override the specified
+        * alignment though. If that is greater then use it.
+        */
+       if (flags & SLAB_HWCACHE_ALIGN) {
+               unsigned long ralign = cache_line_size();
+               while (size <= ralign / 2)
+                       ralign /= 2;
+               align = max(align, ralign);
+       }
+
+       if (align < ARCH_SLAB_MINALIGN)
+               align = ARCH_SLAB_MINALIGN;
+
+       return ALIGN(align, sizeof(void *));
+}
+
+
 /*
  * kmem_cache_create - Create a cache.
  * @name: A string which is used in /proc/slabinfo to identify this cache.
@@ -124,7 +152,7 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align
        s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
        if (s) {
                s->object_size = s->size = size;
-               s->align = align;
+               s->align = calculate_alignment(flags, align, size);
                s->ctor = ctor;
                s->name = kstrdup(name, GFP_KERNEL);
                if (!s->name) {
@@ -211,7 +239,7 @@ void __init create_boot_cache(struct kmem_cache *s, const char *name, size_t siz
 
        s->name = name;
        s->size = s->object_size = size;
-       s->align = ARCH_KMALLOC_MINALIGN;
+       s->align = calculate_alignment(flags, ARCH_KMALLOC_MINALIGN, size);
        err = __kmem_cache_create(s, flags);
 
        if (err)

diff --git a/mm/slob.c b/mm/slob.c
index 87e16c4d9143621f608f45f675b4be467add5961..795bab7d391da90fbd68cfd7c4730485db69b294 100644 (file)
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -123,7 +123,6 @@ static inline void clear_slob_page_free(struct page *sp)
 
 #define SLOB_UNIT sizeof(slob_t)
 #define SLOB_UNITS(size) (((size) + SLOB_UNIT - 1)/SLOB_UNIT)
-#define SLOB_ALIGN L1_CACHE_BYTES
 
 /*
  * struct slob_rcu is inserted at the tail of allocated slob blocks, which
@@ -527,20 +526,11 @@ EXPORT_SYMBOL(ksize);
 
 int __kmem_cache_create(struct kmem_cache *c, unsigned long flags)
 {
-       size_t align = c->size;
-
        if (flags & SLAB_DESTROY_BY_RCU) {
                /* leave room for rcu footer at the end of object */
                c->size += sizeof(struct slob_rcu);
        }
        c->flags = flags;
-       /* ignore alignment unless it's forced */
-       c->align = (flags & SLAB_HWCACHE_ALIGN) ? SLOB_ALIGN : 0;
-       if (c->align < ARCH_SLAB_MINALIGN)
-               c->align = ARCH_SLAB_MINALIGN;
-       if (c->align < align)
-               c->align = align;
-
        return 0;
 }
 

diff --git a/mm/slub.c b/mm/slub.c
index c82453ac812a2bdcc37ec70e2682afa61a935898..9640edd2cc78e067acd29432b44ba8e19e5ff815 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2760,32 +2760,6 @@ static inline int calculate_order(int size, int reserved)
        return -ENOSYS;
 }
 
-/*
- * Figure out what the alignment of the objects will be.
- */
-static unsigned long calculate_alignment(unsigned long flags,
-               unsigned long align, unsigned long size)
-{
-       /*
-        * If the user wants hardware cache aligned objects then follow that
-        * suggestion if the object is sufficiently large.
-        *
-        * The hardware cache alignment cannot override the specified
-        * alignment though. If that is greater then use it.
-        */
-       if (flags & SLAB_HWCACHE_ALIGN) {
-               unsigned long ralign = cache_line_size();
-               while (size <= ralign / 2)
-                       ralign /= 2;
-               align = max(align, ralign);
-       }
-
-       if (align < ARCH_SLAB_MINALIGN)
-               align = ARCH_SLAB_MINALIGN;
-
-       return ALIGN(align, sizeof(void *));
-}
-
 static void
 init_kmem_cache_node(struct kmem_cache_node *n)
 {
@@ -2919,7 +2893,6 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order)
 {
        unsigned long flags = s->flags;
        unsigned long size = s->object_size;
-       unsigned long align = s->align;
        int order;
 
        /*
@@ -2990,20 +2963,12 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order)
                size += sizeof(void *);
 #endif
 
-       /*
-        * Determine the alignment based on various parameters that the
-        * user specified and the dynamic determination of cache line size
-        * on bootup.
-        */
-       align = calculate_alignment(flags, align, s->object_size);
-       s->align = align;
-
        /*
         * SLUB stores one object immediately after another beginning from
         * offset 0. In order to align the objects we have to simply size
         * each object to conform to the alignment.
         */
-       size = ALIGN(size, align);
+       size = ALIGN(size, s->align);
        s->size = size;
        if (forced_order >= 0)
                order = forced_order;
@@ -3032,7 +2997,6 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order)
                s->max = s->oo;
 
        return !!oo_objects(s->oo);
-
 }
 
 static int kmem_cache_open(struct kmem_cache *s, unsigned long flags)