slub: Remove CONFIG_CMPXCHG_LOCAL ifdeffery

Remove the #ifdefs. This means that the irqsafe_cpu_cmpxchg_double() is used
everywhere.

There may be performance implications since:

A. We now have to manage a transaction ID for all arches

B. The interrupt holdoff for arches not supporting CONFIG_CMPXCHG_LOCAL is reduced
to a very short irqoff section.

There are no multiple irqoff/irqon sequences as a result of this change. Even in the fallback
case we only have to do one disable and enable like before.

Signed-off-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>

diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h
index 45ca123e800219777001f75113324f6cfd60486f..ca0c076b2374db1112196fdab837e6f6a7fb2701 100644 (file)
--- a/include/linux/slub_def.h
+++ b/include/linux/slub_def.h
@@ -37,9 +37,7 @@ enum stat_item {
 
 struct kmem_cache_cpu {
        void **freelist;        /* Pointer to next available object */
-#ifdef CONFIG_CMPXCHG_LOCAL
        unsigned long tid;      /* Globally unique transaction id */
-#endif
        struct page *page;      /* The slab from which we are allocating */
        int node;               /* The node of the page (or -1 for debug) */
 #ifdef CONFIG_SLUB_STATS

diff --git a/mm/slub.c b/mm/slub.c
index c952fac112e858a2b050575710995dbbd7c02ae8..461199f019d6e8a4101692dc5ed2586d9f0b275f 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1551,7 +1551,6 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
        }
 }
 
-#ifdef CONFIG_CMPXCHG_LOCAL
 #ifdef CONFIG_PREEMPT
 /*
  * Calculate the next globally unique transaction for disambiguiation
@@ -1611,17 +1610,12 @@ static inline void note_cmpxchg_failure(const char *n,
        stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
 }
 
-#endif
-
 void init_kmem_cache_cpus(struct kmem_cache *s)
 {
-#ifdef CONFIG_CMPXCHG_LOCAL
        int cpu;
 
        for_each_possible_cpu(cpu)
                per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
-#endif
-
 }
 /*
  * Remove the cpu slab
@@ -1654,9 +1648,7 @@ static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
                page->inuse--;
        }
        c->page = NULL;
-#ifdef CONFIG_CMPXCHG_LOCAL
        c->tid = next_tid(c->tid);
-#endif
        unfreeze_slab(s, page, tail);
 }
 
@@ -1791,7 +1783,6 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 {
        void **object;
        struct page *page;
-#ifdef CONFIG_CMPXCHG_LOCAL
        unsigned long flags;
 
        local_irq_save(flags);
@@ -1802,7 +1793,6 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
         * pointer.
         */
        c = this_cpu_ptr(s->cpu_slab);
-#endif
 #endif
 
        /* We handle __GFP_ZERO in the caller */
@@ -1831,10 +1821,8 @@ load_freelist:
 
 unlock_out:
        slab_unlock(page);
-#ifdef CONFIG_CMPXCHG_LOCAL
        c->tid = next_tid(c->tid);
        local_irq_restore(flags);
-#endif
        stat(s, ALLOC_SLOWPATH);
        return object;
 
@@ -1873,9 +1861,7 @@ load_from_page:
        }
        if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
                slab_out_of_memory(s, gfpflags, node);
-#ifdef CONFIG_CMPXCHG_LOCAL
        local_irq_restore(flags);
-#endif
        return NULL;
 debug:
        if (!alloc_debug_processing(s, page, object, addr))
@@ -1902,20 +1888,12 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
 {
        void **object;
        struct kmem_cache_cpu *c;
-#ifdef CONFIG_CMPXCHG_LOCAL
        unsigned long tid;
-#else
-       unsigned long flags;
-#endif
 
        if (slab_pre_alloc_hook(s, gfpflags))
                return NULL;
 
-#ifndef CONFIG_CMPXCHG_LOCAL
-       local_irq_save(flags);
-#else
 redo:
-#endif
 
        /*
         * Must read kmem_cache cpu data via this cpu ptr. Preemption is
@@ -1925,7 +1903,6 @@ redo:
         */
        c = __this_cpu_ptr(s->cpu_slab);
 
-#ifdef CONFIG_CMPXCHG_LOCAL
        /*
         * The transaction ids are globally unique per cpu and per operation on
         * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
@@ -1934,7 +1911,6 @@ redo:
         */
        tid = c->tid;
        barrier();
-#endif
 
        object = c->freelist;
        if (unlikely(!object || !node_match(c, node)))
@@ -1942,7 +1918,6 @@ redo:
                object = __slab_alloc(s, gfpflags, node, addr, c);
 
        else {
-#ifdef CONFIG_CMPXCHG_LOCAL
                /*
                 * The cmpxchg will only match if there was no additonal
                 * operation and if we are on the right processor.
@@ -1963,16 +1938,9 @@ redo:
                        note_cmpxchg_failure("slab_alloc", s, tid);
                        goto redo;
                }
-#else
-               c->freelist = get_freepointer(s, object);
-#endif
                stat(s, ALLOC_FASTPATH);
        }
 
-#ifndef CONFIG_CMPXCHG_LOCAL
-       local_irq_restore(flags);
-#endif
-
        if (unlikely(gfpflags & __GFP_ZERO) && object)
                memset(object, 0, s->objsize);
 
@@ -2049,11 +2017,9 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
 {
        void *prior;
        void **object = (void *)x;
-#ifdef CONFIG_CMPXCHG_LOCAL
        unsigned long flags;
 
        local_irq_save(flags);
-#endif
        slab_lock(page);
        stat(s, FREE_SLOWPATH);
 
@@ -2084,9 +2050,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
 
 out_unlock:
        slab_unlock(page);
-#ifdef CONFIG_CMPXCHG_LOCAL
        local_irq_restore(flags);
-#endif
        return;
 
 slab_empty:
@@ -2098,9 +2062,7 @@ slab_empty:
                stat(s, FREE_REMOVE_PARTIAL);
        }
        slab_unlock(page);
-#ifdef CONFIG_CMPXCHG_LOCAL
        local_irq_restore(flags);
-#endif
        stat(s, FREE_SLAB);
        discard_slab(s, page);
 }
@@ -2121,20 +2083,11 @@ static __always_inline void slab_free(struct kmem_cache *s,
 {
        void **object = (void *)x;
        struct kmem_cache_cpu *c;
-#ifdef CONFIG_CMPXCHG_LOCAL
        unsigned long tid;
-#else
-       unsigned long flags;
-#endif
 
        slab_free_hook(s, x);
 
-#ifndef CONFIG_CMPXCHG_LOCAL
-       local_irq_save(flags);
-
-#else
 redo:
-#endif
 
        /*
         * Determine the currently cpus per cpu slab.
@@ -2144,15 +2097,12 @@ redo:
         */
        c = __this_cpu_ptr(s->cpu_slab);
 
-#ifdef CONFIG_CMPXCHG_LOCAL
        tid = c->tid;
        barrier();
-#endif
 
        if (likely(page == c->page && c->node != NUMA_NO_NODE)) {
                set_freepointer(s, object, c->freelist);
 
-#ifdef CONFIG_CMPXCHG_LOCAL
                if (unlikely(!this_cpu_cmpxchg_double(
                                s->cpu_slab->freelist, s->cpu_slab->tid,
                                c->freelist, tid,
@@ -2161,16 +2111,10 @@ redo:
                        note_cmpxchg_failure("slab_free", s, tid);
                        goto redo;
                }
-#else
-               c->freelist = object;
-#endif
                stat(s, FREE_FASTPATH);
        } else
                __slab_free(s, page, x, addr);
 
-#ifndef CONFIG_CMPXCHG_LOCAL
-       local_irq_restore(flags);
-#endif
 }
 
 void kmem_cache_free(struct kmem_cache *s, void *x)