slab: implement bulk alloc in SLAB allocator

This patch implements the alloc side of bulk API for the SLAB allocator.

Further optimization are still possible by changing the call to
__do_cache_alloc() into something that can return multiple objects.
This optimization is left for later, given end results already show in
the area of 80% speedup.

Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Vladimir Davydov <vdavydov@virtuozzo.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/mm/slab.c b/mm/slab.c
index f872208a0912b19a1606676a27c1827448a3c49c..0c2e2cafbc83cd87553feb1f162e69c0372b3c17 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -3392,9 +3392,42 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
 EXPORT_SYMBOL(kmem_cache_free_bulk);
 
 int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
-                                                               void **p)
+                         void **p)
 {
-       return __kmem_cache_alloc_bulk(s, flags, size, p);
+       size_t i;
+
+       s = slab_pre_alloc_hook(s, flags);
+       if (!s)
+               return 0;
+
+       cache_alloc_debugcheck_before(s, flags);
+
+       local_irq_disable();
+       for (i = 0; i < size; i++) {
+               void *objp = __do_cache_alloc(s, flags);
+
+               /* this call could be done outside IRQ disabled section */
+               objp = cache_alloc_debugcheck_after(s, flags, objp, _RET_IP_);
+
+               if (unlikely(!objp))
+                       goto error;
+               p[i] = objp;
+       }
+       local_irq_enable();
+
+       /* Clear memory outside IRQ disabled section */
+       if (unlikely(flags & __GFP_ZERO))
+               for (i = 0; i < size; i++)
+                       memset(p[i], 0, s->object_size);
+
+       slab_post_alloc_hook(s, flags, size, p);
+       /* FIXME: Trace call missing. Christoph would like a bulk variant */
+       return size;
+error:
+       local_irq_enable();
+       slab_post_alloc_hook(s, flags, i, p);
+       __kmem_cache_free_bulk(s, i, p);
+       return 0;
 }
 EXPORT_SYMBOL(kmem_cache_alloc_bulk);