mm/mempool.c

   1 /*
   2  *  linux/mm/mempool.c
   3  *
   4  *  memory buffer pool support. Such pools are mostly used
   5  *  for guaranteed, deadlock-free memory allocations during
   6  *  extreme VM load.
   7  *
   8  *  started by Ingo Molnar, Copyright (C) 2001
   9  */
  10
  11 #include <linux/mm.h>
  12 #include <linux/slab.h>
  13 #include <linux/export.h>
  14 #include <linux/mempool.h>
  15 #include <linux/blkdev.h>
  16 #include <linux/writeback.h>
  17
  18 static void add_element(mempool_t *pool, void *element)
  19 {
  20         BUG_ON(pool->curr_nr >= pool->min_nr);
  21         pool->elements[pool->curr_nr++] = element;
  22 }
  23
  24 static void *remove_element(mempool_t *pool)
  25 {
  26         BUG_ON(pool->curr_nr <= 0);
  27         return pool->elements[--pool->curr_nr];
  28 }
  29
  30 /**
  31  * mempool_destroy - deallocate a memory pool
  32  * @pool:      pointer to the memory pool which was allocated via
  33  *             mempool_create().
  34  *
  35  * Free all reserved elements in @pool and @pool itself.  This function
  36  * only sleeps if the free_fn() function sleeps.
  37  */
  38 void mempool_destroy(mempool_t *pool)
  39 {
  40         while (pool->curr_nr) {
  41                 void *element = remove_element(pool);
  42                 pool->free(element, pool->pool_data);
  43         }
  44         kfree(pool->elements);
  45         kfree(pool);
  46 }
  47 EXPORT_SYMBOL(mempool_destroy);
  48
  49 /**
  50  * mempool_create - create a memory pool
  51  * @min_nr:    the minimum number of elements guaranteed to be
  52  *             allocated for this pool.
  53  * @alloc_fn:  user-defined element-allocation function.
  54  * @free_fn:   user-defined element-freeing function.
  55  * @pool_data: optional private data available to the user-defined functions.
  56  *
  57  * this function creates and allocates a guaranteed size, preallocated
  58  * memory pool. The pool can be used from the mempool_alloc() and mempool_free()
  59  * functions. This function might sleep. Both the alloc_fn() and the free_fn()
  60  * functions might sleep - as long as the mempool_alloc() function is not called
  61  * from IRQ contexts.
  62  */
  63 mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
  64                                 mempool_free_t *free_fn, void *pool_data)
  65 {
  66         return  mempool_create_node(min_nr,alloc_fn,free_fn, pool_data,-1);
  67 }
  68 EXPORT_SYMBOL(mempool_create);
  69
  70 mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn,
  71                         mempool_free_t *free_fn, void *pool_data, int node_id)
  72 {
  73         mempool_t *pool;
  74         pool = kmalloc_node(sizeof(*pool), GFP_KERNEL | __GFP_ZERO, node_id);
  75         if (!pool)
  76                 return NULL;
  77         pool->elements = kmalloc_node(min_nr * sizeof(void *),
  78                                         GFP_KERNEL, node_id);
  79         if (!pool->elements) {
  80                 kfree(pool);
  81                 return NULL;
  82         }
  83         spin_lock_init(&pool->lock);
  84         pool->min_nr = min_nr;
  85         pool->pool_data = pool_data;
  86         init_waitqueue_head(&pool->wait);
  87         pool->alloc = alloc_fn;
  88         pool->free = free_fn;
  89
  90         /*
  91          * First pre-allocate the guaranteed number of buffers.
  92          */
  93         while (pool->curr_nr < pool->min_nr) {
  94                 void *element;
  95
  96                 element = pool->alloc(GFP_KERNEL, pool->pool_data);
  97                 if (unlikely(!element)) {
  98                         mempool_destroy(pool);
  99                         return NULL;
 100                 }
 101                 add_element(pool, element);
 102         }
 103         return pool;
 104 }
 105 EXPORT_SYMBOL(mempool_create_node);
 106
 107 /**
 108  * mempool_resize - resize an existing memory pool
 109  * @pool:       pointer to the memory pool which was allocated via
 110  *              mempool_create().
 111  * @new_min_nr: the new minimum number of elements guaranteed to be
 112  *              allocated for this pool.
 113  * @gfp_mask:   the usual allocation bitmask.
 114  *
 115  * This function shrinks/grows the pool. In the case of growing,
 116  * it cannot be guaranteed that the pool will be grown to the new
 117  * size immediately, but new mempool_free() calls will refill it.
 118  *
 119  * Note, the caller must guarantee that no mempool_destroy is called
 120  * while this function is running. mempool_alloc() & mempool_free()
 121  * might be called (eg. from IRQ contexts) while this function executes.
 122  */
 123 int mempool_resize(mempool_t *pool, int new_min_nr, gfp_t gfp_mask)
 124 {
 125         void *element;
 126         void **new_elements;
 127         unsigned long flags;
 128
 129         BUG_ON(new_min_nr <= 0);
 130
 131         spin_lock_irqsave(&pool->lock, flags);
 132         if (new_min_nr <= pool->min_nr) {
 133                 while (new_min_nr < pool->curr_nr) {
 134                         element = remove_element(pool);
 135                         spin_unlock_irqrestore(&pool->lock, flags);
 136                         pool->free(element, pool->pool_data);
 137                         spin_lock_irqsave(&pool->lock, flags);
 138                 }
 139                 pool->min_nr = new_min_nr;
 140                 goto out_unlock;
 141         }
 142         spin_unlock_irqrestore(&pool->lock, flags);
 143
 144         /* Grow the pool */
 145         new_elements = kmalloc(new_min_nr * sizeof(*new_elements), gfp_mask);
 146         if (!new_elements)
 147                 return -ENOMEM;
 148
 149         spin_lock_irqsave(&pool->lock, flags);
 150         if (unlikely(new_min_nr <= pool->min_nr)) {
 151                 /* Raced, other resize will do our work */
 152                 spin_unlock_irqrestore(&pool->lock, flags);
 153                 kfree(new_elements);
 154                 goto out;
 155         }
 156         memcpy(new_elements, pool->elements,
 157                         pool->curr_nr * sizeof(*new_elements));
 158         kfree(pool->elements);
 159         pool->elements = new_elements;
 160         pool->min_nr = new_min_nr;
 161
 162         while (pool->curr_nr < pool->min_nr) {
 163                 spin_unlock_irqrestore(&pool->lock, flags);
 164                 element = pool->alloc(gfp_mask, pool->pool_data);
 165                 if (!element)
 166                         goto out;
 167                 spin_lock_irqsave(&pool->lock, flags);
 168                 if (pool->curr_nr < pool->min_nr) {
 169                         add_element(pool, element);
 170                 } else {
 171                         spin_unlock_irqrestore(&pool->lock, flags);
 172                         pool->free(element, pool->pool_data);   /* Raced */
 173                         goto out;
 174                 }
 175         }
 176 out_unlock:
 177         spin_unlock_irqrestore(&pool->lock, flags);
 178 out:
 179         return 0;
 180 }
 181 EXPORT_SYMBOL(mempool_resize);
 182
 183 /**
 184  * mempool_alloc - allocate an element from a specific memory pool
 185  * @pool:      pointer to the memory pool which was allocated via
 186  *             mempool_create().
 187  * @gfp_mask:  the usual allocation bitmask.
 188  *
 189  * this function only sleeps if the alloc_fn() function sleeps or
 190  * returns NULL. Note that due to preallocation, this function
 191  * *never* fails when called from process contexts. (it might
 192  * fail if called from an IRQ context.)
 193  */
 194 void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
 195 {
 196         void *element;
 197         unsigned long flags;
 198         wait_queue_t wait;
 199         gfp_t gfp_temp;
 200
 201         might_sleep_if(gfp_mask & __GFP_WAIT);
 202
 203         gfp_mask |= __GFP_NOMEMALLOC;   /* don't allocate emergency reserves */
 204         gfp_mask |= __GFP_NORETRY;      /* don't loop in __alloc_pages */
 205         gfp_mask |= __GFP_NOWARN;       /* failures are OK */
 206
 207         gfp_temp = gfp_mask & ~(__GFP_WAIT|__GFP_IO);
 208
 209 repeat_alloc:
 210
 211         element = pool->alloc(gfp_temp, pool->pool_data);
 212         if (likely(element != NULL))
 213                 return element;
 214
 215         spin_lock_irqsave(&pool->lock, flags);
 216         if (likely(pool->curr_nr)) {
 217                 element = remove_element(pool);
 218                 spin_unlock_irqrestore(&pool->lock, flags);
 219                 /* paired with rmb in mempool_free(), read comment there */
 220                 smp_wmb();
 221                 return element;
 222         }
 223
 224         /*
 225          * We use gfp mask w/o __GFP_WAIT or IO for the first round.  If
 226          * alloc failed with that and @pool was empty, retry immediately.
 227          */
 228         if (gfp_temp != gfp_mask) {
 229                 spin_unlock_irqrestore(&pool->lock, flags);
 230                 gfp_temp = gfp_mask;
 231                 goto repeat_alloc;
 232         }
 233
 234         /* We must not sleep if !__GFP_WAIT */
 235         if (!(gfp_mask & __GFP_WAIT)) {
 236                 spin_unlock_irqrestore(&pool->lock, flags);
 237                 return NULL;
 238         }
 239
 240         /* Let's wait for someone else to return an element to @pool */
 241         init_wait(&wait);
 242         prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
 243
 244         spin_unlock_irqrestore(&pool->lock, flags);
 245
 246         /*
 247          * FIXME: this should be io_schedule().  The timeout is there as a
 248          * workaround for some DM problems in 2.6.18.
 249          */
 250         io_schedule_timeout(5*HZ);
 251
 252         finish_wait(&pool->wait, &wait);
 253         goto repeat_alloc;
 254 }
 255 EXPORT_SYMBOL(mempool_alloc);
 256
 257 /**
 258  * mempool_free - return an element to the pool.
 259  * @element:   pool element pointer.
 260  * @pool:      pointer to the memory pool which was allocated via
 261  *             mempool_create().
 262  *
 263  * this function only sleeps if the free_fn() function sleeps.
 264  */
 265 void mempool_free(void *element, mempool_t *pool)
 266 {
 267         unsigned long flags;
 268
 269         if (unlikely(element == NULL))
 270                 return;
 271
 272         /*
 273          * Paired with the wmb in mempool_alloc().  The preceding read is
 274          * for @element and the following @pool->curr_nr.  This ensures
 275          * that the visible value of @pool->curr_nr is from after the
 276          * allocation of @element.  This is necessary for fringe cases
 277          * where @element was passed to this task without going through
 278          * barriers.
 279          *
 280          * For example, assume @p is %NULL at the beginning and one task
 281          * performs "p = mempool_alloc(...);" while another task is doing
 282          * "while (!p) cpu_relax(); mempool_free(p, ...);".  This function
 283          * may end up using curr_nr value which is from before allocation
 284          * of @p without the following rmb.
 285          */
 286         smp_rmb();
 287
 288         /*
 289          * For correctness, we need a test which is guaranteed to trigger
 290          * if curr_nr + #allocated == min_nr.  Testing curr_nr < min_nr
 291          * without locking achieves that and refilling as soon as possible
 292          * is desirable.
 293          *
 294          * Because curr_nr visible here is always a value after the
 295          * allocation of @element, any task which decremented curr_nr below
 296          * min_nr is guaranteed to see curr_nr < min_nr unless curr_nr gets
 297          * incremented to min_nr afterwards.  If curr_nr gets incremented
 298          * to min_nr after the allocation of @element, the elements
 299          * allocated after that are subject to the same guarantee.
 300          *
 301          * Waiters happen iff curr_nr is 0 and the above guarantee also
 302          * ensures that there will be frees which return elements to the
 303          * pool waking up the waiters.
 304          */
 305         if (pool->curr_nr < pool->min_nr) {
 306                 spin_lock_irqsave(&pool->lock, flags);
 307                 if (pool->curr_nr < pool->min_nr) {
 308                         add_element(pool, element);
 309                         spin_unlock_irqrestore(&pool->lock, flags);
 310                         wake_up(&pool->wait);
 311                         return;
 312                 }
 313                 spin_unlock_irqrestore(&pool->lock, flags);
 314         }
 315         pool->free(element, pool->pool_data);
 316 }
 317 EXPORT_SYMBOL(mempool_free);
 318
 319 /*
 320  * A commonly used alloc and free fn.
 321  */
 322 void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data)
 323 {
 324         struct kmem_cache *mem = pool_data;
 325         return kmem_cache_alloc(mem, gfp_mask);
 326 }
 327 EXPORT_SYMBOL(mempool_alloc_slab);
 328
 329 void mempool_free_slab(void *element, void *pool_data)
 330 {
 331         struct kmem_cache *mem = pool_data;
 332         kmem_cache_free(mem, element);
 333 }
 334 EXPORT_SYMBOL(mempool_free_slab);
 335
 336 /*
 337  * A commonly used alloc and free fn that kmalloc/kfrees the amount of memory
 338  * specified by pool_data
 339  */
 340 void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data)
 341 {
 342         size_t size = (size_t)pool_data;
 343         return kmalloc(size, gfp_mask);
 344 }
 345 EXPORT_SYMBOL(mempool_kmalloc);
 346
 347 void mempool_kfree(void *element, void *pool_data)
 348 {
 349         kfree(element);
 350 }
 351 EXPORT_SYMBOL(mempool_kfree);
 352
 353 /*
 354  * A simple mempool-backed page allocator that allocates pages
 355  * of the order specified by pool_data.
 356  */
 357 void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data)
 358 {
 359         int order = (int)(long)pool_data;
 360         return alloc_pages(gfp_mask, order);
 361 }
 362 EXPORT_SYMBOL(mempool_alloc_pages);
 363
 364 void mempool_free_pages(void *element, void *pool_data)
 365 {
 366         int order = (int)(long)pool_data;
 367         __free_pages(element, order);
 368 }
 369 EXPORT_SYMBOL(mempool_free_pages);