mm/slub.c: convert vnsprintf-static to va_format
[GitHub/exynos8895/android_kernel_samsung_universal8895.git] / mm / slub.c
CommitLineData
81819f0f
CL
1/*
2 * SLUB: A slab allocator that limits cache line use instead of queuing
3 * objects in per cpu and per node lists.
4 *
881db7fb
CL
5 * The allocator synchronizes using per slab locks or atomic operatios
6 * and only uses a centralized lock to manage a pool of partial slabs.
81819f0f 7 *
cde53535 8 * (C) 2007 SGI, Christoph Lameter
881db7fb 9 * (C) 2011 Linux Foundation, Christoph Lameter
81819f0f
CL
10 */
11
12#include <linux/mm.h>
1eb5ac64 13#include <linux/swap.h> /* struct reclaim_state */
81819f0f
CL
14#include <linux/module.h>
15#include <linux/bit_spinlock.h>
16#include <linux/interrupt.h>
17#include <linux/bitops.h>
18#include <linux/slab.h>
97d06609 19#include "slab.h"
7b3c3a50 20#include <linux/proc_fs.h>
3ac38faa 21#include <linux/notifier.h>
81819f0f 22#include <linux/seq_file.h>
5a896d9e 23#include <linux/kmemcheck.h>
81819f0f
CL
24#include <linux/cpu.h>
25#include <linux/cpuset.h>
26#include <linux/mempolicy.h>
27#include <linux/ctype.h>
3ac7fe5a 28#include <linux/debugobjects.h>
81819f0f 29#include <linux/kallsyms.h>
b9049e23 30#include <linux/memory.h>
f8bd2258 31#include <linux/math64.h>
773ff60e 32#include <linux/fault-inject.h>
bfa71457 33#include <linux/stacktrace.h>
4de900b4 34#include <linux/prefetch.h>
2633d7a0 35#include <linux/memcontrol.h>
81819f0f 36
4a92379b
RK
37#include <trace/events/kmem.h>
38
072bb0aa
MG
39#include "internal.h"
40
81819f0f
CL
41/*
42 * Lock order:
18004c5d 43 * 1. slab_mutex (Global Mutex)
881db7fb
CL
44 * 2. node->list_lock
45 * 3. slab_lock(page) (Only on some arches and for debugging)
81819f0f 46 *
18004c5d 47 * slab_mutex
881db7fb 48 *
18004c5d 49 * The role of the slab_mutex is to protect the list of all the slabs
881db7fb
CL
50 * and to synchronize major metadata changes to slab cache structures.
51 *
52 * The slab_lock is only used for debugging and on arches that do not
53 * have the ability to do a cmpxchg_double. It only protects the second
54 * double word in the page struct. Meaning
55 * A. page->freelist -> List of object free in a page
56 * B. page->counters -> Counters of objects
57 * C. page->frozen -> frozen state
58 *
59 * If a slab is frozen then it is exempt from list management. It is not
60 * on any list. The processor that froze the slab is the one who can
61 * perform list operations on the page. Other processors may put objects
62 * onto the freelist but the processor that froze the slab is the only
63 * one that can retrieve the objects from the page's freelist.
81819f0f
CL
64 *
65 * The list_lock protects the partial and full list on each node and
66 * the partial slab counter. If taken then no new slabs may be added or
67 * removed from the lists nor make the number of partial slabs be modified.
68 * (Note that the total number of slabs is an atomic value that may be
69 * modified without taking the list lock).
70 *
71 * The list_lock is a centralized lock and thus we avoid taking it as
72 * much as possible. As long as SLUB does not have to handle partial
73 * slabs, operations can continue without any centralized lock. F.e.
74 * allocating a long series of objects that fill up slabs does not require
75 * the list lock.
81819f0f
CL
76 * Interrupts are disabled during allocation and deallocation in order to
77 * make the slab allocator safe to use in the context of an irq. In addition
78 * interrupts are disabled to ensure that the processor does not change
79 * while handling per_cpu slabs, due to kernel preemption.
80 *
81 * SLUB assigns one slab for allocation to each processor.
82 * Allocations only occur from these slabs called cpu slabs.
83 *
672bba3a
CL
84 * Slabs with free elements are kept on a partial list and during regular
85 * operations no list for full slabs is used. If an object in a full slab is
81819f0f 86 * freed then the slab will show up again on the partial lists.
672bba3a
CL
87 * We track full slabs for debugging purposes though because otherwise we
88 * cannot scan all objects.
81819f0f
CL
89 *
90 * Slabs are freed when they become empty. Teardown and setup is
91 * minimal so we rely on the page allocators per cpu caches for
92 * fast frees and allocs.
93 *
94 * Overloading of page flags that are otherwise used for LRU management.
95 *
4b6f0750
CL
96 * PageActive The slab is frozen and exempt from list processing.
97 * This means that the slab is dedicated to a purpose
98 * such as satisfying allocations for a specific
99 * processor. Objects may be freed in the slab while
100 * it is frozen but slab_free will then skip the usual
101 * list operations. It is up to the processor holding
102 * the slab to integrate the slab into the slab lists
103 * when the slab is no longer needed.
104 *
105 * One use of this flag is to mark slabs that are
106 * used for allocations. Then such a slab becomes a cpu
107 * slab. The cpu slab may be equipped with an additional
dfb4f096 108 * freelist that allows lockless access to
894b8788
CL
109 * free objects in addition to the regular freelist
110 * that requires the slab lock.
81819f0f
CL
111 *
112 * PageError Slab requires special handling due to debug
113 * options set. This moves slab handling out of
894b8788 114 * the fast path and disables lockless freelists.
81819f0f
CL
115 */
116
af537b0a
CL
117static inline int kmem_cache_debug(struct kmem_cache *s)
118{
5577bd8a 119#ifdef CONFIG_SLUB_DEBUG
af537b0a 120 return unlikely(s->flags & SLAB_DEBUG_FLAGS);
5577bd8a 121#else
af537b0a 122 return 0;
5577bd8a 123#endif
af537b0a 124}
5577bd8a 125
345c905d
JK
126static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
127{
128#ifdef CONFIG_SLUB_CPU_PARTIAL
129 return !kmem_cache_debug(s);
130#else
131 return false;
132#endif
133}
134
81819f0f
CL
135/*
136 * Issues still to be resolved:
137 *
81819f0f
CL
138 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
139 *
81819f0f
CL
140 * - Variable sizing of the per node arrays
141 */
142
143/* Enable to test recovery from slab corruption on boot */
144#undef SLUB_RESILIENCY_TEST
145
b789ef51
CL
146/* Enable to log cmpxchg failures */
147#undef SLUB_DEBUG_CMPXCHG
148
2086d26a
CL
149/*
150 * Mininum number of partial slabs. These will be left on the partial
151 * lists even if they are empty. kmem_cache_shrink may reclaim them.
152 */
76be8950 153#define MIN_PARTIAL 5
e95eed57 154
2086d26a
CL
155/*
156 * Maximum number of desirable partial slabs.
157 * The existence of more partial slabs makes kmem_cache_shrink
721ae22a 158 * sort the partial list by the number of objects in use.
2086d26a
CL
159 */
160#define MAX_PARTIAL 10
161
81819f0f
CL
162#define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \
163 SLAB_POISON | SLAB_STORE_USER)
672bba3a 164
fa5ec8a1 165/*
3de47213
DR
166 * Debugging flags that require metadata to be stored in the slab. These get
167 * disabled when slub_debug=O is used and a cache's min order increases with
168 * metadata.
fa5ec8a1 169 */
3de47213 170#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
fa5ec8a1 171
81819f0f
CL
172/*
173 * Set of flags that will prevent slab merging
174 */
175#define SLUB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
4c13dd3b
DM
176 SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE | \
177 SLAB_FAILSLAB)
81819f0f
CL
178
179#define SLUB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \
5a896d9e 180 SLAB_CACHE_DMA | SLAB_NOTRACK)
81819f0f 181
210b5c06
CG
182#define OO_SHIFT 16
183#define OO_MASK ((1 << OO_SHIFT) - 1)
50d5c41c 184#define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */
210b5c06 185
81819f0f 186/* Internal SLUB flags */
f90ec390 187#define __OBJECT_POISON 0x80000000UL /* Poison object */
b789ef51 188#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */
81819f0f 189
81819f0f
CL
190#ifdef CONFIG_SMP
191static struct notifier_block slab_notifier;
192#endif
193
02cbc874
CL
194/*
195 * Tracking user of a slab.
196 */
d6543e39 197#define TRACK_ADDRS_COUNT 16
02cbc874 198struct track {
ce71e27c 199 unsigned long addr; /* Called from address */
d6543e39
BG
200#ifdef CONFIG_STACKTRACE
201 unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */
202#endif
02cbc874
CL
203 int cpu; /* Was running on cpu */
204 int pid; /* Pid context */
205 unsigned long when; /* When did the operation occur */
206};
207
208enum track_item { TRACK_ALLOC, TRACK_FREE };
209
ab4d5ed5 210#ifdef CONFIG_SYSFS
81819f0f
CL
211static int sysfs_slab_add(struct kmem_cache *);
212static int sysfs_slab_alias(struct kmem_cache *, const char *);
107dab5c 213static void memcg_propagate_slab_attrs(struct kmem_cache *s);
81819f0f 214#else
0c710013
CL
215static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
216static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
217 { return 0; }
107dab5c 218static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { }
81819f0f
CL
219#endif
220
4fdccdfb 221static inline void stat(const struct kmem_cache *s, enum stat_item si)
8ff12cfc
CL
222{
223#ifdef CONFIG_SLUB_STATS
88da03a6
CL
224 /*
225 * The rmw is racy on a preemptible kernel but this is acceptable, so
226 * avoid this_cpu_add()'s irq-disable overhead.
227 */
228 raw_cpu_inc(s->cpu_slab->stat[si]);
8ff12cfc
CL
229#endif
230}
231
81819f0f
CL
232/********************************************************************
233 * Core slab cache functions
234 *******************************************************************/
235
81819f0f
CL
236static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
237{
81819f0f 238 return s->node[node];
81819f0f
CL
239}
240
6446faa2 241/* Verify that a pointer has an address that is valid within a slab page */
02cbc874
CL
242static inline int check_valid_pointer(struct kmem_cache *s,
243 struct page *page, const void *object)
244{
245 void *base;
246
a973e9dd 247 if (!object)
02cbc874
CL
248 return 1;
249
a973e9dd 250 base = page_address(page);
39b26464 251 if (object < base || object >= base + page->objects * s->size ||
02cbc874
CL
252 (object - base) % s->size) {
253 return 0;
254 }
255
256 return 1;
257}
258
7656c72b
CL
259static inline void *get_freepointer(struct kmem_cache *s, void *object)
260{
261 return *(void **)(object + s->offset);
262}
263
0ad9500e
ED
264static void prefetch_freepointer(const struct kmem_cache *s, void *object)
265{
266 prefetch(object + s->offset);
267}
268
1393d9a1
CL
269static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
270{
271 void *p;
272
273#ifdef CONFIG_DEBUG_PAGEALLOC
274 probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p));
275#else
276 p = get_freepointer(s, object);
277#endif
278 return p;
279}
280
7656c72b
CL
281static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
282{
283 *(void **)(object + s->offset) = fp;
284}
285
286/* Loop over all objects in a slab */
224a88be
CL
287#define for_each_object(__p, __s, __addr, __objects) \
288 for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\
7656c72b
CL
289 __p += (__s)->size)
290
7656c72b
CL
291/* Determine object index from a given position */
292static inline int slab_index(void *p, struct kmem_cache *s, void *addr)
293{
294 return (p - addr) / s->size;
295}
296
d71f606f
MK
297static inline size_t slab_ksize(const struct kmem_cache *s)
298{
299#ifdef CONFIG_SLUB_DEBUG
300 /*
301 * Debugging requires use of the padding between object
302 * and whatever may come after it.
303 */
304 if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
3b0efdfa 305 return s->object_size;
d71f606f
MK
306
307#endif
308 /*
309 * If we have the need to store the freelist pointer
310 * back there or track user information then we can
311 * only use the space before that information.
312 */
313 if (s->flags & (SLAB_DESTROY_BY_RCU | SLAB_STORE_USER))
314 return s->inuse;
315 /*
316 * Else we can use all the padding etc for the allocation
317 */
318 return s->size;
319}
320
ab9a0f19
LJ
321static inline int order_objects(int order, unsigned long size, int reserved)
322{
323 return ((PAGE_SIZE << order) - reserved) / size;
324}
325
834f3d11 326static inline struct kmem_cache_order_objects oo_make(int order,
ab9a0f19 327 unsigned long size, int reserved)
834f3d11
CL
328{
329 struct kmem_cache_order_objects x = {
ab9a0f19 330 (order << OO_SHIFT) + order_objects(order, size, reserved)
834f3d11
CL
331 };
332
333 return x;
334}
335
336static inline int oo_order(struct kmem_cache_order_objects x)
337{
210b5c06 338 return x.x >> OO_SHIFT;
834f3d11
CL
339}
340
341static inline int oo_objects(struct kmem_cache_order_objects x)
342{
210b5c06 343 return x.x & OO_MASK;
834f3d11
CL
344}
345
881db7fb
CL
346/*
347 * Per slab locking using the pagelock
348 */
349static __always_inline void slab_lock(struct page *page)
350{
351 bit_spin_lock(PG_locked, &page->flags);
352}
353
354static __always_inline void slab_unlock(struct page *page)
355{
356 __bit_spin_unlock(PG_locked, &page->flags);
357}
358
a0320865
DH
359static inline void set_page_slub_counters(struct page *page, unsigned long counters_new)
360{
361 struct page tmp;
362 tmp.counters = counters_new;
363 /*
364 * page->counters can cover frozen/inuse/objects as well
365 * as page->_count. If we assign to ->counters directly
366 * we run the risk of losing updates to page->_count, so
367 * be careful and only assign to the fields we need.
368 */
369 page->frozen = tmp.frozen;
370 page->inuse = tmp.inuse;
371 page->objects = tmp.objects;
372}
373
1d07171c
CL
374/* Interrupts must be disabled (for the fallback code to work right) */
375static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
376 void *freelist_old, unsigned long counters_old,
377 void *freelist_new, unsigned long counters_new,
378 const char *n)
379{
380 VM_BUG_ON(!irqs_disabled());
2565409f
HC
381#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
382 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
1d07171c 383 if (s->flags & __CMPXCHG_DOUBLE) {
cdcd6298 384 if (cmpxchg_double(&page->freelist, &page->counters,
1d07171c
CL
385 freelist_old, counters_old,
386 freelist_new, counters_new))
387 return 1;
388 } else
389#endif
390 {
391 slab_lock(page);
d0e0ac97
CG
392 if (page->freelist == freelist_old &&
393 page->counters == counters_old) {
1d07171c 394 page->freelist = freelist_new;
a0320865 395 set_page_slub_counters(page, counters_new);
1d07171c
CL
396 slab_unlock(page);
397 return 1;
398 }
399 slab_unlock(page);
400 }
401
402 cpu_relax();
403 stat(s, CMPXCHG_DOUBLE_FAIL);
404
405#ifdef SLUB_DEBUG_CMPXCHG
f9f58285 406 pr_info("%s %s: cmpxchg double redo ", n, s->name);
1d07171c
CL
407#endif
408
409 return 0;
410}
411
b789ef51
CL
412static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
413 void *freelist_old, unsigned long counters_old,
414 void *freelist_new, unsigned long counters_new,
415 const char *n)
416{
2565409f
HC
417#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
418 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
b789ef51 419 if (s->flags & __CMPXCHG_DOUBLE) {
cdcd6298 420 if (cmpxchg_double(&page->freelist, &page->counters,
b789ef51
CL
421 freelist_old, counters_old,
422 freelist_new, counters_new))
423 return 1;
424 } else
425#endif
426 {
1d07171c
CL
427 unsigned long flags;
428
429 local_irq_save(flags);
881db7fb 430 slab_lock(page);
d0e0ac97
CG
431 if (page->freelist == freelist_old &&
432 page->counters == counters_old) {
b789ef51 433 page->freelist = freelist_new;
a0320865 434 set_page_slub_counters(page, counters_new);
881db7fb 435 slab_unlock(page);
1d07171c 436 local_irq_restore(flags);
b789ef51
CL
437 return 1;
438 }
881db7fb 439 slab_unlock(page);
1d07171c 440 local_irq_restore(flags);
b789ef51
CL
441 }
442
443 cpu_relax();
444 stat(s, CMPXCHG_DOUBLE_FAIL);
445
446#ifdef SLUB_DEBUG_CMPXCHG
f9f58285 447 pr_info("%s %s: cmpxchg double redo ", n, s->name);
b789ef51
CL
448#endif
449
450 return 0;
451}
452
41ecc55b 453#ifdef CONFIG_SLUB_DEBUG
5f80b13a
CL
454/*
455 * Determine a map of object in use on a page.
456 *
881db7fb 457 * Node listlock must be held to guarantee that the page does
5f80b13a
CL
458 * not vanish from under us.
459 */
460static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map)
461{
462 void *p;
463 void *addr = page_address(page);
464
465 for (p = page->freelist; p; p = get_freepointer(s, p))
466 set_bit(slab_index(p, s, addr), map);
467}
468
41ecc55b
CL
469/*
470 * Debug settings:
471 */
f0630fff
CL
472#ifdef CONFIG_SLUB_DEBUG_ON
473static int slub_debug = DEBUG_DEFAULT_FLAGS;
474#else
41ecc55b 475static int slub_debug;
f0630fff 476#endif
41ecc55b
CL
477
478static char *slub_debug_slabs;
fa5ec8a1 479static int disable_higher_order_debug;
41ecc55b 480
81819f0f
CL
481/*
482 * Object debugging
483 */
484static void print_section(char *text, u8 *addr, unsigned int length)
485{
ffc79d28
SAS
486 print_hex_dump(KERN_ERR, text, DUMP_PREFIX_ADDRESS, 16, 1, addr,
487 length, 1);
81819f0f
CL
488}
489
81819f0f
CL
490static struct track *get_track(struct kmem_cache *s, void *object,
491 enum track_item alloc)
492{
493 struct track *p;
494
495 if (s->offset)
496 p = object + s->offset + sizeof(void *);
497 else
498 p = object + s->inuse;
499
500 return p + alloc;
501}
502
503static void set_track(struct kmem_cache *s, void *object,
ce71e27c 504 enum track_item alloc, unsigned long addr)
81819f0f 505{
1a00df4a 506 struct track *p = get_track(s, object, alloc);
81819f0f 507
81819f0f 508 if (addr) {
d6543e39
BG
509#ifdef CONFIG_STACKTRACE
510 struct stack_trace trace;
511 int i;
512
513 trace.nr_entries = 0;
514 trace.max_entries = TRACK_ADDRS_COUNT;
515 trace.entries = p->addrs;
516 trace.skip = 3;
517 save_stack_trace(&trace);
518
519 /* See rant in lockdep.c */
520 if (trace.nr_entries != 0 &&
521 trace.entries[trace.nr_entries - 1] == ULONG_MAX)
522 trace.nr_entries--;
523
524 for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++)
525 p->addrs[i] = 0;
526#endif
81819f0f
CL
527 p->addr = addr;
528 p->cpu = smp_processor_id();
88e4ccf2 529 p->pid = current->pid;
81819f0f
CL
530 p->when = jiffies;
531 } else
532 memset(p, 0, sizeof(struct track));
533}
534
81819f0f
CL
535static void init_tracking(struct kmem_cache *s, void *object)
536{
24922684
CL
537 if (!(s->flags & SLAB_STORE_USER))
538 return;
539
ce71e27c
EGM
540 set_track(s, object, TRACK_FREE, 0UL);
541 set_track(s, object, TRACK_ALLOC, 0UL);
81819f0f
CL
542}
543
544static void print_track(const char *s, struct track *t)
545{
546 if (!t->addr)
547 return;
548
f9f58285
FF
549 pr_err("INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
550 s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid);
d6543e39
BG
551#ifdef CONFIG_STACKTRACE
552 {
553 int i;
554 for (i = 0; i < TRACK_ADDRS_COUNT; i++)
555 if (t->addrs[i])
f9f58285 556 pr_err("\t%pS\n", (void *)t->addrs[i]);
d6543e39
BG
557 else
558 break;
559 }
560#endif
24922684
CL
561}
562
563static void print_tracking(struct kmem_cache *s, void *object)
564{
565 if (!(s->flags & SLAB_STORE_USER))
566 return;
567
568 print_track("Allocated", get_track(s, object, TRACK_ALLOC));
569 print_track("Freed", get_track(s, object, TRACK_FREE));
570}
571
572static void print_page_info(struct page *page)
573{
f9f58285 574 pr_err("INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n",
d0e0ac97 575 page, page->objects, page->inuse, page->freelist, page->flags);
24922684
CL
576
577}
578
579static void slab_bug(struct kmem_cache *s, char *fmt, ...)
580{
ecc42fbe 581 struct va_format vaf;
24922684 582 va_list args;
24922684
CL
583
584 va_start(args, fmt);
ecc42fbe
FF
585 vaf.fmt = fmt;
586 vaf.va = &args;
f9f58285 587 pr_err("=============================================================================\n");
ecc42fbe 588 pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf);
f9f58285 589 pr_err("-----------------------------------------------------------------------------\n\n");
645df230 590
373d4d09 591 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
ecc42fbe 592 va_end(args);
81819f0f
CL
593}
594
24922684
CL
595static void slab_fix(struct kmem_cache *s, char *fmt, ...)
596{
ecc42fbe 597 struct va_format vaf;
24922684 598 va_list args;
24922684
CL
599
600 va_start(args, fmt);
ecc42fbe
FF
601 vaf.fmt = fmt;
602 vaf.va = &args;
603 pr_err("FIX %s: %pV\n", s->name, &vaf);
24922684 604 va_end(args);
24922684
CL
605}
606
607static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
81819f0f
CL
608{
609 unsigned int off; /* Offset of last byte */
a973e9dd 610 u8 *addr = page_address(page);
24922684
CL
611
612 print_tracking(s, p);
613
614 print_page_info(page);
615
f9f58285
FF
616 pr_err("INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
617 p, p - addr, get_freepointer(s, p));
24922684
CL
618
619 if (p > addr + 16)
ffc79d28 620 print_section("Bytes b4 ", p - 16, 16);
81819f0f 621
3b0efdfa 622 print_section("Object ", p, min_t(unsigned long, s->object_size,
ffc79d28 623 PAGE_SIZE));
81819f0f 624 if (s->flags & SLAB_RED_ZONE)
3b0efdfa
CL
625 print_section("Redzone ", p + s->object_size,
626 s->inuse - s->object_size);
81819f0f 627
81819f0f
CL
628 if (s->offset)
629 off = s->offset + sizeof(void *);
630 else
631 off = s->inuse;
632
24922684 633 if (s->flags & SLAB_STORE_USER)
81819f0f 634 off += 2 * sizeof(struct track);
81819f0f
CL
635
636 if (off != s->size)
637 /* Beginning of the filler is the free pointer */
ffc79d28 638 print_section("Padding ", p + off, s->size - off);
24922684
CL
639
640 dump_stack();
81819f0f
CL
641}
642
643static void object_err(struct kmem_cache *s, struct page *page,
644 u8 *object, char *reason)
645{
3dc50637 646 slab_bug(s, "%s", reason);
24922684 647 print_trailer(s, page, object);
81819f0f
CL
648}
649
d0e0ac97
CG
650static void slab_err(struct kmem_cache *s, struct page *page,
651 const char *fmt, ...)
81819f0f
CL
652{
653 va_list args;
654 char buf[100];
655
24922684
CL
656 va_start(args, fmt);
657 vsnprintf(buf, sizeof(buf), fmt, args);
81819f0f 658 va_end(args);
3dc50637 659 slab_bug(s, "%s", buf);
24922684 660 print_page_info(page);
81819f0f
CL
661 dump_stack();
662}
663
f7cb1933 664static void init_object(struct kmem_cache *s, void *object, u8 val)
81819f0f
CL
665{
666 u8 *p = object;
667
668 if (s->flags & __OBJECT_POISON) {
3b0efdfa
CL
669 memset(p, POISON_FREE, s->object_size - 1);
670 p[s->object_size - 1] = POISON_END;
81819f0f
CL
671 }
672
673 if (s->flags & SLAB_RED_ZONE)
3b0efdfa 674 memset(p + s->object_size, val, s->inuse - s->object_size);
81819f0f
CL
675}
676
24922684
CL
677static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
678 void *from, void *to)
679{
680 slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
681 memset(from, data, to - from);
682}
683
684static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
685 u8 *object, char *what,
06428780 686 u8 *start, unsigned int value, unsigned int bytes)
24922684
CL
687{
688 u8 *fault;
689 u8 *end;
690
79824820 691 fault = memchr_inv(start, value, bytes);
24922684
CL
692 if (!fault)
693 return 1;
694
695 end = start + bytes;
696 while (end > fault && end[-1] == value)
697 end--;
698
699 slab_bug(s, "%s overwritten", what);
f9f58285 700 pr_err("INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
24922684
CL
701 fault, end - 1, fault[0], value);
702 print_trailer(s, page, object);
703
704 restore_bytes(s, what, value, fault, end);
705 return 0;
81819f0f
CL
706}
707
81819f0f
CL
708/*
709 * Object layout:
710 *
711 * object address
712 * Bytes of the object to be managed.
713 * If the freepointer may overlay the object then the free
714 * pointer is the first word of the object.
672bba3a 715 *
81819f0f
CL
716 * Poisoning uses 0x6b (POISON_FREE) and the last byte is
717 * 0xa5 (POISON_END)
718 *
3b0efdfa 719 * object + s->object_size
81819f0f 720 * Padding to reach word boundary. This is also used for Redzoning.
672bba3a 721 * Padding is extended by another word if Redzoning is enabled and
3b0efdfa 722 * object_size == inuse.
672bba3a 723 *
81819f0f
CL
724 * We fill with 0xbb (RED_INACTIVE) for inactive objects and with
725 * 0xcc (RED_ACTIVE) for objects in use.
726 *
727 * object + s->inuse
672bba3a
CL
728 * Meta data starts here.
729 *
81819f0f
CL
730 * A. Free pointer (if we cannot overwrite object on free)
731 * B. Tracking data for SLAB_STORE_USER
672bba3a 732 * C. Padding to reach required alignment boundary or at mininum
6446faa2 733 * one word if debugging is on to be able to detect writes
672bba3a
CL
734 * before the word boundary.
735 *
736 * Padding is done using 0x5a (POISON_INUSE)
81819f0f
CL
737 *
738 * object + s->size
672bba3a 739 * Nothing is used beyond s->size.
81819f0f 740 *
3b0efdfa 741 * If slabcaches are merged then the object_size and inuse boundaries are mostly
672bba3a 742 * ignored. And therefore no slab options that rely on these boundaries
81819f0f
CL
743 * may be used with merged slabcaches.
744 */
745
81819f0f
CL
746static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
747{
748 unsigned long off = s->inuse; /* The end of info */
749
750 if (s->offset)
751 /* Freepointer is placed after the object. */
752 off += sizeof(void *);
753
754 if (s->flags & SLAB_STORE_USER)
755 /* We also have user information there */
756 off += 2 * sizeof(struct track);
757
758 if (s->size == off)
759 return 1;
760
24922684
CL
761 return check_bytes_and_report(s, page, p, "Object padding",
762 p + off, POISON_INUSE, s->size - off);
81819f0f
CL
763}
764
39b26464 765/* Check the pad bytes at the end of a slab page */
81819f0f
CL
766static int slab_pad_check(struct kmem_cache *s, struct page *page)
767{
24922684
CL
768 u8 *start;
769 u8 *fault;
770 u8 *end;
771 int length;
772 int remainder;
81819f0f
CL
773
774 if (!(s->flags & SLAB_POISON))
775 return 1;
776
a973e9dd 777 start = page_address(page);
ab9a0f19 778 length = (PAGE_SIZE << compound_order(page)) - s->reserved;
39b26464
CL
779 end = start + length;
780 remainder = length % s->size;
81819f0f
CL
781 if (!remainder)
782 return 1;
783
79824820 784 fault = memchr_inv(end - remainder, POISON_INUSE, remainder);
24922684
CL
785 if (!fault)
786 return 1;
787 while (end > fault && end[-1] == POISON_INUSE)
788 end--;
789
790 slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
ffc79d28 791 print_section("Padding ", end - remainder, remainder);
24922684 792
8a3d271d 793 restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end);
24922684 794 return 0;
81819f0f
CL
795}
796
797static int check_object(struct kmem_cache *s, struct page *page,
f7cb1933 798 void *object, u8 val)
81819f0f
CL
799{
800 u8 *p = object;
3b0efdfa 801 u8 *endobject = object + s->object_size;
81819f0f
CL
802
803 if (s->flags & SLAB_RED_ZONE) {
24922684 804 if (!check_bytes_and_report(s, page, object, "Redzone",
3b0efdfa 805 endobject, val, s->inuse - s->object_size))
81819f0f 806 return 0;
81819f0f 807 } else {
3b0efdfa 808 if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
3adbefee 809 check_bytes_and_report(s, page, p, "Alignment padding",
d0e0ac97
CG
810 endobject, POISON_INUSE,
811 s->inuse - s->object_size);
3adbefee 812 }
81819f0f
CL
813 }
814
815 if (s->flags & SLAB_POISON) {
f7cb1933 816 if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
24922684 817 (!check_bytes_and_report(s, page, p, "Poison", p,
3b0efdfa 818 POISON_FREE, s->object_size - 1) ||
24922684 819 !check_bytes_and_report(s, page, p, "Poison",
3b0efdfa 820 p + s->object_size - 1, POISON_END, 1)))
81819f0f 821 return 0;
81819f0f
CL
822 /*
823 * check_pad_bytes cleans up on its own.
824 */
825 check_pad_bytes(s, page, p);
826 }
827
f7cb1933 828 if (!s->offset && val == SLUB_RED_ACTIVE)
81819f0f
CL
829 /*
830 * Object and freepointer overlap. Cannot check
831 * freepointer while object is allocated.
832 */
833 return 1;
834
835 /* Check free pointer validity */
836 if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
837 object_err(s, page, p, "Freepointer corrupt");
838 /*
9f6c708e 839 * No choice but to zap it and thus lose the remainder
81819f0f 840 * of the free objects in this slab. May cause
672bba3a 841 * another error because the object count is now wrong.
81819f0f 842 */
a973e9dd 843 set_freepointer(s, p, NULL);
81819f0f
CL
844 return 0;
845 }
846 return 1;
847}
848
849static int check_slab(struct kmem_cache *s, struct page *page)
850{
39b26464
CL
851 int maxobj;
852
81819f0f
CL
853 VM_BUG_ON(!irqs_disabled());
854
855 if (!PageSlab(page)) {
24922684 856 slab_err(s, page, "Not a valid slab page");
81819f0f
CL
857 return 0;
858 }
39b26464 859
ab9a0f19 860 maxobj = order_objects(compound_order(page), s->size, s->reserved);
39b26464
CL
861 if (page->objects > maxobj) {
862 slab_err(s, page, "objects %u > max %u",
863 s->name, page->objects, maxobj);
864 return 0;
865 }
866 if (page->inuse > page->objects) {
24922684 867 slab_err(s, page, "inuse %u > max %u",
39b26464 868 s->name, page->inuse, page->objects);
81819f0f
CL
869 return 0;
870 }
871 /* Slab_pad_check fixes things up after itself */
872 slab_pad_check(s, page);
873 return 1;
874}
875
876/*
672bba3a
CL
877 * Determine if a certain object on a page is on the freelist. Must hold the
878 * slab lock to guarantee that the chains are in a consistent state.
81819f0f
CL
879 */
880static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
881{
882 int nr = 0;
881db7fb 883 void *fp;
81819f0f 884 void *object = NULL;
224a88be 885 unsigned long max_objects;
81819f0f 886
881db7fb 887 fp = page->freelist;
39b26464 888 while (fp && nr <= page->objects) {
81819f0f
CL
889 if (fp == search)
890 return 1;
891 if (!check_valid_pointer(s, page, fp)) {
892 if (object) {
893 object_err(s, page, object,
894 "Freechain corrupt");
a973e9dd 895 set_freepointer(s, object, NULL);
81819f0f 896 } else {
24922684 897 slab_err(s, page, "Freepointer corrupt");
a973e9dd 898 page->freelist = NULL;
39b26464 899 page->inuse = page->objects;
24922684 900 slab_fix(s, "Freelist cleared");
81819f0f
CL
901 return 0;
902 }
903 break;
904 }
905 object = fp;
906 fp = get_freepointer(s, object);
907 nr++;
908 }
909
ab9a0f19 910 max_objects = order_objects(compound_order(page), s->size, s->reserved);
210b5c06
CG
911 if (max_objects > MAX_OBJS_PER_PAGE)
912 max_objects = MAX_OBJS_PER_PAGE;
224a88be
CL
913
914 if (page->objects != max_objects) {
915 slab_err(s, page, "Wrong number of objects. Found %d but "
916 "should be %d", page->objects, max_objects);
917 page->objects = max_objects;
918 slab_fix(s, "Number of objects adjusted.");
919 }
39b26464 920 if (page->inuse != page->objects - nr) {
70d71228 921 slab_err(s, page, "Wrong object count. Counter is %d but "
39b26464
CL
922 "counted were %d", page->inuse, page->objects - nr);
923 page->inuse = page->objects - nr;
24922684 924 slab_fix(s, "Object count adjusted.");
81819f0f
CL
925 }
926 return search == NULL;
927}
928
0121c619
CL
929static void trace(struct kmem_cache *s, struct page *page, void *object,
930 int alloc)
3ec09742
CL
931{
932 if (s->flags & SLAB_TRACE) {
f9f58285 933 pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
3ec09742
CL
934 s->name,
935 alloc ? "alloc" : "free",
936 object, page->inuse,
937 page->freelist);
938
939 if (!alloc)
d0e0ac97
CG
940 print_section("Object ", (void *)object,
941 s->object_size);
3ec09742
CL
942
943 dump_stack();
944 }
945}
946
c016b0bd
CL
947/*
948 * Hooks for other subsystems that check memory allocations. In a typical
949 * production configuration these hooks all should produce no code at all.
950 */
d56791b3
RB
951static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
952{
953 kmemleak_alloc(ptr, size, 1, flags);
954}
955
956static inline void kfree_hook(const void *x)
957{
958 kmemleak_free(x);
959}
960
c016b0bd
CL
961static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
962{
c1d50836 963 flags &= gfp_allowed_mask;
c016b0bd
CL
964 lockdep_trace_alloc(flags);
965 might_sleep_if(flags & __GFP_WAIT);
966
3b0efdfa 967 return should_failslab(s->object_size, flags, s->flags);
c016b0bd
CL
968}
969
d0e0ac97
CG
970static inline void slab_post_alloc_hook(struct kmem_cache *s,
971 gfp_t flags, void *object)
c016b0bd 972{
c1d50836 973 flags &= gfp_allowed_mask;
b3d41885 974 kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));
3b0efdfa 975 kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags);
c016b0bd
CL
976}
977
978static inline void slab_free_hook(struct kmem_cache *s, void *x)
979{
980 kmemleak_free_recursive(x, s->flags);
c016b0bd 981
d3f661d6 982 /*
d1756174 983 * Trouble is that we may no longer disable interrupts in the fast path
d3f661d6
CL
984 * So in order to make the debug calls that expect irqs to be
985 * disabled we need to disable interrupts temporarily.
986 */
987#if defined(CONFIG_KMEMCHECK) || defined(CONFIG_LOCKDEP)
988 {
989 unsigned long flags;
990
991 local_irq_save(flags);
3b0efdfa
CL
992 kmemcheck_slab_free(s, x, s->object_size);
993 debug_check_no_locks_freed(x, s->object_size);
d3f661d6
CL
994 local_irq_restore(flags);
995 }
996#endif
f9b615de 997 if (!(s->flags & SLAB_DEBUG_OBJECTS))
3b0efdfa 998 debug_check_no_obj_freed(x, s->object_size);
c016b0bd
CL
999}
1000
643b1138 1001/*
672bba3a 1002 * Tracking of fully allocated slabs for debugging purposes.
643b1138 1003 */
5cc6eee8
CL
1004static void add_full(struct kmem_cache *s,
1005 struct kmem_cache_node *n, struct page *page)
643b1138 1006{
5cc6eee8
CL
1007 if (!(s->flags & SLAB_STORE_USER))
1008 return;
1009
255d0884 1010 lockdep_assert_held(&n->list_lock);
643b1138 1011 list_add(&page->lru, &n->full);
643b1138
CL
1012}
1013
c65c1877 1014static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page)
643b1138 1015{
643b1138
CL
1016 if (!(s->flags & SLAB_STORE_USER))
1017 return;
1018
255d0884 1019 lockdep_assert_held(&n->list_lock);
643b1138 1020 list_del(&page->lru);
643b1138
CL
1021}
1022
0f389ec6
CL
1023/* Tracking of the number of slabs for debugging purposes */
1024static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1025{
1026 struct kmem_cache_node *n = get_node(s, node);
1027
1028 return atomic_long_read(&n->nr_slabs);
1029}
1030
26c02cf0
AB
1031static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1032{
1033 return atomic_long_read(&n->nr_slabs);
1034}
1035
205ab99d 1036static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
0f389ec6
CL
1037{
1038 struct kmem_cache_node *n = get_node(s, node);
1039
1040 /*
1041 * May be called early in order to allocate a slab for the
1042 * kmem_cache_node structure. Solve the chicken-egg
1043 * dilemma by deferring the increment of the count during
1044 * bootstrap (see early_kmem_cache_node_alloc).
1045 */
338b2642 1046 if (likely(n)) {
0f389ec6 1047 atomic_long_inc(&n->nr_slabs);
205ab99d
CL
1048 atomic_long_add(objects, &n->total_objects);
1049 }
0f389ec6 1050}
205ab99d 1051static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
0f389ec6
CL
1052{
1053 struct kmem_cache_node *n = get_node(s, node);
1054
1055 atomic_long_dec(&n->nr_slabs);
205ab99d 1056 atomic_long_sub(objects, &n->total_objects);
0f389ec6
CL
1057}
1058
1059/* Object debug checks for alloc/free paths */
3ec09742
CL
1060static void setup_object_debug(struct kmem_cache *s, struct page *page,
1061 void *object)
1062{
1063 if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)))
1064 return;
1065
f7cb1933 1066 init_object(s, object, SLUB_RED_INACTIVE);
3ec09742
CL
1067 init_tracking(s, object);
1068}
1069
d0e0ac97
CG
1070static noinline int alloc_debug_processing(struct kmem_cache *s,
1071 struct page *page,
ce71e27c 1072 void *object, unsigned long addr)
81819f0f
CL
1073{
1074 if (!check_slab(s, page))
1075 goto bad;
1076
81819f0f
CL
1077 if (!check_valid_pointer(s, page, object)) {
1078 object_err(s, page, object, "Freelist Pointer check fails");
70d71228 1079 goto bad;
81819f0f
CL
1080 }
1081
f7cb1933 1082 if (!check_object(s, page, object, SLUB_RED_INACTIVE))
81819f0f 1083 goto bad;
81819f0f 1084
3ec09742
CL
1085 /* Success perform special debug activities for allocs */
1086 if (s->flags & SLAB_STORE_USER)
1087 set_track(s, object, TRACK_ALLOC, addr);
1088 trace(s, page, object, 1);
f7cb1933 1089 init_object(s, object, SLUB_RED_ACTIVE);
81819f0f 1090 return 1;
3ec09742 1091
81819f0f
CL
1092bad:
1093 if (PageSlab(page)) {
1094 /*
1095 * If this is a slab page then lets do the best we can
1096 * to avoid issues in the future. Marking all objects
672bba3a 1097 * as used avoids touching the remaining objects.
81819f0f 1098 */
24922684 1099 slab_fix(s, "Marking all objects used");
39b26464 1100 page->inuse = page->objects;
a973e9dd 1101 page->freelist = NULL;
81819f0f
CL
1102 }
1103 return 0;
1104}
1105
19c7ff9e
CL
1106static noinline struct kmem_cache_node *free_debug_processing(
1107 struct kmem_cache *s, struct page *page, void *object,
1108 unsigned long addr, unsigned long *flags)
81819f0f 1109{
19c7ff9e 1110 struct kmem_cache_node *n = get_node(s, page_to_nid(page));
5c2e4bbb 1111
19c7ff9e 1112 spin_lock_irqsave(&n->list_lock, *flags);
881db7fb
CL
1113 slab_lock(page);
1114
81819f0f
CL
1115 if (!check_slab(s, page))
1116 goto fail;
1117
1118 if (!check_valid_pointer(s, page, object)) {
70d71228 1119 slab_err(s, page, "Invalid object pointer 0x%p", object);
81819f0f
CL
1120 goto fail;
1121 }
1122
1123 if (on_freelist(s, page, object)) {
24922684 1124 object_err(s, page, object, "Object already free");
81819f0f
CL
1125 goto fail;
1126 }
1127
f7cb1933 1128 if (!check_object(s, page, object, SLUB_RED_ACTIVE))
5c2e4bbb 1129 goto out;
81819f0f 1130
1b4f59e3 1131 if (unlikely(s != page->slab_cache)) {
3adbefee 1132 if (!PageSlab(page)) {
70d71228
CL
1133 slab_err(s, page, "Attempt to free object(0x%p) "
1134 "outside of slab", object);
1b4f59e3 1135 } else if (!page->slab_cache) {
f9f58285
FF
1136 pr_err("SLUB <none>: no slab for object 0x%p.\n",
1137 object);
70d71228 1138 dump_stack();
06428780 1139 } else
24922684
CL
1140 object_err(s, page, object,
1141 "page slab pointer corrupt.");
81819f0f
CL
1142 goto fail;
1143 }
3ec09742 1144
3ec09742
CL
1145 if (s->flags & SLAB_STORE_USER)
1146 set_track(s, object, TRACK_FREE, addr);
1147 trace(s, page, object, 0);
f7cb1933 1148 init_object(s, object, SLUB_RED_INACTIVE);
5c2e4bbb 1149out:
881db7fb 1150 slab_unlock(page);
19c7ff9e
CL
1151 /*
1152 * Keep node_lock to preserve integrity
1153 * until the object is actually freed
1154 */
1155 return n;
3ec09742 1156
81819f0f 1157fail:
19c7ff9e
CL
1158 slab_unlock(page);
1159 spin_unlock_irqrestore(&n->list_lock, *flags);
24922684 1160 slab_fix(s, "Object at 0x%p not freed", object);
19c7ff9e 1161 return NULL;
81819f0f
CL
1162}
1163
41ecc55b
CL
1164static int __init setup_slub_debug(char *str)
1165{
f0630fff
CL
1166 slub_debug = DEBUG_DEFAULT_FLAGS;
1167 if (*str++ != '=' || !*str)
1168 /*
1169 * No options specified. Switch on full debugging.
1170 */
1171 goto out;
1172
1173 if (*str == ',')
1174 /*
1175 * No options but restriction on slabs. This means full
1176 * debugging for slabs matching a pattern.
1177 */
1178 goto check_slabs;
1179
fa5ec8a1
DR
1180 if (tolower(*str) == 'o') {
1181 /*
1182 * Avoid enabling debugging on caches if its minimum order
1183 * would increase as a result.
1184 */
1185 disable_higher_order_debug = 1;
1186 goto out;
1187 }
1188
f0630fff
CL
1189 slub_debug = 0;
1190 if (*str == '-')
1191 /*
1192 * Switch off all debugging measures.
1193 */
1194 goto out;
1195
1196 /*
1197 * Determine which debug features should be switched on
1198 */
06428780 1199 for (; *str && *str != ','; str++) {
f0630fff
CL
1200 switch (tolower(*str)) {
1201 case 'f':
1202 slub_debug |= SLAB_DEBUG_FREE;
1203 break;
1204 case 'z':
1205 slub_debug |= SLAB_RED_ZONE;
1206 break;
1207 case 'p':
1208 slub_debug |= SLAB_POISON;
1209 break;
1210 case 'u':
1211 slub_debug |= SLAB_STORE_USER;
1212 break;
1213 case 't':
1214 slub_debug |= SLAB_TRACE;
1215 break;
4c13dd3b
DM
1216 case 'a':
1217 slub_debug |= SLAB_FAILSLAB;
1218 break;
f0630fff 1219 default:
f9f58285
FF
1220 pr_err("slub_debug option '%c' unknown. skipped\n",
1221 *str);
f0630fff 1222 }
41ecc55b
CL
1223 }
1224
f0630fff 1225check_slabs:
41ecc55b
CL
1226 if (*str == ',')
1227 slub_debug_slabs = str + 1;
f0630fff 1228out:
41ecc55b
CL
1229 return 1;
1230}
1231
1232__setup("slub_debug", setup_slub_debug);
1233
3b0efdfa 1234static unsigned long kmem_cache_flags(unsigned long object_size,
ba0268a8 1235 unsigned long flags, const char *name,
51cc5068 1236 void (*ctor)(void *))
41ecc55b
CL
1237{
1238 /*
e153362a 1239 * Enable debugging if selected on the kernel commandline.
41ecc55b 1240 */
c6f58d9b
CL
1241 if (slub_debug && (!slub_debug_slabs || (name &&
1242 !strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs)))))
3de47213 1243 flags |= slub_debug;
ba0268a8
CL
1244
1245 return flags;
41ecc55b
CL
1246}
1247#else
3ec09742
CL
1248static inline void setup_object_debug(struct kmem_cache *s,
1249 struct page *page, void *object) {}
41ecc55b 1250
3ec09742 1251static inline int alloc_debug_processing(struct kmem_cache *s,
ce71e27c 1252 struct page *page, void *object, unsigned long addr) { return 0; }
41ecc55b 1253
19c7ff9e
CL
1254static inline struct kmem_cache_node *free_debug_processing(
1255 struct kmem_cache *s, struct page *page, void *object,
1256 unsigned long addr, unsigned long *flags) { return NULL; }
41ecc55b 1257
41ecc55b
CL
1258static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
1259 { return 1; }
1260static inline int check_object(struct kmem_cache *s, struct page *page,
f7cb1933 1261 void *object, u8 val) { return 1; }
5cc6eee8
CL
1262static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
1263 struct page *page) {}
c65c1877
PZ
1264static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
1265 struct page *page) {}
3b0efdfa 1266static inline unsigned long kmem_cache_flags(unsigned long object_size,
ba0268a8 1267 unsigned long flags, const char *name,
51cc5068 1268 void (*ctor)(void *))
ba0268a8
CL
1269{
1270 return flags;
1271}
41ecc55b 1272#define slub_debug 0
0f389ec6 1273
fdaa45e9
IM
1274#define disable_higher_order_debug 0
1275
0f389ec6
CL
1276static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1277 { return 0; }
26c02cf0
AB
1278static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1279 { return 0; }
205ab99d
CL
1280static inline void inc_slabs_node(struct kmem_cache *s, int node,
1281 int objects) {}
1282static inline void dec_slabs_node(struct kmem_cache *s, int node,
1283 int objects) {}
7d550c56 1284
d56791b3
RB
1285static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
1286{
1287 kmemleak_alloc(ptr, size, 1, flags);
1288}
1289
1290static inline void kfree_hook(const void *x)
1291{
1292 kmemleak_free(x);
1293}
1294
7d550c56
CL
1295static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
1296 { return 0; }
1297
1298static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags,
d56791b3
RB
1299 void *object)
1300{
1301 kmemleak_alloc_recursive(object, s->object_size, 1, s->flags,
1302 flags & gfp_allowed_mask);
1303}
7d550c56 1304
d56791b3
RB
1305static inline void slab_free_hook(struct kmem_cache *s, void *x)
1306{
1307 kmemleak_free_recursive(x, s->flags);
1308}
7d550c56 1309
ab4d5ed5 1310#endif /* CONFIG_SLUB_DEBUG */
205ab99d 1311
81819f0f
CL
1312/*
1313 * Slab allocation and freeing
1314 */
65c3376a
CL
1315static inline struct page *alloc_slab_page(gfp_t flags, int node,
1316 struct kmem_cache_order_objects oo)
1317{
1318 int order = oo_order(oo);
1319
b1eeab67
VN
1320 flags |= __GFP_NOTRACK;
1321
2154a336 1322 if (node == NUMA_NO_NODE)
65c3376a
CL
1323 return alloc_pages(flags, order);
1324 else
6b65aaf3 1325 return alloc_pages_exact_node(node, flags, order);
65c3376a
CL
1326}
1327
81819f0f
CL
1328static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
1329{
06428780 1330 struct page *page;
834f3d11 1331 struct kmem_cache_order_objects oo = s->oo;
ba52270d 1332 gfp_t alloc_gfp;
81819f0f 1333
7e0528da
CL
1334 flags &= gfp_allowed_mask;
1335
1336 if (flags & __GFP_WAIT)
1337 local_irq_enable();
1338
b7a49f0d 1339 flags |= s->allocflags;
e12ba74d 1340
ba52270d
PE
1341 /*
1342 * Let the initial higher-order allocation fail under memory pressure
1343 * so we fall-back to the minimum order allocation.
1344 */
1345 alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
1346
1347 page = alloc_slab_page(alloc_gfp, node, oo);
65c3376a
CL
1348 if (unlikely(!page)) {
1349 oo = s->min;
80c3a998 1350 alloc_gfp = flags;
65c3376a
CL
1351 /*
1352 * Allocation may have failed due to fragmentation.
1353 * Try a lower order alloc if possible
1354 */
80c3a998 1355 page = alloc_slab_page(alloc_gfp, node, oo);
81819f0f 1356
7e0528da
CL
1357 if (page)
1358 stat(s, ORDER_FALLBACK);
65c3376a 1359 }
5a896d9e 1360
737b719e 1361 if (kmemcheck_enabled && page
5086c389 1362 && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
b1eeab67
VN
1363 int pages = 1 << oo_order(oo);
1364
80c3a998 1365 kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node);
b1eeab67
VN
1366
1367 /*
1368 * Objects from caches that have a constructor don't get
1369 * cleared when they're allocated, so we need to do it here.
1370 */
1371 if (s->ctor)
1372 kmemcheck_mark_uninitialized_pages(page, pages);
1373 else
1374 kmemcheck_mark_unallocated_pages(page, pages);
5a896d9e
VN
1375 }
1376
737b719e
DR
1377 if (flags & __GFP_WAIT)
1378 local_irq_disable();
1379 if (!page)
1380 return NULL;
1381
834f3d11 1382 page->objects = oo_objects(oo);
81819f0f
CL
1383 mod_zone_page_state(page_zone(page),
1384 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
1385 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
65c3376a 1386 1 << oo_order(oo));
81819f0f
CL
1387
1388 return page;
1389}
1390
1391static void setup_object(struct kmem_cache *s, struct page *page,
1392 void *object)
1393{
3ec09742 1394 setup_object_debug(s, page, object);
4f104934 1395 if (unlikely(s->ctor))
51cc5068 1396 s->ctor(object);
81819f0f
CL
1397}
1398
1399static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
1400{
1401 struct page *page;
81819f0f 1402 void *start;
81819f0f
CL
1403 void *last;
1404 void *p;
1f458cbf 1405 int order;
81819f0f 1406
6cb06229 1407 BUG_ON(flags & GFP_SLAB_BUG_MASK);
81819f0f 1408
6cb06229
CL
1409 page = allocate_slab(s,
1410 flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
81819f0f
CL
1411 if (!page)
1412 goto out;
1413
1f458cbf 1414 order = compound_order(page);
205ab99d 1415 inc_slabs_node(s, page_to_nid(page), page->objects);
1f458cbf 1416 memcg_bind_pages(s, order);
1b4f59e3 1417 page->slab_cache = s;
c03f94cc 1418 __SetPageSlab(page);
072bb0aa
MG
1419 if (page->pfmemalloc)
1420 SetPageSlabPfmemalloc(page);
81819f0f
CL
1421
1422 start = page_address(page);
81819f0f
CL
1423
1424 if (unlikely(s->flags & SLAB_POISON))
1f458cbf 1425 memset(start, POISON_INUSE, PAGE_SIZE << order);
81819f0f
CL
1426
1427 last = start;
224a88be 1428 for_each_object(p, s, start, page->objects) {
81819f0f
CL
1429 setup_object(s, page, last);
1430 set_freepointer(s, last, p);
1431 last = p;
1432 }
1433 setup_object(s, page, last);
a973e9dd 1434 set_freepointer(s, last, NULL);
81819f0f
CL
1435
1436 page->freelist = start;
e6e82ea1 1437 page->inuse = page->objects;
8cb0a506 1438 page->frozen = 1;
81819f0f 1439out:
81819f0f
CL
1440 return page;
1441}
1442
1443static void __free_slab(struct kmem_cache *s, struct page *page)
1444{
834f3d11
CL
1445 int order = compound_order(page);
1446 int pages = 1 << order;
81819f0f 1447
af537b0a 1448 if (kmem_cache_debug(s)) {
81819f0f
CL
1449 void *p;
1450
1451 slab_pad_check(s, page);
224a88be
CL
1452 for_each_object(p, s, page_address(page),
1453 page->objects)
f7cb1933 1454 check_object(s, page, p, SLUB_RED_INACTIVE);
81819f0f
CL
1455 }
1456
b1eeab67 1457 kmemcheck_free_shadow(page, compound_order(page));
5a896d9e 1458
81819f0f
CL
1459 mod_zone_page_state(page_zone(page),
1460 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
1461 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
06428780 1462 -pages);
81819f0f 1463
072bb0aa 1464 __ClearPageSlabPfmemalloc(page);
49bd5221 1465 __ClearPageSlab(page);
1f458cbf
GC
1466
1467 memcg_release_pages(s, order);
22b751c3 1468 page_mapcount_reset(page);
1eb5ac64
NP
1469 if (current->reclaim_state)
1470 current->reclaim_state->reclaimed_slab += pages;
d79923fa 1471 __free_memcg_kmem_pages(page, order);
81819f0f
CL
1472}
1473
da9a638c
LJ
1474#define need_reserve_slab_rcu \
1475 (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head))
1476
81819f0f
CL
1477static void rcu_free_slab(struct rcu_head *h)
1478{
1479 struct page *page;
1480
da9a638c
LJ
1481 if (need_reserve_slab_rcu)
1482 page = virt_to_head_page(h);
1483 else
1484 page = container_of((struct list_head *)h, struct page, lru);
1485
1b4f59e3 1486 __free_slab(page->slab_cache, page);
81819f0f
CL
1487}
1488
1489static void free_slab(struct kmem_cache *s, struct page *page)
1490{
1491 if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) {
da9a638c
LJ
1492 struct rcu_head *head;
1493
1494 if (need_reserve_slab_rcu) {
1495 int order = compound_order(page);
1496 int offset = (PAGE_SIZE << order) - s->reserved;
1497
1498 VM_BUG_ON(s->reserved != sizeof(*head));
1499 head = page_address(page) + offset;
1500 } else {
1501 /*
1502 * RCU free overloads the RCU head over the LRU
1503 */
1504 head = (void *)&page->lru;
1505 }
81819f0f
CL
1506
1507 call_rcu(head, rcu_free_slab);
1508 } else
1509 __free_slab(s, page);
1510}
1511
1512static void discard_slab(struct kmem_cache *s, struct page *page)
1513{
205ab99d 1514 dec_slabs_node(s, page_to_nid(page), page->objects);
81819f0f
CL
1515 free_slab(s, page);
1516}
1517
1518/*
5cc6eee8 1519 * Management of partially allocated slabs.
81819f0f 1520 */
1e4dd946
SR
1521static inline void
1522__add_partial(struct kmem_cache_node *n, struct page *page, int tail)
81819f0f 1523{
e95eed57 1524 n->nr_partial++;
136333d1 1525 if (tail == DEACTIVATE_TO_TAIL)
7c2e132c
CL
1526 list_add_tail(&page->lru, &n->partial);
1527 else
1528 list_add(&page->lru, &n->partial);
81819f0f
CL
1529}
1530
1e4dd946
SR
1531static inline void add_partial(struct kmem_cache_node *n,
1532 struct page *page, int tail)
62e346a8 1533{
c65c1877 1534 lockdep_assert_held(&n->list_lock);
1e4dd946
SR
1535 __add_partial(n, page, tail);
1536}
c65c1877 1537
1e4dd946
SR
1538static inline void
1539__remove_partial(struct kmem_cache_node *n, struct page *page)
1540{
62e346a8
CL
1541 list_del(&page->lru);
1542 n->nr_partial--;
1543}
1544
1e4dd946
SR
1545static inline void remove_partial(struct kmem_cache_node *n,
1546 struct page *page)
1547{
1548 lockdep_assert_held(&n->list_lock);
1549 __remove_partial(n, page);
1550}
1551
81819f0f 1552/*
7ced3719
CL
1553 * Remove slab from the partial list, freeze it and
1554 * return the pointer to the freelist.
81819f0f 1555 *
497b66f2 1556 * Returns a list of objects or NULL if it fails.
81819f0f 1557 */
497b66f2 1558static inline void *acquire_slab(struct kmem_cache *s,
acd19fd1 1559 struct kmem_cache_node *n, struct page *page,
633b0764 1560 int mode, int *objects)
81819f0f 1561{
2cfb7455
CL
1562 void *freelist;
1563 unsigned long counters;
1564 struct page new;
1565
c65c1877
PZ
1566 lockdep_assert_held(&n->list_lock);
1567
2cfb7455
CL
1568 /*
1569 * Zap the freelist and set the frozen bit.
1570 * The old freelist is the list of objects for the
1571 * per cpu allocation list.
1572 */
7ced3719
CL
1573 freelist = page->freelist;
1574 counters = page->counters;
1575 new.counters = counters;
633b0764 1576 *objects = new.objects - new.inuse;
23910c50 1577 if (mode) {
7ced3719 1578 new.inuse = page->objects;
23910c50
PE
1579 new.freelist = NULL;
1580 } else {
1581 new.freelist = freelist;
1582 }
2cfb7455 1583
a0132ac0 1584 VM_BUG_ON(new.frozen);
7ced3719 1585 new.frozen = 1;
2cfb7455 1586
7ced3719 1587 if (!__cmpxchg_double_slab(s, page,
2cfb7455 1588 freelist, counters,
02d7633f 1589 new.freelist, new.counters,
7ced3719 1590 "acquire_slab"))
7ced3719 1591 return NULL;
2cfb7455
CL
1592
1593 remove_partial(n, page);
7ced3719 1594 WARN_ON(!freelist);
49e22585 1595 return freelist;
81819f0f
CL
1596}
1597
633b0764 1598static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);
8ba00bb6 1599static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);
49e22585 1600
81819f0f 1601/*
672bba3a 1602 * Try to allocate a partial slab from a specific node.
81819f0f 1603 */
8ba00bb6
JK
1604static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
1605 struct kmem_cache_cpu *c, gfp_t flags)
81819f0f 1606{
49e22585
CL
1607 struct page *page, *page2;
1608 void *object = NULL;
633b0764
JK
1609 int available = 0;
1610 int objects;
81819f0f
CL
1611
1612 /*
1613 * Racy check. If we mistakenly see no partial slabs then we
1614 * just allocate an empty slab. If we mistakenly try to get a
672bba3a
CL
1615 * partial slab and there is none available then get_partials()
1616 * will return NULL.
81819f0f
CL
1617 */
1618 if (!n || !n->nr_partial)
1619 return NULL;
1620
1621 spin_lock(&n->list_lock);
49e22585 1622 list_for_each_entry_safe(page, page2, &n->partial, lru) {
8ba00bb6 1623 void *t;
49e22585 1624
8ba00bb6
JK
1625 if (!pfmemalloc_match(page, flags))
1626 continue;
1627
633b0764 1628 t = acquire_slab(s, n, page, object == NULL, &objects);
49e22585
CL
1629 if (!t)
1630 break;
1631
633b0764 1632 available += objects;
12d79634 1633 if (!object) {
49e22585 1634 c->page = page;
49e22585 1635 stat(s, ALLOC_FROM_PARTIAL);
49e22585 1636 object = t;
49e22585 1637 } else {
633b0764 1638 put_cpu_partial(s, page, 0);
8028dcea 1639 stat(s, CPU_PARTIAL_NODE);
49e22585 1640 }
345c905d
JK
1641 if (!kmem_cache_has_cpu_partial(s)
1642 || available > s->cpu_partial / 2)
49e22585
CL
1643 break;
1644
497b66f2 1645 }
81819f0f 1646 spin_unlock(&n->list_lock);
497b66f2 1647 return object;
81819f0f
CL
1648}
1649
1650/*
672bba3a 1651 * Get a page from somewhere. Search in increasing NUMA distances.
81819f0f 1652 */
de3ec035 1653static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
acd19fd1 1654 struct kmem_cache_cpu *c)
81819f0f
CL
1655{
1656#ifdef CONFIG_NUMA
1657 struct zonelist *zonelist;
dd1a239f 1658 struct zoneref *z;
54a6eb5c
MG
1659 struct zone *zone;
1660 enum zone_type high_zoneidx = gfp_zone(flags);
497b66f2 1661 void *object;
cc9a6c87 1662 unsigned int cpuset_mems_cookie;
81819f0f
CL
1663
1664 /*
672bba3a
CL
1665 * The defrag ratio allows a configuration of the tradeoffs between
1666 * inter node defragmentation and node local allocations. A lower
1667 * defrag_ratio increases the tendency to do local allocations
1668 * instead of attempting to obtain partial slabs from other nodes.
81819f0f 1669 *
672bba3a
CL
1670 * If the defrag_ratio is set to 0 then kmalloc() always
1671 * returns node local objects. If the ratio is higher then kmalloc()
1672 * may return off node objects because partial slabs are obtained
1673 * from other nodes and filled up.
81819f0f 1674 *
6446faa2 1675 * If /sys/kernel/slab/xx/defrag_ratio is set to 100 (which makes
672bba3a
CL
1676 * defrag_ratio = 1000) then every (well almost) allocation will
1677 * first attempt to defrag slab caches on other nodes. This means
1678 * scanning over all nodes to look for partial slabs which may be
1679 * expensive if we do it every time we are trying to find a slab
1680 * with available objects.
81819f0f 1681 */
9824601e
CL
1682 if (!s->remote_node_defrag_ratio ||
1683 get_cycles() % 1024 > s->remote_node_defrag_ratio)
81819f0f
CL
1684 return NULL;
1685
cc9a6c87 1686 do {
d26914d1 1687 cpuset_mems_cookie = read_mems_allowed_begin();
2a389610 1688 zonelist = node_zonelist(mempolicy_slab_node(), flags);
cc9a6c87
MG
1689 for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
1690 struct kmem_cache_node *n;
1691
1692 n = get_node(s, zone_to_nid(zone));
1693
1694 if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
1695 n->nr_partial > s->min_partial) {
8ba00bb6 1696 object = get_partial_node(s, n, c, flags);
cc9a6c87
MG
1697 if (object) {
1698 /*
d26914d1
MG
1699 * Don't check read_mems_allowed_retry()
1700 * here - if mems_allowed was updated in
1701 * parallel, that was a harmless race
1702 * between allocation and the cpuset
1703 * update
cc9a6c87 1704 */
cc9a6c87
MG
1705 return object;
1706 }
c0ff7453 1707 }
81819f0f 1708 }
d26914d1 1709 } while (read_mems_allowed_retry(cpuset_mems_cookie));
81819f0f
CL
1710#endif
1711 return NULL;
1712}
1713
1714/*
1715 * Get a partial page, lock it and return it.
1716 */
497b66f2 1717static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,
acd19fd1 1718 struct kmem_cache_cpu *c)
81819f0f 1719{
497b66f2 1720 void *object;
2154a336 1721 int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node;
81819f0f 1722
8ba00bb6 1723 object = get_partial_node(s, get_node(s, searchnode), c, flags);
497b66f2
CL
1724 if (object || node != NUMA_NO_NODE)
1725 return object;
81819f0f 1726
acd19fd1 1727 return get_any_partial(s, flags, c);
81819f0f
CL
1728}
1729
8a5ec0ba
CL
1730#ifdef CONFIG_PREEMPT
1731/*
1732 * Calculate the next globally unique transaction for disambiguiation
1733 * during cmpxchg. The transactions start with the cpu number and are then
1734 * incremented by CONFIG_NR_CPUS.
1735 */
1736#define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS)
1737#else
1738/*
1739 * No preemption supported therefore also no need to check for
1740 * different cpus.
1741 */
1742#define TID_STEP 1
1743#endif
1744
1745static inline unsigned long next_tid(unsigned long tid)
1746{
1747 return tid + TID_STEP;
1748}
1749
1750static inline unsigned int tid_to_cpu(unsigned long tid)
1751{
1752 return tid % TID_STEP;
1753}
1754
1755static inline unsigned long tid_to_event(unsigned long tid)
1756{
1757 return tid / TID_STEP;
1758}
1759
1760static inline unsigned int init_tid(int cpu)
1761{
1762 return cpu;
1763}
1764
1765static inline void note_cmpxchg_failure(const char *n,
1766 const struct kmem_cache *s, unsigned long tid)
1767{
1768#ifdef SLUB_DEBUG_CMPXCHG
1769 unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);
1770
f9f58285 1771 pr_info("%s %s: cmpxchg redo ", n, s->name);
8a5ec0ba
CL
1772
1773#ifdef CONFIG_PREEMPT
1774 if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
f9f58285 1775 pr_warn("due to cpu change %d -> %d\n",
8a5ec0ba
CL
1776 tid_to_cpu(tid), tid_to_cpu(actual_tid));
1777 else
1778#endif
1779 if (tid_to_event(tid) != tid_to_event(actual_tid))
f9f58285 1780 pr_warn("due to cpu running other code. Event %ld->%ld\n",
8a5ec0ba
CL
1781 tid_to_event(tid), tid_to_event(actual_tid));
1782 else
f9f58285 1783 pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
8a5ec0ba
CL
1784 actual_tid, tid, next_tid(tid));
1785#endif
4fdccdfb 1786 stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
8a5ec0ba
CL
1787}
1788
788e1aad 1789static void init_kmem_cache_cpus(struct kmem_cache *s)
8a5ec0ba 1790{
8a5ec0ba
CL
1791 int cpu;
1792
1793 for_each_possible_cpu(cpu)
1794 per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
8a5ec0ba 1795}
2cfb7455 1796
81819f0f
CL
1797/*
1798 * Remove the cpu slab
1799 */
d0e0ac97
CG
1800static void deactivate_slab(struct kmem_cache *s, struct page *page,
1801 void *freelist)
81819f0f 1802{
2cfb7455 1803 enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
2cfb7455
CL
1804 struct kmem_cache_node *n = get_node(s, page_to_nid(page));
1805 int lock = 0;
1806 enum slab_modes l = M_NONE, m = M_NONE;
2cfb7455 1807 void *nextfree;
136333d1 1808 int tail = DEACTIVATE_TO_HEAD;
2cfb7455
CL
1809 struct page new;
1810 struct page old;
1811
1812 if (page->freelist) {
84e554e6 1813 stat(s, DEACTIVATE_REMOTE_FREES);
136333d1 1814 tail = DEACTIVATE_TO_TAIL;
2cfb7455
CL
1815 }
1816
894b8788 1817 /*
2cfb7455
CL
1818 * Stage one: Free all available per cpu objects back
1819 * to the page freelist while it is still frozen. Leave the
1820 * last one.
1821 *
1822 * There is no need to take the list->lock because the page
1823 * is still frozen.
1824 */
1825 while (freelist && (nextfree = get_freepointer(s, freelist))) {
1826 void *prior;
1827 unsigned long counters;
1828
1829 do {
1830 prior = page->freelist;
1831 counters = page->counters;
1832 set_freepointer(s, freelist, prior);
1833 new.counters = counters;
1834 new.inuse--;
a0132ac0 1835 VM_BUG_ON(!new.frozen);
2cfb7455 1836
1d07171c 1837 } while (!__cmpxchg_double_slab(s, page,
2cfb7455
CL
1838 prior, counters,
1839 freelist, new.counters,
1840 "drain percpu freelist"));
1841
1842 freelist = nextfree;
1843 }
1844
894b8788 1845 /*
2cfb7455
CL
1846 * Stage two: Ensure that the page is unfrozen while the
1847 * list presence reflects the actual number of objects
1848 * during unfreeze.
1849 *
1850 * We setup the list membership and then perform a cmpxchg
1851 * with the count. If there is a mismatch then the page
1852 * is not unfrozen but the page is on the wrong list.
1853 *
1854 * Then we restart the process which may have to remove
1855 * the page from the list that we just put it on again
1856 * because the number of objects in the slab may have
1857 * changed.
894b8788 1858 */
2cfb7455 1859redo:
894b8788 1860
2cfb7455
CL
1861 old.freelist = page->freelist;
1862 old.counters = page->counters;
a0132ac0 1863 VM_BUG_ON(!old.frozen);
7c2e132c 1864
2cfb7455
CL
1865 /* Determine target state of the slab */
1866 new.counters = old.counters;
1867 if (freelist) {
1868 new.inuse--;
1869 set_freepointer(s, freelist, old.freelist);
1870 new.freelist = freelist;
1871 } else
1872 new.freelist = old.freelist;
1873
1874 new.frozen = 0;
1875
81107188 1876 if (!new.inuse && n->nr_partial > s->min_partial)
2cfb7455
CL
1877 m = M_FREE;
1878 else if (new.freelist) {
1879 m = M_PARTIAL;
1880 if (!lock) {
1881 lock = 1;
1882 /*
1883 * Taking the spinlock removes the possiblity
1884 * that acquire_slab() will see a slab page that
1885 * is frozen
1886 */
1887 spin_lock(&n->list_lock);
1888 }
1889 } else {
1890 m = M_FULL;
1891 if (kmem_cache_debug(s) && !lock) {
1892 lock = 1;
1893 /*
1894 * This also ensures that the scanning of full
1895 * slabs from diagnostic functions will not see
1896 * any frozen slabs.
1897 */
1898 spin_lock(&n->list_lock);
1899 }
1900 }
1901
1902 if (l != m) {
1903
1904 if (l == M_PARTIAL)
1905
1906 remove_partial(n, page);
1907
1908 else if (l == M_FULL)
894b8788 1909
c65c1877 1910 remove_full(s, n, page);
2cfb7455
CL
1911
1912 if (m == M_PARTIAL) {
1913
1914 add_partial(n, page, tail);
136333d1 1915 stat(s, tail);
2cfb7455
CL
1916
1917 } else if (m == M_FULL) {
894b8788 1918
2cfb7455
CL
1919 stat(s, DEACTIVATE_FULL);
1920 add_full(s, n, page);
1921
1922 }
1923 }
1924
1925 l = m;
1d07171c 1926 if (!__cmpxchg_double_slab(s, page,
2cfb7455
CL
1927 old.freelist, old.counters,
1928 new.freelist, new.counters,
1929 "unfreezing slab"))
1930 goto redo;
1931
2cfb7455
CL
1932 if (lock)
1933 spin_unlock(&n->list_lock);
1934
1935 if (m == M_FREE) {
1936 stat(s, DEACTIVATE_EMPTY);
1937 discard_slab(s, page);
1938 stat(s, FREE_SLAB);
894b8788 1939 }
81819f0f
CL
1940}
1941
d24ac77f
JK
1942/*
1943 * Unfreeze all the cpu partial slabs.
1944 *
59a09917
CL
1945 * This function must be called with interrupts disabled
1946 * for the cpu using c (or some other guarantee must be there
1947 * to guarantee no concurrent accesses).
d24ac77f 1948 */
59a09917
CL
1949static void unfreeze_partials(struct kmem_cache *s,
1950 struct kmem_cache_cpu *c)
49e22585 1951{
345c905d 1952#ifdef CONFIG_SLUB_CPU_PARTIAL
43d77867 1953 struct kmem_cache_node *n = NULL, *n2 = NULL;
9ada1934 1954 struct page *page, *discard_page = NULL;
49e22585
CL
1955
1956 while ((page = c->partial)) {
49e22585
CL
1957 struct page new;
1958 struct page old;
1959
1960 c->partial = page->next;
43d77867
JK
1961
1962 n2 = get_node(s, page_to_nid(page));
1963 if (n != n2) {
1964 if (n)
1965 spin_unlock(&n->list_lock);
1966
1967 n = n2;
1968 spin_lock(&n->list_lock);
1969 }
49e22585
CL
1970
1971 do {
1972
1973 old.freelist = page->freelist;
1974 old.counters = page->counters;
a0132ac0 1975 VM_BUG_ON(!old.frozen);
49e22585
CL
1976
1977 new.counters = old.counters;
1978 new.freelist = old.freelist;
1979
1980 new.frozen = 0;
1981
d24ac77f 1982 } while (!__cmpxchg_double_slab(s, page,
49e22585
CL
1983 old.freelist, old.counters,
1984 new.freelist, new.counters,
1985 "unfreezing slab"));
1986
43d77867 1987 if (unlikely(!new.inuse && n->nr_partial > s->min_partial)) {
9ada1934
SL
1988 page->next = discard_page;
1989 discard_page = page;
43d77867
JK
1990 } else {
1991 add_partial(n, page, DEACTIVATE_TO_TAIL);
1992 stat(s, FREE_ADD_PARTIAL);
49e22585
CL
1993 }
1994 }
1995
1996 if (n)
1997 spin_unlock(&n->list_lock);
9ada1934
SL
1998
1999 while (discard_page) {
2000 page = discard_page;
2001 discard_page = discard_page->next;
2002
2003 stat(s, DEACTIVATE_EMPTY);
2004 discard_slab(s, page);
2005 stat(s, FREE_SLAB);
2006 }
345c905d 2007#endif
49e22585
CL
2008}
2009
2010/*
2011 * Put a page that was just frozen (in __slab_free) into a partial page
2012 * slot if available. This is done without interrupts disabled and without
2013 * preemption disabled. The cmpxchg is racy and may put the partial page
2014 * onto a random cpus partial slot.
2015 *
2016 * If we did not find a slot then simply move all the partials to the
2017 * per node partial list.
2018 */
633b0764 2019static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
49e22585 2020{
345c905d 2021#ifdef CONFIG_SLUB_CPU_PARTIAL
49e22585
CL
2022 struct page *oldpage;
2023 int pages;
2024 int pobjects;
2025
2026 do {
2027 pages = 0;
2028 pobjects = 0;
2029 oldpage = this_cpu_read(s->cpu_slab->partial);
2030
2031 if (oldpage) {
2032 pobjects = oldpage->pobjects;
2033 pages = oldpage->pages;
2034 if (drain && pobjects > s->cpu_partial) {
2035 unsigned long flags;
2036 /*
2037 * partial array is full. Move the existing
2038 * set to the per node partial list.
2039 */
2040 local_irq_save(flags);
59a09917 2041 unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
49e22585 2042 local_irq_restore(flags);
e24fc410 2043 oldpage = NULL;
49e22585
CL
2044 pobjects = 0;
2045 pages = 0;
8028dcea 2046 stat(s, CPU_PARTIAL_DRAIN);
49e22585
CL
2047 }
2048 }
2049
2050 pages++;
2051 pobjects += page->objects - page->inuse;
2052
2053 page->pages = pages;
2054 page->pobjects = pobjects;
2055 page->next = oldpage;
2056
d0e0ac97
CG
2057 } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page)
2058 != oldpage);
345c905d 2059#endif
49e22585
CL
2060}
2061
dfb4f096 2062static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
81819f0f 2063{
84e554e6 2064 stat(s, CPUSLAB_FLUSH);
c17dda40
CL
2065 deactivate_slab(s, c->page, c->freelist);
2066
2067 c->tid = next_tid(c->tid);
2068 c->page = NULL;
2069 c->freelist = NULL;
81819f0f
CL
2070}
2071
2072/*
2073 * Flush cpu slab.
6446faa2 2074 *
81819f0f
CL
2075 * Called from IPI handler with interrupts disabled.
2076 */
0c710013 2077static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
81819f0f 2078{
9dfc6e68 2079 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
81819f0f 2080
49e22585
CL
2081 if (likely(c)) {
2082 if (c->page)
2083 flush_slab(s, c);
2084
59a09917 2085 unfreeze_partials(s, c);
49e22585 2086 }
81819f0f
CL
2087}
2088
2089static void flush_cpu_slab(void *d)
2090{
2091 struct kmem_cache *s = d;
81819f0f 2092
dfb4f096 2093 __flush_cpu_slab(s, smp_processor_id());
81819f0f
CL
2094}
2095
a8364d55
GBY
2096static bool has_cpu_slab(int cpu, void *info)
2097{
2098 struct kmem_cache *s = info;
2099 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
2100
02e1a9cd 2101 return c->page || c->partial;
a8364d55
GBY
2102}
2103
81819f0f
CL
2104static void flush_all(struct kmem_cache *s)
2105{
a8364d55 2106 on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC);
81819f0f
CL
2107}
2108
dfb4f096
CL
2109/*
2110 * Check if the objects in a per cpu structure fit numa
2111 * locality expectations.
2112 */
57d437d2 2113static inline int node_match(struct page *page, int node)
dfb4f096
CL
2114{
2115#ifdef CONFIG_NUMA
4d7868e6 2116 if (!page || (node != NUMA_NO_NODE && page_to_nid(page) != node))
dfb4f096
CL
2117 return 0;
2118#endif
2119 return 1;
2120}
2121
781b2ba6
PE
2122static int count_free(struct page *page)
2123{
2124 return page->objects - page->inuse;
2125}
2126
2127static unsigned long count_partial(struct kmem_cache_node *n,
2128 int (*get_count)(struct page *))
2129{
2130 unsigned long flags;
2131 unsigned long x = 0;
2132 struct page *page;
2133
2134 spin_lock_irqsave(&n->list_lock, flags);
2135 list_for_each_entry(page, &n->partial, lru)
2136 x += get_count(page);
2137 spin_unlock_irqrestore(&n->list_lock, flags);
2138 return x;
2139}
2140
26c02cf0
AB
2141static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
2142{
2143#ifdef CONFIG_SLUB_DEBUG
2144 return atomic_long_read(&n->total_objects);
2145#else
2146 return 0;
2147#endif
2148}
2149
781b2ba6
PE
2150static noinline void
2151slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
2152{
2153 int node;
2154
f9f58285 2155 pr_warn("SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n",
781b2ba6 2156 nid, gfpflags);
f9f58285
FF
2157 pr_warn(" cache: %s, object size: %d, buffer size: %d, default order: %d, min order: %d\n",
2158 s->name, s->object_size, s->size, oo_order(s->oo),
2159 oo_order(s->min));
781b2ba6 2160
3b0efdfa 2161 if (oo_order(s->min) > get_order(s->object_size))
f9f58285
FF
2162 pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n",
2163 s->name);
fa5ec8a1 2164
781b2ba6
PE
2165 for_each_online_node(node) {
2166 struct kmem_cache_node *n = get_node(s, node);
2167 unsigned long nr_slabs;
2168 unsigned long nr_objs;
2169 unsigned long nr_free;
2170
2171 if (!n)
2172 continue;
2173
26c02cf0
AB
2174 nr_free = count_partial(n, count_free);
2175 nr_slabs = node_nr_slabs(n);
2176 nr_objs = node_nr_objs(n);
781b2ba6 2177
f9f58285 2178 pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n",
781b2ba6
PE
2179 node, nr_slabs, nr_objs, nr_free);
2180 }
2181}
2182
497b66f2
CL
2183static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
2184 int node, struct kmem_cache_cpu **pc)
2185{
6faa6833 2186 void *freelist;
188fd063
CL
2187 struct kmem_cache_cpu *c = *pc;
2188 struct page *page;
497b66f2 2189
188fd063 2190 freelist = get_partial(s, flags, node, c);
497b66f2 2191
188fd063
CL
2192 if (freelist)
2193 return freelist;
2194
2195 page = new_slab(s, flags, node);
497b66f2
CL
2196 if (page) {
2197 c = __this_cpu_ptr(s->cpu_slab);
2198 if (c->page)
2199 flush_slab(s, c);
2200
2201 /*
2202 * No other reference to the page yet so we can
2203 * muck around with it freely without cmpxchg
2204 */
6faa6833 2205 freelist = page->freelist;
497b66f2
CL
2206 page->freelist = NULL;
2207
2208 stat(s, ALLOC_SLAB);
497b66f2
CL
2209 c->page = page;
2210 *pc = c;
2211 } else
6faa6833 2212 freelist = NULL;
497b66f2 2213
6faa6833 2214 return freelist;
497b66f2
CL
2215}
2216
072bb0aa
MG
2217static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags)
2218{
2219 if (unlikely(PageSlabPfmemalloc(page)))
2220 return gfp_pfmemalloc_allowed(gfpflags);
2221
2222 return true;
2223}
2224
213eeb9f 2225/*
d0e0ac97
CG
2226 * Check the page->freelist of a page and either transfer the freelist to the
2227 * per cpu freelist or deactivate the page.
213eeb9f
CL
2228 *
2229 * The page is still frozen if the return value is not NULL.
2230 *
2231 * If this function returns NULL then the page has been unfrozen.
d24ac77f
JK
2232 *
2233 * This function must be called with interrupt disabled.
213eeb9f
CL
2234 */
2235static inline void *get_freelist(struct kmem_cache *s, struct page *page)
2236{
2237 struct page new;
2238 unsigned long counters;
2239 void *freelist;
2240
2241 do {
2242 freelist = page->freelist;
2243 counters = page->counters;
6faa6833 2244
213eeb9f 2245 new.counters = counters;
a0132ac0 2246 VM_BUG_ON(!new.frozen);
213eeb9f
CL
2247
2248 new.inuse = page->objects;
2249 new.frozen = freelist != NULL;
2250
d24ac77f 2251 } while (!__cmpxchg_double_slab(s, page,
213eeb9f
CL
2252 freelist, counters,
2253 NULL, new.counters,
2254 "get_freelist"));
2255
2256 return freelist;
2257}
2258
81819f0f 2259/*
894b8788
CL
2260 * Slow path. The lockless freelist is empty or we need to perform
2261 * debugging duties.
2262 *
894b8788
CL
2263 * Processing is still very fast if new objects have been freed to the
2264 * regular freelist. In that case we simply take over the regular freelist
2265 * as the lockless freelist and zap the regular freelist.
81819f0f 2266 *
894b8788
CL
2267 * If that is not working then we fall back to the partial lists. We take the
2268 * first element of the freelist as the object to allocate now and move the
2269 * rest of the freelist to the lockless freelist.
81819f0f 2270 *
894b8788 2271 * And if we were unable to get a new slab from the partial slab lists then
6446faa2
CL
2272 * we need to allocate a new slab. This is the slowest path since it involves
2273 * a call to the page allocator and the setup of a new slab.
81819f0f 2274 */
ce71e27c
EGM
2275static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
2276 unsigned long addr, struct kmem_cache_cpu *c)
81819f0f 2277{
6faa6833 2278 void *freelist;
f6e7def7 2279 struct page *page;
8a5ec0ba
CL
2280 unsigned long flags;
2281
2282 local_irq_save(flags);
2283#ifdef CONFIG_PREEMPT
2284 /*
2285 * We may have been preempted and rescheduled on a different
2286 * cpu before disabling interrupts. Need to reload cpu area
2287 * pointer.
2288 */
2289 c = this_cpu_ptr(s->cpu_slab);
8a5ec0ba 2290#endif
81819f0f 2291
f6e7def7
CL
2292 page = c->page;
2293 if (!page)
81819f0f 2294 goto new_slab;
49e22585 2295redo:
6faa6833 2296
57d437d2 2297 if (unlikely(!node_match(page, node))) {
e36a2652 2298 stat(s, ALLOC_NODE_MISMATCH);
f6e7def7 2299 deactivate_slab(s, page, c->freelist);
c17dda40
CL
2300 c->page = NULL;
2301 c->freelist = NULL;
fc59c053
CL
2302 goto new_slab;
2303 }
6446faa2 2304
072bb0aa
MG
2305 /*
2306 * By rights, we should be searching for a slab page that was
2307 * PFMEMALLOC but right now, we are losing the pfmemalloc
2308 * information when the page leaves the per-cpu allocator
2309 */
2310 if (unlikely(!pfmemalloc_match(page, gfpflags))) {
2311 deactivate_slab(s, page, c->freelist);
2312 c->page = NULL;
2313 c->freelist = NULL;
2314 goto new_slab;
2315 }
2316
73736e03 2317 /* must check again c->freelist in case of cpu migration or IRQ */
6faa6833
CL
2318 freelist = c->freelist;
2319 if (freelist)
73736e03 2320 goto load_freelist;
03e404af 2321
2cfb7455 2322 stat(s, ALLOC_SLOWPATH);
03e404af 2323
f6e7def7 2324 freelist = get_freelist(s, page);
6446faa2 2325
6faa6833 2326 if (!freelist) {
03e404af
CL
2327 c->page = NULL;
2328 stat(s, DEACTIVATE_BYPASS);
fc59c053 2329 goto new_slab;
03e404af 2330 }
6446faa2 2331
84e554e6 2332 stat(s, ALLOC_REFILL);
6446faa2 2333
894b8788 2334load_freelist:
507effea
CL
2335 /*
2336 * freelist is pointing to the list of objects to be used.
2337 * page is pointing to the page from which the objects are obtained.
2338 * That page must be frozen for per cpu allocations to work.
2339 */
a0132ac0 2340 VM_BUG_ON(!c->page->frozen);
6faa6833 2341 c->freelist = get_freepointer(s, freelist);
8a5ec0ba
CL
2342 c->tid = next_tid(c->tid);
2343 local_irq_restore(flags);
6faa6833 2344 return freelist;
81819f0f 2345
81819f0f 2346new_slab:
2cfb7455 2347
49e22585 2348 if (c->partial) {
f6e7def7
CL
2349 page = c->page = c->partial;
2350 c->partial = page->next;
49e22585
CL
2351 stat(s, CPU_PARTIAL_ALLOC);
2352 c->freelist = NULL;
2353 goto redo;
81819f0f
CL
2354 }
2355
188fd063 2356 freelist = new_slab_objects(s, gfpflags, node, &c);
01ad8a7b 2357
f4697436
CL
2358 if (unlikely(!freelist)) {
2359 if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
2360 slab_out_of_memory(s, gfpflags, node);
2cfb7455 2361
f4697436
CL
2362 local_irq_restore(flags);
2363 return NULL;
81819f0f 2364 }
2cfb7455 2365
f6e7def7 2366 page = c->page;
5091b74a 2367 if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags)))
4b6f0750 2368 goto load_freelist;
2cfb7455 2369
497b66f2 2370 /* Only entered in the debug case */
d0e0ac97
CG
2371 if (kmem_cache_debug(s) &&
2372 !alloc_debug_processing(s, page, freelist, addr))
497b66f2 2373 goto new_slab; /* Slab failed checks. Next slab needed */
894b8788 2374
f6e7def7 2375 deactivate_slab(s, page, get_freepointer(s, freelist));
c17dda40
CL
2376 c->page = NULL;
2377 c->freelist = NULL;
a71ae47a 2378 local_irq_restore(flags);
6faa6833 2379 return freelist;
894b8788
CL
2380}
2381
2382/*
2383 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
2384 * have the fastpath folded into their functions. So no function call
2385 * overhead for requests that can be satisfied on the fastpath.
2386 *
2387 * The fastpath works by first checking if the lockless freelist can be used.
2388 * If not then __slab_alloc is called for slow processing.
2389 *
2390 * Otherwise we can simply pick the next object from the lockless free list.
2391 */
2b847c3c 2392static __always_inline void *slab_alloc_node(struct kmem_cache *s,
ce71e27c 2393 gfp_t gfpflags, int node, unsigned long addr)
894b8788 2394{
894b8788 2395 void **object;
dfb4f096 2396 struct kmem_cache_cpu *c;
57d437d2 2397 struct page *page;
8a5ec0ba 2398 unsigned long tid;
1f84260c 2399
c016b0bd 2400 if (slab_pre_alloc_hook(s, gfpflags))
773ff60e 2401 return NULL;
1f84260c 2402
d79923fa 2403 s = memcg_kmem_get_cache(s, gfpflags);
8a5ec0ba 2404redo:
8a5ec0ba
CL
2405 /*
2406 * Must read kmem_cache cpu data via this cpu ptr. Preemption is
2407 * enabled. We may switch back and forth between cpus while
2408 * reading from one cpu area. That does not matter as long
2409 * as we end up on the original cpu again when doing the cmpxchg.
7cccd80b
CL
2410 *
2411 * Preemption is disabled for the retrieval of the tid because that
2412 * must occur from the current processor. We cannot allow rescheduling
2413 * on a different processor between the determination of the pointer
2414 * and the retrieval of the tid.
8a5ec0ba 2415 */
7cccd80b 2416 preempt_disable();
9dfc6e68 2417 c = __this_cpu_ptr(s->cpu_slab);
8a5ec0ba 2418
8a5ec0ba
CL
2419 /*
2420 * The transaction ids are globally unique per cpu and per operation on
2421 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
2422 * occurs on the right processor and that there was no operation on the
2423 * linked list in between.
2424 */
2425 tid = c->tid;
7cccd80b 2426 preempt_enable();
8a5ec0ba 2427
9dfc6e68 2428 object = c->freelist;
57d437d2 2429 page = c->page;
ac6434e6 2430 if (unlikely(!object || !node_match(page, node)))
dfb4f096 2431 object = __slab_alloc(s, gfpflags, node, addr, c);
894b8788
CL
2432
2433 else {
0ad9500e
ED
2434 void *next_object = get_freepointer_safe(s, object);
2435
8a5ec0ba 2436 /*
25985edc 2437 * The cmpxchg will only match if there was no additional
8a5ec0ba
CL
2438 * operation and if we are on the right processor.
2439 *
d0e0ac97
CG
2440 * The cmpxchg does the following atomically (without lock
2441 * semantics!)
8a5ec0ba
CL
2442 * 1. Relocate first pointer to the current per cpu area.
2443 * 2. Verify that tid and freelist have not been changed
2444 * 3. If they were not changed replace tid and freelist
2445 *
d0e0ac97
CG
2446 * Since this is without lock semantics the protection is only
2447 * against code executing on this cpu *not* from access by
2448 * other cpus.
8a5ec0ba 2449 */
933393f5 2450 if (unlikely(!this_cpu_cmpxchg_double(
8a5ec0ba
CL
2451 s->cpu_slab->freelist, s->cpu_slab->tid,
2452 object, tid,
0ad9500e 2453 next_object, next_tid(tid)))) {
8a5ec0ba
CL
2454
2455 note_cmpxchg_failure("slab_alloc", s, tid);
2456 goto redo;
2457 }
0ad9500e 2458 prefetch_freepointer(s, next_object);
84e554e6 2459 stat(s, ALLOC_FASTPATH);
894b8788 2460 }
8a5ec0ba 2461
74e2134f 2462 if (unlikely(gfpflags & __GFP_ZERO) && object)
3b0efdfa 2463 memset(object, 0, s->object_size);
d07dbea4 2464
c016b0bd 2465 slab_post_alloc_hook(s, gfpflags, object);
5a896d9e 2466
894b8788 2467 return object;
81819f0f
CL
2468}
2469
2b847c3c
EG
2470static __always_inline void *slab_alloc(struct kmem_cache *s,
2471 gfp_t gfpflags, unsigned long addr)
2472{
2473 return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr);
2474}
2475
81819f0f
CL
2476void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
2477{
2b847c3c 2478 void *ret = slab_alloc(s, gfpflags, _RET_IP_);
5b882be4 2479
d0e0ac97
CG
2480 trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size,
2481 s->size, gfpflags);
5b882be4
EGM
2482
2483 return ret;
81819f0f
CL
2484}
2485EXPORT_SYMBOL(kmem_cache_alloc);
2486
0f24f128 2487#ifdef CONFIG_TRACING
4a92379b
RK
2488void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
2489{
2b847c3c 2490 void *ret = slab_alloc(s, gfpflags, _RET_IP_);
4a92379b
RK
2491 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
2492 return ret;
2493}
2494EXPORT_SYMBOL(kmem_cache_alloc_trace);
5b882be4
EGM
2495#endif
2496
81819f0f
CL
2497#ifdef CONFIG_NUMA
2498void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
2499{
2b847c3c 2500 void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);
5b882be4 2501
ca2b84cb 2502 trace_kmem_cache_alloc_node(_RET_IP_, ret,
3b0efdfa 2503 s->object_size, s->size, gfpflags, node);
5b882be4
EGM
2504
2505 return ret;
81819f0f
CL
2506}
2507EXPORT_SYMBOL(kmem_cache_alloc_node);
81819f0f 2508
0f24f128 2509#ifdef CONFIG_TRACING
4a92379b 2510void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
5b882be4 2511 gfp_t gfpflags,
4a92379b 2512 int node, size_t size)
5b882be4 2513{
2b847c3c 2514 void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);
4a92379b
RK
2515
2516 trace_kmalloc_node(_RET_IP_, ret,
2517 size, s->size, gfpflags, node);
2518 return ret;
5b882be4 2519}
4a92379b 2520EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
5b882be4 2521#endif
5d1f57e4 2522#endif
5b882be4 2523
81819f0f 2524/*
894b8788
CL
2525 * Slow patch handling. This may still be called frequently since objects
2526 * have a longer lifetime than the cpu slabs in most processing loads.
81819f0f 2527 *
894b8788
CL
2528 * So we still attempt to reduce cache line usage. Just take the slab
2529 * lock and free the item. If there is no additional partial page
2530 * handling required then we can return immediately.
81819f0f 2531 */
894b8788 2532static void __slab_free(struct kmem_cache *s, struct page *page,
ff12059e 2533 void *x, unsigned long addr)
81819f0f
CL
2534{
2535 void *prior;
2536 void **object = (void *)x;
2cfb7455 2537 int was_frozen;
2cfb7455
CL
2538 struct page new;
2539 unsigned long counters;
2540 struct kmem_cache_node *n = NULL;
61728d1e 2541 unsigned long uninitialized_var(flags);
81819f0f 2542
8a5ec0ba 2543 stat(s, FREE_SLOWPATH);
81819f0f 2544
19c7ff9e
CL
2545 if (kmem_cache_debug(s) &&
2546 !(n = free_debug_processing(s, page, x, addr, &flags)))
80f08c19 2547 return;
6446faa2 2548
2cfb7455 2549 do {
837d678d
JK
2550 if (unlikely(n)) {
2551 spin_unlock_irqrestore(&n->list_lock, flags);
2552 n = NULL;
2553 }
2cfb7455
CL
2554 prior = page->freelist;
2555 counters = page->counters;
2556 set_freepointer(s, object, prior);
2557 new.counters = counters;
2558 was_frozen = new.frozen;
2559 new.inuse--;
837d678d 2560 if ((!new.inuse || !prior) && !was_frozen) {
49e22585 2561
c65c1877 2562 if (kmem_cache_has_cpu_partial(s) && !prior) {
49e22585
CL
2563
2564 /*
d0e0ac97
CG
2565 * Slab was on no list before and will be
2566 * partially empty
2567 * We can defer the list move and instead
2568 * freeze it.
49e22585
CL
2569 */
2570 new.frozen = 1;
2571
c65c1877 2572 } else { /* Needs to be taken off a list */
49e22585
CL
2573
2574 n = get_node(s, page_to_nid(page));
2575 /*
2576 * Speculatively acquire the list_lock.
2577 * If the cmpxchg does not succeed then we may
2578 * drop the list_lock without any processing.
2579 *
2580 * Otherwise the list_lock will synchronize with
2581 * other processors updating the list of slabs.
2582 */
2583 spin_lock_irqsave(&n->list_lock, flags);
2584
2585 }
2cfb7455 2586 }
81819f0f 2587
2cfb7455
CL
2588 } while (!cmpxchg_double_slab(s, page,
2589 prior, counters,
2590 object, new.counters,
2591 "__slab_free"));
81819f0f 2592
2cfb7455 2593 if (likely(!n)) {
49e22585
CL
2594
2595 /*
2596 * If we just froze the page then put it onto the
2597 * per cpu partial list.
2598 */
8028dcea 2599 if (new.frozen && !was_frozen) {
49e22585 2600 put_cpu_partial(s, page, 1);
8028dcea
AS
2601 stat(s, CPU_PARTIAL_FREE);
2602 }
49e22585 2603 /*
2cfb7455
CL
2604 * The list lock was not taken therefore no list
2605 * activity can be necessary.
2606 */
2607 if (was_frozen)
2608 stat(s, FREE_FROZEN);
80f08c19 2609 return;
2cfb7455 2610 }
81819f0f 2611
837d678d
JK
2612 if (unlikely(!new.inuse && n->nr_partial > s->min_partial))
2613 goto slab_empty;
2614
81819f0f 2615 /*
837d678d
JK
2616 * Objects left in the slab. If it was not on the partial list before
2617 * then add it.
81819f0f 2618 */
345c905d
JK
2619 if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
2620 if (kmem_cache_debug(s))
c65c1877 2621 remove_full(s, n, page);
837d678d
JK
2622 add_partial(n, page, DEACTIVATE_TO_TAIL);
2623 stat(s, FREE_ADD_PARTIAL);
8ff12cfc 2624 }
80f08c19 2625 spin_unlock_irqrestore(&n->list_lock, flags);
81819f0f
CL
2626 return;
2627
2628slab_empty:
a973e9dd 2629 if (prior) {
81819f0f 2630 /*
6fbabb20 2631 * Slab on the partial list.
81819f0f 2632 */
5cc6eee8 2633 remove_partial(n, page);
84e554e6 2634 stat(s, FREE_REMOVE_PARTIAL);
c65c1877 2635 } else {
6fbabb20 2636 /* Slab must be on the full list */
c65c1877
PZ
2637 remove_full(s, n, page);
2638 }
2cfb7455 2639
80f08c19 2640 spin_unlock_irqrestore(&n->list_lock, flags);
84e554e6 2641 stat(s, FREE_SLAB);
81819f0f 2642 discard_slab(s, page);
81819f0f
CL
2643}
2644
894b8788
CL
2645/*
2646 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
2647 * can perform fastpath freeing without additional function calls.
2648 *
2649 * The fastpath is only possible if we are freeing to the current cpu slab
2650 * of this processor. This typically the case if we have just allocated
2651 * the item before.
2652 *
2653 * If fastpath is not possible then fall back to __slab_free where we deal
2654 * with all sorts of special processing.
2655 */
06428780 2656static __always_inline void slab_free(struct kmem_cache *s,
ce71e27c 2657 struct page *page, void *x, unsigned long addr)
894b8788
CL
2658{
2659 void **object = (void *)x;
dfb4f096 2660 struct kmem_cache_cpu *c;
8a5ec0ba 2661 unsigned long tid;
1f84260c 2662
c016b0bd
CL
2663 slab_free_hook(s, x);
2664
8a5ec0ba
CL
2665redo:
2666 /*
2667 * Determine the currently cpus per cpu slab.
2668 * The cpu may change afterward. However that does not matter since
2669 * data is retrieved via this pointer. If we are on the same cpu
2670 * during the cmpxchg then the free will succedd.
2671 */
7cccd80b 2672 preempt_disable();
9dfc6e68 2673 c = __this_cpu_ptr(s->cpu_slab);
c016b0bd 2674
8a5ec0ba 2675 tid = c->tid;
7cccd80b 2676 preempt_enable();
c016b0bd 2677
442b06bc 2678 if (likely(page == c->page)) {
ff12059e 2679 set_freepointer(s, object, c->freelist);
8a5ec0ba 2680
933393f5 2681 if (unlikely(!this_cpu_cmpxchg_double(
8a5ec0ba
CL
2682 s->cpu_slab->freelist, s->cpu_slab->tid,
2683 c->freelist, tid,
2684 object, next_tid(tid)))) {
2685
2686 note_cmpxchg_failure("slab_free", s, tid);
2687 goto redo;
2688 }
84e554e6 2689 stat(s, FREE_FASTPATH);
894b8788 2690 } else
ff12059e 2691 __slab_free(s, page, x, addr);
894b8788 2692
894b8788
CL
2693}
2694
81819f0f
CL
2695void kmem_cache_free(struct kmem_cache *s, void *x)
2696{
b9ce5ef4
GC
2697 s = cache_from_obj(s, x);
2698 if (!s)
79576102 2699 return;
b9ce5ef4 2700 slab_free(s, virt_to_head_page(x), x, _RET_IP_);
ca2b84cb 2701 trace_kmem_cache_free(_RET_IP_, x);
81819f0f
CL
2702}
2703EXPORT_SYMBOL(kmem_cache_free);
2704
81819f0f 2705/*
672bba3a
CL
2706 * Object placement in a slab is made very easy because we always start at
2707 * offset 0. If we tune the size of the object to the alignment then we can
2708 * get the required alignment by putting one properly sized object after
2709 * another.
81819f0f
CL
2710 *
2711 * Notice that the allocation order determines the sizes of the per cpu
2712 * caches. Each processor has always one slab available for allocations.
2713 * Increasing the allocation order reduces the number of times that slabs
672bba3a 2714 * must be moved on and off the partial lists and is therefore a factor in
81819f0f 2715 * locking overhead.
81819f0f
CL
2716 */
2717
2718/*
2719 * Mininum / Maximum order of slab pages. This influences locking overhead
2720 * and slab fragmentation. A higher order reduces the number of partial slabs
2721 * and increases the number of allocations possible without having to
2722 * take the list_lock.
2723 */
2724static int slub_min_order;
114e9e89 2725static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
9b2cd506 2726static int slub_min_objects;
81819f0f
CL
2727
2728/*
2729 * Merge control. If this is set then no merging of slab caches will occur.
672bba3a 2730 * (Could be removed. This was introduced to pacify the merge skeptics.)
81819f0f
CL
2731 */
2732static int slub_nomerge;
2733
81819f0f
CL
2734/*
2735 * Calculate the order of allocation given an slab object size.
2736 *
672bba3a
CL
2737 * The order of allocation has significant impact on performance and other
2738 * system components. Generally order 0 allocations should be preferred since
2739 * order 0 does not cause fragmentation in the page allocator. Larger objects
2740 * be problematic to put into order 0 slabs because there may be too much
c124f5b5 2741 * unused space left. We go to a higher order if more than 1/16th of the slab
672bba3a
CL
2742 * would be wasted.
2743 *
2744 * In order to reach satisfactory performance we must ensure that a minimum
2745 * number of objects is in one slab. Otherwise we may generate too much
2746 * activity on the partial lists which requires taking the list_lock. This is
2747 * less a concern for large slabs though which are rarely used.
81819f0f 2748 *
672bba3a
CL
2749 * slub_max_order specifies the order where we begin to stop considering the
2750 * number of objects in a slab as critical. If we reach slub_max_order then
2751 * we try to keep the page order as low as possible. So we accept more waste
2752 * of space in favor of a small page order.
81819f0f 2753 *
672bba3a
CL
2754 * Higher order allocations also allow the placement of more objects in a
2755 * slab and thereby reduce object handling overhead. If the user has
2756 * requested a higher mininum order then we start with that one instead of
2757 * the smallest order which will fit the object.
81819f0f 2758 */
5e6d444e 2759static inline int slab_order(int size, int min_objects,
ab9a0f19 2760 int max_order, int fract_leftover, int reserved)
81819f0f
CL
2761{
2762 int order;
2763 int rem;
6300ea75 2764 int min_order = slub_min_order;
81819f0f 2765
ab9a0f19 2766 if (order_objects(min_order, size, reserved) > MAX_OBJS_PER_PAGE)
210b5c06 2767 return get_order(size * MAX_OBJS_PER_PAGE) - 1;
39b26464 2768
6300ea75 2769 for (order = max(min_order,
5e6d444e
CL
2770 fls(min_objects * size - 1) - PAGE_SHIFT);
2771 order <= max_order; order++) {
81819f0f 2772
5e6d444e 2773 unsigned long slab_size = PAGE_SIZE << order;
81819f0f 2774
ab9a0f19 2775 if (slab_size < min_objects * size + reserved)
81819f0f
CL
2776 continue;
2777
ab9a0f19 2778 rem = (slab_size - reserved) % size;
81819f0f 2779
5e6d444e 2780 if (rem <= slab_size / fract_leftover)
81819f0f
CL
2781 break;
2782
2783 }
672bba3a 2784
81819f0f
CL
2785 return order;
2786}
2787
ab9a0f19 2788static inline int calculate_order(int size, int reserved)
5e6d444e
CL
2789{
2790 int order;
2791 int min_objects;
2792 int fraction;
e8120ff1 2793 int max_objects;
5e6d444e
CL
2794
2795 /*
2796 * Attempt to find best configuration for a slab. This
2797 * works by first attempting to generate a layout with
2798 * the best configuration and backing off gradually.
2799 *
2800 * First we reduce the acceptable waste in a slab. Then
2801 * we reduce the minimum objects required in a slab.
2802 */
2803 min_objects = slub_min_objects;
9b2cd506
CL
2804 if (!min_objects)
2805 min_objects = 4 * (fls(nr_cpu_ids) + 1);
ab9a0f19 2806 max_objects = order_objects(slub_max_order, size, reserved);
e8120ff1
ZY
2807 min_objects = min(min_objects, max_objects);
2808
5e6d444e 2809 while (min_objects > 1) {
c124f5b5 2810 fraction = 16;
5e6d444e
CL
2811 while (fraction >= 4) {
2812 order = slab_order(size, min_objects,
ab9a0f19 2813 slub_max_order, fraction, reserved);
5e6d444e
CL
2814 if (order <= slub_max_order)
2815 return order;
2816 fraction /= 2;
2817 }
5086c389 2818 min_objects--;
5e6d444e
CL
2819 }
2820
2821 /*
2822 * We were unable to place multiple objects in a slab. Now
2823 * lets see if we can place a single object there.
2824 */
ab9a0f19 2825 order = slab_order(size, 1, slub_max_order, 1, reserved);
5e6d444e
CL
2826 if (order <= slub_max_order)
2827 return order;
2828
2829 /*
2830 * Doh this slab cannot be placed using slub_max_order.
2831 */
ab9a0f19 2832 order = slab_order(size, 1, MAX_ORDER, 1, reserved);
818cf590 2833 if (order < MAX_ORDER)
5e6d444e
CL
2834 return order;
2835 return -ENOSYS;
2836}
2837
5595cffc 2838static void
4053497d 2839init_kmem_cache_node(struct kmem_cache_node *n)
81819f0f
CL
2840{
2841 n->nr_partial = 0;
81819f0f
CL
2842 spin_lock_init(&n->list_lock);
2843 INIT_LIST_HEAD(&n->partial);
8ab1372f 2844#ifdef CONFIG_SLUB_DEBUG
0f389ec6 2845 atomic_long_set(&n->nr_slabs, 0);
02b71b70 2846 atomic_long_set(&n->total_objects, 0);
643b1138 2847 INIT_LIST_HEAD(&n->full);
8ab1372f 2848#endif
81819f0f
CL
2849}
2850
55136592 2851static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
4c93c355 2852{
6c182dc0 2853 BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
95a05b42 2854 KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));
4c93c355 2855
8a5ec0ba 2856 /*
d4d84fef
CM
2857 * Must align to double word boundary for the double cmpxchg
2858 * instructions to work; see __pcpu_double_call_return_bool().
8a5ec0ba 2859 */
d4d84fef
CM
2860 s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
2861 2 * sizeof(void *));
8a5ec0ba
CL
2862
2863 if (!s->cpu_slab)
2864 return 0;
2865
2866 init_kmem_cache_cpus(s);
4c93c355 2867
8a5ec0ba 2868 return 1;
4c93c355 2869}
4c93c355 2870
51df1142
CL
2871static struct kmem_cache *kmem_cache_node;
2872
81819f0f
CL
2873/*
2874 * No kmalloc_node yet so do it by hand. We know that this is the first
2875 * slab on the node for this slabcache. There are no concurrent accesses
2876 * possible.
2877 *
721ae22a
ZYW
2878 * Note that this function only works on the kmem_cache_node
2879 * when allocating for the kmem_cache_node. This is used for bootstrapping
4c93c355 2880 * memory on a fresh node that has no slab structures yet.
81819f0f 2881 */
55136592 2882static void early_kmem_cache_node_alloc(int node)
81819f0f
CL
2883{
2884 struct page *page;
2885 struct kmem_cache_node *n;
2886
51df1142 2887 BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
81819f0f 2888
51df1142 2889 page = new_slab(kmem_cache_node, GFP_NOWAIT, node);
81819f0f
CL
2890
2891 BUG_ON(!page);
a2f92ee7 2892 if (page_to_nid(page) != node) {
f9f58285
FF
2893 pr_err("SLUB: Unable to allocate memory from node %d\n", node);
2894 pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
a2f92ee7
CL
2895 }
2896
81819f0f
CL
2897 n = page->freelist;
2898 BUG_ON(!n);
51df1142 2899 page->freelist = get_freepointer(kmem_cache_node, n);
e6e82ea1 2900 page->inuse = 1;
8cb0a506 2901 page->frozen = 0;
51df1142 2902 kmem_cache_node->node[node] = n;
8ab1372f 2903#ifdef CONFIG_SLUB_DEBUG
f7cb1933 2904 init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
51df1142 2905 init_tracking(kmem_cache_node, n);
8ab1372f 2906#endif
4053497d 2907 init_kmem_cache_node(n);
51df1142 2908 inc_slabs_node(kmem_cache_node, node, page->objects);
6446faa2 2909
67b6c900 2910 /*
1e4dd946
SR
2911 * No locks need to be taken here as it has just been
2912 * initialized and there is no concurrent access.
67b6c900 2913 */
1e4dd946 2914 __add_partial(n, page, DEACTIVATE_TO_HEAD);
81819f0f
CL
2915}
2916
2917static void free_kmem_cache_nodes(struct kmem_cache *s)
2918{
2919 int node;
2920
f64dc58c 2921 for_each_node_state(node, N_NORMAL_MEMORY) {
81819f0f 2922 struct kmem_cache_node *n = s->node[node];
51df1142 2923
73367bd8 2924 if (n)
51df1142
CL
2925 kmem_cache_free(kmem_cache_node, n);
2926
81819f0f
CL
2927 s->node[node] = NULL;
2928 }
2929}
2930
55136592 2931static int init_kmem_cache_nodes(struct kmem_cache *s)
81819f0f
CL
2932{
2933 int node;
81819f0f 2934
f64dc58c 2935 for_each_node_state(node, N_NORMAL_MEMORY) {
81819f0f
CL
2936 struct kmem_cache_node *n;
2937
73367bd8 2938 if (slab_state == DOWN) {
55136592 2939 early_kmem_cache_node_alloc(node);
73367bd8
AD
2940 continue;
2941 }
51df1142 2942 n = kmem_cache_alloc_node(kmem_cache_node,
55136592 2943 GFP_KERNEL, node);
81819f0f 2944
73367bd8
AD
2945 if (!n) {
2946 free_kmem_cache_nodes(s);
2947 return 0;
81819f0f 2948 }
73367bd8 2949
81819f0f 2950 s->node[node] = n;
4053497d 2951 init_kmem_cache_node(n);
81819f0f
CL
2952 }
2953 return 1;
2954}
81819f0f 2955
c0bdb232 2956static void set_min_partial(struct kmem_cache *s, unsigned long min)
3b89d7d8
DR
2957{
2958 if (min < MIN_PARTIAL)
2959 min = MIN_PARTIAL;
2960 else if (min > MAX_PARTIAL)
2961 min = MAX_PARTIAL;
2962 s->min_partial = min;
2963}
2964
81819f0f
CL
2965/*
2966 * calculate_sizes() determines the order and the distribution of data within
2967 * a slab object.
2968 */
06b285dc 2969static int calculate_sizes(struct kmem_cache *s, int forced_order)
81819f0f
CL
2970{
2971 unsigned long flags = s->flags;
3b0efdfa 2972 unsigned long size = s->object_size;
834f3d11 2973 int order;
81819f0f 2974
d8b42bf5
CL
2975 /*
2976 * Round up object size to the next word boundary. We can only
2977 * place the free pointer at word boundaries and this determines
2978 * the possible location of the free pointer.
2979 */
2980 size = ALIGN(size, sizeof(void *));
2981
2982#ifdef CONFIG_SLUB_DEBUG
81819f0f
CL
2983 /*
2984 * Determine if we can poison the object itself. If the user of
2985 * the slab may touch the object after free or before allocation
2986 * then we should never poison the object itself.
2987 */
2988 if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) &&
c59def9f 2989 !s->ctor)
81819f0f
CL
2990 s->flags |= __OBJECT_POISON;
2991 else
2992 s->flags &= ~__OBJECT_POISON;
2993
81819f0f
CL
2994
2995 /*
672bba3a 2996 * If we are Redzoning then check if there is some space between the
81819f0f 2997 * end of the object and the free pointer. If not then add an
672bba3a 2998 * additional word to have some bytes to store Redzone information.
81819f0f 2999 */
3b0efdfa 3000 if ((flags & SLAB_RED_ZONE) && size == s->object_size)
81819f0f 3001 size += sizeof(void *);
41ecc55b 3002#endif
81819f0f
CL
3003
3004 /*
672bba3a
CL
3005 * With that we have determined the number of bytes in actual use
3006 * by the object. This is the potential offset to the free pointer.
81819f0f
CL
3007 */
3008 s->inuse = size;
3009
3010 if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) ||
c59def9f 3011 s->ctor)) {
81819f0f
CL
3012 /*
3013 * Relocate free pointer after the object if it is not
3014 * permitted to overwrite the first word of the object on
3015 * kmem_cache_free.
3016 *
3017 * This is the case if we do RCU, have a constructor or
3018 * destructor or are poisoning the objects.
3019 */
3020 s->offset = size;
3021 size += sizeof(void *);
3022 }
3023
c12b3c62 3024#ifdef CONFIG_SLUB_DEBUG
81819f0f
CL
3025 if (flags & SLAB_STORE_USER)
3026 /*
3027 * Need to store information about allocs and frees after
3028 * the object.
3029 */
3030 size += 2 * sizeof(struct track);
3031
be7b3fbc 3032 if (flags & SLAB_RED_ZONE)
81819f0f
CL
3033 /*
3034 * Add some empty padding so that we can catch
3035 * overwrites from earlier objects rather than let
3036 * tracking information or the free pointer be
0211a9c8 3037 * corrupted if a user writes before the start
81819f0f
CL
3038 * of the object.
3039 */
3040 size += sizeof(void *);
41ecc55b 3041#endif
672bba3a 3042
81819f0f
CL
3043 /*
3044 * SLUB stores one object immediately after another beginning from
3045 * offset 0. In order to align the objects we have to simply size
3046 * each object to conform to the alignment.
3047 */
45906855 3048 size = ALIGN(size, s->align);
81819f0f 3049 s->size = size;
06b285dc
CL
3050 if (forced_order >= 0)
3051 order = forced_order;
3052 else
ab9a0f19 3053 order = calculate_order(size, s->reserved);
81819f0f 3054
834f3d11 3055 if (order < 0)
81819f0f
CL
3056 return 0;
3057
b7a49f0d 3058 s->allocflags = 0;
834f3d11 3059 if (order)
b7a49f0d
CL
3060 s->allocflags |= __GFP_COMP;
3061
3062 if (s->flags & SLAB_CACHE_DMA)
2c59dd65 3063 s->allocflags |= GFP_DMA;
b7a49f0d
CL
3064
3065 if (s->flags & SLAB_RECLAIM_ACCOUNT)
3066 s->allocflags |= __GFP_RECLAIMABLE;
3067
81819f0f
CL
3068 /*
3069 * Determine the number of objects per slab
3070 */
ab9a0f19
LJ
3071 s->oo = oo_make(order, size, s->reserved);
3072 s->min = oo_make(get_order(size), size, s->reserved);
205ab99d
CL
3073 if (oo_objects(s->oo) > oo_objects(s->max))
3074 s->max = s->oo;
81819f0f 3075
834f3d11 3076 return !!oo_objects(s->oo);
81819f0f
CL
3077}
3078
8a13a4cc 3079static int kmem_cache_open(struct kmem_cache *s, unsigned long flags)
81819f0f 3080{
8a13a4cc 3081 s->flags = kmem_cache_flags(s->size, flags, s->name, s->ctor);
ab9a0f19 3082 s->reserved = 0;
81819f0f 3083
da9a638c
LJ
3084 if (need_reserve_slab_rcu && (s->flags & SLAB_DESTROY_BY_RCU))
3085 s->reserved = sizeof(struct rcu_head);
81819f0f 3086
06b285dc 3087 if (!calculate_sizes(s, -1))
81819f0f 3088 goto error;
3de47213
DR
3089 if (disable_higher_order_debug) {
3090 /*
3091 * Disable debugging flags that store metadata if the min slab
3092 * order increased.
3093 */
3b0efdfa 3094 if (get_order(s->size) > get_order(s->object_size)) {
3de47213
DR
3095 s->flags &= ~DEBUG_METADATA_FLAGS;
3096 s->offset = 0;
3097 if (!calculate_sizes(s, -1))
3098 goto error;
3099 }
3100 }
81819f0f 3101
2565409f
HC
3102#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
3103 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
b789ef51
CL
3104 if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0)
3105 /* Enable fast mode */
3106 s->flags |= __CMPXCHG_DOUBLE;
3107#endif
3108
3b89d7d8
DR
3109 /*
3110 * The larger the object size is, the more pages we want on the partial
3111 * list to avoid pounding the page allocator excessively.
3112 */
49e22585
CL
3113 set_min_partial(s, ilog2(s->size) / 2);
3114
3115 /*
3116 * cpu_partial determined the maximum number of objects kept in the
3117 * per cpu partial lists of a processor.
3118 *
3119 * Per cpu partial lists mainly contain slabs that just have one
3120 * object freed. If they are used for allocation then they can be
3121 * filled up again with minimal effort. The slab will never hit the
3122 * per node partial lists and therefore no locking will be required.
3123 *
3124 * This setting also determines
3125 *
3126 * A) The number of objects from per cpu partial slabs dumped to the
3127 * per node list when we reach the limit.
9f264904 3128 * B) The number of objects in cpu partial slabs to extract from the
d0e0ac97
CG
3129 * per node list when we run out of per cpu objects. We only fetch
3130 * 50% to keep some capacity around for frees.
49e22585 3131 */
345c905d 3132 if (!kmem_cache_has_cpu_partial(s))
8f1e33da
CL
3133 s->cpu_partial = 0;
3134 else if (s->size >= PAGE_SIZE)
49e22585
CL
3135 s->cpu_partial = 2;
3136 else if (s->size >= 1024)
3137 s->cpu_partial = 6;
3138 else if (s->size >= 256)
3139 s->cpu_partial = 13;
3140 else
3141 s->cpu_partial = 30;
3142
81819f0f 3143#ifdef CONFIG_NUMA
e2cb96b7 3144 s->remote_node_defrag_ratio = 1000;
81819f0f 3145#endif
55136592 3146 if (!init_kmem_cache_nodes(s))
dfb4f096 3147 goto error;
81819f0f 3148
55136592 3149 if (alloc_kmem_cache_cpus(s))
278b1bb1 3150 return 0;
ff12059e 3151
4c93c355 3152 free_kmem_cache_nodes(s);
81819f0f
CL
3153error:
3154 if (flags & SLAB_PANIC)
3155 panic("Cannot create slab %s size=%lu realsize=%u "
3156 "order=%u offset=%u flags=%lx\n",
d0e0ac97
CG
3157 s->name, (unsigned long)s->size, s->size,
3158 oo_order(s->oo), s->offset, flags);
278b1bb1 3159 return -EINVAL;
81819f0f 3160}
81819f0f 3161
33b12c38
CL
3162static void list_slab_objects(struct kmem_cache *s, struct page *page,
3163 const char *text)
3164{
3165#ifdef CONFIG_SLUB_DEBUG
3166 void *addr = page_address(page);
3167 void *p;
a5dd5c11
NK
3168 unsigned long *map = kzalloc(BITS_TO_LONGS(page->objects) *
3169 sizeof(long), GFP_ATOMIC);
bbd7d57b
ED
3170 if (!map)
3171 return;
945cf2b6 3172 slab_err(s, page, text, s->name);
33b12c38 3173 slab_lock(page);
33b12c38 3174
5f80b13a 3175 get_map(s, page, map);
33b12c38
CL
3176 for_each_object(p, s, addr, page->objects) {
3177
3178 if (!test_bit(slab_index(p, s, addr), map)) {
f9f58285 3179 pr_err("INFO: Object 0x%p @offset=%tu\n", p, p - addr);
33b12c38
CL
3180 print_tracking(s, p);
3181 }
3182 }
3183 slab_unlock(page);
bbd7d57b 3184 kfree(map);
33b12c38
CL
3185#endif
3186}
3187
81819f0f 3188/*
599870b1 3189 * Attempt to free all partial slabs on a node.
69cb8e6b
CL
3190 * This is called from kmem_cache_close(). We must be the last thread
3191 * using the cache and therefore we do not need to lock anymore.
81819f0f 3192 */
599870b1 3193static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
81819f0f 3194{
81819f0f
CL
3195 struct page *page, *h;
3196
33b12c38 3197 list_for_each_entry_safe(page, h, &n->partial, lru) {
81819f0f 3198 if (!page->inuse) {
1e4dd946 3199 __remove_partial(n, page);
81819f0f 3200 discard_slab(s, page);
33b12c38
CL
3201 } else {
3202 list_slab_objects(s, page,
945cf2b6 3203 "Objects remaining in %s on kmem_cache_close()");
599870b1 3204 }
33b12c38 3205 }
81819f0f
CL
3206}
3207
3208/*
672bba3a 3209 * Release all resources used by a slab cache.
81819f0f 3210 */
0c710013 3211static inline int kmem_cache_close(struct kmem_cache *s)
81819f0f
CL
3212{
3213 int node;
3214
3215 flush_all(s);
81819f0f 3216 /* Attempt to free all objects */
f64dc58c 3217 for_each_node_state(node, N_NORMAL_MEMORY) {
81819f0f
CL
3218 struct kmem_cache_node *n = get_node(s, node);
3219
599870b1
CL
3220 free_partial(s, n);
3221 if (n->nr_partial || slabs_node(s, node))
81819f0f
CL
3222 return 1;
3223 }
945cf2b6 3224 free_percpu(s->cpu_slab);
81819f0f
CL
3225 free_kmem_cache_nodes(s);
3226 return 0;
3227}
3228
945cf2b6 3229int __kmem_cache_shutdown(struct kmem_cache *s)
81819f0f 3230{
41a21285 3231 return kmem_cache_close(s);
81819f0f 3232}
81819f0f
CL
3233
3234/********************************************************************
3235 * Kmalloc subsystem
3236 *******************************************************************/
3237
81819f0f
CL
3238static int __init setup_slub_min_order(char *str)
3239{
06428780 3240 get_option(&str, &slub_min_order);
81819f0f
CL
3241
3242 return 1;
3243}
3244
3245__setup("slub_min_order=", setup_slub_min_order);
3246
3247static int __init setup_slub_max_order(char *str)
3248{
06428780 3249 get_option(&str, &slub_max_order);
818cf590 3250 slub_max_order = min(slub_max_order, MAX_ORDER - 1);
81819f0f
CL
3251
3252 return 1;
3253}
3254
3255__setup("slub_max_order=", setup_slub_max_order);
3256
3257static int __init setup_slub_min_objects(char *str)
3258{
06428780 3259 get_option(&str, &slub_min_objects);
81819f0f
CL
3260
3261 return 1;
3262}
3263
3264__setup("slub_min_objects=", setup_slub_min_objects);
3265
3266static int __init setup_slub_nomerge(char *str)
3267{
3268 slub_nomerge = 1;
3269 return 1;
3270}
3271
3272__setup("slub_nomerge", setup_slub_nomerge);
3273
81819f0f
CL
3274void *__kmalloc(size_t size, gfp_t flags)
3275{
aadb4bc4 3276 struct kmem_cache *s;
5b882be4 3277 void *ret;
81819f0f 3278
95a05b42 3279 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
eada35ef 3280 return kmalloc_large(size, flags);
aadb4bc4 3281
2c59dd65 3282 s = kmalloc_slab(size, flags);
aadb4bc4
CL
3283
3284 if (unlikely(ZERO_OR_NULL_PTR(s)))
6cb8f913
CL
3285 return s;
3286
2b847c3c 3287 ret = slab_alloc(s, flags, _RET_IP_);
5b882be4 3288
ca2b84cb 3289 trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
5b882be4
EGM
3290
3291 return ret;
81819f0f
CL
3292}
3293EXPORT_SYMBOL(__kmalloc);
3294
5d1f57e4 3295#ifdef CONFIG_NUMA
f619cfe1
CL
3296static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
3297{
b1eeab67 3298 struct page *page;
e4f7c0b4 3299 void *ptr = NULL;
f619cfe1 3300
d79923fa 3301 flags |= __GFP_COMP | __GFP_NOTRACK | __GFP_KMEMCG;
b1eeab67 3302 page = alloc_pages_node(node, flags, get_order(size));
f619cfe1 3303 if (page)
e4f7c0b4
CM
3304 ptr = page_address(page);
3305
d56791b3 3306 kmalloc_large_node_hook(ptr, size, flags);
e4f7c0b4 3307 return ptr;
f619cfe1
CL
3308}
3309
81819f0f
CL
3310void *__kmalloc_node(size_t size, gfp_t flags, int node)
3311{
aadb4bc4 3312 struct kmem_cache *s;
5b882be4 3313 void *ret;
81819f0f 3314
95a05b42 3315 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
5b882be4
EGM
3316 ret = kmalloc_large_node(size, flags, node);
3317
ca2b84cb
EGM
3318 trace_kmalloc_node(_RET_IP_, ret,
3319 size, PAGE_SIZE << get_order(size),
3320 flags, node);
5b882be4
EGM
3321
3322 return ret;
3323 }
aadb4bc4 3324
2c59dd65 3325 s = kmalloc_slab(size, flags);
aadb4bc4
CL
3326
3327 if (unlikely(ZERO_OR_NULL_PTR(s)))
6cb8f913
CL
3328 return s;
3329
2b847c3c 3330 ret = slab_alloc_node(s, flags, node, _RET_IP_);
5b882be4 3331
ca2b84cb 3332 trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
5b882be4
EGM
3333
3334 return ret;
81819f0f
CL
3335}
3336EXPORT_SYMBOL(__kmalloc_node);
3337#endif
3338
3339size_t ksize(const void *object)
3340{
272c1d21 3341 struct page *page;
81819f0f 3342
ef8b4520 3343 if (unlikely(object == ZERO_SIZE_PTR))
272c1d21
CL
3344 return 0;
3345
294a80a8 3346 page = virt_to_head_page(object);
294a80a8 3347
76994412
PE
3348 if (unlikely(!PageSlab(page))) {
3349 WARN_ON(!PageCompound(page));
294a80a8 3350 return PAGE_SIZE << compound_order(page);
76994412 3351 }
81819f0f 3352
1b4f59e3 3353 return slab_ksize(page->slab_cache);
81819f0f 3354}
b1aabecd 3355EXPORT_SYMBOL(ksize);
81819f0f
CL
3356
3357void kfree(const void *x)
3358{
81819f0f 3359 struct page *page;
5bb983b0 3360 void *object = (void *)x;
81819f0f 3361
2121db74
PE
3362 trace_kfree(_RET_IP_, x);
3363
2408c550 3364 if (unlikely(ZERO_OR_NULL_PTR(x)))
81819f0f
CL
3365 return;
3366
b49af68f 3367 page = virt_to_head_page(x);
aadb4bc4 3368 if (unlikely(!PageSlab(page))) {
0937502a 3369 BUG_ON(!PageCompound(page));
d56791b3 3370 kfree_hook(x);
d79923fa 3371 __free_memcg_kmem_pages(page, compound_order(page));
aadb4bc4
CL
3372 return;
3373 }
1b4f59e3 3374 slab_free(page->slab_cache, page, object, _RET_IP_);
81819f0f
CL
3375}
3376EXPORT_SYMBOL(kfree);
3377
2086d26a 3378/*
672bba3a
CL
3379 * kmem_cache_shrink removes empty slabs from the partial lists and sorts
3380 * the remaining slabs by the number of items in use. The slabs with the
3381 * most items in use come first. New allocations will then fill those up
3382 * and thus they can be removed from the partial lists.
3383 *
3384 * The slabs with the least items are placed last. This results in them
3385 * being allocated from last increasing the chance that the last objects
3386 * are freed in them.
2086d26a
CL
3387 */
3388int kmem_cache_shrink(struct kmem_cache *s)
3389{
3390 int node;
3391 int i;
3392 struct kmem_cache_node *n;
3393 struct page *page;
3394 struct page *t;
205ab99d 3395 int objects = oo_objects(s->max);
2086d26a 3396 struct list_head *slabs_by_inuse =
834f3d11 3397 kmalloc(sizeof(struct list_head) * objects, GFP_KERNEL);
2086d26a
CL
3398 unsigned long flags;
3399
3400 if (!slabs_by_inuse)
3401 return -ENOMEM;
3402
3403 flush_all(s);
f64dc58c 3404 for_each_node_state(node, N_NORMAL_MEMORY) {
2086d26a
CL
3405 n = get_node(s, node);
3406
3407 if (!n->nr_partial)
3408 continue;
3409
834f3d11 3410 for (i = 0; i < objects; i++)
2086d26a
CL
3411 INIT_LIST_HEAD(slabs_by_inuse + i);
3412
3413 spin_lock_irqsave(&n->list_lock, flags);
3414
3415 /*
672bba3a 3416 * Build lists indexed by the items in use in each slab.
2086d26a 3417 *
672bba3a
CL
3418 * Note that concurrent frees may occur while we hold the
3419 * list_lock. page->inuse here is the upper limit.
2086d26a
CL
3420 */
3421 list_for_each_entry_safe(page, t, &n->partial, lru) {
69cb8e6b
CL
3422 list_move(&page->lru, slabs_by_inuse + page->inuse);
3423 if (!page->inuse)
3424 n->nr_partial--;
2086d26a
CL
3425 }
3426
2086d26a 3427 /*
672bba3a
CL
3428 * Rebuild the partial list with the slabs filled up most
3429 * first and the least used slabs at the end.
2086d26a 3430 */
69cb8e6b 3431 for (i = objects - 1; i > 0; i--)
2086d26a
CL
3432 list_splice(slabs_by_inuse + i, n->partial.prev);
3433
2086d26a 3434 spin_unlock_irqrestore(&n->list_lock, flags);
69cb8e6b
CL
3435
3436 /* Release empty slabs */
3437 list_for_each_entry_safe(page, t, slabs_by_inuse, lru)
3438 discard_slab(s, page);
2086d26a
CL
3439 }
3440
3441 kfree(slabs_by_inuse);
3442 return 0;
3443}
3444EXPORT_SYMBOL(kmem_cache_shrink);
3445
b9049e23
YG
3446static int slab_mem_going_offline_callback(void *arg)
3447{
3448 struct kmem_cache *s;
3449
18004c5d 3450 mutex_lock(&slab_mutex);
b9049e23
YG
3451 list_for_each_entry(s, &slab_caches, list)
3452 kmem_cache_shrink(s);
18004c5d 3453 mutex_unlock(&slab_mutex);
b9049e23
YG
3454
3455 return 0;
3456}
3457
3458static void slab_mem_offline_callback(void *arg)
3459{
3460 struct kmem_cache_node *n;
3461 struct kmem_cache *s;
3462 struct memory_notify *marg = arg;
3463 int offline_node;
3464
b9d5ab25 3465 offline_node = marg->status_change_nid_normal;
b9049e23
YG
3466
3467 /*
3468 * If the node still has available memory. we need kmem_cache_node
3469 * for it yet.
3470 */
3471 if (offline_node < 0)
3472 return;
3473
18004c5d 3474 mutex_lock(&slab_mutex);
b9049e23
YG
3475 list_for_each_entry(s, &slab_caches, list) {
3476 n = get_node(s, offline_node);
3477 if (n) {
3478 /*
3479 * if n->nr_slabs > 0, slabs still exist on the node
3480 * that is going down. We were unable to free them,
c9404c9c 3481 * and offline_pages() function shouldn't call this
b9049e23
YG
3482 * callback. So, we must fail.
3483 */
0f389ec6 3484 BUG_ON(slabs_node(s, offline_node));
b9049e23
YG
3485
3486 s->node[offline_node] = NULL;
8de66a0c 3487 kmem_cache_free(kmem_cache_node, n);
b9049e23
YG
3488 }
3489 }
18004c5d 3490 mutex_unlock(&slab_mutex);
b9049e23
YG
3491}
3492
3493static int slab_mem_going_online_callback(void *arg)
3494{
3495 struct kmem_cache_node *n;
3496 struct kmem_cache *s;
3497 struct memory_notify *marg = arg;
b9d5ab25 3498 int nid = marg->status_change_nid_normal;
b9049e23
YG
3499 int ret = 0;
3500
3501 /*
3502 * If the node's memory is already available, then kmem_cache_node is
3503 * already created. Nothing to do.
3504 */
3505 if (nid < 0)
3506 return 0;
3507
3508 /*
0121c619 3509 * We are bringing a node online. No memory is available yet. We must
b9049e23
YG
3510 * allocate a kmem_cache_node structure in order to bring the node
3511 * online.
3512 */
18004c5d 3513 mutex_lock(&slab_mutex);
b9049e23
YG
3514 list_for_each_entry(s, &slab_caches, list) {
3515 /*
3516 * XXX: kmem_cache_alloc_node will fallback to other nodes
3517 * since memory is not yet available from the node that
3518 * is brought up.
3519 */
8de66a0c 3520 n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
b9049e23
YG
3521 if (!n) {
3522 ret = -ENOMEM;
3523 goto out;
3524 }
4053497d 3525 init_kmem_cache_node(n);
b9049e23
YG
3526 s->node[nid] = n;
3527 }
3528out:
18004c5d 3529 mutex_unlock(&slab_mutex);
b9049e23
YG
3530 return ret;
3531}
3532
3533static int slab_memory_callback(struct notifier_block *self,
3534 unsigned long action, void *arg)
3535{
3536 int ret = 0;
3537
3538 switch (action) {
3539 case MEM_GOING_ONLINE:
3540 ret = slab_mem_going_online_callback(arg);
3541 break;
3542 case MEM_GOING_OFFLINE:
3543 ret = slab_mem_going_offline_callback(arg);
3544 break;
3545 case MEM_OFFLINE:
3546 case MEM_CANCEL_ONLINE:
3547 slab_mem_offline_callback(arg);
3548 break;
3549 case MEM_ONLINE:
3550 case MEM_CANCEL_OFFLINE:
3551 break;
3552 }
dc19f9db
KH
3553 if (ret)
3554 ret = notifier_from_errno(ret);
3555 else
3556 ret = NOTIFY_OK;
b9049e23
YG
3557 return ret;
3558}
3559
3ac38faa
AM
3560static struct notifier_block slab_memory_callback_nb = {
3561 .notifier_call = slab_memory_callback,
3562 .priority = SLAB_CALLBACK_PRI,
3563};
b9049e23 3564
81819f0f
CL
3565/********************************************************************
3566 * Basic setup of slabs
3567 *******************************************************************/
3568
51df1142
CL
3569/*
3570 * Used for early kmem_cache structures that were allocated using
dffb4d60
CL
3571 * the page allocator. Allocate them properly then fix up the pointers
3572 * that may be pointing to the wrong kmem_cache structure.
51df1142
CL
3573 */
3574
dffb4d60 3575static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
51df1142
CL
3576{
3577 int node;
dffb4d60 3578 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
51df1142 3579
dffb4d60 3580 memcpy(s, static_cache, kmem_cache->object_size);
51df1142 3581
7d557b3c
GC
3582 /*
3583 * This runs very early, and only the boot processor is supposed to be
3584 * up. Even if it weren't true, IRQs are not up so we couldn't fire
3585 * IPIs around.
3586 */
3587 __flush_cpu_slab(s, smp_processor_id());
51df1142
CL
3588 for_each_node_state(node, N_NORMAL_MEMORY) {
3589 struct kmem_cache_node *n = get_node(s, node);
3590 struct page *p;
3591
3592 if (n) {
3593 list_for_each_entry(p, &n->partial, lru)
1b4f59e3 3594 p->slab_cache = s;
51df1142 3595
607bf324 3596#ifdef CONFIG_SLUB_DEBUG
51df1142 3597 list_for_each_entry(p, &n->full, lru)
1b4f59e3 3598 p->slab_cache = s;
51df1142
CL
3599#endif
3600 }
3601 }
dffb4d60
CL
3602 list_add(&s->list, &slab_caches);
3603 return s;
51df1142
CL
3604}
3605
81819f0f
CL
3606void __init kmem_cache_init(void)
3607{
dffb4d60
CL
3608 static __initdata struct kmem_cache boot_kmem_cache,
3609 boot_kmem_cache_node;
51df1142 3610
fc8d8620
SG
3611 if (debug_guardpage_minorder())
3612 slub_max_order = 0;
3613
dffb4d60
CL
3614 kmem_cache_node = &boot_kmem_cache_node;
3615 kmem_cache = &boot_kmem_cache;
51df1142 3616
dffb4d60
CL
3617 create_boot_cache(kmem_cache_node, "kmem_cache_node",
3618 sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN);
b9049e23 3619
3ac38faa 3620 register_hotmemory_notifier(&slab_memory_callback_nb);
81819f0f
CL
3621
3622 /* Able to allocate the per node structures */
3623 slab_state = PARTIAL;
3624
dffb4d60
CL
3625 create_boot_cache(kmem_cache, "kmem_cache",
3626 offsetof(struct kmem_cache, node) +
3627 nr_node_ids * sizeof(struct kmem_cache_node *),
3628 SLAB_HWCACHE_ALIGN);
8a13a4cc 3629
dffb4d60 3630 kmem_cache = bootstrap(&boot_kmem_cache);
81819f0f 3631
51df1142
CL
3632 /*
3633 * Allocate kmem_cache_node properly from the kmem_cache slab.
3634 * kmem_cache_node is separately allocated so no need to
3635 * update any list pointers.
3636 */
dffb4d60 3637 kmem_cache_node = bootstrap(&boot_kmem_cache_node);
51df1142
CL
3638
3639 /* Now we can use the kmem_cache to allocate kmalloc slabs */
f97d5f63 3640 create_kmalloc_caches(0);
81819f0f
CL
3641
3642#ifdef CONFIG_SMP
3643 register_cpu_notifier(&slab_notifier);
9dfc6e68 3644#endif
81819f0f 3645
f9f58285 3646 pr_info("SLUB: HWalign=%d, Order=%d-%d, MinObjects=%d, CPUs=%d, Nodes=%d\n",
f97d5f63 3647 cache_line_size(),
81819f0f
CL
3648 slub_min_order, slub_max_order, slub_min_objects,
3649 nr_cpu_ids, nr_node_ids);
3650}
3651
7e85ee0c
PE
3652void __init kmem_cache_init_late(void)
3653{
7e85ee0c
PE
3654}
3655
81819f0f
CL
3656/*
3657 * Find a mergeable slab cache
3658 */
3659static int slab_unmergeable(struct kmem_cache *s)
3660{
3661 if (slub_nomerge || (s->flags & SLUB_NEVER_MERGE))
3662 return 1;
3663
a44cb944
VD
3664 if (!is_root_cache(s))
3665 return 1;
3666
c59def9f 3667 if (s->ctor)
81819f0f
CL
3668 return 1;
3669
8ffa6875
CL
3670 /*
3671 * We may have set a slab to be unmergeable during bootstrap.
3672 */
3673 if (s->refcount < 0)
3674 return 1;
3675
81819f0f
CL
3676 return 0;
3677}
3678
a44cb944
VD
3679static struct kmem_cache *find_mergeable(size_t size, size_t align,
3680 unsigned long flags, const char *name, void (*ctor)(void *))
81819f0f 3681{
5b95a4ac 3682 struct kmem_cache *s;
81819f0f
CL
3683
3684 if (slub_nomerge || (flags & SLUB_NEVER_MERGE))
3685 return NULL;
3686
c59def9f 3687 if (ctor)
81819f0f
CL
3688 return NULL;
3689
3690 size = ALIGN(size, sizeof(void *));
3691 align = calculate_alignment(flags, align, size);
3692 size = ALIGN(size, align);
ba0268a8 3693 flags = kmem_cache_flags(size, flags, name, NULL);
81819f0f 3694
5b95a4ac 3695 list_for_each_entry(s, &slab_caches, list) {
81819f0f
CL
3696 if (slab_unmergeable(s))
3697 continue;
3698
3699 if (size > s->size)
3700 continue;
3701
ba0268a8 3702 if ((flags & SLUB_MERGE_SAME) != (s->flags & SLUB_MERGE_SAME))
a44cb944 3703 continue;
81819f0f
CL
3704 /*
3705 * Check if alignment is compatible.
3706 * Courtesy of Adrian Drzewiecki
3707 */
06428780 3708 if ((s->size & ~(align - 1)) != s->size)
81819f0f
CL
3709 continue;
3710
3711 if (s->size - size >= sizeof(void *))
3712 continue;
3713
3714 return s;
3715 }
3716 return NULL;
3717}
3718
2633d7a0 3719struct kmem_cache *
a44cb944
VD
3720__kmem_cache_alias(const char *name, size_t size, size_t align,
3721 unsigned long flags, void (*ctor)(void *))
81819f0f
CL
3722{
3723 struct kmem_cache *s;
3724
a44cb944 3725 s = find_mergeable(size, align, flags, name, ctor);
81819f0f 3726 if (s) {
84d0ddd6
VD
3727 int i;
3728 struct kmem_cache *c;
3729
81819f0f 3730 s->refcount++;
84d0ddd6 3731
81819f0f
CL
3732 /*
3733 * Adjust the object sizes so that we clear
3734 * the complete object on kzalloc.
3735 */
3b0efdfa 3736 s->object_size = max(s->object_size, (int)size);
81819f0f 3737 s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
6446faa2 3738
84d0ddd6
VD
3739 for_each_memcg_cache_index(i) {
3740 c = cache_from_memcg_idx(s, i);
3741 if (!c)
3742 continue;
3743 c->object_size = s->object_size;
3744 c->inuse = max_t(int, c->inuse,
3745 ALIGN(size, sizeof(void *)));
3746 }
3747
7b8f3b66 3748 if (sysfs_slab_alias(s, name)) {
7b8f3b66 3749 s->refcount--;
cbb79694 3750 s = NULL;
7b8f3b66 3751 }
a0e1d1be 3752 }
6446faa2 3753
cbb79694
CL
3754 return s;
3755}
84c1cf62 3756
8a13a4cc 3757int __kmem_cache_create(struct kmem_cache *s, unsigned long flags)
cbb79694 3758{
aac3a166
PE
3759 int err;
3760
3761 err = kmem_cache_open(s, flags);
3762 if (err)
3763 return err;
20cea968 3764
45530c44
CL
3765 /* Mutex is not taken during early boot */
3766 if (slab_state <= UP)
3767 return 0;
3768
107dab5c 3769 memcg_propagate_slab_attrs(s);
aac3a166 3770 err = sysfs_slab_add(s);
aac3a166
PE
3771 if (err)
3772 kmem_cache_close(s);
20cea968 3773
aac3a166 3774 return err;
81819f0f 3775}
81819f0f 3776
81819f0f 3777#ifdef CONFIG_SMP
81819f0f 3778/*
672bba3a
CL
3779 * Use the cpu notifier to insure that the cpu slabs are flushed when
3780 * necessary.
81819f0f 3781 */
0db0628d 3782static int slab_cpuup_callback(struct notifier_block *nfb,
81819f0f
CL
3783 unsigned long action, void *hcpu)
3784{
3785 long cpu = (long)hcpu;
5b95a4ac
CL
3786 struct kmem_cache *s;
3787 unsigned long flags;
81819f0f
CL
3788
3789 switch (action) {
3790 case CPU_UP_CANCELED:
8bb78442 3791 case CPU_UP_CANCELED_FROZEN:
81819f0f 3792 case CPU_DEAD:
8bb78442 3793 case CPU_DEAD_FROZEN:
18004c5d 3794 mutex_lock(&slab_mutex);
5b95a4ac
CL
3795 list_for_each_entry(s, &slab_caches, list) {
3796 local_irq_save(flags);
3797 __flush_cpu_slab(s, cpu);
3798 local_irq_restore(flags);
3799 }
18004c5d 3800 mutex_unlock(&slab_mutex);
81819f0f
CL
3801 break;
3802 default:
3803 break;
3804 }
3805 return NOTIFY_OK;
3806}
3807
0db0628d 3808static struct notifier_block slab_notifier = {
3adbefee 3809 .notifier_call = slab_cpuup_callback
06428780 3810};
81819f0f
CL
3811
3812#endif
3813
ce71e27c 3814void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
81819f0f 3815{
aadb4bc4 3816 struct kmem_cache *s;
94b528d0 3817 void *ret;
aadb4bc4 3818
95a05b42 3819 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
eada35ef
PE
3820 return kmalloc_large(size, gfpflags);
3821
2c59dd65 3822 s = kmalloc_slab(size, gfpflags);
81819f0f 3823
2408c550 3824 if (unlikely(ZERO_OR_NULL_PTR(s)))
6cb8f913 3825 return s;
81819f0f 3826
2b847c3c 3827 ret = slab_alloc(s, gfpflags, caller);
94b528d0 3828
25985edc 3829 /* Honor the call site pointer we received. */
ca2b84cb 3830 trace_kmalloc(caller, ret, size, s->size, gfpflags);
94b528d0
EGM
3831
3832 return ret;
81819f0f
CL
3833}
3834
5d1f57e4 3835#ifdef CONFIG_NUMA
81819f0f 3836void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
ce71e27c 3837 int node, unsigned long caller)
81819f0f 3838{
aadb4bc4 3839 struct kmem_cache *s;
94b528d0 3840 void *ret;
aadb4bc4 3841
95a05b42 3842 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
d3e14aa3
XF
3843 ret = kmalloc_large_node(size, gfpflags, node);
3844
3845 trace_kmalloc_node(caller, ret,
3846 size, PAGE_SIZE << get_order(size),
3847 gfpflags, node);
3848
3849 return ret;
3850 }
eada35ef 3851
2c59dd65 3852 s = kmalloc_slab(size, gfpflags);
81819f0f 3853
2408c550 3854 if (unlikely(ZERO_OR_NULL_PTR(s)))
6cb8f913 3855 return s;
81819f0f 3856
2b847c3c 3857 ret = slab_alloc_node(s, gfpflags, node, caller);
94b528d0 3858
25985edc 3859 /* Honor the call site pointer we received. */
ca2b84cb 3860 trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
94b528d0
EGM
3861
3862 return ret;
81819f0f 3863}
5d1f57e4 3864#endif
81819f0f 3865
ab4d5ed5 3866#ifdef CONFIG_SYSFS
205ab99d
CL
3867static int count_inuse(struct page *page)
3868{
3869 return page->inuse;
3870}
3871
3872static int count_total(struct page *page)
3873{
3874 return page->objects;
3875}
ab4d5ed5 3876#endif
205ab99d 3877
ab4d5ed5 3878#ifdef CONFIG_SLUB_DEBUG
434e245d
CL
3879static int validate_slab(struct kmem_cache *s, struct page *page,
3880 unsigned long *map)
53e15af0
CL
3881{
3882 void *p;
a973e9dd 3883 void *addr = page_address(page);
53e15af0
CL
3884
3885 if (!check_slab(s, page) ||
3886 !on_freelist(s, page, NULL))
3887 return 0;
3888
3889 /* Now we know that a valid freelist exists */
39b26464 3890 bitmap_zero(map, page->objects);
53e15af0 3891
5f80b13a
CL
3892 get_map(s, page, map);
3893 for_each_object(p, s, addr, page->objects) {
3894 if (test_bit(slab_index(p, s, addr), map))
3895 if (!check_object(s, page, p, SLUB_RED_INACTIVE))
3896 return 0;
53e15af0
CL
3897 }
3898
224a88be 3899 for_each_object(p, s, addr, page->objects)
7656c72b 3900 if (!test_bit(slab_index(p, s, addr), map))
37d57443 3901 if (!check_object(s, page, p, SLUB_RED_ACTIVE))
53e15af0
CL
3902 return 0;
3903 return 1;
3904}
3905
434e245d
CL
3906static void validate_slab_slab(struct kmem_cache *s, struct page *page,
3907 unsigned long *map)
53e15af0 3908{
881db7fb
CL
3909 slab_lock(page);
3910 validate_slab(s, page, map);
3911 slab_unlock(page);
53e15af0
CL
3912}
3913
434e245d
CL
3914static int validate_slab_node(struct kmem_cache *s,
3915 struct kmem_cache_node *n, unsigned long *map)
53e15af0
CL
3916{
3917 unsigned long count = 0;
3918 struct page *page;
3919 unsigned long flags;
3920
3921 spin_lock_irqsave(&n->list_lock, flags);
3922
3923 list_for_each_entry(page, &n->partial, lru) {
434e245d 3924 validate_slab_slab(s, page, map);
53e15af0
CL
3925 count++;
3926 }
3927 if (count != n->nr_partial)
f9f58285
FF
3928 pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
3929 s->name, count, n->nr_partial);
53e15af0
CL
3930
3931 if (!(s->flags & SLAB_STORE_USER))
3932 goto out;
3933
3934 list_for_each_entry(page, &n->full, lru) {
434e245d 3935 validate_slab_slab(s, page, map);
53e15af0
CL
3936 count++;
3937 }
3938 if (count != atomic_long_read(&n->nr_slabs))
f9f58285
FF
3939 pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
3940 s->name, count, atomic_long_read(&n->nr_slabs));
53e15af0
CL
3941
3942out:
3943 spin_unlock_irqrestore(&n->list_lock, flags);
3944 return count;
3945}
3946
434e245d 3947static long validate_slab_cache(struct kmem_cache *s)
53e15af0
CL
3948{
3949 int node;
3950 unsigned long count = 0;
205ab99d 3951 unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
434e245d
CL
3952 sizeof(unsigned long), GFP_KERNEL);
3953
3954 if (!map)
3955 return -ENOMEM;
53e15af0
CL
3956
3957 flush_all(s);
f64dc58c 3958 for_each_node_state(node, N_NORMAL_MEMORY) {
53e15af0
CL
3959 struct kmem_cache_node *n = get_node(s, node);
3960
434e245d 3961 count += validate_slab_node(s, n, map);
53e15af0 3962 }
434e245d 3963 kfree(map);
53e15af0
CL
3964 return count;
3965}
88a420e4 3966/*
672bba3a 3967 * Generate lists of code addresses where slabcache objects are allocated
88a420e4
CL
3968 * and freed.
3969 */
3970
3971struct location {
3972 unsigned long count;
ce71e27c 3973 unsigned long addr;
45edfa58
CL
3974 long long sum_time;
3975 long min_time;
3976 long max_time;
3977 long min_pid;
3978 long max_pid;
174596a0 3979 DECLARE_BITMAP(cpus, NR_CPUS);
45edfa58 3980 nodemask_t nodes;
88a420e4
CL
3981};
3982
3983struct loc_track {
3984 unsigned long max;
3985 unsigned long count;
3986 struct location *loc;
3987};
3988
3989static void free_loc_track(struct loc_track *t)
3990{
3991 if (t->max)
3992 free_pages((unsigned long)t->loc,
3993 get_order(sizeof(struct location) * t->max));
3994}
3995
68dff6a9 3996static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
88a420e4
CL
3997{
3998 struct location *l;
3999 int order;
4000
88a420e4
CL
4001 order = get_order(sizeof(struct location) * max);
4002
68dff6a9 4003 l = (void *)__get_free_pages(flags, order);
88a420e4
CL
4004 if (!l)
4005 return 0;
4006
4007 if (t->count) {
4008 memcpy(l, t->loc, sizeof(struct location) * t->count);
4009 free_loc_track(t);
4010 }
4011 t->max = max;
4012 t->loc = l;
4013 return 1;
4014}
4015
4016static int add_location(struct loc_track *t, struct kmem_cache *s,
45edfa58 4017 const struct track *track)
88a420e4
CL
4018{
4019 long start, end, pos;
4020 struct location *l;
ce71e27c 4021 unsigned long caddr;
45edfa58 4022 unsigned long age = jiffies - track->when;
88a420e4
CL
4023
4024 start = -1;
4025 end = t->count;
4026
4027 for ( ; ; ) {
4028 pos = start + (end - start + 1) / 2;
4029
4030 /*
4031 * There is nothing at "end". If we end up there
4032 * we need to add something to before end.
4033 */
4034 if (pos == end)
4035 break;
4036
4037 caddr = t->loc[pos].addr;
45edfa58
CL
4038 if (track->addr == caddr) {
4039
4040 l = &t->loc[pos];
4041 l->count++;
4042 if (track->when) {
4043 l->sum_time += age;
4044 if (age < l->min_time)
4045 l->min_time = age;
4046 if (age > l->max_time)
4047 l->max_time = age;
4048
4049 if (track->pid < l->min_pid)
4050 l->min_pid = track->pid;
4051 if (track->pid > l->max_pid)
4052 l->max_pid = track->pid;
4053
174596a0
RR
4054 cpumask_set_cpu(track->cpu,
4055 to_cpumask(l->cpus));
45edfa58
CL
4056 }
4057 node_set(page_to_nid(virt_to_page(track)), l->nodes);
88a420e4
CL
4058 return 1;
4059 }
4060
45edfa58 4061 if (track->addr < caddr)
88a420e4
CL
4062 end = pos;
4063 else
4064 start = pos;
4065 }
4066
4067 /*
672bba3a 4068 * Not found. Insert new tracking element.
88a420e4 4069 */
68dff6a9 4070 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
88a420e4
CL
4071 return 0;
4072
4073 l = t->loc + pos;
4074 if (pos < t->count)
4075 memmove(l + 1, l,
4076 (t->count - pos) * sizeof(struct location));
4077 t->count++;
4078 l->count = 1;
45edfa58
CL
4079 l->addr = track->addr;
4080 l->sum_time = age;
4081 l->min_time = age;
4082 l->max_time = age;
4083 l->min_pid = track->pid;
4084 l->max_pid = track->pid;
174596a0
RR
4085 cpumask_clear(to_cpumask(l->cpus));
4086 cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
45edfa58
CL
4087 nodes_clear(l->nodes);
4088 node_set(page_to_nid(virt_to_page(track)), l->nodes);
88a420e4
CL
4089 return 1;
4090}
4091
4092static void process_slab(struct loc_track *t, struct kmem_cache *s,
bbd7d57b 4093 struct page *page, enum track_item alloc,
a5dd5c11 4094 unsigned long *map)
88a420e4 4095{
a973e9dd 4096 void *addr = page_address(page);
88a420e4
CL
4097 void *p;
4098
39b26464 4099 bitmap_zero(map, page->objects);
5f80b13a 4100 get_map(s, page, map);
88a420e4 4101
224a88be 4102 for_each_object(p, s, addr, page->objects)
45edfa58
CL
4103 if (!test_bit(slab_index(p, s, addr), map))
4104 add_location(t, s, get_track(s, p, alloc));
88a420e4
CL
4105}
4106
4107static int list_locations(struct kmem_cache *s, char *buf,
4108 enum track_item alloc)
4109{
e374d483 4110 int len = 0;
88a420e4 4111 unsigned long i;
68dff6a9 4112 struct loc_track t = { 0, 0, NULL };
88a420e4 4113 int node;
bbd7d57b
ED
4114 unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
4115 sizeof(unsigned long), GFP_KERNEL);
88a420e4 4116
bbd7d57b
ED
4117 if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location),
4118 GFP_TEMPORARY)) {
4119 kfree(map);
68dff6a9 4120 return sprintf(buf, "Out of memory\n");
bbd7d57b 4121 }
88a420e4
CL
4122 /* Push back cpu slabs */
4123 flush_all(s);
4124
f64dc58c 4125 for_each_node_state(node, N_NORMAL_MEMORY) {
88a420e4
CL
4126 struct kmem_cache_node *n = get_node(s, node);
4127 unsigned long flags;
4128 struct page *page;
4129
9e86943b 4130 if (!atomic_long_read(&n->nr_slabs))
88a420e4
CL
4131 continue;
4132
4133 spin_lock_irqsave(&n->list_lock, flags);
4134 list_for_each_entry(page, &n->partial, lru)
bbd7d57b 4135 process_slab(&t, s, page, alloc, map);
88a420e4 4136 list_for_each_entry(page, &n->full, lru)
bbd7d57b 4137 process_slab(&t, s, page, alloc, map);
88a420e4
CL
4138 spin_unlock_irqrestore(&n->list_lock, flags);
4139 }
4140
4141 for (i = 0; i < t.count; i++) {
45edfa58 4142 struct location *l = &t.loc[i];
88a420e4 4143
9c246247 4144 if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100)
88a420e4 4145 break;
e374d483 4146 len += sprintf(buf + len, "%7ld ", l->count);
45edfa58
CL
4147
4148 if (l->addr)
62c70bce 4149 len += sprintf(buf + len, "%pS", (void *)l->addr);
88a420e4 4150 else
e374d483 4151 len += sprintf(buf + len, "<not-available>");
45edfa58
CL
4152
4153 if (l->sum_time != l->min_time) {
e374d483 4154 len += sprintf(buf + len, " age=%ld/%ld/%ld",
f8bd2258
RZ
4155 l->min_time,
4156 (long)div_u64(l->sum_time, l->count),
4157 l->max_time);
45edfa58 4158 } else
e374d483 4159 len += sprintf(buf + len, " age=%ld",
45edfa58
CL
4160 l->min_time);
4161
4162 if (l->min_pid != l->max_pid)
e374d483 4163 len += sprintf(buf + len, " pid=%ld-%ld",
45edfa58
CL
4164 l->min_pid, l->max_pid);
4165 else
e374d483 4166 len += sprintf(buf + len, " pid=%ld",
45edfa58
CL
4167 l->min_pid);
4168
174596a0
RR
4169 if (num_online_cpus() > 1 &&
4170 !cpumask_empty(to_cpumask(l->cpus)) &&
e374d483
HH
4171 len < PAGE_SIZE - 60) {
4172 len += sprintf(buf + len, " cpus=");
d0e0ac97
CG
4173 len += cpulist_scnprintf(buf + len,
4174 PAGE_SIZE - len - 50,
174596a0 4175 to_cpumask(l->cpus));
45edfa58
CL
4176 }
4177
62bc62a8 4178 if (nr_online_nodes > 1 && !nodes_empty(l->nodes) &&
e374d483
HH
4179 len < PAGE_SIZE - 60) {
4180 len += sprintf(buf + len, " nodes=");
d0e0ac97
CG
4181 len += nodelist_scnprintf(buf + len,
4182 PAGE_SIZE - len - 50,
4183 l->nodes);
45edfa58
CL
4184 }
4185
e374d483 4186 len += sprintf(buf + len, "\n");
88a420e4
CL
4187 }
4188
4189 free_loc_track(&t);
bbd7d57b 4190 kfree(map);
88a420e4 4191 if (!t.count)
e374d483
HH
4192 len += sprintf(buf, "No data\n");
4193 return len;
88a420e4 4194}
ab4d5ed5 4195#endif
88a420e4 4196
a5a84755
CL
4197#ifdef SLUB_RESILIENCY_TEST
4198static void resiliency_test(void)
4199{
4200 u8 *p;
4201
95a05b42 4202 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || KMALLOC_SHIFT_HIGH < 10);
a5a84755 4203
f9f58285
FF
4204 pr_err("SLUB resiliency testing\n");
4205 pr_err("-----------------------\n");
4206 pr_err("A. Corruption after allocation\n");
a5a84755
CL
4207
4208 p = kzalloc(16, GFP_KERNEL);
4209 p[16] = 0x12;
f9f58285
FF
4210 pr_err("\n1. kmalloc-16: Clobber Redzone/next pointer 0x12->0x%p\n\n",
4211 p + 16);
a5a84755
CL
4212
4213 validate_slab_cache(kmalloc_caches[4]);
4214
4215 /* Hmmm... The next two are dangerous */
4216 p = kzalloc(32, GFP_KERNEL);
4217 p[32 + sizeof(void *)] = 0x34;
f9f58285
FF
4218 pr_err("\n2. kmalloc-32: Clobber next pointer/next slab 0x34 -> -0x%p\n",
4219 p);
4220 pr_err("If allocated object is overwritten then not detectable\n\n");
a5a84755
CL
4221
4222 validate_slab_cache(kmalloc_caches[5]);
4223 p = kzalloc(64, GFP_KERNEL);
4224 p += 64 + (get_cycles() & 0xff) * sizeof(void *);
4225 *p = 0x56;
f9f58285
FF
4226 pr_err("\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n",
4227 p);
4228 pr_err("If allocated object is overwritten then not detectable\n\n");
a5a84755
CL
4229 validate_slab_cache(kmalloc_caches[6]);
4230
f9f58285 4231 pr_err("\nB. Corruption after free\n");
a5a84755
CL
4232 p = kzalloc(128, GFP_KERNEL);
4233 kfree(p);
4234 *p = 0x78;
f9f58285 4235 pr_err("1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p);
a5a84755
CL
4236 validate_slab_cache(kmalloc_caches[7]);
4237
4238 p = kzalloc(256, GFP_KERNEL);
4239 kfree(p);
4240 p[50] = 0x9a;
f9f58285 4241 pr_err("\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", p);
a5a84755
CL
4242 validate_slab_cache(kmalloc_caches[8]);
4243
4244 p = kzalloc(512, GFP_KERNEL);
4245 kfree(p);
4246 p[512] = 0xab;
f9f58285 4247 pr_err("\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p);
a5a84755
CL
4248 validate_slab_cache(kmalloc_caches[9]);
4249}
4250#else
4251#ifdef CONFIG_SYSFS
4252static void resiliency_test(void) {};
4253#endif
4254#endif
4255
ab4d5ed5 4256#ifdef CONFIG_SYSFS
81819f0f 4257enum slab_stat_type {
205ab99d
CL
4258 SL_ALL, /* All slabs */
4259 SL_PARTIAL, /* Only partially allocated slabs */
4260 SL_CPU, /* Only slabs used for cpu caches */
4261 SL_OBJECTS, /* Determine allocated objects not slabs */
4262 SL_TOTAL /* Determine object capacity not slabs */
81819f0f
CL
4263};
4264
205ab99d 4265#define SO_ALL (1 << SL_ALL)
81819f0f
CL
4266#define SO_PARTIAL (1 << SL_PARTIAL)
4267#define SO_CPU (1 << SL_CPU)
4268#define SO_OBJECTS (1 << SL_OBJECTS)
205ab99d 4269#define SO_TOTAL (1 << SL_TOTAL)
81819f0f 4270
62e5c4b4
CG
4271static ssize_t show_slab_objects(struct kmem_cache *s,
4272 char *buf, unsigned long flags)
81819f0f
CL
4273{
4274 unsigned long total = 0;
81819f0f
CL
4275 int node;
4276 int x;
4277 unsigned long *nodes;
81819f0f 4278
e35e1a97 4279 nodes = kzalloc(sizeof(unsigned long) * nr_node_ids, GFP_KERNEL);
62e5c4b4
CG
4280 if (!nodes)
4281 return -ENOMEM;
81819f0f 4282
205ab99d
CL
4283 if (flags & SO_CPU) {
4284 int cpu;
81819f0f 4285
205ab99d 4286 for_each_possible_cpu(cpu) {
d0e0ac97
CG
4287 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
4288 cpu);
ec3ab083 4289 int node;
49e22585 4290 struct page *page;
dfb4f096 4291
bc6697d8 4292 page = ACCESS_ONCE(c->page);
ec3ab083
CL
4293 if (!page)
4294 continue;
205ab99d 4295
ec3ab083
CL
4296 node = page_to_nid(page);
4297 if (flags & SO_TOTAL)
4298 x = page->objects;
4299 else if (flags & SO_OBJECTS)
4300 x = page->inuse;
4301 else
4302 x = 1;
49e22585 4303
ec3ab083
CL
4304 total += x;
4305 nodes[node] += x;
4306
4307 page = ACCESS_ONCE(c->partial);
49e22585 4308 if (page) {
8afb1474
LZ
4309 node = page_to_nid(page);
4310 if (flags & SO_TOTAL)
4311 WARN_ON_ONCE(1);
4312 else if (flags & SO_OBJECTS)
4313 WARN_ON_ONCE(1);
4314 else
4315 x = page->pages;
bc6697d8
ED
4316 total += x;
4317 nodes[node] += x;
49e22585 4318 }
81819f0f
CL
4319 }
4320 }
4321
04d94879 4322 lock_memory_hotplug();
ab4d5ed5 4323#ifdef CONFIG_SLUB_DEBUG
205ab99d
CL
4324 if (flags & SO_ALL) {
4325 for_each_node_state(node, N_NORMAL_MEMORY) {
4326 struct kmem_cache_node *n = get_node(s, node);
4327
d0e0ac97
CG
4328 if (flags & SO_TOTAL)
4329 x = atomic_long_read(&n->total_objects);
4330 else if (flags & SO_OBJECTS)
4331 x = atomic_long_read(&n->total_objects) -
4332 count_partial(n, count_free);
81819f0f 4333 else
205ab99d 4334 x = atomic_long_read(&n->nr_slabs);
81819f0f
CL
4335 total += x;
4336 nodes[node] += x;
4337 }
4338
ab4d5ed5
CL
4339 } else
4340#endif
4341 if (flags & SO_PARTIAL) {
205ab99d
CL
4342 for_each_node_state(node, N_NORMAL_MEMORY) {
4343 struct kmem_cache_node *n = get_node(s, node);
81819f0f 4344
205ab99d
CL
4345 if (flags & SO_TOTAL)
4346 x = count_partial(n, count_total);
4347 else if (flags & SO_OBJECTS)
4348 x = count_partial(n, count_inuse);
81819f0f 4349 else
205ab99d 4350 x = n->nr_partial;
81819f0f
CL
4351 total += x;
4352 nodes[node] += x;
4353 }
4354 }
81819f0f
CL
4355 x = sprintf(buf, "%lu", total);
4356#ifdef CONFIG_NUMA
f64dc58c 4357 for_each_node_state(node, N_NORMAL_MEMORY)
81819f0f
CL
4358 if (nodes[node])
4359 x += sprintf(buf + x, " N%d=%lu",
4360 node, nodes[node]);
4361#endif
04d94879 4362 unlock_memory_hotplug();
81819f0f
CL
4363 kfree(nodes);
4364 return x + sprintf(buf + x, "\n");
4365}
4366
ab4d5ed5 4367#ifdef CONFIG_SLUB_DEBUG
81819f0f
CL
4368static int any_slab_objects(struct kmem_cache *s)
4369{
4370 int node;
81819f0f 4371
dfb4f096 4372 for_each_online_node(node) {
81819f0f
CL
4373 struct kmem_cache_node *n = get_node(s, node);
4374
dfb4f096
CL
4375 if (!n)
4376 continue;
4377
4ea33e2d 4378 if (atomic_long_read(&n->total_objects))
81819f0f
CL
4379 return 1;
4380 }
4381 return 0;
4382}
ab4d5ed5 4383#endif
81819f0f
CL
4384
4385#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
497888cf 4386#define to_slab(n) container_of(n, struct kmem_cache, kobj)
81819f0f
CL
4387
4388struct slab_attribute {
4389 struct attribute attr;
4390 ssize_t (*show)(struct kmem_cache *s, char *buf);
4391 ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
4392};
4393
4394#define SLAB_ATTR_RO(_name) \
ab067e99
VK
4395 static struct slab_attribute _name##_attr = \
4396 __ATTR(_name, 0400, _name##_show, NULL)
81819f0f
CL
4397
4398#define SLAB_ATTR(_name) \
4399 static struct slab_attribute _name##_attr = \
ab067e99 4400 __ATTR(_name, 0600, _name##_show, _name##_store)
81819f0f 4401
81819f0f
CL
4402static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
4403{
4404 return sprintf(buf, "%d\n", s->size);
4405}
4406SLAB_ATTR_RO(slab_size);
4407
4408static ssize_t align_show(struct kmem_cache *s, char *buf)
4409{
4410 return sprintf(buf, "%d\n", s->align);
4411}
4412SLAB_ATTR_RO(align);
4413
4414static ssize_t object_size_show(struct kmem_cache *s, char *buf)
4415{
3b0efdfa 4416 return sprintf(buf, "%d\n", s->object_size);
81819f0f
CL
4417}
4418SLAB_ATTR_RO(object_size);
4419
4420static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
4421{
834f3d11 4422 return sprintf(buf, "%d\n", oo_objects(s->oo));
81819f0f
CL
4423}
4424SLAB_ATTR_RO(objs_per_slab);
4425
06b285dc
CL
4426static ssize_t order_store(struct kmem_cache *s,
4427 const char *buf, size_t length)
4428{
0121c619
CL
4429 unsigned long order;
4430 int err;
4431
3dbb95f7 4432 err = kstrtoul(buf, 10, &order);
0121c619
CL
4433 if (err)
4434 return err;
06b285dc
CL
4435
4436 if (order > slub_max_order || order < slub_min_order)
4437 return -EINVAL;
4438
4439 calculate_sizes(s, order);
4440 return length;
4441}
4442
81819f0f
CL
4443static ssize_t order_show(struct kmem_cache *s, char *buf)
4444{
834f3d11 4445 return sprintf(buf, "%d\n", oo_order(s->oo));
81819f0f 4446}
06b285dc 4447SLAB_ATTR(order);
81819f0f 4448
73d342b1
DR
4449static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
4450{
4451 return sprintf(buf, "%lu\n", s->min_partial);
4452}
4453
4454static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
4455 size_t length)
4456{
4457 unsigned long min;
4458 int err;
4459
3dbb95f7 4460 err = kstrtoul(buf, 10, &min);
73d342b1
DR
4461 if (err)
4462 return err;
4463
c0bdb232 4464 set_min_partial(s, min);
73d342b1
DR
4465 return length;
4466}
4467SLAB_ATTR(min_partial);
4468
49e22585
CL
4469static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
4470{
4471 return sprintf(buf, "%u\n", s->cpu_partial);
4472}
4473
4474static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
4475 size_t length)
4476{
4477 unsigned long objects;
4478 int err;
4479
3dbb95f7 4480 err = kstrtoul(buf, 10, &objects);
49e22585
CL
4481 if (err)
4482 return err;
345c905d 4483 if (objects && !kmem_cache_has_cpu_partial(s))
74ee4ef1 4484 return -EINVAL;
49e22585
CL
4485
4486 s->cpu_partial = objects;
4487 flush_all(s);
4488 return length;
4489}
4490SLAB_ATTR(cpu_partial);
4491
81819f0f
CL
4492static ssize_t ctor_show(struct kmem_cache *s, char *buf)
4493{
62c70bce
JP
4494 if (!s->ctor)
4495 return 0;
4496 return sprintf(buf, "%pS\n", s->ctor);
81819f0f
CL
4497}
4498SLAB_ATTR_RO(ctor);
4499
81819f0f
CL
4500static ssize_t aliases_show(struct kmem_cache *s, char *buf)
4501{
4502 return sprintf(buf, "%d\n", s->refcount - 1);
4503}
4504SLAB_ATTR_RO(aliases);
4505
81819f0f
CL
4506static ssize_t partial_show(struct kmem_cache *s, char *buf)
4507{
d9acf4b7 4508 return show_slab_objects(s, buf, SO_PARTIAL);
81819f0f
CL
4509}
4510SLAB_ATTR_RO(partial);
4511
4512static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
4513{
d9acf4b7 4514 return show_slab_objects(s, buf, SO_CPU);
81819f0f
CL
4515}
4516SLAB_ATTR_RO(cpu_slabs);
4517
4518static ssize_t objects_show(struct kmem_cache *s, char *buf)
4519{
205ab99d 4520 return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
81819f0f
CL
4521}
4522SLAB_ATTR_RO(objects);
4523
205ab99d
CL
4524static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
4525{
4526 return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
4527}
4528SLAB_ATTR_RO(objects_partial);
4529
49e22585
CL
4530static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
4531{
4532 int objects = 0;
4533 int pages = 0;
4534 int cpu;
4535 int len;
4536
4537 for_each_online_cpu(cpu) {
4538 struct page *page = per_cpu_ptr(s->cpu_slab, cpu)->partial;
4539
4540 if (page) {
4541 pages += page->pages;
4542 objects += page->pobjects;
4543 }
4544 }
4545
4546 len = sprintf(buf, "%d(%d)", objects, pages);
4547
4548#ifdef CONFIG_SMP
4549 for_each_online_cpu(cpu) {
4550 struct page *page = per_cpu_ptr(s->cpu_slab, cpu) ->partial;
4551
4552 if (page && len < PAGE_SIZE - 20)
4553 len += sprintf(buf + len, " C%d=%d(%d)", cpu,
4554 page->pobjects, page->pages);
4555 }
4556#endif
4557 return len + sprintf(buf + len, "\n");
4558}
4559SLAB_ATTR_RO(slabs_cpu_partial);
4560
a5a84755
CL
4561static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
4562{
4563 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
4564}
4565
4566static ssize_t reclaim_account_store(struct kmem_cache *s,
4567 const char *buf, size_t length)
4568{
4569 s->flags &= ~SLAB_RECLAIM_ACCOUNT;
4570 if (buf[0] == '1')
4571 s->flags |= SLAB_RECLAIM_ACCOUNT;
4572 return length;
4573}
4574SLAB_ATTR(reclaim_account);
4575
4576static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
4577{
4578 return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
4579}
4580SLAB_ATTR_RO(hwcache_align);
4581
4582#ifdef CONFIG_ZONE_DMA
4583static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
4584{
4585 return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
4586}
4587SLAB_ATTR_RO(cache_dma);
4588#endif
4589
4590static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
4591{
4592 return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU));
4593}
4594SLAB_ATTR_RO(destroy_by_rcu);
4595
ab9a0f19
LJ
4596static ssize_t reserved_show(struct kmem_cache *s, char *buf)
4597{
4598 return sprintf(buf, "%d\n", s->reserved);
4599}
4600SLAB_ATTR_RO(reserved);
4601
ab4d5ed5 4602#ifdef CONFIG_SLUB_DEBUG
a5a84755
CL
4603static ssize_t slabs_show(struct kmem_cache *s, char *buf)
4604{
4605 return show_slab_objects(s, buf, SO_ALL);
4606}
4607SLAB_ATTR_RO(slabs);
4608
205ab99d
CL
4609static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
4610{
4611 return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
4612}
4613SLAB_ATTR_RO(total_objects);
4614
81819f0f
CL
4615static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
4616{
4617 return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE));
4618}
4619
4620static ssize_t sanity_checks_store(struct kmem_cache *s,
4621 const char *buf, size_t length)
4622{
4623 s->flags &= ~SLAB_DEBUG_FREE;
b789ef51
CL
4624 if (buf[0] == '1') {
4625 s->flags &= ~__CMPXCHG_DOUBLE;
81819f0f 4626 s->flags |= SLAB_DEBUG_FREE;
b789ef51 4627 }
81819f0f
CL
4628 return length;
4629}
4630SLAB_ATTR(sanity_checks);
4631
4632static ssize_t trace_show(struct kmem_cache *s, char *buf)
4633{
4634 return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE));
4635}
4636
4637static ssize_t trace_store(struct kmem_cache *s, const char *buf,
4638 size_t length)
4639{
4640 s->flags &= ~SLAB_TRACE;
b789ef51
CL
4641 if (buf[0] == '1') {
4642 s->flags &= ~__CMPXCHG_DOUBLE;
81819f0f 4643 s->flags |= SLAB_TRACE;
b789ef51 4644 }
81819f0f
CL
4645 return length;
4646}
4647SLAB_ATTR(trace);
4648
81819f0f
CL
4649static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
4650{
4651 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
4652}
4653
4654static ssize_t red_zone_store(struct kmem_cache *s,
4655 const char *buf, size_t length)
4656{
4657 if (any_slab_objects(s))
4658 return -EBUSY;
4659
4660 s->flags &= ~SLAB_RED_ZONE;
b789ef51
CL
4661 if (buf[0] == '1') {
4662 s->flags &= ~__CMPXCHG_DOUBLE;
81819f0f 4663 s->flags |= SLAB_RED_ZONE;
b789ef51 4664 }
06b285dc 4665 calculate_sizes(s, -1);
81819f0f
CL
4666 return length;
4667}
4668SLAB_ATTR(red_zone);
4669
4670static ssize_t poison_show(struct kmem_cache *s, char *buf)
4671{
4672 return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
4673}
4674
4675static ssize_t poison_store(struct kmem_cache *s,
4676 const char *buf, size_t length)
4677{
4678 if (any_slab_objects(s))
4679 return -EBUSY;
4680
4681 s->flags &= ~SLAB_POISON;
b789ef51
CL
4682 if (buf[0] == '1') {
4683 s->flags &= ~__CMPXCHG_DOUBLE;
81819f0f 4684 s->flags |= SLAB_POISON;
b789ef51 4685 }
06b285dc 4686 calculate_sizes(s, -1);
81819f0f
CL
4687 return length;
4688}
4689SLAB_ATTR(poison);
4690
4691static ssize_t store_user_show(struct kmem_cache *s, char *buf)
4692{
4693 return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
4694}
4695
4696static ssize_t store_user_store(struct kmem_cache *s,
4697 const char *buf, size_t length)
4698{
4699 if (any_slab_objects(s))
4700 return -EBUSY;
4701
4702 s->flags &= ~SLAB_STORE_USER;
b789ef51
CL
4703 if (buf[0] == '1') {
4704 s->flags &= ~__CMPXCHG_DOUBLE;
81819f0f 4705 s->flags |= SLAB_STORE_USER;
b789ef51 4706 }
06b285dc 4707 calculate_sizes(s, -1);
81819f0f
CL
4708 return length;
4709}
4710SLAB_ATTR(store_user);
4711
53e15af0
CL
4712static ssize_t validate_show(struct kmem_cache *s, char *buf)
4713{
4714 return 0;
4715}
4716
4717static ssize_t validate_store(struct kmem_cache *s,
4718 const char *buf, size_t length)
4719{
434e245d
CL
4720 int ret = -EINVAL;
4721
4722 if (buf[0] == '1') {
4723 ret = validate_slab_cache(s);
4724 if (ret >= 0)
4725 ret = length;
4726 }
4727 return ret;
53e15af0
CL
4728}
4729SLAB_ATTR(validate);
a5a84755
CL
4730
4731static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf)
4732{
4733 if (!(s->flags & SLAB_STORE_USER))
4734 return -ENOSYS;
4735 return list_locations(s, buf, TRACK_ALLOC);
4736}
4737SLAB_ATTR_RO(alloc_calls);
4738
4739static ssize_t free_calls_show(struct kmem_cache *s, char *buf)
4740{
4741 if (!(s->flags & SLAB_STORE_USER))
4742 return -ENOSYS;
4743 return list_locations(s, buf, TRACK_FREE);
4744}
4745SLAB_ATTR_RO(free_calls);
4746#endif /* CONFIG_SLUB_DEBUG */
4747
4748#ifdef CONFIG_FAILSLAB
4749static ssize_t failslab_show(struct kmem_cache *s, char *buf)
4750{
4751 return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
4752}
4753
4754static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
4755 size_t length)
4756{
4757 s->flags &= ~SLAB_FAILSLAB;
4758 if (buf[0] == '1')
4759 s->flags |= SLAB_FAILSLAB;
4760 return length;
4761}
4762SLAB_ATTR(failslab);
ab4d5ed5 4763#endif
53e15af0 4764
2086d26a
CL
4765static ssize_t shrink_show(struct kmem_cache *s, char *buf)
4766{
4767 return 0;
4768}
4769
4770static ssize_t shrink_store(struct kmem_cache *s,
4771 const char *buf, size_t length)
4772{
4773 if (buf[0] == '1') {
4774 int rc = kmem_cache_shrink(s);
4775
4776 if (rc)
4777 return rc;
4778 } else
4779 return -EINVAL;
4780 return length;
4781}
4782SLAB_ATTR(shrink);
4783
81819f0f 4784#ifdef CONFIG_NUMA
9824601e 4785static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
81819f0f 4786{
9824601e 4787 return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10);
81819f0f
CL
4788}
4789
9824601e 4790static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
81819f0f
CL
4791 const char *buf, size_t length)
4792{
0121c619
CL
4793 unsigned long ratio;
4794 int err;
4795
3dbb95f7 4796 err = kstrtoul(buf, 10, &ratio);
0121c619
CL
4797 if (err)
4798 return err;
4799
e2cb96b7 4800 if (ratio <= 100)
0121c619 4801 s->remote_node_defrag_ratio = ratio * 10;
81819f0f 4802
81819f0f
CL
4803 return length;
4804}
9824601e 4805SLAB_ATTR(remote_node_defrag_ratio);
81819f0f
CL
4806#endif
4807
8ff12cfc 4808#ifdef CONFIG_SLUB_STATS
8ff12cfc
CL
4809static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
4810{
4811 unsigned long sum = 0;
4812 int cpu;
4813 int len;
4814 int *data = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL);
4815
4816 if (!data)
4817 return -ENOMEM;
4818
4819 for_each_online_cpu(cpu) {
9dfc6e68 4820 unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
8ff12cfc
CL
4821
4822 data[cpu] = x;
4823 sum += x;
4824 }
4825
4826 len = sprintf(buf, "%lu", sum);
4827
50ef37b9 4828#ifdef CONFIG_SMP
8ff12cfc
CL
4829 for_each_online_cpu(cpu) {
4830 if (data[cpu] && len < PAGE_SIZE - 20)
50ef37b9 4831 len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]);
8ff12cfc 4832 }
50ef37b9 4833#endif
8ff12cfc
CL
4834 kfree(data);
4835 return len + sprintf(buf + len, "\n");
4836}
4837
78eb00cc
DR
4838static void clear_stat(struct kmem_cache *s, enum stat_item si)
4839{
4840 int cpu;
4841
4842 for_each_online_cpu(cpu)
9dfc6e68 4843 per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
78eb00cc
DR
4844}
4845
8ff12cfc
CL
4846#define STAT_ATTR(si, text) \
4847static ssize_t text##_show(struct kmem_cache *s, char *buf) \
4848{ \
4849 return show_stat(s, buf, si); \
4850} \
78eb00cc
DR
4851static ssize_t text##_store(struct kmem_cache *s, \
4852 const char *buf, size_t length) \
4853{ \
4854 if (buf[0] != '0') \
4855 return -EINVAL; \
4856 clear_stat(s, si); \
4857 return length; \
4858} \
4859SLAB_ATTR(text); \
8ff12cfc
CL
4860
4861STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
4862STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
4863STAT_ATTR(FREE_FASTPATH, free_fastpath);
4864STAT_ATTR(FREE_SLOWPATH, free_slowpath);
4865STAT_ATTR(FREE_FROZEN, free_frozen);
4866STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
4867STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
4868STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
4869STAT_ATTR(ALLOC_SLAB, alloc_slab);
4870STAT_ATTR(ALLOC_REFILL, alloc_refill);
e36a2652 4871STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
8ff12cfc
CL
4872STAT_ATTR(FREE_SLAB, free_slab);
4873STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
4874STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
4875STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
4876STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
4877STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
4878STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
03e404af 4879STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
65c3376a 4880STAT_ATTR(ORDER_FALLBACK, order_fallback);
b789ef51
CL
4881STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
4882STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
49e22585
CL
4883STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
4884STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
8028dcea
AS
4885STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
4886STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
8ff12cfc
CL
4887#endif
4888
06428780 4889static struct attribute *slab_attrs[] = {
81819f0f
CL
4890 &slab_size_attr.attr,
4891 &object_size_attr.attr,
4892 &objs_per_slab_attr.attr,
4893 &order_attr.attr,
73d342b1 4894 &min_partial_attr.attr,
49e22585 4895 &cpu_partial_attr.attr,
81819f0f 4896 &objects_attr.attr,
205ab99d 4897 &objects_partial_attr.attr,
81819f0f
CL
4898 &partial_attr.attr,
4899 &cpu_slabs_attr.attr,
4900 &ctor_attr.attr,
81819f0f
CL
4901 &aliases_attr.attr,
4902 &align_attr.attr,
81819f0f
CL
4903 &hwcache_align_attr.attr,
4904 &reclaim_account_attr.attr,
4905 &destroy_by_rcu_attr.attr,
a5a84755 4906 &shrink_attr.attr,
ab9a0f19 4907 &reserved_attr.attr,
49e22585 4908 &slabs_cpu_partial_attr.attr,
ab4d5ed5 4909#ifdef CONFIG_SLUB_DEBUG
a5a84755
CL
4910 &total_objects_attr.attr,
4911 &slabs_attr.attr,
4912 &sanity_checks_attr.attr,
4913 &trace_attr.attr,
81819f0f
CL
4914 &red_zone_attr.attr,
4915 &poison_attr.attr,
4916 &store_user_attr.attr,
53e15af0 4917 &validate_attr.attr,
88a420e4
CL
4918 &alloc_calls_attr.attr,
4919 &free_calls_attr.attr,
ab4d5ed5 4920#endif
81819f0f
CL
4921#ifdef CONFIG_ZONE_DMA
4922 &cache_dma_attr.attr,
4923#endif
4924#ifdef CONFIG_NUMA
9824601e 4925 &remote_node_defrag_ratio_attr.attr,
8ff12cfc
CL
4926#endif
4927#ifdef CONFIG_SLUB_STATS
4928 &alloc_fastpath_attr.attr,
4929 &alloc_slowpath_attr.attr,
4930 &free_fastpath_attr.attr,
4931 &free_slowpath_attr.attr,
4932 &free_frozen_attr.attr,
4933 &free_add_partial_attr.attr,
4934 &free_remove_partial_attr.attr,
4935 &alloc_from_partial_attr.attr,
4936 &alloc_slab_attr.attr,
4937 &alloc_refill_attr.attr,
e36a2652 4938 &alloc_node_mismatch_attr.attr,
8ff12cfc
CL
4939 &free_slab_attr.attr,
4940 &cpuslab_flush_attr.attr,
4941 &deactivate_full_attr.attr,
4942 &deactivate_empty_attr.attr,
4943 &deactivate_to_head_attr.attr,
4944 &deactivate_to_tail_attr.attr,
4945 &deactivate_remote_frees_attr.attr,
03e404af 4946 &deactivate_bypass_attr.attr,
65c3376a 4947 &order_fallback_attr.attr,
b789ef51
CL
4948 &cmpxchg_double_fail_attr.attr,
4949 &cmpxchg_double_cpu_fail_attr.attr,
49e22585
CL
4950 &cpu_partial_alloc_attr.attr,
4951 &cpu_partial_free_attr.attr,
8028dcea
AS
4952 &cpu_partial_node_attr.attr,
4953 &cpu_partial_drain_attr.attr,
81819f0f 4954#endif
4c13dd3b
DM
4955#ifdef CONFIG_FAILSLAB
4956 &failslab_attr.attr,
4957#endif
4958
81819f0f
CL
4959 NULL
4960};
4961
4962static struct attribute_group slab_attr_group = {
4963 .attrs = slab_attrs,
4964};
4965
4966static ssize_t slab_attr_show(struct kobject *kobj,
4967 struct attribute *attr,
4968 char *buf)
4969{
4970 struct slab_attribute *attribute;
4971 struct kmem_cache *s;
4972 int err;
4973
4974 attribute = to_slab_attr(attr);
4975 s = to_slab(kobj);
4976
4977 if (!attribute->show)
4978 return -EIO;
4979
4980 err = attribute->show(s, buf);
4981
4982 return err;
4983}
4984
4985static ssize_t slab_attr_store(struct kobject *kobj,
4986 struct attribute *attr,
4987 const char *buf, size_t len)
4988{
4989 struct slab_attribute *attribute;
4990 struct kmem_cache *s;
4991 int err;
4992
4993 attribute = to_slab_attr(attr);
4994 s = to_slab(kobj);
4995
4996 if (!attribute->store)
4997 return -EIO;
4998
4999 err = attribute->store(s, buf, len);
107dab5c
GC
5000#ifdef CONFIG_MEMCG_KMEM
5001 if (slab_state >= FULL && err >= 0 && is_root_cache(s)) {
5002 int i;
81819f0f 5003
107dab5c
GC
5004 mutex_lock(&slab_mutex);
5005 if (s->max_attr_size < len)
5006 s->max_attr_size = len;
5007
ebe945c2
GC
5008 /*
5009 * This is a best effort propagation, so this function's return
5010 * value will be determined by the parent cache only. This is
5011 * basically because not all attributes will have a well
5012 * defined semantics for rollbacks - most of the actions will
5013 * have permanent effects.
5014 *
5015 * Returning the error value of any of the children that fail
5016 * is not 100 % defined, in the sense that users seeing the
5017 * error code won't be able to know anything about the state of
5018 * the cache.
5019 *
5020 * Only returning the error code for the parent cache at least
5021 * has well defined semantics. The cache being written to
5022 * directly either failed or succeeded, in which case we loop
5023 * through the descendants with best-effort propagation.
5024 */
107dab5c 5025 for_each_memcg_cache_index(i) {
2ade4de8 5026 struct kmem_cache *c = cache_from_memcg_idx(s, i);
107dab5c
GC
5027 if (c)
5028 attribute->store(c, buf, len);
5029 }
5030 mutex_unlock(&slab_mutex);
5031 }
5032#endif
81819f0f
CL
5033 return err;
5034}
5035
107dab5c
GC
5036static void memcg_propagate_slab_attrs(struct kmem_cache *s)
5037{
5038#ifdef CONFIG_MEMCG_KMEM
5039 int i;
5040 char *buffer = NULL;
93030d83 5041 struct kmem_cache *root_cache;
107dab5c 5042
93030d83 5043 if (is_root_cache(s))
107dab5c
GC
5044 return;
5045
93030d83
VD
5046 root_cache = s->memcg_params->root_cache;
5047
107dab5c
GC
5048 /*
5049 * This mean this cache had no attribute written. Therefore, no point
5050 * in copying default values around
5051 */
93030d83 5052 if (!root_cache->max_attr_size)
107dab5c
GC
5053 return;
5054
5055 for (i = 0; i < ARRAY_SIZE(slab_attrs); i++) {
5056 char mbuf[64];
5057 char *buf;
5058 struct slab_attribute *attr = to_slab_attr(slab_attrs[i]);
5059
5060 if (!attr || !attr->store || !attr->show)
5061 continue;
5062
5063 /*
5064 * It is really bad that we have to allocate here, so we will
5065 * do it only as a fallback. If we actually allocate, though,
5066 * we can just use the allocated buffer until the end.
5067 *
5068 * Most of the slub attributes will tend to be very small in
5069 * size, but sysfs allows buffers up to a page, so they can
5070 * theoretically happen.
5071 */
5072 if (buffer)
5073 buf = buffer;
93030d83 5074 else if (root_cache->max_attr_size < ARRAY_SIZE(mbuf))
107dab5c
GC
5075 buf = mbuf;
5076 else {
5077 buffer = (char *) get_zeroed_page(GFP_KERNEL);
5078 if (WARN_ON(!buffer))
5079 continue;
5080 buf = buffer;
5081 }
5082
93030d83 5083 attr->show(root_cache, buf);
107dab5c
GC
5084 attr->store(s, buf, strlen(buf));
5085 }
5086
5087 if (buffer)
5088 free_page((unsigned long)buffer);
5089#endif
5090}
5091
41a21285
CL
5092static void kmem_cache_release(struct kobject *k)
5093{
5094 slab_kmem_cache_release(to_slab(k));
5095}
5096
52cf25d0 5097static const struct sysfs_ops slab_sysfs_ops = {
81819f0f
CL
5098 .show = slab_attr_show,
5099 .store = slab_attr_store,
5100};
5101
5102static struct kobj_type slab_ktype = {
5103 .sysfs_ops = &slab_sysfs_ops,
41a21285 5104 .release = kmem_cache_release,
81819f0f
CL
5105};
5106
5107static int uevent_filter(struct kset *kset, struct kobject *kobj)
5108{
5109 struct kobj_type *ktype = get_ktype(kobj);
5110
5111 if (ktype == &slab_ktype)
5112 return 1;
5113 return 0;
5114}
5115
9cd43611 5116static const struct kset_uevent_ops slab_uevent_ops = {
81819f0f
CL
5117 .filter = uevent_filter,
5118};
5119
27c3a314 5120static struct kset *slab_kset;
81819f0f 5121
9a41707b
VD
5122static inline struct kset *cache_kset(struct kmem_cache *s)
5123{
5124#ifdef CONFIG_MEMCG_KMEM
5125 if (!is_root_cache(s))
5126 return s->memcg_params->root_cache->memcg_kset;
5127#endif
5128 return slab_kset;
5129}
5130
81819f0f
CL
5131#define ID_STR_LENGTH 64
5132
5133/* Create a unique string id for a slab cache:
6446faa2
CL
5134 *
5135 * Format :[flags-]size
81819f0f
CL
5136 */
5137static char *create_unique_id(struct kmem_cache *s)
5138{
5139 char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
5140 char *p = name;
5141
5142 BUG_ON(!name);
5143
5144 *p++ = ':';
5145 /*
5146 * First flags affecting slabcache operations. We will only
5147 * get here for aliasable slabs so we do not need to support
5148 * too many flags. The flags here must cover all flags that
5149 * are matched during merging to guarantee that the id is
5150 * unique.
5151 */
5152 if (s->flags & SLAB_CACHE_DMA)
5153 *p++ = 'd';
5154 if (s->flags & SLAB_RECLAIM_ACCOUNT)
5155 *p++ = 'a';
5156 if (s->flags & SLAB_DEBUG_FREE)
5157 *p++ = 'F';
5a896d9e
VN
5158 if (!(s->flags & SLAB_NOTRACK))
5159 *p++ = 't';
81819f0f
CL
5160 if (p != name + 1)
5161 *p++ = '-';
5162 p += sprintf(p, "%07d", s->size);
2633d7a0
GC
5163
5164#ifdef CONFIG_MEMCG_KMEM
5165 if (!is_root_cache(s))
d0e0ac97
CG
5166 p += sprintf(p, "-%08d",
5167 memcg_cache_id(s->memcg_params->memcg));
2633d7a0
GC
5168#endif
5169
81819f0f
CL
5170 BUG_ON(p > name + ID_STR_LENGTH - 1);
5171 return name;
5172}
5173
5174static int sysfs_slab_add(struct kmem_cache *s)
5175{
5176 int err;
5177 const char *name;
45530c44 5178 int unmergeable = slab_unmergeable(s);
81819f0f 5179
81819f0f
CL
5180 if (unmergeable) {
5181 /*
5182 * Slabcache can never be merged so we can use the name proper.
5183 * This is typically the case for debug situations. In that
5184 * case we can catch duplicate names easily.
5185 */
27c3a314 5186 sysfs_remove_link(&slab_kset->kobj, s->name);
81819f0f
CL
5187 name = s->name;
5188 } else {
5189 /*
5190 * Create a unique name for the slab as a target
5191 * for the symlinks.
5192 */
5193 name = create_unique_id(s);
5194 }
5195
9a41707b 5196 s->kobj.kset = cache_kset(s);
26e4f205 5197 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
54b6a731
DJ
5198 if (err)
5199 goto out_put_kobj;
81819f0f
CL
5200
5201 err = sysfs_create_group(&s->kobj, &slab_attr_group);
54b6a731
DJ
5202 if (err)
5203 goto out_del_kobj;
9a41707b
VD
5204
5205#ifdef CONFIG_MEMCG_KMEM
5206 if (is_root_cache(s)) {
5207 s->memcg_kset = kset_create_and_add("cgroup", NULL, &s->kobj);
5208 if (!s->memcg_kset) {
54b6a731
DJ
5209 err = -ENOMEM;
5210 goto out_del_kobj;
9a41707b
VD
5211 }
5212 }
5213#endif
5214
81819f0f
CL
5215 kobject_uevent(&s->kobj, KOBJ_ADD);
5216 if (!unmergeable) {
5217 /* Setup first alias */
5218 sysfs_slab_alias(s, s->name);
81819f0f 5219 }
54b6a731
DJ
5220out:
5221 if (!unmergeable)
5222 kfree(name);
5223 return err;
5224out_del_kobj:
5225 kobject_del(&s->kobj);
5226out_put_kobj:
5227 kobject_put(&s->kobj);
5228 goto out;
81819f0f
CL
5229}
5230
41a21285 5231void sysfs_slab_remove(struct kmem_cache *s)
81819f0f 5232{
97d06609 5233 if (slab_state < FULL)
2bce6485
CL
5234 /*
5235 * Sysfs has not been setup yet so no need to remove the
5236 * cache from sysfs.
5237 */
5238 return;
5239
9a41707b
VD
5240#ifdef CONFIG_MEMCG_KMEM
5241 kset_unregister(s->memcg_kset);
5242#endif
81819f0f
CL
5243 kobject_uevent(&s->kobj, KOBJ_REMOVE);
5244 kobject_del(&s->kobj);
151c602f 5245 kobject_put(&s->kobj);
81819f0f
CL
5246}
5247
5248/*
5249 * Need to buffer aliases during bootup until sysfs becomes
9f6c708e 5250 * available lest we lose that information.
81819f0f
CL
5251 */
5252struct saved_alias {
5253 struct kmem_cache *s;
5254 const char *name;
5255 struct saved_alias *next;
5256};
5257
5af328a5 5258static struct saved_alias *alias_list;
81819f0f
CL
5259
5260static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
5261{
5262 struct saved_alias *al;
5263
97d06609 5264 if (slab_state == FULL) {
81819f0f
CL
5265 /*
5266 * If we have a leftover link then remove it.
5267 */
27c3a314
GKH
5268 sysfs_remove_link(&slab_kset->kobj, name);
5269 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
81819f0f
CL
5270 }
5271
5272 al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
5273 if (!al)
5274 return -ENOMEM;
5275
5276 al->s = s;
5277 al->name = name;
5278 al->next = alias_list;
5279 alias_list = al;
5280 return 0;
5281}
5282
5283static int __init slab_sysfs_init(void)
5284{
5b95a4ac 5285 struct kmem_cache *s;
81819f0f
CL
5286 int err;
5287
18004c5d 5288 mutex_lock(&slab_mutex);
2bce6485 5289
0ff21e46 5290 slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
27c3a314 5291 if (!slab_kset) {
18004c5d 5292 mutex_unlock(&slab_mutex);
f9f58285 5293 pr_err("Cannot register slab subsystem.\n");
81819f0f
CL
5294 return -ENOSYS;
5295 }
5296
97d06609 5297 slab_state = FULL;
26a7bd03 5298
5b95a4ac 5299 list_for_each_entry(s, &slab_caches, list) {
26a7bd03 5300 err = sysfs_slab_add(s);
5d540fb7 5301 if (err)
f9f58285
FF
5302 pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
5303 s->name);
26a7bd03 5304 }
81819f0f
CL
5305
5306 while (alias_list) {
5307 struct saved_alias *al = alias_list;
5308
5309 alias_list = alias_list->next;
5310 err = sysfs_slab_alias(al->s, al->name);
5d540fb7 5311 if (err)
f9f58285
FF
5312 pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
5313 al->name);
81819f0f
CL
5314 kfree(al);
5315 }
5316
18004c5d 5317 mutex_unlock(&slab_mutex);
81819f0f
CL
5318 resiliency_test();
5319 return 0;
5320}
5321
5322__initcall(slab_sysfs_init);
ab4d5ed5 5323#endif /* CONFIG_SYSFS */
57ed3eda
PE
5324
5325/*
5326 * The /proc/slabinfo ABI
5327 */
158a9624 5328#ifdef CONFIG_SLABINFO
0d7561c6 5329void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
57ed3eda 5330{
57ed3eda 5331 unsigned long nr_slabs = 0;
205ab99d
CL
5332 unsigned long nr_objs = 0;
5333 unsigned long nr_free = 0;
57ed3eda
PE
5334 int node;
5335
57ed3eda
PE
5336 for_each_online_node(node) {
5337 struct kmem_cache_node *n = get_node(s, node);
5338
5339 if (!n)
5340 continue;
5341
c17fd13e
WL
5342 nr_slabs += node_nr_slabs(n);
5343 nr_objs += node_nr_objs(n);
205ab99d 5344 nr_free += count_partial(n, count_free);
57ed3eda
PE
5345 }
5346
0d7561c6
GC
5347 sinfo->active_objs = nr_objs - nr_free;
5348 sinfo->num_objs = nr_objs;
5349 sinfo->active_slabs = nr_slabs;
5350 sinfo->num_slabs = nr_slabs;
5351 sinfo->objects_per_slab = oo_objects(s->oo);
5352 sinfo->cache_order = oo_order(s->oo);
57ed3eda
PE
5353}
5354
0d7561c6 5355void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)
7b3c3a50 5356{
7b3c3a50
AD
5357}
5358
b7454ad3
GC
5359ssize_t slabinfo_write(struct file *file, const char __user *buffer,
5360 size_t count, loff_t *ppos)
7b3c3a50 5361{
b7454ad3 5362 return -EIO;
7b3c3a50 5363}
158a9624 5364#endif /* CONFIG_SLABINFO */