X-Git-Url: https://git.stricted.de/?a=blobdiff_plain;f=mm%2Fslub.c;h=01900d906cd1806a51bc0fd0a81448e44c1f7287;hb=446a42c9b20fe9549c51817f73c1572680ae0a5f;hp=57707f01bcfb72395fd7414de8059dcb341cbdbd;hpb=e6eea1536e077b2b7bfc8c173f09d01ef72d2c7b;p=GitHub%2Fmt8127%2Fandroid_kernel_alcatel_ttab.git diff --git a/mm/slub.c b/mm/slub.c index 57707f01bcfb..01900d906cd1 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -33,11 +33,22 @@ #include #include #include +#include #include +#include #include "internal.h" - +#ifdef CONFIG_ARM64 +#ifdef BUG +#undef BUG +#define BUG() \ + do { \ + pr_alert("BUG: failure at %s:%d/%s()!\n", __FILE__, __LINE__, __func__); \ + *(char *)0 = 0;\ + } while(0) +#endif +#endif /* * Lock order: * 1. slab_mutex (Global Mutex) @@ -196,7 +207,7 @@ struct track { unsigned long when; /* When did the operation occur */ }; -enum track_item { TRACK_ALLOC, TRACK_FREE }; +enum track_item { TRACK_FREE, TRACK_ALLOC }; #ifdef CONFIG_SYSFS static int sysfs_slab_add(struct kmem_cache *); @@ -346,6 +357,21 @@ static __always_inline void slab_unlock(struct page *page) __bit_spin_unlock(PG_locked, &page->flags); } +static inline void set_page_slub_counters(struct page *page, unsigned long counters_new) +{ + struct page tmp; + tmp.counters = counters_new; + /* + * page->counters can cover frozen/inuse/objects as well + * as page->_count. If we assign to ->counters directly + * we run the risk of losing updates to page->_count, so + * be careful and only assign to the fields we need. + */ + page->frozen = tmp.frozen; + page->inuse = tmp.inuse; + page->objects = tmp.objects; +} + /* Interrupts must be disabled (for the fallback code to work right) */ static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page, void *freelist_old, unsigned long counters_old, @@ -366,7 +392,7 @@ static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page slab_lock(page); if (page->freelist == freelist_old && page->counters == counters_old) { page->freelist = freelist_new; - page->counters = counters_new; + set_page_slub_counters(page, counters_new); slab_unlock(page); return 1; } @@ -404,7 +430,7 @@ static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page, slab_lock(page); if (page->freelist == freelist_old && page->counters == counters_old) { page->freelist = freelist_new; - page->counters = counters_new; + set_page_slub_counters(page, counters_new); slab_unlock(page); local_irq_restore(flags); return 1; @@ -618,6 +644,7 @@ static void object_err(struct kmem_cache *s, struct page *page, { slab_bug(s, "%s", reason); print_trailer(s, page, object); + BUG(); } static void slab_err(struct kmem_cache *s, struct page *page, const char *fmt, ...) @@ -631,6 +658,7 @@ static void slab_err(struct kmem_cache *s, struct page *page, const char *fmt, . slab_bug(s, "%s", buf); print_page_info(page); dump_stack(); + BUG(); } static void init_object(struct kmem_cache *s, void *object, u8 val) @@ -673,7 +701,10 @@ static int check_bytes_and_report(struct kmem_cache *s, struct page *page, fault, end - 1, fault[0], value); print_trailer(s, page, object); + /* trigger BUG before restore_bytes */ + BUG(); restore_bytes(s, what, value, fault, end); + return 0; } @@ -830,13 +861,18 @@ static int check_slab(struct kmem_cache *s, struct page *page) maxobj = order_objects(compound_order(page), s->size, s->reserved); if (page->objects > maxobj) { + pr_alert("page->objects: %d, maxobj: %d, comporder: %d", page->objects, + maxobj, compound_order(page)); + pr_alert("s->size %d, s->reserved: %d", s->size, s->reserved); + print_section("page: ", (void *)page, sizeof(struct page)); + print_section("kmem_cache: ", (void *)s, sizeof(struct kmem_cache)); slab_err(s, page, "objects %u > max %u", - s->name, page->objects, maxobj); + page->objects, maxobj); return 0; } if (page->inuse > page->objects) { slab_err(s, page, "inuse %u > max %u", - s->name, page->inuse, page->objects); + page->inuse, page->objects); return 0; } /* Slab_pad_check fixes things up after itself */ @@ -1201,8 +1237,12 @@ static unsigned long kmem_cache_flags(unsigned long object_size, /* * Enable debugging if selected on the kernel commandline. */ - if (slub_debug && (!slub_debug_slabs || - !strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs)))) + if(flags & SLAB_NO_DEBUG) { + return flags; + } + + if (slub_debug && (!slub_debug_slabs || (name && + !strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs))))) flags |= slub_debug; return flags; @@ -1265,7 +1305,11 @@ static inline struct page *alloc_slab_page(gfp_t flags, int node, flags |= __GFP_NOTRACK; if (node == NUMA_NO_NODE) +#ifndef CONFIG_MTK_PAGERECORDER return alloc_pages(flags, order); +#else + return alloc_pages_nopagedebug(flags, order); +#endif else return alloc_pages_exact_node(node, flags, order); } @@ -3945,9 +3989,22 @@ static long validate_slab_cache(struct kmem_cache *s) * and freed. */ +#ifdef CONFIG_MTK_MEMCFG +#define MTK_MEMCFG_SLABTRACE_CNT 4 +/* MTK_MEMCFG_SLABTRACE_CNT should be always <= TRACK_ADDRS_COUNT */ +#if (MTK_MEMCFG_SLABTRACE_CNT > TRACK_ADDRS_COUNT) +#error (MTK_MEMCFG_SLABTRACE_CNT > TRACK_ADDRS_COUNT) +#endif +#endif + struct location { unsigned long count; unsigned long addr; +#ifdef CONFIG_MTK_MEMCFG +#ifdef CONFIG_STACKTRACE + unsigned long addrs[MTK_MEMCFG_SLABTRACE_CNT]; /* Called from address */ +#endif +#endif long long sum_time; long min_time; long max_time; @@ -3966,8 +4023,13 @@ struct loc_track { static void free_loc_track(struct loc_track *t) { if (t->max) +#ifndef CONFIG_MTK_PAGERECORDER free_pages((unsigned long)t->loc, get_order(sizeof(struct location) * t->max)); +#else + __free_pages_nopagedebug((struct page *)t->loc, + get_order(sizeof(struct location) * t->max)); +#endif } static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags) @@ -3977,7 +4039,11 @@ static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags) order = get_order(sizeof(struct location) * max); +#ifndef CONFIG_MTK_PAGERECORDER l = (void *)__get_free_pages(flags, order); +#else + l = (void *)__get_free_pages_nopagedebug(flags, order); +#endif if (!l) return 0; @@ -4285,7 +4351,13 @@ static ssize_t show_slab_objects(struct kmem_cache *s, page = ACCESS_ONCE(c->partial); if (page) { - x = page->pobjects; + node = page_to_nid(page); + if (flags & SO_TOTAL) + WARN_ON_ONCE(1); + else if (flags & SO_OBJECTS) + WARN_ON_ONCE(1); + else + x = page->pages; total += x; nodes[node] += x; } @@ -5304,4 +5376,192 @@ ssize_t slabinfo_write(struct file *file, const char __user *buffer, { return -EIO; } + +#ifdef CONFIG_MTK_MEMCFG + +static int mtk_memcfg_add_location(struct loc_track *t, struct kmem_cache *s, + const struct track *track) +{ + long start, end, pos; + struct location *l; + unsigned long (*caddrs)[MTK_MEMCFG_SLABTRACE_CNT]; /* Called from addresses */ + unsigned long taddrs[MTK_MEMCFG_SLABTRACE_CNT] + = { [0 ... MTK_MEMCFG_SLABTRACE_CNT - 1] = 0,}; /* Called from addresses of track */ + unsigned long age = jiffies - track->when; + int i, cnt; + + start = -1; + end = t->count; + + /* find the index of track->addr */ + for (i = 0; i < TRACK_ADDRS_COUNT; i++) { + if ((track->addr == track->addrs[i]) || + (track->addr - 4 == track->addrs[i])) + break; + } + cnt = min(MTK_MEMCFG_SLABTRACE_CNT, TRACK_ADDRS_COUNT - i); + memcpy(taddrs, track->addrs + i, (cnt * sizeof (unsigned long))); + + for ( ; ; ) { + pos = start + (end - start + 1) / 2; + + /* + * There is nothing at "end". If we end up there + * we need to add something to before end. + */ + if (pos == end) + break; + + caddrs = &(t->loc[pos].addrs); + if (!memcmp(caddrs, taddrs, MTK_MEMCFG_SLABTRACE_CNT * sizeof (unsigned long))) { + + l = &t->loc[pos]; + l->count++; + if (track->when) { + l->sum_time += age; + if (age < l->min_time) + l->min_time = age; + if (age > l->max_time) + l->max_time = age; + + if (track->pid < l->min_pid) + l->min_pid = track->pid; + if (track->pid > l->max_pid) + l->max_pid = track->pid; + + cpumask_set_cpu(track->cpu, + to_cpumask(l->cpus)); + } + node_set(page_to_nid(virt_to_page(track)), l->nodes); + return 1; + } + + if (memcmp(caddrs, taddrs, MTK_MEMCFG_SLABTRACE_CNT * sizeof (unsigned long)) < 0) + end = pos; + else + start = pos; + } + + /* + * Not found. Insert new tracking element. + */ + if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC)) + return 0; + + l = t->loc + pos; + if (pos < t->count) + memmove(l + 1, l, + (t->count - pos) * sizeof(struct location)); + t->count++; + l->count = 1; + l->addr = track->addr; + memcpy(l->addrs, taddrs, MTK_MEMCFG_SLABTRACE_CNT * sizeof (unsigned long)); + l->sum_time = age; + l->min_time = age; + l->max_time = age; + l->min_pid = track->pid; + l->max_pid = track->pid; + cpumask_clear(to_cpumask(l->cpus)); + cpumask_set_cpu(track->cpu, to_cpumask(l->cpus)); + nodes_clear(l->nodes); + node_set(page_to_nid(virt_to_page(track)), l->nodes); + return 1; +} + +static void mtk_memcfg_process_slab(struct loc_track *t, struct kmem_cache *s, + struct page *page, enum track_item alloc, + unsigned long *map) +{ + void *addr = page_address(page); + void *p; + + bitmap_zero(map, page->objects); + get_map(s, page, map); + + for_each_object(p, s, addr, page->objects) + if (!test_bit(slab_index(p, s, addr), map)) + mtk_memcfg_add_location(t, s, get_track(s, p, alloc)); +} + +static int mtk_memcfg_list_locations(struct kmem_cache *s, struct seq_file *m, + enum track_item alloc) +{ + unsigned long i, j; + struct loc_track t = { 0, 0, NULL }; + int node; + unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) * + sizeof(unsigned long), GFP_KERNEL); + + if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location), + GFP_TEMPORARY)) { + kfree(map); + return seq_printf(m, "Out of memory\n"); + } + /* Push back cpu slabs */ + flush_all(s); + + for_each_node_state(node, N_NORMAL_MEMORY) { + struct kmem_cache_node *n = get_node(s, node); + unsigned long flags; + struct page *page; + + if (!atomic_long_read(&n->nr_slabs)) + continue; + + spin_lock_irqsave(&n->list_lock, flags); + list_for_each_entry(page, &n->partial, lru) + mtk_memcfg_process_slab(&t, s, page, alloc, map); + list_for_each_entry(page, &n->full, lru) + mtk_memcfg_process_slab(&t, s, page, alloc, map); + spin_unlock_irqrestore(&n->list_lock, flags); + } + + for (i = 0; i < t.count; i++) { + struct location *l = &t.loc[i]; + + seq_printf(m, "%7ld ", l->count); + + if (l->addr) + seq_printf(m, "%pS", (void *)l->addr); + else + seq_printf(m, ""); + + for (j = 0; j < MTK_MEMCFG_SLABTRACE_CNT; j++) + if (l->addrs[j]) + seq_printf(m, " %p", (void *)l->addrs[j]); + + seq_printf(m, "\n"); + } + + free_loc_track(&t); + kfree(map); + + if (!t.count) + seq_printf(m, "No data\n"); + return 0; +} + +static int mtk_memcfg_slabtrace_show(struct seq_file *m, void *p) +{ + struct kmem_cache *s; + mutex_lock(&slab_mutex); + list_for_each_entry(s, &slab_caches, list) { + seq_printf(m, "========== kmem_cache: %s alloc_calls ==========\n", s->name); + if (!(s->flags & SLAB_STORE_USER)) { + continue; + } else { + mtk_memcfg_list_locations(s, m, TRACK_ALLOC); + } + } + mutex_unlock(&slab_mutex); + return 0; +} + +int slabtrace_open(struct inode *inode, struct file *file) +{ + return single_open(file, mtk_memcfg_slabtrace_show, NULL); +} + +#endif + #endif /* CONFIG_SLABINFO */