slub: Make CONFIG_DEBUG_PAGE_ALLOC work with new fastpath

Fastpath can do a speculative access to a page that CONFIG_DEBUG_PAGE_ALLOC may have
marked as invalid to retrieve the pointer to the next free object.

Use probe_kernel_read in that case in order not to cause a page fault.

Cc: <stable@kernel.org> # 38.x
Reported-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>

diff --git a/mm/slub.c b/mm/slub.c
index 8657ab838b829a844a2fda412a084fdde8835502..97bb5b8d935fb0f6a9c029032a73d56076ee7208 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -261,6 +261,18 @@ static inline void *get_freepointer(struct kmem_cache *s, void *object)
        return *(void **)(object + s->offset);
 }
 
+static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
+{
+       void *p;
+
+#ifdef CONFIG_DEBUG_PAGEALLOC
+       probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p));
+#else
+       p = get_freepointer(s, object);
+#endif
+       return p;
+}
+
 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
 {
        *(void **)(object + s->offset) = fp;
@@ -1933,7 +1945,7 @@ redo:
                if (unlikely(!this_cpu_cmpxchg_double(
                                s->cpu_slab->freelist, s->cpu_slab->tid,
                                object, tid,
-                               get_freepointer(s, object), next_tid(tid)))) {
+                               get_freepointer_safe(s, object), next_tid(tid)))) {
 
                        note_cmpxchg_failure("slab_alloc", s, tid);
                        goto redo;