mm: NUMA aware alloc_thread_info_node()

Add a node parameter to alloc_thread_info(), and change its name to
alloc_thread_info_node()

This change is needed to allow NUMA aware kthread_create_on_cpu()

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: David S. Miller <davem@davemloft.net>
Reviewed-by: Andi Kleen <ak@linux.intel.com>
Acked-by: Rusty Russell <rusty@rustcorp.com.au>
Cc: Tejun Heo <tj@kernel.org>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: David Howells <dhowells@redhat.com>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/cris/include/asm/thread_info.h b/arch/cris/include/asm/thread_info.h
index 91776069ca807b30c3902922b70889d9e5bb5746..29b74a10583039b474e3fd941665af8f88fd41e9 100644 (file)
--- a/arch/cris/include/asm/thread_info.h
+++ b/arch/cris/include/asm/thread_info.h
@@ -68,7 +68,7 @@ struct thread_info {
 #define init_thread_info       (init_thread_union.thread_info)
 
 /* thread information allocation */
-#define alloc_thread_info(tsk) ((struct thread_info *) __get_free_pages(GFP_KERNEL,1))
+#define alloc_thread_info(tsk, node) ((struct thread_info *) __get_free_pages(GFP_KERNEL,1))
 #define free_thread_info(ti) free_pages((unsigned long) (ti), 1)
 
 #endif /* !__ASSEMBLY__ */

diff --git a/arch/frv/include/asm/thread_info.h b/arch/frv/include/asm/thread_info.h
index 11f33ead29bf3244b5169cf2073692782b8a9233..8582e9c7531c8f3433f0749fec1861cf92847919 100644 (file)
--- a/arch/frv/include/asm/thread_info.h
+++ b/arch/frv/include/asm/thread_info.h
@@ -84,16 +84,11 @@ register struct thread_info *__current_thread_info asm("gr15");
 
 /* thread information allocation */
 #ifdef CONFIG_DEBUG_STACK_USAGE
-#define alloc_thread_info(tsk)                                 \
-       ({                                                      \
-               struct thread_info *ret;                        \
-                                                               \
-               ret = kzalloc(THREAD_SIZE, GFP_KERNEL);         \
-                                                               \
-               ret;                                            \
-       })
+#define alloc_thread_info_node(tsk, node)                      \
+               kzalloc_node(THREAD_SIZE, GFP_KERNEL, node)
 #else
-#define alloc_thread_info(tsk) kmalloc(THREAD_SIZE, GFP_KERNEL)
+#define alloc_thread_info_node(tsk)                            \
+               kmalloc_node(THREAD_SIZE, GFP_KERNEL, node)
 #endif
 
 #define free_thread_info(info) kfree(info)

diff --git a/arch/ia64/include/asm/thread_info.h b/arch/ia64/include/asm/thread_info.h
index 342004bbefe72344a28d03506cd3a5cf6cbc9218..6392908e8f9816c2d5d7432720fc8497054580e4 100644 (file)
--- a/arch/ia64/include/asm/thread_info.h
+++ b/arch/ia64/include/asm/thread_info.h
@@ -59,11 +59,12 @@ struct thread_info {
 #ifndef ASM_OFFSETS_C
 /* how to get the thread information struct from C */
 #define current_thread_info()  ((struct thread_info *) ((char *) current + IA64_TASK_SIZE))
-#define alloc_thread_info(tsk) ((struct thread_info *) ((char *) (tsk) + IA64_TASK_SIZE))
+#define alloc_thread_info_node(tsk, node)      \
+               ((struct thread_info *) ((char *) (tsk) + IA64_TASK_SIZE))
 #define task_thread_info(tsk)  ((struct thread_info *) ((char *) (tsk) + IA64_TASK_SIZE))
 #else
 #define current_thread_info()  ((struct thread_info *) 0)
-#define alloc_thread_info(tsk) ((struct thread_info *) 0)
+#define alloc_thread_info_node(tsk, node)      ((struct thread_info *) 0)
 #define task_thread_info(tsk)  ((struct thread_info *) 0)
 #endif
 #define free_thread_info(ti)   /* nothing */

diff --git a/arch/m32r/include/asm/thread_info.h b/arch/m32r/include/asm/thread_info.h
index 71faff5bcc278e4dff72c942629a6a1b5dd6a789..0227dba44068b87b8e4406f05d57340fcdec3c87 100644 (file)
--- a/arch/m32r/include/asm/thread_info.h
+++ b/arch/m32r/include/asm/thread_info.h
@@ -96,16 +96,11 @@ static inline struct thread_info *current_thread_info(void)
 
 /* thread information allocation */
 #ifdef CONFIG_DEBUG_STACK_USAGE
-#define alloc_thread_info(tsk)                                 \
-       ({                                                      \
-               struct thread_info *ret;                        \
-                                                               \
-               ret = kzalloc(THREAD_SIZE, GFP_KERNEL);         \
-                                                               \
-               ret;                                            \
-        })
+#define alloc_thread_info_node(tsk, node)                      \
+               kzalloc_node(THREAD_SIZE, GFP_KERNEL, node)
 #else
-#define alloc_thread_info(tsk) kmalloc(THREAD_SIZE, GFP_KERNEL)
+#define alloc_thread_info_node(tsk, node)                      \
+               kmalloc_node(THREAD_SIZE, GFP_KERNEL, node)
 #endif
 
 #define free_thread_info(info) kfree(info)

diff --git a/arch/mips/include/asm/thread_info.h b/arch/mips/include/asm/thread_info.h
index d309556cacf8ac9348f8068b99197673f570b772..d71160de4d10f19fc73e54d0a0a18dd442ecb44f 100644 (file)
--- a/arch/mips/include/asm/thread_info.h
+++ b/arch/mips/include/asm/thread_info.h
@@ -88,9 +88,11 @@ register struct thread_info *__current_thread_info __asm__("$28");
 #define __HAVE_ARCH_THREAD_INFO_ALLOCATOR
 
 #ifdef CONFIG_DEBUG_STACK_USAGE
-#define alloc_thread_info(tsk) kzalloc(THREAD_SIZE, GFP_KERNEL)
+#define alloc_thread_info_node(tsk, node) \
+               kzalloc_node(THREAD_SIZE, GFP_KERNEL, node)
 #else
-#define alloc_thread_info(tsk) kmalloc(THREAD_SIZE, GFP_KERNEL)
+#define alloc_thread_info_node(tsk, node) \
+               kmalloc_node(THREAD_SIZE, GFP_KERNEL, node)
 #endif
 
 #define free_thread_info(info) kfree(info)

diff --git a/arch/mn10300/include/asm/thread_info.h b/arch/mn10300/include/asm/thread_info.h
index aa07a4a5d7949406550ba001f7a8faaa631633ec..8d53f09c878d6592e9b7eeeffc3430a457783293 100644 (file)
--- a/arch/mn10300/include/asm/thread_info.h
+++ b/arch/mn10300/include/asm/thread_info.h
@@ -124,9 +124,11 @@ static inline unsigned long current_stack_pointer(void)
 
 /* thread information allocation */
 #ifdef CONFIG_DEBUG_STACK_USAGE
-#define alloc_thread_info(tsk) kzalloc(THREAD_SIZE, GFP_KERNEL)
+#define alloc_thread_info_node(tsk, node)                      \
+               kzalloc_node(THREAD_SIZE, GFP_KERNEL, node)
 #else
-#define alloc_thread_info(tsk) kmalloc(THREAD_SIZE, GFP_KERNEL)
+#define alloc_thread_info_node(tsk, node)                      \
+               kmalloc_node(THREAD_SIZE, GFP_KERNEL, node)
 #endif
 
 #define free_thread_info(ti)   kfree((ti))

diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h
index 65eb85976a03a9199e3316f5e59424d3105822c7..d8529ef13b2329352a6b85a92adb2574ed768607 100644 (file)
--- a/arch/powerpc/include/asm/thread_info.h
+++ b/arch/powerpc/include/asm/thread_info.h
@@ -72,7 +72,7 @@ struct thread_info {
 
 #define __HAVE_ARCH_THREAD_INFO_ALLOCATOR
 
-extern struct thread_info *alloc_thread_info(struct task_struct *tsk);
+extern struct thread_info *alloc_thread_info_node(struct task_struct *tsk, int node);
 extern void free_thread_info(struct thread_info *ti);
 
 #endif /* THREAD_SHIFT < PAGE_SHIFT */

diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 8303a6c65ef7e85f230bac363bb722c49e460211..f74f355a9617a97ea05e174efaa9f67f4db3e275 100644 (file)
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -1218,11 +1218,11 @@ void __ppc64_runlatch_off(void)
 
 static struct kmem_cache *thread_info_cache;
 
-struct thread_info *alloc_thread_info(struct task_struct *tsk)
+struct thread_info *alloc_thread_info_node(struct task_struct *tsk, int node)
 {
        struct thread_info *ti;
 
-       ti = kmem_cache_alloc(thread_info_cache, GFP_KERNEL);
+       ti = kmem_cache_alloc_node(thread_info_cache, GFP_KERNEL, node);
        if (unlikely(ti == NULL))
                return NULL;
 #ifdef CONFIG_DEBUG_STACK_USAGE

diff --git a/arch/score/include/asm/thread_info.h b/arch/score/include/asm/thread_info.h
index 8570d08f58c1f25cc8194d85cbe987133153526c..2205c62284dba93c87e9726465abc29ccd272e72 100644 (file)
--- a/arch/score/include/asm/thread_info.h
+++ b/arch/score/include/asm/thread_info.h
@@ -71,7 +71,7 @@ struct thread_info {
 register struct thread_info *__current_thread_info __asm__("r28");
 #define current_thread_info()  __current_thread_info
 
-#define alloc_thread_info(tsk) kmalloc(THREAD_SIZE, GFP_KERNEL)
+#define alloc_thread_info_node(tsk, node) kmalloc_node(THREAD_SIZE, GFP_KERNEL, node)
 #define free_thread_info(info) kfree(info)
 
 #endif /* !__ASSEMBLY__ */

diff --git a/arch/sh/include/asm/thread_info.h b/arch/sh/include/asm/thread_info.h
index c228946926edba4025a52cf379a335050059ed29..ea2d5089de1ed27492c5fca9aefd195a2f787831 100644 (file)
--- a/arch/sh/include/asm/thread_info.h
+++ b/arch/sh/include/asm/thread_info.h
@@ -95,7 +95,7 @@ static inline struct thread_info *current_thread_info(void)
 
 #endif
 
-extern struct thread_info *alloc_thread_info(struct task_struct *tsk);
+extern struct thread_info *alloc_thread_info_node(struct task_struct *tsk, int node);
 extern void free_thread_info(struct thread_info *ti);
 extern void arch_task_cache_init(void);
 #define arch_task_cache_init arch_task_cache_init

diff --git a/arch/sh/kernel/process.c b/arch/sh/kernel/process.c
index dcb126dc76fd7b72102f4d5624e7a07a9664a82f..f39ad57296b718cd918599a678c41ad860920bab 100644 (file)
--- a/arch/sh/kernel/process.c
+++ b/arch/sh/kernel/process.c
@@ -32,16 +32,16 @@ void free_thread_xstate(struct task_struct *tsk)
 #if THREAD_SHIFT < PAGE_SHIFT
 static struct kmem_cache *thread_info_cache;
 
-struct thread_info *alloc_thread_info(struct task_struct *tsk)
+struct thread_info *alloc_thread_info(struct task_struct *tsk, int node)
 {
        struct thread_info *ti;
-
-       ti = kmem_cache_alloc(thread_info_cache, GFP_KERNEL);
-       if (unlikely(ti == NULL))
-               return NULL;
 #ifdef CONFIG_DEBUG_STACK_USAGE
-       memset(ti, 0, THREAD_SIZE);
+       gfp_t mask = GFP_KERNEL | __GFP_ZERO;
+#else
+       gfp_t mask = GFP_KERNEL;
 #endif
+
+       ti = kmem_cache_alloc_node(thread_info_cache, mask, node);
        return ti;
 }
 
@@ -64,7 +64,9 @@ struct thread_info *alloc_thread_info(struct task_struct *tsk)
 #else
        gfp_t mask = GFP_KERNEL;
 #endif
-       return (struct thread_info *)__get_free_pages(mask, THREAD_SIZE_ORDER);
+       struct page *page = alloc_pages_node(node, mask, THREAD_SIZE_ORDER);
+
+       return page ? page_address(page) : NULL;
 }
 
 void free_thread_info(struct thread_info *ti)

diff --git a/arch/sparc/include/asm/thread_info_32.h b/arch/sparc/include/asm/thread_info_32.h
index 9dd0318d3ddfe5313eb6317f7c41965c93aece46..fa5753233410baaf9c489fe7968675b418fcdde1 100644 (file)
--- a/arch/sparc/include/asm/thread_info_32.h
+++ b/arch/sparc/include/asm/thread_info_32.h
@@ -82,8 +82,8 @@ register struct thread_info *current_thread_info_reg asm("g6");
 
 #define __HAVE_ARCH_THREAD_INFO_ALLOCATOR
 
-BTFIXUPDEF_CALL(struct thread_info *, alloc_thread_info, void)
-#define alloc_thread_info(tsk) BTFIXUP_CALL(alloc_thread_info)()
+BTFIXUPDEF_CALL(struct thread_info *, alloc_thread_info_node, int)
+#define alloc_thread_info_node(tsk, node) BTFIXUP_CALL(alloc_thread_info_node)(node)
 
 BTFIXUPDEF_CALL(void, free_thread_info, struct thread_info *)
 #define free_thread_info(ti) BTFIXUP_CALL(free_thread_info)(ti)
@@ -92,7 +92,7 @@ BTFIXUPDEF_CALL(void, free_thread_info, struct thread_info *)
 
 /*
  * Size of kernel stack for each process.
- * Observe the order of get_free_pages() in alloc_thread_info().
+ * Observe the order of get_free_pages() in alloc_thread_info_node().
  * The sun4 has 8K stack too, because it's short on memory, and 16K is a waste.
  */
 #define THREAD_SIZE            8192

diff --git a/arch/sparc/include/asm/thread_info_64.h b/arch/sparc/include/asm/thread_info_64.h
index fb2ea7705a46fc228d5d985d30d7686f0135b3ea..60d86be1a53381bd0fd09c3b133c0c0c80e60bd1 100644 (file)
--- a/arch/sparc/include/asm/thread_info_64.h
+++ b/arch/sparc/include/asm/thread_info_64.h
@@ -146,21 +146,21 @@ register struct thread_info *current_thread_info_reg asm("g6");
 #define __HAVE_ARCH_THREAD_INFO_ALLOCATOR
 
 #ifdef CONFIG_DEBUG_STACK_USAGE
-#define alloc_thread_info(tsk)                                 \
-({                                                             \
-       struct thread_info *ret;                                \
-                                                               \
-       ret = (struct thread_info *)                            \
-         __get_free_pages(GFP_KERNEL, __THREAD_INFO_ORDER);    \
-       if (ret)                                                \
-               memset(ret, 0, PAGE_SIZE<<__THREAD_INFO_ORDER); \
-       ret;                                                    \
-})
+#define THREAD_FLAGS (GFP_KERNEL | __GFP_ZERO)
 #else
-#define alloc_thread_info(tsk) \
-       ((struct thread_info *)__get_free_pages(GFP_KERNEL, __THREAD_INFO_ORDER))
+#define THREAD_FLAGS (GFP_KERNEL)
 #endif
 
+#define alloc_thread_info_node(tsk, node)                              \
+({                                                                     \
+       struct page *page = alloc_pages_node(node, THREAD_FLAGS,        \
+                                            __THREAD_INFO_ORDER);      \
+       struct thread_info *ret;                                        \
+                                                                       \
+       ret = page ? page_address(page) : NULL;                         \
+       ret;                                                            \
+})
+
 #define free_thread_info(ti) \
        free_pages((unsigned long)(ti),__THREAD_INFO_ORDER)
 

diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c
index 92319aa8b66212c35b6ee377db990057d05ee0df..fe09fd8be6956e5f00fe0fddc5c9bbf6a11882c5 100644 (file)
--- a/arch/sparc/mm/srmmu.c
+++ b/arch/sparc/mm/srmmu.c
@@ -650,7 +650,7 @@ static void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len)
  * mappings on the kernel stack without any special code as we did
  * need on the sun4c.
  */
-static struct thread_info *srmmu_alloc_thread_info(void)
+static struct thread_info *srmmu_alloc_thread_info_node(int node)
 {
        struct thread_info *ret;
 
@@ -2271,7 +2271,7 @@ void __init ld_mmu_srmmu(void)
 
        BTFIXUPSET_CALL(mmu_info, srmmu_mmu_info, BTFIXUPCALL_NORM);
 
-       BTFIXUPSET_CALL(alloc_thread_info, srmmu_alloc_thread_info, BTFIXUPCALL_NORM);
+       BTFIXUPSET_CALL(alloc_thread_info_node, srmmu_alloc_thread_info_node, BTFIXUPCALL_NORM);
        BTFIXUPSET_CALL(free_thread_info, srmmu_free_thread_info, BTFIXUPCALL_NORM);
 
        BTFIXUPSET_CALL(pte_to_pgoff, srmmu_pte_to_pgoff, BTFIXUPCALL_NORM);

diff --git a/arch/sparc/mm/sun4c.c b/arch/sparc/mm/sun4c.c
index b5137cc2aba310ad5b0c877d58edabf01cc46010..a2350b5e68aa66c237999dca7790a7e7e0282c48 100644 (file)
--- a/arch/sparc/mm/sun4c.c
+++ b/arch/sparc/mm/sun4c.c
@@ -922,7 +922,7 @@ static inline void garbage_collect(int entry)
        free_locked_segment(BUCKET_ADDR(entry));
 }
 
-static struct thread_info *sun4c_alloc_thread_info(void)
+static struct thread_info *sun4c_alloc_thread_info_node(int node)
 {
        unsigned long addr, pages;
        int entry;
@@ -2155,7 +2155,7 @@ void __init ld_mmu_sun4c(void)
        BTFIXUPSET_CALL(__swp_offset, sun4c_swp_offset, BTFIXUPCALL_NORM);
        BTFIXUPSET_CALL(__swp_entry, sun4c_swp_entry, BTFIXUPCALL_NORM);
 
-       BTFIXUPSET_CALL(alloc_thread_info, sun4c_alloc_thread_info, BTFIXUPCALL_NORM);
+       BTFIXUPSET_CALL(alloc_thread_info_node, sun4c_alloc_thread_info_node, BTFIXUPCALL_NORM);
        BTFIXUPSET_CALL(free_thread_info, sun4c_free_thread_info, BTFIXUPCALL_NORM);
 
        BTFIXUPSET_CALL(mmu_info, sun4c_mmu_info, BTFIXUPCALL_NORM);

diff --git a/arch/tile/include/asm/thread_info.h b/arch/tile/include/asm/thread_info.h
index 9e8e9c4dfa2af284a62bcc430af5e845ee03fb80..3405b52853b869ba7bae0fcd7e091051fdaa30a7 100644 (file)
--- a/arch/tile/include/asm/thread_info.h
+++ b/arch/tile/include/asm/thread_info.h
@@ -84,7 +84,7 @@ register unsigned long stack_pointer __asm__("sp");
   ((struct thread_info *)(stack_pointer & -THREAD_SIZE))
 
 #define __HAVE_ARCH_THREAD_INFO_ALLOCATOR
-extern struct thread_info *alloc_thread_info(struct task_struct *task);
+extern struct thread_info *alloc_thread_info_node(struct task_struct *task, int node);
 extern void free_thread_info(struct thread_info *info);
 
 /* Sit on a nap instruction until interrupted. */

diff --git a/arch/tile/kernel/process.c b/arch/tile/kernel/process.c
index b9cd962e1d307c45af9b342f36941d8cc67e9e04..d0065103eb7bd48e98003e2d594c7c8b2572f0c4 100644 (file)
--- a/arch/tile/kernel/process.c
+++ b/arch/tile/kernel/process.c
@@ -109,7 +109,7 @@ void cpu_idle(void)
        }
 }
 
-struct thread_info *alloc_thread_info(struct task_struct *task)
+struct thread_info *alloc_thread_info_node(struct task_struct *task, int node)
 {
        struct page *page;
        gfp_t flags = GFP_KERNEL;
@@ -118,7 +118,7 @@ struct thread_info *alloc_thread_info(struct task_struct *task)
        flags |= __GFP_ZERO;
 #endif
 
-       page = alloc_pages(flags, THREAD_SIZE_ORDER);
+       page = alloc_pages_node(node, flags, THREAD_SIZE_ORDER);
        if (!page)
                return NULL;
 

diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index f0b6e5dbc5a03b97f821cb71a6d4b95c2bc51b77..1f2e61e28981b7a32d8f9f6f409475460a2ca73d 100644 (file)
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -161,8 +161,14 @@ struct thread_info {
 
 #define __HAVE_ARCH_THREAD_INFO_ALLOCATOR
 
-#define alloc_thread_info(tsk)                                         \
-       ((struct thread_info *)__get_free_pages(THREAD_FLAGS, THREAD_ORDER))
+#define alloc_thread_info_node(tsk, node)                              \
+({                                                                     \
+       struct page *page = alloc_pages_node(node, THREAD_FLAGS,        \
+                                            THREAD_ORDER);             \
+       struct thread_info *ret = page ? page_address(page) : NULL;     \
+                                                                       \
+       ret;                                                            \
+})
 
 #ifdef CONFIG_X86_32
 

diff --git a/kernel/fork.c b/kernel/fork.c
index cffbe8a4e1fc185da3628a45ecdd381651f8ca00..cbc6adc6e891b96d347ab2aef78d35808807b546 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -117,14 +117,17 @@ static struct kmem_cache *task_struct_cachep;
 #endif
 
 #ifndef __HAVE_ARCH_THREAD_INFO_ALLOCATOR
-static inline struct thread_info *alloc_thread_info(struct task_struct *tsk)
+static struct thread_info *alloc_thread_info_node(struct task_struct *tsk,
+                                                 int node)
 {
 #ifdef CONFIG_DEBUG_STACK_USAGE
        gfp_t mask = GFP_KERNEL | __GFP_ZERO;
 #else
        gfp_t mask = GFP_KERNEL;
 #endif
-       return (struct thread_info *)__get_free_pages(mask, THREAD_SIZE_ORDER);
+       struct page *page = alloc_pages_node(node, mask, THREAD_SIZE_ORDER);
+
+       return page ? page_address(page) : NULL;
 }
 
 static inline void free_thread_info(struct thread_info *ti)
@@ -260,7 +263,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig)
        if (!tsk)
                return NULL;
 
-       ti = alloc_thread_info(tsk);
+       ti = alloc_thread_info_node(tsk, node);
        if (!ti) {
                free_task_struct(tsk);
                return NULL;