sched/core: Allow putting thread_info into task_struct

If an arch opts in by setting CONFIG_THREAD_INFO_IN_TASK_STRUCT,
then thread_info is defined as a single 'u32 flags' and is the first
entry of task_struct.  thread_info::task is removed (it serves no
purpose if thread_info is embedded in task_struct), and
thread_info::cpu gets its own slot in task_struct.

This is heavily based on a patch written by Linus.

Originally-from: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Jann Horn <jann@thejh.net>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/a0898196f0476195ca02713691a5037a14f2aac5.1473801993.git.luto@kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/include/linux/init_task.h b/include/linux/init_task.h
index f8834f820ec2475f0c4dab07df153034ccd7d26c..9c04d44eeb3c14fd2f144054d207358434b55d0d 100644 (file)
--- a/include/linux/init_task.h
+++ b/include/linux/init_task.h
@@ -15,6 +15,8 @@
 #include <net/net_namespace.h>
 #include <linux/sched/rt.h>
 
+#include <asm/thread_info.h>
+
 #ifdef CONFIG_SMP
 # define INIT_PUSHABLE_TASKS(tsk)                                      \
        .pushable_tasks = PLIST_NODE_INIT(tsk.pushable_tasks, MAX_PRIO),
@@ -183,12 +185,19 @@ extern struct task_group root_task_group;
 # define INIT_KASAN(tsk)
 #endif
 
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+# define INIT_TASK_TI(tsk) .thread_info = INIT_THREAD_INFO(tsk),
+#else
+# define INIT_TASK_TI(tsk)
+#endif
+
 /*
  *  INIT_TASK is used to set up the first task table, touch at
  * your own risk!. Base=0, limit=0x1fffff (=2MB)
  */
 #define INIT_TASK(tsk) \
 {                                                                      \
+       INIT_TASK_TI(tsk)                                               \
        .state          = 0,                                            \
        .stack          = init_stack,                                   \
        .usage          = ATOMIC_INIT(2),                               \

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 20f9f47bcfd07c3069e7653270a805a101a15a41..a287e8b13549874034fb71fadc02249292db50cc 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1458,6 +1458,13 @@ struct tlbflush_unmap_batch {
 };
 
 struct task_struct {
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+       /*
+        * For reasons of header soup (see current_thread_info()), this
+        * must be the first element of task_struct.
+        */
+       struct thread_info thread_info;
+#endif
        volatile long state;    /* -1 unrunnable, 0 runnable, >0 stopped */
        void *stack;
        atomic_t usage;
@@ -1467,6 +1474,9 @@ struct task_struct {
 #ifdef CONFIG_SMP
        struct llist_node wake_entry;
        int on_cpu;
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+       unsigned int cpu;       /* current CPU */
+#endif
        unsigned int wakee_flips;
        unsigned long wakee_flip_decay_ts;
        struct task_struct *last_wakee;
@@ -2588,7 +2598,9 @@ extern void set_curr_task(int cpu, struct task_struct *p);
 void yield(void);
 
 union thread_union {
+#ifndef CONFIG_THREAD_INFO_IN_TASK
        struct thread_info thread_info;
+#endif
        unsigned long stack[THREAD_SIZE/sizeof(long)];
 };
 
@@ -3076,10 +3088,26 @@ static inline void threadgroup_change_end(struct task_struct *tsk)
        cgroup_threadgroup_change_end(tsk);
 }
 
-#ifndef __HAVE_THREAD_FUNCTIONS
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+
+static inline struct thread_info *task_thread_info(struct task_struct *task)
+{
+       return &task->thread_info;
+}
+static inline void *task_stack_page(const struct task_struct *task)
+{
+       return task->stack;
+}
+#define setup_thread_stack(new,old)    do { } while(0)
+static inline unsigned long *end_of_stack(const struct task_struct *task)
+{
+       return task->stack;
+}
+
+#elif !defined(__HAVE_THREAD_FUNCTIONS)
 
 #define task_thread_info(task) ((struct thread_info *)(task)->stack)
-#define task_stack_page(task)  ((task)->stack)
+#define task_stack_page(task)  ((void *)(task)->stack)
 
 static inline void setup_thread_stack(struct task_struct *p, struct task_struct *org)
 {
@@ -3379,7 +3407,11 @@ static inline void ptrace_signal_wake_up(struct task_struct *t, bool resume)
 
 static inline unsigned int task_cpu(const struct task_struct *p)
 {
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+       return p->cpu;
+#else
        return task_thread_info(p)->cpu;
+#endif
 }
 
 static inline int task_node(const struct task_struct *p)

diff --git a/include/linux/thread_info.h b/include/linux/thread_info.h
index 2b5b10eed74ff52fc5b11fe911be7d84b08eda77..e2d0fd81b1bae8843c6485344b455ca1eb3c55eb 100644 (file)
--- a/include/linux/thread_info.h
+++ b/include/linux/thread_info.h
@@ -13,6 +13,21 @@
 struct timespec;
 struct compat_timespec;
 
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+struct thread_info {
+       u32                     flags;          /* low level flags */
+};
+
+#define INIT_THREAD_INFO(tsk)                  \
+{                                              \
+       .flags          = 0,                    \
+}
+#endif
+
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+#define current_thread_info() ((struct thread_info *)current)
+#endif
+
 /*
  * System call restart block.
  */

diff --git a/init/Kconfig b/init/Kconfig
index cac3f096050d5b27a9bd5de8600469f3c5aa0ada..ec8d43894b02056d4c7874dadb640b989f787f3d 100644 (file)
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -26,6 +26,13 @@ config IRQ_WORK
 config BUILDTIME_EXTABLE_SORT
        bool
 
+config THREAD_INFO_IN_TASK
+       bool
+       help
+         Select this to move thread_info off the stack into task_struct.  To
+         make this work, an arch will need to remove all thread_info fields
+         except flags and fix any runtime bugs.
+
 menu "General setup"
 
 config BROKEN

diff --git a/init/init_task.c b/init/init_task.c
index ba0a7f362d9e3ffbf00a45c7baa5c2f0a2186b8f..11f83be1fa7997bb2a38e01f9d4a69f21f37bf93 100644 (file)
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -22,5 +22,8 @@ EXPORT_SYMBOL(init_task);
  * Initial thread structure. Alignment of this is handled by a special
  * linker map entry.
  */
-union thread_union init_thread_union __init_task_data =
-       { INIT_THREAD_INFO(init_task) };
+union thread_union init_thread_union __init_task_data = {
+#ifndef CONFIG_THREAD_INFO_IN_TASK
+       INIT_THREAD_INFO(init_task)
+#endif
+};

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index c64fc5114004f6a893a1bf942a49f3df1d6c8fea..3655c9625e5b39c2709dd00f736fb1cc99b11f7b 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1000,7 +1000,11 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
         * per-task data have been completed by this moment.
         */
        smp_wmb();
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+       p->cpu = cpu;
+#else
        task_thread_info(p)->cpu = cpu;
+#endif
        p->wake_cpu = cpu;
 #endif
 }