arm64: add support for kernel mode NEON in any context without preemption

Change-Id: I6be8365f65f6dc8d894e21174fdd3d6070bfae25
Signed-off-by: Jeongtae Park <jtp.park@samsung.com>

diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index 5155f21e15e3657cb331cbaca50912cc711ef467..82760fc0cd3ecd15c98b20f1006227e4d2e08abe 100644 (file)
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -41,6 +41,23 @@ struct fpsimd_state {
        unsigned int cpu;
 };
 
+struct fpsimd_kernel_state {
+       __uint128_t vregs[32];
+       u32 fpsr;
+       u32 fpcr;
+       /*
+        * indicate the depth of using FP/SIMD registers in kernel mode.
+        * above kernel state should be preserved at first time
+        * before FP/SIMD registers be used by other tasks
+        * and the state should be restored before they be used by own.
+        *
+        * a kernel thread which uses FP/SIMD registers have to
+        * set this depth and it could utilize for a tasks executes
+        * some NEON instructions without preemption disable.
+        */
+       atomic_t depth;
+};
+
 /*
  * Struct for stacking the bottom 'n' FP/SIMD registers.
  */
@@ -85,6 +102,12 @@ extern void fpsimd_load_partial_state(struct fpsimd_partial_state *state);
 extern void __efi_fpsimd_begin(void);
 extern void __efi_fpsimd_end(void);
 
+void fpsimd_set_task_using(struct task_struct *t);
+void fpsimd_clr_task_using(struct task_struct *t);
+
+void fpsimd_get(void);
+void fpsimd_put(void);
+
 #endif
 
 #endif

diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index fda6f5812281c2d6f549283d4277ab04e4cb153d..907ccfd0a4ffdab5f9abf5584e160979f93a83ff 100644 (file)
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -108,6 +108,7 @@ struct thread_struct {
        unsigned long           tp2_value;
 #endif
        struct fpsimd_state     fpsimd_state;
+       struct fpsimd_kernel_state fpsimd_kernel_state;
        unsigned long           fault_address;  /* fault info */
        unsigned long           fault_code;     /* ESR_EL1 value */
        struct debug_info       debug;          /* debugging */

diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 08275689013700c7a6810d3c5d7aeb65c5e19518..b81abcc6d83609b974fabb75e0a33bd462e0b4ac 100644 (file)
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -130,6 +130,13 @@ void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
 
 void fpsimd_thread_switch(struct task_struct *next)
 {
+       struct fpsimd_state *cur_st = &current->thread.fpsimd_state;
+       struct fpsimd_kernel_state *cur_kst
+                       = &current->thread.fpsimd_kernel_state;
+       struct fpsimd_state *nxt_st = &next->thread.fpsimd_state;
+       struct fpsimd_kernel_state *nxt_kst
+                       = &next->thread.fpsimd_kernel_state;
+
        if (!system_supports_fpsimd())
                return;
        /*
@@ -138,7 +145,15 @@ void fpsimd_thread_switch(struct task_struct *next)
         * 'current'.
         */
        if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE))
-               fpsimd_save_state(&current->thread.fpsimd_state);
+               fpsimd_save_state(cur_st);
+
+       if (atomic_read(&cur_kst->depth))
+               fpsimd_save_state((struct fpsimd_state *)cur_kst);
+
+       if (atomic_read(&nxt_kst->depth)) {
+               fpsimd_load_state((struct fpsimd_state *)nxt_kst);
+               this_cpu_write(fpsimd_last_state, (struct fpsimd_state *)nxt_kst);
+       }
 
        if (next->mm) {
                /*
@@ -148,10 +163,8 @@ void fpsimd_thread_switch(struct task_struct *next)
                 * the TIF_FOREIGN_FPSTATE flag so the state will be loaded
                 * upon the next return to userland.
                 */
-               struct fpsimd_state *st = &next->thread.fpsimd_state;
-
-               if (__this_cpu_read(fpsimd_last_state) == st
-                   && st->cpu == smp_processor_id())
+               if (__this_cpu_read(fpsimd_last_state) == nxt_st
+                   && nxt_st->cpu == smp_processor_id())
                        clear_ti_thread_flag(task_thread_info(next),
                                             TIF_FOREIGN_FPSTATE);
                else
@@ -231,6 +244,42 @@ void fpsimd_flush_task_state(struct task_struct *t)
        t->thread.fpsimd_state.cpu = NR_CPUS;
 }
 
+void fpsimd_set_task_using(struct task_struct *t)
+{
+       atomic_set(&t->thread.fpsimd_kernel_state.depth, 1);
+}
+
+void fpsimd_clr_task_using(struct task_struct *t)
+{
+       atomic_set(&t->thread.fpsimd_kernel_state.depth, 0);
+}
+
+void fpsimd_get(void)
+{
+       if (in_interrupt())
+               return;
+
+       if (atomic_inc_return(&current->thread.fpsimd_kernel_state.depth) == 1) {
+               preempt_disable();
+               if (current->mm &&
+                   !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE)) {
+                       fpsimd_save_state(&current->thread.fpsimd_state);
+                       fpsimd_flush_task_state(current);
+               }
+               this_cpu_write(fpsimd_last_state, NULL);
+               preempt_enable();
+       }
+}
+
+void fpsimd_put(void)
+{
+       if (in_interrupt())
+               return;
+
+       BUG_ON(atomic_dec_return(
+               &current->thread.fpsimd_kernel_state.depth) < 0);
+}
+
 #ifdef CONFIG_KERNEL_MODE_NEON
 
 static DEFINE_PER_CPU(struct fpsimd_partial_state, hardirq_fpsimdstate);
@@ -340,13 +389,22 @@ static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
 {
        switch (cmd) {
        case CPU_PM_ENTER:
-               if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE))
+               if ((current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE))
+                    || atomic_read(&current->thread.fpsimd_kernel_state.depth)) {
                        fpsimd_save_state(&current->thread.fpsimd_state);
+               }
                this_cpu_write(fpsimd_last_state, NULL);
                break;
        case CPU_PM_EXIT:
                if (current->mm)
                        set_thread_flag(TIF_FOREIGN_FPSTATE);
+
+               if (atomic_read(&current->thread.fpsimd_kernel_state.depth)) {
+                       fpsimd_load_state(&current->thread.fpsimd_state);
+                       this_cpu_write(fpsimd_last_state,
+                                       &current->thread.fpsimd_state);
+                       current->thread.fpsimd_state.cpu = smp_processor_id();
+               }
                break;
        case CPU_PM_ENTER_FAILED:
        default: