From: Jeongtae Park Date: Fri, 1 Jun 2018 00:11:00 +0000 (+0900) Subject: arm64: add support for kernel mode NEON in any context without preemption X-Git-Url: https://git.stricted.de/?a=commitdiff_plain;h=bfe01b370640d48e189190f44a0c2653b9cbe4ff;p=GitHub%2FLineageOS%2Fandroid_kernel_motorola_exynos9610.git arm64: add support for kernel mode NEON in any context without preemption Change-Id: I6be8365f65f6dc8d894e21174fdd3d6070bfae25 Signed-off-by: Jeongtae Park --- diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 5155f21e15e3..82760fc0cd3e 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -41,6 +41,23 @@ struct fpsimd_state { unsigned int cpu; }; +struct fpsimd_kernel_state { + __uint128_t vregs[32]; + u32 fpsr; + u32 fpcr; + /* + * indicate the depth of using FP/SIMD registers in kernel mode. + * above kernel state should be preserved at first time + * before FP/SIMD registers be used by other tasks + * and the state should be restored before they be used by own. + * + * a kernel thread which uses FP/SIMD registers have to + * set this depth and it could utilize for a tasks executes + * some NEON instructions without preemption disable. + */ + atomic_t depth; +}; + /* * Struct for stacking the bottom 'n' FP/SIMD registers. */ @@ -85,6 +102,12 @@ extern void fpsimd_load_partial_state(struct fpsimd_partial_state *state); extern void __efi_fpsimd_begin(void); extern void __efi_fpsimd_end(void); +void fpsimd_set_task_using(struct task_struct *t); +void fpsimd_clr_task_using(struct task_struct *t); + +void fpsimd_get(void); +void fpsimd_put(void); + #endif #endif diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index fda6f5812281..907ccfd0a4ff 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -108,6 +108,7 @@ struct thread_struct { unsigned long tp2_value; #endif struct fpsimd_state fpsimd_state; + struct fpsimd_kernel_state fpsimd_kernel_state; unsigned long fault_address; /* fault info */ unsigned long fault_code; /* ESR_EL1 value */ struct debug_info debug; /* debugging */ diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 082756890137..b81abcc6d836 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -130,6 +130,13 @@ void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs) void fpsimd_thread_switch(struct task_struct *next) { + struct fpsimd_state *cur_st = ¤t->thread.fpsimd_state; + struct fpsimd_kernel_state *cur_kst + = ¤t->thread.fpsimd_kernel_state; + struct fpsimd_state *nxt_st = &next->thread.fpsimd_state; + struct fpsimd_kernel_state *nxt_kst + = &next->thread.fpsimd_kernel_state; + if (!system_supports_fpsimd()) return; /* @@ -138,7 +145,15 @@ void fpsimd_thread_switch(struct task_struct *next) * 'current'. */ if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE)) - fpsimd_save_state(¤t->thread.fpsimd_state); + fpsimd_save_state(cur_st); + + if (atomic_read(&cur_kst->depth)) + fpsimd_save_state((struct fpsimd_state *)cur_kst); + + if (atomic_read(&nxt_kst->depth)) { + fpsimd_load_state((struct fpsimd_state *)nxt_kst); + this_cpu_write(fpsimd_last_state, (struct fpsimd_state *)nxt_kst); + } if (next->mm) { /* @@ -148,10 +163,8 @@ void fpsimd_thread_switch(struct task_struct *next) * the TIF_FOREIGN_FPSTATE flag so the state will be loaded * upon the next return to userland. */ - struct fpsimd_state *st = &next->thread.fpsimd_state; - - if (__this_cpu_read(fpsimd_last_state) == st - && st->cpu == smp_processor_id()) + if (__this_cpu_read(fpsimd_last_state) == nxt_st + && nxt_st->cpu == smp_processor_id()) clear_ti_thread_flag(task_thread_info(next), TIF_FOREIGN_FPSTATE); else @@ -231,6 +244,42 @@ void fpsimd_flush_task_state(struct task_struct *t) t->thread.fpsimd_state.cpu = NR_CPUS; } +void fpsimd_set_task_using(struct task_struct *t) +{ + atomic_set(&t->thread.fpsimd_kernel_state.depth, 1); +} + +void fpsimd_clr_task_using(struct task_struct *t) +{ + atomic_set(&t->thread.fpsimd_kernel_state.depth, 0); +} + +void fpsimd_get(void) +{ + if (in_interrupt()) + return; + + if (atomic_inc_return(¤t->thread.fpsimd_kernel_state.depth) == 1) { + preempt_disable(); + if (current->mm && + !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE)) { + fpsimd_save_state(¤t->thread.fpsimd_state); + fpsimd_flush_task_state(current); + } + this_cpu_write(fpsimd_last_state, NULL); + preempt_enable(); + } +} + +void fpsimd_put(void) +{ + if (in_interrupt()) + return; + + BUG_ON(atomic_dec_return( + ¤t->thread.fpsimd_kernel_state.depth) < 0); +} + #ifdef CONFIG_KERNEL_MODE_NEON static DEFINE_PER_CPU(struct fpsimd_partial_state, hardirq_fpsimdstate); @@ -340,13 +389,22 @@ static int fpsimd_cpu_pm_notifier(struct notifier_block *self, { switch (cmd) { case CPU_PM_ENTER: - if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE)) + if ((current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE)) + || atomic_read(¤t->thread.fpsimd_kernel_state.depth)) { fpsimd_save_state(¤t->thread.fpsimd_state); + } this_cpu_write(fpsimd_last_state, NULL); break; case CPU_PM_EXIT: if (current->mm) set_thread_flag(TIF_FOREIGN_FPSTATE); + + if (atomic_read(¤t->thread.fpsimd_kernel_state.depth)) { + fpsimd_load_state(¤t->thread.fpsimd_state); + this_cpu_write(fpsimd_last_state, + ¤t->thread.fpsimd_state); + current->thread.fpsimd_state.cpu = smp_processor_id(); + } break; case CPU_PM_ENTER_FAILED: default: