From: Jeongtae Park Date: Thu, 10 Nov 2016 06:39:58 +0000 (+0900) Subject: arm64: add support for kernel mode NEON in any context without preemption X-Git-Url: https://git.stricted.de/?a=commitdiff_plain;h=2d0467372776ada0b23824e3ac2ac2b595fc33b3;p=GitHub%2FLineageOS%2Fandroid_kernel_motorola_exynos9610.git arm64: add support for kernel mode NEON in any context without preemption Change-Id: I09a5d88cc304ca1dba9fabf3f4c29aacebd88771 Signed-off-by: Jeongtae Park Signed-off-by: Wooki Min --- diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 410c48163c6a..89dd572e39cd 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -41,6 +41,22 @@ struct fpsimd_state { unsigned int cpu; }; +struct fpsimd_kernel_state { + __uint128_t vregs[32]; + u32 fpsr; + u32 fpcr; + /* + * indicate the depth of using FP/SIMD registers in kernel mode. + * above kernel state should be preserved at first time + * before FP/SIMD registers be used by other tasks + * and the state should be restored before they be used by own. + * + * a kernel thread which uses FP/SIMD registers have to + * set this depth and it could utilize for a tasks executes + * some NEON instructions without preemption disable. + */ + atomic_t depth; +}; #if defined(__KERNEL__) && defined(CONFIG_COMPAT) /* Masks for extracting the FPSR and FPCR from the FPSCR */ @@ -71,6 +87,12 @@ extern void fpsimd_flush_task_state(struct task_struct *target); extern void __efi_fpsimd_begin(void); extern void __efi_fpsimd_end(void); +void fpsimd_set_task_using(struct task_struct *t); +void fpsimd_clr_task_using(struct task_struct *t); + +void fpsimd_get(void); +void fpsimd_put(void); + #endif #endif diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index fda6f5812281..907ccfd0a4ff 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -108,6 +108,7 @@ struct thread_struct { unsigned long tp2_value; #endif struct fpsimd_state fpsimd_state; + struct fpsimd_kernel_state fpsimd_kernel_state; unsigned long fault_address; /* fault info */ unsigned long fault_code; /* ESR_EL1 value */ struct debug_info debug; /* debugging */ diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 5d547deb6996..8903c01777d4 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -137,15 +137,31 @@ void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs) void fpsimd_thread_switch(struct task_struct *next) { + struct fpsimd_state *cur_st = ¤t->thread.fpsimd_state; + struct fpsimd_kernel_state *cur_kst + = ¤t->thread.fpsimd_kernel_state; + struct fpsimd_state *nxt_st = &next->thread.fpsimd_state; + struct fpsimd_kernel_state *nxt_kst + = &next->thread.fpsimd_kernel_state; + if (!system_supports_fpsimd()) return; + /* * Save the current FPSIMD state to memory, but only if whatever is in * the registers is in fact the most recent userland FPSIMD state of * 'current'. */ if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE)) - fpsimd_save_state(¤t->thread.fpsimd_state); + fpsimd_save_state(cur_st); + + if (atomic_read(&cur_kst->depth)) + fpsimd_save_state((struct fpsimd_state *)cur_kst); + + if (atomic_read(&nxt_kst->depth)) { + fpsimd_load_state((struct fpsimd_state *)nxt_kst); + this_cpu_write(fpsimd_last_state, (struct fpsimd_state *)nxt_kst); + } if (next->mm) { /* @@ -155,10 +171,8 @@ void fpsimd_thread_switch(struct task_struct *next) * the TIF_FOREIGN_FPSTATE flag so the state will be loaded * upon the next return to userland. */ - struct fpsimd_state *st = &next->thread.fpsimd_state; - - if (__this_cpu_read(fpsimd_last_state) == st - && st->cpu == smp_processor_id()) + if (__this_cpu_read(fpsimd_last_state) == nxt_st + && nxt_st->cpu == smp_processor_id()) clear_ti_thread_flag(task_thread_info(next), TIF_FOREIGN_FPSTATE); else @@ -252,6 +266,42 @@ void fpsimd_flush_task_state(struct task_struct *t) t->thread.fpsimd_state.cpu = NR_CPUS; } +void fpsimd_set_task_using(struct task_struct *t) +{ + atomic_set(&t->thread.fpsimd_kernel_state.depth, 1); +} + +void fpsimd_clr_task_using(struct task_struct *t) +{ + atomic_set(&t->thread.fpsimd_kernel_state.depth, 0); +} + +void fpsimd_get(void) +{ + if (in_interrupt()) + return; + + if (atomic_inc_return(¤t->thread.fpsimd_kernel_state.depth) == 1) { + preempt_disable(); + if (current->mm && + !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE)) { + fpsimd_save_state(¤t->thread.fpsimd_state); + fpsimd_flush_task_state(current); + } + this_cpu_write(fpsimd_last_state, NULL); + preempt_enable(); + } +} + +void fpsimd_put(void) +{ + if (in_interrupt()) + return; + + BUG_ON(atomic_dec_return( + ¤t->thread.fpsimd_kernel_state.depth) < 0); +} + #ifdef CONFIG_KERNEL_MODE_NEON DEFINE_PER_CPU(bool, kernel_neon_busy); @@ -382,13 +432,22 @@ static int fpsimd_cpu_pm_notifier(struct notifier_block *self, { switch (cmd) { case CPU_PM_ENTER: - if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE)) + if ((current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE)) + || atomic_read(¤t->thread.fpsimd_kernel_state.depth)) { fpsimd_save_state(¤t->thread.fpsimd_state); + } this_cpu_write(fpsimd_last_state, NULL); break; case CPU_PM_EXIT: if (current->mm) set_thread_flag(TIF_FOREIGN_FPSTATE); + + if (atomic_read(¤t->thread.fpsimd_kernel_state.depth)) { + fpsimd_load_state(¤t->thread.fpsimd_state); + this_cpu_write(fpsimd_last_state, + ¤t->thread.fpsimd_state); + current->thread.fpsimd_state.cpu = smp_processor_id(); + } break; case CPU_PM_ENTER_FAILED: default: