unsigned int cpu;
};
-struct fpsimd_kernel_state {
- __uint128_t vregs[32];
- u32 fpsr;
- u32 fpcr;
- /*
- * indicate the depth of using FP/SIMD registers in kernel mode.
- * above kernel state should be preserved at first time
- * before FP/SIMD registers be used by other tasks
- * and the state should be restored before they be used by own.
- *
- * a kernel thread which uses FP/SIMD registers have to
- * set this depth and it could utilize for a tasks executes
- * some NEON instructions without preemption disable.
- */
- atomic_t depth;
-};
#if defined(__KERNEL__) && defined(CONFIG_COMPAT)
/* Masks for extracting the FPSR and FPCR from the FPSCR */
extern void __efi_fpsimd_begin(void);
extern void __efi_fpsimd_end(void);
-void fpsimd_set_task_using(struct task_struct *t);
-void fpsimd_clr_task_using(struct task_struct *t);
-
-void fpsimd_get(void);
-void fpsimd_put(void);
-
#endif
#endif
void fpsimd_thread_switch(struct task_struct *next)
{
- struct fpsimd_state *cur_st = ¤t->thread.fpsimd_state;
- struct fpsimd_kernel_state *cur_kst
- = ¤t->thread.fpsimd_kernel_state;
- struct fpsimd_state *nxt_st = &next->thread.fpsimd_state;
- struct fpsimd_kernel_state *nxt_kst
- = &next->thread.fpsimd_kernel_state;
-
if (!system_supports_fpsimd())
return;
-
/*
* Save the current FPSIMD state to memory, but only if whatever is in
* the registers is in fact the most recent userland FPSIMD state of
* 'current'.
*/
if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE))
- fpsimd_save_state(cur_st);
-
- if (atomic_read(&cur_kst->depth))
- fpsimd_save_state((struct fpsimd_state *)cur_kst);
-
- if (atomic_read(&nxt_kst->depth)) {
- fpsimd_load_state((struct fpsimd_state *)nxt_kst);
- this_cpu_write(fpsimd_last_state, (struct fpsimd_state *)nxt_kst);
- }
+ fpsimd_save_state(¤t->thread.fpsimd_state);
if (next->mm) {
/*
* the TIF_FOREIGN_FPSTATE flag so the state will be loaded
* upon the next return to userland.
*/
- if (__this_cpu_read(fpsimd_last_state) == nxt_st
- && nxt_st->cpu == smp_processor_id())
+ struct fpsimd_state *st = &next->thread.fpsimd_state;
+
+ if (__this_cpu_read(fpsimd_last_state) == st
+ && st->cpu == smp_processor_id())
clear_ti_thread_flag(task_thread_info(next),
TIF_FOREIGN_FPSTATE);
else
t->thread.fpsimd_state.cpu = NR_CPUS;
}
-void fpsimd_set_task_using(struct task_struct *t)
-{
- atomic_set(&t->thread.fpsimd_kernel_state.depth, 1);
-}
-
-void fpsimd_clr_task_using(struct task_struct *t)
-{
- atomic_set(&t->thread.fpsimd_kernel_state.depth, 0);
-}
-
-void fpsimd_get(void)
-{
- if (in_interrupt())
- return;
-
- if (atomic_inc_return(¤t->thread.fpsimd_kernel_state.depth) == 1) {
- preempt_disable();
- if (current->mm &&
- !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE)) {
- fpsimd_save_state(¤t->thread.fpsimd_state);
- fpsimd_flush_task_state(current);
- }
- this_cpu_write(fpsimd_last_state, NULL);
- preempt_enable();
- }
-}
-
-void fpsimd_put(void)
-{
- if (in_interrupt())
- return;
-
- BUG_ON(atomic_dec_return(
- ¤t->thread.fpsimd_kernel_state.depth) < 0);
-}
-
#ifdef CONFIG_KERNEL_MODE_NEON
DEFINE_PER_CPU(bool, kernel_neon_busy);
{
switch (cmd) {
case CPU_PM_ENTER:
- if ((current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE))
- || atomic_read(¤t->thread.fpsimd_kernel_state.depth)) {
+ if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE))
fpsimd_save_state(¤t->thread.fpsimd_state);
- }
this_cpu_write(fpsimd_last_state, NULL);
break;
case CPU_PM_EXIT:
if (current->mm)
set_thread_flag(TIF_FOREIGN_FPSTATE);
-
- if (atomic_read(¤t->thread.fpsimd_kernel_state.depth)) {
- fpsimd_load_state(¤t->thread.fpsimd_state);
- this_cpu_write(fpsimd_last_state,
- ¤t->thread.fpsimd_state);
- current->thread.fpsimd_state.cpu = smp_processor_id();
- }
break;
case CPU_PM_ENTER_FAILED:
default: