#define STKFRM (PPC_MIN_STKFRM + 16)
- .macro inst32 op
-reg = 0
- .rept 32
-20: \op reg,0,r4
- b 3f
- EX_TABLE(20b,99f)
-reg = reg + 1
- .endr
- .endm
-
-/* Get the contents of frN into fr0; N is in r3. */
+/* Get the contents of frN into *p; N is in r3 and p is in r4. */
_GLOBAL(get_fpr)
mflr r0
+ mfmsr r6
+ ori r7, r6, MSR_FP
+ MTMSRD(r7)
+ isync
rlwinm r3,r3,3,0xf8
bcl 20,31,1f
- blr /* fr0 is already in fr0 */
- nop
-reg = 1
- .rept 31
- fmr fr0,reg
- blr
+reg = 0
+ .rept 32
+ stfd reg, 0(r4)
+ b 2f
reg = reg + 1
.endr
1: mflr r5
mtctr r5
mtlr r0
bctr
+2: MTMSRD(r6)
+ isync
+ blr
-/* Put the contents of fr0 into frN; N is in r3. */
+/* Put the contents of *p into frN; N is in r3 and p is in r4. */
_GLOBAL(put_fpr)
mflr r0
+ mfmsr r6
+ ori r7, r6, MSR_FP
+ MTMSRD(r7)
+ isync
rlwinm r3,r3,3,0xf8
bcl 20,31,1f
- blr /* fr0 is already in fr0 */
- nop
-reg = 1
- .rept 31
- fmr reg,fr0
- blr
+reg = 0
+ .rept 32
+ lfd reg, 0(r4)
+ b 2f
reg = reg + 1
.endr
1: mflr r5
mtctr r5
mtlr r0
bctr
-
-/* Load FP reg N from float at *p. N is in r3, p in r4. */
-_GLOBAL(do_lfs)
- PPC_STLU r1,-STKFRM(r1)
- mflr r0
- PPC_STL r0,STKFRM+PPC_LR_STKOFF(r1)
- mfmsr r6
- ori r7,r6,MSR_FP
- cmpwi cr7,r3,0
- MTMSRD(r7)
- isync
- beq cr7,1f
- stfd fr0,STKFRM-16(r1)
-1: li r9,-EFAULT
-2: lfs fr0,0(r4)
- li r9,0
-3: bl put_fpr
- beq cr7,4f
- lfd fr0,STKFRM-16(r1)
-4: PPC_LL r0,STKFRM+PPC_LR_STKOFF(r1)
- mtlr r0
- MTMSRD(r6)
- isync
- mr r3,r9
- addi r1,r1,STKFRM
- blr
- EX_TABLE(2b,3b)
-
-/* Load FP reg N from double at *p. N is in r3, p in r4. */
-_GLOBAL(do_lfd)
- PPC_STLU r1,-STKFRM(r1)
- mflr r0
- PPC_STL r0,STKFRM+PPC_LR_STKOFF(r1)
- mfmsr r6
- ori r7,r6,MSR_FP
- cmpwi cr7,r3,0
- MTMSRD(r7)
- isync
- beq cr7,1f
- stfd fr0,STKFRM-16(r1)
-1: li r9,-EFAULT
-2: lfd fr0,0(r4)
- li r9,0
-3: beq cr7,4f
- bl put_fpr
- lfd fr0,STKFRM-16(r1)
-4: PPC_LL r0,STKFRM+PPC_LR_STKOFF(r1)
- mtlr r0
- MTMSRD(r6)
- isync
- mr r3,r9
- addi r1,r1,STKFRM
- blr
- EX_TABLE(2b,3b)
-
-/* Store FP reg N to float at *p. N is in r3, p in r4. */
-_GLOBAL(do_stfs)
- PPC_STLU r1,-STKFRM(r1)
- mflr r0
- PPC_STL r0,STKFRM+PPC_LR_STKOFF(r1)
- mfmsr r6
- ori r7,r6,MSR_FP
- cmpwi cr7,r3,0
- MTMSRD(r7)
- isync
- beq cr7,1f
- stfd fr0,STKFRM-16(r1)
- bl get_fpr
-1: li r9,-EFAULT
-2: stfs fr0,0(r4)
- li r9,0
-3: beq cr7,4f
- lfd fr0,STKFRM-16(r1)
-4: PPC_LL r0,STKFRM+PPC_LR_STKOFF(r1)
- mtlr r0
- MTMSRD(r6)
+2: MTMSRD(r6)
isync
- mr r3,r9
- addi r1,r1,STKFRM
blr
- EX_TABLE(2b,3b)
-/* Store FP reg N to double at *p. N is in r3, p in r4. */
-_GLOBAL(do_stfd)
- PPC_STLU r1,-STKFRM(r1)
+#ifdef CONFIG_ALTIVEC
+/* Get the contents of vrN into *p; N is in r3 and p is in r4. */
+_GLOBAL(get_vr)
mflr r0
- PPC_STL r0,STKFRM+PPC_LR_STKOFF(r1)
mfmsr r6
- ori r7,r6,MSR_FP
- cmpwi cr7,r3,0
+ oris r7, r6, MSR_VEC@h
MTMSRD(r7)
isync
- beq cr7,1f
- stfd fr0,STKFRM-16(r1)
- bl get_fpr
-1: li r9,-EFAULT
-2: stfd fr0,0(r4)
- li r9,0
-3: beq cr7,4f
- lfd fr0,STKFRM-16(r1)
-4: PPC_LL r0,STKFRM+PPC_LR_STKOFF(r1)
- mtlr r0
- MTMSRD(r6)
- isync
- mr r3,r9
- addi r1,r1,STKFRM
- blr
- EX_TABLE(2b,3b)
-
-#ifdef CONFIG_ALTIVEC
-/* Get the contents of vrN into v0; N is in r3. Doesn't touch r3 or r4. */
-_GLOBAL(get_vr)
- mflr r0
rlwinm r6,r3,3,0xf8
bcl 20,31,1f
- blr /* v0 is already in v0 */
- nop
-reg = 1
- .rept 31
- vor v0,reg,reg /* assembler doesn't know vmr? */
- blr
+reg = 0
+ .rept 32
+ stvx reg, 0, r4
+ b 2f
reg = reg + 1
.endr
1: mflr r5
mtctr r5
mtlr r0
bctr
+2: MTMSRD(r6)
+ isync
+ blr
-/* Put the contents of v0 into vrN; N is in r3. Doesn't touch r3 or r4. */
+/* Put the contents of *p into vrN; N is in r3 and p is in r4. */
_GLOBAL(put_vr)
mflr r0
+ mfmsr r6
+ oris r7, r6, MSR_VEC@h
+ MTMSRD(r7)
+ isync
rlwinm r6,r3,3,0xf8
bcl 20,31,1f
- blr /* v0 is already in v0 */
- nop
-reg = 1
- .rept 31
- vor reg,v0,v0
- blr
+reg = 0
+ .rept 32
+ lvx reg, 0, r4
+ b 2f
reg = reg + 1
.endr
1: mflr r5
mtctr r5
mtlr r0
bctr
-
-/* Load vector reg N from *p. N is in r3, p in r4. */
-_GLOBAL(do_lvx)
- PPC_STLU r1,-STKFRM(r1)
- mflr r0
- PPC_STL r0,STKFRM+PPC_LR_STKOFF(r1)
- mfmsr r6
- oris r7,r6,MSR_VEC@h
- cmpwi cr7,r3,0
- li r8,STKFRM-16
- MTMSRD(r7)
- isync
- beq cr7,1f
- stvx v0,r1,r8
-1: li r9,-EFAULT
-2: lvx v0,0,r4
- li r9,0
-3: beq cr7,4f
- bl put_vr
- lvx v0,r1,r8
-4: PPC_LL r0,STKFRM+PPC_LR_STKOFF(r1)
- mtlr r0
- MTMSRD(r6)
- isync
- mr r3,r9
- addi r1,r1,STKFRM
- blr
- EX_TABLE(2b,3b)
-
-/* Store vector reg N to *p. N is in r3, p in r4. */
-_GLOBAL(do_stvx)
- PPC_STLU r1,-STKFRM(r1)
- mflr r0
- PPC_STL r0,STKFRM+PPC_LR_STKOFF(r1)
- mfmsr r6
- oris r7,r6,MSR_VEC@h
- cmpwi cr7,r3,0
- li r8,STKFRM-16
- MTMSRD(r7)
- isync
- beq cr7,1f
- stvx v0,r1,r8
- bl get_vr
-1: li r9,-EFAULT
-2: stvx v0,0,r4
- li r9,0
-3: beq cr7,4f
- lvx v0,r1,r8
-4: PPC_LL r0,STKFRM+PPC_LR_STKOFF(r1)
- mtlr r0
- MTMSRD(r6)
+2: MTMSRD(r6)
isync
- mr r3,r9
- addi r1,r1,STKFRM
blr
- EX_TABLE(2b,3b)
#endif /* CONFIG_ALTIVEC */
#ifdef CONFIG_VSX
mr r3,r9
addi r1,r1,STKFRM
blr
- EX_TABLE(2b,3b)
#endif /* CONFIG_VSX */
/* Convert single-precision to double, without disturbing FPRs. */
/*
* Functions in ldstfp.S
*/
-extern int do_lfs(int rn, unsigned long ea);
-extern int do_lfd(int rn, unsigned long ea);
-extern int do_stfs(int rn, unsigned long ea);
-extern int do_stfd(int rn, unsigned long ea);
-extern int do_lvx(int rn, unsigned long ea);
-extern int do_stvx(int rn, unsigned long ea);
+extern void get_fpr(int rn, double *p);
+extern void put_fpr(int rn, const double *p);
+extern void get_vr(int rn, __vector128 *p);
+extern void put_vr(int rn, __vector128 *p);
extern void load_vsrn(int vsr, const void *p);
extern void store_vsrn(int vsr, void *p);
extern void conv_sp_to_dp(const float *sp, double *dp);
#ifdef CONFIG_PPC_FPU
/*
- * Check the address and alignment, and call func to do the actual
- * load or store.
+ * These access either the real FP register or the image in the
+ * thread_struct, depending on regs->msr & MSR_FP.
*/
-static int do_fp_load(int rn, int (*func)(int, unsigned long),
- unsigned long ea, int nb,
- struct pt_regs *regs)
+static int do_fp_load(int rn, unsigned long ea, int nb, struct pt_regs *regs)
{
int err;
- u8 buf[sizeof(double)] __attribute__((aligned(sizeof(double))));
+ union {
+ float f;
+ double d;
+ unsigned long l;
+ u8 b[sizeof(double)];
+ } u;
if (!address_ok(regs, ea, nb))
return -EFAULT;
- if (ea & 3) {
- err = copy_mem_in(buf, ea, nb);
- if (err)
- return err;
- ea = (unsigned long) buf;
- }
- return (*func)(rn, ea);
+ err = copy_mem_in(u.b, ea, nb);
+ if (err)
+ return err;
+ preempt_disable();
+ if (nb == 4)
+ conv_sp_to_dp(&u.f, &u.d);
+ if (regs->msr & MSR_FP)
+ put_fpr(rn, &u.d);
+ else
+ current->thread.TS_FPR(rn) = u.l;
+ preempt_enable();
+ return 0;
}
NOKPROBE_SYMBOL(do_fp_load);
-static int do_fp_store(int rn, int (*func)(int, unsigned long),
- unsigned long ea, int nb,
- struct pt_regs *regs)
+static int do_fp_store(int rn, unsigned long ea, int nb, struct pt_regs *regs)
{
- int err;
- u8 buf[sizeof(double)] __attribute__((aligned(sizeof(double))));
+ union {
+ float f;
+ double d;
+ unsigned long l;
+ u8 b[sizeof(double)];
+ } u;
if (!address_ok(regs, ea, nb))
return -EFAULT;
- if ((ea & 3) == 0)
- return (*func)(rn, ea);
- err = (*func)(rn, (unsigned long) buf);
- if (!err)
- err = copy_mem_out(buf, ea, nb);
- return err;
+ preempt_disable();
+ if (regs->msr & MSR_FP)
+ get_fpr(rn, &u.d);
+ else
+ u.l = current->thread.TS_FPR(rn);
+ if (nb == 4)
+ conv_dp_to_sp(&u.d, &u.f);
+ preempt_enable();
+ return copy_mem_out(u.b, ea, nb);
}
NOKPROBE_SYMBOL(do_fp_store);
#endif
#ifdef CONFIG_ALTIVEC
/* For Altivec/VMX, no need to worry about alignment */
-static nokprobe_inline int do_vec_load(int rn, int (*func)(int, unsigned long),
- unsigned long ea, struct pt_regs *regs)
+static nokprobe_inline int do_vec_load(int rn, unsigned long ea,
+ int size, struct pt_regs *regs)
{
+ int err;
+ union {
+ __vector128 v;
+ u8 b[sizeof(__vector128)];
+ } u = {};
+
if (!address_ok(regs, ea & ~0xfUL, 16))
return -EFAULT;
- return (*func)(rn, ea);
+ /* align to multiple of size */
+ ea &= ~(size - 1);
+ err = copy_mem_in(u.b, ea, size);
+ if (err)
+ return err;
+
+ preempt_disable();
+ if (regs->msr & MSR_VEC)
+ put_vr(rn, &u.v);
+ else
+ current->thread.vr_state.vr[rn] = u.v;
+ preempt_enable();
+ return 0;
}
-static nokprobe_inline int do_vec_store(int rn, int (*func)(int, unsigned long),
- unsigned long ea, struct pt_regs *regs)
+static nokprobe_inline int do_vec_store(int rn, unsigned long ea,
+ int size, struct pt_regs *regs)
{
+ union {
+ __vector128 v;
+ u8 b[sizeof(__vector128)];
+ } u;
+
if (!address_ok(regs, ea & ~0xfUL, 16))
return -EFAULT;
- return (*func)(rn, ea);
+ /* align to multiple of size */
+ ea &= ~(size - 1);
+
+ preempt_disable();
+ if (regs->msr & MSR_VEC)
+ get_vr(rn, &u.v);
+ else
+ u.v = current->thread.vr_state.vr[rn];
+ preempt_enable();
+ return copy_mem_out(u.b, ea, size);
}
#endif /* CONFIG_ALTIVEC */
}
EXPORT_SYMBOL_GPL(emulate_vsx_store);
NOKPROBE_SYMBOL(emulate_vsx_store);
+
+static nokprobe_inline int do_vsx_load(struct instruction_op *op,
+ unsigned long ea, struct pt_regs *regs)
+{
+ int reg = op->reg;
+ u8 mem[16];
+ union vsx_reg buf;
+ int size = GETSIZE(op->type);
+
+ if (!address_ok(regs, ea, size) || copy_mem_in(mem, ea, size))
+ return -EFAULT;
+
+ emulate_vsx_load(op, &buf, mem);
+ preempt_disable();
+ if (reg < 32) {
+ /* FP regs + extensions */
+ if (regs->msr & MSR_FP) {
+ load_vsrn(reg, &buf);
+ } else {
+ current->thread.fp_state.fpr[reg][0] = buf.d[0];
+ current->thread.fp_state.fpr[reg][1] = buf.d[1];
+ }
+ } else {
+ if (regs->msr & MSR_VEC)
+ load_vsrn(reg, &buf);
+ else
+ current->thread.vr_state.vr[reg - 32] = buf.v;
+ }
+ preempt_enable();
+ return 0;
+}
+
+static nokprobe_inline int do_vsx_store(struct instruction_op *op,
+ unsigned long ea, struct pt_regs *regs)
+{
+ int reg = op->reg;
+ u8 mem[16];
+ union vsx_reg buf;
+ int size = GETSIZE(op->type);
+
+ if (!address_ok(regs, ea, size))
+ return -EFAULT;
+
+ preempt_disable();
+ if (reg < 32) {
+ /* FP regs + extensions */
+ if (regs->msr & MSR_FP) {
+ store_vsrn(reg, &buf);
+ } else {
+ buf.d[0] = current->thread.fp_state.fpr[reg][0];
+ buf.d[1] = current->thread.fp_state.fpr[reg][1];
+ }
+ } else {
+ if (regs->msr & MSR_VEC)
+ store_vsrn(reg, &buf);
+ else
+ buf.v = current->thread.vr_state.vr[reg - 32];
+ }
+ preempt_enable();
+ emulate_vsx_store(op, &buf, mem);
+ return copy_mem_out(mem, ea, size);
+}
#endif /* CONFIG_VSX */
#define __put_user_asmx(x, addr, err, op, cr) \
#ifdef CONFIG_PPC_FPU
case LOAD_FP:
- if (!(regs->msr & MSR_FP))
+ /*
+ * If the instruction is in userspace, we can emulate it even
+ * if the VMX state is not live, because we have the state
+ * stored in the thread_struct. If the instruction is in
+ * the kernel, we must not touch the state in the thread_struct.
+ */
+ if (!(regs->msr & MSR_PR) && !(regs->msr & MSR_FP))
return 0;
- if (size == 4)
- err = do_fp_load(op.reg, do_lfs, ea, size, regs);
- else
- err = do_fp_load(op.reg, do_lfd, ea, size, regs);
+ err = do_fp_load(op.reg, ea, size, regs);
goto ldst_done;
#endif
#ifdef CONFIG_ALTIVEC
case LOAD_VMX:
- if (!(regs->msr & MSR_VEC))
+ if (!(regs->msr & MSR_PR) && !(regs->msr & MSR_VEC))
return 0;
- err = do_vec_load(op.reg, do_lvx, ea, regs);
+ err = do_vec_load(op.reg, ea, size, regs);
goto ldst_done;
#endif
#ifdef CONFIG_VSX
case LOAD_VSX: {
- u8 mem[16];
- union vsx_reg buf;
unsigned long msrbit = MSR_VSX;
/*
*/
if (op.reg >= 32 && (op.vsx_flags & VSX_CHECK_VEC))
msrbit = MSR_VEC;
- if (!(regs->msr & msrbit))
+ if (!(regs->msr & MSR_PR) && !(regs->msr & msrbit))
return 0;
- if (!address_ok(regs, ea, size) ||
- copy_mem_in(mem, ea, size))
- return 0;
-
- emulate_vsx_load(&op, &buf, mem);
- load_vsrn(op.reg, &buf);
+ err = do_vsx_load(&op, ea, regs);
goto ldst_done;
}
#endif
#ifdef CONFIG_PPC_FPU
case STORE_FP:
- if (!(regs->msr & MSR_FP))
+ if (!(regs->msr & MSR_PR) && !(regs->msr & MSR_FP))
return 0;
- if (size == 4)
- err = do_fp_store(op.reg, do_stfs, ea, size, regs);
- else
- err = do_fp_store(op.reg, do_stfd, ea, size, regs);
+ err = do_fp_store(op.reg, ea, size, regs);
goto ldst_done;
#endif
#ifdef CONFIG_ALTIVEC
case STORE_VMX:
- if (!(regs->msr & MSR_VEC))
+ if (!(regs->msr & MSR_PR) && !(regs->msr & MSR_VEC))
return 0;
- err = do_vec_store(op.reg, do_stvx, ea, regs);
+ err = do_vec_store(op.reg, ea, size, regs);
goto ldst_done;
#endif
#ifdef CONFIG_VSX
case STORE_VSX: {
- u8 mem[16];
- union vsx_reg buf;
unsigned long msrbit = MSR_VSX;
/*
*/
if (op.reg >= 32 && (op.vsx_flags & VSX_CHECK_VEC))
msrbit = MSR_VEC;
- if (!(regs->msr & msrbit))
- return 0;
- if (!address_ok(regs, ea, size))
- return 0;
-
- store_vsrn(op.reg, &buf);
- emulate_vsx_store(&op, &buf, mem);
- if (copy_mem_out(mem, ea, size))
+ if (!(regs->msr & MSR_PR) && !(regs->msr & msrbit))
return 0;
+ err = do_vsx_store(&op, ea, regs);
goto ldst_done;
}
#endif
projects / GitHub / LineageOS / android_kernel_motorola_exynos9610.git / commitdiff