[GitHub/MotorolaMobilityLLC/kernel-slsi.git] / arch / arm / include / asm / futex.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_ARM_FUTEX_H
#define _ASM_ARM_FUTEX_H

#ifdef __KERNEL__

#include <linux/futex.h>
#include <linux/uaccess.h>
#include <asm/errno.h>

#define __futex_atomic_ex_table(err_reg)			\
	"3:\n"							\
	"	.pushsection __ex_table,\"a\"\n"		\
	"	.align	3\n"					\
	"	.long	1b, 4f, 2b, 4f\n"			\
	"	.popsection\n"					\
	"	.pushsection .text.fixup,\"ax\"\n"		\
	"	.align	2\n"					\
	"4:	mov	%0, " err_reg "\n"			\
	"	b	3b\n"					\
	"	.popsection"

#ifdef CONFIG_SMP

#define __futex_atomic_op(insn, ret, oldval, tmp, uaddr, oparg)	\
({								\
	unsigned int __ua_flags;				\
	smp_mb();						\
	prefetchw(uaddr);					\
	__ua_flags = uaccess_save_and_enable();			\
	__asm__ __volatile__(					\
	"1:	ldrex	%1, [%3]\n"				\
	"	" insn "\n"					\
	"2:	strex	%2, %0, [%3]\n"				\
	"	teq	%2, #0\n"				\
	"	bne	1b\n"					\
	"	mov	%0, #0\n"				\
	__futex_atomic_ex_table("%5")				\
	: "=&r" (ret), "=&r" (oldval), "=&r" (tmp)		\
	: "r" (uaddr), "r" (oparg), "Ir" (-EFAULT)		\
	: "cc", "memory");					\
	uaccess_restore(__ua_flags);				\
})

static inline int
futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
			      u32 oldval, u32 newval)
{
	unsigned int __ua_flags;
	int ret;
	u32 val;

	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
		return -EFAULT;

	smp_mb();
	/* Prefetching cannot fault */
	prefetchw(uaddr);
	__ua_flags = uaccess_save_and_enable();
	__asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n"
	"1:	ldrex	%1, [%4]\n"
	"	teq	%1, %2\n"
	"	ite	eq	@ explicit IT needed for the 2b label\n"
	"2:	strexeq	%0, %3, [%4]\n"
	"	movne	%0, #0\n"
	"	teq	%0, #0\n"
	"	bne	1b\n"
	__futex_atomic_ex_table("%5")
	: "=&r" (ret), "=&r" (val)
	: "r" (oldval), "r" (newval), "r" (uaddr), "Ir" (-EFAULT)
	: "cc", "memory");
	uaccess_restore(__ua_flags);
	smp_mb();

	*uval = val;
	return ret;
}

#else /* !SMP, we can work around lack of atomic ops by disabling preemption */

#include <linux/preempt.h>
#include <asm/domain.h>

#define __futex_atomic_op(insn, ret, oldval, tmp, uaddr, oparg)	\
({								\
	unsigned int __ua_flags = uaccess_save_and_enable();	\
	__asm__ __volatile__(					\
	"1:	" TUSER(ldr) "	%1, [%3]\n"			\
	"	" insn "\n"					\
	"2:	" TUSER(str) "	%0, [%3]\n"			\
	"	mov	%0, #0\n"				\
	__futex_atomic_ex_table("%5")				\
	: "=&r" (ret), "=&r" (oldval), "=&r" (tmp)		\
	: "r" (uaddr), "r" (oparg), "Ir" (-EFAULT)		\
	: "cc", "memory");					\
	uaccess_restore(__ua_flags);				\
})

static inline int
futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
			      u32 oldval, u32 newval)
{
	unsigned int __ua_flags;
	int ret = 0;
	u32 val;

	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
		return -EFAULT;

	preempt_disable();
	__ua_flags = uaccess_save_and_enable();
	__asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n"
	"1:	" TUSER(ldr) "	%1, [%4]\n"
	"	teq	%1, %2\n"
	"	it	eq	@ explicit IT needed for the 2b label\n"
	"2:	" TUSER(streq) "	%3, [%4]\n"
	__futex_atomic_ex_table("%5")
	: "+r" (ret), "=&r" (val)
	: "r" (oldval), "r" (newval), "r" (uaddr), "Ir" (-EFAULT)
	: "cc", "memory");
	uaccess_restore(__ua_flags);

	*uval = val;
	preempt_enable();

	return ret;
}

#endif /* !SMP */

static inline int
arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
{
	int oldval = 0, ret, tmp;

#ifndef CONFIG_SMP
	preempt_disable();
#endif
	pagefault_disable();

	switch (op) {
	case FUTEX_OP_SET:
		__futex_atomic_op("mov	%0, %4", ret, oldval, tmp, uaddr, oparg);
		break;
	case FUTEX_OP_ADD:
		__futex_atomic_op("add	%0, %1, %4", ret, oldval, tmp, uaddr, oparg);
		break;
	case FUTEX_OP_OR:
		__futex_atomic_op("orr	%0, %1, %4", ret, oldval, tmp, uaddr, oparg);
		break;
	case FUTEX_OP_ANDN:
		__futex_atomic_op("and	%0, %1, %4", ret, oldval, tmp, uaddr, ~oparg);
		break;
	case FUTEX_OP_XOR:
		__futex_atomic_op("eor	%0, %1, %4", ret, oldval, tmp, uaddr, oparg);
		break;
	default:
		ret = -ENOSYS;
	}

	pagefault_enable();
#ifndef CONFIG_SMP
	preempt_enable();
#endif

	if (!ret)
		*oval = oldval;

	return ret;
}

#endif /* __KERNEL__ */
#endif /* _ASM_ARM_FUTEX_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
e589ed23 MP	2	#ifndef _ASM_ARM_FUTEX_H
	3	#define _ASM_ARM_FUTEX_H
	4
	5	#ifdef __KERNEL__
	6
c1b0db56 WD	7	#include <linux/futex.h>
	8	#include <linux/uaccess.h>
	9	#include <asm/errno.h>
	10
	11	#define __futex_atomic_ex_table(err_reg) \
	12	"3:\n" \
	13	" .pushsection __ex_table,\"a\"\n" \
	14	" .align 3\n" \
	15	" .long 1b, 4f, 2b, 4f\n" \
	16	" .popsection\n" \
c4a84ae3	17	" .pushsection .text.fixup,\"ax\"\n" \
667d1b48	18	" .align 2\n" \
c1b0db56 WD	19	"4: mov %0, " err_reg "\n" \
	20	" b 3b\n" \
	21	" .popsection"
	22
e589ed23	23	#ifdef CONFIG_SMP
4732efbe	24
df77abca	25	#define __futex_atomic_op(insn, ret, oldval, tmp, uaddr, oparg) \
3fba7e23 RK	26	({ \
3fba7e23 RK	27	unsigned int __ua_flags; \
c1b0db56	28	smp_mb(); \
c32ffce0	29	prefetchw(uaddr); \
3fba7e23	30	__ua_flags = uaccess_save_and_enable(); \
c1b0db56	31	__asm__ __volatile__( \
df77abca	32	"1: ldrex %1, [%3]\n" \
c1b0db56	33	" " insn "\n" \
df77abca WD	34	"2: strex %2, %0, [%3]\n" \
df77abca WD	35	" teq %2, #0\n" \
c1b0db56 WD	36	" bne 1b\n" \
c1b0db56 WD	37	" mov %0, #0\n" \
df77abca WD	38	__futex_atomic_ex_table("%5") \
df77abca WD	39	: "=&r" (ret), "=&r" (oldval), "=&r" (tmp) \
c1b0db56	40	: "r" (uaddr), "r" (oparg), "Ir" (-EFAULT) \
3fba7e23 RK	41	: "cc", "memory"); \
	42	uaccess_restore(__ua_flags); \
	43	})
c1b0db56 WD	44
	45	static inline int
	46	futex_atomic_cmpxchg_inatomic(u32 uval, u32 __user uaddr,
	47	u32 oldval, u32 newval)
	48	{
3fba7e23	49	unsigned int __ua_flags;
c1b0db56 WD	50	int ret;
	51	u32 val;
	52
	53	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
	54	return -EFAULT;
	55
	56	smp_mb();
c32ffce0 WD	57	/* Prefetching cannot fault */
c32ffce0 WD	58	prefetchw(uaddr);
3fba7e23	59	__ua_flags = uaccess_save_and_enable();
c1b0db56 WD	60	__asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n"
	61	"1: ldrex %1, [%4]\n"
	62	" teq %1, %2\n"
	63	" ite eq @ explicit IT needed for the 2b label\n"
	64	"2: strexeq %0, %3, [%4]\n"
	65	" movne %0, #0\n"
	66	" teq %0, #0\n"
	67	" bne 1b\n"
	68	__futex_atomic_ex_table("%5")
	69	: "=&r" (ret), "=&r" (val)
	70	: "r" (oldval), "r" (newval), "r" (uaddr), "Ir" (-EFAULT)
	71	: "cc", "memory");
3fba7e23	72	uaccess_restore(__ua_flags);
c1b0db56 WD	73	smp_mb();
	74
	75	*uval = val;
	76	return ret;
	77	}
4732efbe	78
e589ed23 MP	79	#else /* !SMP, we can work around lack of atomic ops by disabling preemption */
e589ed23 MP	80
e589ed23	81	#include <linux/preempt.h>
247055aa	82	#include <asm/domain.h>
e589ed23	83
df77abca	84	#define __futex_atomic_op(insn, ret, oldval, tmp, uaddr, oparg) \
3fba7e23 RK	85	({ \
3fba7e23 RK	86	unsigned int __ua_flags = uaccess_save_and_enable(); \
e589ed23	87	__asm__ __volatile__( \
4e7682d0	88	"1: " TUSER(ldr) " %1, [%3]\n" \
e589ed23	89	" " insn "\n" \
4e7682d0	90	"2: " TUSER(str) " %0, [%3]\n" \
e589ed23	91	" mov %0, #0\n" \
df77abca WD	92	__futex_atomic_ex_table("%5") \
df77abca WD	93	: "=&r" (ret), "=&r" (oldval), "=&r" (tmp) \
e589ed23	94	: "r" (uaddr), "r" (oparg), "Ir" (-EFAULT) \
3fba7e23 RK	95	: "cc", "memory"); \
	96	uaccess_restore(__ua_flags); \
	97	})
e589ed23	98
c1b0db56 WD	99	static inline int
	100	futex_atomic_cmpxchg_inatomic(u32 uval, u32 __user uaddr,
	101	u32 oldval, u32 newval)
	102	{
3fba7e23	103	unsigned int __ua_flags;
c1b0db56 WD	104	int ret = 0;
	105	u32 val;
	106
	107	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
	108	return -EFAULT;
	109
39919b01	110	preempt_disable();
3fba7e23	111	__ua_flags = uaccess_save_and_enable();
c1b0db56	112	__asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n"
4e7682d0	113	"1: " TUSER(ldr) " %1, [%4]\n"
c1b0db56 WD	114	" teq %1, %2\n"
c1b0db56 WD	115	" it eq @ explicit IT needed for the 2b label\n"
4e7682d0	116	"2: " TUSER(streq) " %3, [%4]\n"
c1b0db56 WD	117	__futex_atomic_ex_table("%5")
	118	: "+r" (ret), "=&r" (val)
	119	: "r" (oldval), "r" (newval), "r" (uaddr), "Ir" (-EFAULT)
	120	: "cc", "memory");
3fba7e23	121	uaccess_restore(__ua_flags);
c1b0db56 WD	122
c1b0db56 WD	123	*uval = val;
39919b01 DH	124	preempt_enable();
39919b01 DH	125
c1b0db56 WD	126	return ret;
	127	}
	128
	129	#endif /* !SMP */
	130
e589ed23	131	static inline int
30d6e0a4	132	arch_futex_atomic_op_inuser(int op, int oparg, int oval, u32 __user uaddr)
e589ed23	133	{
df77abca	134	int oldval = 0, ret, tmp;
e589ed23	135
388b0e0a DH	136	#ifndef CONFIG_SMP
	137	preempt_disable();
	138	#endif
	139	pagefault_disable();
e589ed23 MP	140
	141	switch (op) {
	142	case FUTEX_OP_SET:
df77abca	143	__futex_atomic_op("mov %0, %4", ret, oldval, tmp, uaddr, oparg);
e589ed23 MP	144	break;
e589ed23 MP	145	case FUTEX_OP_ADD:
df77abca	146	__futex_atomic_op("add %0, %1, %4", ret, oldval, tmp, uaddr, oparg);
e589ed23 MP	147	break;
e589ed23 MP	148	case FUTEX_OP_OR:
df77abca	149	__futex_atomic_op("orr %0, %1, %4", ret, oldval, tmp, uaddr, oparg);
e589ed23 MP	150	break;
e589ed23 MP	151	case FUTEX_OP_ANDN:
df77abca	152	__futex_atomic_op("and %0, %1, %4", ret, oldval, tmp, uaddr, ~oparg);
e589ed23 MP	153	break;
e589ed23 MP	154	case FUTEX_OP_XOR:
df77abca	155	__futex_atomic_op("eor %0, %1, %4", ret, oldval, tmp, uaddr, oparg);
e589ed23 MP	156	break;
	157	default:
	158	ret = -ENOSYS;
	159	}
	160
388b0e0a DH	161	pagefault_enable();
	162	#ifndef CONFIG_SMP
	163	preempt_enable();
	164	#endif
e589ed23	165
30d6e0a4 JS	166	if (!ret)
	167	*oval = oldval;
	168
e589ed23 MP	169	return ret;
	170	}
	171
e589ed23 MP	172	#endif /* __KERNEL__ */
e589ed23 MP	173	#endif /* _ASM_ARM_FUTEX_H */