arm64: futex: Bound number of LDXR/STXR loops in FUTEX_WAKE_OP

commit 03110a5cb2161690ae5ac04994d47ed0cd6cef75 upstream.

Our futex implementation makes use of LDXR/STXR loops to perform atomic
updates to user memory from atomic context. This can lead to latency
problems if we end up spinning around the LL/SC sequence at the expense
of doing something useful.

Rework our futex atomic operations so that we return -EAGAIN if we fail
to update the futex word after 128 attempts. The core futex code will
reschedule if necessary and we'll try again later.

Fixes: 6170a97460db ("arm64: Atomic operations")
Signed-off-by: Will Deacon <will.deacon@arm.com>
[bwh: Backported to 4.9: adjust context]
Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h
index 86a43450f014c1aa0337dd4606af12a7288a0294..bdf5ec2b8356527f26179a3b298d2976d3e7a4d1 100644 (file)
--- a/arch/arm64/include/asm/futex.h
+++ b/arch/arm64/include/asm/futex.h
@@ -26,7 +26,12 @@
 #include <asm/errno.h>
 #include <asm/sysreg.h>
 
+#define FUTEX_MAX_LOOPS        128 /* What's the largest number you can think of? */
+
 #define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)                \
+do {                                                                   \
+       unsigned int loops = FUTEX_MAX_LOOPS;                           \
+                                                                       \
        asm volatile(                                                   \
        ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN,            \
                    CONFIG_ARM64_PAN)                                   \
@@ -34,21 +39,26 @@
 "1:    ldxr    %w1, %2\n"                                              \
        insn "\n"                                                       \
 "2:    stlxr   %w0, %w3, %2\n"                                         \
-"      cbnz    %w0, 1b\n"                                              \
-"      dmb     ish\n"                                                  \
+"      cbz     %w0, 3f\n"                                              \
+"      sub     %w4, %w4, %w0\n"                                        \
+"      cbnz    %w4, 1b\n"                                              \
+"      mov     %w0, %w7\n"                                             \
 "3:\n"                                                                 \
+"      dmb     ish\n"                                                  \
 "      .pushsection .fixup,\"ax\"\n"                                   \
 "      .align  2\n"                                                    \
-"4:    mov     %w0, %w5\n"                                             \
+"4:    mov     %w0, %w6\n"                                             \
 "      b       3b\n"                                                   \
 "      .popsection\n"                                                  \
        _ASM_EXTABLE(1b, 4b)                                            \
        _ASM_EXTABLE(2b, 4b)                                            \
        ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,            \
                    CONFIG_ARM64_PAN)                                   \
-       : "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp)       \
-       : "r" (oparg), "Ir" (-EFAULT)                                   \
-       : "memory")
+       : "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp),      \
+         "+r" (loops)                                                  \
+       : "r" (oparg), "Ir" (-EFAULT), "Ir" (-EAGAIN)                   \
+       : "memory");                                                    \
+} while (0)
 
 static inline int
 arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
@@ -59,23 +69,23 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
 
        switch (op) {
        case FUTEX_OP_SET:
-               __futex_atomic_op("mov  %w3, %w4",
+               __futex_atomic_op("mov  %w3, %w5",
                                  ret, oldval, uaddr, tmp, oparg);
                break;
        case FUTEX_OP_ADD:
-               __futex_atomic_op("add  %w3, %w1, %w4",
+               __futex_atomic_op("add  %w3, %w1, %w5",
                                  ret, oldval, uaddr, tmp, oparg);
                break;
        case FUTEX_OP_OR:
-               __futex_atomic_op("orr  %w3, %w1, %w4",
+               __futex_atomic_op("orr  %w3, %w1, %w5",
                                  ret, oldval, uaddr, tmp, oparg);
                break;
        case FUTEX_OP_ANDN:
-               __futex_atomic_op("and  %w3, %w1, %w4",
+               __futex_atomic_op("and  %w3, %w1, %w5",
                                  ret, oldval, uaddr, tmp, ~oparg);
                break;
        case FUTEX_OP_XOR:
-               __futex_atomic_op("eor  %w3, %w1, %w4",
+               __futex_atomic_op("eor  %w3, %w1, %w5",
                                  ret, oldval, uaddr, tmp, oparg);
                break;
        default:
@@ -95,6 +105,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr,
                              u32 oldval, u32 newval)
 {
        int ret = 0;
+       unsigned int loops = FUTEX_MAX_LOOPS;
        u32 val, tmp;
        u32 __user *uaddr;
 
@@ -106,21 +117,25 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr,
 ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, CONFIG_ARM64_PAN)
 "      prfm    pstl1strm, %2\n"
 "1:    ldxr    %w1, %2\n"
-"      sub     %w3, %w1, %w4\n"
-"      cbnz    %w3, 3f\n"
-"2:    stlxr   %w3, %w5, %2\n"
-"      cbnz    %w3, 1b\n"
-"      dmb     ish\n"
+"      sub     %w3, %w1, %w5\n"
+"      cbnz    %w3, 4f\n"
+"2:    stlxr   %w3, %w6, %2\n"
+"      cbz     %w3, 3f\n"
+"      sub     %w4, %w4, %w3\n"
+"      cbnz    %w4, 1b\n"
+"      mov     %w0, %w8\n"
 "3:\n"
+"      dmb     ish\n"
+"4:\n"
 "      .pushsection .fixup,\"ax\"\n"
-"4:    mov     %w0, %w6\n"
-"      b       3b\n"
+"5:    mov     %w0, %w7\n"
+"      b       4b\n"
 "      .popsection\n"
-       _ASM_EXTABLE(1b, 4b)
-       _ASM_EXTABLE(2b, 4b)
+       _ASM_EXTABLE(1b, 5b)
+       _ASM_EXTABLE(2b, 5b)
 ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, CONFIG_ARM64_PAN)
-       : "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp)
-       : "r" (oldval), "r" (newval), "Ir" (-EFAULT)
+       : "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)
+       : "r" (oldval), "r" (newval), "Ir" (-EFAULT), "Ir" (-EAGAIN)
        : "memory");
 
        *uval = val;