locking/atomic, arch/arm: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor...

Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.

This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Will Deacon <will.deacon@arm.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Davidlohr Bueso <dbueso@suse.de>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h
index 9e10c4567eb4dd3c898486f5fc34777c3760adc9..0feb110ec542950ac6285bf41e9155b575633d82 100644 (file)
--- a/arch/arm/include/asm/atomic.h
+++ b/arch/arm/include/asm/atomic.h
@@ -77,8 +77,36 @@ static inline int atomic_##op##_return_relaxed(int i, atomic_t *v)   \
        return result;                                                  \
 }
 
+#define ATOMIC_FETCH_OP(op, c_op, asm_op)                              \
+static inline int atomic_fetch_##op##_relaxed(int i, atomic_t *v)      \
+{                                                                      \
+       unsigned long tmp;                                              \
+       int result, val;                                                \
+                                                                       \
+       prefetchw(&v->counter);                                         \
+                                                                       \
+       __asm__ __volatile__("@ atomic_fetch_" #op "\n"                 \
+"1:    ldrex   %0, [%4]\n"                                             \
+"      " #asm_op "     %1, %0, %5\n"                                   \
+"      strex   %2, %1, [%4]\n"                                         \
+"      teq     %2, #0\n"                                               \
+"      bne     1b"                                                     \
+       : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Qo" (v->counter)  \
+       : "r" (&v->counter), "Ir" (i)                                   \
+       : "cc");                                                        \
+                                                                       \
+       return result;                                                  \
+}
+
 #define atomic_add_return_relaxed      atomic_add_return_relaxed
 #define atomic_sub_return_relaxed      atomic_sub_return_relaxed
+#define atomic_fetch_add_relaxed       atomic_fetch_add_relaxed
+#define atomic_fetch_sub_relaxed       atomic_fetch_sub_relaxed
+
+#define atomic_fetch_and_relaxed       atomic_fetch_and_relaxed
+#define atomic_fetch_andnot_relaxed    atomic_fetch_andnot_relaxed
+#define atomic_fetch_or_relaxed                atomic_fetch_or_relaxed
+#define atomic_fetch_xor_relaxed       atomic_fetch_xor_relaxed
 
 static inline int atomic_cmpxchg_relaxed(atomic_t *ptr, int old, int new)
 {
@@ -159,6 +187,22 @@ static inline int atomic_##op##_return(int i, atomic_t *v)         \
        return val;                                                     \
 }
 
+#define ATOMIC_FETCH_OP(op, c_op, asm_op)                              \
+static inline int atomic_fetch_##op(int i, atomic_t *v)                        \
+{                                                                      \
+       unsigned long flags;                                            \
+       int val;                                                        \
+                                                                       \
+       raw_local_irq_save(flags);                                      \
+       val = v->counter;                                               \
+       v->counter c_op i;                                              \
+       raw_local_irq_restore(flags);                                   \
+                                                                       \
+       return val;                                                     \
+}
+
+#define atomic_fetch_or atomic_fetch_or
+
 static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
 {
        int ret;
@@ -187,19 +231,26 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u)
 
 #define ATOMIC_OPS(op, c_op, asm_op)                                   \
        ATOMIC_OP(op, c_op, asm_op)                                     \
-       ATOMIC_OP_RETURN(op, c_op, asm_op)
+       ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
+       ATOMIC_FETCH_OP(op, c_op, asm_op)
 
 ATOMIC_OPS(add, +=, add)
 ATOMIC_OPS(sub, -=, sub)
 
 #define atomic_andnot atomic_andnot
 
-ATOMIC_OP(and, &=, and)
-ATOMIC_OP(andnot, &= ~, bic)
-ATOMIC_OP(or,  |=, orr)
-ATOMIC_OP(xor, ^=, eor)
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op, c_op, asm_op)                                   \
+       ATOMIC_OP(op, c_op, asm_op)                                     \
+       ATOMIC_FETCH_OP(op, c_op, asm_op)
+
+ATOMIC_OPS(and, &=, and)
+ATOMIC_OPS(andnot, &= ~, bic)
+ATOMIC_OPS(or,  |=, orr)
+ATOMIC_OPS(xor, ^=, eor)
 
 #undef ATOMIC_OPS
+#undef ATOMIC_FETCH_OP
 #undef ATOMIC_OP_RETURN
 #undef ATOMIC_OP
 
@@ -317,24 +368,61 @@ atomic64_##op##_return_relaxed(long long i, atomic64_t *v)                \
        return result;                                                  \
 }
 
+#define ATOMIC64_FETCH_OP(op, op1, op2)                                        \
+static inline long long                                                        \
+atomic64_fetch_##op##_relaxed(long long i, atomic64_t *v)              \
+{                                                                      \
+       long long result, val;                                          \
+       unsigned long tmp;                                              \
+                                                                       \
+       prefetchw(&v->counter);                                         \
+                                                                       \
+       __asm__ __volatile__("@ atomic64_fetch_" #op "\n"               \
+"1:    ldrexd  %0, %H0, [%4]\n"                                        \
+"      " #op1 " %Q1, %Q0, %Q5\n"                                       \
+"      " #op2 " %R1, %R0, %R5\n"                                       \
+"      strexd  %2, %1, %H1, [%4]\n"                                    \
+"      teq     %2, #0\n"                                               \
+"      bne     1b"                                                     \
+       : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Qo" (v->counter)  \
+       : "r" (&v->counter), "r" (i)                                    \
+       : "cc");                                                        \
+                                                                       \
+       return result;                                                  \
+}
+
 #define ATOMIC64_OPS(op, op1, op2)                                     \
        ATOMIC64_OP(op, op1, op2)                                       \
-       ATOMIC64_OP_RETURN(op, op1, op2)
+       ATOMIC64_OP_RETURN(op, op1, op2)                                \
+       ATOMIC64_FETCH_OP(op, op1, op2)
 
 ATOMIC64_OPS(add, adds, adc)
 ATOMIC64_OPS(sub, subs, sbc)
 
 #define atomic64_add_return_relaxed    atomic64_add_return_relaxed
 #define atomic64_sub_return_relaxed    atomic64_sub_return_relaxed
+#define atomic64_fetch_add_relaxed     atomic64_fetch_add_relaxed
+#define atomic64_fetch_sub_relaxed     atomic64_fetch_sub_relaxed
+
+#undef ATOMIC64_OPS
+#define ATOMIC64_OPS(op, op1, op2)                                     \
+       ATOMIC64_OP(op, op1, op2)                                       \
+       ATOMIC64_FETCH_OP(op, op1, op2)
 
 #define atomic64_andnot atomic64_andnot
 
-ATOMIC64_OP(and, and, and)
-ATOMIC64_OP(andnot, bic, bic)
-ATOMIC64_OP(or,  orr, orr)
-ATOMIC64_OP(xor, eor, eor)
+ATOMIC64_OPS(and, and, and)
+ATOMIC64_OPS(andnot, bic, bic)
+ATOMIC64_OPS(or,  orr, orr)
+ATOMIC64_OPS(xor, eor, eor)
+
+#define atomic64_fetch_and_relaxed     atomic64_fetch_and_relaxed
+#define atomic64_fetch_andnot_relaxed  atomic64_fetch_andnot_relaxed
+#define atomic64_fetch_or_relaxed      atomic64_fetch_or_relaxed
+#define atomic64_fetch_xor_relaxed     atomic64_fetch_xor_relaxed
 
 #undef ATOMIC64_OPS
+#undef ATOMIC64_FETCH_OP
 #undef ATOMIC64_OP_RETURN
 #undef ATOMIC64_OP