arch/s390/include/asm/atomic.h

   1 #ifndef __ARCH_S390_ATOMIC__
   2 #define __ARCH_S390_ATOMIC__
   3
   4 /*
   5  * Copyright 1999,2009 IBM Corp.
   6  * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
   7  *            Denis Joseph Barrow,
   8  *            Arnd Bergmann <arndb@de.ibm.com>,
   9  *
  10  * Atomic operations that C can't guarantee us.
  11  * Useful for resource counting etc.
  12  * s390 uses 'Compare And Swap' for atomicity in SMP environment.
  13  *
  14  */
  15
  16 #include <linux/compiler.h>
  17 #include <linux/types.h>
  18 #include <asm/system.h>
  19
  20 #define ATOMIC_INIT(i)  { (i) }
  21
  22 #define __CS_LOOP(ptr, op_val, op_string) ({                            \
  23         int old_val, new_val;                                           \
  24         asm volatile(                                                   \
  25                 "       l       %0,%2\n"                                \
  26                 "0:     lr      %1,%0\n"                                \
  27                 op_string "     %1,%3\n"                                \
  28                 "       cs      %0,%1,%2\n"                             \
  29                 "       jl      0b"                                     \
  30                 : "=&d" (old_val), "=&d" (new_val),                     \
  31                   "=Q" (((atomic_t *)(ptr))->counter)                   \
  32                 : "d" (op_val),  "Q" (((atomic_t *)(ptr))->counter)     \
  33                 : "cc", "memory");                                      \
  34         new_val;                                                        \
  35 })
  36
  37 static inline int atomic_read(const atomic_t *v)
  38 {
  39         int c;
  40
  41         asm volatile(
  42                 "       l       %0,%1\n"
  43                 : "=d" (c) : "Q" (v->counter));
  44         return c;
  45 }
  46
  47 static inline void atomic_set(atomic_t *v, int i)
  48 {
  49         asm volatile(
  50                 "       st      %1,%0\n"
  51                 : "=Q" (v->counter) : "d" (i));
  52 }
  53
  54 static inline int atomic_add_return(int i, atomic_t *v)
  55 {
  56         return __CS_LOOP(v, i, "ar");
  57 }
  58 #define atomic_add(_i, _v)              atomic_add_return(_i, _v)
  59 #define atomic_add_negative(_i, _v)     (atomic_add_return(_i, _v) < 0)
  60 #define atomic_inc(_v)                  atomic_add_return(1, _v)
  61 #define atomic_inc_return(_v)           atomic_add_return(1, _v)
  62 #define atomic_inc_and_test(_v)         (atomic_add_return(1, _v) == 0)
  63
  64 static inline int atomic_sub_return(int i, atomic_t *v)
  65 {
  66         return __CS_LOOP(v, i, "sr");
  67 }
  68 #define atomic_sub(_i, _v)              atomic_sub_return(_i, _v)
  69 #define atomic_sub_and_test(_i, _v)     (atomic_sub_return(_i, _v) == 0)
  70 #define atomic_dec(_v)                  atomic_sub_return(1, _v)
  71 #define atomic_dec_return(_v)           atomic_sub_return(1, _v)
  72 #define atomic_dec_and_test(_v)         (atomic_sub_return(1, _v) == 0)
  73
  74 static inline void atomic_clear_mask(unsigned long mask, atomic_t *v)
  75 {
  76         __CS_LOOP(v, ~mask, "nr");
  77 }
  78
  79 static inline void atomic_set_mask(unsigned long mask, atomic_t *v)
  80 {
  81         __CS_LOOP(v, mask, "or");
  82 }
  83
  84 #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
  85
  86 static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
  87 {
  88         asm volatile(
  89                 "       cs      %0,%2,%1"
  90                 : "+d" (old), "=Q" (v->counter)
  91                 : "d" (new), "Q" (v->counter)
  92                 : "cc", "memory");
  93         return old;
  94 }
  95
  96 static inline int atomic_add_unless(atomic_t *v, int a, int u)
  97 {
  98         int c, old;
  99         c = atomic_read(v);
 100         for (;;) {
 101                 if (unlikely(c == u))
 102                         break;
 103                 old = atomic_cmpxchg(v, c, c + a);
 104                 if (likely(old == c))
 105                         break;
 106                 c = old;
 107         }
 108         return c != u;
 109 }
 110
 111 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
 112
 113 #undef __CS_LOOP
 114
 115 #define ATOMIC64_INIT(i)  { (i) }
 116
 117 #ifdef CONFIG_64BIT
 118
 119 #define __CSG_LOOP(ptr, op_val, op_string) ({                           \
 120         long long old_val, new_val;                                     \
 121         asm volatile(                                                   \
 122                 "       lg      %0,%2\n"                                \
 123                 "0:     lgr     %1,%0\n"                                \
 124                 op_string "     %1,%3\n"                                \
 125                 "       csg     %0,%1,%2\n"                             \
 126                 "       jl      0b"                                     \
 127                 : "=&d" (old_val), "=&d" (new_val),                     \
 128                   "=Q" (((atomic_t *)(ptr))->counter)                   \
 129                 : "d" (op_val), "Q" (((atomic_t *)(ptr))->counter)      \
 130                 : "cc", "memory");                                      \
 131         new_val;                                                        \
 132 })
 133
 134 static inline long long atomic64_read(const atomic64_t *v)
 135 {
 136         long long c;
 137
 138         asm volatile(
 139                 "       lg      %0,%1\n"
 140                 : "=d" (c) : "Q" (v->counter));
 141         return c;
 142 }
 143
 144 static inline void atomic64_set(atomic64_t *v, long long i)
 145 {
 146         asm volatile(
 147                 "       stg     %1,%0\n"
 148                 : "=Q" (v->counter) : "d" (i));
 149 }
 150
 151 static inline long long atomic64_add_return(long long i, atomic64_t *v)
 152 {
 153         return __CSG_LOOP(v, i, "agr");
 154 }
 155
 156 static inline long long atomic64_sub_return(long long i, atomic64_t *v)
 157 {
 158         return __CSG_LOOP(v, i, "sgr");
 159 }
 160
 161 static inline void atomic64_clear_mask(unsigned long mask, atomic64_t *v)
 162 {
 163         __CSG_LOOP(v, ~mask, "ngr");
 164 }
 165
 166 static inline void atomic64_set_mask(unsigned long mask, atomic64_t *v)
 167 {
 168         __CSG_LOOP(v, mask, "ogr");
 169 }
 170
 171 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
 172
 173 static inline long long atomic64_cmpxchg(atomic64_t *v,
 174                                              long long old, long long new)
 175 {
 176         asm volatile(
 177                 "       csg     %0,%2,%1"
 178                 : "+d" (old), "=Q" (v->counter)
 179                 : "d" (new), "Q" (v->counter)
 180                 : "cc", "memory");
 181         return old;
 182 }
 183
 184 #undef __CSG_LOOP
 185
 186 #else /* CONFIG_64BIT */
 187
 188 typedef struct {
 189         long long counter;
 190 } atomic64_t;
 191
 192 static inline long long atomic64_read(const atomic64_t *v)
 193 {
 194         register_pair rp;
 195
 196         asm volatile(
 197                 "       lm      %0,%N0,%1"
 198                 : "=&d" (rp) : "Q" (v->counter) );
 199         return rp.pair;
 200 }
 201
 202 static inline void atomic64_set(atomic64_t *v, long long i)
 203 {
 204         register_pair rp = {.pair = i};
 205
 206         asm volatile(
 207                 "       stm     %1,%N1,%0"
 208                 : "=Q" (v->counter) : "d" (rp) );
 209 }
 210
 211 static inline long long atomic64_xchg(atomic64_t *v, long long new)
 212 {
 213         register_pair rp_new = {.pair = new};
 214         register_pair rp_old;
 215
 216         asm volatile(
 217                 "       lm      %0,%N0,%1\n"
 218                 "0:     cds     %0,%2,%1\n"
 219                 "       jl      0b\n"
 220                 : "=&d" (rp_old), "=Q" (v->counter)
 221                 : "d" (rp_new), "Q" (v->counter)
 222                 : "cc");
 223         return rp_old.pair;
 224 }
 225
 226 static inline long long atomic64_cmpxchg(atomic64_t *v,
 227                                          long long old, long long new)
 228 {
 229         register_pair rp_old = {.pair = old};
 230         register_pair rp_new = {.pair = new};
 231
 232         asm volatile(
 233                 "       cds     %0,%2,%1"
 234                 : "+&d" (rp_old), "=Q" (v->counter)
 235                 : "d" (rp_new), "Q" (v->counter)
 236                 : "cc");
 237         return rp_old.pair;
 238 }
 239
 240
 241 static inline long long atomic64_add_return(long long i, atomic64_t *v)
 242 {
 243         long long old, new;
 244
 245         do {
 246                 old = atomic64_read(v);
 247                 new = old + i;
 248         } while (atomic64_cmpxchg(v, old, new) != old);
 249         return new;
 250 }
 251
 252 static inline long long atomic64_sub_return(long long i, atomic64_t *v)
 253 {
 254         long long old, new;
 255
 256         do {
 257                 old = atomic64_read(v);
 258                 new = old - i;
 259         } while (atomic64_cmpxchg(v, old, new) != old);
 260         return new;
 261 }
 262
 263 static inline void atomic64_set_mask(unsigned long long mask, atomic64_t *v)
 264 {
 265         long long old, new;
 266
 267         do {
 268                 old = atomic64_read(v);
 269                 new = old | mask;
 270         } while (atomic64_cmpxchg(v, old, new) != old);
 271 }
 272
 273 static inline void atomic64_clear_mask(unsigned long long mask, atomic64_t *v)
 274 {
 275         long long old, new;
 276
 277         do {
 278                 old = atomic64_read(v);
 279                 new = old & mask;
 280         } while (atomic64_cmpxchg(v, old, new) != old);
 281 }
 282
 283 #endif /* CONFIG_64BIT */
 284
 285 static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u)
 286 {
 287         long long c, old;
 288
 289         c = atomic64_read(v);
 290         for (;;) {
 291                 if (unlikely(c == u))
 292                         break;
 293                 old = atomic64_cmpxchg(v, c, c + a);
 294                 if (likely(old == c))
 295                         break;
 296                 c = old;
 297         }
 298         return c != u;
 299 }
 300
 301 static inline long long atomic64_dec_if_positive(atomic64_t *v)
 302 {
 303         long long c, old, dec;
 304
 305         c = atomic64_read(v);
 306         for (;;) {
 307                 dec = c - 1;
 308                 if (unlikely(dec < 0))
 309                         break;
 310                 old = atomic64_cmpxchg((v), c, dec);
 311                 if (likely(old == c))
 312                         break;
 313                 c = old;
 314         }
 315         return dec;
 316 }
 317
 318 #define atomic64_add(_i, _v)            atomic64_add_return(_i, _v)
 319 #define atomic64_add_negative(_i, _v)   (atomic64_add_return(_i, _v) < 0)
 320 #define atomic64_inc(_v)                atomic64_add_return(1, _v)
 321 #define atomic64_inc_return(_v)         atomic64_add_return(1, _v)
 322 #define atomic64_inc_and_test(_v)       (atomic64_add_return(1, _v) == 0)
 323 #define atomic64_sub(_i, _v)            atomic64_sub_return(_i, _v)
 324 #define atomic64_sub_and_test(_i, _v)   (atomic64_sub_return(_i, _v) == 0)
 325 #define atomic64_dec(_v)                atomic64_sub_return(1, _v)
 326 #define atomic64_dec_return(_v)         atomic64_sub_return(1, _v)
 327 #define atomic64_dec_and_test(_v)       (atomic64_sub_return(1, _v) == 0)
 328 #define atomic64_inc_not_zero(v)        atomic64_add_unless((v), 1, 0)
 329
 330 #define smp_mb__before_atomic_dec()     smp_mb()
 331 #define smp_mb__after_atomic_dec()      smp_mb()
 332 #define smp_mb__before_atomic_inc()     smp_mb()
 333 #define smp_mb__after_atomic_inc()      smp_mb()
 334
 335 #include <asm-generic/atomic-long.h>
 336
 337 #endif /* __ARCH_S390_ATOMIC__  */