arch/arm64/include/asm/cmpxchg.h

   1 /*
   2  * Based on arch/arm/include/asm/cmpxchg.h
   3  *
   4  * Copyright (C) 2012 ARM Ltd.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18 #ifndef __ASM_CMPXCHG_H
  19 #define __ASM_CMPXCHG_H
  20
  21 #include <linux/bug.h>
  22
  23 #include <asm/barrier.h>
  24
  25 static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
  26 {
  27         unsigned long ret, tmp;
  28
  29         switch (size) {
  30         case 1:
  31                 asm volatile("//        __xchg1\n"
  32                 "1:     ldaxrb  %w0, %2\n"
  33                 "       stlxrb  %w1, %w3, %2\n"
  34                 "       cbnz    %w1, 1b\n"
  35                         : "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr)
  36                         : "r" (x)
  37                         : "cc", "memory");
  38                 break;
  39         case 2:
  40                 asm volatile("//        __xchg2\n"
  41                 "1:     ldaxrh  %w0, %2\n"
  42                 "       stlxrh  %w1, %w3, %2\n"
  43                 "       cbnz    %w1, 1b\n"
  44                         : "=&r" (ret), "=&r" (tmp), "+Q" (*(u16 *)ptr)
  45                         : "r" (x)
  46                         : "cc", "memory");
  47                 break;
  48         case 4:
  49                 asm volatile("//        __xchg4\n"
  50                 "1:     ldaxr   %w0, %2\n"
  51                 "       stlxr   %w1, %w3, %2\n"
  52                 "       cbnz    %w1, 1b\n"
  53                         : "=&r" (ret), "=&r" (tmp), "+Q" (*(u32 *)ptr)
  54                         : "r" (x)
  55                         : "cc", "memory");
  56                 break;
  57         case 8:
  58                 asm volatile("//        __xchg8\n"
  59                 "1:     ldaxr   %0, %2\n"
  60                 "       stlxr   %w1, %3, %2\n"
  61                 "       cbnz    %w1, 1b\n"
  62                         : "=&r" (ret), "=&r" (tmp), "+Q" (*(u64 *)ptr)
  63                         : "r" (x)
  64                         : "cc", "memory");
  65                 break;
  66         default:
  67                 BUILD_BUG();
  68         }
  69
  70         return ret;
  71 }
  72
  73 #define xchg(ptr,x) \
  74 ({ \
  75         __typeof__(*(ptr)) __ret; \
  76         __ret = (__typeof__(*(ptr))) \
  77                 __xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \
  78         __ret; \
  79 })
  80
  81 static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
  82                                       unsigned long new, int size)
  83 {
  84         unsigned long oldval = 0, res;
  85
  86         switch (size) {
  87         case 1:
  88                 do {
  89                         asm volatile("// __cmpxchg1\n"
  90                         "       ldxrb   %w1, %2\n"
  91                         "       mov     %w0, #0\n"
  92                         "       cmp     %w1, %w3\n"
  93                         "       b.ne    1f\n"
  94                         "       stxrb   %w0, %w4, %2\n"
  95                         "1:\n"
  96                                 : "=&r" (res), "=&r" (oldval), "+Q" (*(u8 *)ptr)
  97                                 : "Ir" (old), "r" (new)
  98                                 : "cc");
  99                 } while (res);
 100                 break;
 101
 102         case 2:
 103                 do {
 104                         asm volatile("// __cmpxchg2\n"
 105                         "       ldxrh   %w1, %2\n"
 106                         "       mov     %w0, #0\n"
 107                         "       cmp     %w1, %w3\n"
 108                         "       b.ne    1f\n"
 109                         "       stxrh   %w0, %w4, %2\n"
 110                         "1:\n"
 111                                 : "=&r" (res), "=&r" (oldval), "+Q" (*(u16 *)ptr)
 112                                 : "Ir" (old), "r" (new)
 113                                 : "cc");
 114                 } while (res);
 115                 break;
 116
 117         case 4:
 118                 do {
 119                         asm volatile("// __cmpxchg4\n"
 120                         "       ldxr    %w1, %2\n"
 121                         "       mov     %w0, #0\n"
 122                         "       cmp     %w1, %w3\n"
 123                         "       b.ne    1f\n"
 124                         "       stxr    %w0, %w4, %2\n"
 125                         "1:\n"
 126                                 : "=&r" (res), "=&r" (oldval), "+Q" (*(u32 *)ptr)
 127                                 : "Ir" (old), "r" (new)
 128                                 : "cc");
 129                 } while (res);
 130                 break;
 131
 132         case 8:
 133                 do {
 134                         asm volatile("// __cmpxchg8\n"
 135                         "       ldxr    %1, %2\n"
 136                         "       mov     %w0, #0\n"
 137                         "       cmp     %1, %3\n"
 138                         "       b.ne    1f\n"
 139                         "       stxr    %w0, %4, %2\n"
 140                         "1:\n"
 141                                 : "=&r" (res), "=&r" (oldval), "+Q" (*(u64 *)ptr)
 142                                 : "Ir" (old), "r" (new)
 143                                 : "cc");
 144                 } while (res);
 145                 break;
 146
 147         default:
 148                 BUILD_BUG();
 149         }
 150
 151         return oldval;
 152 }
 153
 154 static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
 155                                          unsigned long new, int size)
 156 {
 157         unsigned long ret;
 158
 159         smp_mb();
 160         ret = __cmpxchg(ptr, old, new, size);
 161         smp_mb();
 162
 163         return ret;
 164 }
 165
 166 #define cmpxchg(ptr, o, n) \
 167 ({ \
 168         __typeof__(*(ptr)) __ret; \
 169         __ret = (__typeof__(*(ptr))) \
 170                 __cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \
 171                              sizeof(*(ptr))); \
 172         __ret; \
 173 })
 174
 175 #define cmpxchg_local(ptr, o, n) \
 176 ({ \
 177         __typeof__(*(ptr)) __ret; \
 178         __ret = (__typeof__(*(ptr))) \
 179                 __cmpxchg((ptr), (unsigned long)(o), \
 180                           (unsigned long)(n), sizeof(*(ptr))); \
 181         __ret; \
 182 })
 183
 184 #define cmpxchg64(ptr,o,n)              cmpxchg((ptr),(o),(n))
 185 #define cmpxchg64_local(ptr,o,n)        cmpxchg_local((ptr),(o),(n))
 186
 187 #endif  /* __ASM_CMPXCHG_H */