kernel/spinlock.c

   1 /*
   2  * Copyright (2004) Linus Torvalds
   3  *
   4  * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
   5  *
   6  * Copyright (2004, 2005) Ingo Molnar
   7  *
   8  * This file contains the spinlock/rwlock implementations for the
   9  * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
  10  *
  11  * Note that some architectures have special knowledge about the
  12  * stack frames of these functions in their profile_pc. If you
  13  * change anything significant here that could change the stack
  14  * frame contact the architecture maintainers.
  15  */
  16
  17 #include <linux/linkage.h>
  18 #include <linux/preempt.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/debug_locks.h>
  22 #include <linux/module.h>
  23
  24 int __lockfunc _spin_trylock(spinlock_t *lock)
  25 {
  26         preempt_disable();
  27         if (_raw_spin_trylock(lock)) {
  28                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  29                 return 1;
  30         }
  31
  32         preempt_enable();
  33         return 0;
  34 }
  35 EXPORT_SYMBOL(_spin_trylock);
  36
  37 int __lockfunc _read_trylock(rwlock_t *lock)
  38 {
  39         preempt_disable();
  40         if (_raw_read_trylock(lock)) {
  41                 rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
  42                 return 1;
  43         }
  44
  45         preempt_enable();
  46         return 0;
  47 }
  48 EXPORT_SYMBOL(_read_trylock);
  49
  50 int __lockfunc _write_trylock(rwlock_t *lock)
  51 {
  52         preempt_disable();
  53         if (_raw_write_trylock(lock)) {
  54                 rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  55                 return 1;
  56         }
  57
  58         preempt_enable();
  59         return 0;
  60 }
  61 EXPORT_SYMBOL(_write_trylock);
  62
  63 /*
  64  * If lockdep is enabled then we use the non-preemption spin-ops
  65  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
  66  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
  67  */
  68 #if !defined(CONFIG_PREEMPT) || !defined(CONFIG_SMP) || \
  69         defined(CONFIG_DEBUG_LOCK_ALLOC)
  70
  71 void __lockfunc _read_lock(rwlock_t *lock)
  72 {
  73         preempt_disable();
  74         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
  75         _raw_read_lock(lock);
  76 }
  77 EXPORT_SYMBOL(_read_lock);
  78
  79 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
  80 {
  81         unsigned long flags;
  82
  83         local_irq_save(flags);
  84         preempt_disable();
  85         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
  86         /*
  87          * On lockdep we dont want the hand-coded irq-enable of
  88          * _raw_spin_lock_flags() code, because lockdep assumes
  89          * that interrupts are not re-enabled during lock-acquire:
  90          */
  91 #ifdef CONFIG_PROVE_LOCKING
  92         _raw_spin_lock(lock);
  93 #else
  94         _raw_spin_lock_flags(lock, &flags);
  95 #endif
  96         return flags;
  97 }
  98 EXPORT_SYMBOL(_spin_lock_irqsave);
  99
 100 void __lockfunc _spin_lock_irq(spinlock_t *lock)
 101 {
 102         local_irq_disable();
 103         preempt_disable();
 104         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 105         _raw_spin_lock(lock);
 106 }
 107 EXPORT_SYMBOL(_spin_lock_irq);
 108
 109 void __lockfunc _spin_lock_bh(spinlock_t *lock)
 110 {
 111         local_bh_disable();
 112         preempt_disable();
 113         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 114         _raw_spin_lock(lock);
 115 }
 116 EXPORT_SYMBOL(_spin_lock_bh);
 117
 118 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
 119 {
 120         unsigned long flags;
 121
 122         local_irq_save(flags);
 123         preempt_disable();
 124         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
 125         _raw_read_lock(lock);
 126         return flags;
 127 }
 128 EXPORT_SYMBOL(_read_lock_irqsave);
 129
 130 void __lockfunc _read_lock_irq(rwlock_t *lock)
 131 {
 132         local_irq_disable();
 133         preempt_disable();
 134         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
 135         _raw_read_lock(lock);
 136 }
 137 EXPORT_SYMBOL(_read_lock_irq);
 138
 139 void __lockfunc _read_lock_bh(rwlock_t *lock)
 140 {
 141         local_bh_disable();
 142         preempt_disable();
 143         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
 144         _raw_read_lock(lock);
 145 }
 146 EXPORT_SYMBOL(_read_lock_bh);
 147
 148 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
 149 {
 150         unsigned long flags;
 151
 152         local_irq_save(flags);
 153         preempt_disable();
 154         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 155         _raw_write_lock(lock);
 156         return flags;
 157 }
 158 EXPORT_SYMBOL(_write_lock_irqsave);
 159
 160 void __lockfunc _write_lock_irq(rwlock_t *lock)
 161 {
 162         local_irq_disable();
 163         preempt_disable();
 164         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 165         _raw_write_lock(lock);
 166 }
 167 EXPORT_SYMBOL(_write_lock_irq);
 168
 169 void __lockfunc _write_lock_bh(rwlock_t *lock)
 170 {
 171         local_bh_disable();
 172         preempt_disable();
 173         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 174         _raw_write_lock(lock);
 175 }
 176 EXPORT_SYMBOL(_write_lock_bh);
 177
 178 void __lockfunc _spin_lock(spinlock_t *lock)
 179 {
 180         preempt_disable();
 181         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 182         _raw_spin_lock(lock);
 183 }
 184
 185 EXPORT_SYMBOL(_spin_lock);
 186
 187 void __lockfunc _write_lock(rwlock_t *lock)
 188 {
 189         preempt_disable();
 190         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 191         _raw_write_lock(lock);
 192 }
 193
 194 EXPORT_SYMBOL(_write_lock);
 195
 196 #else /* CONFIG_PREEMPT: */
 197
 198 /*
 199  * This could be a long-held lock. We both prepare to spin for a long
 200  * time (making _this_ CPU preemptable if possible), and we also signal
 201  * towards that other CPU that it should break the lock ASAP.
 202  *
 203  * (We do this in a function because inlining it would be excessive.)
 204  */
 205
 206 #define BUILD_LOCK_OPS(op, locktype)                                    \
 207 void __lockfunc _##op##_lock(locktype##_t *lock)                        \
 208 {                                                                       \
 209         for (;;) {                                                      \
 210                 preempt_disable();                                      \
 211                 if (likely(_raw_##op##_trylock(lock)))                  \
 212                         break;                                          \
 213                 preempt_enable();                                       \
 214                                                                         \
 215                 if (!(lock)->break_lock)                                \
 216                         (lock)->break_lock = 1;                         \
 217                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
 218                         cpu_relax();                                    \
 219         }                                                               \
 220         (lock)->break_lock = 0;                                         \
 221 }                                                                       \
 222                                                                         \
 223 EXPORT_SYMBOL(_##op##_lock);                                            \
 224                                                                         \
 225 unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock)       \
 226 {                                                                       \
 227         unsigned long flags;                                            \
 228                                                                         \
 229         for (;;) {                                                      \
 230                 preempt_disable();                                      \
 231                 local_irq_save(flags);                                  \
 232                 if (likely(_raw_##op##_trylock(lock)))                  \
 233                         break;                                          \
 234                 local_irq_restore(flags);                               \
 235                 preempt_enable();                                       \
 236                                                                         \
 237                 if (!(lock)->break_lock)                                \
 238                         (lock)->break_lock = 1;                         \
 239                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
 240                         cpu_relax();                                    \
 241         }                                                               \
 242         (lock)->break_lock = 0;                                         \
 243         return flags;                                                   \
 244 }                                                                       \
 245                                                                         \
 246 EXPORT_SYMBOL(_##op##_lock_irqsave);                                    \
 247                                                                         \
 248 void __lockfunc _##op##_lock_irq(locktype##_t *lock)                    \
 249 {                                                                       \
 250         _##op##_lock_irqsave(lock);                                     \
 251 }                                                                       \
 252                                                                         \
 253 EXPORT_SYMBOL(_##op##_lock_irq);                                        \
 254                                                                         \
 255 void __lockfunc _##op##_lock_bh(locktype##_t *lock)                     \
 256 {                                                                       \
 257         unsigned long flags;                                            \
 258                                                                         \
 259         /*                                                      */      \
 260         /* Careful: we must exclude softirqs too, hence the     */      \
 261         /* irq-disabling. We use the generic preemption-aware   */      \
 262         /* function:                                            */      \
 263         /**/                                                            \
 264         flags = _##op##_lock_irqsave(lock);                             \
 265         local_bh_disable();                                             \
 266         local_irq_restore(flags);                                       \
 267 }                                                                       \
 268                                                                         \
 269 EXPORT_SYMBOL(_##op##_lock_bh)
 270
 271 /*
 272  * Build preemption-friendly versions of the following
 273  * lock-spinning functions:
 274  *
 275  *         _[spin|read|write]_lock()
 276  *         _[spin|read|write]_lock_irq()
 277  *         _[spin|read|write]_lock_irqsave()
 278  *         _[spin|read|write]_lock_bh()
 279  */
 280 BUILD_LOCK_OPS(spin, spinlock);
 281 BUILD_LOCK_OPS(read, rwlock);
 282 BUILD_LOCK_OPS(write, rwlock);
 283
 284 #endif /* CONFIG_PREEMPT */
 285
 286 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 287
 288 void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
 289 {
 290         preempt_disable();
 291         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
 292         _raw_spin_lock(lock);
 293 }
 294
 295 EXPORT_SYMBOL(_spin_lock_nested);
 296
 297 #endif
 298
 299 void __lockfunc _spin_unlock(spinlock_t *lock)
 300 {
 301         spin_release(&lock->dep_map, 1, _RET_IP_);
 302         _raw_spin_unlock(lock);
 303         preempt_enable();
 304 }
 305 EXPORT_SYMBOL(_spin_unlock);
 306
 307 void __lockfunc _write_unlock(rwlock_t *lock)
 308 {
 309         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 310         _raw_write_unlock(lock);
 311         preempt_enable();
 312 }
 313 EXPORT_SYMBOL(_write_unlock);
 314
 315 void __lockfunc _read_unlock(rwlock_t *lock)
 316 {
 317         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 318         _raw_read_unlock(lock);
 319         preempt_enable();
 320 }
 321 EXPORT_SYMBOL(_read_unlock);
 322
 323 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
 324 {
 325         spin_release(&lock->dep_map, 1, _RET_IP_);
 326         _raw_spin_unlock(lock);
 327         local_irq_restore(flags);
 328         preempt_enable();
 329 }
 330 EXPORT_SYMBOL(_spin_unlock_irqrestore);
 331
 332 void __lockfunc _spin_unlock_irq(spinlock_t *lock)
 333 {
 334         spin_release(&lock->dep_map, 1, _RET_IP_);
 335         _raw_spin_unlock(lock);
 336         local_irq_enable();
 337         preempt_enable();
 338 }
 339 EXPORT_SYMBOL(_spin_unlock_irq);
 340
 341 void __lockfunc _spin_unlock_bh(spinlock_t *lock)
 342 {
 343         spin_release(&lock->dep_map, 1, _RET_IP_);
 344         _raw_spin_unlock(lock);
 345         preempt_enable_no_resched();
 346         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 347 }
 348 EXPORT_SYMBOL(_spin_unlock_bh);
 349
 350 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 351 {
 352         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 353         _raw_read_unlock(lock);
 354         local_irq_restore(flags);
 355         preempt_enable();
 356 }
 357 EXPORT_SYMBOL(_read_unlock_irqrestore);
 358
 359 void __lockfunc _read_unlock_irq(rwlock_t *lock)
 360 {
 361         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 362         _raw_read_unlock(lock);
 363         local_irq_enable();
 364         preempt_enable();
 365 }
 366 EXPORT_SYMBOL(_read_unlock_irq);
 367
 368 void __lockfunc _read_unlock_bh(rwlock_t *lock)
 369 {
 370         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 371         _raw_read_unlock(lock);
 372         preempt_enable_no_resched();
 373         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 374 }
 375 EXPORT_SYMBOL(_read_unlock_bh);
 376
 377 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 378 {
 379         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 380         _raw_write_unlock(lock);
 381         local_irq_restore(flags);
 382         preempt_enable();
 383 }
 384 EXPORT_SYMBOL(_write_unlock_irqrestore);
 385
 386 void __lockfunc _write_unlock_irq(rwlock_t *lock)
 387 {
 388         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 389         _raw_write_unlock(lock);
 390         local_irq_enable();
 391         preempt_enable();
 392 }
 393 EXPORT_SYMBOL(_write_unlock_irq);
 394
 395 void __lockfunc _write_unlock_bh(rwlock_t *lock)
 396 {
 397         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 398         _raw_write_unlock(lock);
 399         preempt_enable_no_resched();
 400         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 401 }
 402 EXPORT_SYMBOL(_write_unlock_bh);
 403
 404 int __lockfunc _spin_trylock_bh(spinlock_t *lock)
 405 {
 406         local_bh_disable();
 407         preempt_disable();
 408         if (_raw_spin_trylock(lock)) {
 409                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
 410                 return 1;
 411         }
 412
 413         preempt_enable_no_resched();
 414         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 415         return 0;
 416 }
 417 EXPORT_SYMBOL(_spin_trylock_bh);
 418
 419 int in_lock_functions(unsigned long addr)
 420 {
 421         /* Linker adds these: start and end of __lockfunc functions */
 422         extern char __lock_text_start[], __lock_text_end[];
 423
 424         return addr >= (unsigned long)__lock_text_start
 425         && addr < (unsigned long)__lock_text_end;
 426 }
 427 EXPORT_SYMBOL(in_lock_functions);