Merge tag 'v3.10.89' into update

[GitHub/mt8127/android_kernel_alcatel_ttab.git] / include / linux / percpu-rwsem.h

diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h
index bd1e86071e57c27219b2746ab715d419c60fcc2c..3e88c9a7d57f7fdaf5c1e8ee3194df6ae3677f13 100644 (file)
--- a/include/linux/percpu-rwsem.h
+++ b/include/linux/percpu-rwsem.h
@@ -1,83 +1,34 @@
 #ifndef _LINUX_PERCPU_RWSEM_H
 #define _LINUX_PERCPU_RWSEM_H
 
-#include <linux/mutex.h>
+#include <linux/atomic.h>
+#include <linux/rwsem.h>
 #include <linux/percpu.h>
-#include <linux/rcupdate.h>
-#include <linux/delay.h>
+#include <linux/wait.h>
+#include <linux/lockdep.h>
 
 struct percpu_rw_semaphore {
-       unsigned __percpu *counters;
-       bool locked;
-       struct mutex mtx;
+       unsigned int __percpu   *fast_read_ctr;
+       atomic_t                write_ctr;
+       struct rw_semaphore     rw_sem;
+       atomic_t                slow_read_ctr;
+       wait_queue_head_t       write_waitq;
 };
 
-#define light_mb()     barrier()
-#define heavy_mb()     synchronize_sched_expedited()
+extern void percpu_down_read(struct percpu_rw_semaphore *);
+extern void percpu_up_read(struct percpu_rw_semaphore *);
 
-static inline void percpu_down_read(struct percpu_rw_semaphore *p)
-{
-       rcu_read_lock_sched();
-       if (unlikely(p->locked)) {
-               rcu_read_unlock_sched();
-               mutex_lock(&p->mtx);
-               this_cpu_inc(*p->counters);
-               mutex_unlock(&p->mtx);
-               return;
-       }
-       this_cpu_inc(*p->counters);
-       rcu_read_unlock_sched();
-       light_mb(); /* A, between read of p->locked and read of data, paired with D */
-}
+extern void percpu_down_write(struct percpu_rw_semaphore *);
+extern void percpu_up_write(struct percpu_rw_semaphore *);
 
-static inline void percpu_up_read(struct percpu_rw_semaphore *p)
-{
-       light_mb(); /* B, between read of the data and write to p->counter, paired with C */
-       this_cpu_dec(*p->counters);
-}
+extern int __percpu_init_rwsem(struct percpu_rw_semaphore *,
+                               const char *, struct lock_class_key *);
+extern void percpu_free_rwsem(struct percpu_rw_semaphore *);
 
-static inline unsigned __percpu_count(unsigned __percpu *counters)
-{
-       unsigned total = 0;
-       int cpu;
-
-       for_each_possible_cpu(cpu)
-               total += ACCESS_ONCE(*per_cpu_ptr(counters, cpu));
-
-       return total;
-}
-
-static inline void percpu_down_write(struct percpu_rw_semaphore *p)
-{
-       mutex_lock(&p->mtx);
-       p->locked = true;
-       synchronize_sched_expedited(); /* make sure that all readers exit the rcu_read_lock_sched region */
-       while (__percpu_count(p->counters))
-               msleep(1);
-       heavy_mb(); /* C, between read of p->counter and write to data, paired with B */
-}
-
-static inline void percpu_up_write(struct percpu_rw_semaphore *p)
-{
-       heavy_mb(); /* D, between write to data and write to p->locked, paired with A */
-       p->locked = false;
-       mutex_unlock(&p->mtx);
-}
-
-static inline int percpu_init_rwsem(struct percpu_rw_semaphore *p)
-{
-       p->counters = alloc_percpu(unsigned);
-       if (unlikely(!p->counters))
-               return -ENOMEM;
-       p->locked = false;
-       mutex_init(&p->mtx);
-       return 0;
-}
-
-static inline void percpu_free_rwsem(struct percpu_rw_semaphore *p)
-{
-       free_percpu(p->counters);
-       p->counters = NULL; /* catch use after free bugs */
-}
+#define percpu_init_rwsem(brw) \
+({                                                             \
+       static struct lock_class_key rwsem_key;                 \
+       __percpu_init_rwsem(brw, #brw, &rwsem_key);             \
+})
 
 #endif