Fix RCU list iterator use of 'rcu_dereference()'

The RCU iterators used 'rcu_dereference()' on an already-fetched RCU
pointer value, which defeats the whole point of the exercise.

When we dereference a pointer protected by RCU, we need to make sure
that we only fetch the value _once_, because if the compiler ends up
re-loading it due to register pressure, the newly reloaded value could
be different from the previously fetched one, and you get inconsistent
results.

Cleaned-up, fixed, and the pointless list_for_each_safe_rcu #define
deleted by Paul Kenney.

Acked-by: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/list.h b/include/linux/list.h
index 75ce2cb4ff6ebc92a8fcaa557a9bd7b6b9b9b1b6..dac16f99c70115ba4e4f1cee8b18f5f1b0f050c8 100644 (file)
--- a/include/linux/list.h
+++ b/include/linux/list.h
@@ -631,31 +631,14 @@ static inline void list_splice_init_rcu(struct list_head *list,
  * as long as the traversal is guarded by rcu_read_lock().
  */
 #define list_for_each_rcu(pos, head) \
-       for (pos = (head)->next; \
-               prefetch(rcu_dereference(pos)->next), pos != (head); \
-               pos = pos->next)
+       for (pos = rcu_dereference((head)->next); \
+               prefetch(pos->next), pos != (head); \
+               pos = rcu_dereference(pos->next))
 
 #define __list_for_each_rcu(pos, head) \
-       for (pos = (head)->next; \
-               rcu_dereference(pos) != (head); \
-               pos = pos->next)
-
-/**
- * list_for_each_safe_rcu
- * @pos:       the &struct list_head to use as a loop cursor.
- * @n:         another &struct list_head to use as temporary storage
- * @head:      the head for your list.
- *
- * Iterate over an rcu-protected list, safe against removal of list entry.
- *
- * This list-traversal primitive may safely run concurrently with
- * the _rcu list-mutation primitives such as list_add_rcu()
- * as long as the traversal is guarded by rcu_read_lock().
- */
-#define list_for_each_safe_rcu(pos, n, head) \
-       for (pos = (head)->next; \
-               n = rcu_dereference(pos)->next, pos != (head); \
-               pos = n)
+       for (pos = rcu_dereference((head)->next); \
+               pos != (head); \
+               pos = rcu_dereference(pos->next))
 
 /**
  * list_for_each_entry_rcu     -       iterate over rcu list of given type
@@ -668,10 +651,9 @@ static inline void list_splice_init_rcu(struct list_head *list,
  * as long as the traversal is guarded by rcu_read_lock().
  */
 #define list_for_each_entry_rcu(pos, head, member) \
-       for (pos = list_entry((head)->next, typeof(*pos), member); \
-               prefetch(rcu_dereference(pos)->member.next), \
-                       &pos->member != (head); \
-               pos = list_entry(pos->member.next, typeof(*pos), member))
+       for (pos = list_entry(rcu_dereference((head)->next), typeof(*pos), member); \
+               prefetch(pos->member.next), &pos->member != (head); \
+               pos = list_entry(rcu_dereference(pos->member.next), typeof(*pos), member))
 
 
 /**
@@ -686,9 +668,9 @@ static inline void list_splice_init_rcu(struct list_head *list,
  * as long as the traversal is guarded by rcu_read_lock().
  */
 #define list_for_each_continue_rcu(pos, head) \
-       for ((pos) = (pos)->next; \
-               prefetch(rcu_dereference((pos))->next), (pos) != (head); \
-               (pos) = (pos)->next)
+       for ((pos) = rcu_dereference((pos)->next); \
+               prefetch((pos)->next), (pos) != (head); \
+               (pos) = rcu_dereference((pos)->next))
 
 /*
  * Double linked lists with a single pointer list head.
@@ -986,10 +968,10 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev,
  * as long as the traversal is guarded by rcu_read_lock().
  */
 #define hlist_for_each_entry_rcu(tpos, pos, head, member)               \
-       for (pos = (head)->first;                                        \
-            rcu_dereference(pos) && ({ prefetch(pos->next); 1;}) &&     \
+       for (pos = rcu_dereference((head)->first);                       \
+               pos && ({ prefetch(pos->next); 1;}) &&                   \
                ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
-            pos = pos->next)
+            pos = rcu_dereference(pos->next))
 
 #else
 #warning "don't include kernel headers in userspace"