list: Use WRITE_ONCE() when adding to lists and hlists

Code that does lockless emptiness testing of non-RCU lists is relying
on the list-addition code to write the list head's ->next pointer
atomically.  This commit therefore adds WRITE_ONCE() to list-addition
pointer stores that could affect the head's ->next pointer.

Reported-by: Dmitry Vyukov <dvyukov@google.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/include/linux/list.h b/include/linux/list.h
index 993395a2e55c5483c890b40e216193d95c85bd64..d7e31fe398b3b7d980ec7d6b09dccd68f747a566 100644 (file)
--- a/include/linux/list.h
+++ b/include/linux/list.h
@@ -42,7 +42,7 @@ static inline void __list_add(struct list_head *new,
        next->prev = new;
        new->next = next;
        new->prev = prev;
-       prev->next = new;
+       WRITE_ONCE(prev->next, new);
 }
 #else
 extern void __list_add(struct list_head *new,
@@ -642,7 +642,7 @@ static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)
        n->next = first;
        if (first)
                first->pprev = &n->next;
-       h->first = n;
+       WRITE_ONCE(h->first, n);
        n->pprev = &h->first;
 }
 
@@ -653,14 +653,14 @@ static inline void hlist_add_before(struct hlist_node *n,
        n->pprev = next->pprev;
        n->next = next;
        next->pprev = &n->next;
-       *(n->pprev) = n;
+       WRITE_ONCE(*(n->pprev), n);
 }
 
 static inline void hlist_add_behind(struct hlist_node *n,
                                    struct hlist_node *prev)
 {
        n->next = prev->next;
-       prev->next = n;
+       WRITE_ONCE(prev->next, n);
        n->pprev = &prev->next;
 
        if (n->next)

diff --git a/lib/list_debug.c b/lib/list_debug.c
index c24c2f7e296fa15e5431ef7dc4a31f8551e37c15..3859bf63561c63936947b007fe3ee20e822509a1 100644 (file)
--- a/lib/list_debug.c
+++ b/lib/list_debug.c
@@ -37,7 +37,7 @@ void __list_add(struct list_head *new,
        next->prev = new;
        new->next = next;
        new->prev = prev;
-       prev->next = new;
+       WRITE_ONCE(prev->next, new);
 }
 EXPORT_SYMBOL(__list_add);