llist: Move cpu_relax() to after the cmpxchg()

If in llist_add()/etc. functions the first cmpxchg() call succeeds, it is
not necessary to use cpu_relax() before the cmpxchg(). So cpu_relax() in
a busy loop involving cmpxchg() should go after cmpxchg() instead of before
that.

This patch fixes this for all involved llist functions.

Signed-off-by: Huang Ying <ying.huang@intel.com>
Acked-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/r/1315461646-1379-4-git-send-email-ying.huang@intel.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/include/linux/llist.h b/include/linux/llist.h
index 65fca1cbf5146624c0c7bc2f6b691d052f0bf6a7..ca91875286bf91c5e330ca0e65b01f59fc7471f7 100644 (file)
--- a/include/linux/llist.h
+++ b/include/linux/llist.h
@@ -148,11 +148,14 @@ static inline void llist_add(struct llist_node *new, struct llist_head *head)
        struct llist_node *entry, *old_entry;
 
        entry = head->first;
-       do {
+       for (;;) {
                old_entry = entry;
                new->next = entry;
+               entry = cmpxchg(&head->first, old_entry, new);
+               if (entry == old_entry)
+                       break;
                cpu_relax();
-       } while ((entry = cmpxchg(&head->first, old_entry, new)) != old_entry);
+       }
 }
 
 /**

diff --git a/lib/llist.c b/lib/llist.c
index b445f2c8596ac20aba66e3bc58b2816f0c095cb1..6c69f1d14c4bf41f2d993d74812b920d19a64f54 100644 (file)
--- a/lib/llist.c
+++ b/lib/llist.c
@@ -41,11 +41,14 @@ void llist_add_batch(struct llist_node *new_first, struct llist_node *new_last,
        struct llist_node *entry, *old_entry;
 
        entry = head->first;
-       do {
+       for (;;) {
                old_entry = entry;
                new_last->next = entry;
+               entry = cmpxchg(&head->first, old_entry, new_first);
+               if (entry == old_entry)
+                       break;
                cpu_relax();
-       } while ((entry = cmpxchg(&head->first, old_entry, new_first)) != old_entry);
+       }
 }
 EXPORT_SYMBOL_GPL(llist_add_batch);
 
@@ -68,13 +71,16 @@ struct llist_node *llist_del_first(struct llist_head *head)
        struct llist_node *entry, *old_entry, *next;
 
        entry = head->first;
-       do {
+       for (;;) {
                if (entry == NULL)
                        return NULL;
                old_entry = entry;
                next = entry->next;
+               entry = cmpxchg(&head->first, old_entry, next);
+               if (entry == old_entry)
+                       break;
                cpu_relax();
-       } while ((entry = cmpxchg(&head->first, old_entry, next)) != old_entry);
+       }
 
        return entry;
 }