llist: Make some llist functions inline

Because llist code will be used in performance critical scheduler
code path, make llist_add() and llist_del_all() inline to avoid
function calling overhead and related 'glue' overhead.

Signed-off-by: Huang Ying <ying.huang@intel.com>
Acked-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/r/1315461646-1379-2-git-send-email-ying.huang@intel.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/drivers/acpi/apei/Kconfig b/drivers/acpi/apei/Kconfig
index e3f47872ec222949d13f239fdaf9e00eb5985984..f0c1ce95a0eccf1bc3987d688ad9bb1cdfd66532 100644 (file)
--- a/drivers/acpi/apei/Kconfig
+++ b/drivers/acpi/apei/Kconfig
@@ -14,7 +14,6 @@ config ACPI_APEI_GHES
        depends on ACPI_APEI && X86
        select ACPI_HED
        select IRQ_WORK
-       select LLIST
        select GENERIC_ALLOCATOR
        help
          Generic Hardware Error Source provides a way to report

diff --git a/include/linux/llist.h b/include/linux/llist.h
index aa0c8b5b3cd0d2b2df5804d1cf94aebb76580d2c..3eccdfd6609651f9b331030f95560d3d3f3bfca2 100644 (file)
--- a/include/linux/llist.h
+++ b/include/linux/llist.h
@@ -37,8 +37,28 @@
  * architectures that don't have NMI-safe cmpxchg implementation, the
  * list can NOT be used in NMI handler.  So code uses the list in NMI
  * handler should depend on CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG.
+ *
+ * Copyright 2010,2011 Intel Corp.
+ *   Author: Huang Ying <ying.huang@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version
+ * 2 as published by the Free Software Foundation;
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
+#include <linux/kernel.h>
+#include <asm/system.h>
+#include <asm/processor.h>
+
 struct llist_head {
        struct llist_node *first;
 };
@@ -113,14 +133,46 @@ static inline void init_llist_head(struct llist_head *list)
  * test whether the list is empty without deleting something from the
  * list.
  */
-static inline int llist_empty(const struct llist_head *head)
+static inline bool llist_empty(const struct llist_head *head)
 {
        return ACCESS_ONCE(head->first) == NULL;
 }
 
-void llist_add(struct llist_node *new, struct llist_head *head);
-void llist_add_batch(struct llist_node *new_first, struct llist_node *new_last,
-                    struct llist_head *head);
-struct llist_node *llist_del_first(struct llist_head *head);
-struct llist_node *llist_del_all(struct llist_head *head);
+/**
+ * llist_add - add a new entry
+ * @new:       new entry to be added
+ * @head:      the head for your lock-less list
+ */
+static inline void llist_add(struct llist_node *new, struct llist_head *head)
+{
+       struct llist_node *entry, *old_entry;
+
+#ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
+       BUG_ON(in_nmi());
+#endif
+
+       entry = head->first;
+       do {
+               old_entry = entry;
+               new->next = entry;
+               cpu_relax();
+       } while ((entry = cmpxchg(&head->first, old_entry, new)) != old_entry);
+}
+
+/**
+ * llist_del_all - delete all entries from lock-less list
+ * @head:      the head of lock-less list to delete all entries
+ *
+ * If list is empty, return NULL, otherwise, delete all entries and
+ * return the pointer to the first entry.  The order of entries
+ * deleted is from the newest to the oldest added one.
+ */
+static inline struct llist_node *llist_del_all(struct llist_head *head)
+{
+#ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
+       BUG_ON(in_nmi());
+#endif
+
+       return xchg(&head->first, NULL);
+}
 #endif /* LLIST_H */

diff --git a/lib/Kconfig b/lib/Kconfig
index 6c695ff9caba7579aaf9f4814abff70ba7c21084..32f3e5ae2be543c59ae3e50682ca504010eb87a3 100644 (file)
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -276,7 +276,4 @@ config CORDIC
          so its calculations are in fixed point. Modules can select this
          when they require this function. Module will be called cordic.
 
-config LLIST
-       bool
-
 endmenu

diff --git a/lib/Makefile b/lib/Makefile
index 3f5bc6d903e088e3a8d17dcc458aec006e7ada61..a4da283f5dc0182a45d370fd76884a86fa37ac42 100644 (file)
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -22,7 +22,7 @@ lib-y += kobject.o kref.o klist.o
 obj-y += bcd.o div64.o sort.o parser.o halfmd4.o debug_locks.o random32.o \
         bust_spinlocks.o hexdump.o kasprintf.o bitmap.o scatterlist.o \
         string_helpers.o gcd.o lcm.o list_sort.o uuid.o flex_array.o \
-        bsearch.o find_last_bit.o find_next_bit.o
+        bsearch.o find_last_bit.o find_next_bit.o llist.o
 obj-y += kstrtox.o
 obj-$(CONFIG_TEST_KSTRTOX) += test-kstrtox.o
 
@@ -115,8 +115,6 @@ obj-$(CONFIG_CPU_RMAP) += cpu_rmap.o
 
 obj-$(CONFIG_CORDIC) += cordic.o
 
-obj-$(CONFIG_LLIST) += llist.o
-
 hostprogs-y    := gen_crc32table
 clean-files    := crc32table.h
 

diff --git a/lib/llist.c b/lib/llist.c
index da445724fa1ffe221421b06c540dd17b9e0df2b7..3e3fa9139c41daa5f91d60fd2d491eff6a697ab2 100644 (file)
--- a/lib/llist.c
+++ b/lib/llist.c
@@ -29,28 +29,6 @@
 
 #include <asm/system.h>
 
-/**
- * llist_add - add a new entry
- * @new:       new entry to be added
- * @head:      the head for your lock-less list
- */
-void llist_add(struct llist_node *new, struct llist_head *head)
-{
-       struct llist_node *entry, *old_entry;
-
-#ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
-       BUG_ON(in_nmi());
-#endif
-
-       entry = head->first;
-       do {
-               old_entry = entry;
-               new->next = entry;
-               cpu_relax();
-       } while ((entry = cmpxchg(&head->first, old_entry, new)) != old_entry);
-}
-EXPORT_SYMBOL_GPL(llist_add);
-
 /**
  * llist_add_batch - add several linked entries in batch
  * @new_first: first entry in batch to be added
@@ -109,21 +87,3 @@ struct llist_node *llist_del_first(struct llist_head *head)
        return entry;
 }
 EXPORT_SYMBOL_GPL(llist_del_first);
-
-/**
- * llist_del_all - delete all entries from lock-less list
- * @head:      the head of lock-less list to delete all entries
- *
- * If list is empty, return NULL, otherwise, delete all entries and
- * return the pointer to the first entry.  The order of entries
- * deleted is from the newest to the oldest added one.
- */
-struct llist_node *llist_del_all(struct llist_head *head)
-{
-#ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
-       BUG_ON(in_nmi());
-#endif
-
-       return xchg(&head->first, NULL);
-}
-EXPORT_SYMBOL_GPL(llist_del_all);