x86: Provide an alternative() based cmpxchg64()

cmpxchg64() today generates, to quote Linus, "barf bag" code.

cmpxchg64() is about to get used in the scheduler to fix a bug there,
but it's a prerequisite that cmpxchg64() first be made non-sucking.

This patch turns cmpxchg64() into an efficient implementation that
uses the alternative() mechanism to just use the raw instruction on
all modern systems.

Note: the fallback is NOT smp safe, just like the current fallback
is not SMP safe. (Interested parties with i486 based SMP systems
are welcome to submit fix patches for that.)

Signed-off-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
[ fixed asm constraint bug ]
Fixed-by: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: John Stultz <johnstul@us.ibm.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <20090930170754.0886ff2e@infradead.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/x86/include/asm/cmpxchg_32.h b/arch/x86/include/asm/cmpxchg_32.h
index 82ceb788a981e7c73a7cb06e3d0bec311e205be7..ee1931be6593aba1d344b7644340a1422d7a4f6d 100644 (file)
--- a/arch/x86/include/asm/cmpxchg_32.h
+++ b/arch/x86/include/asm/cmpxchg_32.h
@@ -312,19 +312,23 @@ static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
 
 extern unsigned long long cmpxchg_486_u64(volatile void *, u64, u64);
 
-#define cmpxchg64(ptr, o, n)                                           \
-({                                                                     \
-       __typeof__(*(ptr)) __ret;                                       \
-       if (likely(boot_cpu_data.x86 > 4))                              \
-               __ret = (__typeof__(*(ptr)))__cmpxchg64((ptr),          \
-                               (unsigned long long)(o),                \
-                               (unsigned long long)(n));               \
-       else                                                            \
-               __ret = (__typeof__(*(ptr)))cmpxchg_486_u64((ptr),      \
-                               (unsigned long long)(o),                \
-                               (unsigned long long)(n));               \
-       __ret;                                                          \
-})
+#define cmpxchg64(ptr, o, n)                                   \
+({                                                             \
+       __typeof__(*(ptr)) __ret;                               \
+       __typeof__(*(ptr)) __old = (o);                         \
+       __typeof__(*(ptr)) __new = (n);                         \
+       alternative_io("call cmpxchg8b_emu",                    \
+                       "lock; cmpxchg8b (%%esi)" ,             \
+                      X86_FEATURE_CX8,                         \
+                      "=A" (__ret),                            \
+                      "S" ((ptr)), "0" (__old),                \
+                      "b" ((unsigned int)__new),               \
+                      "c" ((unsigned int)(__new>>32))          \
+                      : "memory");                             \
+       __ret; })
+
+
+
 #define cmpxchg64_local(ptr, o, n)                                     \
 ({                                                                     \
        __typeof__(*(ptr)) __ret;                                       \

diff --git a/arch/x86/kernel/i386_ksyms_32.c b/arch/x86/kernel/i386_ksyms_32.c
index 43cec6bdda637aa48aef7f590aa48b9df1300396..1736c5a725aafe21959f4903d721788d1cc0354c 100644 (file)
--- a/arch/x86/kernel/i386_ksyms_32.c
+++ b/arch/x86/kernel/i386_ksyms_32.c
@@ -10,6 +10,14 @@
 EXPORT_SYMBOL(mcount);
 #endif
 
+/*
+ * Note, this is a prototype to get at the symbol for
+ * the export, but dont use it from C code, it is used
+ * by assembly code and is not using C calling convention!
+ */
+extern void cmpxchg8b_emu(void);
+EXPORT_SYMBOL(cmpxchg8b_emu);
+
 /* Networking helper routines. */
 EXPORT_SYMBOL(csum_partial_copy_generic);
 

diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 9e609206fac9b668b906c5949a3b018f9804752c..3e549b8ec8c9da4ebf3c02db2289196abc0a5b1b 100644 (file)
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -15,7 +15,7 @@ ifeq ($(CONFIG_X86_32),y)
         obj-y += atomic64_32.o
         lib-y += checksum_32.o
         lib-y += strstr_32.o
-        lib-y += semaphore_32.o string_32.o
+        lib-y += semaphore_32.o string_32.o cmpxchg8b_emu.o
 
         lib-$(CONFIG_X86_USE_3DNOW) += mmx_32.o
 else

diff --git a/arch/x86/lib/cmpxchg8b_emu.S b/arch/x86/lib/cmpxchg8b_emu.S
new file mode 100644 (file)
index 0000000..828cb71
--- /dev/null
+++ b/arch/x86/lib/cmpxchg8b_emu.S
@@ -0,0 +1,57 @@
+/*
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     as published by the Free Software Foundation; version 2
+ *     of the License.
+ *
+ */
+
+#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
+#include <asm/frame.h>
+#include <asm/dwarf2.h>
+
+
+.text
+
+/*
+ * Inputs:
+ * %esi : memory location to compare
+ * %eax : low 32 bits of old value
+ * %edx : high 32 bits of old value
+ * %ebx : low 32 bits of new value
+ * %ecx : high 32 bits of new value
+ */
+ENTRY(cmpxchg8b_emu)
+CFI_STARTPROC
+
+#
+# Emulate 'cmpxchg8b (%esi)' on UP except we don't
+# set the whole ZF thing (caller will just compare
+# eax:edx with the expected value)
+#
+cmpxchg8b_emu:
+       pushfl
+       cli
+
+       cmpl  (%esi), %eax
+       jne not_same
+       cmpl 4(%esi), %edx
+       jne half_same
+
+       movl %ebx,  (%esi)
+       movl %ecx, 4(%esi)
+
+       popfl
+       ret
+
+ not_same:
+       movl  (%esi), %eax
+ half_same:
+       movl 4(%esi), %edx
+
+       popfl
+       ret
+
+CFI_ENDPROC
+ENDPROC(cmpxchg8b_emu)