x86-64: Modify memcpy()/memset() alternatives mechanism

In order to avoid unnecessary chains of branches, rather than
implementing memcpy()/memset()'s access to their alternative
implementations via a jump, patch the (larger) original function
directly.

The memcpy() part of this is slightly subtle: while alternative
instruction patching does itself use memcpy(), with the
replacement block being less than 64-bytes in size the main loop
of the original function doesn't get used for copying memcpy_c()
over memcpy(), and hence we can safely write over its beginning.

Also note that the CFI annotations are fine for both variants of
each of the functions.

Signed-off-by: Jan Beulich <jbeulich@novell.com>
Cc: Nick Piggin <npiggin@suse.de>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
LKML-Reference: <4B2BB8D30200007800026AF2@vpn.id2.novell.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S
index ad5441ed1b57fdfffe6822dede52cad626ae2b39..f82e884928af6643854f64df5899c42ab6fee9b2 100644 (file)
--- a/arch/x86/lib/memcpy_64.S
+++ b/arch/x86/lib/memcpy_64.S
@@ -20,12 +20,11 @@
 /*
  * memcpy_c() - fast string ops (REP MOVSQ) based variant.
  *
- * Calls to this get patched into the kernel image via the
+ * This gets patched over the unrolled variant (below) via the
  * alternative instructions framework:
  */
-       ALIGN
-memcpy_c:
-       CFI_STARTPROC
+       .section .altinstr_replacement, "ax", @progbits
+.Lmemcpy_c:
        movq %rdi, %rax
 
        movl %edx, %ecx
@@ -35,8 +34,8 @@ memcpy_c:
        movl %edx, %ecx
        rep movsb
        ret
-       CFI_ENDPROC
-ENDPROC(memcpy_c)
+.Lmemcpy_e:
+       .previous
 
 ENTRY(__memcpy)
 ENTRY(memcpy)
@@ -128,16 +127,10 @@ ENDPROC(__memcpy)
         * It is also a lot simpler. Use this when possible:
         */
 
-       .section .altinstr_replacement, "ax"
-1:     .byte 0xeb                              /* jmp <disp8> */
-       .byte (memcpy_c - memcpy) - (2f - 1b)   /* offset */
-2:
-       .previous
-
        .section .altinstructions, "a"
        .align 8
        .quad memcpy
-       .quad 1b
+       .quad .Lmemcpy_c
        .byte X86_FEATURE_REP_GOOD
 
        /*
@@ -145,6 +138,6 @@ ENDPROC(__memcpy)
         * so it is silly to overwrite itself with nops - reboot is the
         * only outcome...
         */
-       .byte 2b - 1b
-       .byte 2b - 1b
+       .byte .Lmemcpy_e - .Lmemcpy_c
+       .byte .Lmemcpy_e - .Lmemcpy_c
        .previous

diff --git a/arch/x86/lib/memset_64.S b/arch/x86/lib/memset_64.S
index 2c5948116bd21731d4a85712fddd00d3943fd728..e88d3b81644a8736bcba56acaef56328c39e77ff 100644 (file)
--- a/arch/x86/lib/memset_64.S
+++ b/arch/x86/lib/memset_64.S
@@ -12,9 +12,8 @@
  * 
  * rax   original destination
  */    
-       ALIGN
-memset_c:
-       CFI_STARTPROC
+       .section .altinstr_replacement, "ax", @progbits
+.Lmemset_c:
        movq %rdi,%r9
        movl %edx,%r8d
        andl $7,%r8d
@@ -29,8 +28,8 @@ memset_c:
        rep stosb
        movq %r9,%rax
        ret
-       CFI_ENDPROC
-ENDPROC(memset_c)
+.Lmemset_e:
+       .previous
 
 ENTRY(memset)
 ENTRY(__memset)
@@ -118,16 +117,11 @@ ENDPROC(__memset)
 
 #include <asm/cpufeature.h>
 
-       .section .altinstr_replacement,"ax"
-1:     .byte 0xeb                              /* jmp <disp8> */
-       .byte (memset_c - memset) - (2f - 1b)   /* offset */
-2:
-       .previous
        .section .altinstructions,"a"
        .align 8
        .quad memset
-       .quad 1b
+       .quad .Lmemset_c
        .byte X86_FEATURE_REP_GOOD
        .byte .Lfinal - memset
-       .byte 2b - 1b
+       .byte .Lmemset_e - .Lmemset_c
        .previous