x86/asm/crypto: Create stack frames in crypto functions

The crypto code has several callable non-leaf functions which don't
honor CONFIG_FRAME_POINTER, which can result in bad stack traces.

Create stack frames for them when CONFIG_FRAME_POINTER is enabled.

Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Arnaldo Carvalho de Melo <acme@kernel.org>
Cc: Bernd Petrovitsch <bernd@petrovitsch.priv.at>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Chris J Arges <chris.j.arges@canonical.com>
Cc: David S. Miller <davem@davemloft.net>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: Jiri Slaby <jslaby@suse.cz>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michal Marek <mmarek@suse.cz>
Cc: Namhyung Kim <namhyung@gmail.com>
Cc: Pedro Alves <palves@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: live-patching@vger.kernel.org
Link: http://lkml.kernel.org/r/6c20192bcf1102ae18ae5a242cabf30ce9b29895.1453405861.git.jpoimboe@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index c44cfedbe1c3d4eb2d57892d73bc4af927914217..383a6f84a060f103c7c43923ad84029e0ce2a9f1 100644 (file)
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -31,6 +31,7 @@
 
 #include <linux/linkage.h>
 #include <asm/inst.h>
+#include <asm/frame.h>
 
 /*
  * The following macros are used to move an (un)aligned 16 byte value to/from
@@ -1800,11 +1801,12 @@ ENDPROC(_key_expansion_256b)
  *                   unsigned int key_len)
  */
 ENTRY(aesni_set_key)
+       FRAME_BEGIN
 #ifndef __x86_64__
        pushl KEYP
-       movl 8(%esp), KEYP              # ctx
-       movl 12(%esp), UKEYP            # in_key
-       movl 16(%esp), %edx             # key_len
+       movl (FRAME_OFFSET+8)(%esp), KEYP       # ctx
+       movl (FRAME_OFFSET+12)(%esp), UKEYP     # in_key
+       movl (FRAME_OFFSET+16)(%esp), %edx      # key_len
 #endif
        movups (UKEYP), %xmm0           # user key (first 16 bytes)
        movaps %xmm0, (KEYP)
@@ -1905,6 +1907,7 @@ ENTRY(aesni_set_key)
 #ifndef __x86_64__
        popl KEYP
 #endif
+       FRAME_END
        ret
 ENDPROC(aesni_set_key)
 
@@ -1912,12 +1915,13 @@ ENDPROC(aesni_set_key)
  * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
  */
 ENTRY(aesni_enc)
+       FRAME_BEGIN
 #ifndef __x86_64__
        pushl KEYP
        pushl KLEN
-       movl 12(%esp), KEYP
-       movl 16(%esp), OUTP
-       movl 20(%esp), INP
+       movl (FRAME_OFFSET+12)(%esp), KEYP      # ctx
+       movl (FRAME_OFFSET+16)(%esp), OUTP      # dst
+       movl (FRAME_OFFSET+20)(%esp), INP       # src
 #endif
        movl 480(KEYP), KLEN            # key length
        movups (INP), STATE             # input
@@ -1927,6 +1931,7 @@ ENTRY(aesni_enc)
        popl KLEN
        popl KEYP
 #endif
+       FRAME_END
        ret
 ENDPROC(aesni_enc)
 
@@ -2101,12 +2106,13 @@ ENDPROC(_aesni_enc4)
  * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
  */
 ENTRY(aesni_dec)
+       FRAME_BEGIN
 #ifndef __x86_64__
        pushl KEYP
        pushl KLEN
-       movl 12(%esp), KEYP
-       movl 16(%esp), OUTP
-       movl 20(%esp), INP
+       movl (FRAME_OFFSET+12)(%esp), KEYP      # ctx
+       movl (FRAME_OFFSET+16)(%esp), OUTP      # dst
+       movl (FRAME_OFFSET+20)(%esp), INP       # src
 #endif
        mov 480(KEYP), KLEN             # key length
        add $240, KEYP
@@ -2117,6 +2123,7 @@ ENTRY(aesni_dec)
        popl KLEN
        popl KEYP
 #endif
+       FRAME_END
        ret
 ENDPROC(aesni_dec)
 
@@ -2292,14 +2299,15 @@ ENDPROC(_aesni_dec4)
  *                   size_t len)
  */
 ENTRY(aesni_ecb_enc)
+       FRAME_BEGIN
 #ifndef __x86_64__
        pushl LEN
        pushl KEYP
        pushl KLEN
-       movl 16(%esp), KEYP
-       movl 20(%esp), OUTP
-       movl 24(%esp), INP
-       movl 28(%esp), LEN
+       movl (FRAME_OFFSET+16)(%esp), KEYP      # ctx
+       movl (FRAME_OFFSET+20)(%esp), OUTP      # dst
+       movl (FRAME_OFFSET+24)(%esp), INP       # src
+       movl (FRAME_OFFSET+28)(%esp), LEN       # len
 #endif
        test LEN, LEN           # check length
        jz .Lecb_enc_ret
@@ -2342,6 +2350,7 @@ ENTRY(aesni_ecb_enc)
        popl KEYP
        popl LEN
 #endif
+       FRAME_END
        ret
 ENDPROC(aesni_ecb_enc)
 
@@ -2350,14 +2359,15 @@ ENDPROC(aesni_ecb_enc)
  *                   size_t len);
  */
 ENTRY(aesni_ecb_dec)
+       FRAME_BEGIN
 #ifndef __x86_64__
        pushl LEN
        pushl KEYP
        pushl KLEN
-       movl 16(%esp), KEYP
-       movl 20(%esp), OUTP
-       movl 24(%esp), INP
-       movl 28(%esp), LEN
+       movl (FRAME_OFFSET+16)(%esp), KEYP      # ctx
+       movl (FRAME_OFFSET+20)(%esp), OUTP      # dst
+       movl (FRAME_OFFSET+24)(%esp), INP       # src
+       movl (FRAME_OFFSET+28)(%esp), LEN       # len
 #endif
        test LEN, LEN
        jz .Lecb_dec_ret
@@ -2401,6 +2411,7 @@ ENTRY(aesni_ecb_dec)
        popl KEYP
        popl LEN
 #endif
+       FRAME_END
        ret
 ENDPROC(aesni_ecb_dec)
 
@@ -2409,16 +2420,17 @@ ENDPROC(aesni_ecb_dec)
  *                   size_t len, u8 *iv)
  */
 ENTRY(aesni_cbc_enc)
+       FRAME_BEGIN
 #ifndef __x86_64__
        pushl IVP
        pushl LEN
        pushl KEYP
        pushl KLEN
-       movl 20(%esp), KEYP
-       movl 24(%esp), OUTP
-       movl 28(%esp), INP
-       movl 32(%esp), LEN
-       movl 36(%esp), IVP
+       movl (FRAME_OFFSET+20)(%esp), KEYP      # ctx
+       movl (FRAME_OFFSET+24)(%esp), OUTP      # dst
+       movl (FRAME_OFFSET+28)(%esp), INP       # src
+       movl (FRAME_OFFSET+32)(%esp), LEN       # len
+       movl (FRAME_OFFSET+36)(%esp), IVP       # iv
 #endif
        cmp $16, LEN
        jb .Lcbc_enc_ret
@@ -2443,6 +2455,7 @@ ENTRY(aesni_cbc_enc)
        popl LEN
        popl IVP
 #endif
+       FRAME_END
        ret
 ENDPROC(aesni_cbc_enc)
 
@@ -2451,16 +2464,17 @@ ENDPROC(aesni_cbc_enc)
  *                   size_t len, u8 *iv)
  */
 ENTRY(aesni_cbc_dec)
+       FRAME_BEGIN
 #ifndef __x86_64__
        pushl IVP
        pushl LEN
        pushl KEYP
        pushl KLEN
-       movl 20(%esp), KEYP
-       movl 24(%esp), OUTP
-       movl 28(%esp), INP
-       movl 32(%esp), LEN
-       movl 36(%esp), IVP
+       movl (FRAME_OFFSET+20)(%esp), KEYP      # ctx
+       movl (FRAME_OFFSET+24)(%esp), OUTP      # dst
+       movl (FRAME_OFFSET+28)(%esp), INP       # src
+       movl (FRAME_OFFSET+32)(%esp), LEN       # len
+       movl (FRAME_OFFSET+36)(%esp), IVP       # iv
 #endif
        cmp $16, LEN
        jb .Lcbc_dec_just_ret
@@ -2534,6 +2548,7 @@ ENTRY(aesni_cbc_dec)
        popl LEN
        popl IVP
 #endif
+       FRAME_END
        ret
 ENDPROC(aesni_cbc_dec)
 
@@ -2600,6 +2615,7 @@ ENDPROC(_aesni_inc)
  *                   size_t len, u8 *iv)
  */
 ENTRY(aesni_ctr_enc)
+       FRAME_BEGIN
        cmp $16, LEN
        jb .Lctr_enc_just_ret
        mov 480(KEYP), KLEN
@@ -2653,6 +2669,7 @@ ENTRY(aesni_ctr_enc)
 .Lctr_enc_ret:
        movups IV, (IVP)
 .Lctr_enc_just_ret:
+       FRAME_END
        ret
 ENDPROC(aesni_ctr_enc)
 
@@ -2679,6 +2696,7 @@ ENDPROC(aesni_ctr_enc)
  *                      bool enc, u8 *iv)
  */
 ENTRY(aesni_xts_crypt8)
+       FRAME_BEGIN
        cmpb $0, %cl
        movl $0, %ecx
        movl $240, %r10d
@@ -2779,6 +2797,7 @@ ENTRY(aesni_xts_crypt8)
        pxor INC, STATE4
        movdqu STATE4, 0x70(OUTP)
 
+       FRAME_END
        ret
 ENDPROC(aesni_xts_crypt8)
 

diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
index ce71f9212409f16326ffc92efec812c2be7d731c..aa9e8bd163f61b0440372bfbc26e956f71bf95d3 100644 (file)
--- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
@@ -16,6 +16,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/frame.h>
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -726,6 +727,7 @@ __camellia_enc_blk16:
         *      %xmm0..%xmm15: 16 encrypted blocks, order swapped:
         *       7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8
         */
+       FRAME_BEGIN
 
        leaq 8 * 16(%rax), %rcx;
 
@@ -780,6 +782,7 @@ __camellia_enc_blk16:
                    %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
                    %xmm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 16(%rax));
 
+       FRAME_END
        ret;
 
 .align 8
@@ -812,6 +815,7 @@ __camellia_dec_blk16:
         *      %xmm0..%xmm15: 16 plaintext blocks, order swapped:
         *       7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8
         */
+       FRAME_BEGIN
 
        leaq 8 * 16(%rax), %rcx;
 
@@ -865,6 +869,7 @@ __camellia_dec_blk16:
                    %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
                    %xmm15, (key_table)(CTX), (%rax), 1 * 16(%rax));
 
+       FRAME_END
        ret;
 
 .align 8
@@ -890,6 +895,7 @@ ENTRY(camellia_ecb_enc_16way)
         *      %rsi: dst (16 blocks)
         *      %rdx: src (16 blocks)
         */
+        FRAME_BEGIN
 
        inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
                     %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
@@ -904,6 +910,7 @@ ENTRY(camellia_ecb_enc_16way)
                     %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
                     %xmm8, %rsi);
 
+       FRAME_END
        ret;
 ENDPROC(camellia_ecb_enc_16way)
 
@@ -913,6 +920,7 @@ ENTRY(camellia_ecb_dec_16way)
         *      %rsi: dst (16 blocks)
         *      %rdx: src (16 blocks)
         */
+        FRAME_BEGIN
 
        cmpl $16, key_length(CTX);
        movl $32, %r8d;
@@ -932,6 +940,7 @@ ENTRY(camellia_ecb_dec_16way)
                     %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
                     %xmm8, %rsi);
 
+       FRAME_END
        ret;
 ENDPROC(camellia_ecb_dec_16way)
 
@@ -941,6 +950,7 @@ ENTRY(camellia_cbc_dec_16way)
         *      %rsi: dst (16 blocks)
         *      %rdx: src (16 blocks)
         */
+       FRAME_BEGIN
 
        cmpl $16, key_length(CTX);
        movl $32, %r8d;
@@ -981,6 +991,7 @@ ENTRY(camellia_cbc_dec_16way)
                     %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
                     %xmm8, %rsi);
 
+       FRAME_END
        ret;
 ENDPROC(camellia_cbc_dec_16way)
 
@@ -997,6 +1008,7 @@ ENTRY(camellia_ctr_16way)
         *      %rdx: src (16 blocks)
         *      %rcx: iv (little endian, 128bit)
         */
+       FRAME_BEGIN
 
        subq $(16 * 16), %rsp;
        movq %rsp, %rax;
@@ -1092,6 +1104,7 @@ ENTRY(camellia_ctr_16way)
                     %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
                     %xmm8, %rsi);
 
+       FRAME_END
        ret;
 ENDPROC(camellia_ctr_16way)
 
@@ -1112,6 +1125,7 @@ camellia_xts_crypt_16way:
         *      %r8: index for input whitening key
         *      %r9: pointer to  __camellia_enc_blk16 or __camellia_dec_blk16
         */
+       FRAME_BEGIN
 
        subq $(16 * 16), %rsp;
        movq %rsp, %rax;
@@ -1234,6 +1248,7 @@ camellia_xts_crypt_16way:
                     %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
                     %xmm8, %rsi);
 
+       FRAME_END
        ret;
 ENDPROC(camellia_xts_crypt_16way)
 

diff --git a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
index 0e0b8863a34bd168c618941407b2890c385a442d..16186c18656dfdc67fc7433a48dcc51ee62d9bea 100644 (file)
--- a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
@@ -11,6 +11,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/frame.h>
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -766,6 +767,7 @@ __camellia_enc_blk32:
         *      %ymm0..%ymm15: 32 encrypted blocks, order swapped:
         *       7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8
         */
+       FRAME_BEGIN
 
        leaq 8 * 32(%rax), %rcx;
 
@@ -820,6 +822,7 @@ __camellia_enc_blk32:
                    %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
                    %ymm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 32(%rax));
 
+       FRAME_END
        ret;
 
 .align 8
@@ -852,6 +855,7 @@ __camellia_dec_blk32:
         *      %ymm0..%ymm15: 16 plaintext blocks, order swapped:
         *       7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8
         */
+       FRAME_BEGIN
 
        leaq 8 * 32(%rax), %rcx;
 
@@ -905,6 +909,7 @@ __camellia_dec_blk32:
                    %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
                    %ymm15, (key_table)(CTX), (%rax), 1 * 32(%rax));
 
+       FRAME_END
        ret;
 
 .align 8
@@ -930,6 +935,7 @@ ENTRY(camellia_ecb_enc_32way)
         *      %rsi: dst (32 blocks)
         *      %rdx: src (32 blocks)
         */
+       FRAME_BEGIN
 
        vzeroupper;
 
@@ -948,6 +954,7 @@ ENTRY(camellia_ecb_enc_32way)
 
        vzeroupper;
 
+       FRAME_END
        ret;
 ENDPROC(camellia_ecb_enc_32way)
 
@@ -957,6 +964,7 @@ ENTRY(camellia_ecb_dec_32way)
         *      %rsi: dst (32 blocks)
         *      %rdx: src (32 blocks)
         */
+       FRAME_BEGIN
 
        vzeroupper;
 
@@ -980,6 +988,7 @@ ENTRY(camellia_ecb_dec_32way)
 
        vzeroupper;
 
+       FRAME_END
        ret;
 ENDPROC(camellia_ecb_dec_32way)
 
@@ -989,6 +998,7 @@ ENTRY(camellia_cbc_dec_32way)
         *      %rsi: dst (32 blocks)
         *      %rdx: src (32 blocks)
         */
+       FRAME_BEGIN
 
        vzeroupper;
 
@@ -1046,6 +1056,7 @@ ENTRY(camellia_cbc_dec_32way)
 
        vzeroupper;
 
+       FRAME_END
        ret;
 ENDPROC(camellia_cbc_dec_32way)
 
@@ -1070,6 +1081,7 @@ ENTRY(camellia_ctr_32way)
         *      %rdx: src (32 blocks)
         *      %rcx: iv (little endian, 128bit)
         */
+       FRAME_BEGIN
 
        vzeroupper;
 
@@ -1184,6 +1196,7 @@ ENTRY(camellia_ctr_32way)
 
        vzeroupper;
 
+       FRAME_END
        ret;
 ENDPROC(camellia_ctr_32way)
 
@@ -1216,6 +1229,7 @@ camellia_xts_crypt_32way:
         *      %r8: index for input whitening key
         *      %r9: pointer to  __camellia_enc_blk32 or __camellia_dec_blk32
         */
+       FRAME_BEGIN
 
        vzeroupper;
 
@@ -1349,6 +1363,7 @@ camellia_xts_crypt_32way:
 
        vzeroupper;
 
+       FRAME_END
        ret;
 ENDPROC(camellia_xts_crypt_32way)
 

diff --git a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
index c35fd5d6ecd26f4b01180484c520c19a10637389..14fa1966bf01d4b89eb68456c21a5e750884d785 100644 (file)
--- a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
@@ -24,6 +24,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/frame.h>
 
 .file "cast5-avx-x86_64-asm_64.S"
 
@@ -365,6 +366,7 @@ ENTRY(cast5_ecb_enc_16way)
         *      %rsi: dst
         *      %rdx: src
         */
+       FRAME_BEGIN
 
        movq %rsi, %r11;
 
@@ -388,6 +390,7 @@ ENTRY(cast5_ecb_enc_16way)
        vmovdqu RR4, (6*4*4)(%r11);
        vmovdqu RL4, (7*4*4)(%r11);
 
+       FRAME_END
        ret;
 ENDPROC(cast5_ecb_enc_16way)
 
@@ -398,6 +401,7 @@ ENTRY(cast5_ecb_dec_16way)
         *      %rdx: src
         */
 
+       FRAME_BEGIN
        movq %rsi, %r11;
 
        vmovdqu (0*4*4)(%rdx), RL1;
@@ -420,6 +424,7 @@ ENTRY(cast5_ecb_dec_16way)
        vmovdqu RR4, (6*4*4)(%r11);
        vmovdqu RL4, (7*4*4)(%r11);
 
+       FRAME_END
        ret;
 ENDPROC(cast5_ecb_dec_16way)
 
@@ -429,6 +434,7 @@ ENTRY(cast5_cbc_dec_16way)
         *      %rsi: dst
         *      %rdx: src
         */
+       FRAME_BEGIN
 
        pushq %r12;
 
@@ -469,6 +475,7 @@ ENTRY(cast5_cbc_dec_16way)
 
        popq %r12;
 
+       FRAME_END
        ret;
 ENDPROC(cast5_cbc_dec_16way)
 
@@ -479,6 +486,7 @@ ENTRY(cast5_ctr_16way)
         *      %rdx: src
         *      %rcx: iv (big endian, 64bit)
         */
+       FRAME_BEGIN
 
        pushq %r12;
 
@@ -542,5 +550,6 @@ ENTRY(cast5_ctr_16way)
 
        popq %r12;
 
+       FRAME_END
        ret;
 ENDPROC(cast5_ctr_16way)

diff --git a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
index e3531f833951bbc2535d1f6c51fe58d428c3b637..c419389889cddf24d60080940148bddecc8f6889 100644 (file)
--- a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
@@ -24,6 +24,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/frame.h>
 #include "glue_helper-asm-avx.S"
 
 .file "cast6-avx-x86_64-asm_64.S"
@@ -349,6 +350,7 @@ ENTRY(cast6_ecb_enc_8way)
         *      %rsi: dst
         *      %rdx: src
         */
+       FRAME_BEGIN
 
        movq %rsi, %r11;
 
@@ -358,6 +360,7 @@ ENTRY(cast6_ecb_enc_8way)
 
        store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+       FRAME_END
        ret;
 ENDPROC(cast6_ecb_enc_8way)
 
@@ -367,6 +370,7 @@ ENTRY(cast6_ecb_dec_8way)
         *      %rsi: dst
         *      %rdx: src
         */
+       FRAME_BEGIN
 
        movq %rsi, %r11;
 
@@ -376,6 +380,7 @@ ENTRY(cast6_ecb_dec_8way)
 
        store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+       FRAME_END
        ret;
 ENDPROC(cast6_ecb_dec_8way)
 
@@ -385,6 +390,7 @@ ENTRY(cast6_cbc_dec_8way)
         *      %rsi: dst
         *      %rdx: src
         */
+       FRAME_BEGIN
 
        pushq %r12;
 
@@ -399,6 +405,7 @@ ENTRY(cast6_cbc_dec_8way)
 
        popq %r12;
 
+       FRAME_END
        ret;
 ENDPROC(cast6_cbc_dec_8way)
 
@@ -409,6 +416,7 @@ ENTRY(cast6_ctr_8way)
         *      %rdx: src
         *      %rcx: iv (little endian, 128bit)
         */
+       FRAME_BEGIN
 
        pushq %r12;
 
@@ -424,6 +432,7 @@ ENTRY(cast6_ctr_8way)
 
        popq %r12;
 
+       FRAME_END
        ret;
 ENDPROC(cast6_ctr_8way)
 
@@ -434,6 +443,7 @@ ENTRY(cast6_xts_enc_8way)
         *      %rdx: src
         *      %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
         */
+       FRAME_BEGIN
 
        movq %rsi, %r11;
 
@@ -446,6 +456,7 @@ ENTRY(cast6_xts_enc_8way)
        /* dst <= regs xor IVs(in dst) */
        store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+       FRAME_END
        ret;
 ENDPROC(cast6_xts_enc_8way)
 
@@ -456,6 +467,7 @@ ENTRY(cast6_xts_dec_8way)
         *      %rdx: src
         *      %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
         */
+       FRAME_BEGIN
 
        movq %rsi, %r11;
 
@@ -468,5 +480,6 @@ ENTRY(cast6_xts_dec_8way)
        /* dst <= regs xor IVs(in dst) */
        store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+       FRAME_END
        ret;
 ENDPROC(cast6_xts_dec_8way)

diff --git a/arch/x86/crypto/ghash-clmulni-intel_asm.S b/arch/x86/crypto/ghash-clmulni-intel_asm.S
index 5d1e0075ac24fa5eae975930647f6fc14c5b256d..eed55c8cca4f8e64b2ef203453ab8c2b34e75722 100644 (file)
--- a/arch/x86/crypto/ghash-clmulni-intel_asm.S
+++ b/arch/x86/crypto/ghash-clmulni-intel_asm.S
@@ -18,6 +18,7 @@
 
 #include <linux/linkage.h>
 #include <asm/inst.h>
+#include <asm/frame.h>
 
 .data
 
@@ -94,6 +95,7 @@ ENDPROC(__clmul_gf128mul_ble)
 
 /* void clmul_ghash_mul(char *dst, const u128 *shash) */
 ENTRY(clmul_ghash_mul)
+       FRAME_BEGIN
        movups (%rdi), DATA
        movups (%rsi), SHASH
        movaps .Lbswap_mask, BSWAP
@@ -101,6 +103,7 @@ ENTRY(clmul_ghash_mul)
        call __clmul_gf128mul_ble
        PSHUFB_XMM BSWAP DATA
        movups DATA, (%rdi)
+       FRAME_END
        ret
 ENDPROC(clmul_ghash_mul)
 
@@ -109,6 +112,7 @@ ENDPROC(clmul_ghash_mul)
  *                        const u128 *shash);
  */
 ENTRY(clmul_ghash_update)
+       FRAME_BEGIN
        cmp $16, %rdx
        jb .Lupdate_just_ret    # check length
        movaps .Lbswap_mask, BSWAP
@@ -128,5 +132,6 @@ ENTRY(clmul_ghash_update)
        PSHUFB_XMM BSWAP DATA
        movups DATA, (%rdi)
 .Lupdate_just_ret:
+       FRAME_END
        ret
 ENDPROC(clmul_ghash_update)

diff --git a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
index 2f202f49872b2f6a6ed4ed5a6c6d01a8f0958660..8be571808342b85df775ac897a2e38ad9f35cab4 100644 (file)
--- a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
@@ -24,6 +24,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/frame.h>
 #include "glue_helper-asm-avx.S"
 
 .file "serpent-avx-x86_64-asm_64.S"
@@ -681,6 +682,7 @@ ENTRY(serpent_ecb_enc_8way_avx)
         *      %rsi: dst
         *      %rdx: src
         */
+       FRAME_BEGIN
 
        load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
@@ -688,6 +690,7 @@ ENTRY(serpent_ecb_enc_8way_avx)
 
        store_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+       FRAME_END
        ret;
 ENDPROC(serpent_ecb_enc_8way_avx)
 
@@ -697,6 +700,7 @@ ENTRY(serpent_ecb_dec_8way_avx)
         *      %rsi: dst
         *      %rdx: src
         */
+       FRAME_BEGIN
 
        load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
@@ -704,6 +708,7 @@ ENTRY(serpent_ecb_dec_8way_avx)
 
        store_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
 
+       FRAME_END
        ret;
 ENDPROC(serpent_ecb_dec_8way_avx)
 
@@ -713,6 +718,7 @@ ENTRY(serpent_cbc_dec_8way_avx)
         *      %rsi: dst
         *      %rdx: src
         */
+       FRAME_BEGIN
 
        load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
@@ -720,6 +726,7 @@ ENTRY(serpent_cbc_dec_8way_avx)
 
        store_cbc_8way(%rdx, %rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
 
+       FRAME_END
        ret;
 ENDPROC(serpent_cbc_dec_8way_avx)
 
@@ -730,6 +737,7 @@ ENTRY(serpent_ctr_8way_avx)
         *      %rdx: src
         *      %rcx: iv (little endian, 128bit)
         */
+       FRAME_BEGIN
 
        load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2,
                      RD2, RK0, RK1, RK2);
@@ -738,6 +746,7 @@ ENTRY(serpent_ctr_8way_avx)
 
        store_ctr_8way(%rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+       FRAME_END
        ret;
 ENDPROC(serpent_ctr_8way_avx)
 
@@ -748,6 +757,7 @@ ENTRY(serpent_xts_enc_8way_avx)
         *      %rdx: src
         *      %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
         */
+       FRAME_BEGIN
 
        /* regs <= src, dst <= IVs, regs <= regs xor IVs */
        load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2,
@@ -758,6 +768,7 @@ ENTRY(serpent_xts_enc_8way_avx)
        /* dst <= regs xor IVs(in dst) */
        store_xts_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+       FRAME_END
        ret;
 ENDPROC(serpent_xts_enc_8way_avx)
 
@@ -768,6 +779,7 @@ ENTRY(serpent_xts_dec_8way_avx)
         *      %rdx: src
         *      %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
         */
+       FRAME_BEGIN
 
        /* regs <= src, dst <= IVs, regs <= regs xor IVs */
        load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2,
@@ -778,5 +790,6 @@ ENTRY(serpent_xts_dec_8way_avx)
        /* dst <= regs xor IVs(in dst) */
        store_xts_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
 
+       FRAME_END
        ret;
 ENDPROC(serpent_xts_dec_8way_avx)

diff --git a/arch/x86/crypto/serpent-avx2-asm_64.S b/arch/x86/crypto/serpent-avx2-asm_64.S
index b222085cccac756efc94be3e6780a67c9dd61779..97c48add33ed965e14561f663d98061fd3670ccd 100644 (file)
--- a/arch/x86/crypto/serpent-avx2-asm_64.S
+++ b/arch/x86/crypto/serpent-avx2-asm_64.S
@@ -15,6 +15,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/frame.h>
 #include "glue_helper-asm-avx2.S"
 
 .file "serpent-avx2-asm_64.S"
@@ -673,6 +674,7 @@ ENTRY(serpent_ecb_enc_16way)
         *      %rsi: dst
         *      %rdx: src
         */
+       FRAME_BEGIN
 
        vzeroupper;
 
@@ -684,6 +686,7 @@ ENTRY(serpent_ecb_enc_16way)
 
        vzeroupper;
 
+       FRAME_END
        ret;
 ENDPROC(serpent_ecb_enc_16way)
 
@@ -693,6 +696,7 @@ ENTRY(serpent_ecb_dec_16way)
         *      %rsi: dst
         *      %rdx: src
         */
+       FRAME_BEGIN
 
        vzeroupper;
 
@@ -704,6 +708,7 @@ ENTRY(serpent_ecb_dec_16way)
 
        vzeroupper;
 
+       FRAME_END
        ret;
 ENDPROC(serpent_ecb_dec_16way)
 
@@ -713,6 +718,7 @@ ENTRY(serpent_cbc_dec_16way)
         *      %rsi: dst
         *      %rdx: src
         */
+       FRAME_BEGIN
 
        vzeroupper;
 
@@ -725,6 +731,7 @@ ENTRY(serpent_cbc_dec_16way)
 
        vzeroupper;
 
+       FRAME_END
        ret;
 ENDPROC(serpent_cbc_dec_16way)
 
@@ -735,6 +742,7 @@ ENTRY(serpent_ctr_16way)
         *      %rdx: src (16 blocks)
         *      %rcx: iv (little endian, 128bit)
         */
+       FRAME_BEGIN
 
        vzeroupper;
 
@@ -748,6 +756,7 @@ ENTRY(serpent_ctr_16way)
 
        vzeroupper;
 
+       FRAME_END
        ret;
 ENDPROC(serpent_ctr_16way)
 
@@ -758,6 +767,7 @@ ENTRY(serpent_xts_enc_16way)
         *      %rdx: src (16 blocks)
         *      %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
         */
+       FRAME_BEGIN
 
        vzeroupper;
 
@@ -772,6 +782,7 @@ ENTRY(serpent_xts_enc_16way)
 
        vzeroupper;
 
+       FRAME_END
        ret;
 ENDPROC(serpent_xts_enc_16way)
 
@@ -782,6 +793,7 @@ ENTRY(serpent_xts_dec_16way)
         *      %rdx: src (16 blocks)
         *      %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
         */
+       FRAME_BEGIN
 
        vzeroupper;
 
@@ -796,5 +808,6 @@ ENTRY(serpent_xts_dec_16way)
 
        vzeroupper;
 
+       FRAME_END
        ret;
 ENDPROC(serpent_xts_dec_16way)

diff --git a/arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S
index 672eaeb258a503d9671560b1cff11e599721271b..96df6a39d7e240d241d1c46077a1336d2717f80d 100644 (file)
--- a/arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S
+++ b/arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S
@@ -52,6 +52,7 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <linux/linkage.h>
+#include <asm/frame.h>
 #include "sha1_mb_mgr_datastruct.S"
 
 
@@ -103,6 +104,7 @@ offset = \_offset
 # JOB* sha1_mb_mgr_flush_avx2(MB_MGR *state)
 # arg 1 : rcx : state
 ENTRY(sha1_mb_mgr_flush_avx2)
+       FRAME_BEGIN
        push    %rbx
 
        # If bit (32+3) is set, then all lanes are empty
@@ -212,6 +214,7 @@ len_is_0:
 
 return:
        pop     %rbx
+       FRAME_END
        ret
 
 return_null:

diff --git a/arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S
index c3b944781d3f971556794b4b9fbbc36e5e0f5b83..1435acf46f22d33798bca3dfdd7d4fea798152d2 100644 (file)
--- a/arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S
+++ b/arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S
@@ -53,6 +53,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/frame.h>
 #include "sha1_mb_mgr_datastruct.S"
 
 
@@ -98,6 +99,7 @@ lane_data       = %r10
 # arg 1 : rcx : state
 # arg 2 : rdx : job
 ENTRY(sha1_mb_mgr_submit_avx2)
+       FRAME_BEGIN
        push    %rbx
        push    %r12
 
@@ -192,6 +194,7 @@ len_is_0:
 return:
        pop     %r12
        pop     %rbx
+       FRAME_END
        ret
 
 return_null:

diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
index 05058134c443176ec717cbdec6eb50235530fb03..dc66273e610d972d54efbd3f4b6b6f865015c92b 100644 (file)
--- a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
@@ -24,6 +24,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/frame.h>
 #include "glue_helper-asm-avx.S"
 
 .file "twofish-avx-x86_64-asm_64.S"
@@ -333,6 +334,7 @@ ENTRY(twofish_ecb_enc_8way)
         *      %rsi: dst
         *      %rdx: src
         */
+       FRAME_BEGIN
 
        movq %rsi, %r11;
 
@@ -342,6 +344,7 @@ ENTRY(twofish_ecb_enc_8way)
 
        store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
 
+       FRAME_END
        ret;
 ENDPROC(twofish_ecb_enc_8way)
 
@@ -351,6 +354,7 @@ ENTRY(twofish_ecb_dec_8way)
         *      %rsi: dst
         *      %rdx: src
         */
+       FRAME_BEGIN
 
        movq %rsi, %r11;
 
@@ -360,6 +364,7 @@ ENTRY(twofish_ecb_dec_8way)
 
        store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+       FRAME_END
        ret;
 ENDPROC(twofish_ecb_dec_8way)
 
@@ -369,6 +374,7 @@ ENTRY(twofish_cbc_dec_8way)
         *      %rsi: dst
         *      %rdx: src
         */
+       FRAME_BEGIN
 
        pushq %r12;
 
@@ -383,6 +389,7 @@ ENTRY(twofish_cbc_dec_8way)
 
        popq %r12;
 
+       FRAME_END
        ret;
 ENDPROC(twofish_cbc_dec_8way)
 
@@ -393,6 +400,7 @@ ENTRY(twofish_ctr_8way)
         *      %rdx: src
         *      %rcx: iv (little endian, 128bit)
         */
+       FRAME_BEGIN
 
        pushq %r12;
 
@@ -408,6 +416,7 @@ ENTRY(twofish_ctr_8way)
 
        popq %r12;
 
+       FRAME_END
        ret;
 ENDPROC(twofish_ctr_8way)
 
@@ -418,6 +427,7 @@ ENTRY(twofish_xts_enc_8way)
         *      %rdx: src
         *      %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
         */
+       FRAME_BEGIN
 
        movq %rsi, %r11;
 
@@ -430,6 +440,7 @@ ENTRY(twofish_xts_enc_8way)
        /* dst <= regs xor IVs(in dst) */
        store_xts_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
 
+       FRAME_END
        ret;
 ENDPROC(twofish_xts_enc_8way)
 
@@ -440,6 +451,7 @@ ENTRY(twofish_xts_dec_8way)
         *      %rdx: src
         *      %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
         */
+       FRAME_BEGIN
 
        movq %rsi, %r11;
 
@@ -452,5 +464,6 @@ ENTRY(twofish_xts_dec_8way)
        /* dst <= regs xor IVs(in dst) */
        store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+       FRAME_END
        ret;
 ENDPROC(twofish_xts_dec_8way)