x86/mm, kexec: Fix memory corruption with SME on successive kexecs

After issuing successive kexecs it was found that the SHA hash failed
verification when booting the kexec'd kernel.  When SME is enabled, the
change from using pages that were marked encrypted to now being marked as
not encrypted (through new identify mapped page tables) results in memory
corruption if there are any cache entries for the previously encrypted
pages. This is because separate cache entries can exist for the same
physical location but tagged both with and without the encryption bit.

To prevent this, issue a wbinvd if SME is active before copying the pages
from the source location to the destination location to clear any possible
cache entry conflicts.

Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
Cc: <kexec@lists.infradead.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brijesh Singh <brijesh.singh@amd.com>
Cc: Dave Young <dyoung@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/e7fb8610af3a93e8f8ae6f214cd9249adc0df2b4.1501186516.git.thomas.lendacky@amd.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/arch/x86/include/asm/kexec.h b/arch/x86/include/asm/kexec.h
index e8183acf931faf7ec4cc444de386f9db826ec952..942c1f444da88ddeb182e57f582a068c15cb2717 100644 (file)
--- a/arch/x86/include/asm/kexec.h
+++ b/arch/x86/include/asm/kexec.h
@@ -147,7 +147,8 @@ unsigned long
 relocate_kernel(unsigned long indirection_page,
                unsigned long page_list,
                unsigned long start_address,
-               unsigned int preserve_context);
+               unsigned int preserve_context,
+               unsigned int sme_active);
 #endif
 
 #define ARCH_HAS_KIMAGE_ARCH

diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c
index 9cf8daacc0469ca1644eb7da98f762b7c58f88de..1f790cf9d38fe0e10e46eaf9b5bef945d25a9370 100644 (file)
--- a/arch/x86/kernel/machine_kexec_64.c
+++ b/arch/x86/kernel/machine_kexec_64.c
@@ -335,7 +335,8 @@ void machine_kexec(struct kimage *image)
        image->start = relocate_kernel((unsigned long)image->head,
                                       (unsigned long)page_list,
                                       image->start,
-                                      image->preserve_context);
+                                      image->preserve_context,
+                                      sme_active());
 
 #ifdef CONFIG_KEXEC_JUMP
        if (image->preserve_context)

diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S
index 98111b38ebfd6eb9949242c5aae7b18bbbdb4489..307d3bac5f04ece485ac1fe42226ee111c0c6e85 100644 (file)
--- a/arch/x86/kernel/relocate_kernel_64.S
+++ b/arch/x86/kernel/relocate_kernel_64.S
@@ -47,6 +47,7 @@ relocate_kernel:
         * %rsi page_list
         * %rdx start address
         * %rcx preserve_context
+        * %r8  sme_active
         */
 
        /* Save the CPU context, used for jumping back */
@@ -71,6 +72,9 @@ relocate_kernel:
        pushq $0
        popfq
 
+       /* Save SME active flag */
+       movq    %r8, %r12
+
        /*
         * get physical address of control page now
         * this is impossible after page table switch
@@ -132,6 +136,16 @@ identity_mapped:
        /* Flush the TLB (needed?) */
        movq    %r9, %cr3
 
+       /*
+        * If SME is active, there could be old encrypted cache line
+        * entries that will conflict with the now unencrypted memory
+        * used by kexec. Flush the caches before copying the kernel.
+        */
+       testq   %r12, %r12
+       jz 1f
+       wbinvd
+1:
+
        movq    %rcx, %r11
        call    swap_pages