arm64: Relax the kernel cache requirements for boot

With system caches for the host OS or architected caches for guest OS we
cannot easily guarantee that there are no dirty or stale cache lines for
the areas of memory written by the kernel during boot with the MMU off
(therefore non-cacheable accesses).

This patch adds the necessary cache maintenance during boot and relaxes
the booting requirements.

Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/Documentation/arm64/booting.txt b/Documentation/arm64/booting.txt
index a9691cc48fe3943bb42fc40797c524cd0843690e..beb754e87c6552e31077d08cb1f3e99d24a09c47 100644 (file)
--- a/Documentation/arm64/booting.txt
+++ b/Documentation/arm64/booting.txt
@@ -111,8 +111,14 @@ Before jumping into the kernel, the following conditions must be met:
 - Caches, MMUs
   The MMU must be off.
   Instruction cache may be on or off.
-  Data cache must be off and invalidated.
-  External caches (if present) must be configured and disabled.
+  The address range corresponding to the loaded kernel image must be
+  cleaned to the PoC. In the presence of a system cache or other
+  coherent masters with caches enabled, this will typically require
+  cache maintenance by VA rather than set/way operations.
+  System caches which respect the architected cache maintenance by VA
+  operations must be configured and may be enabled.
+  System caches which do not respect architected cache maintenance by VA
+  operations (not recommended) must be configured and disabled.
 
 - Architected timers
   CNTFRQ must be programmed with the timer frequency and CNTVOFF must

diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 61035d6814cbcca9a7bb9be06e92e44446256f25..26109682d2fad765c50fcf56b12a2c0b7ef8c6eb 100644 (file)
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -26,6 +26,7 @@
 #include <asm/assembler.h>
 #include <asm/ptrace.h>
 #include <asm/asm-offsets.h>
+#include <asm/cache.h>
 #include <asm/cputype.h>
 #include <asm/memory.h>
 #include <asm/thread_info.h>
@@ -229,7 +230,11 @@ ENTRY(set_cpu_boot_mode_flag)
        cmp     w20, #BOOT_CPU_MODE_EL2
        b.ne    1f
        add     x1, x1, #4
-1:     str     w20, [x1]                       // This CPU has booted in EL1
+1:     dc      cvac, x1                        // Clean potentially dirty cache line
+       dsb     sy
+       str     w20, [x1]                       // This CPU has booted in EL1
+       dc      civac, x1                       // Clean&invalidate potentially stale cache line
+       dsb     sy
        ret
 ENDPROC(set_cpu_boot_mode_flag)
 
@@ -240,8 +245,9 @@ ENDPROC(set_cpu_boot_mode_flag)
  * This is not in .bss, because we set it sufficiently early that the boot-time
  * zeroing of .bss would clobber it.
  */
-       .pushsection    .data
+       .pushsection    .data..cacheline_aligned
 ENTRY(__boot_cpu_mode)
+       .align  L1_CACHE_SHIFT
        .long   BOOT_CPU_MODE_EL2
        .long   0
        .popsection
@@ -408,6 +414,15 @@ ENDPROC(__calc_phys_offset)
  */
 __create_page_tables:
        pgtbl   x25, x26, x24                   // idmap_pg_dir and swapper_pg_dir addresses
+       mov     x27, lr
+
+       /*
+        * Invalidate the idmap and swapper page tables to avoid potential
+        * dirty cache lines being evicted.
+        */
+       mov     x0, x25
+       add     x1, x26, #SWAPPER_DIR_SIZE
+       bl      __inval_cache_range
 
        /*
         * Clear the idmap and swapper page tables.
@@ -470,6 +485,17 @@ __create_page_tables:
        add     x0, x26, #2 * PAGE_SIZE         // section table address
        create_pgd_entry x26, x0, x5, x6, x7
 #endif
+
+       /*
+        * Since the page tables have been populated with non-cacheable
+        * accesses (MMU disabled), invalidate the idmap and swapper page
+        * tables again to remove any speculatively loaded cache lines.
+        */
+       mov     x0, x25
+       add     x1, x26, #SWAPPER_DIR_SIZE
+       bl      __inval_cache_range
+
+       mov     lr, x27
        ret
 ENDPROC(__create_page_tables)
        .ltorg

diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index c46f48b33c1409d8ea6dfe3c48327a00b6376a06..e803a62e0e45e317e72def339358a4c78f608f48 100644 (file)
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -167,6 +167,14 @@ ENTRY(__flush_dcache_area)
        ret
 ENDPROC(__flush_dcache_area)
 
+/*
+ *     __inval_cache_range(start, end)
+ *     - start   - start address of region
+ *     - end     - end address of region
+ */
+ENTRY(__inval_cache_range)
+       /* FALLTHROUGH */
+
 /*
  *     __dma_inv_range(start, end)
  *     - start   - virtual start address of region
@@ -183,6 +191,7 @@ __dma_inv_range:
        b.lo    1b
        dsb     sy
        ret
+ENDPROC(__inval_cache_range)
 ENDPROC(__dma_inv_range)
 
 /*