arm64: mm: move vmemmap region right below the linear region

This moves the vmemmap region right below PAGE_OFFSET, aka the start
of the linear region, and redefines its size to be a power of two.
Due to the placement of PAGE_OFFSET in the middle of the address space,
whose size is a power of two as well, this guarantees that virt to
page conversions and vice versa can be implemented efficiently, by
masking and shifting rather than ordinary arithmetic.

Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>

diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index 12f8a00fb3f1767a645a04358dcaca08fd4f6b43..8a2ab195ca77b0525ec45ff3772b53d2faf205c8 100644 (file)
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -39,6 +39,21 @@
  */
 #define PCI_IO_SIZE            SZ_16M
 
+/*
+ * Log2 of the upper bound of the size of a struct page. Used for sizing
+ * the vmemmap region only, does not affect actual memory footprint.
+ * We don't use sizeof(struct page) directly since taking its size here
+ * requires its definition to be available at this point in the inclusion
+ * chain, and it may not be a power of 2 in the first place.
+ */
+#define STRUCT_PAGE_MAX_SHIFT  6
+
+/*
+ * VMEMMAP_SIZE - allows the whole linear region to be covered by
+ *                a struct page array
+ */
+#define VMEMMAP_SIZE (UL(1) << (VA_BITS - PAGE_SHIFT - 1 + STRUCT_PAGE_MAX_SHIFT))
+
 /*
  * PAGE_OFFSET - the virtual address of the start of the kernel image (top
  *              (VA_BITS - 1))
@@ -54,7 +69,8 @@
 #define MODULES_END            (MODULES_VADDR + MODULES_VSIZE)
 #define MODULES_VADDR          (VA_START + KASAN_SHADOW_SIZE)
 #define MODULES_VSIZE          (SZ_128M)
-#define PCI_IO_END             (PAGE_OFFSET - SZ_2M)
+#define VMEMMAP_START          (PAGE_OFFSET - VMEMMAP_SIZE)
+#define PCI_IO_END             (VMEMMAP_START - SZ_2M)
 #define PCI_IO_START           (PCI_IO_END - PCI_IO_SIZE)
 #define FIXADDR_TOP            (PCI_IO_START - SZ_2M)
 #define TASK_SIZE_64           (UL(1) << VA_BITS)

diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 377257a8d393598033d22c85713a0a05a752b8ff..2abfa4d09e652ff5cc122d179a2f36502d88c71e 100644 (file)
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -24,20 +24,15 @@
 #include <asm/pgtable-prot.h>
 
 /*
- * VMALLOC and SPARSEMEM_VMEMMAP ranges.
+ * VMALLOC range.
  *
- * VMEMAP_SIZE: allows the whole linear region to be covered by a struct page array
- *     (rounded up to PUD_SIZE).
  * VMALLOC_START: beginning of the kernel vmalloc space
- * VMALLOC_END: extends to the available space below vmmemmap, PCI I/O space,
- *     fixed mappings and modules
+ * VMALLOC_END: extends to the available space below vmmemmap, PCI I/O space
+ *     and fixed mappings
  */
-#define VMEMMAP_SIZE           ALIGN((1UL << (VA_BITS - PAGE_SHIFT - 1)) * sizeof(struct page), PUD_SIZE)
-
 #define VMALLOC_START          (MODULES_END)
 #define VMALLOC_END            (PAGE_OFFSET - PUD_SIZE - VMEMMAP_SIZE - SZ_64K)
 
-#define VMEMMAP_START          (VMALLOC_END + SZ_64K)
 #define vmemmap                        ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT))
 
 #define FIRST_USER_ADDRESS     0UL

diff --git a/arch/arm64/mm/dump.c b/arch/arm64/mm/dump.c
index f9271cb2f5e3d94a620946e4e36971b33a43f390..a21f47421b0cf0be5a36a014fc6d3add78113704 100644 (file)
--- a/arch/arm64/mm/dump.c
+++ b/arch/arm64/mm/dump.c
@@ -37,14 +37,14 @@ enum address_markers_idx {
        MODULES_END_NR,
        VMALLOC_START_NR,
        VMALLOC_END_NR,
-#ifdef CONFIG_SPARSEMEM_VMEMMAP
-       VMEMMAP_START_NR,
-       VMEMMAP_END_NR,
-#endif
        FIXADDR_START_NR,
        FIXADDR_END_NR,
        PCI_START_NR,
        PCI_END_NR,
+#ifdef CONFIG_SPARSEMEM_VMEMMAP
+       VMEMMAP_START_NR,
+       VMEMMAP_END_NR,
+#endif
        KERNEL_SPACE_NR,
 };
 
@@ -53,14 +53,14 @@ static struct addr_marker address_markers[] = {
        { MODULES_END,          "Modules end" },
        { VMALLOC_START,        "vmalloc() Area" },
        { VMALLOC_END,          "vmalloc() End" },
-#ifdef CONFIG_SPARSEMEM_VMEMMAP
-       { 0,                    "vmemmap start" },
-       { 0,                    "vmemmap end" },
-#endif
        { FIXADDR_START,        "Fixmap start" },
        { FIXADDR_TOP,          "Fixmap end" },
        { PCI_IO_START,         "PCI I/O start" },
        { PCI_IO_END,           "PCI I/O end" },
+#ifdef CONFIG_SPARSEMEM_VMEMMAP
+       { 0,                    "vmemmap start" },
+       { 0,                    "vmemmap end" },
+#endif
        { PAGE_OFFSET,          "Linear Mapping" },
        { -1,                   NULL },
 };

diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 89376f3c65a3d3f0cb23a0b89afbadf0c3fbf48c..d55d720dba79ecef05f7c03685382315182d1a0e 100644 (file)
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -412,6 +412,10 @@ void __init mem_init(void)
                MLK_ROUNDUP(__start_rodata, _etext),
                MLK_ROUNDUP(__init_begin, __init_end),
                MLK_ROUNDUP(_sdata, _edata));
+       pr_cont("    fixed   : 0x%16lx - 0x%16lx   (%6ld KB)\n",
+               MLK(FIXADDR_START, FIXADDR_TOP));
+       pr_cont("    PCI I/O : 0x%16lx - 0x%16lx   (%6ld MB)\n",
+               MLM(PCI_IO_START, PCI_IO_END));
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
        pr_cont("    vmemmap : 0x%16lx - 0x%16lx   (%6ld GB maximum)\n"
                "              0x%16lx - 0x%16lx   (%6ld MB actual)\n",
@@ -420,10 +424,6 @@ void __init mem_init(void)
                MLM((unsigned long)phys_to_page(memblock_start_of_DRAM()),
                    (unsigned long)virt_to_page(high_memory)));
 #endif
-       pr_cont("    fixed   : 0x%16lx - 0x%16lx   (%6ld KB)\n",
-               MLK(FIXADDR_START, FIXADDR_TOP));
-       pr_cont("    PCI I/O : 0x%16lx - 0x%16lx   (%6ld MB)\n",
-               MLM(PCI_IO_START, PCI_IO_END));
        pr_cont("    memory  : 0x%16lx - 0x%16lx   (%6ld MB)\n",
                MLM(__phys_to_virt(memblock_start_of_DRAM()),
                    (unsigned long)high_memory));
@@ -440,6 +440,12 @@ void __init mem_init(void)
        BUILD_BUG_ON(TASK_SIZE_32                       > TASK_SIZE_64);
 #endif
 
+       /*
+        * Make sure we chose the upper bound of sizeof(struct page)
+        * correctly.
+        */
+       BUILD_BUG_ON(sizeof(struct page) > (1 << STRUCT_PAGE_MAX_SHIFT));
+
        if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) {
                extern int sysctl_overcommit_memory;
                /*