x86/mm/ident_map: Add 5-level paging support

Add additional page table level handing. It's mostly mechanical.

The only quirk is that with p4d folded, 'pgd' is equal to 'p4d' in
kernel_ident_mapping_init(). The pgd entry has to point to the pud
page table in this case.

Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-mm@kvack.org
Link: http://lkml.kernel.org/r/20170313143309.16020-5-kirill.shutemov@linux.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/arch/x86/mm/ident_map.c b/arch/x86/mm/ident_map.c
index 4473cb4f8b906dcae083a4b29c38b72d9b7d56d3..04210a29dd6060959745c4d974c4c99e6292786d 100644 (file)
--- a/arch/x86/mm/ident_map.c
+++ b/arch/x86/mm/ident_map.c
@@ -45,6 +45,34 @@ static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page,
        return 0;
 }
 
+static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page,
+                         unsigned long addr, unsigned long end)
+{
+       unsigned long next;
+
+       for (; addr < end; addr = next) {
+               p4d_t *p4d = p4d_page + p4d_index(addr);
+               pud_t *pud;
+
+               next = (addr & P4D_MASK) + P4D_SIZE;
+               if (next > end)
+                       next = end;
+
+               if (p4d_present(*p4d)) {
+                       pud = pud_offset(p4d, 0);
+                       ident_pud_init(info, pud, addr, next);
+                       continue;
+               }
+               pud = (pud_t *)info->alloc_pgt_page(info->context);
+               if (!pud)
+                       return -ENOMEM;
+               ident_pud_init(info, pud, addr, next);
+               set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE));
+       }
+
+       return 0;
+}
+
 int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
                              unsigned long pstart, unsigned long pend)
 {
@@ -55,27 +83,36 @@ int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
 
        for (; addr < end; addr = next) {
                pgd_t *pgd = pgd_page + pgd_index(addr);
-               pud_t *pud;
+               p4d_t *p4d;
 
                next = (addr & PGDIR_MASK) + PGDIR_SIZE;
                if (next > end)
                        next = end;
 
                if (pgd_present(*pgd)) {
-                       pud = pud_offset(pgd, 0);
-                       result = ident_pud_init(info, pud, addr, next);
+                       p4d = p4d_offset(pgd, 0);
+                       result = ident_p4d_init(info, p4d, addr, next);
                        if (result)
                                return result;
                        continue;
                }
 
-               pud = (pud_t *)info->alloc_pgt_page(info->context);
-               if (!pud)
+               p4d = (p4d_t *)info->alloc_pgt_page(info->context);
+               if (!p4d)
                        return -ENOMEM;
-               result = ident_pud_init(info, pud, addr, next);
+               result = ident_p4d_init(info, p4d, addr, next);
                if (result)
                        return result;
-               set_pgd(pgd, __pgd(__pa(pud) | _KERNPG_TABLE));
+               if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
+                       set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE));
+               } else {
+                       /*
+                        * With p4d folded, pgd is equal to p4d.
+                        * The pgd entry has to point to the pud page table in this case.
+                        */
+                       pud_t *pud = pud_offset(p4d, 0);
+                       set_pgd(pgd, __pgd(__pa(pud) | _KERNPG_TABLE));
+               }
        }
 
        return 0;