vmemmap: generify initialisation via helpers

Convert the common vmemmap population into initialisation helpers for use by
architecture vmemmap populators.  All architecture implementing the
SPARSEMEM_VMEMMAP variant supply an architecture specific vmemmap_populate()
initialiser, which may make use of the helpers.

This allows us to clean up and remove the initialisation Kconfig entries.
With this patch there is a single SPARSEMEM_VMEMMAP_ENABLE Kconfig option to
indicate use of that variant.

Signed-off-by: Andy Whitcroft <apw@shadowen.org>
Acked-by: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/mm.h b/include/linux/mm.h
index d216abbd0574f0c92cb740277e38867c4ce5d418..fbff8e481cc44318acb8036e61cbe58dc82f935c 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1219,10 +1219,15 @@ extern int randomize_va_space;
 const char * arch_vma_name(struct vm_area_struct *vma);
 
 struct page *sparse_early_mem_map_populate(unsigned long pnum, int nid);
-int vmemmap_populate(struct page *start_page, unsigned long pages, int node);
-int vmemmap_populate_pmd(pud_t *, unsigned long, unsigned long, int);
+pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
+pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
+pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
+pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
 void *vmemmap_alloc_block(unsigned long size, int node);
 void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
+int vmemmap_populate_basepages(struct page *start_page,
+                                               unsigned long pages, int node);
+int vmemmap_populate(struct page *start_page, unsigned long pages, int node);
 
 #endif /* __KERNEL__ */
 #endif /* _LINUX_MM_H */

diff --git a/mm/Kconfig b/mm/Kconfig
index a7609cbcb00d8ce3fb81f366d7c835af970452e7..b0673066841223074db7e14c50b5a03db2de02a9 100644 (file)
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -112,6 +112,19 @@ config SPARSEMEM_EXTREME
        def_bool y
        depends on SPARSEMEM && !SPARSEMEM_STATIC
 
+#
+# SPARSEMEM_VMEMMAP uses a virtually mapped mem_map to optimise pfn_to_page
+# and page_to_pfn.  The most efficient option where kernel virtual space is
+# not under pressure.
+#
+config SPARSEMEM_VMEMMAP_ENABLE
+       def_bool n
+
+config SPARSEMEM_VMEMMAP
+       bool
+       depends on SPARSEMEM
+       default y if (SPARSEMEM_VMEMMAP_ENABLE)
+
 # eventually, we can have this option just 'select SPARSEMEM'
 config MEMORY_HOTPLUG
        bool "Allow for memory hot-add"

diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 7bb7a4b96d74cb712a188c8d45d6c2cf447210f0..4f2d4854f84050a9096282ec66b5e54fea463098 100644 (file)
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -14,21 +14,8 @@
  * case the overhead consists of a few additional pages that are
  * allocated to create a view of memory for vmemmap.
  *
- * Special Kconfig settings:
- *
- * CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP
- *
- *     The architecture has its own functions to populate the memory
- *     map and provides a vmemmap_populate function.
- *
- * CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP_PMD
- *
- *     The architecture provides functions to populate the pmd level
- *     of the vmemmap mappings.  Allowing mappings using large pages
- *     where available.
- *
- *     If neither are set then PAGE_SIZE mappings are generated which
- *     require one PTE/TLB per PAGE_SIZE chunk of the virtual memory map.
+ * The architecture is expected to provide a vmemmap_populate() function
+ * to instantiate the mapping.
  */
 #include <linux/mm.h>
 #include <linux/mmzone.h>
@@ -60,7 +47,6 @@ void * __meminit vmemmap_alloc_block(unsigned long size, int node)
                                __pa(MAX_DMA_ADDRESS));
 }
 
-#ifndef CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP
 void __meminit vmemmap_verify(pte_t *pte, int node,
                                unsigned long start, unsigned long end)
 {
@@ -72,103 +58,84 @@ void __meminit vmemmap_verify(pte_t *pte, int node,
                        "page_structs\n", start, end - 1);
 }
 
-#ifndef CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP_PMD
-static int __meminit vmemmap_populate_pte(pmd_t *pmd, unsigned long addr,
-                                       unsigned long end, int node)
+pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node)
 {
-       pte_t *pte;
-
-       for (pte = pte_offset_kernel(pmd, addr); addr < end;
-                                               pte++, addr += PAGE_SIZE)
-               if (pte_none(*pte)) {
-                       pte_t entry;
-                       void *p = vmemmap_alloc_block(PAGE_SIZE, node);
-                       if (!p)
-                               return -ENOMEM;
-
-                       entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL);
-                       set_pte(pte, entry);
-
-               } else
-                       vmemmap_verify(pte, node, addr + PAGE_SIZE, end);
-
-       return 0;
+       pte_t *pte = pte_offset_kernel(pmd, addr);
+       if (pte_none(*pte)) {
+               pte_t entry;
+               void *p = vmemmap_alloc_block(PAGE_SIZE, node);
+               if (!p)
+                       return 0;
+               entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL);
+               set_pte_at(&init_mm, addr, pte, entry);
+       }
+       return pte;
 }
 
-int __meminit vmemmap_populate_pmd(pud_t *pud, unsigned long addr,
-                                               unsigned long end, int node)
+pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node)
 {
-       pmd_t *pmd;
-       int error = 0;
-       unsigned long next;
-
-       for (pmd = pmd_offset(pud, addr); addr < end && !error;
-                                               pmd++, addr = next) {
-               if (pmd_none(*pmd)) {
-                       void *p = vmemmap_alloc_block(PAGE_SIZE, node);
-                       if (!p)
-                               return -ENOMEM;
-
-                       pmd_populate_kernel(&init_mm, pmd, p);
-               } else
-                       vmemmap_verify((pte_t *)pmd, node,
-                                       pmd_addr_end(addr, end), end);
-               next = pmd_addr_end(addr, end);
-               error = vmemmap_populate_pte(pmd, addr, next, node);
+       pmd_t *pmd = pmd_offset(pud, addr);
+       if (pmd_none(*pmd)) {
+               void *p = vmemmap_alloc_block(PAGE_SIZE, node);
+               if (!p)
+                       return 0;
+               pmd_populate_kernel(&init_mm, pmd, p);
        }
-       return error;
+       return pmd;
 }
-#endif /* CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP_PMD */
 
-static int __meminit vmemmap_populate_pud(pgd_t *pgd, unsigned long addr,
-                                               unsigned long end, int node)
+pud_t * __meminit vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node)
 {
-       pud_t *pud;
-       int error = 0;
-       unsigned long next;
-
-       for (pud = pud_offset(pgd, addr); addr < end && !error;
-                                               pud++, addr = next) {
-               if (pud_none(*pud)) {
-                       void *p = vmemmap_alloc_block(PAGE_SIZE, node);
-                       if (!p)
-                               return -ENOMEM;
+       pud_t *pud = pud_offset(pgd, addr);
+       if (pud_none(*pud)) {
+               void *p = vmemmap_alloc_block(PAGE_SIZE, node);
+               if (!p)
+                       return 0;
+               pud_populate(&init_mm, pud, p);
+       }
+       return pud;
+}
 
-                       pud_populate(&init_mm, pud, p);
-               }
-               next = pud_addr_end(addr, end);
-               error = vmemmap_populate_pmd(pud, addr, next, node);
+pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node)
+{
+       pgd_t *pgd = pgd_offset_k(addr);
+       if (pgd_none(*pgd)) {
+               void *p = vmemmap_alloc_block(PAGE_SIZE, node);
+               if (!p)
+                       return 0;
+               pgd_populate(&init_mm, pgd, p);
        }
-       return error;
+       return pgd;
 }
 
-int __meminit vmemmap_populate(struct page *start_page,
-                                               unsigned long nr, int node)
+int __meminit vmemmap_populate_basepages(struct page *start_page,
+                                               unsigned long size, int node)
 {
-       pgd_t *pgd;
        unsigned long addr = (unsigned long)start_page;
-       unsigned long end = (unsigned long)(start_page + nr);
-       unsigned long next;
-       int error = 0;
-
-       printk(KERN_DEBUG "[%lx-%lx] Virtual memory section"
-               " (%ld pages) node %d\n", addr, end - 1, nr, node);
-
-       for (pgd = pgd_offset_k(addr); addr < end && !error;
-                                       pgd++, addr = next) {
-               if (pgd_none(*pgd)) {
-                       void *p = vmemmap_alloc_block(PAGE_SIZE, node);
-                       if (!p)
-                               return -ENOMEM;
+       unsigned long end = (unsigned long)(start_page + size);
+       pgd_t *pgd;
+       pud_t *pud;
+       pmd_t *pmd;
+       pte_t *pte;
 
-                       pgd_populate(&init_mm, pgd, p);
-               }
-               next = pgd_addr_end(addr,end);
-               error = vmemmap_populate_pud(pgd, addr, next, node);
+       for (; addr < end; addr += PAGE_SIZE) {
+               pgd = vmemmap_pgd_populate(addr, node);
+               if (!pgd)
+                       return -ENOMEM;
+               pud = vmemmap_pud_populate(pgd, addr, node);
+               if (!pud)
+                       return -ENOMEM;
+               pmd = vmemmap_pmd_populate(pud, addr, node);
+               if (!pmd)
+                       return -ENOMEM;
+               pte = vmemmap_pte_populate(pmd, addr, node);
+               if (!pte)
+                       return -ENOMEM;
+               vmemmap_verify(pte, node, addr, addr + PAGE_SIZE);
        }
-       return error;
+
+       return 0;
 }
-#endif /* !CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP */
 
 struct page __init *sparse_early_mem_map_populate(unsigned long pnum, int nid)
 {