[GitHub/mt8127/android_kernel_alcatel_ttab.git] / arch / tile / mm / hugetlbpage.c

/*
 * Copyright 2010 Tilera Corporation. All Rights Reserved.
 *
 *   This program is free software; you can redistribute it and/or
 *   modify it under the terms of the GNU General Public License
 *   as published by the Free Software Foundation, version 2.
 *
 *   This program is distributed in the hope that it will be useful, but
 *   WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
 *   NON INFRINGEMENT.  See the GNU General Public License for
 *   more details.
 *
 * TILE Huge TLB Page Support for Kernel.
 * Taken from i386 hugetlb implementation:
 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
 */

#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/sysctl.h>
#include <linux/mman.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>
#include <asm/setup.h>

#ifdef CONFIG_HUGETLB_SUPER_PAGES

/*
 * Provide an additional huge page size (in addition to the regular default
 * huge page size) if no "hugepagesz" arguments are specified.
 * Note that it must be smaller than the default huge page size so
 * that it's possible to allocate them on demand from the buddy allocator.
 * You can change this to 64K (on a 16K build), 256K, 1M, or 4M,
 * or not define it at all.
 */
#define ADDITIONAL_HUGE_SIZE (1024 * 1024UL)

/* "Extra" page-size multipliers, one per level of the page table. */
int huge_shift[HUGE_SHIFT_ENTRIES] = {
#ifdef ADDITIONAL_HUGE_SIZE
#define ADDITIONAL_HUGE_SHIFT __builtin_ctzl(ADDITIONAL_HUGE_SIZE / PAGE_SIZE)
	[HUGE_SHIFT_PAGE] = ADDITIONAL_HUGE_SHIFT
#endif
};

/*
 * This routine is a hybrid of pte_alloc_map() and pte_alloc_kernel().
 * It assumes that L2 PTEs are never in HIGHMEM (we don't support that).
 * It locks the user pagetable, and bumps up the mm->nr_ptes field,
 * but otherwise allocate the page table using the kernel versions.
 */
static pte_t *pte_alloc_hugetlb(struct mm_struct *mm, pmd_t *pmd,
				unsigned long address)
{
	pte_t *new;

	if (pmd_none(*pmd)) {
		new = pte_alloc_one_kernel(mm, address);
		if (!new)
			return NULL;

		smp_wmb(); /* See comment in __pte_alloc */

		spin_lock(&mm->page_table_lock);
		if (likely(pmd_none(*pmd))) {  /* Has another populated it ? */
			mm->nr_ptes++;
			pmd_populate_kernel(mm, pmd, new);
			new = NULL;
		} else
			VM_BUG_ON(pmd_trans_splitting(*pmd));
		spin_unlock(&mm->page_table_lock);
		if (new)
			pte_free_kernel(mm, new);
	}

	return pte_offset_kernel(pmd, address);
}
#endif

pte_t *huge_pte_alloc(struct mm_struct *mm,
		      unsigned long addr, unsigned long sz)
{
	pgd_t *pgd;
	pud_t *pud;

	addr &= -sz;   /* Mask off any low bits in the address. */

	pgd = pgd_offset(mm, addr);
	pud = pud_alloc(mm, pgd, addr);

#ifdef CONFIG_HUGETLB_SUPER_PAGES
	if (sz >= PGDIR_SIZE) {
		BUG_ON(sz != PGDIR_SIZE &&
		       sz != PGDIR_SIZE << huge_shift[HUGE_SHIFT_PGDIR]);
		return (pte_t *)pud;
	} else {
		pmd_t *pmd = pmd_alloc(mm, pud, addr);
		if (sz >= PMD_SIZE) {
			BUG_ON(sz != PMD_SIZE &&
			       sz != (PMD_SIZE << huge_shift[HUGE_SHIFT_PMD]));
			return (pte_t *)pmd;
		}
		else {
			if (sz != PAGE_SIZE << huge_shift[HUGE_SHIFT_PAGE])
				panic("Unexpected page size %#lx\n", sz);
			return pte_alloc_hugetlb(mm, pmd, addr);
		}
	}
#else
	BUG_ON(sz != PMD_SIZE);
	return (pte_t *) pmd_alloc(mm, pud, addr);
#endif
}

static pte_t *get_pte(pte_t *base, int index, int level)
{
	pte_t *ptep = base + index;
#ifdef CONFIG_HUGETLB_SUPER_PAGES
	if (!pte_present(*ptep) && huge_shift[level] != 0) {
		unsigned long mask = -1UL << huge_shift[level];
		pte_t *super_ptep = base + (index & mask);
		pte_t pte = *super_ptep;
		if (pte_present(pte) && pte_super(pte))
			ptep = super_ptep;
	}
#endif
	return ptep;
}

pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
#ifdef CONFIG_HUGETLB_SUPER_PAGES
	pte_t *pte;
#endif

	/* Get the top-level page table entry. */
	pgd = (pgd_t *)get_pte((pte_t *)mm->pgd, pgd_index(addr), 0);
	if (!pgd_present(*pgd))
		return NULL;

	/* We don't have four levels. */
	pud = pud_offset(pgd, addr);
#ifndef __PAGETABLE_PUD_FOLDED
# error support fourth page table level
#endif

	/* Check for an L0 huge PTE, if we have three levels. */
#ifndef __PAGETABLE_PMD_FOLDED
	if (pud_huge(*pud))
		return (pte_t *)pud;

	pmd = (pmd_t *)get_pte((pte_t *)pud_page_vaddr(*pud),
			       pmd_index(addr), 1);
	if (!pmd_present(*pmd))
		return NULL;
#else
	pmd = pmd_offset(pud, addr);
#endif

	/* Check for an L1 huge PTE. */
	if (pmd_huge(*pmd))
		return (pte_t *)pmd;

#ifdef CONFIG_HUGETLB_SUPER_PAGES
	/* Check for an L2 huge PTE. */
	pte = get_pte((pte_t *)pmd_page_vaddr(*pmd), pte_index(addr), 2);
	if (!pte_present(*pte))
		return NULL;
	if (pte_super(*pte))
		return pte;
#endif

	return NULL;
}

struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
			      int write)
{
	return ERR_PTR(-EINVAL);
}

int pmd_huge(pmd_t pmd)
{
	return !!(pmd_val(pmd) & _PAGE_HUGE_PAGE);
}

int pud_huge(pud_t pud)
{
	return !!(pud_val(pud) & _PAGE_HUGE_PAGE);
}

struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
			     pmd_t *pmd, int write)
{
	struct page *page;

	page = pte_page(*(pte_t *)pmd);
	if (page)
		page += ((address & ~PMD_MASK) >> PAGE_SHIFT);
	return page;
}

struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address,
			     pud_t *pud, int write)
{
	struct page *page;

	page = pte_page(*(pte_t *)pud);
	if (page)
		page += ((address & ~PUD_MASK) >> PAGE_SHIFT);
	return page;
}

int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
{
	return 0;
}

#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
		unsigned long addr, unsigned long len,
		unsigned long pgoff, unsigned long flags)
{
	struct hstate *h = hstate_file(file);
	struct vm_unmapped_area_info info;

	info.flags = 0;
	info.length = len;
	info.low_limit = TASK_UNMAPPED_BASE;
	info.high_limit = TASK_SIZE;
	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	info.align_offset = 0;
	return vm_unmapped_area(&info);
}

static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
		unsigned long addr0, unsigned long len,
		unsigned long pgoff, unsigned long flags)
{
	struct hstate *h = hstate_file(file);
	struct vm_unmapped_area_info info;
	unsigned long addr;

	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
	info.length = len;
	info.low_limit = PAGE_SIZE;
	info.high_limit = current->mm->mmap_base;
	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	info.align_offset = 0;
	addr = vm_unmapped_area(&info);

	/*
	 * A failed mmap() very likely causes application failure,
	 * so fall back to the bottom-up function here. This scenario
	 * can happen with large stack limits and large mmap()
	 * allocations.
	 */
	if (addr & ~PAGE_MASK) {
		VM_BUG_ON(addr != -ENOMEM);
		info.flags = 0;
		info.low_limit = TASK_UNMAPPED_BASE;
		info.high_limit = TASK_SIZE;
		addr = vm_unmapped_area(&info);
	}

	return addr;
}

unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
		unsigned long len, unsigned long pgoff, unsigned long flags)
{
	struct hstate *h = hstate_file(file);
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma;

	if (len & ~huge_page_mask(h))
		return -EINVAL;
	if (len > TASK_SIZE)
		return -ENOMEM;

	if (flags & MAP_FIXED) {
		if (prepare_hugepage_range(file, addr, len))
			return -EINVAL;
		return addr;
	}

	if (addr) {
		addr = ALIGN(addr, huge_page_size(h));
		vma = find_vma(mm, addr);
		if (TASK_SIZE - len >= addr &&
		    (!vma || addr + len <= vm_start_gap(vma)))
			return addr;
	}
	if (current->mm->get_unmapped_area == arch_get_unmapped_area)
		return hugetlb_get_unmapped_area_bottomup(file, addr, len,
				pgoff, flags);
	else
		return hugetlb_get_unmapped_area_topdown(file, addr, len,
				pgoff, flags);
}
#endif /* HAVE_ARCH_HUGETLB_UNMAPPED_AREA */

#ifdef CONFIG_HUGETLB_SUPER_PAGES
static __init int __setup_hugepagesz(unsigned long ps)
{
	int log_ps = __builtin_ctzl(ps);
	int level, base_shift;

	if ((1UL << log_ps) != ps || (log_ps & 1) != 0) {
		pr_warn("Not enabling %ld byte huge pages;"
			" must be a power of four.\n", ps);
		return -EINVAL;
	}

	if (ps > 64*1024*1024*1024UL) {
		pr_warn("Not enabling %ld MB huge pages;"
			" largest legal value is 64 GB .\n", ps >> 20);
		return -EINVAL;
	} else if (ps >= PUD_SIZE) {
		static long hv_jpage_size;
		if (hv_jpage_size == 0)
			hv_jpage_size = hv_sysconf(HV_SYSCONF_PAGE_SIZE_JUMBO);
		if (hv_jpage_size != PUD_SIZE) {
			pr_warn("Not enabling >= %ld MB huge pages:"
				" hypervisor reports size %ld\n",
				PUD_SIZE >> 20, hv_jpage_size);
			return -EINVAL;
		}
		level = 0;
		base_shift = PUD_SHIFT;
	} else if (ps >= PMD_SIZE) {
		level = 1;
		base_shift = PMD_SHIFT;
	} else if (ps > PAGE_SIZE) {
		level = 2;
		base_shift = PAGE_SHIFT;
	} else {
		pr_err("hugepagesz: huge page size %ld too small\n", ps);
		return -EINVAL;
	}

	if (log_ps != base_shift) {
		int shift_val = log_ps - base_shift;
		if (huge_shift[level] != 0) {
			int old_shift = base_shift + huge_shift[level];
			pr_warn("Not enabling %ld MB huge pages;"
				" already have size %ld MB.\n",
				ps >> 20, (1UL << old_shift) >> 20);
			return -EINVAL;
		}
		if (hv_set_pte_super_shift(level, shift_val) != 0) {
			pr_warn("Not enabling %ld MB huge pages;"
				" no hypervisor support.\n", ps >> 20);
			return -EINVAL;
		}
		printk(KERN_DEBUG "Enabled %ld MB huge pages\n", ps >> 20);
		huge_shift[level] = shift_val;
	}

	hugetlb_add_hstate(log_ps - PAGE_SHIFT);

	return 0;
}

static bool saw_hugepagesz;

static __init int setup_hugepagesz(char *opt)
{
	if (!saw_hugepagesz) {
		saw_hugepagesz = true;
		memset(huge_shift, 0, sizeof(huge_shift));
	}
	return __setup_hugepagesz(memparse(opt, NULL));
}
__setup("hugepagesz=", setup_hugepagesz);

#ifdef ADDITIONAL_HUGE_SIZE
/*
 * Provide an additional huge page size if no "hugepagesz" args are given.
 * In that case, all the cores have properly set up their hv super_shift
 * already, but we need to notify the hugetlb code to enable the
 * new huge page size from the Linux point of view.
 */
static __init int add_default_hugepagesz(void)
{
	if (!saw_hugepagesz) {
		BUILD_BUG_ON(ADDITIONAL_HUGE_SIZE >= PMD_SIZE ||
			     ADDITIONAL_HUGE_SIZE <= PAGE_SIZE);
		BUILD_BUG_ON((PAGE_SIZE << ADDITIONAL_HUGE_SHIFT) !=
			     ADDITIONAL_HUGE_SIZE);
		BUILD_BUG_ON(ADDITIONAL_HUGE_SHIFT & 1);
		hugetlb_add_hstate(ADDITIONAL_HUGE_SHIFT);
	}
	return 0;
}
arch_initcall(add_default_hugepagesz);
#endif

#endif /* CONFIG_HUGETLB_SUPER_PAGES */
Commit	Line	Data
867e359b CM	1	/*
	2	* Copyright 2010 Tilera Corporation. All Rights Reserved.
	3	*
	4	* This program is free software; you can redistribute it and/or
	5	* modify it under the terms of the GNU General Public License
	6	* as published by the Free Software Foundation, version 2.
	7	*
	8	* This program is distributed in the hope that it will be useful, but
	9	* WITHOUT ANY WARRANTY; without even the implied warranty of
	10	* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
	11	* NON INFRINGEMENT. See the GNU General Public License for
	12	* more details.
	13	*
	14	* TILE Huge TLB Page Support for Kernel.
	15	* Taken from i386 hugetlb implementation:
	16	* Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
	17	*/
	18
	19	#include <linux/init.h>
	20	#include <linux/fs.h>
	21	#include <linux/mm.h>
	22	#include <linux/hugetlb.h>
	23	#include <linux/pagemap.h>
867e359b CM	24	#include <linux/slab.h>
	25	#include <linux/err.h>
	26	#include <linux/sysctl.h>
	27	#include <linux/mman.h>
	28	#include <asm/tlb.h>
	29	#include <asm/tlbflush.h>
621b1955 CM	30	#include <asm/setup.h>
	31
	32	#ifdef CONFIG_HUGETLB_SUPER_PAGES
	33
	34	/*
	35	* Provide an additional huge page size (in addition to the regular default
	36	* huge page size) if no "hugepagesz" arguments are specified.
	37	* Note that it must be smaller than the default huge page size so
	38	* that it's possible to allocate them on demand from the buddy allocator.
	39	* You can change this to 64K (on a 16K build), 256K, 1M, or 4M,
	40	* or not define it at all.
	41	*/
	42	#define ADDITIONAL_HUGE_SIZE (1024 * 1024UL)
	43
	44	/* "Extra" page-size multipliers, one per level of the page table. */
	45	int huge_shift[HUGE_SHIFT_ENTRIES] = {
	46	#ifdef ADDITIONAL_HUGE_SIZE
	47	#define ADDITIONAL_HUGE_SHIFT __builtin_ctzl(ADDITIONAL_HUGE_SIZE / PAGE_SIZE)
	48	[HUGE_SHIFT_PAGE] = ADDITIONAL_HUGE_SHIFT
	49	#endif
	50	};
	51
	52	/*
	53	* This routine is a hybrid of pte_alloc_map() and pte_alloc_kernel().
	54	* It assumes that L2 PTEs are never in HIGHMEM (we don't support that).
	55	* It locks the user pagetable, and bumps up the mm->nr_ptes field,
	56	* but otherwise allocate the page table using the kernel versions.
	57	*/
	58	static pte_t pte_alloc_hugetlb(struct mm_struct mm, pmd_t *pmd,
	59	unsigned long address)
	60	{
	61	pte_t *new;
	62
	63	if (pmd_none(*pmd)) {
	64	new = pte_alloc_one_kernel(mm, address);
	65	if (!new)
	66	return NULL;
	67
	68	smp_wmb(); /* See comment in __pte_alloc */
	69
	70	spin_lock(&mm->page_table_lock);
	71	if (likely(pmd_none(pmd))) { / Has another populated it ? */
	72	mm->nr_ptes++;
	73	pmd_populate_kernel(mm, pmd, new);
	74	new = NULL;
	75	} else
	76	VM_BUG_ON(pmd_trans_splitting(*pmd));
	77	spin_unlock(&mm->page_table_lock);
	78	if (new)
	79	pte_free_kernel(mm, new);
	80	}
	81
	82	return pte_offset_kernel(pmd, address);
	83	}
	84	#endif
867e359b CM	85
	86	pte_t huge_pte_alloc(struct mm_struct mm,
	87	unsigned long addr, unsigned long sz)
	88	{
	89	pgd_t *pgd;
	90	pud_t *pud;
867e359b	91
621b1955	92	addr &= -sz; /* Mask off any low bits in the address. */
867e359b CM	93
	94	pgd = pgd_offset(mm, addr);
	95	pud = pud_alloc(mm, pgd, addr);
867e359b	96
621b1955 CM	97	#ifdef CONFIG_HUGETLB_SUPER_PAGES
	98	if (sz >= PGDIR_SIZE) {
	99	BUG_ON(sz != PGDIR_SIZE &&
	100	sz != PGDIR_SIZE << huge_shift[HUGE_SHIFT_PGDIR]);
	101	return (pte_t *)pud;
	102	} else {
	103	pmd_t *pmd = pmd_alloc(mm, pud, addr);
	104	if (sz >= PMD_SIZE) {
	105	BUG_ON(sz != PMD_SIZE &&
	106	sz != (PMD_SIZE << huge_shift[HUGE_SHIFT_PMD]));
	107	return (pte_t *)pmd;
	108	}
	109	else {
	110	if (sz != PAGE_SIZE << huge_shift[HUGE_SHIFT_PAGE])
	111	panic("Unexpected page size %#lx\n", sz);
	112	return pte_alloc_hugetlb(mm, pmd, addr);
	113	}
	114	}
	115	#else
	116	BUG_ON(sz != PMD_SIZE);
	117	return (pte_t *) pmd_alloc(mm, pud, addr);
	118	#endif
867e359b CM	119	}
867e359b CM	120
621b1955	121	static pte_t get_pte(pte_t base, int index, int level)
867e359b	122	{
621b1955 CM	123	pte_t *ptep = base + index;
	124	#ifdef CONFIG_HUGETLB_SUPER_PAGES
	125	if (!pte_present(*ptep) && huge_shift[level] != 0) {
	126	unsigned long mask = -1UL << huge_shift[level];
	127	pte_t *super_ptep = base + (index & mask);
	128	pte_t pte = *super_ptep;
	129	if (pte_present(pte) && pte_super(pte))
	130	ptep = super_ptep;
867e359b	131	}
621b1955 CM	132	#endif
621b1955 CM	133	return ptep;
867e359b CM	134	}
867e359b CM	135
621b1955	136	pte_t huge_pte_offset(struct mm_struct mm, unsigned long addr)
867e359b	137	{
621b1955 CM	138	pgd_t *pgd;
	139	pud_t *pud;
	140	pmd_t *pmd;
	141	#ifdef CONFIG_HUGETLB_SUPER_PAGES
	142	pte_t *pte;
	143	#endif
867e359b	144
621b1955 CM	145	/* Get the top-level page table entry. */
	146	pgd = (pgd_t )get_pte((pte_t )mm->pgd, pgd_index(addr), 0);
	147	if (!pgd_present(*pgd))
	148	return NULL;
867e359b	149
621b1955 CM	150	/* We don't have four levels. */
	151	pud = pud_offset(pgd, addr);
	152	#ifndef __PAGETABLE_PUD_FOLDED
	153	# error support fourth page table level
	154	#endif
867e359b	155
621b1955 CM	156	/* Check for an L0 huge PTE, if we have three levels. */
	157	#ifndef __PAGETABLE_PMD_FOLDED
	158	if (pud_huge(*pud))
	159	return (pte_t *)pud;
867e359b	160
621b1955 CM	161	pmd = (pmd_t )get_pte((pte_t )pud_page_vaddr(*pud),
	162	pmd_index(addr), 1);
	163	if (!pmd_present(*pmd))
	164	return NULL;
	165	#else
	166	pmd = pmd_offset(pud, addr);
	167	#endif
867e359b	168
621b1955 CM	169	/* Check for an L1 huge PTE. */
	170	if (pmd_huge(*pmd))
	171	return (pte_t *)pmd;
	172
	173	#ifdef CONFIG_HUGETLB_SUPER_PAGES
	174	/* Check for an L2 huge PTE. */
	175	pte = get_pte((pte_t )pmd_page_vaddr(pmd), pte_index(addr), 2);
	176	if (!pte_present(*pte))
	177	return NULL;
	178	if (pte_super(*pte))
	179	return pte;
	180	#endif
867e359b	181
867e359b CM	182	return NULL;
	183	}
	184
867e359b CM	185	struct page follow_huge_addr(struct mm_struct mm, unsigned long address,
	186	int write)
	187	{
	188	return ERR_PTR(-EINVAL);
	189	}
	190
	191	int pmd_huge(pmd_t pmd)
	192	{
	193	return !!(pmd_val(pmd) & _PAGE_HUGE_PAGE);
	194	}
	195
	196	int pud_huge(pud_t pud)
	197	{
	198	return !!(pud_val(pud) & _PAGE_HUGE_PAGE);
	199	}
	200
	201	struct page follow_huge_pmd(struct mm_struct mm, unsigned long address,
	202	pmd_t *pmd, int write)
	203	{
	204	struct page *page;
	205
	206	page = pte_page((pte_t )pmd);
	207	if (page)
	208	page += ((address & ~PMD_MASK) >> PAGE_SHIFT);
	209	return page;
	210	}
	211
	212	struct page follow_huge_pud(struct mm_struct mm, unsigned long address,
	213	pud_t *pud, int write)
	214	{
	215	struct page *page;
	216
	217	page = pte_page((pte_t )pud);
	218	if (page)
	219	page += ((address & ~PUD_MASK) >> PAGE_SHIFT);
	220	return page;
	221	}
	222
	223	int huge_pmd_unshare(struct mm_struct mm, unsigned long addr, pte_t *ptep)
	224	{
	225	return 0;
	226	}
	227
867e359b CM	228	#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
	229	static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
	230	unsigned long addr, unsigned long len,
	231	unsigned long pgoff, unsigned long flags)
	232	{
	233	struct hstate *h = hstate_file(file);
dd529596 ML	234	struct vm_unmapped_area_info info;
	235
	236	info.flags = 0;
	237	info.length = len;
	238	info.low_limit = TASK_UNMAPPED_BASE;
	239	info.high_limit = TASK_SIZE;
	240	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	241	info.align_offset = 0;
	242	return vm_unmapped_area(&info);
867e359b CM	243	}
	244
	245	static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
	246	unsigned long addr0, unsigned long len,
	247	unsigned long pgoff, unsigned long flags)
	248	{
	249	struct hstate *h = hstate_file(file);
dd529596 ML	250	struct vm_unmapped_area_info info;
dd529596 ML	251	unsigned long addr;
867e359b	252
dd529596 ML	253	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
	254	info.length = len;
	255	info.low_limit = PAGE_SIZE;
	256	info.high_limit = current->mm->mmap_base;
	257	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	258	info.align_offset = 0;
	259	addr = vm_unmapped_area(&info);
867e359b	260
867e359b CM	261	/*
	262	* A failed mmap() very likely causes application failure,
	263	* so fall back to the bottom-up function here. This scenario
	264	* can happen with large stack limits and large mmap()
	265	* allocations.
	266	*/
dd529596 ML	267	if (addr & ~PAGE_MASK) {
	268	VM_BUG_ON(addr != -ENOMEM);
	269	info.flags = 0;
	270	info.low_limit = TASK_UNMAPPED_BASE;
	271	info.high_limit = TASK_SIZE;
	272	addr = vm_unmapped_area(&info);
	273	}
867e359b CM	274
	275	return addr;
	276	}
	277
	278	unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
	279	unsigned long len, unsigned long pgoff, unsigned long flags)
	280	{
	281	struct hstate *h = hstate_file(file);
	282	struct mm_struct *mm = current->mm;
	283	struct vm_area_struct *vma;
	284
	285	if (len & ~huge_page_mask(h))
	286	return -EINVAL;
	287	if (len > TASK_SIZE)
	288	return -ENOMEM;
	289
	290	if (flags & MAP_FIXED) {
	291	if (prepare_hugepage_range(file, addr, len))
	292	return -EINVAL;
	293	return addr;
	294	}
	295
	296	if (addr) {
	297	addr = ALIGN(addr, huge_page_size(h));
	298	vma = find_vma(mm, addr);
	299	if (TASK_SIZE - len >= addr &&
1ad9a25d	300	(!vma \|\| addr + len <= vm_start_gap(vma)))
867e359b CM	301	return addr;
	302	}
	303	if (current->mm->get_unmapped_area == arch_get_unmapped_area)
	304	return hugetlb_get_unmapped_area_bottomup(file, addr, len,
	305	pgoff, flags);
	306	else
	307	return hugetlb_get_unmapped_area_topdown(file, addr, len,
	308	pgoff, flags);
	309	}
621b1955	310	#endif /* HAVE_ARCH_HUGETLB_UNMAPPED_AREA */
867e359b	311
621b1955 CM	312	#ifdef CONFIG_HUGETLB_SUPER_PAGES
621b1955 CM	313	static __init int __setup_hugepagesz(unsigned long ps)
867e359b	314	{
621b1955 CM	315	int log_ps = __builtin_ctzl(ps);
	316	int level, base_shift;
	317
	318	if ((1UL << log_ps) != ps \|\| (log_ps & 1) != 0) {
	319	pr_warn("Not enabling %ld byte huge pages;"
	320	" must be a power of four.\n", ps);
	321	return -EINVAL;
	322	}
	323
	324	if (ps > 6410241024*1024UL) {
	325	pr_warn("Not enabling %ld MB huge pages;"
	326	" largest legal value is 64 GB .\n", ps >> 20);
	327	return -EINVAL;
	328	} else if (ps >= PUD_SIZE) {
	329	static long hv_jpage_size;
	330	if (hv_jpage_size == 0)
	331	hv_jpage_size = hv_sysconf(HV_SYSCONF_PAGE_SIZE_JUMBO);
	332	if (hv_jpage_size != PUD_SIZE) {
	333	pr_warn("Not enabling >= %ld MB huge pages:"
	334	" hypervisor reports size %ld\n",
	335	PUD_SIZE >> 20, hv_jpage_size);
	336	return -EINVAL;
	337	}
	338	level = 0;
	339	base_shift = PUD_SHIFT;
	340	} else if (ps >= PMD_SIZE) {
	341	level = 1;
	342	base_shift = PMD_SHIFT;
	343	} else if (ps > PAGE_SIZE) {
	344	level = 2;
	345	base_shift = PAGE_SHIFT;
867e359b	346	} else {
621b1955 CM	347	pr_err("hugepagesz: huge page size %ld too small\n", ps);
621b1955 CM	348	return -EINVAL;
867e359b	349	}
621b1955 CM	350
	351	if (log_ps != base_shift) {
	352	int shift_val = log_ps - base_shift;
	353	if (huge_shift[level] != 0) {
	354	int old_shift = base_shift + huge_shift[level];
	355	pr_warn("Not enabling %ld MB huge pages;"
	356	" already have size %ld MB.\n",
	357	ps >> 20, (1UL << old_shift) >> 20);
	358	return -EINVAL;
	359	}
	360	if (hv_set_pte_super_shift(level, shift_val) != 0) {
	361	pr_warn("Not enabling %ld MB huge pages;"
	362	" no hypervisor support.\n", ps >> 20);
	363	return -EINVAL;
	364	}
	365	printk(KERN_DEBUG "Enabled %ld MB huge pages\n", ps >> 20);
	366	huge_shift[level] = shift_val;
	367	}
	368
	369	hugetlb_add_hstate(log_ps - PAGE_SHIFT);
	370
	371	return 0;
	372	}
	373
	374	static bool saw_hugepagesz;
	375
	376	static __init int setup_hugepagesz(char *opt)
	377	{
	378	if (!saw_hugepagesz) {
	379	saw_hugepagesz = true;
	380	memset(huge_shift, 0, sizeof(huge_shift));
	381	}
	382	return __setup_hugepagesz(memparse(opt, NULL));
867e359b CM	383	}
	384	__setup("hugepagesz=", setup_hugepagesz);
	385
621b1955 CM	386	#ifdef ADDITIONAL_HUGE_SIZE
	387	/*
	388	* Provide an additional huge page size if no "hugepagesz" args are given.
	389	* In that case, all the cores have properly set up their hv super_shift
	390	* already, but we need to notify the hugetlb code to enable the
	391	* new huge page size from the Linux point of view.
	392	*/
	393	static __init int add_default_hugepagesz(void)
	394	{
	395	if (!saw_hugepagesz) {
	396	BUILD_BUG_ON(ADDITIONAL_HUGE_SIZE >= PMD_SIZE \|\|
	397	ADDITIONAL_HUGE_SIZE <= PAGE_SIZE);
	398	BUILD_BUG_ON((PAGE_SIZE << ADDITIONAL_HUGE_SHIFT) !=
	399	ADDITIONAL_HUGE_SIZE);
	400	BUILD_BUG_ON(ADDITIONAL_HUGE_SHIFT & 1);
	401	hugetlb_add_hstate(ADDITIONAL_HUGE_SHIFT);
	402	}
	403	return 0;
	404	}
	405	arch_initcall(add_default_hugepagesz);
	406	#endif
	407
	408	#endif /* CONFIG_HUGETLB_SUPER_PAGES */