[GitHub/LineageOS/android_kernel_motorola_exynos9610.git] / arch / x86 / mm / ident_map.c

/*
 * Helper routines for building identity mapping page tables. This is
 * included by both the compressed kernel and the regular kernel.
 */

static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page,
			   unsigned long addr, unsigned long end)
{
	addr &= PMD_MASK;
	for (; addr < end; addr += PMD_SIZE) {
		pmd_t *pmd = pmd_page + pmd_index(addr);

		if (pmd_present(*pmd))
			continue;

		set_pmd(pmd, __pmd((addr - info->offset) | info->page_flag));
	}
}

static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page,
			  unsigned long addr, unsigned long end)
{
	unsigned long next;

	for (; addr < end; addr = next) {
		pud_t *pud = pud_page + pud_index(addr);
		pmd_t *pmd;

		next = (addr & PUD_MASK) + PUD_SIZE;
		if (next > end)
			next = end;

		if (info->direct_gbpages) {
			pud_t pudval;

			if (pud_present(*pud))
				continue;

			addr &= PUD_MASK;
			pudval = __pud((addr - info->offset) | info->page_flag);
			set_pud(pud, pudval);
			continue;
		}

		if (pud_present(*pud)) {
			pmd = pmd_offset(pud, 0);
			ident_pmd_init(info, pmd, addr, next);
			continue;
		}
		pmd = (pmd_t *)info->alloc_pgt_page(info->context);
		if (!pmd)
			return -ENOMEM;
		ident_pmd_init(info, pmd, addr, next);
		set_pud(pud, __pud(__pa(pmd) | info->kernpg_flag));
	}

	return 0;
}

static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page,
			  unsigned long addr, unsigned long end)
{
	unsigned long next;

	for (; addr < end; addr = next) {
		p4d_t *p4d = p4d_page + p4d_index(addr);
		pud_t *pud;

		next = (addr & P4D_MASK) + P4D_SIZE;
		if (next > end)
			next = end;

		if (p4d_present(*p4d)) {
			pud = pud_offset(p4d, 0);
			ident_pud_init(info, pud, addr, next);
			continue;
		}
		pud = (pud_t *)info->alloc_pgt_page(info->context);
		if (!pud)
			return -ENOMEM;
		ident_pud_init(info, pud, addr, next);
		set_p4d(p4d, __p4d(__pa(pud) | info->kernpg_flag));
	}

	return 0;
}

int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
			      unsigned long pstart, unsigned long pend)
{
	unsigned long addr = pstart + info->offset;
	unsigned long end = pend + info->offset;
	unsigned long next;
	int result;

	/* Set the default pagetable flags if not supplied */
	if (!info->kernpg_flag)
		info->kernpg_flag = _KERNPG_TABLE;

	for (; addr < end; addr = next) {
		pgd_t *pgd = pgd_page + pgd_index(addr);
		p4d_t *p4d;

		next = (addr & PGDIR_MASK) + PGDIR_SIZE;
		if (next > end)
			next = end;

		if (pgd_present(*pgd)) {
			p4d = p4d_offset(pgd, 0);
			result = ident_p4d_init(info, p4d, addr, next);
			if (result)
				return result;
			continue;
		}

		p4d = (p4d_t *)info->alloc_pgt_page(info->context);
		if (!p4d)
			return -ENOMEM;
		result = ident_p4d_init(info, p4d, addr, next);
		if (result)
			return result;
		if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
			set_pgd(pgd, __pgd(__pa(p4d) | info->kernpg_flag));
		} else {
			/*
			 * With p4d folded, pgd is equal to p4d.
			 * The pgd entry has to point to the pud page table in this case.
			 */
			pud_t *pud = pud_offset(p4d, 0);
			set_pgd(pgd, __pgd(__pa(pud) | info->kernpg_flag));
		}
	}

	return 0;
}
Commit	Line	Data
cb18ef0d KC	1	/*
	2	* Helper routines for building identity mapping page tables. This is
	3	* included by both the compressed kernel and the regular kernel.
	4	*/
cf4fb15b	5
e4630fdd	6	static void ident_pmd_init(struct x86_mapping_info info, pmd_t pmd_page,
cf4fb15b YL	7	unsigned long addr, unsigned long end)
	8	{
	9	addr &= PMD_MASK;
	10	for (; addr < end; addr += PMD_SIZE) {
	11	pmd_t *pmd = pmd_page + pmd_index(addr);
	12
e4630fdd RW	13	if (pmd_present(*pmd))
	14	continue;
	15
66aad4fd	16	set_pmd(pmd, __pmd((addr - info->offset) \| info->page_flag));
cf4fb15b YL	17	}
cf4fb15b YL	18	}
cb18ef0d	19
cf4fb15b YL	20	static int ident_pud_init(struct x86_mapping_info info, pud_t pud_page,
	21	unsigned long addr, unsigned long end)
	22	{
	23	unsigned long next;
	24
	25	for (; addr < end; addr = next) {
	26	pud_t *pud = pud_page + pud_index(addr);
	27	pmd_t *pmd;
	28
	29	next = (addr & PUD_MASK) + PUD_SIZE;
	30	if (next > end)
	31	next = end;
	32
66aad4fd XP	33	if (info->direct_gbpages) {
	34	pud_t pudval;
	35
	36	if (pud_present(*pud))
	37	continue;
	38
	39	addr &= PUD_MASK;
	40	pudval = __pud((addr - info->offset) \| info->page_flag);
	41	set_pud(pud, pudval);
	42	continue;
	43	}
	44
cf4fb15b YL	45	if (pud_present(*pud)) {
cf4fb15b YL	46	pmd = pmd_offset(pud, 0);
e4630fdd	47	ident_pmd_init(info, pmd, addr, next);
cf4fb15b YL	48	continue;
	49	}
	50	pmd = (pmd_t *)info->alloc_pgt_page(info->context);
	51	if (!pmd)
	52	return -ENOMEM;
e4630fdd	53	ident_pmd_init(info, pmd, addr, next);
bba4ed01	54	set_pud(pud, __pud(__pa(pmd) \| info->kernpg_flag));
cf4fb15b YL	55	}
	56
	57	return 0;
	58	}
	59
ea3b5e60 KS	60	static int ident_p4d_init(struct x86_mapping_info info, p4d_t p4d_page,
	61	unsigned long addr, unsigned long end)
	62	{
	63	unsigned long next;
	64
	65	for (; addr < end; addr = next) {
	66	p4d_t *p4d = p4d_page + p4d_index(addr);
	67	pud_t *pud;
	68
	69	next = (addr & P4D_MASK) + P4D_SIZE;
	70	if (next > end)
	71	next = end;
	72
	73	if (p4d_present(*p4d)) {
	74	pud = pud_offset(p4d, 0);
	75	ident_pud_init(info, pud, addr, next);
	76	continue;
	77	}
	78	pud = (pud_t *)info->alloc_pgt_page(info->context);
	79	if (!pud)
	80	return -ENOMEM;
	81	ident_pud_init(info, pud, addr, next);
bba4ed01	82	set_p4d(p4d, __p4d(__pa(pud) \| info->kernpg_flag));
ea3b5e60 KS	83	}
	84
	85	return 0;
	86	}
	87
cf4fb15b	88	int kernel_ident_mapping_init(struct x86_mapping_info info, pgd_t pgd_page,
e4630fdd	89	unsigned long pstart, unsigned long pend)
cf4fb15b	90	{
e4630fdd RW	91	unsigned long addr = pstart + info->offset;
e4630fdd RW	92	unsigned long end = pend + info->offset;
cf4fb15b YL	93	unsigned long next;
cf4fb15b YL	94	int result;
cf4fb15b	95
bba4ed01 TL	96	/* Set the default pagetable flags if not supplied */
	97	if (!info->kernpg_flag)
	98	info->kernpg_flag = _KERNPG_TABLE;
	99
cf4fb15b	100	for (; addr < end; addr = next) {
e4630fdd	101	pgd_t *pgd = pgd_page + pgd_index(addr);
ea3b5e60	102	p4d_t *p4d;
cf4fb15b YL	103
	104	next = (addr & PGDIR_MASK) + PGDIR_SIZE;
	105	if (next > end)
	106	next = end;
	107
	108	if (pgd_present(*pgd)) {
ea3b5e60 KS	109	p4d = p4d_offset(pgd, 0);
ea3b5e60 KS	110	result = ident_p4d_init(info, p4d, addr, next);
cf4fb15b YL	111	if (result)
	112	return result;
	113	continue;
	114	}
	115
ea3b5e60 KS	116	p4d = (p4d_t *)info->alloc_pgt_page(info->context);
ea3b5e60 KS	117	if (!p4d)
cf4fb15b	118	return -ENOMEM;
ea3b5e60	119	result = ident_p4d_init(info, p4d, addr, next);
cf4fb15b YL	120	if (result)
cf4fb15b YL	121	return result;
ea3b5e60	122	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
bba4ed01	123	set_pgd(pgd, __pgd(__pa(p4d) \| info->kernpg_flag));
ea3b5e60 KS	124	} else {
	125	/*
	126	* With p4d folded, pgd is equal to p4d.
	127	* The pgd entry has to point to the pud page table in this case.
	128	*/
	129	pud_t *pud = pud_offset(p4d, 0);
bba4ed01	130	set_pgd(pgd, __pgd(__pa(pud) \| info->kernpg_flag));
ea3b5e60	131	}
cf4fb15b YL	132	}
	133
	134	return 0;
	135	}