[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / misc / mediatek / gpu / mt8127 / mali / mali / common / mali_mmu_page_directory.c

/*
 * This confidential and proprietary software may be used only as
 * authorised by a licensing agreement from ARM Limited
 * (C) COPYRIGHT 2011-2015 ARM Limited
 * ALL RIGHTS RESERVED
 * The entire notice above must be reproduced on all authorised
 * copies and copies may only be made to the extent permitted
 * by a licensing agreement from ARM Limited.
 */

#include "mali_kernel_common.h"
#include "mali_osk.h"
#include "mali_ukk.h"
#include "mali_uk_types.h"
#include "mali_mmu_page_directory.h"
#include "mali_memory.h"
#include "mali_l2_cache.h"

static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data);

u32 mali_allocate_empty_page(mali_io_address *virt_addr)
{
	_mali_osk_errcode_t err;
	mali_io_address mapping;
	mali_dma_addr address;

	if (_MALI_OSK_ERR_OK != mali_mmu_get_table_page(&address, &mapping)) {
		/* Allocation failed */
		MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to get table page for empty pgdir\n"));
		return 0;
	}

	MALI_DEBUG_ASSERT_POINTER(mapping);

	err = fill_page(mapping, 0);
	if (_MALI_OSK_ERR_OK != err) {
		mali_mmu_release_table_page(address, mapping);
		MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to zero page\n"));
		return 0;
	}

	*virt_addr = mapping;
	return address;
}

void mali_free_empty_page(mali_dma_addr address, mali_io_address virt_addr)
{
	if (MALI_INVALID_PAGE != address) {
		mali_mmu_release_table_page(address, virt_addr);
	}
}

_mali_osk_errcode_t mali_create_fault_flush_pages(mali_dma_addr *page_directory,
		mali_io_address *page_directory_mapping,
		mali_dma_addr *page_table, mali_io_address *page_table_mapping,
		mali_dma_addr *data_page, mali_io_address *data_page_mapping)
{
	_mali_osk_errcode_t err;

	err = mali_mmu_get_table_page(data_page, data_page_mapping);
	if (_MALI_OSK_ERR_OK == err) {
		err = mali_mmu_get_table_page(page_table, page_table_mapping);
		if (_MALI_OSK_ERR_OK == err) {
			err = mali_mmu_get_table_page(page_directory, page_directory_mapping);
			if (_MALI_OSK_ERR_OK == err) {
				fill_page(*data_page_mapping, 0);
				fill_page(*page_table_mapping, *data_page | MALI_MMU_FLAGS_DEFAULT);
				fill_page(*page_directory_mapping, *page_table | MALI_MMU_FLAGS_PRESENT);
				MALI_SUCCESS;
			}
			mali_mmu_release_table_page(*page_table, *page_table_mapping);
			*page_table = MALI_INVALID_PAGE;
		}
		mali_mmu_release_table_page(*data_page, *data_page_mapping);
		*data_page = MALI_INVALID_PAGE;
	}
	return err;
}

void mali_destroy_fault_flush_pages(
	mali_dma_addr *page_directory, mali_io_address *page_directory_mapping,
	mali_dma_addr *page_table, mali_io_address *page_table_mapping,
	mali_dma_addr *data_page, mali_io_address *data_page_mapping)
{
	if (MALI_INVALID_PAGE != *page_directory) {
		mali_mmu_release_table_page(*page_directory, *page_directory_mapping);
		*page_directory = MALI_INVALID_PAGE;
		*page_directory_mapping = NULL;
	}

	if (MALI_INVALID_PAGE != *page_table) {
		mali_mmu_release_table_page(*page_table, *page_table_mapping);
		*page_table = MALI_INVALID_PAGE;
		*page_table_mapping = NULL;
	}

	if (MALI_INVALID_PAGE != *data_page) {
		mali_mmu_release_table_page(*data_page, *data_page_mapping);
		*data_page = MALI_INVALID_PAGE;
		*data_page_mapping = NULL;
	}
}

static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data)
{
	int i;
	MALI_DEBUG_ASSERT_POINTER(mapping);

	for (i = 0; i < MALI_MMU_PAGE_SIZE / 4; i++) {
		_mali_osk_mem_iowrite32_relaxed(mapping, i * sizeof(u32), data);
	}
	_mali_osk_mem_barrier();
	MALI_SUCCESS;
}

_mali_osk_errcode_t mali_mmu_pagedir_map(struct mali_page_directory *pagedir, u32 mali_address, u32 size)
{
	const int first_pde = MALI_MMU_PDE_ENTRY(mali_address);
	const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1);
	_mali_osk_errcode_t err;
	mali_io_address pde_mapping;
	mali_dma_addr pde_phys;
	int i;

	if (last_pde < first_pde)
		return _MALI_OSK_ERR_INVALID_ARGS;

	for (i = first_pde; i <= last_pde; i++) {
		if (0 == (_mali_osk_mem_ioread32(pagedir->page_directory_mapped,
						 i * sizeof(u32)) & MALI_MMU_FLAGS_PRESENT)) {
			/* Page table not present */
			MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]);
			MALI_DEBUG_ASSERT(NULL == pagedir->page_entries_mapped[i]);

			err = mali_mmu_get_table_page(&pde_phys, &pde_mapping);
			if (_MALI_OSK_ERR_OK != err) {
				MALI_PRINT_ERROR(("Failed to allocate page table page.\n"));
				return err;
			}
			pagedir->page_entries_mapped[i] = pde_mapping;

			/* Update PDE, mark as present */
			_mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32),
							pde_phys | MALI_MMU_FLAGS_PRESENT);

			MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]);
			pagedir->page_entries_usage_count[i] = 1;
		} else {
			pagedir->page_entries_usage_count[i]++;
		}
	}
	_mali_osk_write_mem_barrier();

	return _MALI_OSK_ERR_OK;
}

MALI_STATIC_INLINE void mali_mmu_zero_pte(mali_io_address page_table, u32 mali_address, u32 size)
{
	int i;
	const int first_pte = MALI_MMU_PTE_ENTRY(mali_address);
	const int last_pte = MALI_MMU_PTE_ENTRY(mali_address + size - 1);

	for (i = first_pte; i <= last_pte; i++) {
		_mali_osk_mem_iowrite32_relaxed(page_table, i * sizeof(u32), 0);
	}
}

static u32 mali_page_directory_get_phys_address(struct mali_page_directory *pagedir, u32 index)
{
	return (_mali_osk_mem_ioread32(pagedir->page_directory_mapped,
				       index * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK);
}


_mali_osk_errcode_t mali_mmu_pagedir_unmap(struct mali_page_directory *pagedir, u32 mali_address, u32 size)
{
	const int first_pde = MALI_MMU_PDE_ENTRY(mali_address);
	const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1);
	u32 left = size;
	int i;
	mali_bool pd_changed = MALI_FALSE;
	u32 pages_to_invalidate[3]; /* hard-coded to 3: max two pages from the PT level plus max one page from PD level */
	u32 num_pages_inv = 0;
	mali_bool invalidate_all = MALI_FALSE; /* safety mechanism in case page_entries_usage_count is unreliable */

	/* For all page directory entries in range. */
	for (i = first_pde; i <= last_pde; i++) {
		u32 size_in_pde, offset;

		MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[i]);
		MALI_DEBUG_ASSERT(0 != pagedir->page_entries_usage_count[i]);

		/* Offset into page table, 0 if mali_address is 4MiB aligned */
		offset = (mali_address & (MALI_MMU_VIRTUAL_PAGE_SIZE - 1));
		if (left < MALI_MMU_VIRTUAL_PAGE_SIZE - offset) {
			size_in_pde = left;
		} else {
			size_in_pde = MALI_MMU_VIRTUAL_PAGE_SIZE - offset;
		}

		pagedir->page_entries_usage_count[i]--;

		/* If entire page table is unused, free it */
		if (0 == pagedir->page_entries_usage_count[i]) {
			u32 page_phys;
			void *page_virt;
			MALI_DEBUG_PRINT(4, ("Releasing page table as this is the last reference\n"));
			/* last reference removed, no need to zero out each PTE  */

			page_phys = MALI_MMU_ENTRY_ADDRESS(_mali_osk_mem_ioread32(pagedir->page_directory_mapped, i * sizeof(u32)));
			page_virt = pagedir->page_entries_mapped[i];
			pagedir->page_entries_mapped[i] = NULL;
			_mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0);

			mali_mmu_release_table_page(page_phys, page_virt);
			pd_changed = MALI_TRUE;
		} else {
			MALI_DEBUG_ASSERT(num_pages_inv < 2);
			if (num_pages_inv < 2) {
				pages_to_invalidate[num_pages_inv] = mali_page_directory_get_phys_address(pagedir, i);
				num_pages_inv++;
			} else {
				invalidate_all = MALI_TRUE;
			}

			/* If part of the page table is still in use, zero the relevant PTEs */
			mali_mmu_zero_pte(pagedir->page_entries_mapped[i], mali_address, size_in_pde);
		}

		left -= size_in_pde;
		mali_address += size_in_pde;
	}
	_mali_osk_write_mem_barrier();

	/* L2 pages invalidation */
	if (MALI_TRUE == pd_changed) {
		MALI_DEBUG_ASSERT(num_pages_inv < 3);
		if (num_pages_inv < 3) {
			pages_to_invalidate[num_pages_inv] = pagedir->page_directory;
			num_pages_inv++;
		} else {
			invalidate_all = MALI_TRUE;
		}
	}

	if (invalidate_all) {
		mali_l2_cache_invalidate_all();
	} else {
		mali_l2_cache_invalidate_all_pages(pages_to_invalidate, num_pages_inv);
	}

	MALI_SUCCESS;
}

struct mali_page_directory *mali_mmu_pagedir_alloc(void)
{
	struct mali_page_directory *pagedir;
	_mali_osk_errcode_t err;
	mali_dma_addr phys;

	pagedir = _mali_osk_calloc(1, sizeof(struct mali_page_directory));
	if (NULL == pagedir) {
		return NULL;
	}

	err = mali_mmu_get_table_page(&phys, &pagedir->page_directory_mapped);
	if (_MALI_OSK_ERR_OK != err) {
		_mali_osk_free(pagedir);
		return NULL;
	}

	pagedir->page_directory = (u32)phys;

	/* Zero page directory */
	fill_page(pagedir->page_directory_mapped, 0);

	return pagedir;
}

void mali_mmu_pagedir_free(struct mali_page_directory *pagedir)
{
	const int num_page_table_entries = sizeof(pagedir->page_entries_mapped) / sizeof(pagedir->page_entries_mapped[0]);
	int i;

	/* Free referenced page tables and zero PDEs. */
	for (i = 0; i < num_page_table_entries; i++) {
		if (pagedir->page_directory_mapped && (_mali_osk_mem_ioread32(
				pagedir->page_directory_mapped,
				sizeof(u32)*i) & MALI_MMU_FLAGS_PRESENT)) {
			mali_dma_addr phys = _mali_osk_mem_ioread32(pagedir->page_directory_mapped,
					     i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK;
			_mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0);
			mali_mmu_release_table_page(phys, pagedir->page_entries_mapped[i]);
		}
	}
	_mali_osk_write_mem_barrier();

	/* Free the page directory page. */
	mali_mmu_release_table_page(pagedir->page_directory, pagedir->page_directory_mapped);

	_mali_osk_free(pagedir);
}


void mali_mmu_pagedir_update(struct mali_page_directory *pagedir, u32 mali_address,
			     mali_dma_addr phys_address, u32 size, u32 permission_bits)
{
	u32 end_address = mali_address + size;
	u32 mali_phys = (u32)phys_address;

	/* Map physical pages into MMU page tables */
	for (; mali_address < end_address; mali_address += MALI_MMU_PAGE_SIZE, mali_phys += MALI_MMU_PAGE_SIZE) {
		MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)]);
		_mali_osk_mem_iowrite32_relaxed(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)],
						MALI_MMU_PTE_ENTRY(mali_address) * sizeof(u32),
						mali_phys | permission_bits);
	}
}

void mali_mmu_pagedir_diag(struct mali_page_directory *pagedir, u32 fault_addr)
{
#if defined(DEBUG)
	u32 pde_index, pte_index;
	u32 pde, pte;

	pde_index = MALI_MMU_PDE_ENTRY(fault_addr);
	pte_index = MALI_MMU_PTE_ENTRY(fault_addr);


	pde = _mali_osk_mem_ioread32(pagedir->page_directory_mapped,
				     pde_index * sizeof(u32));


	if (pde & MALI_MMU_FLAGS_PRESENT) {
		u32 pte_addr = MALI_MMU_ENTRY_ADDRESS(pde);

		pte = _mali_osk_mem_ioread32(pagedir->page_entries_mapped[pde_index],
					     pte_index * sizeof(u32));

		MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table present: %08x\n"
				     "\t\tPTE: %08x, page %08x is %s\n",
				     fault_addr, pte_addr, pte,
				     MALI_MMU_ENTRY_ADDRESS(pte),
				     pte & MALI_MMU_FLAGS_DEFAULT ? "rw" : "not present"));
	} else {
		MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table not present: %08x\n",
				     fault_addr, pde));
	}
#else
	MALI_IGNORE(pagedir);
	MALI_IGNORE(fault_addr);
#endif
}

/* For instrumented */
struct dump_info {
	u32 buffer_left;
	u32 register_writes_size;
	u32 page_table_dump_size;
	u32 *buffer;
};

static _mali_osk_errcode_t writereg(u32 where, u32 what, const char *comment, struct dump_info *info)
{
	if (NULL != info) {
		info->register_writes_size += sizeof(u32) * 2; /* two 32-bit words */

		if (NULL != info->buffer) {
			/* check that we have enough space */
			if (info->buffer_left < sizeof(u32) * 2) MALI_ERROR(_MALI_OSK_ERR_NOMEM);

			*info->buffer = where;
			info->buffer++;

			*info->buffer = what;
			info->buffer++;

			info->buffer_left -= sizeof(u32) * 2;
		}
	}

	MALI_SUCCESS;
}

static _mali_osk_errcode_t mali_mmu_dump_page(mali_io_address page, u32 phys_addr, struct dump_info *info)
{
	if (NULL != info) {
		/* 4096 for the page and 4 bytes for the address */
		const u32 page_size_in_elements = MALI_MMU_PAGE_SIZE / 4;
		const u32 page_size_in_bytes = MALI_MMU_PAGE_SIZE;
		const u32 dump_size_in_bytes = MALI_MMU_PAGE_SIZE + 4;

		info->page_table_dump_size += dump_size_in_bytes;

		if (NULL != info->buffer) {
			if (info->buffer_left < dump_size_in_bytes) MALI_ERROR(_MALI_OSK_ERR_NOMEM);

			*info->buffer = phys_addr;
			info->buffer++;

			_mali_osk_memcpy(info->buffer, page, page_size_in_bytes);
			info->buffer += page_size_in_elements;

			info->buffer_left -= dump_size_in_bytes;
		}
	}

	MALI_SUCCESS;
}

static _mali_osk_errcode_t dump_mmu_page_table(struct mali_page_directory *pagedir, struct dump_info *info)
{
	MALI_DEBUG_ASSERT_POINTER(pagedir);
	MALI_DEBUG_ASSERT_POINTER(info);

	if (NULL != pagedir->page_directory_mapped) {
		int i;

		MALI_CHECK_NO_ERROR(
			mali_mmu_dump_page(pagedir->page_directory_mapped, pagedir->page_directory, info)
		);

		for (i = 0; i < 1024; i++) {
			if (NULL != pagedir->page_entries_mapped[i]) {
				MALI_CHECK_NO_ERROR(
					mali_mmu_dump_page(pagedir->page_entries_mapped[i],
							   _mali_osk_mem_ioread32(pagedir->page_directory_mapped,
									   i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK, info)
				);
			}
		}
	}

	MALI_SUCCESS;
}

static _mali_osk_errcode_t dump_mmu_registers(struct mali_page_directory *pagedir, struct dump_info *info)
{
	MALI_CHECK_NO_ERROR(writereg(0x00000000, pagedir->page_directory,
				     "set the page directory address", info));
	MALI_CHECK_NO_ERROR(writereg(0x00000008, 4, "zap???", info));
	MALI_CHECK_NO_ERROR(writereg(0x00000008, 0, "enable paging", info));
	MALI_SUCCESS;
}

_mali_osk_errcode_t _mali_ukk_query_mmu_page_table_dump_size(_mali_uk_query_mmu_page_table_dump_size_s *args)
{
	struct dump_info info = { 0, 0, 0, NULL };
	struct mali_session_data *session_data;

	session_data = (struct mali_session_data *)(uintptr_t)(args->ctx);
	MALI_DEBUG_ASSERT_POINTER(session_data);
	MALI_DEBUG_ASSERT_POINTER(args);

	MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info));
	MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info));
	args->size = info.register_writes_size + info.page_table_dump_size;
	MALI_SUCCESS;
}

_mali_osk_errcode_t _mali_ukk_dump_mmu_page_table(_mali_uk_dump_mmu_page_table_s *args)
{
	struct dump_info info = { 0, 0, 0, NULL };
	struct mali_session_data *session_data;

	MALI_DEBUG_ASSERT_POINTER(args);

	session_data = (struct mali_session_data *)(uintptr_t)(args->ctx);
	MALI_DEBUG_ASSERT_POINTER(session_data);

	info.buffer_left = args->size;
	info.buffer = (u32 *)(uintptr_t)args->buffer;

	args->register_writes = (uintptr_t)info.buffer;
	MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info));

	args->page_table_dump = (uintptr_t)info.buffer;
	MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info));

	args->register_writes_size = info.register_writes_size;
	args->page_table_dump_size = info.page_table_dump_size;

	MALI_SUCCESS;
}
Commit	Line	Data
6fa3eb70 S	1	/*
	2	* This confidential and proprietary software may be used only as
	3	* authorised by a licensing agreement from ARM Limited
bdc132d7	4	* (C) COPYRIGHT 2011-2015 ARM Limited
6fa3eb70 S	5	* ALL RIGHTS RESERVED
	6	* The entire notice above must be reproduced on all authorised
	7	* copies and copies may only be made to the extent permitted
	8	* by a licensing agreement from ARM Limited.
	9	*/
	10
	11	#include "mali_kernel_common.h"
	12	#include "mali_osk.h"
bdc132d7	13	#include "mali_ukk.h"
6fa3eb70 S	14	#include "mali_uk_types.h"
	15	#include "mali_mmu_page_directory.h"
	16	#include "mali_memory.h"
	17	#include "mali_l2_cache.h"
	18
	19	static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data);
	20
	21	u32 mali_allocate_empty_page(mali_io_address *virt_addr)
	22	{
	23	_mali_osk_errcode_t err;
	24	mali_io_address mapping;
bdc132d7	25	mali_dma_addr address;
6fa3eb70	26
bdc132d7	27	if (_MALI_OSK_ERR_OK != mali_mmu_get_table_page(&address, &mapping)) {
6fa3eb70 S	28	/* Allocation failed */
	29	MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to get table page for empty pgdir\n"));
	30	return 0;
	31	}
	32
bdc132d7	33	MALI_DEBUG_ASSERT_POINTER(mapping);
6fa3eb70 S	34
	35	err = fill_page(mapping, 0);
	36	if (_MALI_OSK_ERR_OK != err) {
	37	mali_mmu_release_table_page(address, mapping);
	38	MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to zero page\n"));
	39	return 0;
	40	}
	41
	42	*virt_addr = mapping;
	43	return address;
	44	}
	45
bdc132d7	46	void mali_free_empty_page(mali_dma_addr address, mali_io_address virt_addr)
6fa3eb70 S	47	{
	48	if (MALI_INVALID_PAGE != address) {
	49	mali_mmu_release_table_page(address, virt_addr);
	50	}
	51	}
	52
bdc132d7 S	53	_mali_osk_errcode_t mali_create_fault_flush_pages(mali_dma_addr *page_directory,
	54	mali_io_address *page_directory_mapping,
	55	mali_dma_addr page_table, mali_io_address page_table_mapping,
	56	mali_dma_addr data_page, mali_io_address data_page_mapping)
6fa3eb70 S	57	{
	58	_mali_osk_errcode_t err;
	59
	60	err = mali_mmu_get_table_page(data_page, data_page_mapping);
	61	if (_MALI_OSK_ERR_OK == err) {
	62	err = mali_mmu_get_table_page(page_table, page_table_mapping);
	63	if (_MALI_OSK_ERR_OK == err) {
	64	err = mali_mmu_get_table_page(page_directory, page_directory_mapping);
	65	if (_MALI_OSK_ERR_OK == err) {
	66	fill_page(*data_page_mapping, 0);
	67	fill_page(page_table_mapping, data_page \| MALI_MMU_FLAGS_DEFAULT);
	68	fill_page(page_directory_mapping, page_table \| MALI_MMU_FLAGS_PRESENT);
	69	MALI_SUCCESS;
	70	}
	71	mali_mmu_release_table_page(page_table, page_table_mapping);
	72	*page_table = MALI_INVALID_PAGE;
	73	}
	74	mali_mmu_release_table_page(data_page, data_page_mapping);
	75	*data_page = MALI_INVALID_PAGE;
	76	}
	77	return err;
	78	}
	79
bdc132d7 S	80	void mali_destroy_fault_flush_pages(
	81	mali_dma_addr page_directory, mali_io_address page_directory_mapping,
	82	mali_dma_addr page_table, mali_io_address page_table_mapping,
	83	mali_dma_addr data_page, mali_io_address data_page_mapping)
6fa3eb70 S	84	{
	85	if (MALI_INVALID_PAGE != *page_directory) {
	86	mali_mmu_release_table_page(page_directory, page_directory_mapping);
	87	*page_directory = MALI_INVALID_PAGE;
	88	*page_directory_mapping = NULL;
	89	}
	90
	91	if (MALI_INVALID_PAGE != *page_table) {
	92	mali_mmu_release_table_page(page_table, page_table_mapping);
	93	*page_table = MALI_INVALID_PAGE;
	94	*page_table_mapping = NULL;
	95	}
	96
	97	if (MALI_INVALID_PAGE != *data_page) {
	98	mali_mmu_release_table_page(data_page, data_page_mapping);
	99	*data_page = MALI_INVALID_PAGE;
	100	*data_page_mapping = NULL;
	101	}
	102	}
	103
	104	static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data)
	105	{
	106	int i;
bdc132d7	107	MALI_DEBUG_ASSERT_POINTER(mapping);
6fa3eb70	108
bdc132d7 S	109	for (i = 0; i < MALI_MMU_PAGE_SIZE / 4; i++) {
bdc132d7 S	110	_mali_osk_mem_iowrite32_relaxed(mapping, i * sizeof(u32), data);
6fa3eb70 S	111	}
	112	_mali_osk_mem_barrier();
	113	MALI_SUCCESS;
	114	}
	115
	116	_mali_osk_errcode_t mali_mmu_pagedir_map(struct mali_page_directory *pagedir, u32 mali_address, u32 size)
	117	{
	118	const int first_pde = MALI_MMU_PDE_ENTRY(mali_address);
	119	const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1);
	120	_mali_osk_errcode_t err;
	121	mali_io_address pde_mapping;
bdc132d7	122	mali_dma_addr pde_phys;
6fa3eb70 S	123	int i;
6fa3eb70 S	124
bdc132d7 S	125	if (last_pde < first_pde)
bdc132d7 S	126	return _MALI_OSK_ERR_INVALID_ARGS;
6fa3eb70	127
bdc132d7 S	128	for (i = first_pde; i <= last_pde; i++) {
	129	if (0 == (_mali_osk_mem_ioread32(pagedir->page_directory_mapped,
	130	i * sizeof(u32)) & MALI_MMU_FLAGS_PRESENT)) {
6fa3eb70 S	131	/* Page table not present */
	132	MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]);
	133	MALI_DEBUG_ASSERT(NULL == pagedir->page_entries_mapped[i]);
	134
	135	err = mali_mmu_get_table_page(&pde_phys, &pde_mapping);
bdc132d7	136	if (_MALI_OSK_ERR_OK != err) {
6fa3eb70 S	137	MALI_PRINT_ERROR(("Failed to allocate page table page.\n"));
	138	return err;
	139	}
	140	pagedir->page_entries_mapped[i] = pde_mapping;
	141
	142	/* Update PDE, mark as present */
bdc132d7 S	143	_mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32),
bdc132d7 S	144	pde_phys \| MALI_MMU_FLAGS_PRESENT);
6fa3eb70 S	145
	146	MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]);
	147	pagedir->page_entries_usage_count[i] = 1;
	148	} else {
	149	pagedir->page_entries_usage_count[i]++;
	150	}
	151	}
	152	_mali_osk_write_mem_barrier();
	153
bdc132d7	154	return _MALI_OSK_ERR_OK;
6fa3eb70 S	155	}
	156
	157	MALI_STATIC_INLINE void mali_mmu_zero_pte(mali_io_address page_table, u32 mali_address, u32 size)
	158	{
	159	int i;
	160	const int first_pte = MALI_MMU_PTE_ENTRY(mali_address);
	161	const int last_pte = MALI_MMU_PTE_ENTRY(mali_address + size - 1);
	162
	163	for (i = first_pte; i <= last_pte; i++) {
	164	_mali_osk_mem_iowrite32_relaxed(page_table, i * sizeof(u32), 0);
	165	}
	166	}
	167
bdc132d7 S	168	static u32 mali_page_directory_get_phys_address(struct mali_page_directory *pagedir, u32 index)
	169	{
	170	return (_mali_osk_mem_ioread32(pagedir->page_directory_mapped,
	171	index * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK);
	172	}
	173
	174
6fa3eb70 S	175	_mali_osk_errcode_t mali_mmu_pagedir_unmap(struct mali_page_directory *pagedir, u32 mali_address, u32 size)
	176	{
	177	const int first_pde = MALI_MMU_PDE_ENTRY(mali_address);
	178	const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1);
	179	u32 left = size;
	180	int i;
	181	mali_bool pd_changed = MALI_FALSE;
	182	u32 pages_to_invalidate[3]; /* hard-coded to 3: max two pages from the PT level plus max one page from PD level */
	183	u32 num_pages_inv = 0;
	184	mali_bool invalidate_all = MALI_FALSE; /* safety mechanism in case page_entries_usage_count is unreliable */
	185
	186	/* For all page directory entries in range. */
	187	for (i = first_pde; i <= last_pde; i++) {
	188	u32 size_in_pde, offset;
	189
	190	MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[i]);
	191	MALI_DEBUG_ASSERT(0 != pagedir->page_entries_usage_count[i]);
	192
	193	/* Offset into page table, 0 if mali_address is 4MiB aligned */
	194	offset = (mali_address & (MALI_MMU_VIRTUAL_PAGE_SIZE - 1));
	195	if (left < MALI_MMU_VIRTUAL_PAGE_SIZE - offset) {
	196	size_in_pde = left;
	197	} else {
	198	size_in_pde = MALI_MMU_VIRTUAL_PAGE_SIZE - offset;
	199	}
	200
	201	pagedir->page_entries_usage_count[i]--;
	202
	203	/* If entire page table is unused, free it */
	204	if (0 == pagedir->page_entries_usage_count[i]) {
	205	u32 page_phys;
	206	void *page_virt;
	207	MALI_DEBUG_PRINT(4, ("Releasing page table as this is the last reference\n"));
	208	/* last reference removed, no need to zero out each PTE */
	209
bdc132d7	210	page_phys = MALI_MMU_ENTRY_ADDRESS(_mali_osk_mem_ioread32(pagedir->page_directory_mapped, i * sizeof(u32)));
6fa3eb70 S	211	page_virt = pagedir->page_entries_mapped[i];
6fa3eb70 S	212	pagedir->page_entries_mapped[i] = NULL;
bdc132d7	213	_mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0);
6fa3eb70 S	214
	215	mali_mmu_release_table_page(page_phys, page_virt);
	216	pd_changed = MALI_TRUE;
	217	} else {
	218	MALI_DEBUG_ASSERT(num_pages_inv < 2);
	219	if (num_pages_inv < 2) {
	220	pages_to_invalidate[num_pages_inv] = mali_page_directory_get_phys_address(pagedir, i);
	221	num_pages_inv++;
	222	} else {
	223	invalidate_all = MALI_TRUE;
	224	}
	225
	226	/* If part of the page table is still in use, zero the relevant PTEs */
	227	mali_mmu_zero_pte(pagedir->page_entries_mapped[i], mali_address, size_in_pde);
	228	}
	229
	230	left -= size_in_pde;
	231	mali_address += size_in_pde;
	232	}
	233	_mali_osk_write_mem_barrier();
	234
	235	/* L2 pages invalidation */
	236	if (MALI_TRUE == pd_changed) {
	237	MALI_DEBUG_ASSERT(num_pages_inv < 3);
	238	if (num_pages_inv < 3) {
	239	pages_to_invalidate[num_pages_inv] = pagedir->page_directory;
	240	num_pages_inv++;
	241	} else {
	242	invalidate_all = MALI_TRUE;
	243	}
	244	}
	245
	246	if (invalidate_all) {
	247	mali_l2_cache_invalidate_all();
	248	} else {
	249	mali_l2_cache_invalidate_all_pages(pages_to_invalidate, num_pages_inv);
	250	}
	251
	252	MALI_SUCCESS;
	253	}
	254
	255	struct mali_page_directory *mali_mmu_pagedir_alloc(void)
	256	{
	257	struct mali_page_directory *pagedir;
bdc132d7 S	258	_mali_osk_errcode_t err;
bdc132d7 S	259	mali_dma_addr phys;
6fa3eb70 S	260
6fa3eb70 S	261	pagedir = _mali_osk_calloc(1, sizeof(struct mali_page_directory));
bdc132d7	262	if (NULL == pagedir) {
6fa3eb70 S	263	return NULL;
	264	}
	265
bdc132d7 S	266	err = mali_mmu_get_table_page(&phys, &pagedir->page_directory_mapped);
bdc132d7 S	267	if (_MALI_OSK_ERR_OK != err) {
6fa3eb70 S	268	_mali_osk_free(pagedir);
	269	return NULL;
	270	}
	271
bdc132d7 S	272	pagedir->page_directory = (u32)phys;
bdc132d7 S	273
6fa3eb70 S	274	/* Zero page directory */
	275	fill_page(pagedir->page_directory_mapped, 0);
	276
	277	return pagedir;
	278	}
	279
	280	void mali_mmu_pagedir_free(struct mali_page_directory *pagedir)
	281	{
	282	const int num_page_table_entries = sizeof(pagedir->page_entries_mapped) / sizeof(pagedir->page_entries_mapped[0]);
	283	int i;
	284
	285	/* Free referenced page tables and zero PDEs. */
	286	for (i = 0; i < num_page_table_entries; i++) {
bdc132d7 S	287	if (pagedir->page_directory_mapped && (_mali_osk_mem_ioread32(
	288	pagedir->page_directory_mapped,
	289	sizeof(u32)*i) & MALI_MMU_FLAGS_PRESENT)) {
	290	mali_dma_addr phys = _mali_osk_mem_ioread32(pagedir->page_directory_mapped,
	291	i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK;
6fa3eb70 S	292	_mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0);
	293	mali_mmu_release_table_page(phys, pagedir->page_entries_mapped[i]);
	294	}
	295	}
	296	_mali_osk_write_mem_barrier();
	297
	298	/* Free the page directory page. */
	299	mali_mmu_release_table_page(pagedir->page_directory, pagedir->page_directory_mapped);
	300
	301	_mali_osk_free(pagedir);
	302	}
	303
	304
bdc132d7 S	305	void mali_mmu_pagedir_update(struct mali_page_directory *pagedir, u32 mali_address,
bdc132d7 S	306	mali_dma_addr phys_address, u32 size, u32 permission_bits)
6fa3eb70 S	307	{
6fa3eb70 S	308	u32 end_address = mali_address + size;
bdc132d7	309	u32 mali_phys = (u32)phys_address;
6fa3eb70 S	310
6fa3eb70 S	311	/* Map physical pages into MMU page tables */
bdc132d7	312	for (; mali_address < end_address; mali_address += MALI_MMU_PAGE_SIZE, mali_phys += MALI_MMU_PAGE_SIZE) {
6fa3eb70 S	313	MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)]);
6fa3eb70 S	314	_mali_osk_mem_iowrite32_relaxed(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)],
bdc132d7 S	315	MALI_MMU_PTE_ENTRY(mali_address) * sizeof(u32),
bdc132d7 S	316	mali_phys \| permission_bits);
6fa3eb70 S	317	}
	318	}
	319
bdc132d7	320	void mali_mmu_pagedir_diag(struct mali_page_directory *pagedir, u32 fault_addr)
6fa3eb70	321	{
bdc132d7 S	322	#if defined(DEBUG)
	323	u32 pde_index, pte_index;
	324	u32 pde, pte;
	325
	326	pde_index = MALI_MMU_PDE_ENTRY(fault_addr);
	327	pte_index = MALI_MMU_PTE_ENTRY(fault_addr);
	328
	329
	330	pde = _mali_osk_mem_ioread32(pagedir->page_directory_mapped,
	331	pde_index * sizeof(u32));
	332
	333
	334	if (pde & MALI_MMU_FLAGS_PRESENT) {
	335	u32 pte_addr = MALI_MMU_ENTRY_ADDRESS(pde);
	336
	337	pte = _mali_osk_mem_ioread32(pagedir->page_entries_mapped[pde_index],
	338	pte_index * sizeof(u32));
	339
	340	MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table present: %08x\n"
	341	"\t\tPTE: %08x, page %08x is %s\n",
	342	fault_addr, pte_addr, pte,
	343	MALI_MMU_ENTRY_ADDRESS(pte),
	344	pte & MALI_MMU_FLAGS_DEFAULT ? "rw" : "not present"));
	345	} else {
	346	MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table not present: %08x\n",
	347	fault_addr, pde));
	348	}
	349	#else
	350	MALI_IGNORE(pagedir);
	351	MALI_IGNORE(fault_addr);
	352	#endif
6fa3eb70 S	353	}
	354
	355	/* For instrumented */
	356	struct dump_info {
	357	u32 buffer_left;
	358	u32 register_writes_size;
	359	u32 page_table_dump_size;
	360	u32 *buffer;
	361	};
	362
	363	static _mali_osk_errcode_t writereg(u32 where, u32 what, const char comment, struct dump_info info)
	364	{
	365	if (NULL != info) {
bdc132d7	366	info->register_writes_size += sizeof(u32) * 2; /* two 32-bit words */
6fa3eb70 S	367
	368	if (NULL != info->buffer) {
	369	/* check that we have enough space */
bdc132d7	370	if (info->buffer_left < sizeof(u32) * 2) MALI_ERROR(_MALI_OSK_ERR_NOMEM);
6fa3eb70 S	371
	372	*info->buffer = where;
	373	info->buffer++;
	374
	375	*info->buffer = what;
	376	info->buffer++;
	377
bdc132d7	378	info->buffer_left -= sizeof(u32) * 2;
6fa3eb70 S	379	}
	380	}
	381
	382	MALI_SUCCESS;
	383	}
	384
bdc132d7	385	static _mali_osk_errcode_t mali_mmu_dump_page(mali_io_address page, u32 phys_addr, struct dump_info *info)
6fa3eb70 S	386	{
	387	if (NULL != info) {
	388	/* 4096 for the page and 4 bytes for the address */
	389	const u32 page_size_in_elements = MALI_MMU_PAGE_SIZE / 4;
	390	const u32 page_size_in_bytes = MALI_MMU_PAGE_SIZE;
	391	const u32 dump_size_in_bytes = MALI_MMU_PAGE_SIZE + 4;
	392
	393	info->page_table_dump_size += dump_size_in_bytes;
	394
	395	if (NULL != info->buffer) {
	396	if (info->buffer_left < dump_size_in_bytes) MALI_ERROR(_MALI_OSK_ERR_NOMEM);
	397
	398	*info->buffer = phys_addr;
	399	info->buffer++;
	400
	401	_mali_osk_memcpy(info->buffer, page, page_size_in_bytes);
	402	info->buffer += page_size_in_elements;
	403
	404	info->buffer_left -= dump_size_in_bytes;
	405	}
	406	}
	407
	408	MALI_SUCCESS;
	409	}
	410
bdc132d7	411	static _mali_osk_errcode_t dump_mmu_page_table(struct mali_page_directory pagedir, struct dump_info info)
6fa3eb70 S	412	{
	413	MALI_DEBUG_ASSERT_POINTER(pagedir);
	414	MALI_DEBUG_ASSERT_POINTER(info);
	415
	416	if (NULL != pagedir->page_directory_mapped) {
	417	int i;
	418
	419	MALI_CHECK_NO_ERROR(
bdc132d7	420	mali_mmu_dump_page(pagedir->page_directory_mapped, pagedir->page_directory, info)
6fa3eb70 S	421	);
	422
	423	for (i = 0; i < 1024; i++) {
	424	if (NULL != pagedir->page_entries_mapped[i]) {
	425	MALI_CHECK_NO_ERROR(
bdc132d7 S	426	mali_mmu_dump_page(pagedir->page_entries_mapped[i],
	427	_mali_osk_mem_ioread32(pagedir->page_directory_mapped,
	428	i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK, info)
6fa3eb70 S	429	);
	430	}
	431	}
	432	}
	433
	434	MALI_SUCCESS;
	435	}
	436
bdc132d7	437	static _mali_osk_errcode_t dump_mmu_registers(struct mali_page_directory pagedir, struct dump_info info)
6fa3eb70 S	438	{
6fa3eb70 S	439	MALI_CHECK_NO_ERROR(writereg(0x00000000, pagedir->page_directory,
bdc132d7	440	"set the page directory address", info));
6fa3eb70 S	441	MALI_CHECK_NO_ERROR(writereg(0x00000008, 4, "zap???", info));
	442	MALI_CHECK_NO_ERROR(writereg(0x00000008, 0, "enable paging", info));
	443	MALI_SUCCESS;
	444	}
	445
bdc132d7	446	_mali_osk_errcode_t _mali_ukk_query_mmu_page_table_dump_size(_mali_uk_query_mmu_page_table_dump_size_s *args)
6fa3eb70 S	447	{
6fa3eb70 S	448	struct dump_info info = { 0, 0, 0, NULL };
bdc132d7	449	struct mali_session_data *session_data;
6fa3eb70	450
bdc132d7 S	451	session_data = (struct mali_session_data *)(uintptr_t)(args->ctx);
bdc132d7 S	452	MALI_DEBUG_ASSERT_POINTER(session_data);
6fa3eb70	453	MALI_DEBUG_ASSERT_POINTER(args);
6fa3eb70 S	454
	455	MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info));
	456	MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info));
	457	args->size = info.register_writes_size + info.page_table_dump_size;
	458	MALI_SUCCESS;
	459	}
	460
bdc132d7	461	_mali_osk_errcode_t _mali_ukk_dump_mmu_page_table(_mali_uk_dump_mmu_page_table_s *args)
6fa3eb70 S	462	{
6fa3eb70 S	463	struct dump_info info = { 0, 0, 0, NULL };
bdc132d7	464	struct mali_session_data *session_data;
6fa3eb70 S	465
6fa3eb70 S	466	MALI_DEBUG_ASSERT_POINTER(args);
6fa3eb70	467
bdc132d7 S	468	session_data = (struct mali_session_data *)(uintptr_t)(args->ctx);
bdc132d7 S	469	MALI_DEBUG_ASSERT_POINTER(session_data);
6fa3eb70 S	470
6fa3eb70 S	471	info.buffer_left = args->size;
bdc132d7	472	info.buffer = (u32 *)(uintptr_t)args->buffer;
6fa3eb70	473
bdc132d7	474	args->register_writes = (uintptr_t)info.buffer;
6fa3eb70 S	475	MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info));
6fa3eb70 S	476
bdc132d7	477	args->page_table_dump = (uintptr_t)info.buffer;
6fa3eb70 S	478	MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info));
	479
	480	args->register_writes_size = info.register_writes_size;
	481	args->page_table_dump_size = info.page_table_dump_size;
	482
	483	MALI_SUCCESS;
	484	}