ARM: Fix build after memfd_create syscall

[GitHub/LineageOS/android_kernel_samsung_universal7580.git] / drivers / iommu / exynos-iovmm-v6.c

/* linux/drivers/iommu/exynos_iovmm.c
 *
 * Copyright (c) 2011-2012 Samsung Electronics Co., Ltd.
 *		http://www.samsung.com
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#ifdef CONFIG_EXYNOS_IOMMU_DEBUG
#define DEBUG
#endif

#include <linux/kernel.h>
#include <linux/hardirq.h>
#include <linux/slab.h>
#include <linux/scatterlist.h>
#include <linux/err.h>
#include <linux/debugfs.h>
#include <linux/delay.h>

#include <linux/exynos_iovmm.h>
#include <plat/cpu.h>

#include "exynos-iommu.h"

/* IOVM region: [0x1000000, 0xE0000000) */
#define IOVA_START_V6	0x10000000
#define IOVM_SIZE_V6	(0xE0000000  - IOVA_START_V6)
#define sg_physically_continuous(sg) (sg_next(sg) == NULL)

/* alloc_iovm_region - Allocate IO virtual memory region
 * vmm: virtual memory allocator
 * size: total size to allocate vm region from @vmm.
 * align: alignment constraints of the allocated virtual address
 * max_align: maximum alignment of allocated virtual address. allocated address
 *            does not need to satisfy larger alignment than max_align.
 * section_offset: page size-aligned offset of iova start address within an 1MB
 *            boundary. The caller of alloc_iovm_region will obtain the
 *            allocated iova + section_offset. This is provided just for the
 *            physically contiguous memory.
 * page_offset: must be smaller than PAGE_SIZE. Just a valut to be added to the
 *         allocated virtual address. This does not effect to the allocaded size
 *         and address.
 *
 * This function returns allocated IO virtual address that satisfies the given
 * constraints: the caller will get the allocated virtual address plus
 * (section_offset + page_offset). Returns 0 if this function is not able
 * to allocate IO virtual memory.
 */
static dma_addr_t alloc_iovm_region(struct exynos_iovmm *vmm, size_t size,
			size_t section_offset,
			off_t page_offset)
{
	u32 index = 0;
	u32 vstart;
	u32 vsize;
	unsigned long end, i;
	struct exynos_vm_region *region;
	size_t align = SECT_SIZE;

	BUG_ON(page_offset >= PAGE_SIZE);

	/* To avoid allocating prefetched iovm region */
	vsize = (ALIGN(size + SZ_128K, SZ_128K) + section_offset) >> PAGE_SHIFT;
	align >>= PAGE_SHIFT;
	section_offset >>= PAGE_SHIFT;

	spin_lock(&vmm->bitmap_lock);
again:
	index = find_next_zero_bit(vmm->vm_map[0],
			IOVM_NUM_PAGES(vmm->iovm_size[0]), index);

	if (align) {
		index = ALIGN(index, align);
		if (index >= IOVM_NUM_PAGES(vmm->iovm_size[0])) {
			spin_unlock(&vmm->bitmap_lock);
			return 0;
		}

		if (test_bit(index, vmm->vm_map[0]))
			goto again;
	}

	end = index + vsize;

	if (end >= IOVM_NUM_PAGES(vmm->iovm_size[0])) {
		spin_unlock(&vmm->bitmap_lock);
		return 0;
	}

	i = find_next_bit(vmm->vm_map[0], end, index);
	if (i < end) {
		index = i + 1;
		goto again;
	}

	bitmap_set(vmm->vm_map[0], index, vsize);

	spin_unlock(&vmm->bitmap_lock);

	vstart = (index << PAGE_SHIFT) + vmm->iova_start[0] + page_offset;

	region = kmalloc(sizeof(*region), GFP_KERNEL);
	if (unlikely(!region)) {
		spin_lock(&vmm->bitmap_lock);
		bitmap_clear(vmm->vm_map[0], index, vsize);
		spin_unlock(&vmm->bitmap_lock);
		return 0;
	}

	INIT_LIST_HEAD(&region->node);
	region->start = vstart;
	region->size = vsize << PAGE_SHIFT;
	region->dummy_size = region->size - size;
	region->section_off = section_offset << PAGE_SHIFT;

	spin_lock(&vmm->vmlist_lock);
	list_add_tail(&region->node, &vmm->regions_list);
	vmm->allocated_size[0] += region->size;
	vmm->num_areas[0]++;
	vmm->num_map++;
	spin_unlock(&vmm->vmlist_lock);

	return region->start + region->section_off;
}

struct exynos_vm_region *find_iovm_region(struct exynos_iovmm *vmm,
							dma_addr_t iova)
{
	struct exynos_vm_region *region;

	spin_lock(&vmm->vmlist_lock);

	list_for_each_entry(region, &vmm->regions_list, node) {
		if (region->start <= iova &&
			(region->start + region->size) > iova) {
			spin_unlock(&vmm->vmlist_lock);
			return region;
		}
	}

	spin_unlock(&vmm->vmlist_lock);

	return NULL;
}

static struct exynos_vm_region *remove_iovm_region(struct exynos_iovmm *vmm,
							dma_addr_t iova)
{
	struct exynos_vm_region *region;

	spin_lock(&vmm->vmlist_lock);

	list_for_each_entry(region, &vmm->regions_list, node) {
		if (region->start + region->section_off == iova) {
			list_del(&region->node);
			vmm->allocated_size[0] -= region->size;
			vmm->num_areas[0]--;
			vmm->num_unmap++;
			spin_unlock(&vmm->vmlist_lock);
			return region;
		}
	}

	spin_unlock(&vmm->vmlist_lock);

	return NULL;
}

static void free_iovm_region(struct exynos_iovmm *vmm,
				struct exynos_vm_region *region)
{
	if (!region)
		return;

	spin_lock(&vmm->bitmap_lock);
	bitmap_clear(vmm->vm_map[0],
		(region->start - vmm->iova_start[0]) >> PAGE_SHIFT,
			region->size >> PAGE_SHIFT);
	spin_unlock(&vmm->bitmap_lock);

	SYSMMU_EVENT_LOG_IOVMM_UNMAP(IOVMM_TO_LOG(vmm),
			region->start, region->start + region->size);

	kfree(region);
}

static dma_addr_t add_iovm_region(struct exynos_iovmm *vmm,
					dma_addr_t start, size_t size)
{
	struct exynos_vm_region *region, *pos;

	region = kmalloc(sizeof(*region), GFP_KERNEL);
	if (!region)
		return 0;

	INIT_LIST_HEAD(&region->node);
	region->start = start;
	region->size = size;

	spin_lock(&vmm->vmlist_lock);

	list_for_each_entry(pos, &vmm->regions_list, node) {
		if ((start < (pos->start + pos->size)) &&
					((start + size) > pos->start)) {
			spin_unlock(&vmm->vmlist_lock);
			kfree(region);
			return 0;
		}
	}

	list_add(&region->node, &vmm->regions_list);

	spin_unlock(&vmm->vmlist_lock);

	return start;
}

static void show_iovm_regions(struct exynos_iovmm *vmm)
{
	struct exynos_vm_region *pos;

	pr_err("LISTING IOVMM REGIONS...\n");
	spin_lock(&vmm->vmlist_lock);
	list_for_each_entry(pos, &vmm->regions_list, node) {
		pr_err("REGION: %#x (SIZE: %#x, +[%#x, %#x])\n",
				pos->start, pos->size,
				pos->section_off, pos->dummy_size);
	}
	spin_unlock(&vmm->vmlist_lock);
	pr_err("END OF LISTING IOVMM REGIONS...\n");
}

int iovmm_activate(struct device *dev)
{
	struct exynos_iovmm *vmm = exynos_get_iovmm(dev);

	if (!vmm) {
		dev_err(dev, "%s: IOVMM not found\n", __func__);
		return -EINVAL;
	}

	return iommu_attach_device(vmm->domain, dev);
}

void iovmm_deactivate(struct device *dev)
{
	struct exynos_iovmm *vmm = exynos_get_iovmm(dev);

	if (!vmm) {
		dev_err(dev, "%s: IOVMM not found\n", __func__);
		return;
	}

	iommu_detach_device(vmm->domain, dev);
}

/* iovmm_map - allocate and map IO virtual memory for the given device
 * dev: device that has IO virtual address space managed by IOVMM
 * sg: list of physically contiguous memory chunks. The preceding chunk needs to
 *     be larger than the following chunks in sg for efficient mapping and
 *     performance. If elements of sg are more than one, physical address of
 *     each chunk needs to be aligned by its size for efficent mapping and TLB
 *     utilization.
 * offset: offset in bytes to be mapped and accessed by dev.
 * size: size in bytes to be mapped and accessed by dev.
 *
 * This function allocates IO virtual memory for the given device and maps the
 * given physical memory conveyed by sg into the allocated IO memory region.
 * Returns allocated IO virtual address if it allocates and maps successfull.
 * Otherwise, minus error number. Caller must check if the return value of this
 * function with IS_ERR_VALUE().
 */
dma_addr_t iovmm_map(struct device *dev, struct scatterlist *sg, off_t offset,
		size_t size, enum dma_data_direction direction, int id)
{
	off_t start_off;
	dma_addr_t addr, start = 0;
	size_t mapped_size = 0;
	struct exynos_iovmm *vmm = exynos_get_iovmm(dev);
	size_t section_offset = 0; /* section offset of contig. mem */
	int ret = 0;
	int idx;
	struct scatterlist *tsg;
	struct exynos_vm_region *region;

	if (vmm == NULL) {
		dev_err(dev, "%s: IOVMM not found\n", __func__);
		return -EINVAL;
	}

	for (; (sg != NULL) && (sg->length < offset); sg = sg_next(sg))
		offset -= sg->length;

	if (sg == NULL) {
		dev_err(dev, "IOVMM: invalid offset to %s.\n", __func__);
		return -EINVAL;
	}

	tsg = sg;

	start_off = offset_in_page(sg_phys(sg) + offset);
	size = PAGE_ALIGN(size + start_off);

	if (sg_physically_continuous(sg)) {
		size_t aligned_pad_size;
		phys_addr_t phys = page_to_phys(sg_page(sg));
		section_offset = phys & (~SECT_MASK);
		aligned_pad_size = ALIGN(phys, SECT_SIZE) - phys;
		if ((sg->length - aligned_pad_size) < SECT_SIZE) {
			aligned_pad_size = ALIGN(phys, LPAGE_SIZE) - phys;
			if ((sg->length - aligned_pad_size) >= LPAGE_SIZE)
				section_offset = phys & (~LPAGE_MASK);
			else
				section_offset = 0;
		}
	}
	start = alloc_iovm_region(vmm, size, section_offset, start_off);
	if (!start) {
		spin_lock(&vmm->vmlist_lock);
		dev_err(dev, "%s: Not enough IOVM space to allocate %#zx\n",
				__func__, size);
		dev_err(dev, "%s: Total %#zx, Allocated %#zx , Chunks %d\n",
				__func__, vmm->iovm_size[0],
				vmm->allocated_size[0], vmm->num_areas[0]);
		spin_unlock(&vmm->vmlist_lock);
		ret = -ENOMEM;
		goto err_map_nomem;
	}

	addr = start - start_off;

	do {
		phys_addr_t phys;
		size_t len;

		phys = sg_phys(sg);
		len = sg->length;

		/* if back to back sg entries are contiguous consolidate them */
		while (sg_next(sg) &&
		       sg_phys(sg) + sg->length == sg_phys(sg_next(sg))) {
			len += sg_next(sg)->length;
			sg = sg_next(sg);
		}

		if (offset > 0) {
			len -= offset;
			phys += offset;
			offset = 0;
		}

		if (offset_in_page(phys)) {
			len += offset_in_page(phys);
			phys = round_down(phys, PAGE_SIZE);
		}

		len = PAGE_ALIGN(len);

		if (len > (size - mapped_size))
			len = size - mapped_size;

		ret = iommu_map(vmm->domain, addr, phys, len, 0);
		if (ret) {
			dev_err(dev, "iommu_map failed w/ err: %d\n", ret);
			break;
		}

		addr += len;
		mapped_size += len;
	} while ((sg = sg_next(sg)) && (mapped_size < size));

	BUG_ON(mapped_size > size);

	if (mapped_size < size) {
		dev_err(dev, "mapped_size(%#zx) is smaller than size(%#zx)\n",
				mapped_size, size);
		if (!ret) {
			dev_err(dev, "ret: %d\n", ret);
			ret = -EINVAL;
		}
		goto err_map_map;
	}

	region = find_iovm_region(vmm, start);
	BUG_ON(!region);

	/*
	 * If pretched SLPD is a fault SLPD in zero_l2_table, FLPD cache
	 * or prefetch buffer caches the address of zero_l2_table.
	 * This function replaces the zero_l2_table with new L2 page
	 * table to write valid mappings.
	 * Accessing the valid area may cause page fault since FLPD
	 * cache may still caches zero_l2_table for the valid area
	 * instead of new L2 page table that have the mapping
	 * information of the valid area
	 * Thus any replacement of zero_l2_table with other valid L2
	 * page table must involve FLPD cache invalidation if the System
	 * MMU have prefetch feature and FLPD cache (version 3.3).
	 * FLPD cache invalidation is performed with TLB invalidation
	 * by VPN without blocking. It is safe to invalidate TLB without
	 * blocking because the target address of TLB invalidation is
	 * not currently mapped.
	 */

	exynos_sysmmu_tlb_invalidate(vmm->domain, region->start, region->size);

	TRACE_LOG_DEV(dev, "IOVMM: Allocated VM region @ %#x/%#x bytes.\n",
								start, size);

	SYSMMU_EVENT_LOG_IOVMM_MAP(IOVMM_TO_LOG(vmm), start, start + size,
					region->size - size);

	return start;

err_map_map:
	iommu_unmap(vmm->domain, start - start_off, mapped_size);
	free_iovm_region(vmm, remove_iovm_region(vmm, start));

	dev_err(dev,
	"Failed(%d) to map IOVMM REGION %pa (SIZE: %#zx, mapped: %#zx)\n",
		ret, &start, size, mapped_size);
	idx = 0;
	do {
		pr_err("SGLIST[%d].size = %#x\n", idx++, tsg->length);
	} while ((tsg = sg_next(tsg)));

	show_iovm_regions(vmm);

err_map_nomem:
	TRACE_LOG_DEV(dev,
		"IOVMM: Failed to allocated VM region for %#x bytes.\n", size);
	return (dma_addr_t)ret;
}

void iovmm_unmap(struct device *dev, dma_addr_t iova)
{
	struct exynos_iovmm *vmm = exynos_get_iovmm(dev);
	struct exynos_vm_region *region;
	size_t unmap_size;

	/* This function must not be called in IRQ handlers */
	BUG_ON(in_irq());

	if (vmm == NULL) {
		dev_err(dev, "%s: IOVMM not found\n", __func__);
		return;
	}

	region = remove_iovm_region(vmm, iova);
	if (region) {
		u32 start = region->start + region->section_off;
		u32 size = region->size - region->dummy_size;

		/* clear page offset */
		if (WARN_ON(start != iova)) {
			dev_err(dev, "IOVMM: "
			   "iova %pa and region %#x(+%#x)@%#x(-%#x) mismatch\n",
				&iova, region->size, region->dummy_size,
				region->start, region->section_off);
			show_iovm_regions(vmm);
			/* reinsert iovm region */
			add_iovm_region(vmm, region->start, region->size);
			kfree(region);
			return;
		}
		unmap_size = iommu_unmap(vmm->domain, start & SPAGE_MASK, size);
		if (unlikely(unmap_size != size)) {
			dev_err(dev,
				"Failed to unmap REGION of %#x:\n", start);
			dev_err(dev, "(SIZE: %#x, iova: %pa, unmapped: %#zx)\n",
					 size, &iova, unmap_size);
			show_iovm_regions(vmm);
			kfree(region);
			BUG();
			return;
		}

		exynos_sysmmu_tlb_invalidate(vmm->domain, region->start, region->size);

		/* 60us is required to guarantee that PTW ends itself */
		udelay(60);

		free_iovm_region(vmm, region);

		TRACE_LOG_DEV(dev, "IOVMM: Unmapped %#x bytes from %#x.\n",
						unmap_size, iova);
	} else {
		dev_err(dev, "IOVMM: No IOVM region %pa to free.\n", &iova);
	}
}

int iovmm_map_oto(struct device *dev, phys_addr_t phys, size_t size)
{
	struct exynos_iovmm *vmm = exynos_get_iovmm(dev);
	int ret;

	BUG_ON(!IS_ALIGNED(phys, PAGE_SIZE));
	BUG_ON(!IS_ALIGNED(size, PAGE_SIZE));

	if (vmm == NULL) {
		dev_err(dev, "%s: IOVMM not found\n", __func__);
		return -EINVAL;
	}

	if (WARN_ON((phys + size) >= IOVA_START_V6)) {
		dev_err(dev,
			"Unable to create one to one mapping for %#zx @ %pa\n",
			size, &phys);
		return -EINVAL;
	}

	if (!add_iovm_region(vmm, (dma_addr_t)phys, size))
		return -EADDRINUSE;

	ret = iommu_map(vmm->domain, (dma_addr_t)phys, phys, size, 0);
	if (ret < 0)
		free_iovm_region(vmm,
				remove_iovm_region(vmm, (dma_addr_t)phys));

	return ret;
}

void iovmm_unmap_oto(struct device *dev, phys_addr_t phys)
{
	struct exynos_iovmm *vmm = exynos_get_iovmm(dev);
	struct exynos_vm_region *region;
	size_t unmap_size;

	/* This function must not be called in IRQ handlers */
	BUG_ON(in_irq());
	BUG_ON(!IS_ALIGNED(phys, PAGE_SIZE));

	if (vmm == NULL) {
		dev_err(dev, "%s: IOVMM not found\n", __func__);
		return;
	}

	region = remove_iovm_region(vmm, (dma_addr_t)phys);
	if (region) {
		unmap_size = iommu_unmap(vmm->domain, (dma_addr_t)phys,
							region->size);
		WARN_ON(unmap_size != region->size);

		exynos_sysmmu_tlb_invalidate(vmm->domain, (dma_addr_t)phys,
					     region->size);

		free_iovm_region(vmm, region);

		TRACE_LOG_DEV(dev, "IOVMM: Unmapped %#x bytes from %#x.\n",
						unmap_size, phys);
	}
}

static struct dentry *exynos_iovmm_debugfs_root;
static struct dentry *exynos_iommu_debugfs_root;

static int exynos_iovmm_create_debugfs(void)
{
	exynos_iovmm_debugfs_root = debugfs_create_dir("iovmm", NULL);
	if (!exynos_iovmm_debugfs_root)
		pr_err("IOVMM: Failed to create debugfs entry\n");
	else
		pr_info("IOVMM: Created debugfs entry at debugfs/iovmm\n");

	exynos_iommu_debugfs_root = debugfs_create_dir("iommu", NULL);
	if (!exynos_iommu_debugfs_root)
		pr_err("IOMMU: Failed to create debugfs entry\n");
	else
		pr_info("IOMMU: Created debugfs entry at debugfs/iommu\n");

	return 0;
}
arch_initcall(exynos_iovmm_create_debugfs);

static int iovmm_debug_show(struct seq_file *s, void *unused)
{
	struct exynos_iovmm *vmm = s->private;
	int i = 0;

	seq_printf(s, "%.6s  %10.s  %10.s  %10.s  %6.s\n",
			"REGION", "VASTART", "SIZE", "FREE", "CHUNKS");
	seq_puts(s, "---------------------------------------------\n");

	spin_lock(&vmm->vmlist_lock);
	while (i < vmm->inplanes) {
		seq_printf(s, "%3s[%d]  %#x  %#10zx  %#10zx  %d\n",
				"in", i, vmm->iova_start[i], vmm->iovm_size[i],
				vmm->iovm_size[i] - vmm->allocated_size[i],
				vmm->num_areas[i]);
		i++;
	}
	while (i < (vmm->inplanes + vmm->onplanes)) {
		seq_printf(s, "%3s[%d]  %#x  %#10zx  %#10zx  %d\n",
				"out", i - vmm->inplanes, vmm->iova_start[i],
				vmm->iovm_size[i],
				vmm->iovm_size[i] - vmm->allocated_size[i],
				vmm->num_areas[i]);
		i++;
	}
	seq_puts(s, "---------------------------------------------\n");
	seq_printf(s, "Total number of mappings  : %d\n", vmm->num_map);
	seq_printf(s, "Total number of unmappings: %d\n", vmm->num_unmap);
	spin_unlock(&vmm->vmlist_lock);

	return 0;
}

static int iovmm_debug_open(struct inode *inode, struct file *file)
{
	return single_open(file, iovmm_debug_show, inode->i_private);
}

static ssize_t iovmm_debug_write(struct file *filp, const char __user *p,
				size_t len, loff_t *off)
{
	struct seq_file *s = filp->private_data;
	struct exynos_iovmm *vmm = s->private;
	/* clears the map count in IOVMM */
	spin_lock(&vmm->vmlist_lock);
	vmm->num_map = 0;
	vmm->num_unmap = 0;
	spin_unlock(&vmm->vmlist_lock);
	return len;
}

static const struct file_operations iovmm_debug_fops = {
	.open = iovmm_debug_open,
	.read = seq_read,
	.write = iovmm_debug_write,
	.llseek = seq_lseek,
	.release = single_release,
};

static void iovmm_register_debugfs(struct exynos_iovmm *vmm)
{
	if (!exynos_iovmm_debugfs_root)
		return;

	debugfs_create_file(vmm->domain_name, 0664,
			exynos_iovmm_debugfs_root, vmm, &iovmm_debug_fops);
}

int exynos_create_iovmm(struct device *dev, int inplanes, int onplanes)
{
	return 0;
}

struct exynos_iovmm *exynos_create_single_iovmm(const char *name)
{
	struct exynos_iovmm *vmm;
	int ret = 0;

	vmm = kzalloc(sizeof(*vmm), GFP_KERNEL);
	if (!vmm) {
		ret = -ENOMEM;
		goto err_alloc_vmm;
	}

	vmm->iovm_size[0] = IOVM_SIZE_V6;
	vmm->iova_start[0] = IOVA_START_V6;
	vmm->vm_map[0] = kzalloc(IOVM_BITMAP_SIZE(IOVM_SIZE_V6), GFP_KERNEL);
	if (!vmm->vm_map[0]) {
		ret = -ENOMEM;
		goto err_setup_domain;
	}

	vmm->inplanes = 1;
	vmm->onplanes = 0;
	vmm->domain = iommu_domain_alloc(&platform_bus_type);
	if (!vmm->domain) {
		ret = -ENOMEM;
		goto err_setup_domain;
	}

	ret = exynos_iommu_init_event_log(IOVMM_TO_LOG(vmm), IOVMM_LOG_LEN);
	if (!ret) {
		iovmm_add_log_to_debugfs(exynos_iovmm_debugfs_root,
					IOVMM_TO_LOG(vmm), name);

		iommu_add_log_to_debugfs(exynos_iommu_debugfs_root,
				IOMMU_TO_LOG(vmm->domain), name);
	} else {
		goto err_init_event_log;
	}

	spin_lock_init(&vmm->vmlist_lock);
	spin_lock_init(&vmm->bitmap_lock);

	INIT_LIST_HEAD(&vmm->regions_list);

	vmm->domain_name = name;

	iovmm_register_debugfs(vmm);

	pr_debug("%s IOVMM: Created %#x B IOVMM from %#x.\n",
			name, IOVM_SIZE_V6, IOVA_START_V6);
	return vmm;

err_init_event_log:
	iommu_domain_free(vmm->domain);
err_setup_domain:
	kfree(vmm);
err_alloc_vmm:
	pr_err("%s IOVMM: Failed to create IOVMM (%d)\n", name, ret);

	return ERR_PTR(ret);
}

void iovmm_set_fault_handler(struct device *dev,
			     iommu_fault_handler_t handler, void *token)
{
	int ret;

	ret = exynos_sysmmu_add_fault_notifier(dev, handler, token);
	if (ret)
		dev_err(dev, "Failed to %s's fault notifier\n", dev_name(dev));
}