From 3fe41bf08a3bf9f512f44177e36b6f1b4f90a962 Mon Sep 17 00:00:00 2001 From: Janghyuck Kim Date: Fri, 22 Apr 2016 14:21:02 +0900 Subject: [PATCH] [COMMON] iommu/exynos: enable IOVMM IOVMM is Exynos I/O Virtual Memory Manager. It manages device virtual address region like allocation and free. IOVMM is connected with exynos iommu domain. Some dummy function is added in exynos-iommu to avoid compilie error. Change-Id: Icdd0fe1acc81240b97361ae495276d920c129ad9 Signed-off-by: Janghyuck Kim --- drivers/iommu/Kconfig | 9 + drivers/iommu/Makefile | 1 + drivers/iommu/exynos-iommu.c | 18 + drivers/iommu/exynos-iommu.h | 70 ++++ drivers/iommu/exynos-iovmm.c | 725 +++++++++++++++++++++++++++++++++++ include/linux/exynos_iovmm.h | 313 +++++++++++++++ 6 files changed, 1136 insertions(+) create mode 100644 drivers/iommu/exynos-iovmm.c create mode 100644 include/linux/exynos_iovmm.h diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index f3a21343e636..8b03da83f9c3 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -266,6 +266,15 @@ config EXYNOS_IOMMU If unsure, say N here. +config EXYNOS_IOVMM + bool "IO Virtual Memory Manager for Exynos IOMMU" + depends on EXYNOS_IOMMU + default y + help + Supporting the users of Exynos IOMMU for allocating and mapping + an IO virtual memory region with a physical memory region + and managing the allocated virtual memory regions. + config EXYNOS_IOMMU_DEBUG bool "Debugging log for Exynos IOMMU" depends on EXYNOS_IOMMU diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 1fb695854809..eb21d4150336 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -25,6 +25,7 @@ obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o obj-$(CONFIG_ROCKCHIP_IOMMU) += rockchip-iommu.o obj-$(CONFIG_TEGRA_IOMMU_GART) += tegra-gart.o obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-smmu.o +obj-$(CONFIG_EXYNOS_IOVMM) += exynos-iovmm.o obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o obj-$(CONFIG_S390_IOMMU) += s390-iommu.o diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 6552a37a3a9e..9ffecb6cd95d 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -29,6 +29,24 @@ static struct kmem_cache *lv2table_kmem_cache; +void exynos_sysmmu_tlb_invalidate(struct iommu_domain *iommu_domain, + dma_addr_t d_start, size_t size) +{ + return; +} + +int exynos_iommu_map_userptr(struct iommu_domain *dom, unsigned long addr, + dma_addr_t d_iova, size_t size, int prot) +{ + return 0; +} + +void exynos_iommu_unmap_userptr(struct iommu_domain *dom, + dma_addr_t d_iova, size_t size) +{ + return; +} + static int __init exynos_sysmmu_probe(struct platform_device *pdev) { /* Dummy */ diff --git a/drivers/iommu/exynos-iommu.h b/drivers/iommu/exynos-iommu.h index 8f6d312d5816..30b61a63fdf3 100644 --- a/drivers/iommu/exynos-iommu.h +++ b/drivers/iommu/exynos-iommu.h @@ -23,6 +23,10 @@ typedef u32 sysmmu_pte_t; +#define IOVM_NUM_PAGES(vmsize) (vmsize / PAGE_SIZE) +#define IOVM_BITMAP_SIZE(vmsize) \ + ((IOVM_NUM_PAGES(vmsize) + BITS_PER_BYTE) / BITS_PER_BYTE) + #define SECT_ORDER 20 #define LPAGE_ORDER 16 #define SPAGE_ORDER 12 @@ -66,6 +70,7 @@ struct exynos_iommu_owner { struct iommu_domain *domain; /* domain of owner */ struct device *master; /* master device */ struct list_head client; /* node for owner clients_list */ + struct exynos_iovmm *vmm_data; }; /* @@ -85,4 +90,69 @@ struct sysmmu_drvdata { phys_addr_t pgtable; /* assigned page table structure */ unsigned int version; /* our version */ }; + +struct exynos_vm_region { + struct list_head node; + u32 start; + u32 size; + u32 section_off; + u32 dummy_size; +}; + +struct exynos_iovmm { + struct iommu_domain *domain; /* iommu domain for this iovmm */ + size_t iovm_size; /* iovm bitmap size per plane */ + u32 iova_start; /* iovm start address per plane */ + unsigned long *vm_map; /* iovm biatmap per plane */ + struct list_head regions_list; /* list of exynos_vm_region */ + spinlock_t vmlist_lock; /* lock for updating regions_list */ + spinlock_t bitmap_lock; /* lock for manipulating bitmaps */ + struct device *dev; /* peripheral device that has this iovmm */ + size_t allocated_size; + int num_areas; + unsigned int num_map; + unsigned int num_unmap; + const char *domain_name; +}; + +void exynos_sysmmu_tlb_invalidate(struct iommu_domain *domain, dma_addr_t start, + size_t size); +int exynos_iommu_map_userptr(struct iommu_domain *dom, unsigned long addr, + dma_addr_t iova, size_t size, int prot); +void exynos_iommu_unmap_userptr(struct iommu_domain *dom, + dma_addr_t iova, size_t size); + +#if defined(CONFIG_EXYNOS_IOVMM) +static inline struct exynos_iovmm *exynos_get_iovmm(struct device *dev) +{ + if (!dev->archdata.iommu) { + dev_err(dev, "%s: System MMU is not configured\n", __func__); + return NULL; + } + + return ((struct exynos_iommu_owner *)dev->archdata.iommu)->vmm_data; +} + +struct exynos_vm_region *find_iovm_region(struct exynos_iovmm *vmm, + dma_addr_t iova); + +struct exynos_iovmm *exynos_create_single_iovmm(const char *name); +#else +static inline struct exynos_iovmm *exynos_get_iovmm(struct device *dev) +{ + return NULL; +} + +struct exynos_vm_region *find_iovm_region(struct exynos_iovmm *vmm, + dma_addr_t iova) +{ + return NULL; +} + +static inline struct exynos_iovmm *exynos_create_single_iovmm(const char *name) +{ + return NULL; +} +#endif /* CONFIG_EXYNOS_IOVMM */ + #endif /* _EXYNOS_IOMMU_H_ */ diff --git a/drivers/iommu/exynos-iovmm.c b/drivers/iommu/exynos-iovmm.c new file mode 100644 index 000000000000..7f0f4963fcc4 --- /dev/null +++ b/drivers/iommu/exynos-iovmm.c @@ -0,0 +1,725 @@ +/* linux/drivers/iommu/exynos_iovmm.c + * + * Copyright (c) 2011-2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifdef CONFIG_EXYNOS_IOMMU_DEBUG +#define DEBUG +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "exynos-iommu.h" + +/* IOVM region: [0x1000000, 0xD0000000) */ +#define IOVA_START 0x10000000 +#define IOVM_SIZE (0xD0000000 - IOVA_START) +#define sg_physically_continuous(sg) (sg_next(sg) == NULL) + +/* alloc_iovm_region - Allocate IO virtual memory region + * vmm: virtual memory allocator + * size: total size to allocate vm region from @vmm. + * align: alignment constraints of the allocated virtual address + * max_align: maximum alignment of allocated virtual address. allocated address + * does not need to satisfy larger alignment than max_align. + * section_offset: page size-aligned offset of iova start address within an 1MB + * boundary. The caller of alloc_iovm_region will obtain the + * allocated iova + section_offset. This is provided just for the + * physically contiguous memory. + * page_offset: must be smaller than PAGE_SIZE. Just a valut to be added to the + * allocated virtual address. This does not effect to the allocaded size + * and address. + * + * This function returns allocated IO virtual address that satisfies the given + * constraints: the caller will get the allocated virtual address plus + * (section_offset + page_offset). Returns 0 if this function is not able + * to allocate IO virtual memory. + */ +static dma_addr_t alloc_iovm_region(struct exynos_iovmm *vmm, size_t size, + size_t section_offset, + off_t page_offset) +{ + u32 index = 0; + u32 vstart; + u32 vsize; + unsigned long end, i; + struct exynos_vm_region *region; + size_t align = SZ_1M; + + BUG_ON(page_offset >= PAGE_SIZE); + + /* To avoid allocating prefetched iovm region */ + vsize = (ALIGN(size + SZ_128K, SZ_128K) + section_offset) >> PAGE_SHIFT; + align >>= PAGE_SHIFT; + section_offset >>= PAGE_SHIFT; + + spin_lock(&vmm->bitmap_lock); +again: + index = find_next_zero_bit(vmm->vm_map, + IOVM_NUM_PAGES(vmm->iovm_size), index); + + if (align) { + index = ALIGN(index, align); + if (index >= IOVM_NUM_PAGES(vmm->iovm_size)) { + spin_unlock(&vmm->bitmap_lock); + return 0; + } + + if (test_bit(index, vmm->vm_map)) + goto again; + } + + end = index + vsize; + + if (end >= IOVM_NUM_PAGES(vmm->iovm_size)) { + spin_unlock(&vmm->bitmap_lock); + return 0; + } + + i = find_next_bit(vmm->vm_map, end, index); + if (i < end) { + index = i + 1; + goto again; + } + + bitmap_set(vmm->vm_map, index, vsize); + + spin_unlock(&vmm->bitmap_lock); + + vstart = (index << PAGE_SHIFT) + vmm->iova_start + page_offset; + + region = kmalloc(sizeof(*region), GFP_KERNEL); + if (unlikely(!region)) { + spin_lock(&vmm->bitmap_lock); + bitmap_clear(vmm->vm_map, index, vsize); + spin_unlock(&vmm->bitmap_lock); + return 0; + } + + INIT_LIST_HEAD(®ion->node); + region->start = vstart; + region->size = vsize << PAGE_SHIFT; + region->dummy_size = region->size - size; + region->section_off = section_offset << PAGE_SHIFT; + + spin_lock(&vmm->vmlist_lock); + list_add_tail(®ion->node, &vmm->regions_list); + vmm->allocated_size += region->size; + vmm->num_areas++; + vmm->num_map++; + spin_unlock(&vmm->vmlist_lock); + + return region->start + region->section_off; +} + +struct exynos_vm_region *find_iovm_region(struct exynos_iovmm *vmm, + dma_addr_t iova) +{ + struct exynos_vm_region *region; + + spin_lock(&vmm->vmlist_lock); + + list_for_each_entry(region, &vmm->regions_list, node) { + if (region->start <= iova && + (region->start + region->size) > iova) { + spin_unlock(&vmm->vmlist_lock); + return region; + } + } + + spin_unlock(&vmm->vmlist_lock); + + return NULL; +} + +static struct exynos_vm_region *remove_iovm_region(struct exynos_iovmm *vmm, + dma_addr_t iova) +{ + struct exynos_vm_region *region; + + spin_lock(&vmm->vmlist_lock); + + list_for_each_entry(region, &vmm->regions_list, node) { + if (region->start + region->section_off == iova) { + list_del(®ion->node); + vmm->allocated_size -= region->size; + vmm->num_areas--; + vmm->num_unmap++; + spin_unlock(&vmm->vmlist_lock); + return region; + } + } + + spin_unlock(&vmm->vmlist_lock); + + return NULL; +} + +static void free_iovm_region(struct exynos_iovmm *vmm, + struct exynos_vm_region *region) +{ + if (!region) + return; + + spin_lock(&vmm->bitmap_lock); + bitmap_clear(vmm->vm_map, + (region->start - vmm->iova_start) >> PAGE_SHIFT, + region->size >> PAGE_SHIFT); + spin_unlock(&vmm->bitmap_lock); + + kfree(region); +} + +static dma_addr_t add_iovm_region(struct exynos_iovmm *vmm, + dma_addr_t start, size_t size) +{ + struct exynos_vm_region *region, *pos; + + region = kmalloc(sizeof(*region), GFP_KERNEL); + if (!region) + return 0; + + INIT_LIST_HEAD(®ion->node); + region->start = start; + region->size = size; + + spin_lock(&vmm->vmlist_lock); + + list_for_each_entry(pos, &vmm->regions_list, node) { + if ((start < (pos->start + pos->size)) && + ((start + size) > pos->start)) { + spin_unlock(&vmm->vmlist_lock); + kfree(region); + return 0; + } + } + + list_add(®ion->node, &vmm->regions_list); + + spin_unlock(&vmm->vmlist_lock); + + return start; +} + +static void show_iovm_regions(struct exynos_iovmm *vmm) +{ + struct exynos_vm_region *pos; + + pr_err("LISTING IOVMM REGIONS...\n"); + spin_lock(&vmm->vmlist_lock); + list_for_each_entry(pos, &vmm->regions_list, node) { + pr_err("REGION: %#x (SIZE: %#x, +[%#x, %#x])\n", + pos->start, pos->size, + pos->section_off, pos->dummy_size); + } + spin_unlock(&vmm->vmlist_lock); + pr_err("END OF LISTING IOVMM REGIONS...\n"); +} + +int iovmm_activate(struct device *dev) +{ + struct exynos_iovmm *vmm = exynos_get_iovmm(dev); + + if (!vmm) { + dev_err(dev, "%s: IOVMM not found\n", __func__); + return -EINVAL; + } + + return iommu_attach_device(vmm->domain, dev); +} + +void iovmm_deactivate(struct device *dev) +{ + struct exynos_iovmm *vmm = exynos_get_iovmm(dev); + + if (!vmm) { + dev_err(dev, "%s: IOVMM not found\n", __func__); + return; + } + + iommu_detach_device(vmm->domain, dev); +} + +struct iommu_domain *get_domain_from_dev(struct device *dev) +{ + struct exynos_iovmm *vmm = exynos_get_iovmm(dev); + + if (!vmm) { + dev_err(dev, "%s: IOVMM not found\n", __func__); + return NULL; + } + + return vmm->domain; +} + +/* iovmm_map - allocate and map IO virtual memory for the given device + * dev: device that has IO virtual address space managed by IOVMM + * sg: list of physically contiguous memory chunks. The preceding chunk needs to + * be larger than the following chunks in sg for efficient mapping and + * performance. If elements of sg are more than one, physical address of + * each chunk needs to be aligned by its size for efficent mapping and TLB + * utilization. + * offset: offset in bytes to be mapped and accessed by dev. + * size: size in bytes to be mapped and accessed by dev. + * + * This function allocates IO virtual memory for the given device and maps the + * given physical memory conveyed by sg into the allocated IO memory region. + * Returns allocated IO virtual address if it allocates and maps successfull. + * Otherwise, minus error number. Caller must check if the return value of this + * function with IS_ERR_VALUE(). + */ +dma_addr_t iovmm_map(struct device *dev, struct scatterlist *sg, off_t offset, + size_t size, enum dma_data_direction direction, int prot) +{ + off_t start_off; + dma_addr_t addr, start = 0; + size_t mapped_size = 0; + struct exynos_iovmm *vmm = exynos_get_iovmm(dev); + size_t section_offset = 0; /* section offset of contig. mem */ + int ret = 0; + int idx; + struct scatterlist *tsg; + struct exynos_vm_region *region; + + if (vmm == NULL) { + dev_err(dev, "%s: IOVMM not found\n", __func__); + return -EINVAL; + } + + for (; (sg != NULL) && (sg->length < offset); sg = sg_next(sg)) + offset -= sg->length; + + if (sg == NULL) { + dev_err(dev, "IOVMM: invalid offset to %s.\n", __func__); + return -EINVAL; + } + + tsg = sg; + + start_off = offset_in_page(sg_phys(sg) + offset); + size = PAGE_ALIGN(size + start_off); + + if (sg_physically_continuous(sg)) { + size_t aligned_pad_size; + phys_addr_t phys = page_to_phys(sg_page(sg)); + section_offset = phys & (~SECT_MASK); + aligned_pad_size = ALIGN(phys, SECT_SIZE) - phys; + if ((sg->length - aligned_pad_size) < SECT_SIZE) { + aligned_pad_size = ALIGN(phys, LPAGE_SIZE) - phys; + if ((sg->length - aligned_pad_size) >= LPAGE_SIZE) + section_offset = phys & (~LPAGE_MASK); + else + section_offset = 0; + } + } + start = alloc_iovm_region(vmm, size, section_offset, start_off); + if (!start) { + spin_lock(&vmm->vmlist_lock); + dev_err(dev, "%s: Not enough IOVM space to allocate %#zx\n", + __func__, size); + dev_err(dev, "%s: Total %#zx, Allocated %#zx , Chunks %d\n", + __func__, vmm->iovm_size, + vmm->allocated_size, vmm->num_areas); + spin_unlock(&vmm->vmlist_lock); + ret = -ENOMEM; + goto err_map_nomem; + } + + addr = start - start_off; + + do { + phys_addr_t phys; + size_t len; + + phys = sg_phys(sg); + len = sg->length; + + /* if back to back sg entries are contiguous consolidate them */ + while (sg_next(sg) && + sg_phys(sg) + sg->length == sg_phys(sg_next(sg))) { + len += sg_next(sg)->length; + sg = sg_next(sg); + } + + if (offset > 0) { + len -= offset; + phys += offset; + offset = 0; + } + + if (offset_in_page(phys)) { + len += offset_in_page(phys); + phys = round_down(phys, PAGE_SIZE); + } + + len = PAGE_ALIGN(len); + + if (len > (size - mapped_size)) + len = size - mapped_size; + + ret = iommu_map(vmm->domain, addr, phys, len, prot); + if (ret) { + dev_err(dev, "iommu_map failed w/ err: %d\n", ret); + break; + } + + addr += len; + mapped_size += len; + } while ((sg = sg_next(sg)) && (mapped_size < size)); + + BUG_ON(mapped_size > size); + + if (mapped_size < size) { + dev_err(dev, "mapped_size(%#zx) is smaller than size(%#zx)\n", + mapped_size, size); + if (!ret) { + dev_err(dev, "ret: %d\n", ret); + ret = -EINVAL; + } + goto err_map_map; + } + + region = find_iovm_region(vmm, start); + BUG_ON(!region); + + /* + * If pretched SLPD is a fault SLPD in zero_l2_table, FLPD cache + * or prefetch buffer caches the address of zero_l2_table. + * This function replaces the zero_l2_table with new L2 page + * table to write valid mappings. + * Accessing the valid area may cause page fault since FLPD + * cache may still caches zero_l2_table for the valid area + * instead of new L2 page table that have the mapping + * information of the valid area + * Thus any replacement of zero_l2_table with other valid L2 + * page table must involve FLPD cache invalidation if the System + * MMU have prefetch feature and FLPD cache (version 3.3). + * FLPD cache invalidation is performed with TLB invalidation + * by VPN without blocking. It is safe to invalidate TLB without + * blocking because the target address of TLB invalidation is + * not currently mapped. + */ + + /* TODO: for sysmmu v6, remove it later */ + exynos_sysmmu_tlb_invalidate(vmm->domain, region->start, region->size); + + dev_dbg(dev, "IOVMM: Allocated VM region @ %#x/%#x bytes.\n", + (unsigned int)start, (unsigned int)size); + + return start; + +err_map_map: + iommu_unmap(vmm->domain, start - start_off, mapped_size); + free_iovm_region(vmm, remove_iovm_region(vmm, start)); + + dev_err(dev, + "Failed(%d) to map IOVMM REGION %pa (SIZE: %#zx, mapped: %#zx)\n", + ret, &start, size, mapped_size); + idx = 0; + do { + pr_err("SGLIST[%d].size = %#x\n", idx++, tsg->length); + } while ((tsg = sg_next(tsg))); + + show_iovm_regions(vmm); + +err_map_nomem: + dev_dbg(dev, + "IOVMM: Failed to allocated VM region for %#x bytes.\n", + (unsigned int)size); + return (dma_addr_t)ret; +} + +void iovmm_unmap(struct device *dev, dma_addr_t iova) +{ + struct exynos_iovmm *vmm = exynos_get_iovmm(dev); + struct exynos_vm_region *region; + size_t unmap_size; + + /* This function must not be called in IRQ handlers */ + BUG_ON(in_irq()); + + if (vmm == NULL) { + dev_err(dev, "%s: IOVMM not found\n", __func__); + return; + } + + region = remove_iovm_region(vmm, iova); + if (region) { + u32 start = region->start + region->section_off; + u32 size = region->size - region->dummy_size; + + /* clear page offset */ + if (WARN_ON(start != iova)) { + dev_err(dev, "IOVMM: " + "iova %pa and region %#x(+%#x)@%#x(-%#x) mismatch\n", + &iova, region->size, region->dummy_size, + region->start, region->section_off); + show_iovm_regions(vmm); + /* reinsert iovm region */ + add_iovm_region(vmm, region->start, region->size); + kfree(region); + return; + } + unmap_size = iommu_unmap(vmm->domain, start & SPAGE_MASK, size); + if (unlikely(unmap_size != size)) { + dev_err(dev, + "Failed to unmap REGION of %#x:\n", start); + dev_err(dev, "(SIZE: %#x, iova: %pa, unmapped: %#zx)\n", + size, &iova, unmap_size); + show_iovm_regions(vmm); + kfree(region); + BUG(); + return; + } + + exynos_sysmmu_tlb_invalidate(vmm->domain, region->start, region->size); + + /* TODO: for sysmmu v6, remove it later */ + /* 60us is required to guarantee that PTW ends itself */ + udelay(60); + + free_iovm_region(vmm, region); + + dev_dbg(dev, "IOVMM: Unmapped %#x bytes from %#x.\n", + (unsigned int)unmap_size, (unsigned int)iova); + } else { + dev_err(dev, "IOVMM: No IOVM region %pa to free.\n", &iova); + } +} + +/* + * NOTE: + * exynos_iovmm_map_userptr() should be called under current->mm.mmap_sem held. + */ +dma_addr_t exynos_iovmm_map_userptr(struct device *dev, unsigned long vaddr, + size_t size, int prot) +{ + struct exynos_iovmm *vmm = exynos_get_iovmm(dev); + unsigned long eaddr = vaddr + size; + off_t offset = offset_in_page(vaddr); + int ret = -EINVAL; + struct vm_area_struct *vma; + dma_addr_t start; + struct exynos_vm_region *region; + + vma = find_vma(current->mm, vaddr); + if (vaddr < vma->vm_start) { + dev_err(dev, "%s: invalid address %#lx\n", __func__, vaddr); + goto err; + } + + if (!!(vma->vm_flags & VM_PFNMAP)) + prot |= IOMMU_PFNMAP; + + while (eaddr > vma->vm_end) { + if (!!(vma->vm_flags & VM_PFNMAP)) { + dev_err(dev, "%s: non-linear pfnmap is not supported\n", + __func__); + goto err; + } + + if ((vma->vm_next == NULL) || + (vma->vm_end != vma->vm_next->vm_start)) { + dev_err(dev, "%s: invalid size %zu\n", __func__, size); + goto err; + } + + vma = vma->vm_next; + } + + size = PAGE_ALIGN(size + offset); + start = alloc_iovm_region(vmm, size, 0, offset); + if (!start) { + spin_lock(&vmm->vmlist_lock); + dev_err(dev, "%s: Not enough IOVM space to allocate %#zx\n", + __func__, size); + dev_err(dev, "%s: Total %#zx, Allocated %#zx , Chunks %d\n", + __func__, vmm->iovm_size, + vmm->allocated_size, vmm->num_areas); + spin_unlock(&vmm->vmlist_lock); + ret = -ENOMEM; + goto err; + } + + ret = exynos_iommu_map_userptr(vmm->domain, vaddr - offset, + start - offset, size, prot); + if (ret < 0) + goto err_map; + + region = find_iovm_region(vmm, start); + BUG_ON(!region); + + return start; +err_map: + free_iovm_region(vmm, remove_iovm_region(vmm, start)); +err: + return (dma_addr_t)ret; +} + +void exynos_iovmm_unmap_userptr(struct device *dev, dma_addr_t iova) +{ + struct exynos_iovmm *vmm = exynos_get_iovmm(dev); + struct exynos_vm_region *region; + + region = remove_iovm_region(vmm, iova); + if (region) { + u32 start = region->start + region->section_off; + u32 size = region->size - region->dummy_size; + + /* clear page offset */ + if (WARN_ON(start != iova)) { + dev_err(dev, "IOVMM: " + "iova %pa and region %#x(+%#x)@%#x(-%#x) mismatch\n", + &iova, region->size, region->dummy_size, + region->start, region->section_off); + show_iovm_regions(vmm); + /* reinsert iovm region */ + add_iovm_region(vmm, region->start, region->size); + kfree(region); + return; + } + + exynos_iommu_unmap_userptr(vmm->domain, + start & SPAGE_MASK, size); + + free_iovm_region(vmm, region); + } else { + dev_err(dev, "IOVMM: No IOVM region %pa to free.\n", &iova); + } +} + +static struct dentry *exynos_iovmm_debugfs_root; +static struct dentry *exynos_iommu_debugfs_root; + +static int exynos_iovmm_create_debugfs(void) +{ + exynos_iovmm_debugfs_root = debugfs_create_dir("iovmm", NULL); + if (!exynos_iovmm_debugfs_root) + pr_err("IOVMM: Failed to create debugfs entry\n"); + else + pr_info("IOVMM: Created debugfs entry at debugfs/iovmm\n"); + + exynos_iommu_debugfs_root = debugfs_create_dir("iommu", NULL); + if (!exynos_iommu_debugfs_root) + pr_err("IOMMU: Failed to create debugfs entry\n"); + else + pr_info("IOMMU: Created debugfs entry at debugfs/iommu\n"); + + return 0; +} +arch_initcall(exynos_iovmm_create_debugfs); + +static int iovmm_debug_show(struct seq_file *s, void *unused) +{ + struct exynos_iovmm *vmm = s->private; + + seq_printf(s, "%10.s %10.s %10.s %6.s\n", + "VASTART", "SIZE", "FREE", "CHUNKS"); + seq_puts(s, "---------------------------------------------\n"); + + spin_lock(&vmm->vmlist_lock); + seq_printf(s, " %#x %#10zx %#10zx %d\n", + vmm->iova_start, vmm->iovm_size, + vmm->iovm_size - vmm->allocated_size, + vmm->num_areas); + seq_puts(s, "---------------------------------------------\n"); + seq_printf(s, "Total number of mappings : %d\n", vmm->num_map); + seq_printf(s, "Total number of unmappings: %d\n", vmm->num_unmap); + spin_unlock(&vmm->vmlist_lock); + + return 0; +} + +static int iovmm_debug_open(struct inode *inode, struct file *file) +{ + return single_open(file, iovmm_debug_show, inode->i_private); +} + +static ssize_t iovmm_debug_write(struct file *filp, const char __user *p, + size_t len, loff_t *off) +{ + struct seq_file *s = filp->private_data; + struct exynos_iovmm *vmm = s->private; + /* clears the map count in IOVMM */ + spin_lock(&vmm->vmlist_lock); + vmm->num_map = 0; + vmm->num_unmap = 0; + spin_unlock(&vmm->vmlist_lock); + return len; +} + +static const struct file_operations iovmm_debug_fops = { + .open = iovmm_debug_open, + .read = seq_read, + .write = iovmm_debug_write, + .llseek = seq_lseek, + .release = single_release, +}; + +static void iovmm_register_debugfs(struct exynos_iovmm *vmm) +{ + if (!exynos_iovmm_debugfs_root) + return; + + debugfs_create_file(vmm->domain_name, 0664, + exynos_iovmm_debugfs_root, vmm, &iovmm_debug_fops); +} + +struct exynos_iovmm *exynos_create_single_iovmm(const char *name) +{ + struct exynos_iovmm *vmm; + int ret = 0; + + vmm = kzalloc(sizeof(*vmm), GFP_KERNEL); + if (!vmm) { + ret = -ENOMEM; + goto err_alloc_vmm; + } + + vmm->iovm_size = IOVM_SIZE; + vmm->iova_start = IOVA_START; + vmm->vm_map = kzalloc(IOVM_BITMAP_SIZE(IOVM_SIZE), GFP_KERNEL); + if (!vmm->vm_map) { + ret = -ENOMEM; + goto err_setup_domain; + } + + vmm->domain = iommu_domain_alloc(&platform_bus_type); + if (!vmm->domain) { + ret = -ENOMEM; + goto err_setup_domain; + } + + spin_lock_init(&vmm->vmlist_lock); + spin_lock_init(&vmm->bitmap_lock); + + INIT_LIST_HEAD(&vmm->regions_list); + + vmm->domain_name = name; + + iovmm_register_debugfs(vmm); + + pr_debug("%s IOVMM: Created %#x B IOVMM from %#x.\n", + name, IOVM_SIZE, IOVA_START); + return vmm; + +err_setup_domain: + kfree(vmm); +err_alloc_vmm: + pr_err("%s IOVMM: Failed to create IOVMM (%d)\n", name, ret); + + return ERR_PTR(ret); +} diff --git a/include/linux/exynos_iovmm.h b/include/linux/exynos_iovmm.h new file mode 100644 index 000000000000..d219a9c37bc3 --- /dev/null +++ b/include/linux/exynos_iovmm.h @@ -0,0 +1,313 @@ +/* /include/linux/exynos_iovmm.h + * + * Copyright (c) 2011 Samsung Electronics Co., Ltd. + * http://www.samsung.com + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __ASM_PLAT_IOVMM_H +#define __ASM_PLAT_IOVMM_H + +#include +#include + +#include +#include + +#define IOMMU_PFNMAP (1 << 5) /* VM_PFNMAP is set */ + +struct scatterlist; +struct device; + +typedef u32 exynos_iova_t; + +#define SYSMMU_FAULT_BITS 4 +#define SYSMMU_FAULT_SHIFT 16 +#define SYSMMU_FAULT_MASK ((1 << SYSMMU_FAULT_BITS) - 1) +#define SYSMMU_FAULT_FLAG(id) (((id) & SYSMMU_FAULT_MASK) << SYSMMU_FAULT_SHIFT) +#define SYSMMU_FAULT_ID(fg) (((fg) >> SYSMMU_FAULT_SHIFT) & SYSMMU_FAULT_MASK) + +#define SYSMMU_FAULT_PTW_ACCESS 0 +#define SYSMMU_FAULT_PAGE_FAULT 1 +#define SYSMMU_FAULT_TLB_MULTIHIT 2 +#define SYSMMU_FAULT_ACCESS 3 +#define SYSMMU_FAULT_SECURITY 4 +#define SYSMMU_FAULT_UNKNOWN 5 + +#define IOMMU_FAULT_EXYNOS_PTW_ACCESS SYSMMU_FAULT_FLAG(SYSMMU_FAULT_PTW_ACCESS) +#define IOMMU_FAULT_EXYNOS_PAGE_FAULT SYSMMU_FAULT_FLAG(SYSMMU_FAULT_PAGE_FAULT) +#define IOMMU_FAULT_EXYNOS_TLB_MULTIHIT \ + SYSMMU_FAULT_FLAG(SYSMMU_FAULT_TLB_MULTIHIT) +#define IOMMU_FAULT_EXYNOS_ACCESS SYSMMU_FAULT_FLAG(SYSMMU_FAULT_ACCESS) +#define IOMMU_FAULT_EXYNOS_SECURITY SYSMMU_FAULT_FLAG(SYSMMU_FAULT_SECURITY) +#define IOMMU_FAULT_EXYNOS_UNKNOWN SYSMMU_FAULT_FLAG(SYSMMU_FAULT_UNKOWN) + +/* TODO: PB related for sysmmu v6, remove it later */ +#define SYSMMU_PBUFCFG_TLB_UPDATE (1 << 16) +#define SYSMMU_PBUFCFG_ASCENDING (1 << 12) +#define SYSMMU_PBUFCFG_DESCENDING (0 << 12) +#define SYSMMU_PBUFCFG_PREFETCH (1 << 8) +#define SYSMMU_PBUFCFG_WRITE (1 << 4) +#define SYSMMU_PBUFCFG_READ (0 << 4) + +#define SYSMMU_PBUFCFG_DEFAULT_INPUT (SYSMMU_PBUFCFG_TLB_UPDATE | \ + SYSMMU_PBUFCFG_ASCENDING | \ + SYSMMU_PBUFCFG_PREFETCH | \ + SYSMMU_PBUFCFG_READ) +#define SYSMMU_PBUFCFG_DEFAULT_OUTPUT (SYSMMU_PBUFCFG_TLB_UPDATE | \ + SYSMMU_PBUFCFG_ASCENDING | \ + SYSMMU_PBUFCFG_PREFETCH | \ + SYSMMU_PBUFCFG_WRITE) + +#define SYSMMU_PBUFCFG_ASCENDING_INPUT (SYSMMU_PBUFCFG_TLB_UPDATE | \ + SYSMMU_PBUFCFG_ASCENDING | \ + SYSMMU_PBUFCFG_PREFETCH | \ + SYSMMU_PBUFCFG_READ) + +#define SYSMMU_PBUFCFG_DESCENDING_INPUT (SYSMMU_PBUFCFG_TLB_UPDATE | \ + SYSMMU_PBUFCFG_DESCENDING | \ + SYSMMU_PBUFCFG_PREFETCH | \ + SYSMMU_PBUFCFG_READ) +/* SYSMMU PPC Event ID */ +enum sysmmu_ppc_event { + READ_TOTAL, + READ_L1TLB_MISS, + READ_L2TLB_MISS, + READ_FLPD_MISS, + READ_PB_LOOKUP, + READ_PB_MISS, + READ_BLOCK_NUM_BY_PREFETCH, + READ_BLOCK_CYCLE_BRY_PREFETCH, + READ_TLB_MISS, + READ_FLPD_MISS_PREFETCH, + WRITE_TOTAL = 0x10, + WRITE_L1TLB_MISS, + WRITE_L2TLB_MISS, + WRITE_FLPD_MISS, + WRITE_PB_LOOKUP, + WRITE_PB_MISS, + WRITE_BLOCK_NUM_BY_PREFETCH, + WRITE_BLOCK_CYCLE_BY_PREFETCH, + WRITE_TLB_MISS, + WRITE_FLPD_MISS_PREFETCH, + TOTAL_ID_NUM, +}; + +struct sysmmu_prefbuf { + unsigned long base; + unsigned long size; + unsigned long config; +}; + +#if defined(CONFIG_EXYNOS_IOVMM) +int iovmm_activate(struct device *dev); +void iovmm_deactivate(struct device *dev); +struct iommu_domain *get_domain_from_dev(struct device *dev); + +/* iovmm_map() - Maps a list of physical memory chunks + * @dev: the owner of the IO address space where the mapping is created + * @sg: list of physical memory chunks to map + * @offset: length in bytes where the mapping starts + * @size: how much memory to map in bytes. @offset + @size must not exceed + * total size of @sg + * @direction: dma data direction for iova + * @prot: iommu mapping property + * + * This function returns mapped IO address in the address space of @dev. + * Returns minus error number if mapping fails. + * Caller must check its return code with IS_ERROR_VALUE() if the function + * succeeded. + * + * The caller of this function must ensure that iovmm_cleanup() is not called + * while this function is called. + * + */ +dma_addr_t iovmm_map(struct device *dev, struct scatterlist *sg, off_t offset, + size_t size, enum dma_data_direction direction, int prot); + +/* iovmm_unmap() - unmaps the given IO address + * @dev: the owner of the IO address space where @iova belongs + * @iova: IO address that needs to be unmapped and freed. + * + * The caller of this function must ensure that iovmm_cleanup() is not called + * while this function is called. + */ +void iovmm_unmap(struct device *dev, dma_addr_t iova); + +/* + * flags to option_iplanes and option_oplanes. + * inplanes and onplanes is 'input planes' and 'output planes', respectively. + * + * default value to option_iplanes: + * (TLB_UPDATE | ASCENDING | PREFETCH) + * default value to option_oplanes: + * (TLB_UPDATE | ASCENDING | PREFETCH | WRITE) + * + * SYSMMU_PBUFCFG_READ and SYSMMU_PBUFCFG_WRITE are ignored because they are + * implicitly set from 'inplanes' and 'onplanes' arguments to + * iovmm_set_prefetch_buffer(). + * + * Guide to setting flags: + * - Clear SYSMMU_BUFCFG_TLB_UPDATE if a buffer is accessed by the device + * for rotation. + * - Set SYSMMU_PBUFCFG_DESCENDING if the device access a buffer in reversed + * order + * - Clear SYSMMU_PBUFCFG_PREFETCH if access to a buffer has poor locality. + * - Otherwise, always set flags as default value. + */ +#else +#define iovmm_activate(dev) (-ENOSYS) +#define iovmm_deactivate(dev) do { } while (0) +#define iovmm_map(dev, sg, offset, size, direction, prot) (-ENOSYS) +#define iovmm_unmap(dev, iova) do { } while (0) +#define get_domain_from_dev(dev) NULL +static inline dma_addr_t exynos_iovmm_map_userptr(struct device *dev, + unsigned long vaddr, size_t size, int prot) +{ +} +#define exynos_iovmm_unmap_userptr(dev, iova) do { } while (0) + +#endif /* CONFIG_EXYNOS_IOVMM */ + +#if defined(CONFIG_EXYNOS_IOMMU) +/** + * exynos_sysmmu_map_user_pages() - maps all pages by fetching from + * user page table entries. + * @dev: The device whose System MMU is about to be disabled. + * @mm: mm struct of user requested to map + * @vaddr: start vaddr in valid vma + * @iova: start io vaddr to be mapped + * @size: size to map + * @write: set if buffer may be written + * @shareable: set shareable bit if true + * + * This function maps all user pages into sysmmu page table. + */ +int exynos_sysmmu_map_user_pages(struct device *dev, + struct mm_struct *mm, + unsigned long vaddr, + exynos_iova_t iova, + size_t size, bool write, + bool shareable); + +/** + * exynos_sysmmu_unmap_user_pages() - unmaps all mapped pages + * @dev: The device whose System MMU is about to be disabled. + * @mm: mm struct of user requested to map + * @vaddr: start vaddr in valid vma + * @iova: start io vaddr to be unmapped + * @size: size to map + * + * This function unmaps all user pages mapped in sysmmu page table. + */ +int exynos_sysmmu_unmap_user_pages(struct device *dev, + struct mm_struct *mm, + unsigned long vaddr, + exynos_iova_t iova, + size_t size); +/** + * exynos_iommu_sync_for_device() + * - maintain cache lines on the given area before DMA + * @dev: The device that is about to see the area + * @iova: The start DMA address of @dev to maintain + * @len: The length of the area + * @dir: Indicate whether @dev read from or write to the area + */ +void exynos_iommu_sync_for_device(struct device *dev, dma_addr_t iova, + size_t len, enum dma_data_direction dir); + +/** + * exynos_iommu_sync_for_cpu() + * - maintain cache lines on the given area after DMA + * @dev: The device that is about to see the area + * @iova: The start DMA address of @dev to maintain + * @len: The length of the area + * @dir: Indicate whether @dev read from or write to the area + */ +void exynos_iommu_sync_for_cpu(struct device *dev, dma_addr_t iova, size_t len, + enum dma_data_direction dir); + +/** + * TODO: description + */ +dma_addr_t exynos_iovmm_map_userptr(struct device *dev, unsigned long vaddr, + size_t size, int prot); +/** + * TODO: description + */ +void exynos_iovmm_unmap_userptr(struct device *dev, dma_addr_t iova); + +/* + * The handle_pte_fault() is called by exynos_sysmmu_map_user_pages(). + * Driver cannot include include/linux/huge_mm.h because + * CONFIG_TRANSPARENT_HUGEPAGE is disabled. + */ +extern int handle_pte_fault(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long address, + pte_t *pte, pmd_t *pmd, unsigned int flags); + +/* + * sysmmu_set_prefetch_buffer_by_region() - set prefetch buffer configuration + * + * @dev: device descriptor of master device + * @pb_reg: array of regions where prefetch buffer contains. + * + * If @dev is NULL or @pb_reg is 0, prefetch buffers is disabled. + * + */ +void sysmmu_set_prefetch_buffer_by_region(struct device *dev, + struct sysmmu_prefbuf pb_reg[], unsigned int num_reg); + +int sysmmu_set_prefetch_buffer_property(struct device *dev, + unsigned int inplanes, unsigned int onplanes, + unsigned int ipoption[], unsigned int opoption[]); +void exynos_sysmmu_show_status(struct device *dev); +void exynos_sysmmu_dump_pgtable(struct device *dev); + +/* + * exynos_sysmmu_set/clear/show_ppc_event() - + * set/clear/show system mmu ppc event + * + * @dev: device descriptor of master device. + * @event: system mmu ppc event id. + * Returns 0 if setting is successful. -EINVAL if the argument is invalid. + * + */ +int exynos_sysmmu_set_ppc_event(struct device *dev, int event); +void exynos_sysmmu_clear_ppc_event(struct device *dev); +void exynos_sysmmu_show_ppc_event(struct device *dev); + +/* + * iovmm_set_fault_handler - register fault handler of dev to iommu controller + * @dev: the device that wants to register fault handler + * @handler: fault handler + * @token: any data the device driver needs to get when fault occurred + */ +void iovmm_set_fault_handler(struct device *dev, + iommu_fault_handler_t handler, void *token); + +#else +#define sysmmu_set_prefetch_buffer_property(dev, inplanes, onplnes, ipoption, opoption) \ + (0) +#define sysmmu_set_prefetch_buffer_by_region(dev, pb_reg, num_reg) \ + do { } while (0) +#define exynos_sysmmu_map_user_pages(dev, mm, vaddr, iova, size, write, sharable) \ + (-ENOSYS) +#define exynos_sysmmu_unmap_user_pages(dev, mm, vaddr, iova, size) \ + do { } while (0) +#define exynos_sysmmu_show_status(dev) do { } while (0) +#define exynos_sysmmu_dump_pgtable(dev) do { } while (0) + +#define exynos_sysmmu_clear_ppc_event(dev) do { } while (0) +#define exynos_sysmmu_show_ppc_event(dev) do { } while (0) +#define exynos_sysmmu_set_ppc_event(dev, event) do { } while (0) +#define iovmm_set_fault_handler(dev, handler, token) do { } while(0) + +#define exynos_iommu_sync_for_device(dev, iova, len, dir) do { } while (0) +#define exynos_iommu_sync_for_cpu(dev, iova, len, dir) do { } while (0) + +#endif +#endif /*__ASM_PLAT_IOVMM_H*/ -- 2.20.1