[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / iommu / tegra-gart.c

/*
 * IOMMU API for GART in Tegra20
 *
 * Copyright (c) 2010-2012, NVIDIA CORPORATION.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 */

#define pr_fmt(fmt)	"%s(): " fmt, __func__

#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/mm.h>
#include <linux/list.h>
#include <linux/device.h>
#include <linux/io.h>
#include <linux/iommu.h>
#include <linux/of.h>

#include <asm/cacheflush.h>

/* bitmap of the page sizes currently supported */
#define GART_IOMMU_PGSIZES	(SZ_4K)

#define GART_REG_BASE		0x24
#define GART_CONFIG		(0x24 - GART_REG_BASE)
#define GART_ENTRY_ADDR		(0x28 - GART_REG_BASE)
#define GART_ENTRY_DATA		(0x2c - GART_REG_BASE)
#define GART_ENTRY_PHYS_ADDR_VALID	(1 << 31)

#define GART_PAGE_SHIFT		12
#define GART_PAGE_SIZE		(1 << GART_PAGE_SHIFT)
#define GART_PAGE_MASK						\
	(~(GART_PAGE_SIZE - 1) & ~GART_ENTRY_PHYS_ADDR_VALID)

struct gart_client {
	struct device		*dev;
	struct list_head	list;
};

struct gart_device {
	void __iomem		*regs;
	u32			*savedata;
	u32			page_count;	/* total remappable size */
	dma_addr_t		iovmm_base;	/* offset to vmm_area */
	spinlock_t		pte_lock;	/* for pagetable */
	struct list_head	client;
	spinlock_t		client_lock;	/* for client list */
	struct device		*dev;
};

static struct gart_device *gart_handle; /* unique for a system */

#define GART_PTE(_pfn)						\
	(GART_ENTRY_PHYS_ADDR_VALID | ((_pfn) << PAGE_SHIFT))

/*
 * Any interaction between any block on PPSB and a block on APB or AHB
 * must have these read-back to ensure the APB/AHB bus transaction is
 * complete before initiating activity on the PPSB block.
 */
#define FLUSH_GART_REGS(gart)	((void)readl((gart)->regs + GART_CONFIG))

#define for_each_gart_pte(gart, iova)					\
	for (iova = gart->iovmm_base;					\
	     iova < gart->iovmm_base + GART_PAGE_SIZE * gart->page_count; \
	     iova += GART_PAGE_SIZE)

static inline void gart_set_pte(struct gart_device *gart,
				unsigned long offs, u32 pte)
{
	writel(offs, gart->regs + GART_ENTRY_ADDR);
	writel(pte, gart->regs + GART_ENTRY_DATA);

	dev_dbg(gart->dev, "%s %08lx:%08x\n",
		 pte ? "map" : "unmap", offs, pte & GART_PAGE_MASK);
}

static inline unsigned long gart_read_pte(struct gart_device *gart,
					  unsigned long offs)
{
	unsigned long pte;

	writel(offs, gart->regs + GART_ENTRY_ADDR);
	pte = readl(gart->regs + GART_ENTRY_DATA);

	return pte;
}

static void do_gart_setup(struct gart_device *gart, const u32 *data)
{
	unsigned long iova;

	for_each_gart_pte(gart, iova)
		gart_set_pte(gart, iova, data ? *(data++) : 0);

	writel(1, gart->regs + GART_CONFIG);
	FLUSH_GART_REGS(gart);
}

#ifdef DEBUG
static void gart_dump_table(struct gart_device *gart)
{
	unsigned long iova;
	unsigned long flags;

	spin_lock_irqsave(&gart->pte_lock, flags);
	for_each_gart_pte(gart, iova) {
		unsigned long pte;

		pte = gart_read_pte(gart, iova);

		dev_dbg(gart->dev, "%s %08lx:%08lx\n",
			(GART_ENTRY_PHYS_ADDR_VALID & pte) ? "v" : " ",
			iova, pte & GART_PAGE_MASK);
	}
	spin_unlock_irqrestore(&gart->pte_lock, flags);
}
#else
static inline void gart_dump_table(struct gart_device *gart)
{
}
#endif

static inline bool gart_iova_range_valid(struct gart_device *gart,
					 unsigned long iova, size_t bytes)
{
	unsigned long iova_start, iova_end, gart_start, gart_end;

	iova_start = iova;
	iova_end = iova_start + bytes - 1;
	gart_start = gart->iovmm_base;
	gart_end = gart_start + gart->page_count * GART_PAGE_SIZE - 1;

	if (iova_start < gart_start)
		return false;
	if (iova_end > gart_end)
		return false;
	return true;
}

static int gart_iommu_attach_dev(struct iommu_domain *domain,
				 struct device *dev)
{
	struct gart_device *gart;
	struct gart_client *client, *c;
	int err = 0;

	gart = gart_handle;
	if (!gart)
		return -EINVAL;
	domain->priv = gart;

	domain->geometry.aperture_start = gart->iovmm_base;
	domain->geometry.aperture_end   = gart->iovmm_base +
					gart->page_count * GART_PAGE_SIZE - 1;
	domain->geometry.force_aperture = true;

	client = devm_kzalloc(gart->dev, sizeof(*c), GFP_KERNEL);
	if (!client)
		return -ENOMEM;
	client->dev = dev;

	spin_lock(&gart->client_lock);
	list_for_each_entry(c, &gart->client, list) {
		if (c->dev == dev) {
			dev_err(gart->dev,
				"%s is already attached\n", dev_name(dev));
			err = -EINVAL;
			goto fail;
		}
	}
	list_add(&client->list, &gart->client);
	spin_unlock(&gart->client_lock);
	dev_dbg(gart->dev, "Attached %s\n", dev_name(dev));
	return 0;

fail:
	devm_kfree(gart->dev, client);
	spin_unlock(&gart->client_lock);
	return err;
}

static void gart_iommu_detach_dev(struct iommu_domain *domain,
				  struct device *dev)
{
	struct gart_device *gart = domain->priv;
	struct gart_client *c;

	spin_lock(&gart->client_lock);

	list_for_each_entry(c, &gart->client, list) {
		if (c->dev == dev) {
			list_del(&c->list);
			devm_kfree(gart->dev, c);
			dev_dbg(gart->dev, "Detached %s\n", dev_name(dev));
			goto out;
		}
	}
	dev_err(gart->dev, "Couldn't find\n");
out:
	spin_unlock(&gart->client_lock);
}

static int gart_iommu_domain_init(struct iommu_domain *domain)
{
	return 0;
}

static void gart_iommu_domain_destroy(struct iommu_domain *domain)
{
	struct gart_device *gart = domain->priv;

	if (!gart)
		return;

	spin_lock(&gart->client_lock);
	if (!list_empty(&gart->client)) {
		struct gart_client *c;

		list_for_each_entry(c, &gart->client, list)
			gart_iommu_detach_dev(domain, c->dev);
	}
	spin_unlock(&gart->client_lock);
	domain->priv = NULL;
}

static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova,
			  phys_addr_t pa, size_t bytes, int prot)
{
	struct gart_device *gart = domain->priv;
	unsigned long flags;
	unsigned long pfn;

	if (!gart_iova_range_valid(gart, iova, bytes))
		return -EINVAL;

	spin_lock_irqsave(&gart->pte_lock, flags);
	pfn = __phys_to_pfn(pa);
	if (!pfn_valid(pfn)) {
		dev_err(gart->dev, "Invalid page: %08x\n", pa);
		spin_unlock_irqrestore(&gart->pte_lock, flags);
		return -EINVAL;
	}
	gart_set_pte(gart, iova, GART_PTE(pfn));
	FLUSH_GART_REGS(gart);
	spin_unlock_irqrestore(&gart->pte_lock, flags);
	return 0;
}

static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
			       size_t bytes)
{
	struct gart_device *gart = domain->priv;
	unsigned long flags;

	if (!gart_iova_range_valid(gart, iova, bytes))
		return 0;

	spin_lock_irqsave(&gart->pte_lock, flags);
	gart_set_pte(gart, iova, 0);
	FLUSH_GART_REGS(gart);
	spin_unlock_irqrestore(&gart->pte_lock, flags);
	return 0;
}

static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain,
					   dma_addr_t iova)
{
	struct gart_device *gart = domain->priv;
	unsigned long pte;
	phys_addr_t pa;
	unsigned long flags;

	if (!gart_iova_range_valid(gart, iova, 0))
		return -EINVAL;

	spin_lock_irqsave(&gart->pte_lock, flags);
	pte = gart_read_pte(gart, iova);
	spin_unlock_irqrestore(&gart->pte_lock, flags);

	pa = (pte & GART_PAGE_MASK);
	if (!pfn_valid(__phys_to_pfn(pa))) {
		dev_err(gart->dev, "No entry for %08llx:%08x\n",
			 (unsigned long long)iova, pa);
		gart_dump_table(gart);
		return -EINVAL;
	}
	return pa;
}

static int gart_iommu_domain_has_cap(struct iommu_domain *domain,
				     unsigned long cap)
{
	return 0;
}

static struct iommu_ops gart_iommu_ops = {
	.domain_init	= gart_iommu_domain_init,
	.domain_destroy	= gart_iommu_domain_destroy,
	.attach_dev	= gart_iommu_attach_dev,
	.detach_dev	= gart_iommu_detach_dev,
	.map		= gart_iommu_map,
	.unmap		= gart_iommu_unmap,
	.iova_to_phys	= gart_iommu_iova_to_phys,
	.domain_has_cap	= gart_iommu_domain_has_cap,
	.pgsize_bitmap	= GART_IOMMU_PGSIZES,
};

static int tegra_gart_suspend(struct device *dev)
{
	struct gart_device *gart = dev_get_drvdata(dev);
	unsigned long iova;
	u32 *data = gart->savedata;
	unsigned long flags;

	spin_lock_irqsave(&gart->pte_lock, flags);
	for_each_gart_pte(gart, iova)
		*(data++) = gart_read_pte(gart, iova);
	spin_unlock_irqrestore(&gart->pte_lock, flags);
	return 0;
}

static int tegra_gart_resume(struct device *dev)
{
	struct gart_device *gart = dev_get_drvdata(dev);
	unsigned long flags;

	spin_lock_irqsave(&gart->pte_lock, flags);
	do_gart_setup(gart, gart->savedata);
	spin_unlock_irqrestore(&gart->pte_lock, flags);
	return 0;
}

static int tegra_gart_probe(struct platform_device *pdev)
{
	struct gart_device *gart;
	struct resource *res, *res_remap;
	void __iomem *gart_regs;
	int err;
	struct device *dev = &pdev->dev;

	if (gart_handle)
		return -EIO;

	BUILD_BUG_ON(PAGE_SHIFT != GART_PAGE_SHIFT);

	/* the GART memory aperture is required */
	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	res_remap = platform_get_resource(pdev, IORESOURCE_MEM, 1);
	if (!res || !res_remap) {
		dev_err(dev, "GART memory aperture expected\n");
		return -ENXIO;
	}

	gart = devm_kzalloc(dev, sizeof(*gart), GFP_KERNEL);
	if (!gart) {
		dev_err(dev, "failed to allocate gart_device\n");
		return -ENOMEM;
	}

	gart_regs = devm_ioremap(dev, res->start, resource_size(res));
	if (!gart_regs) {
		dev_err(dev, "failed to remap GART registers\n");
		err = -ENXIO;
		goto fail;
	}

	gart->dev = &pdev->dev;
	spin_lock_init(&gart->pte_lock);
	spin_lock_init(&gart->client_lock);
	INIT_LIST_HEAD(&gart->client);
	gart->regs = gart_regs;
	gart->iovmm_base = (dma_addr_t)res_remap->start;
	gart->page_count = (resource_size(res_remap) >> GART_PAGE_SHIFT);

	gart->savedata = vmalloc(sizeof(u32) * gart->page_count);
	if (!gart->savedata) {
		dev_err(dev, "failed to allocate context save area\n");
		err = -ENOMEM;
		goto fail;
	}

	platform_set_drvdata(pdev, gart);
	do_gart_setup(gart, NULL);

	gart_handle = gart;
	bus_set_iommu(&platform_bus_type, &gart_iommu_ops);
	return 0;

fail:
	if (gart_regs)
		devm_iounmap(dev, gart_regs);
	if (gart && gart->savedata)
		vfree(gart->savedata);
	devm_kfree(dev, gart);
	return err;
}

static int tegra_gart_remove(struct platform_device *pdev)
{
	struct gart_device *gart = platform_get_drvdata(pdev);
	struct device *dev = gart->dev;

	writel(0, gart->regs + GART_CONFIG);
	if (gart->savedata)
		vfree(gart->savedata);
	if (gart->regs)
		devm_iounmap(dev, gart->regs);
	devm_kfree(dev, gart);
	gart_handle = NULL;
	return 0;
}

const struct dev_pm_ops tegra_gart_pm_ops = {
	.suspend	= tegra_gart_suspend,
	.resume		= tegra_gart_resume,
};

static struct of_device_id tegra_gart_of_match[] = {
	{ .compatible = "nvidia,tegra20-gart", },
	{ },
};
MODULE_DEVICE_TABLE(of, tegra_gart_of_match);

static struct platform_driver tegra_gart_driver = {
	.probe		= tegra_gart_probe,
	.remove		= tegra_gart_remove,
	.driver = {
		.owner	= THIS_MODULE,
		.name	= "tegra-gart",
		.pm	= &tegra_gart_pm_ops,
		.of_match_table = tegra_gart_of_match,
	},
};

static int tegra_gart_init(void)
{
	return platform_driver_register(&tegra_gart_driver);
}

static void __exit tegra_gart_exit(void)
{
	platform_driver_unregister(&tegra_gart_driver);
}

subsys_initcall(tegra_gart_init);
module_exit(tegra_gart_exit);

MODULE_DESCRIPTION("IOMMU API for GART in Tegra20");
MODULE_AUTHOR("Hiroshi DOYU <hdoyu@nvidia.com>");
MODULE_ALIAS("platform:tegra-gart");
MODULE_LICENSE("GPL v2");
Commit	Line	Data
d53e54b4 HD	1	/*
	2	* IOMMU API for GART in Tegra20
	3	*
	4	* Copyright (c) 2010-2012, NVIDIA CORPORATION. All rights reserved.
	5	*
	6	* This program is free software; you can redistribute it and/or modify it
	7	* under the terms and conditions of the GNU General Public License,
	8	* version 2, as published by the Free Software Foundation.
	9	*
	10	* This program is distributed in the hope it will be useful, but WITHOUT
	11	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	12	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	13	* more details.
	14	*
	15	* You should have received a copy of the GNU General Public License along with
	16	* this program; if not, write to the Free Software Foundation, Inc.,
	17	* 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
	18	*/
	19
	20	#define pr_fmt(fmt) "%s(): " fmt, __func__
	21
	22	#include <linux/module.h>
	23	#include <linux/platform_device.h>
	24	#include <linux/spinlock.h>
	25	#include <linux/slab.h>
	26	#include <linux/vmalloc.h>
	27	#include <linux/mm.h>
	28	#include <linux/list.h>
	29	#include <linux/device.h>
	30	#include <linux/io.h>
	31	#include <linux/iommu.h>
7cffae42	32	#include <linux/of.h>
d53e54b4 HD	33
	34	#include <asm/cacheflush.h>
	35
	36	/* bitmap of the page sizes currently supported */
	37	#define GART_IOMMU_PGSIZES (SZ_4K)
	38
774dfc9b HD	39	#define GART_REG_BASE 0x24
	40	#define GART_CONFIG (0x24 - GART_REG_BASE)
	41	#define GART_ENTRY_ADDR (0x28 - GART_REG_BASE)
	42	#define GART_ENTRY_DATA (0x2c - GART_REG_BASE)
d53e54b4 HD	43	#define GART_ENTRY_PHYS_ADDR_VALID (1 << 31)
	44
	45	#define GART_PAGE_SHIFT 12
	46	#define GART_PAGE_SIZE (1 << GART_PAGE_SHIFT)
	47	#define GART_PAGE_MASK \
	48	(~(GART_PAGE_SIZE - 1) & ~GART_ENTRY_PHYS_ADDR_VALID)
	49
	50	struct gart_client {
	51	struct device *dev;
	52	struct list_head list;
	53	};
	54
	55	struct gart_device {
	56	void __iomem *regs;
	57	u32 *savedata;
	58	u32 page_count; /* total remappable size */
	59	dma_addr_t iovmm_base; /* offset to vmm_area */
	60	spinlock_t pte_lock; /* for pagetable */
	61	struct list_head client;
	62	spinlock_t client_lock; /* for client list */
	63	struct device *dev;
	64	};
	65
	66	static struct gart_device gart_handle; / unique for a system */
	67
	68	#define GART_PTE(_pfn) \
	69	(GART_ENTRY_PHYS_ADDR_VALID \| ((_pfn) << PAGE_SHIFT))
	70
	71	/*
	72	* Any interaction between any block on PPSB and a block on APB or AHB
	73	* must have these read-back to ensure the APB/AHB bus transaction is
	74	* complete before initiating activity on the PPSB block.
	75	*/
	76	#define FLUSH_GART_REGS(gart) ((void)readl((gart)->regs + GART_CONFIG))
	77
	78	#define for_each_gart_pte(gart, iova) \
	79	for (iova = gart->iovmm_base; \
	80	iova < gart->iovmm_base + GART_PAGE_SIZE * gart->page_count; \
	81	iova += GART_PAGE_SIZE)
	82
	83	static inline void gart_set_pte(struct gart_device *gart,
	84	unsigned long offs, u32 pte)
	85	{
	86	writel(offs, gart->regs + GART_ENTRY_ADDR);
	87	writel(pte, gart->regs + GART_ENTRY_DATA);
	88
	89	dev_dbg(gart->dev, "%s %08lx:%08x\n",
	90	pte ? "map" : "unmap", offs, pte & GART_PAGE_MASK);
	91	}
	92
	93	static inline unsigned long gart_read_pte(struct gart_device *gart,
	94	unsigned long offs)
	95	{
	96	unsigned long pte;
	97
	98	writel(offs, gart->regs + GART_ENTRY_ADDR);
	99	pte = readl(gart->regs + GART_ENTRY_DATA);
	100
	101	return pte;
	102	}
	103
	104	static void do_gart_setup(struct gart_device gart, const u32 data)
	105	{
	106	unsigned long iova;
107
108	for_each_gart_pte(gart, iova)
109	gart_set_pte(gart, iova, data ? *(data++) : 0);
110
111	writel(1, gart->regs + GART_CONFIG);
112	FLUSH_GART_REGS(gart);
113	}
114
115	#ifdef DEBUG
116	static void gart_dump_table(struct gart_device *gart)
117	{
118	unsigned long iova;
119	unsigned long flags;
120
121	spin_lock_irqsave(&gart->pte_lock, flags);
122	for_each_gart_pte(gart, iova) {
123	unsigned long pte;
124
125	pte = gart_read_pte(gart, iova);
126
127	dev_dbg(gart->dev, "%s %08lx:%08lx\n",
128	(GART_ENTRY_PHYS_ADDR_VALID & pte) ? "v" : " ",
129	iova, pte & GART_PAGE_MASK);
130	}
131	spin_unlock_irqrestore(&gart->pte_lock, flags);
132	}
133	#else
134	static inline void gart_dump_table(struct gart_device *gart)
135	{
136	}
137	#endif
138
139	static inline bool gart_iova_range_valid(struct gart_device *gart,
140	unsigned long iova, size_t bytes)
141	{
142	unsigned long iova_start, iova_end, gart_start, gart_end;
143
144	iova_start = iova;
145	iova_end = iova_start + bytes - 1;
146	gart_start = gart->iovmm_base;
147	gart_end = gart_start + gart->page_count * GART_PAGE_SIZE - 1;
148
149	if (iova_start < gart_start)
150	return false;
151	if (iova_end > gart_end)
152	return false;
153	return true;
154	}
155
156	static int gart_iommu_attach_dev(struct iommu_domain *domain,
157	struct device *dev)
158	{
159	struct gart_device *gart;
160	struct gart_client client, c;
161	int err = 0;
162
543f3f33	163	gart = gart_handle;
d53e54b4 HD	164	if (!gart)
	165	return -EINVAL;
	166	domain->priv = gart;
	167
23349902 HD	168	domain->geometry.aperture_start = gart->iovmm_base;
	169	domain->geometry.aperture_end = gart->iovmm_base +
	170	gart->page_count * GART_PAGE_SIZE - 1;
	171	domain->geometry.force_aperture = true;
	172
d53e54b4 HD	173	client = devm_kzalloc(gart->dev, sizeof(*c), GFP_KERNEL);
	174	if (!client)
	175	return -ENOMEM;
	176	client->dev = dev;
	177
	178	spin_lock(&gart->client_lock);
	179	list_for_each_entry(c, &gart->client, list) {
	180	if (c->dev == dev) {
	181	dev_err(gart->dev,
	182	"%s is already attached\n", dev_name(dev));
	183	err = -EINVAL;
	184	goto fail;
	185	}
	186	}
	187	list_add(&client->list, &gart->client);
	188	spin_unlock(&gart->client_lock);
	189	dev_dbg(gart->dev, "Attached %s\n", dev_name(dev));
	190	return 0;
	191
	192	fail:
	193	devm_kfree(gart->dev, client);
	194	spin_unlock(&gart->client_lock);
	195	return err;
	196	}
	197
	198	static void gart_iommu_detach_dev(struct iommu_domain *domain,
	199	struct device *dev)
	200	{
	201	struct gart_device *gart = domain->priv;
	202	struct gart_client *c;
	203
	204	spin_lock(&gart->client_lock);
	205
	206	list_for_each_entry(c, &gart->client, list) {
	207	if (c->dev == dev) {
	208	list_del(&c->list);
	209	devm_kfree(gart->dev, c);
	210	dev_dbg(gart->dev, "Detached %s\n", dev_name(dev));
	211	goto out;
	212	}
	213	}
	214	dev_err(gart->dev, "Couldn't find\n");
	215	out:
	216	spin_unlock(&gart->client_lock);
	217	}
	218
	219	static int gart_iommu_domain_init(struct iommu_domain *domain)
	220	{
	221	return 0;
	222	}
	223
	224	static void gart_iommu_domain_destroy(struct iommu_domain *domain)
	225	{
	226	struct gart_device *gart = domain->priv;
	227
	228	if (!gart)
	229	return;
	230
	231	spin_lock(&gart->client_lock);
	232	if (!list_empty(&gart->client)) {
	233	struct gart_client *c;
	234
	235	list_for_each_entry(c, &gart->client, list)
	236	gart_iommu_detach_dev(domain, c->dev);
237	}
238	spin_unlock(&gart->client_lock);
239	domain->priv = NULL;
240	}
241
242	static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova,
243	phys_addr_t pa, size_t bytes, int prot)
244	{
245	struct gart_device *gart = domain->priv;
246	unsigned long flags;
247	unsigned long pfn;
248
249	if (!gart_iova_range_valid(gart, iova, bytes))
250	return -EINVAL;
251
252	spin_lock_irqsave(&gart->pte_lock, flags);
253	pfn = __phys_to_pfn(pa);
254	if (!pfn_valid(pfn)) {
255	dev_err(gart->dev, "Invalid page: %08x\n", pa);
09c32533	256	spin_unlock_irqrestore(&gart->pte_lock, flags);
d53e54b4 HD	257	return -EINVAL;
	258	}
	259	gart_set_pte(gart, iova, GART_PTE(pfn));
	260	FLUSH_GART_REGS(gart);
	261	spin_unlock_irqrestore(&gart->pte_lock, flags);
	262	return 0;
	263	}
	264
	265	static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
	266	size_t bytes)
	267	{
	268	struct gart_device *gart = domain->priv;
	269	unsigned long flags;
	270
	271	if (!gart_iova_range_valid(gart, iova, bytes))
	272	return 0;
	273
	274	spin_lock_irqsave(&gart->pte_lock, flags);
	275	gart_set_pte(gart, iova, 0);
	276	FLUSH_GART_REGS(gart);
	277	spin_unlock_irqrestore(&gart->pte_lock, flags);
	278	return 0;
	279	}
	280
	281	static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac	282	dma_addr_t iova)
d53e54b4 HD	283	{
	284	struct gart_device *gart = domain->priv;
	285	unsigned long pte;
	286	phys_addr_t pa;
	287	unsigned long flags;
	288
	289	if (!gart_iova_range_valid(gart, iova, 0))
	290	return -EINVAL;
	291
	292	spin_lock_irqsave(&gart->pte_lock, flags);
	293	pte = gart_read_pte(gart, iova);
	294	spin_unlock_irqrestore(&gart->pte_lock, flags);
	295
	296	pa = (pte & GART_PAGE_MASK);
	297	if (!pfn_valid(__phys_to_pfn(pa))) {
72ca55db VS	298	dev_err(gart->dev, "No entry for %08llx:%08x\n",
72ca55db VS	299	(unsigned long long)iova, pa);
d53e54b4 HD	300	gart_dump_table(gart);
	301	return -EINVAL;
	302	}
	303	return pa;
	304	}
	305
	306	static int gart_iommu_domain_has_cap(struct iommu_domain *domain,
	307	unsigned long cap)
	308	{
	309	return 0;
	310	}
	311
	312	static struct iommu_ops gart_iommu_ops = {
	313	.domain_init = gart_iommu_domain_init,
	314	.domain_destroy = gart_iommu_domain_destroy,
	315	.attach_dev = gart_iommu_attach_dev,
	316	.detach_dev = gart_iommu_detach_dev,
	317	.map = gart_iommu_map,
	318	.unmap = gart_iommu_unmap,
	319	.iova_to_phys = gart_iommu_iova_to_phys,
	320	.domain_has_cap = gart_iommu_domain_has_cap,
	321	.pgsize_bitmap = GART_IOMMU_PGSIZES,
	322	};
	323
	324	static int tegra_gart_suspend(struct device *dev)
	325	{
	326	struct gart_device *gart = dev_get_drvdata(dev);
	327	unsigned long iova;
	328	u32 *data = gart->savedata;
	329	unsigned long flags;
	330
	331	spin_lock_irqsave(&gart->pte_lock, flags);
	332	for_each_gart_pte(gart, iova)
	333	*(data++) = gart_read_pte(gart, iova);
	334	spin_unlock_irqrestore(&gart->pte_lock, flags);
	335	return 0;
	336	}
	337
	338	static int tegra_gart_resume(struct device *dev)
	339	{
	340	struct gart_device *gart = dev_get_drvdata(dev);
	341	unsigned long flags;
	342
	343	spin_lock_irqsave(&gart->pte_lock, flags);
	344	do_gart_setup(gart, gart->savedata);
	345	spin_unlock_irqrestore(&gart->pte_lock, flags);
	346	return 0;
	347	}
	348
	349	static int tegra_gart_probe(struct platform_device *pdev)
	350	{
	351	struct gart_device *gart;
	352	struct resource res, res_remap;
	353	void __iomem *gart_regs;
	354	int err;
	355	struct device *dev = &pdev->dev;
	356
	357	if (gart_handle)
	358	return -EIO;
	359
	360	BUILD_BUG_ON(PAGE_SHIFT != GART_PAGE_SHIFT);
	361
	362	/* the GART memory aperture is required */
	363	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
364	res_remap = platform_get_resource(pdev, IORESOURCE_MEM, 1);
365	if (!res \|\| !res_remap) {
366	dev_err(dev, "GART memory aperture expected\n");
367	return -ENXIO;
368	}
369
370	gart = devm_kzalloc(dev, sizeof(*gart), GFP_KERNEL);
371	if (!gart) {
372	dev_err(dev, "failed to allocate gart_device\n");
373	return -ENOMEM;
374	}
375
376	gart_regs = devm_ioremap(dev, res->start, resource_size(res));
377	if (!gart_regs) {
378	dev_err(dev, "failed to remap GART registers\n");
379	err = -ENXIO;
380	goto fail;
381	}
382
383	gart->dev = &pdev->dev;
384	spin_lock_init(&gart->pte_lock);
385	spin_lock_init(&gart->client_lock);
386	INIT_LIST_HEAD(&gart->client);
387	gart->regs = gart_regs;
388	gart->iovmm_base = (dma_addr_t)res_remap->start;
389	gart->page_count = (resource_size(res_remap) >> GART_PAGE_SHIFT);
390
391	gart->savedata = vmalloc(sizeof(u32) * gart->page_count);
392	if (!gart->savedata) {
393	dev_err(dev, "failed to allocate context save area\n");
394	err = -ENOMEM;
395	goto fail;
396	}
397
398	platform_set_drvdata(pdev, gart);
399	do_gart_setup(gart, NULL);
400
401	gart_handle = gart;
b7d4bec1	402	bus_set_iommu(&platform_bus_type, &gart_iommu_ops);
d53e54b4 HD	403	return 0;
	404
	405	fail:
	406	if (gart_regs)
	407	devm_iounmap(dev, gart_regs);
	408	if (gart && gart->savedata)
	409	vfree(gart->savedata);
	410	devm_kfree(dev, gart);
	411	return err;
	412	}
	413
	414	static int tegra_gart_remove(struct platform_device *pdev)
	415	{
	416	struct gart_device *gart = platform_get_drvdata(pdev);
	417	struct device *dev = gart->dev;
	418
	419	writel(0, gart->regs + GART_CONFIG);
	420	if (gart->savedata)
	421	vfree(gart->savedata);
	422	if (gart->regs)
	423	devm_iounmap(dev, gart->regs);
	424	devm_kfree(dev, gart);
	425	gart_handle = NULL;
	426	return 0;
	427	}
	428
	429	const struct dev_pm_ops tegra_gart_pm_ops = {
	430	.suspend = tegra_gart_suspend,
	431	.resume = tegra_gart_resume,
	432	};
	433
d34d6517	434	static struct of_device_id tegra_gart_of_match[] = {
7cffae42 TR	435	{ .compatible = "nvidia,tegra20-gart", },
	436	{ },
	437	};
	438	MODULE_DEVICE_TABLE(of, tegra_gart_of_match);
7cffae42	439
d53e54b4 HD	440	static struct platform_driver tegra_gart_driver = {
	441	.probe = tegra_gart_probe,
	442	.remove = tegra_gart_remove,
	443	.driver = {
	444	.owner = THIS_MODULE,
	445	.name = "tegra-gart",
	446	.pm = &tegra_gart_pm_ops,
e664e8c0	447	.of_match_table = tegra_gart_of_match,
d53e54b4 HD	448	},
	449	};
	450
d34d6517	451	static int tegra_gart_init(void)
d53e54b4	452	{
d53e54b4 HD	453	return platform_driver_register(&tegra_gart_driver);
	454	}
	455
	456	static void __exit tegra_gart_exit(void)
	457	{
	458	platform_driver_unregister(&tegra_gart_driver);
	459	}
	460
	461	subsys_initcall(tegra_gart_init);
	462	module_exit(tegra_gart_exit);
	463
	464	MODULE_DESCRIPTION("IOMMU API for GART in Tegra20");
	465	MODULE_AUTHOR("Hiroshi DOYU <hdoyu@nvidia.com>");
7cffae42	466	MODULE_ALIAS("platform:tegra-gart");
d53e54b4	467	MODULE_LICENSE("GPL v2");