[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / iommu / tegra-gart.c

/*
 * IOMMU API for GART in Tegra20
 *
 * Copyright (c) 2010-2012, NVIDIA CORPORATION.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 */

#define pr_fmt(fmt)	"%s(): " fmt, __func__

#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/mm.h>
#include <linux/list.h>
#include <linux/device.h>
#include <linux/io.h>
#include <linux/iommu.h>
#include <linux/of.h>

#include <asm/cacheflush.h>

/* bitmap of the page sizes currently supported */
#define GART_IOMMU_PGSIZES	(SZ_4K)

#define GART_CONFIG		0x24
#define GART_ENTRY_ADDR		0x28
#define GART_ENTRY_DATA		0x2c
#define GART_ENTRY_PHYS_ADDR_VALID	(1 << 31)

#define GART_PAGE_SHIFT		12
#define GART_PAGE_SIZE		(1 << GART_PAGE_SHIFT)
#define GART_PAGE_MASK						\
	(~(GART_PAGE_SIZE - 1) & ~GART_ENTRY_PHYS_ADDR_VALID)

struct gart_client {
	struct device		*dev;
	struct list_head	list;
};

struct gart_device {
	void __iomem		*regs;
	u32			*savedata;
	u32			page_count;	/* total remappable size */
	dma_addr_t		iovmm_base;	/* offset to vmm_area */
	spinlock_t		pte_lock;	/* for pagetable */
	struct list_head	client;
	spinlock_t		client_lock;	/* for client list */
	struct device		*dev;
};

static struct gart_device *gart_handle; /* unique for a system */

#define GART_PTE(_pfn)						\
	(GART_ENTRY_PHYS_ADDR_VALID | ((_pfn) << PAGE_SHIFT))

/*
 * Any interaction between any block on PPSB and a block on APB or AHB
 * must have these read-back to ensure the APB/AHB bus transaction is
 * complete before initiating activity on the PPSB block.
 */
#define FLUSH_GART_REGS(gart)	((void)readl((gart)->regs + GART_CONFIG))

#define for_each_gart_pte(gart, iova)					\
	for (iova = gart->iovmm_base;					\
	     iova < gart->iovmm_base + GART_PAGE_SIZE * gart->page_count; \
	     iova += GART_PAGE_SIZE)

static inline void gart_set_pte(struct gart_device *gart,
				unsigned long offs, u32 pte)
{
	writel(offs, gart->regs + GART_ENTRY_ADDR);
	writel(pte, gart->regs + GART_ENTRY_DATA);

	dev_dbg(gart->dev, "%s %08lx:%08x\n",
		 pte ? "map" : "unmap", offs, pte & GART_PAGE_MASK);
}

static inline unsigned long gart_read_pte(struct gart_device *gart,
					  unsigned long offs)
{
	unsigned long pte;

	writel(offs, gart->regs + GART_ENTRY_ADDR);
	pte = readl(gart->regs + GART_ENTRY_DATA);

	return pte;
}

static void do_gart_setup(struct gart_device *gart, const u32 *data)
{
	unsigned long iova;

	for_each_gart_pte(gart, iova)
		gart_set_pte(gart, iova, data ? *(data++) : 0);

	writel(1, gart->regs + GART_CONFIG);
	FLUSH_GART_REGS(gart);
}

#ifdef DEBUG
static void gart_dump_table(struct gart_device *gart)
{
	unsigned long iova;
	unsigned long flags;

	spin_lock_irqsave(&gart->pte_lock, flags);
	for_each_gart_pte(gart, iova) {
		unsigned long pte;

		pte = gart_read_pte(gart, iova);

		dev_dbg(gart->dev, "%s %08lx:%08lx\n",
			(GART_ENTRY_PHYS_ADDR_VALID & pte) ? "v" : " ",
			iova, pte & GART_PAGE_MASK);
	}
	spin_unlock_irqrestore(&gart->pte_lock, flags);
}
#else
static inline void gart_dump_table(struct gart_device *gart)
{
}
#endif

static inline bool gart_iova_range_valid(struct gart_device *gart,
					 unsigned long iova, size_t bytes)
{
	unsigned long iova_start, iova_end, gart_start, gart_end;

	iova_start = iova;
	iova_end = iova_start + bytes - 1;
	gart_start = gart->iovmm_base;
	gart_end = gart_start + gart->page_count * GART_PAGE_SIZE - 1;

	if (iova_start < gart_start)
		return false;
	if (iova_end > gart_end)
		return false;
	return true;
}

static int gart_iommu_attach_dev(struct iommu_domain *domain,
				 struct device *dev)
{
	struct gart_device *gart;
	struct gart_client *client, *c;
	int err = 0;

	gart = gart_handle;
	if (!gart)
		return -EINVAL;
	domain->priv = gart;

	client = devm_kzalloc(gart->dev, sizeof(*c), GFP_KERNEL);
	if (!client)
		return -ENOMEM;
	client->dev = dev;

	spin_lock(&gart->client_lock);
	list_for_each_entry(c, &gart->client, list) {
		if (c->dev == dev) {
			dev_err(gart->dev,
				"%s is already attached\n", dev_name(dev));
			err = -EINVAL;
			goto fail;
		}
	}
	list_add(&client->list, &gart->client);
	spin_unlock(&gart->client_lock);
	dev_dbg(gart->dev, "Attached %s\n", dev_name(dev));
	return 0;

fail:
	devm_kfree(gart->dev, client);
	spin_unlock(&gart->client_lock);
	return err;
}

static void gart_iommu_detach_dev(struct iommu_domain *domain,
				  struct device *dev)
{
	struct gart_device *gart = domain->priv;
	struct gart_client *c;

	spin_lock(&gart->client_lock);

	list_for_each_entry(c, &gart->client, list) {
		if (c->dev == dev) {
			list_del(&c->list);
			devm_kfree(gart->dev, c);
			dev_dbg(gart->dev, "Detached %s\n", dev_name(dev));
			goto out;
		}
	}
	dev_err(gart->dev, "Couldn't find\n");
out:
	spin_unlock(&gart->client_lock);
}

static int gart_iommu_domain_init(struct iommu_domain *domain)
{
	return 0;
}

static void gart_iommu_domain_destroy(struct iommu_domain *domain)
{
	struct gart_device *gart = domain->priv;

	if (!gart)
		return;

	spin_lock(&gart->client_lock);
	if (!list_empty(&gart->client)) {
		struct gart_client *c;

		list_for_each_entry(c, &gart->client, list)
			gart_iommu_detach_dev(domain, c->dev);
	}
	spin_unlock(&gart->client_lock);
	domain->priv = NULL;
}

static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova,
			  phys_addr_t pa, size_t bytes, int prot)
{
	struct gart_device *gart = domain->priv;
	unsigned long flags;
	unsigned long pfn;

	if (!gart_iova_range_valid(gart, iova, bytes))
		return -EINVAL;

	spin_lock_irqsave(&gart->pte_lock, flags);
	pfn = __phys_to_pfn(pa);
	if (!pfn_valid(pfn)) {
		dev_err(gart->dev, "Invalid page: %08x\n", pa);
		spin_unlock_irqrestore(&gart->pte_lock, flags);
		return -EINVAL;
	}
	gart_set_pte(gart, iova, GART_PTE(pfn));
	FLUSH_GART_REGS(gart);
	spin_unlock_irqrestore(&gart->pte_lock, flags);
	return 0;
}

static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
			       size_t bytes)
{
	struct gart_device *gart = domain->priv;
	unsigned long flags;

	if (!gart_iova_range_valid(gart, iova, bytes))
		return 0;

	spin_lock_irqsave(&gart->pte_lock, flags);
	gart_set_pte(gart, iova, 0);
	FLUSH_GART_REGS(gart);
	spin_unlock_irqrestore(&gart->pte_lock, flags);
	return 0;
}

static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain,
					   unsigned long iova)
{
	struct gart_device *gart = domain->priv;
	unsigned long pte;
	phys_addr_t pa;
	unsigned long flags;

	if (!gart_iova_range_valid(gart, iova, 0))
		return -EINVAL;

	spin_lock_irqsave(&gart->pte_lock, flags);
	pte = gart_read_pte(gart, iova);
	spin_unlock_irqrestore(&gart->pte_lock, flags);

	pa = (pte & GART_PAGE_MASK);
	if (!pfn_valid(__phys_to_pfn(pa))) {
		dev_err(gart->dev, "No entry for %08lx:%08x\n", iova, pa);
		gart_dump_table(gart);
		return -EINVAL;
	}
	return pa;
}

static int gart_iommu_domain_has_cap(struct iommu_domain *domain,
				     unsigned long cap)
{
	return 0;
}

static struct iommu_ops gart_iommu_ops = {
	.domain_init	= gart_iommu_domain_init,
	.domain_destroy	= gart_iommu_domain_destroy,
	.attach_dev	= gart_iommu_attach_dev,
	.detach_dev	= gart_iommu_detach_dev,
	.map		= gart_iommu_map,
	.unmap		= gart_iommu_unmap,
	.iova_to_phys	= gart_iommu_iova_to_phys,
	.domain_has_cap	= gart_iommu_domain_has_cap,
	.pgsize_bitmap	= GART_IOMMU_PGSIZES,
};

static int tegra_gart_suspend(struct device *dev)
{
	struct gart_device *gart = dev_get_drvdata(dev);
	unsigned long iova;
	u32 *data = gart->savedata;
	unsigned long flags;

	spin_lock_irqsave(&gart->pte_lock, flags);
	for_each_gart_pte(gart, iova)
		*(data++) = gart_read_pte(gart, iova);
	spin_unlock_irqrestore(&gart->pte_lock, flags);
	return 0;
}

static int tegra_gart_resume(struct device *dev)
{
	struct gart_device *gart = dev_get_drvdata(dev);
	unsigned long flags;

	spin_lock_irqsave(&gart->pte_lock, flags);
	do_gart_setup(gart, gart->savedata);
	spin_unlock_irqrestore(&gart->pte_lock, flags);
	return 0;
}

static int tegra_gart_probe(struct platform_device *pdev)
{
	struct gart_device *gart;
	struct resource *res, *res_remap;
	void __iomem *gart_regs;
	int err;
	struct device *dev = &pdev->dev;

	if (gart_handle)
		return -EIO;

	BUILD_BUG_ON(PAGE_SHIFT != GART_PAGE_SHIFT);

	/* the GART memory aperture is required */
	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	res_remap = platform_get_resource(pdev, IORESOURCE_MEM, 1);
	if (!res || !res_remap) {
		dev_err(dev, "GART memory aperture expected\n");
		return -ENXIO;
	}

	gart = devm_kzalloc(dev, sizeof(*gart), GFP_KERNEL);
	if (!gart) {
		dev_err(dev, "failed to allocate gart_device\n");
		return -ENOMEM;
	}

	gart_regs = devm_ioremap(dev, res->start, resource_size(res));
	if (!gart_regs) {
		dev_err(dev, "failed to remap GART registers\n");
		err = -ENXIO;
		goto fail;
	}

	gart->dev = &pdev->dev;
	spin_lock_init(&gart->pte_lock);
	spin_lock_init(&gart->client_lock);
	INIT_LIST_HEAD(&gart->client);
	gart->regs = gart_regs;
	gart->iovmm_base = (dma_addr_t)res_remap->start;
	gart->page_count = (resource_size(res_remap) >> GART_PAGE_SHIFT);

	gart->savedata = vmalloc(sizeof(u32) * gart->page_count);
	if (!gart->savedata) {
		dev_err(dev, "failed to allocate context save area\n");
		err = -ENOMEM;
		goto fail;
	}

	platform_set_drvdata(pdev, gart);
	do_gart_setup(gart, NULL);

	gart_handle = gart;
	return 0;

fail:
	if (gart_regs)
		devm_iounmap(dev, gart_regs);
	if (gart && gart->savedata)
		vfree(gart->savedata);
	devm_kfree(dev, gart);
	return err;
}

static int tegra_gart_remove(struct platform_device *pdev)
{
	struct gart_device *gart = platform_get_drvdata(pdev);
	struct device *dev = gart->dev;

	writel(0, gart->regs + GART_CONFIG);
	if (gart->savedata)
		vfree(gart->savedata);
	if (gart->regs)
		devm_iounmap(dev, gart->regs);
	devm_kfree(dev, gart);
	gart_handle = NULL;
	return 0;
}

const struct dev_pm_ops tegra_gart_pm_ops = {
	.suspend	= tegra_gart_suspend,
	.resume		= tegra_gart_resume,
};

#ifdef CONFIG_OF
static struct of_device_id tegra_gart_of_match[] __devinitdata = {
	{ .compatible = "nvidia,tegra20-gart", },
	{ },
};
MODULE_DEVICE_TABLE(of, tegra_gart_of_match);
#endif

static struct platform_driver tegra_gart_driver = {
	.probe		= tegra_gart_probe,
	.remove		= tegra_gart_remove,
	.driver = {
		.owner	= THIS_MODULE,
		.name	= "tegra-gart",
		.pm	= &tegra_gart_pm_ops,
		.of_match_table = of_match_ptr(tegra_gart_of_match),
	},
};

static int __devinit tegra_gart_init(void)
{
	bus_set_iommu(&platform_bus_type, &gart_iommu_ops);
	return platform_driver_register(&tegra_gart_driver);
}

static void __exit tegra_gart_exit(void)
{
	platform_driver_unregister(&tegra_gart_driver);
}

subsys_initcall(tegra_gart_init);
module_exit(tegra_gart_exit);

MODULE_DESCRIPTION("IOMMU API for GART in Tegra20");
MODULE_AUTHOR("Hiroshi DOYU <hdoyu@nvidia.com>");
MODULE_ALIAS("platform:tegra-gart");
MODULE_LICENSE("GPL v2");
Commit	Line	Data
d53e54b4 HD	1	/*
	2	* IOMMU API for GART in Tegra20
	3	*
	4	* Copyright (c) 2010-2012, NVIDIA CORPORATION. All rights reserved.
	5	*
	6	* This program is free software; you can redistribute it and/or modify it
	7	* under the terms and conditions of the GNU General Public License,
	8	* version 2, as published by the Free Software Foundation.
	9	*
	10	* This program is distributed in the hope it will be useful, but WITHOUT
	11	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	12	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	13	* more details.
	14	*
	15	* You should have received a copy of the GNU General Public License along with
	16	* this program; if not, write to the Free Software Foundation, Inc.,
	17	* 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
	18	*/
	19
	20	#define pr_fmt(fmt) "%s(): " fmt, __func__
	21
	22	#include <linux/module.h>
	23	#include <linux/platform_device.h>
	24	#include <linux/spinlock.h>
	25	#include <linux/slab.h>
	26	#include <linux/vmalloc.h>
	27	#include <linux/mm.h>
	28	#include <linux/list.h>
	29	#include <linux/device.h>
	30	#include <linux/io.h>
	31	#include <linux/iommu.h>
7cffae42	32	#include <linux/of.h>
d53e54b4 HD	33
	34	#include <asm/cacheflush.h>
	35
	36	/* bitmap of the page sizes currently supported */
	37	#define GART_IOMMU_PGSIZES (SZ_4K)
	38
	39	#define GART_CONFIG 0x24
	40	#define GART_ENTRY_ADDR 0x28
	41	#define GART_ENTRY_DATA 0x2c
	42	#define GART_ENTRY_PHYS_ADDR_VALID (1 << 31)
	43
	44	#define GART_PAGE_SHIFT 12
	45	#define GART_PAGE_SIZE (1 << GART_PAGE_SHIFT)
	46	#define GART_PAGE_MASK \
	47	(~(GART_PAGE_SIZE - 1) & ~GART_ENTRY_PHYS_ADDR_VALID)
	48
	49	struct gart_client {
	50	struct device *dev;
	51	struct list_head list;
	52	};
	53
	54	struct gart_device {
	55	void __iomem *regs;
	56	u32 *savedata;
	57	u32 page_count; /* total remappable size */
	58	dma_addr_t iovmm_base; /* offset to vmm_area */
	59	spinlock_t pte_lock; /* for pagetable */
	60	struct list_head client;
	61	spinlock_t client_lock; /* for client list */
	62	struct device *dev;
	63	};
	64
	65	static struct gart_device gart_handle; / unique for a system */
	66
	67	#define GART_PTE(_pfn) \
	68	(GART_ENTRY_PHYS_ADDR_VALID \| ((_pfn) << PAGE_SHIFT))
	69
	70	/*
	71	* Any interaction between any block on PPSB and a block on APB or AHB
	72	* must have these read-back to ensure the APB/AHB bus transaction is
	73	* complete before initiating activity on the PPSB block.
	74	*/
	75	#define FLUSH_GART_REGS(gart) ((void)readl((gart)->regs + GART_CONFIG))
	76
	77	#define for_each_gart_pte(gart, iova) \
	78	for (iova = gart->iovmm_base; \
	79	iova < gart->iovmm_base + GART_PAGE_SIZE * gart->page_count; \
	80	iova += GART_PAGE_SIZE)
	81
	82	static inline void gart_set_pte(struct gart_device *gart,
	83	unsigned long offs, u32 pte)
	84	{
	85	writel(offs, gart->regs + GART_ENTRY_ADDR);
	86	writel(pte, gart->regs + GART_ENTRY_DATA);
	87
	88	dev_dbg(gart->dev, "%s %08lx:%08x\n",
	89	pte ? "map" : "unmap", offs, pte & GART_PAGE_MASK);
	90	}
	91
	92	static inline unsigned long gart_read_pte(struct gart_device *gart,
	93	unsigned long offs)
	94	{
	95	unsigned long pte;
	96
97	writel(offs, gart->regs + GART_ENTRY_ADDR);
98	pte = readl(gart->regs + GART_ENTRY_DATA);
99
100	return pte;
101	}
102
103	static void do_gart_setup(struct gart_device gart, const u32 data)
104	{
105	unsigned long iova;
106
107	for_each_gart_pte(gart, iova)
108	gart_set_pte(gart, iova, data ? *(data++) : 0);
109
110	writel(1, gart->regs + GART_CONFIG);
111	FLUSH_GART_REGS(gart);
112	}
113
114	#ifdef DEBUG
115	static void gart_dump_table(struct gart_device *gart)
116	{
117	unsigned long iova;
118	unsigned long flags;
119
120	spin_lock_irqsave(&gart->pte_lock, flags);
121	for_each_gart_pte(gart, iova) {
122	unsigned long pte;
123
124	pte = gart_read_pte(gart, iova);
125
126	dev_dbg(gart->dev, "%s %08lx:%08lx\n",
127	(GART_ENTRY_PHYS_ADDR_VALID & pte) ? "v" : " ",
128	iova, pte & GART_PAGE_MASK);
129	}
130	spin_unlock_irqrestore(&gart->pte_lock, flags);
131	}
132	#else
133	static inline void gart_dump_table(struct gart_device *gart)
134	{
135	}
136	#endif
137
138	static inline bool gart_iova_range_valid(struct gart_device *gart,
139	unsigned long iova, size_t bytes)
140	{
141	unsigned long iova_start, iova_end, gart_start, gart_end;
142
143	iova_start = iova;
144	iova_end = iova_start + bytes - 1;
145	gart_start = gart->iovmm_base;
146	gart_end = gart_start + gart->page_count * GART_PAGE_SIZE - 1;
147
148	if (iova_start < gart_start)
149	return false;
150	if (iova_end > gart_end)
151	return false;
152	return true;
153	}
154
155	static int gart_iommu_attach_dev(struct iommu_domain *domain,
156	struct device *dev)
157	{
158	struct gart_device *gart;
159	struct gart_client client, c;
160	int err = 0;
161
543f3f33	162	gart = gart_handle;
d53e54b4 HD	163	if (!gart)
	164	return -EINVAL;
	165	domain->priv = gart;
	166
	167	client = devm_kzalloc(gart->dev, sizeof(*c), GFP_KERNEL);
	168	if (!client)
	169	return -ENOMEM;
	170	client->dev = dev;
	171
	172	spin_lock(&gart->client_lock);
	173	list_for_each_entry(c, &gart->client, list) {
	174	if (c->dev == dev) {
	175	dev_err(gart->dev,
	176	"%s is already attached\n", dev_name(dev));
	177	err = -EINVAL;
	178	goto fail;
	179	}
	180	}
	181	list_add(&client->list, &gart->client);
	182	spin_unlock(&gart->client_lock);
	183	dev_dbg(gart->dev, "Attached %s\n", dev_name(dev));
	184	return 0;
	185
	186	fail:
	187	devm_kfree(gart->dev, client);
	188	spin_unlock(&gart->client_lock);
	189	return err;
	190	}
	191
	192	static void gart_iommu_detach_dev(struct iommu_domain *domain,
	193	struct device *dev)
	194	{
	195	struct gart_device *gart = domain->priv;
	196	struct gart_client *c;
	197
	198	spin_lock(&gart->client_lock);
	199
	200	list_for_each_entry(c, &gart->client, list) {
	201	if (c->dev == dev) {
	202	list_del(&c->list);
	203	devm_kfree(gart->dev, c);
	204	dev_dbg(gart->dev, "Detached %s\n", dev_name(dev));
	205	goto out;
	206	}
	207	}
	208	dev_err(gart->dev, "Couldn't find\n");
	209	out:
	210	spin_unlock(&gart->client_lock);
	211	}
	212
	213	static int gart_iommu_domain_init(struct iommu_domain *domain)
	214	{
	215	return 0;
	216	}
	217
	218	static void gart_iommu_domain_destroy(struct iommu_domain *domain)
	219	{
	220	struct gart_device *gart = domain->priv;
	221
	222	if (!gart)
	223	return;
	224
	225	spin_lock(&gart->client_lock);
	226	if (!list_empty(&gart->client)) {
227	struct gart_client *c;
228
229	list_for_each_entry(c, &gart->client, list)
230	gart_iommu_detach_dev(domain, c->dev);
231	}
232	spin_unlock(&gart->client_lock);
233	domain->priv = NULL;
234	}
235
236	static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova,
237	phys_addr_t pa, size_t bytes, int prot)
238	{
239	struct gart_device *gart = domain->priv;
240	unsigned long flags;
241	unsigned long pfn;
242
243	if (!gart_iova_range_valid(gart, iova, bytes))
244	return -EINVAL;
245
246	spin_lock_irqsave(&gart->pte_lock, flags);
247	pfn = __phys_to_pfn(pa);
248	if (!pfn_valid(pfn)) {
249	dev_err(gart->dev, "Invalid page: %08x\n", pa);
09c32533	250	spin_unlock_irqrestore(&gart->pte_lock, flags);
d53e54b4 HD	251	return -EINVAL;
	252	}
	253	gart_set_pte(gart, iova, GART_PTE(pfn));
	254	FLUSH_GART_REGS(gart);
	255	spin_unlock_irqrestore(&gart->pte_lock, flags);
	256	return 0;
	257	}
	258
	259	static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
	260	size_t bytes)
	261	{
	262	struct gart_device *gart = domain->priv;
	263	unsigned long flags;
	264
	265	if (!gart_iova_range_valid(gart, iova, bytes))
	266	return 0;
	267
	268	spin_lock_irqsave(&gart->pte_lock, flags);
	269	gart_set_pte(gart, iova, 0);
	270	FLUSH_GART_REGS(gart);
	271	spin_unlock_irqrestore(&gart->pte_lock, flags);
	272	return 0;
	273	}
	274
	275	static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain,
	276	unsigned long iova)
	277	{
	278	struct gart_device *gart = domain->priv;
	279	unsigned long pte;
	280	phys_addr_t pa;
	281	unsigned long flags;
	282
	283	if (!gart_iova_range_valid(gart, iova, 0))
	284	return -EINVAL;
	285
	286	spin_lock_irqsave(&gart->pte_lock, flags);
	287	pte = gart_read_pte(gart, iova);
	288	spin_unlock_irqrestore(&gart->pte_lock, flags);
	289
	290	pa = (pte & GART_PAGE_MASK);
	291	if (!pfn_valid(__phys_to_pfn(pa))) {
	292	dev_err(gart->dev, "No entry for %08lx:%08x\n", iova, pa);
	293	gart_dump_table(gart);
	294	return -EINVAL;
	295	}
	296	return pa;
	297	}
	298
	299	static int gart_iommu_domain_has_cap(struct iommu_domain *domain,
	300	unsigned long cap)
	301	{
	302	return 0;
	303	}
	304
	305	static struct iommu_ops gart_iommu_ops = {
	306	.domain_init = gart_iommu_domain_init,
	307	.domain_destroy = gart_iommu_domain_destroy,
	308	.attach_dev = gart_iommu_attach_dev,
	309	.detach_dev = gart_iommu_detach_dev,
	310	.map = gart_iommu_map,
	311	.unmap = gart_iommu_unmap,
	312	.iova_to_phys = gart_iommu_iova_to_phys,
	313	.domain_has_cap = gart_iommu_domain_has_cap,
	314	.pgsize_bitmap = GART_IOMMU_PGSIZES,
315	};
316
317	static int tegra_gart_suspend(struct device *dev)
318	{
319	struct gart_device *gart = dev_get_drvdata(dev);
320	unsigned long iova;
321	u32 *data = gart->savedata;
322	unsigned long flags;
323
324	spin_lock_irqsave(&gart->pte_lock, flags);
325	for_each_gart_pte(gart, iova)
326	*(data++) = gart_read_pte(gart, iova);
327	spin_unlock_irqrestore(&gart->pte_lock, flags);
328	return 0;
329	}
330
331	static int tegra_gart_resume(struct device *dev)
332	{
333	struct gart_device *gart = dev_get_drvdata(dev);
334	unsigned long flags;
335
336	spin_lock_irqsave(&gart->pte_lock, flags);
337	do_gart_setup(gart, gart->savedata);
338	spin_unlock_irqrestore(&gart->pte_lock, flags);
339	return 0;
340	}
341
342	static int tegra_gart_probe(struct platform_device *pdev)
343	{
344	struct gart_device *gart;
345	struct resource res, res_remap;
346	void __iomem *gart_regs;
347	int err;
348	struct device *dev = &pdev->dev;
349
350	if (gart_handle)
351	return -EIO;
352
353	BUILD_BUG_ON(PAGE_SHIFT != GART_PAGE_SHIFT);
354
355	/* the GART memory aperture is required */
356	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
357	res_remap = platform_get_resource(pdev, IORESOURCE_MEM, 1);
358	if (!res \|\| !res_remap) {
359	dev_err(dev, "GART memory aperture expected\n");
360	return -ENXIO;
361	}
362
363	gart = devm_kzalloc(dev, sizeof(*gart), GFP_KERNEL);
364	if (!gart) {
365	dev_err(dev, "failed to allocate gart_device\n");
366	return -ENOMEM;
367	}
368
369	gart_regs = devm_ioremap(dev, res->start, resource_size(res));
370	if (!gart_regs) {
371	dev_err(dev, "failed to remap GART registers\n");
372	err = -ENXIO;
373	goto fail;
374	}
375
376	gart->dev = &pdev->dev;
377	spin_lock_init(&gart->pte_lock);
378	spin_lock_init(&gart->client_lock);
379	INIT_LIST_HEAD(&gart->client);
380	gart->regs = gart_regs;
381	gart->iovmm_base = (dma_addr_t)res_remap->start;
382	gart->page_count = (resource_size(res_remap) >> GART_PAGE_SHIFT);
383
384	gart->savedata = vmalloc(sizeof(u32) * gart->page_count);
385	if (!gart->savedata) {
386	dev_err(dev, "failed to allocate context save area\n");
387	err = -ENOMEM;
388	goto fail;
389	}
390
391	platform_set_drvdata(pdev, gart);
392	do_gart_setup(gart, NULL);
393
394	gart_handle = gart;
395	return 0;
396
397	fail:
398	if (gart_regs)
399	devm_iounmap(dev, gart_regs);
400	if (gart && gart->savedata)
401	vfree(gart->savedata);
402	devm_kfree(dev, gart);
403	return err;
404	}
405
406	static int tegra_gart_remove(struct platform_device *pdev)
407	{
408	struct gart_device *gart = platform_get_drvdata(pdev);
409	struct device *dev = gart->dev;
410
411	writel(0, gart->regs + GART_CONFIG);
412	if (gart->savedata)
413	vfree(gart->savedata);
414	if (gart->regs)
415	devm_iounmap(dev, gart->regs);
416	devm_kfree(dev, gart);
417	gart_handle = NULL;
418	return 0;
419	}
420
421	const struct dev_pm_ops tegra_gart_pm_ops = {
422	.suspend = tegra_gart_suspend,
423	.resume = tegra_gart_resume,
424	};
425
7cffae42 TR	426	#ifdef CONFIG_OF
	427	static struct of_device_id tegra_gart_of_match[] __devinitdata = {
	428	{ .compatible = "nvidia,tegra20-gart", },
	429	{ },
	430	};
	431	MODULE_DEVICE_TABLE(of, tegra_gart_of_match);
	432	#endif
	433
d53e54b4 HD	434	static struct platform_driver tegra_gart_driver = {
	435	.probe = tegra_gart_probe,
	436	.remove = tegra_gart_remove,
	437	.driver = {
	438	.owner = THIS_MODULE,
	439	.name = "tegra-gart",
	440	.pm = &tegra_gart_pm_ops,
7cffae42	441	.of_match_table = of_match_ptr(tegra_gart_of_match),
d53e54b4 HD	442	},
	443	};
	444
	445	static int __devinit tegra_gart_init(void)
	446	{
	447	bus_set_iommu(&platform_bus_type, &gart_iommu_ops);
	448	return platform_driver_register(&tegra_gart_driver);
	449	}
	450
	451	static void __exit tegra_gart_exit(void)
	452	{
	453	platform_driver_unregister(&tegra_gart_driver);
	454	}
	455
	456	subsys_initcall(tegra_gart_init);
	457	module_exit(tegra_gart_exit);
	458
	459	MODULE_DESCRIPTION("IOMMU API for GART in Tegra20");
	460	MODULE_AUTHOR("Hiroshi DOYU <hdoyu@nvidia.com>");
7cffae42	461	MODULE_ALIAS("platform:tegra-gart");
d53e54b4	462	MODULE_LICENSE("GPL v2");