[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / gpu / drm / nouveau / nv50_instmem.c

/*
 * Copyright (C) 2007 Ben Skeggs.
 *
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial
 * portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 */

#include <drm/drmP.h>

#include "nouveau_drv.h"
#include "nouveau_vm.h"

#define BAR1_VM_BASE 0x0020000000ULL
#define BAR1_VM_SIZE pci_resource_len(dev->pdev, 1)
#define BAR3_VM_BASE 0x0000000000ULL
#define BAR3_VM_SIZE pci_resource_len(dev->pdev, 3)

struct nv50_instmem_priv {
	uint32_t save1700[5]; /* 0x1700->0x1710 */

	struct nouveau_gpuobj *bar1_dmaobj;
	struct nouveau_gpuobj *bar3_dmaobj;
};

static void
nv50_channel_del(struct nouveau_channel **pchan)
{
	struct nouveau_channel *chan;

	chan = *pchan;
	*pchan = NULL;
	if (!chan)
		return;

	nouveau_gpuobj_ref(NULL, &chan->ramfc);
	nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd);
	nouveau_gpuobj_ref(NULL, &chan->vm_pd);
	if (drm_mm_initialized(&chan->ramin_heap))
		drm_mm_takedown(&chan->ramin_heap);
	nouveau_gpuobj_ref(NULL, &chan->ramin);
	kfree(chan);
}

static int
nv50_channel_new(struct drm_device *dev, u32 size, struct nouveau_vm *vm,
		 struct nouveau_channel **pchan)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	u32 pgd = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200;
	u32  fc = (dev_priv->chipset == 0x50) ? 0x0000 : 0x4200;
	struct nouveau_channel *chan;
	int ret, i;

	chan = kzalloc(sizeof(*chan), GFP_KERNEL);
	if (!chan)
		return -ENOMEM;
	chan->dev = dev;

	ret = nouveau_gpuobj_new(dev, NULL, size, 0x1000, 0, &chan->ramin);
	if (ret) {
		nv50_channel_del(&chan);
		return ret;
	}

	ret = drm_mm_init(&chan->ramin_heap, 0x6000, chan->ramin->size - 0x6000);
	if (ret) {
		nv50_channel_del(&chan);
		return ret;
	}

	ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst == ~0 ? ~0 :
				      chan->ramin->pinst + pgd,
				      chan->ramin->vinst + pgd,
				      0x4000, NVOBJ_FLAG_ZERO_ALLOC,
				      &chan->vm_pd);
	if (ret) {
		nv50_channel_del(&chan);
		return ret;
	}

	for (i = 0; i < 0x4000; i += 8) {
		nv_wo32(chan->vm_pd, i + 0, 0x00000000);
		nv_wo32(chan->vm_pd, i + 4, 0xdeadcafe);
	}

	ret = nouveau_vm_ref(vm, &chan->vm, chan->vm_pd);
	if (ret) {
		nv50_channel_del(&chan);
		return ret;
	}

	ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst == ~0 ? ~0 :
				      chan->ramin->pinst + fc,
				      chan->ramin->vinst + fc, 0x100,
				      NVOBJ_FLAG_ZERO_ALLOC, &chan->ramfc);
	if (ret) {
		nv50_channel_del(&chan);
		return ret;
	}

	*pchan = chan;
	return 0;
}

int
nv50_instmem_init(struct drm_device *dev)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nv50_instmem_priv *priv;
	struct nouveau_channel *chan;
	struct nouveau_vm *vm;
	int ret, i;
	u32 tmp;

	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
	if (!priv)
		return -ENOMEM;
	dev_priv->engine.instmem.priv = priv;

	/* Save state, will restore at takedown. */
	for (i = 0x1700; i <= 0x1710; i += 4)
		priv->save1700[(i-0x1700)/4] = nv_rd32(dev, i);

	/* Global PRAMIN heap */
	ret = drm_mm_init(&dev_priv->ramin_heap, 0, dev_priv->ramin_size);
	if (ret) {
		NV_ERROR(dev, "Failed to init RAMIN heap\n");
		goto error;
	}

	/* BAR3 */
	ret = nouveau_vm_new(dev, BAR3_VM_BASE, BAR3_VM_SIZE, BAR3_VM_BASE,
			     &dev_priv->bar3_vm);
	if (ret)
		goto error;

	ret = nouveau_gpuobj_new(dev, NULL, (BAR3_VM_SIZE >> 12) * 8,
				 0x1000, NVOBJ_FLAG_DONT_MAP |
				 NVOBJ_FLAG_ZERO_ALLOC,
				 &dev_priv->bar3_vm->pgt[0].obj[0]);
	if (ret)
		goto error;
	dev_priv->bar3_vm->pgt[0].refcount[0] = 1;

	nv50_instmem_map(dev_priv->bar3_vm->pgt[0].obj[0]);

	ret = nv50_channel_new(dev, 128 * 1024, dev_priv->bar3_vm, &chan);
	if (ret)
		goto error;
	dev_priv->channels.ptr[0] = dev_priv->channels.ptr[127] = chan;

	ret = nv50_gpuobj_dma_new(chan, 0x0000, BAR3_VM_BASE, BAR3_VM_SIZE,
				  NV_MEM_TARGET_VM, NV_MEM_ACCESS_VM,
				  NV_MEM_TYPE_VM, NV_MEM_COMP_VM,
				  &priv->bar3_dmaobj);
	if (ret)
		goto error;

	nv_wr32(dev, 0x001704, 0x00000000 | (chan->ramin->vinst >> 12));
	nv_wr32(dev, 0x001704, 0x40000000 | (chan->ramin->vinst >> 12));
	nv_wr32(dev, 0x00170c, 0x80000000 | (priv->bar3_dmaobj->cinst >> 4));

	dev_priv->engine.instmem.flush(dev);
	dev_priv->ramin_available = true;

	tmp = nv_ro32(chan->ramin, 0);
	nv_wo32(chan->ramin, 0, ~tmp);
	if (nv_ro32(chan->ramin, 0) != ~tmp) {
		NV_ERROR(dev, "PRAMIN readback failed\n");
		ret = -EIO;
		goto error;
	}
	nv_wo32(chan->ramin, 0, tmp);

	/* BAR1 */
	ret = nouveau_vm_new(dev, BAR1_VM_BASE, BAR1_VM_SIZE, BAR1_VM_BASE, &vm);
	if (ret)
		goto error;

	ret = nouveau_vm_ref(vm, &dev_priv->bar1_vm, chan->vm_pd);
	if (ret)
		goto error;
	nouveau_vm_ref(NULL, &vm, NULL);

	ret = nv50_gpuobj_dma_new(chan, 0x0000, BAR1_VM_BASE, BAR1_VM_SIZE,
				  NV_MEM_TARGET_VM, NV_MEM_ACCESS_VM,
				  NV_MEM_TYPE_VM, NV_MEM_COMP_VM,
				  &priv->bar1_dmaobj);
	if (ret)
		goto error;

	nv_wr32(dev, 0x001708, 0x80000000 | (priv->bar1_dmaobj->cinst >> 4));
	for (i = 0; i < 8; i++)
		nv_wr32(dev, 0x1900 + (i*4), 0);

	/* Create shared channel VM, space is reserved at the beginning
	 * to catch "NULL pointer" references
	 */
	ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0020000000ULL,
			     &dev_priv->chan_vm);
	if (ret)
		return ret;

	return 0;

error:
	nv50_instmem_takedown(dev);
	return ret;
}

void
nv50_instmem_takedown(struct drm_device *dev)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv;
	struct nouveau_channel *chan = dev_priv->channels.ptr[0];
	int i;

	NV_DEBUG(dev, "\n");

	if (!priv)
		return;

	dev_priv->ramin_available = false;

	nouveau_vm_ref(NULL, &dev_priv->chan_vm, NULL);

	for (i = 0x1700; i <= 0x1710; i += 4)
		nv_wr32(dev, i, priv->save1700[(i - 0x1700) / 4]);

	nouveau_gpuobj_ref(NULL, &priv->bar3_dmaobj);
	nouveau_gpuobj_ref(NULL, &priv->bar1_dmaobj);

	nouveau_vm_ref(NULL, &dev_priv->bar1_vm, chan->vm_pd);
	dev_priv->channels.ptr[127] = 0;
	nv50_channel_del(&dev_priv->channels.ptr[0]);

	nouveau_gpuobj_ref(NULL, &dev_priv->bar3_vm->pgt[0].obj[0]);
	nouveau_vm_ref(NULL, &dev_priv->bar3_vm, NULL);

	if (drm_mm_initialized(&dev_priv->ramin_heap))
		drm_mm_takedown(&dev_priv->ramin_heap);

	dev_priv->engine.instmem.priv = NULL;
	kfree(priv);
}

int
nv50_instmem_suspend(struct drm_device *dev)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;

	dev_priv->ramin_available = false;
	return 0;
}

void
nv50_instmem_resume(struct drm_device *dev)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv;
	struct nouveau_channel *chan = dev_priv->channels.ptr[0];
	int i;

	/* Poke the relevant regs, and pray it works :) */
	nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->vinst >> 12));
	nv_wr32(dev, NV50_PUNK_UNK1710, 0);
	nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->vinst >> 12) |
					 NV50_PUNK_BAR_CFG_BASE_VALID);
	nv_wr32(dev, NV50_PUNK_BAR1_CTXDMA, (priv->bar1_dmaobj->cinst >> 4) |
					NV50_PUNK_BAR1_CTXDMA_VALID);
	nv_wr32(dev, NV50_PUNK_BAR3_CTXDMA, (priv->bar3_dmaobj->cinst >> 4) |
					NV50_PUNK_BAR3_CTXDMA_VALID);

	for (i = 0; i < 8; i++)
		nv_wr32(dev, 0x1900 + (i*4), 0);

	dev_priv->ramin_available = true;
}

struct nv50_gpuobj_node {
	struct nouveau_mem *vram;
	struct nouveau_vma chan_vma;
	u32 align;
};

int
nv50_instmem_get(struct nouveau_gpuobj *gpuobj, struct nouveau_channel *chan,
		 u32 size, u32 align)
{
	struct drm_device *dev = gpuobj->dev;
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
	struct nv50_gpuobj_node *node = NULL;
	int ret;

	node = kzalloc(sizeof(*node), GFP_KERNEL);
	if (!node)
		return -ENOMEM;
	node->align = align;

	size  = (size + 4095) & ~4095;
	align = max(align, (u32)4096);

	ret = vram->get(dev, size, align, 0, 0, &node->vram);
	if (ret) {
		kfree(node);
		return ret;
	}

	gpuobj->vinst = node->vram->offset;

	if (gpuobj->flags & NVOBJ_FLAG_VM) {
		u32 flags = NV_MEM_ACCESS_RW;
		if (!(gpuobj->flags & NVOBJ_FLAG_VM_USER))
			flags |= NV_MEM_ACCESS_SYS;

		ret = nouveau_vm_get(chan->vm, size, 12, flags,
				     &node->chan_vma);
		if (ret) {
			vram->put(dev, &node->vram);
			kfree(node);
			return ret;
		}

		nouveau_vm_map(&node->chan_vma, node->vram);
		gpuobj->linst = node->chan_vma.offset;
	}

	gpuobj->size = size;
	gpuobj->node = node;
	return 0;
}

void
nv50_instmem_put(struct nouveau_gpuobj *gpuobj)
{
	struct drm_device *dev = gpuobj->dev;
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
	struct nv50_gpuobj_node *node;

	node = gpuobj->node;
	gpuobj->node = NULL;

	if (node->chan_vma.node) {
		nouveau_vm_unmap(&node->chan_vma);
		nouveau_vm_put(&node->chan_vma);
	}
	vram->put(dev, &node->vram);
	kfree(node);
}

int
nv50_instmem_map(struct nouveau_gpuobj *gpuobj)
{
	struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
	struct nv50_gpuobj_node *node = gpuobj->node;
	int ret;

	ret = nouveau_vm_get(dev_priv->bar3_vm, gpuobj->size, 12,
			     NV_MEM_ACCESS_RW, &node->vram->bar_vma);
	if (ret)
		return ret;

	nouveau_vm_map(&node->vram->bar_vma, node->vram);
	gpuobj->pinst = node->vram->bar_vma.offset;
	return 0;
}

void
nv50_instmem_unmap(struct nouveau_gpuobj *gpuobj)
{
	struct nv50_gpuobj_node *node = gpuobj->node;

	if (node->vram->bar_vma.node) {
		nouveau_vm_unmap(&node->vram->bar_vma);
		nouveau_vm_put(&node->vram->bar_vma);
	}
}

void
nv50_instmem_flush(struct drm_device *dev)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	unsigned long flags;

	spin_lock_irqsave(&dev_priv->vm_lock, flags);
	nv_wr32(dev, 0x00330c, 0x00000001);
	if (!nv_wait(dev, 0x00330c, 0x00000002, 0x00000000))
		NV_ERROR(dev, "PRAMIN flush timeout\n");
	spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
}

void
nv84_instmem_flush(struct drm_device *dev)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	unsigned long flags;

	spin_lock_irqsave(&dev_priv->vm_lock, flags);
	nv_wr32(dev, 0x070000, 0x00000001);
	if (!nv_wait(dev, 0x070000, 0x00000002, 0x00000000))
		NV_ERROR(dev, "PRAMIN flush timeout\n");
	spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
}
Commit	Line	Data
6ee73861 BS	1	/*
	2	* Copyright (C) 2007 Ben Skeggs.
	3	*
	4	* All Rights Reserved.
	5	*
	6	* Permission is hereby granted, free of charge, to any person obtaining
	7	* a copy of this software and associated documentation files (the
	8	* "Software"), to deal in the Software without restriction, including
	9	* without limitation the rights to use, copy, modify, merge, publish,
	10	* distribute, sublicense, and/or sell copies of the Software, and to
	11	* permit persons to whom the Software is furnished to do so, subject to
	12	* the following conditions:
	13	*
	14	* The above copyright notice and this permission notice (including the
	15	* next paragraph) shall be included in all copies or substantial
	16	* portions of the Software.
	17	*
	18	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	19	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	20	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
	21	* IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
	22	* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
	23	* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
	24	* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
	25	*
	26	*/
	27
760285e7	28	#include <drm/drmP.h>
a11c3198	29
6ee73861	30	#include "nouveau_drv.h"
a11c3198	31	#include "nouveau_vm.h"
6ee73861	32
f869ef88 BS	33	#define BAR1_VM_BASE 0x0020000000ULL
	34	#define BAR1_VM_SIZE pci_resource_len(dev->pdev, 1)
	35	#define BAR3_VM_BASE 0x0000000000ULL
	36	#define BAR3_VM_SIZE pci_resource_len(dev->pdev, 3)
	37
6ee73861 BS	38	struct nv50_instmem_priv {
	39	uint32_t save1700[5]; /* 0x1700->0x1710 */
	40
f869ef88 BS	41	struct nouveau_gpuobj *bar1_dmaobj;
f869ef88 BS	42	struct nouveau_gpuobj *bar3_dmaobj;
6ee73861 BS	43	};
6ee73861 BS	44
fbd2895e BS	45	static void
	46	nv50_channel_del(struct nouveau_channel **pchan)
	47	{
	48	struct nouveau_channel *chan;
6ee73861	49
fbd2895e BS	50	chan = *pchan;
	51	*pchan = NULL;
	52	if (!chan)
	53	return;
	54
	55	nouveau_gpuobj_ref(NULL, &chan->ramfc);
f869ef88	56	nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd);
fbd2895e	57	nouveau_gpuobj_ref(NULL, &chan->vm_pd);
31a5b8ce	58	if (drm_mm_initialized(&chan->ramin_heap))
fbd2895e BS	59	drm_mm_takedown(&chan->ramin_heap);
	60	nouveau_gpuobj_ref(NULL, &chan->ramin);
	61	kfree(chan);
	62	}
	63
	64	static int
f869ef88	65	nv50_channel_new(struct drm_device dev, u32 size, struct nouveau_vm vm,
fbd2895e BS	66	struct nouveau_channel **pchan)
	67	{
	68	struct drm_nouveau_private *dev_priv = dev->dev_private;
	69	u32 pgd = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200;
	70	u32 fc = (dev_priv->chipset == 0x50) ? 0x0000 : 0x4200;
	71	struct nouveau_channel *chan;
f869ef88	72	int ret, i;
fbd2895e BS	73
	74	chan = kzalloc(sizeof(*chan), GFP_KERNEL);
	75	if (!chan)
	76	return -ENOMEM;
	77	chan->dev = dev;
	78
	79	ret = nouveau_gpuobj_new(dev, NULL, size, 0x1000, 0, &chan->ramin);
	80	if (ret) {
	81	nv50_channel_del(&chan);
	82	return ret;
	83	}
	84
d37f60c8	85	ret = drm_mm_init(&chan->ramin_heap, 0x6000, chan->ramin->size - 0x6000);
fbd2895e BS	86	if (ret) {
	87	nv50_channel_del(&chan);
	88	return ret;
	89	}
	90
	91	ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst == ~0 ? ~0 :
	92	chan->ramin->pinst + pgd,
	93	chan->ramin->vinst + pgd,
	94	0x4000, NVOBJ_FLAG_ZERO_ALLOC,
	95	&chan->vm_pd);
	96	if (ret) {
	97	nv50_channel_del(&chan);
	98	return ret;
	99	}
	100
f869ef88 BS	101	for (i = 0; i < 0x4000; i += 8) {
	102	nv_wo32(chan->vm_pd, i + 0, 0x00000000);
	103	nv_wo32(chan->vm_pd, i + 4, 0xdeadcafe);
	104	}
	105
	106	ret = nouveau_vm_ref(vm, &chan->vm, chan->vm_pd);
	107	if (ret) {
	108	nv50_channel_del(&chan);
	109	return ret;
	110	}
	111
fbd2895e BS	112	ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst == ~0 ? ~0 :
	113	chan->ramin->pinst + fc,
	114	chan->ramin->vinst + fc, 0x100,
	115	NVOBJ_FLAG_ZERO_ALLOC, &chan->ramfc);
	116	if (ret) {
	117	nv50_channel_del(&chan);
	118	return ret;
	119	}
	120
	121	*pchan = chan;
	122	return 0;
	123	}
6ee73861 BS	124
	125	int
	126	nv50_instmem_init(struct drm_device *dev)
	127	{
	128	struct drm_nouveau_private *dev_priv = dev->dev_private;
6ee73861	129	struct nv50_instmem_priv *priv;
fbd2895e	130	struct nouveau_channel *chan;
f869ef88	131	struct nouveau_vm *vm;
6ee73861	132	int ret, i;
fbd2895e	133	u32 tmp;
6ee73861 BS	134
	135	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
	136	if (!priv)
	137	return -ENOMEM;
	138	dev_priv->engine.instmem.priv = priv;
	139
	140	/* Save state, will restore at takedown. */
	141	for (i = 0x1700; i <= 0x1710; i += 4)
	142	priv->save1700[(i-0x1700)/4] = nv_rd32(dev, i);
	143
fbd2895e BS	144	/* Global PRAMIN heap */
	145	ret = drm_mm_init(&dev_priv->ramin_heap, 0, dev_priv->ramin_size);
	146	if (ret) {
	147	NV_ERROR(dev, "Failed to init RAMIN heap\n");
f869ef88	148	goto error;
fbd2895e	149	}
615661f3	150
f869ef88 BS	151	/* BAR3 */
f869ef88 BS	152	ret = nouveau_vm_new(dev, BAR3_VM_BASE, BAR3_VM_SIZE, BAR3_VM_BASE,
3ee01281	153	&dev_priv->bar3_vm);
6ee73861	154	if (ret)
f869ef88	155	goto error;
6ee73861	156
f869ef88 BS	157	ret = nouveau_gpuobj_new(dev, NULL, (BAR3_VM_SIZE >> 12) * 8,
	158	0x1000, NVOBJ_FLAG_DONT_MAP \|
	159	NVOBJ_FLAG_ZERO_ALLOC,
3ee01281	160	&dev_priv->bar3_vm->pgt[0].obj[0]);
6ee73861	161	if (ret)
f869ef88	162	goto error;
3ee01281	163	dev_priv->bar3_vm->pgt[0].refcount[0] = 1;
6ee73861	164
3ee01281	165	nv50_instmem_map(dev_priv->bar3_vm->pgt[0].obj[0]);
6ee73861	166
f869ef88	167	ret = nv50_channel_new(dev, 128 * 1024, dev_priv->bar3_vm, &chan);
6ee73861	168	if (ret)
f869ef88 BS	169	goto error;
f869ef88 BS	170	dev_priv->channels.ptr[0] = dev_priv->channels.ptr[127] = chan;
fbd2895e	171
f869ef88 BS	172	ret = nv50_gpuobj_dma_new(chan, 0x0000, BAR3_VM_BASE, BAR3_VM_SIZE,
	173	NV_MEM_TARGET_VM, NV_MEM_ACCESS_VM,
	174	NV_MEM_TYPE_VM, NV_MEM_COMP_VM,
	175	&priv->bar3_dmaobj);
	176	if (ret)
	177	goto error;
6ee73861	178
fbd2895e BS	179	nv_wr32(dev, 0x001704, 0x00000000 \| (chan->ramin->vinst >> 12));
fbd2895e BS	180	nv_wr32(dev, 0x001704, 0x40000000 \| (chan->ramin->vinst >> 12));
f869ef88	181	nv_wr32(dev, 0x00170c, 0x80000000 \| (priv->bar3_dmaobj->cinst >> 4));
76befb8c	182
c45aadab FJ	183	dev_priv->engine.instmem.flush(dev);
	184	dev_priv->ramin_available = true;
	185
	186	tmp = nv_ro32(chan->ramin, 0);
	187	nv_wo32(chan->ramin, 0, ~tmp);
	188	if (nv_ro32(chan->ramin, 0) != ~tmp) {
fbd2895e	189	NV_ERROR(dev, "PRAMIN readback failed\n");
f869ef88 BS	190	ret = -EIO;
f869ef88 BS	191	goto error;
76befb8c	192	}
c45aadab	193	nv_wo32(chan->ramin, 0, tmp);
fbd2895e	194
f869ef88	195	/* BAR1 */
3ee01281	196	ret = nouveau_vm_new(dev, BAR1_VM_BASE, BAR1_VM_SIZE, BAR1_VM_BASE, &vm);
f869ef88 BS	197	if (ret)
	198	goto error;
	199
	200	ret = nouveau_vm_ref(vm, &dev_priv->bar1_vm, chan->vm_pd);
	201	if (ret)
	202	goto error;
	203	nouveau_vm_ref(NULL, &vm, NULL);
	204
	205	ret = nv50_gpuobj_dma_new(chan, 0x0000, BAR1_VM_BASE, BAR1_VM_SIZE,
	206	NV_MEM_TARGET_VM, NV_MEM_ACCESS_VM,
	207	NV_MEM_TYPE_VM, NV_MEM_COMP_VM,
	208	&priv->bar1_dmaobj);
	209	if (ret)
	210	goto error;
	211
	212	nv_wr32(dev, 0x001708, 0x80000000 \| (priv->bar1_dmaobj->cinst >> 4));
	213	for (i = 0; i < 8; i++)
	214	nv_wr32(dev, 0x1900 + (i*4), 0);
	215
b571fe21 BS	216	/* Create shared channel VM, space is reserved at the beginning
b571fe21 BS	217	* to catch "NULL pointer" references
4c136142	218	*/
b571fe21	219	ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0020000000ULL,
3ee01281	220	&dev_priv->chan_vm);
4c136142 BS	221	if (ret)
4c136142 BS	222	return ret;
6ee73861	223
6ee73861	224	return 0;
f869ef88 BS	225
	226	error:
	227	nv50_instmem_takedown(dev);
	228	return ret;
6ee73861 BS	229	}
	230
	231	void
	232	nv50_instmem_takedown(struct drm_device *dev)
	233	{
	234	struct drm_nouveau_private *dev_priv = dev->dev_private;
	235	struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv;
cff5c133	236	struct nouveau_channel *chan = dev_priv->channels.ptr[0];
6ee73861 BS	237	int i;
	238
	239	NV_DEBUG(dev, "\n");
	240
	241	if (!priv)
	242	return;
	243
fbd2895e BS	244	dev_priv->ramin_available = false;
fbd2895e BS	245
4c136142	246	nouveau_vm_ref(NULL, &dev_priv->chan_vm, NULL);
f869ef88	247
6ee73861 BS	248	for (i = 0x1700; i <= 0x1710; i += 4)
	249	nv_wr32(dev, i, priv->save1700[(i - 0x1700) / 4]);
	250
f869ef88 BS	251	nouveau_gpuobj_ref(NULL, &priv->bar3_dmaobj);
f869ef88 BS	252	nouveau_gpuobj_ref(NULL, &priv->bar1_dmaobj);
6ee73861	253
f869ef88 BS	254	nouveau_vm_ref(NULL, &dev_priv->bar1_vm, chan->vm_pd);
	255	dev_priv->channels.ptr[127] = 0;
	256	nv50_channel_del(&dev_priv->channels.ptr[0]);
6ee73861	257
3ee01281	258	nouveau_gpuobj_ref(NULL, &dev_priv->bar3_vm->pgt[0].obj[0]);
f869ef88 BS	259	nouveau_vm_ref(NULL, &dev_priv->bar3_vm, NULL);
f869ef88 BS	260
31a5b8ce	261	if (drm_mm_initialized(&dev_priv->ramin_heap))
f869ef88	262	drm_mm_takedown(&dev_priv->ramin_heap);
6ee73861 BS	263
	264	dev_priv->engine.instmem.priv = NULL;
	265	kfree(priv);
	266	}
	267
	268	int
	269	nv50_instmem_suspend(struct drm_device *dev)
	270	{
	271	struct drm_nouveau_private *dev_priv = dev->dev_private;
6ee73861	272
dc1e5c0d	273	dev_priv->ramin_available = false;
6ee73861 BS	274	return 0;
	275	}
	276
	277	void
	278	nv50_instmem_resume(struct drm_device *dev)
	279	{
	280	struct drm_nouveau_private *dev_priv = dev->dev_private;
	281	struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv;
cff5c133	282	struct nouveau_channel *chan = dev_priv->channels.ptr[0];
6ee73861 BS	283	int i;
6ee73861 BS	284
6ee73861	285	/* Poke the relevant regs, and pray it works :) */
a8eaebc6	286	nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->vinst >> 12));
6ee73861	287	nv_wr32(dev, NV50_PUNK_UNK1710, 0);
a8eaebc6	288	nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->vinst >> 12) \|
6ee73861	289	NV50_PUNK_BAR_CFG_BASE_VALID);
f869ef88	290	nv_wr32(dev, NV50_PUNK_BAR1_CTXDMA, (priv->bar1_dmaobj->cinst >> 4) \|
6ee73861	291	NV50_PUNK_BAR1_CTXDMA_VALID);
f869ef88	292	nv_wr32(dev, NV50_PUNK_BAR3_CTXDMA, (priv->bar3_dmaobj->cinst >> 4) \|
6ee73861 BS	293	NV50_PUNK_BAR3_CTXDMA_VALID);
	294
	295	for (i = 0; i < 8; i++)
	296	nv_wr32(dev, 0x1900 + (i*4), 0);
dc1e5c0d BS	297
dc1e5c0d BS	298	dev_priv->ramin_available = true;
6ee73861 BS	299	}
6ee73861 BS	300
e41115d0	301	struct nv50_gpuobj_node {
d5f42394	302	struct nouveau_mem *vram;
34cf01bc	303	struct nouveau_vma chan_vma;
e41115d0 BS	304	u32 align;
	305	};
	306
6ee73861	307	int
6e32fedc BS	308	nv50_instmem_get(struct nouveau_gpuobj gpuobj, struct nouveau_channel chan,
6e32fedc BS	309	u32 size, u32 align)
6ee73861	310	{
e41115d0	311	struct drm_device *dev = gpuobj->dev;
34cf01bc	312	struct drm_nouveau_private *dev_priv = dev->dev_private;
60d2a88a	313	struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
e41115d0	314	struct nv50_gpuobj_node *node = NULL;
6ee73861 BS	315	int ret;
6ee73861 BS	316
e41115d0 BS	317	node = kzalloc(sizeof(*node), GFP_KERNEL);
	318	if (!node)
	319	return -ENOMEM;
	320	node->align = align;
6ee73861	321
f869ef88 BS	322	size = (size + 4095) & ~4095;
f869ef88 BS	323	align = max(align, (u32)4096);
6ee73861	324
60d2a88a	325	ret = vram->get(dev, size, align, 0, 0, &node->vram);
6ee73861	326	if (ret) {
f869ef88	327	kfree(node);
6ee73861 BS	328	return ret;
	329	}
	330
f869ef88	331	gpuobj->vinst = node->vram->offset;
34cf01bc BS	332
34cf01bc BS	333	if (gpuobj->flags & NVOBJ_FLAG_VM) {
c906ca0f BS	334	u32 flags = NV_MEM_ACCESS_RW;
	335	if (!(gpuobj->flags & NVOBJ_FLAG_VM_USER))
	336	flags \|= NV_MEM_ACCESS_SYS;
	337
6e32fedc	338	ret = nouveau_vm_get(chan->vm, size, 12, flags,
34cf01bc BS	339	&node->chan_vma);
34cf01bc BS	340	if (ret) {
60d2a88a	341	vram->put(dev, &node->vram);
34cf01bc BS	342	kfree(node);
	343	return ret;
	344	}
	345
	346	nouveau_vm_map(&node->chan_vma, node->vram);
f8522fc8	347	gpuobj->linst = node->chan_vma.offset;
34cf01bc BS	348	}
	349
	350	gpuobj->size = size;
	351	gpuobj->node = node;
6ee73861 BS	352	return 0;
	353	}
	354
	355	void
e41115d0	356	nv50_instmem_put(struct nouveau_gpuobj *gpuobj)
6ee73861	357	{
f869ef88	358	struct drm_device *dev = gpuobj->dev;
60d2a88a BS	359	struct drm_nouveau_private *dev_priv = dev->dev_private;
60d2a88a BS	360	struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
e41115d0	361	struct nv50_gpuobj_node *node;
6ee73861	362
e41115d0 BS	363	node = gpuobj->node;
	364	gpuobj->node = NULL;
	365
34cf01bc BS	366	if (node->chan_vma.node) {
	367	nouveau_vm_unmap(&node->chan_vma);
	368	nouveau_vm_put(&node->chan_vma);
	369	}
60d2a88a	370	vram->put(dev, &node->vram);
e41115d0	371	kfree(node);
6ee73861 BS	372	}
	373
	374	int
e41115d0	375	nv50_instmem_map(struct nouveau_gpuobj *gpuobj)
6ee73861	376	{
e41115d0	377	struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
e41115d0	378	struct nv50_gpuobj_node *node = gpuobj->node;
f869ef88	379	int ret;
6ee73861	380
f869ef88 BS	381	ret = nouveau_vm_get(dev_priv->bar3_vm, gpuobj->size, 12,
	382	NV_MEM_ACCESS_RW, &node->vram->bar_vma);
	383	if (ret)
	384	return ret;
6ee73861	385
f869ef88 BS	386	nouveau_vm_map(&node->vram->bar_vma, node->vram);
f869ef88 BS	387	gpuobj->pinst = node->vram->bar_vma.offset;
6ee73861 BS	388	return 0;
	389	}
	390
e41115d0 BS	391	void
e41115d0 BS	392	nv50_instmem_unmap(struct nouveau_gpuobj *gpuobj)
6ee73861	393	{
e41115d0	394	struct nv50_gpuobj_node *node = gpuobj->node;
6ee73861	395
f869ef88 BS	396	if (node->vram->bar_vma.node) {
	397	nouveau_vm_unmap(&node->vram->bar_vma);
	398	nouveau_vm_put(&node->vram->bar_vma);
6ee73861	399	}
6ee73861 BS	400	}
	401
	402	void
f56cb86f	403	nv50_instmem_flush(struct drm_device *dev)
734ee835	404	{
6f70a4c3	405	struct drm_nouveau_private *dev_priv = dev->dev_private;
04eb34a4	406	unsigned long flags;
6f70a4c3	407
04eb34a4	408	spin_lock_irqsave(&dev_priv->vm_lock, flags);
734ee835	409	nv_wr32(dev, 0x00330c, 0x00000001);
4b5c152a	410	if (!nv_wait(dev, 0x00330c, 0x00000002, 0x00000000))
734ee835	411	NV_ERROR(dev, "PRAMIN flush timeout\n");
04eb34a4	412	spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
734ee835 BS	413	}
	414
	415	void
	416	nv84_instmem_flush(struct drm_device *dev)
6ee73861	417	{
6f70a4c3	418	struct drm_nouveau_private *dev_priv = dev->dev_private;
04eb34a4	419	unsigned long flags;
6f70a4c3	420
04eb34a4	421	spin_lock_irqsave(&dev_priv->vm_lock, flags);
f56cb86f	422	nv_wr32(dev, 0x070000, 0x00000001);
4b5c152a	423	if (!nv_wait(dev, 0x070000, 0x00000002, 0x00000000))
f56cb86f	424	NV_ERROR(dev, "PRAMIN flush timeout\n");
04eb34a4	425	spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
6ee73861 BS	426	}
6ee73861 BS	427