[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / gpu / drm / radeon / radeon_fence.c

/*
 * Copyright 2009 Jerome Glisse.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 */
/*
 * Authors:
 *    Jerome Glisse <glisse@freedesktop.org>
 *    Dave Airlie
 */
#include <linux/seq_file.h>
#include <linux/atomic.h>
#include <linux/wait.h>
#include <linux/list.h>
#include <linux/kref.h>
#include <linux/slab.h>
#include "drmP.h"
#include "drm.h"
#include "radeon_reg.h"
#include "radeon.h"
#include "radeon_trace.h"

static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring)
{
	if (rdev->wb.enabled) {
		*rdev->fence_drv[ring].cpu_addr = cpu_to_le32(seq);
	} else {
		WREG32(rdev->fence_drv[ring].scratch_reg, seq);
	}
}

static u32 radeon_fence_read(struct radeon_device *rdev, int ring)
{
	u32 seq = 0;

	if (rdev->wb.enabled) {
		seq = le32_to_cpu(*rdev->fence_drv[ring].cpu_addr);
	} else {
		seq = RREG32(rdev->fence_drv[ring].scratch_reg);
	}
	return seq;
}

int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence)
{
	unsigned long irq_flags;

	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	if (fence->emitted) {
		write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
		return 0;
	}
	fence->seq = atomic_add_return(1, &rdev->fence_drv[fence->ring].seq);
	radeon_fence_ring_emit(rdev, fence->ring, fence);
	trace_radeon_fence_emit(rdev->ddev, fence->seq);
	fence->emitted = true;
	/* are we the first fence on a previusly idle ring? */
	if (list_empty(&rdev->fence_drv[fence->ring].emitted)) {
		rdev->fence_drv[fence->ring].last_activity = jiffies;
	}
	list_move_tail(&fence->list, &rdev->fence_drv[fence->ring].emitted);
	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	return 0;
}

static bool radeon_fence_poll_locked(struct radeon_device *rdev, int ring)
{
	struct radeon_fence *fence;
	struct list_head *i, *n;
	uint32_t seq;
	bool wake = false;

	seq = radeon_fence_read(rdev, ring);
	if (seq == rdev->fence_drv[ring].last_seq)
		return false;

	rdev->fence_drv[ring].last_seq = seq;
	rdev->fence_drv[ring].last_activity = jiffies;

	n = NULL;
	list_for_each(i, &rdev->fence_drv[ring].emitted) {
		fence = list_entry(i, struct radeon_fence, list);
		if (fence->seq == seq) {
			n = i;
			break;
		}
	}
	/* all fence previous to this one are considered as signaled */
	if (n) {
		i = n;
		do {
			n = i->prev;
			list_move_tail(i, &rdev->fence_drv[ring].signaled);
			fence = list_entry(i, struct radeon_fence, list);
			fence->signaled = true;
			i = n;
		} while (i != &rdev->fence_drv[ring].emitted);
		wake = true;
	}
	return wake;
}

static void radeon_fence_destroy(struct kref *kref)
{
	unsigned long irq_flags;
        struct radeon_fence *fence;

	fence = container_of(kref, struct radeon_fence, kref);
	write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
	list_del(&fence->list);
	fence->emitted = false;
	write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
	if (fence->semaphore)
		radeon_semaphore_free(fence->rdev, fence->semaphore);
	kfree(fence);
}

int radeon_fence_create(struct radeon_device *rdev,
			struct radeon_fence **fence,
			int ring)
{
	*fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);
	if ((*fence) == NULL) {
		return -ENOMEM;
	}
	kref_init(&((*fence)->kref));
	(*fence)->rdev = rdev;
	(*fence)->emitted = false;
	(*fence)->signaled = false;
	(*fence)->seq = 0;
	(*fence)->ring = ring;
	(*fence)->semaphore = NULL;
	INIT_LIST_HEAD(&(*fence)->list);
	return 0;
}

bool radeon_fence_signaled(struct radeon_fence *fence)
{
	unsigned long irq_flags;
	bool signaled = false;

	if (!fence)
		return true;

	write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
	signaled = fence->signaled;
	/* if we are shuting down report all fence as signaled */
	if (fence->rdev->shutdown) {
		signaled = true;
	}
	if (!fence->emitted) {
		WARN(1, "Querying an unemitted fence : %p !\n", fence);
		signaled = true;
	}
	if (!signaled) {
		radeon_fence_poll_locked(fence->rdev, fence->ring);
		signaled = fence->signaled;
	}
	write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
	return signaled;
}

int radeon_fence_wait(struct radeon_fence *fence, bool intr)
{
	struct radeon_device *rdev;
	unsigned long irq_flags, timeout;
	u32 seq;
	int i, r;
	bool signaled;

	if (fence == NULL) {
		WARN(1, "Querying an invalid fence : %p !\n", fence);
		return -EINVAL;
	}

	rdev = fence->rdev;
	signaled = radeon_fence_signaled(fence);
	while (!signaled) {
		read_lock_irqsave(&rdev->fence_lock, irq_flags);
		timeout = jiffies - RADEON_FENCE_JIFFIES_TIMEOUT;
		if (time_after(rdev->fence_drv[fence->ring].last_activity, timeout)) {
			/* the normal case, timeout is somewhere before last_activity */
			timeout = rdev->fence_drv[fence->ring].last_activity - timeout;
		} else {
			/* either jiffies wrapped around, or no fence was signaled in the last 500ms
			 * anyway we will just wait for the minimum amount and then check for a lockup */
			timeout = 1;
		}
		/* save current sequence value used to check for GPU lockups */
		seq = rdev->fence_drv[fence->ring].last_seq;
		read_unlock_irqrestore(&rdev->fence_lock, irq_flags);

		trace_radeon_fence_wait_begin(rdev->ddev, seq);
		radeon_irq_kms_sw_irq_get(rdev, fence->ring);
		if (intr) {
			r = wait_event_interruptible_timeout(
				rdev->fence_drv[fence->ring].queue,
				(signaled = radeon_fence_signaled(fence)), timeout);
		} else {
			r = wait_event_timeout(
				rdev->fence_drv[fence->ring].queue,
				(signaled = radeon_fence_signaled(fence)), timeout);
		}
		radeon_irq_kms_sw_irq_put(rdev, fence->ring);
		if (unlikely(r < 0)) {
			return r;
		}
		trace_radeon_fence_wait_end(rdev->ddev, seq);

		if (unlikely(!signaled)) {
			/* we were interrupted for some reason and fence
			 * isn't signaled yet, resume waiting */
			if (r) {
				continue;
			}

			write_lock_irqsave(&rdev->fence_lock, irq_flags);
			/* check if sequence value has changed since last_activity */
			if (seq != rdev->fence_drv[fence->ring].last_seq) {
				write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
				continue;
			}

			/* change sequence value on all rings, so nobody else things there is a lockup */
			for (i = 0; i < RADEON_NUM_RINGS; ++i)
				rdev->fence_drv[i].last_seq -= 0x10000;
			write_unlock_irqrestore(&rdev->fence_lock, irq_flags);

			if (radeon_ring_is_lockup(rdev, fence->ring, &rdev->ring[fence->ring])) {

				/* good news we believe it's a lockup */
				printk(KERN_WARNING "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n",
				     fence->seq, seq);

				/* mark the ring as not ready any more */
				rdev->ring[fence->ring].ready = false;
				r = radeon_gpu_reset(rdev);
				if (r)
					return r;

				write_lock_irqsave(&rdev->fence_lock, irq_flags);
				rdev->fence_drv[fence->ring].last_activity = jiffies;
				write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
			}
		}
	}
	return 0;
}

int radeon_fence_wait_next(struct radeon_device *rdev, int ring)
{
	unsigned long irq_flags;
	struct radeon_fence *fence;
	int r;

	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	if (!rdev->ring[ring].ready) {
		write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
		return -EBUSY;
	}
	if (list_empty(&rdev->fence_drv[ring].emitted)) {
		write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
		return -ENOENT;
	}
	fence = list_entry(rdev->fence_drv[ring].emitted.next,
			   struct radeon_fence, list);
	radeon_fence_ref(fence);
	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	r = radeon_fence_wait(fence, false);
	radeon_fence_unref(&fence);
	return r;
}

int radeon_fence_wait_empty(struct radeon_device *rdev, int ring)
{
	unsigned long irq_flags;
	struct radeon_fence *fence;
	int r;

	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	if (!rdev->ring[ring].ready) {
		write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
		return -EBUSY;
	}
	if (list_empty(&rdev->fence_drv[ring].emitted)) {
		write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
		return 0;
	}
	fence = list_entry(rdev->fence_drv[ring].emitted.prev,
			   struct radeon_fence, list);
	radeon_fence_ref(fence);
	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	r = radeon_fence_wait(fence, false);
	radeon_fence_unref(&fence);
	return r;
}

struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence)
{
	kref_get(&fence->kref);
	return fence;
}

void radeon_fence_unref(struct radeon_fence **fence)
{
	struct radeon_fence *tmp = *fence;

	*fence = NULL;
	if (tmp) {
		kref_put(&tmp->kref, radeon_fence_destroy);
	}
}

void radeon_fence_process(struct radeon_device *rdev, int ring)
{
	unsigned long irq_flags;
	bool wake;

	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	wake = radeon_fence_poll_locked(rdev, ring);
	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	if (wake) {
		wake_up_all(&rdev->fence_drv[ring].queue);
	}
}

int radeon_fence_count_emitted(struct radeon_device *rdev, int ring)
{
	unsigned long irq_flags;
	int not_processed = 0;

	read_lock_irqsave(&rdev->fence_lock, irq_flags);
	if (!rdev->fence_drv[ring].initialized) {
		read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
		return 0;
	}

	if (!list_empty(&rdev->fence_drv[ring].emitted)) {
		struct list_head *ptr;
		list_for_each(ptr, &rdev->fence_drv[ring].emitted) {
			/* count up to 3, that's enought info */
			if (++not_processed >= 3)
				break;
		}
	}
	read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	return not_processed;
}

int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring)
{
	unsigned long irq_flags;
	uint64_t index;
	int r;

	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
	if (rdev->wb.use_event) {
		rdev->fence_drv[ring].scratch_reg = 0;
		index = R600_WB_EVENT_OFFSET + ring * 4;
	} else {
		r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg);
		if (r) {
			dev_err(rdev->dev, "fence failed to get scratch register\n");
			write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
			return r;
		}
		index = RADEON_WB_SCRATCH_OFFSET +
			rdev->fence_drv[ring].scratch_reg -
			rdev->scratch.reg_base;
	}
	rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4];
	rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index;
	radeon_fence_write(rdev, atomic_read(&rdev->fence_drv[ring].seq), ring);
	rdev->fence_drv[ring].initialized = true;
	DRM_INFO("fence driver on ring %d use gpu addr 0x%08Lx and cpu addr 0x%p\n",
		 ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr);
	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	return 0;
}

static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring)
{
	rdev->fence_drv[ring].scratch_reg = -1;
	rdev->fence_drv[ring].cpu_addr = NULL;
	rdev->fence_drv[ring].gpu_addr = 0;
	atomic_set(&rdev->fence_drv[ring].seq, 0);
	INIT_LIST_HEAD(&rdev->fence_drv[ring].emitted);
	INIT_LIST_HEAD(&rdev->fence_drv[ring].signaled);
	init_waitqueue_head(&rdev->fence_drv[ring].queue);
	rdev->fence_drv[ring].initialized = false;
}

int radeon_fence_driver_init(struct radeon_device *rdev)
{
	unsigned long irq_flags;
	int ring;

	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
		radeon_fence_driver_init_ring(rdev, ring);
	}
	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	if (radeon_debugfs_fence_init(rdev)) {
		dev_err(rdev->dev, "fence debugfs file creation failed\n");
	}
	return 0;
}

void radeon_fence_driver_fini(struct radeon_device *rdev)
{
	unsigned long irq_flags;
	int ring;

	for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
		if (!rdev->fence_drv[ring].initialized)
			continue;
		radeon_fence_wait_empty(rdev, ring);
		wake_up_all(&rdev->fence_drv[ring].queue);
		write_lock_irqsave(&rdev->fence_lock, irq_flags);
		radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
		write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
		rdev->fence_drv[ring].initialized = false;
	}
}


/*
 * Fence debugfs
 */
#if defined(CONFIG_DEBUG_FS)
static int radeon_debugfs_fence_info(struct seq_file *m, void *data)
{
	struct drm_info_node *node = (struct drm_info_node *)m->private;
	struct drm_device *dev = node->minor->dev;
	struct radeon_device *rdev = dev->dev_private;
	struct radeon_fence *fence;
	int i;

	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
		if (!rdev->fence_drv[i].initialized)
			continue;

		seq_printf(m, "--- ring %d ---\n", i);
		seq_printf(m, "Last signaled fence 0x%08X\n",
			   radeon_fence_read(rdev, i));
		if (!list_empty(&rdev->fence_drv[i].emitted)) {
			fence = list_entry(rdev->fence_drv[i].emitted.prev,
					   struct radeon_fence, list);
			seq_printf(m, "Last emitted fence %p with 0x%08X\n",
				   fence,  fence->seq);
		}
	}
	return 0;
}

static struct drm_info_list radeon_debugfs_fence_list[] = {
	{"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL},
};
#endif

int radeon_debugfs_fence_init(struct radeon_device *rdev)
{
#if defined(CONFIG_DEBUG_FS)
	return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1);
#else
	return 0;
#endif
}
Commit	Line	Data
771fe6b9 JG	1	/*
	2	* Copyright 2009 Jerome Glisse.
	3	* All Rights Reserved.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining a
	6	* copy of this software and associated documentation files (the
	7	* "Software"), to deal in the Software without restriction, including
	8	* without limitation the rights to use, copy, modify, merge, publish,
	9	* distribute, sub license, and/or sell copies of the Software, and to
	10	* permit persons to whom the Software is furnished to do so, subject to
	11	* the following conditions:
	12	*
	13	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	14	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	15	* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
	16	* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
	17	* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
	18	* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
	19	* USE OR OTHER DEALINGS IN THE SOFTWARE.
	20	*
	21	* The above copyright notice and this permission notice (including the
	22	* next paragraph) shall be included in all copies or substantial portions
	23	* of the Software.
	24	*
	25	*/
	26	/*
	27	* Authors:
	28	* Jerome Glisse <glisse@freedesktop.org>
	29	* Dave Airlie
	30	*/
	31	#include <linux/seq_file.h>
60063497	32	#include <linux/atomic.h>
771fe6b9 JG	33	#include <linux/wait.h>
	34	#include <linux/list.h>
	35	#include <linux/kref.h>
5a0e3ad6	36	#include <linux/slab.h>
771fe6b9 JG	37	#include "drmP.h"
	38	#include "drm.h"
	39	#include "radeon_reg.h"
	40	#include "radeon.h"
99ee7fac	41	#include "radeon_trace.h"
771fe6b9	42
7465280c	43	static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring)
b81157d0 AD	44	{
b81157d0 AD	45	if (rdev->wb.enabled) {
30eb77f4 JG	46	*rdev->fence_drv[ring].cpu_addr = cpu_to_le32(seq);
30eb77f4 JG	47	} else {
7465280c	48	WREG32(rdev->fence_drv[ring].scratch_reg, seq);
30eb77f4	49	}
b81157d0 AD	50	}
b81157d0 AD	51
7465280c	52	static u32 radeon_fence_read(struct radeon_device *rdev, int ring)
b81157d0	53	{
7465280c	54	u32 seq = 0;
b81157d0 AD	55
b81157d0 AD	56	if (rdev->wb.enabled) {
30eb77f4 JG	57	seq = le32_to_cpu(*rdev->fence_drv[ring].cpu_addr);
30eb77f4 JG	58	} else {
7465280c	59	seq = RREG32(rdev->fence_drv[ring].scratch_reg);
30eb77f4	60	}
b81157d0 AD	61	return seq;
	62	}
	63
771fe6b9 JG	64	int radeon_fence_emit(struct radeon_device rdev, struct radeon_fence fence)
	65	{
	66	unsigned long irq_flags;
	67
7465280c	68	write_lock_irqsave(&rdev->fence_lock, irq_flags);
851a6bd9	69	if (fence->emitted) {
7465280c	70	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
771fe6b9 JG	71	return 0;
771fe6b9 JG	72	}
7465280c	73	fence->seq = atomic_add_return(1, &rdev->fence_drv[fence->ring].seq);
25a9e352	74	radeon_fence_ring_emit(rdev, fence->ring, fence);
99ee7fac	75	trace_radeon_fence_emit(rdev->ddev, fence->seq);
851a6bd9	76	fence->emitted = true;
36abacae CK	77	/* are we the first fence on a previusly idle ring? */
	78	if (list_empty(&rdev->fence_drv[fence->ring].emitted)) {
	79	rdev->fence_drv[fence->ring].last_activity = jiffies;
	80	}
7465280c AD	81	list_move_tail(&fence->list, &rdev->fence_drv[fence->ring].emitted);
7465280c AD	82	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
771fe6b9 JG	83	return 0;
	84	}
	85
7465280c	86	static bool radeon_fence_poll_locked(struct radeon_device *rdev, int ring)
771fe6b9 JG	87	{
	88	struct radeon_fence *fence;
	89	struct list_head i, n;
	90	uint32_t seq;
	91	bool wake = false;
	92
7465280c	93	seq = radeon_fence_read(rdev, ring);
36abacae	94	if (seq == rdev->fence_drv[ring].last_seq)
225758d8	95	return false;
36abacae CK	96
	97	rdev->fence_drv[ring].last_seq = seq;
	98	rdev->fence_drv[ring].last_activity = jiffies;
	99
771fe6b9	100	n = NULL;
7465280c	101	list_for_each(i, &rdev->fence_drv[ring].emitted) {
771fe6b9 JG	102	fence = list_entry(i, struct radeon_fence, list);
	103	if (fence->seq == seq) {
	104	n = i;
	105	break;
	106	}
	107	}
	108	/* all fence previous to this one are considered as signaled */
	109	if (n) {
	110	i = n;
	111	do {
	112	n = i->prev;
7465280c	113	list_move_tail(i, &rdev->fence_drv[ring].signaled);
771fe6b9 JG	114	fence = list_entry(i, struct radeon_fence, list);
	115	fence->signaled = true;
	116	i = n;
7465280c	117	} while (i != &rdev->fence_drv[ring].emitted);
771fe6b9 JG	118	wake = true;
	119	}
	120	return wake;
	121	}
	122
	123	static void radeon_fence_destroy(struct kref *kref)
	124	{
	125	unsigned long irq_flags;
	126	struct radeon_fence *fence;
	127
	128	fence = container_of(kref, struct radeon_fence, kref);
7465280c	129	write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
771fe6b9	130	list_del(&fence->list);
851a6bd9	131	fence->emitted = false;
7465280c	132	write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
93504fce CK	133	if (fence->semaphore)
93504fce CK	134	radeon_semaphore_free(fence->rdev, fence->semaphore);
771fe6b9 JG	135	kfree(fence);
	136	}
	137
7465280c AD	138	int radeon_fence_create(struct radeon_device *rdev,
	139	struct radeon_fence **fence,
	140	int ring)
771fe6b9	141	{
771fe6b9 JG	142	*fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);
	143	if ((*fence) == NULL) {
	144	return -ENOMEM;
	145	}
	146	kref_init(&((*fence)->kref));
	147	(*fence)->rdev = rdev;
851a6bd9	148	(*fence)->emitted = false;
771fe6b9 JG	149	(*fence)->signaled = false;
771fe6b9 JG	150	(*fence)->seq = 0;
7465280c	151	(*fence)->ring = ring;
93504fce	152	(*fence)->semaphore = NULL;
771fe6b9	153	INIT_LIST_HEAD(&(*fence)->list);
771fe6b9 JG	154	return 0;
	155	}
	156
771fe6b9 JG	157	bool radeon_fence_signaled(struct radeon_fence *fence)
771fe6b9 JG	158	{
771fe6b9 JG	159	unsigned long irq_flags;
	160	bool signaled = false;
	161
3655d54a	162	if (!fence)
771fe6b9	163	return true;
3655d54a	164
7465280c	165	write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
771fe6b9 JG	166	signaled = fence->signaled;
	167	/* if we are shuting down report all fence as signaled */
	168	if (fence->rdev->shutdown) {
	169	signaled = true;
	170	}
851a6bd9 CK	171	if (!fence->emitted) {
851a6bd9 CK	172	WARN(1, "Querying an unemitted fence : %p !\n", fence);
771fe6b9 JG	173	signaled = true;
	174	}
	175	if (!signaled) {
7465280c	176	radeon_fence_poll_locked(fence->rdev, fence->ring);
771fe6b9 JG	177	signaled = fence->signaled;
771fe6b9 JG	178	}
7465280c	179	write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
771fe6b9 JG	180	return signaled;
	181	}
	182
3ce0a23d	183	int radeon_fence_wait(struct radeon_fence *fence, bool intr)
771fe6b9 JG	184	{
771fe6b9 JG	185	struct radeon_device *rdev;
225758d8 JG	186	unsigned long irq_flags, timeout;
225758d8 JG	187	u32 seq;
36abacae CK	188	int i, r;
36abacae CK	189	bool signaled;
771fe6b9	190
771fe6b9 JG	191	if (fence == NULL) {
771fe6b9 JG	192	WARN(1, "Querying an invalid fence : %p !\n", fence);
36abacae	193	return -EINVAL;
771fe6b9	194	}
36abacae	195
771fe6b9	196	rdev = fence->rdev;
36abacae CK	197	signaled = radeon_fence_signaled(fence);
	198	while (!signaled) {
	199	read_lock_irqsave(&rdev->fence_lock, irq_flags);
	200	timeout = jiffies - RADEON_FENCE_JIFFIES_TIMEOUT;
	201	if (time_after(rdev->fence_drv[fence->ring].last_activity, timeout)) {
	202	/* the normal case, timeout is somewhere before last_activity */
	203	timeout = rdev->fence_drv[fence->ring].last_activity - timeout;
	204	} else {
	205	/* either jiffies wrapped around, or no fence was signaled in the last 500ms
	206	* anyway we will just wait for the minimum amount and then check for a lockup */
	207	timeout = 1;
	208	}
	209	/* save current sequence value used to check for GPU lockups */
	210	seq = rdev->fence_drv[fence->ring].last_seq;
	211	read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	212
	213	trace_radeon_fence_wait_begin(rdev->ddev, seq);
1b37078b	214	radeon_irq_kms_sw_irq_get(rdev, fence->ring);
36abacae CK	215	if (intr) {
	216	r = wait_event_interruptible_timeout(
	217	rdev->fence_drv[fence->ring].queue,
	218	(signaled = radeon_fence_signaled(fence)), timeout);
	219	} else {
	220	r = wait_event_timeout(
	221	rdev->fence_drv[fence->ring].queue,
	222	(signaled = radeon_fence_signaled(fence)), timeout);
	223	}
1b37078b	224	radeon_irq_kms_sw_irq_put(rdev, fence->ring);
90aca4d2	225	if (unlikely(r < 0)) {
5cc6fbab	226	return r;
90aca4d2	227	}
36abacae	228	trace_radeon_fence_wait_end(rdev->ddev, seq);
25a9e352	229
36abacae CK	230	if (unlikely(!signaled)) {
	231	/* we were interrupted for some reason and fence
	232	* isn't signaled yet, resume waiting */
	233	if (r) {
	234	continue;
	235	}
25a9e352	236
36abacae CK	237	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	238	/* check if sequence value has changed since last_activity */
	239	if (seq != rdev->fence_drv[fence->ring].last_seq) {
	240	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	241	continue;
	242	}
	243
	244	/* change sequence value on all rings, so nobody else things there is a lockup */
	245	for (i = 0; i < RADEON_NUM_RINGS; ++i)
	246	rdev->fence_drv[i].last_seq -= 0x10000;
	247	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	248
	249	if (radeon_ring_is_lockup(rdev, fence->ring, &rdev->ring[fence->ring])) {
	250
	251	/* good news we believe it's a lockup */
	252	printk(KERN_WARNING "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n",
	253	fence->seq, seq);
	254
	255	/* mark the ring as not ready any more */
	256	rdev->ring[fence->ring].ready = false;
	257	r = radeon_gpu_reset(rdev);
	258	if (r)
	259	return r;
	260
	261	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	262	rdev->fence_drv[fence->ring].last_activity = jiffies;
	263	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	264	}
771fe6b9	265	}
771fe6b9	266	}
771fe6b9 JG	267	return 0;
	268	}
	269
7465280c	270	int radeon_fence_wait_next(struct radeon_device *rdev, int ring)
771fe6b9 JG	271	{
	272	unsigned long irq_flags;
	273	struct radeon_fence *fence;
	274	int r;
	275
7465280c	276	write_lock_irqsave(&rdev->fence_lock, irq_flags);
25a9e352 CK	277	if (!rdev->ring[ring].ready) {
	278	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	279	return -EBUSY;
	280	}
7465280c AD	281	if (list_empty(&rdev->fence_drv[ring].emitted)) {
7465280c AD	282	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
2f6bfe11	283	return -ENOENT;
771fe6b9	284	}
7465280c	285	fence = list_entry(rdev->fence_drv[ring].emitted.next,
771fe6b9 JG	286	struct radeon_fence, list);
771fe6b9 JG	287	radeon_fence_ref(fence);
7465280c	288	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
771fe6b9 JG	289	r = radeon_fence_wait(fence, false);
	290	radeon_fence_unref(&fence);
	291	return r;
	292	}
	293
adea5c27	294	int radeon_fence_wait_empty(struct radeon_device *rdev, int ring)
771fe6b9 JG	295	{
	296	unsigned long irq_flags;
	297	struct radeon_fence *fence;
	298	int r;
	299
7465280c	300	write_lock_irqsave(&rdev->fence_lock, irq_flags);
25a9e352 CK	301	if (!rdev->ring[ring].ready) {
	302	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	303	return -EBUSY;
	304	}
7465280c AD	305	if (list_empty(&rdev->fence_drv[ring].emitted)) {
7465280c AD	306	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
771fe6b9 JG	307	return 0;
771fe6b9 JG	308	}
7465280c	309	fence = list_entry(rdev->fence_drv[ring].emitted.prev,
771fe6b9 JG	310	struct radeon_fence, list);
771fe6b9 JG	311	radeon_fence_ref(fence);
7465280c	312	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
771fe6b9 JG	313	r = radeon_fence_wait(fence, false);
	314	radeon_fence_unref(&fence);
	315	return r;
	316	}
	317
	318	struct radeon_fence radeon_fence_ref(struct radeon_fence fence)
	319	{
	320	kref_get(&fence->kref);
	321	return fence;
	322	}
	323
	324	void radeon_fence_unref(struct radeon_fence **fence)
	325	{
	326	struct radeon_fence tmp = fence;
	327
	328	*fence = NULL;
	329	if (tmp) {
cdb650a4	330	kref_put(&tmp->kref, radeon_fence_destroy);
771fe6b9 JG	331	}
	332	}
	333
7465280c	334	void radeon_fence_process(struct radeon_device *rdev, int ring)
771fe6b9 JG	335	{
	336	unsigned long irq_flags;
	337	bool wake;
	338
7465280c AD	339	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	340	wake = radeon_fence_poll_locked(rdev, ring);
	341	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
771fe6b9	342	if (wake) {
7465280c	343	wake_up_all(&rdev->fence_drv[ring].queue);
771fe6b9 JG	344	}
	345	}
	346
47492a23 CK	347	int radeon_fence_count_emitted(struct radeon_device *rdev, int ring)
	348	{
	349	unsigned long irq_flags;
	350	int not_processed = 0;
	351
	352	read_lock_irqsave(&rdev->fence_lock, irq_flags);
40e8c738 DA	353	if (!rdev->fence_drv[ring].initialized) {
40e8c738 DA	354	read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
47492a23	355	return 0;
40e8c738	356	}
47492a23 CK	357
	358	if (!list_empty(&rdev->fence_drv[ring].emitted)) {
	359	struct list_head *ptr;
	360	list_for_each(ptr, &rdev->fence_drv[ring].emitted) {
	361	/* count up to 3, that's enought info */
	362	if (++not_processed >= 3)
	363	break;
	364	}
	365	}
	366	read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	367	return not_processed;
	368	}
	369
30eb77f4	370	int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring)
771fe6b9 JG	371	{
771fe6b9 JG	372	unsigned long irq_flags;
30eb77f4 JG	373	uint64_t index;
30eb77f4 JG	374	int r;
771fe6b9	375
30eb77f4 JG	376	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	377	radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
	378	if (rdev->wb.use_event) {
	379	rdev->fence_drv[ring].scratch_reg = 0;
	380	index = R600_WB_EVENT_OFFSET + ring * 4;
	381	} else {
7465280c AD	382	r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg);
	383	if (r) {
	384	dev_err(rdev->dev, "fence failed to get scratch register\n");
	385	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	386	return r;
	387	}
30eb77f4 JG	388	index = RADEON_WB_SCRATCH_OFFSET +
	389	rdev->fence_drv[ring].scratch_reg -
	390	rdev->scratch.reg_base;
7465280c	391	}
30eb77f4 JG	392	rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4];
	393	rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index;
	394	radeon_fence_write(rdev, atomic_read(&rdev->fence_drv[ring].seq), ring);
	395	rdev->fence_drv[ring].initialized = true;
	396	DRM_INFO("fence driver on ring %d use gpu addr 0x%08Lx and cpu addr 0x%p\n",
	397	ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr);
	398	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	399	return 0;
	400	}
	401
	402	static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring)
	403	{
	404	rdev->fence_drv[ring].scratch_reg = -1;
	405	rdev->fence_drv[ring].cpu_addr = NULL;
	406	rdev->fence_drv[ring].gpu_addr = 0;
	407	atomic_set(&rdev->fence_drv[ring].seq, 0);
30eb77f4 JG	408	INIT_LIST_HEAD(&rdev->fence_drv[ring].emitted);
	409	INIT_LIST_HEAD(&rdev->fence_drv[ring].signaled);
	410	init_waitqueue_head(&rdev->fence_drv[ring].queue);
	411	rdev->fence_drv[ring].initialized = false;
	412	}
	413
	414	int radeon_fence_driver_init(struct radeon_device *rdev)
	415	{
	416	unsigned long irq_flags;
	417	int ring;
	418
	419	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	420	for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
	421	radeon_fence_driver_init_ring(rdev, ring);
771fe6b9	422	}
30eb77f4	423	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
771fe6b9	424	if (radeon_debugfs_fence_init(rdev)) {
0a0c7596	425	dev_err(rdev->dev, "fence debugfs file creation failed\n");
771fe6b9 JG	426	}
	427	return 0;
	428	}
	429
	430	void radeon_fence_driver_fini(struct radeon_device *rdev)
	431	{
	432	unsigned long irq_flags;
7465280c AD	433	int ring;
	434
	435	for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
	436	if (!rdev->fence_drv[ring].initialized)
	437	continue;
adea5c27	438	radeon_fence_wait_empty(rdev, ring);
7465280c AD	439	wake_up_all(&rdev->fence_drv[ring].queue);
	440	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	441	radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
	442	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	443	rdev->fence_drv[ring].initialized = false;
	444	}
771fe6b9 JG	445	}
	446
	447
	448	/*
	449	* Fence debugfs
	450	*/
	451	#if defined(CONFIG_DEBUG_FS)
	452	static int radeon_debugfs_fence_info(struct seq_file m, void data)
	453	{
	454	struct drm_info_node node = (struct drm_info_node )m->private;
	455	struct drm_device *dev = node->minor->dev;
	456	struct radeon_device *rdev = dev->dev_private;
	457	struct radeon_fence *fence;
7465280c AD	458	int i;
	459
	460	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
	461	if (!rdev->fence_drv[i].initialized)
	462	continue;
	463
	464	seq_printf(m, "--- ring %d ---\n", i);
	465	seq_printf(m, "Last signaled fence 0x%08X\n",
	466	radeon_fence_read(rdev, i));
	467	if (!list_empty(&rdev->fence_drv[i].emitted)) {
	468	fence = list_entry(rdev->fence_drv[i].emitted.prev,
	469	struct radeon_fence, list);
	470	seq_printf(m, "Last emitted fence %p with 0x%08X\n",
	471	fence, fence->seq);
	472	}
771fe6b9 JG	473	}
	474	return 0;
	475	}
	476
	477	static struct drm_info_list radeon_debugfs_fence_list[] = {
	478	{"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL},
	479	};
	480	#endif
	481
	482	int radeon_debugfs_fence_init(struct radeon_device *rdev)
	483	{
	484	#if defined(CONFIG_DEBUG_FS)
	485	return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1);
	486	#else
	487	return 0;
	488	#endif
	489	}