[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / gpu / drm / ttm / ttm_tt.c

/**************************************************************************
 *
 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 **************************************************************************/
/*
 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
 */

#include <linux/vmalloc.h>
#include <linux/sched.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/swap.h>
#include <linux/slab.h>
#include "drm_cache.h"
#include "ttm/ttm_module.h"
#include "ttm/ttm_bo_driver.h"
#include "ttm/ttm_placement.h"

static int ttm_tt_swapin(struct ttm_tt *ttm);

/**
 * Allocates storage for pointers to the pages that back the ttm.
 *
 * Uses kmalloc if possible. Otherwise falls back to vmalloc.
 */
static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
{
	unsigned long size = ttm->num_pages * sizeof(*ttm->pages);
	ttm->pages = NULL;

	if (size <= PAGE_SIZE)
		ttm->pages = kzalloc(size, GFP_KERNEL);

	if (!ttm->pages) {
		ttm->pages = vmalloc_user(size);
		if (ttm->pages)
			ttm->page_flags |= TTM_PAGE_FLAG_VMALLOC;
	}
}

static void ttm_tt_free_page_directory(struct ttm_tt *ttm)
{
	if (ttm->page_flags & TTM_PAGE_FLAG_VMALLOC) {
		vfree(ttm->pages);
		ttm->page_flags &= ~TTM_PAGE_FLAG_VMALLOC;
	} else {
		kfree(ttm->pages);
	}
	ttm->pages = NULL;
}

static struct page *ttm_tt_alloc_page(unsigned page_flags)
{
	gfp_t gfp_flags = GFP_USER;

	if (page_flags & TTM_PAGE_FLAG_ZERO_ALLOC)
		gfp_flags |= __GFP_ZERO;

	if (page_flags & TTM_PAGE_FLAG_DMA32)
		gfp_flags |= __GFP_DMA32;
	else
		gfp_flags |= __GFP_HIGHMEM;

	return alloc_page(gfp_flags);
}

static void ttm_tt_free_user_pages(struct ttm_tt *ttm)
{
	int write;
	int dirty;
	struct page *page;
	int i;
	struct ttm_backend *be = ttm->be;

	BUG_ON(!(ttm->page_flags & TTM_PAGE_FLAG_USER));
	write = ((ttm->page_flags & TTM_PAGE_FLAG_WRITE) != 0);
	dirty = ((ttm->page_flags & TTM_PAGE_FLAG_USER_DIRTY) != 0);

	if (be)
		be->func->clear(be);

	for (i = 0; i < ttm->num_pages; ++i) {
		page = ttm->pages[i];
		if (page == NULL)
			continue;

		if (page == ttm->dummy_read_page) {
			BUG_ON(write);
			continue;
		}

		if (write && dirty && !PageReserved(page))
			set_page_dirty_lock(page);

		ttm->pages[i] = NULL;
		ttm_mem_global_free(ttm->glob->mem_glob, PAGE_SIZE);
		put_page(page);
	}
	ttm->state = tt_unpopulated;
	ttm->first_himem_page = ttm->num_pages;
	ttm->last_lomem_page = -1;
}

static struct page *__ttm_tt_get_page(struct ttm_tt *ttm, int index)
{
	struct page *p;
	struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
	int ret;

	while (NULL == (p = ttm->pages[index])) {
		p = ttm_tt_alloc_page(ttm->page_flags);

		if (!p)
			return NULL;

		ret = ttm_mem_global_alloc_page(mem_glob, p, false, false);
		if (unlikely(ret != 0))
			goto out_err;

		if (PageHighMem(p))
			ttm->pages[--ttm->first_himem_page] = p;
		else
			ttm->pages[++ttm->last_lomem_page] = p;
	}
	return p;
out_err:
	put_page(p);
	return NULL;
}

struct page *ttm_tt_get_page(struct ttm_tt *ttm, int index)
{
	int ret;

	if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
		ret = ttm_tt_swapin(ttm);
		if (unlikely(ret != 0))
			return NULL;
	}
	return __ttm_tt_get_page(ttm, index);
}

int ttm_tt_populate(struct ttm_tt *ttm)
{
	struct page *page;
	unsigned long i;
	struct ttm_backend *be;
	int ret;

	if (ttm->state != tt_unpopulated)
		return 0;

	if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
		ret = ttm_tt_swapin(ttm);
		if (unlikely(ret != 0))
			return ret;
	}

	be = ttm->be;

	for (i = 0; i < ttm->num_pages; ++i) {
		page = __ttm_tt_get_page(ttm, i);
		if (!page)
			return -ENOMEM;
	}

	be->func->populate(be, ttm->num_pages, ttm->pages,
			   ttm->dummy_read_page);
	ttm->state = tt_unbound;
	return 0;
}
EXPORT_SYMBOL(ttm_tt_populate);

#ifdef CONFIG_X86
static inline int ttm_tt_set_page_caching(struct page *p,
					  enum ttm_caching_state c_old,
					  enum ttm_caching_state c_new)
{
	int ret = 0;

	if (PageHighMem(p))
		return 0;

	if (c_old != tt_cached) {
		/* p isn't in the default caching state, set it to
		 * writeback first to free its current memtype. */

		ret = set_pages_wb(p, 1);
		if (ret)
			return ret;
	}

	if (c_new == tt_wc)
		ret = set_memory_wc((unsigned long) page_address(p), 1);
	else if (c_new == tt_uncached)
		ret = set_pages_uc(p, 1);

	return ret;
}
#else /* CONFIG_X86 */
static inline int ttm_tt_set_page_caching(struct page *p,
					  enum ttm_caching_state c_old,
					  enum ttm_caching_state c_new)
{
	return 0;
}
#endif /* CONFIG_X86 */

/*
 * Change caching policy for the linear kernel map
 * for range of pages in a ttm.
 */

static int ttm_tt_set_caching(struct ttm_tt *ttm,
			      enum ttm_caching_state c_state)
{
	int i, j;
	struct page *cur_page;
	int ret;

	if (ttm->caching_state == c_state)
		return 0;

	if (c_state != tt_cached) {
		ret = ttm_tt_populate(ttm);
		if (unlikely(ret != 0))
			return ret;
	}

	if (ttm->caching_state == tt_cached)
		drm_clflush_pages(ttm->pages, ttm->num_pages);

	for (i = 0; i < ttm->num_pages; ++i) {
		cur_page = ttm->pages[i];
		if (likely(cur_page != NULL)) {
			ret = ttm_tt_set_page_caching(cur_page,
						      ttm->caching_state,
						      c_state);
			if (unlikely(ret != 0))
				goto out_err;
		}
	}

	ttm->caching_state = c_state;

	return 0;

out_err:
	for (j = 0; j < i; ++j) {
		cur_page = ttm->pages[j];
		if (likely(cur_page != NULL)) {
			(void)ttm_tt_set_page_caching(cur_page, c_state,
						      ttm->caching_state);
		}
	}

	return ret;
}

int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement)
{
	enum ttm_caching_state state;

	if (placement & TTM_PL_FLAG_WC)
		state = tt_wc;
	else if (placement & TTM_PL_FLAG_UNCACHED)
		state = tt_uncached;
	else
		state = tt_cached;

	return ttm_tt_set_caching(ttm, state);
}
EXPORT_SYMBOL(ttm_tt_set_placement_caching);

static void ttm_tt_free_alloced_pages(struct ttm_tt *ttm)
{
	int i;
	struct page *cur_page;
	struct ttm_backend *be = ttm->be;

	if (be)
		be->func->clear(be);
	(void)ttm_tt_set_caching(ttm, tt_cached);
	for (i = 0; i < ttm->num_pages; ++i) {
		cur_page = ttm->pages[i];
		ttm->pages[i] = NULL;
		if (cur_page) {
			if (page_count(cur_page) != 1)
				printk(KERN_ERR TTM_PFX
				       "Erroneous page count. "
				       "Leaking pages.\n");
			ttm_mem_global_free_page(ttm->glob->mem_glob,
						 cur_page);
			__free_page(cur_page);
		}
	}
	ttm->state = tt_unpopulated;
	ttm->first_himem_page = ttm->num_pages;
	ttm->last_lomem_page = -1;
}

void ttm_tt_destroy(struct ttm_tt *ttm)
{
	struct ttm_backend *be;

	if (unlikely(ttm == NULL))
		return;

	be = ttm->be;
	if (likely(be != NULL)) {
		be->func->destroy(be);
		ttm->be = NULL;
	}

	if (likely(ttm->pages != NULL)) {
		if (ttm->page_flags & TTM_PAGE_FLAG_USER)
			ttm_tt_free_user_pages(ttm);
		else
			ttm_tt_free_alloced_pages(ttm);

		ttm_tt_free_page_directory(ttm);
	}

	if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTANT_SWAP) &&
	    ttm->swap_storage)
		fput(ttm->swap_storage);

	kfree(ttm);
}

int ttm_tt_set_user(struct ttm_tt *ttm,
		    struct task_struct *tsk,
		    unsigned long start, unsigned long num_pages)
{
	struct mm_struct *mm = tsk->mm;
	int ret;
	int write = (ttm->page_flags & TTM_PAGE_FLAG_WRITE) != 0;
	struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;

	BUG_ON(num_pages != ttm->num_pages);
	BUG_ON((ttm->page_flags & TTM_PAGE_FLAG_USER) == 0);

	/**
	 * Account user pages as lowmem pages for now.
	 */

	ret = ttm_mem_global_alloc(mem_glob, num_pages * PAGE_SIZE,
				   false, false);
	if (unlikely(ret != 0))
		return ret;

	down_read(&mm->mmap_sem);
	ret = get_user_pages(tsk, mm, start, num_pages,
			     write, 0, ttm->pages, NULL);
	up_read(&mm->mmap_sem);

	if (ret != num_pages && write) {
		ttm_tt_free_user_pages(ttm);
		ttm_mem_global_free(mem_glob, num_pages * PAGE_SIZE);
		return -ENOMEM;
	}

	ttm->tsk = tsk;
	ttm->start = start;
	ttm->state = tt_unbound;

	return 0;
}

struct ttm_tt *ttm_tt_create(struct ttm_bo_device *bdev, unsigned long size,
			     uint32_t page_flags, struct page *dummy_read_page)
{
	struct ttm_bo_driver *bo_driver = bdev->driver;
	struct ttm_tt *ttm;

	if (!bo_driver)
		return NULL;

	ttm = kzalloc(sizeof(*ttm), GFP_KERNEL);
	if (!ttm)
		return NULL;

	ttm->glob = bdev->glob;
	ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
	ttm->first_himem_page = ttm->num_pages;
	ttm->last_lomem_page = -1;
	ttm->caching_state = tt_cached;
	ttm->page_flags = page_flags;

	ttm->dummy_read_page = dummy_read_page;

	ttm_tt_alloc_page_directory(ttm);
	if (!ttm->pages) {
		ttm_tt_destroy(ttm);
		printk(KERN_ERR TTM_PFX "Failed allocating page table\n");
		return NULL;
	}
	ttm->be = bo_driver->create_ttm_backend_entry(bdev);
	if (!ttm->be) {
		ttm_tt_destroy(ttm);
		printk(KERN_ERR TTM_PFX "Failed creating ttm backend entry\n");
		return NULL;
	}
	ttm->state = tt_unpopulated;
	return ttm;
}

void ttm_tt_unbind(struct ttm_tt *ttm)
{
	int ret;
	struct ttm_backend *be = ttm->be;

	if (ttm->state == tt_bound) {
		ret = be->func->unbind(be);
		BUG_ON(ret);
		ttm->state = tt_unbound;
	}
}

int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)
{
	int ret = 0;
	struct ttm_backend *be;

	if (!ttm)
		return -EINVAL;

	if (ttm->state == tt_bound)
		return 0;

	be = ttm->be;

	ret = ttm_tt_populate(ttm);
	if (ret)
		return ret;

	ret = be->func->bind(be, bo_mem);
	if (ret) {
		printk(KERN_ERR TTM_PFX "Couldn't bind backend.\n");
		return ret;
	}

	ttm->state = tt_bound;

	if (ttm->page_flags & TTM_PAGE_FLAG_USER)
		ttm->page_flags |= TTM_PAGE_FLAG_USER_DIRTY;
	return 0;
}
EXPORT_SYMBOL(ttm_tt_bind);

static int ttm_tt_swapin(struct ttm_tt *ttm)
{
	struct address_space *swap_space;
	struct file *swap_storage;
	struct page *from_page;
	struct page *to_page;
	void *from_virtual;
	void *to_virtual;
	int i;
	int ret = -ENOMEM;

	if (ttm->page_flags & TTM_PAGE_FLAG_USER) {
		ret = ttm_tt_set_user(ttm, ttm->tsk, ttm->start,
				      ttm->num_pages);
		if (unlikely(ret != 0))
			return ret;

		ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED;
		return 0;
	}

	swap_storage = ttm->swap_storage;
	BUG_ON(swap_storage == NULL);

	swap_space = swap_storage->f_path.dentry->d_inode->i_mapping;

	for (i = 0; i < ttm->num_pages; ++i) {
		from_page = read_mapping_page(swap_space, i, NULL);
		if (IS_ERR(from_page)) {
			ret = PTR_ERR(from_page);
			goto out_err;
		}
		to_page = __ttm_tt_get_page(ttm, i);
		if (unlikely(to_page == NULL))
			goto out_err;

		preempt_disable();
		from_virtual = kmap_atomic(from_page, KM_USER0);
		to_virtual = kmap_atomic(to_page, KM_USER1);
		memcpy(to_virtual, from_virtual, PAGE_SIZE);
		kunmap_atomic(to_virtual, KM_USER1);
		kunmap_atomic(from_virtual, KM_USER0);
		preempt_enable();
		page_cache_release(from_page);
	}

	if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTANT_SWAP))
		fput(swap_storage);
	ttm->swap_storage = NULL;
	ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED;

	return 0;
out_err:
	ttm_tt_free_alloced_pages(ttm);
	return ret;
}

int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistant_swap_storage)
{
	struct address_space *swap_space;
	struct file *swap_storage;
	struct page *from_page;
	struct page *to_page;
	void *from_virtual;
	void *to_virtual;
	int i;
	int ret = -ENOMEM;

	BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated);
	BUG_ON(ttm->caching_state != tt_cached);

	/*
	 * For user buffers, just unpin the pages, as there should be
	 * vma references.
	 */

	if (ttm->page_flags & TTM_PAGE_FLAG_USER) {
		ttm_tt_free_user_pages(ttm);
		ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED;
		ttm->swap_storage = NULL;
		return 0;
	}

	if (!persistant_swap_storage) {
		swap_storage = shmem_file_setup("ttm swap",
						ttm->num_pages << PAGE_SHIFT,
						0);
		if (unlikely(IS_ERR(swap_storage))) {
			printk(KERN_ERR "Failed allocating swap storage.\n");
			return PTR_ERR(swap_storage);
		}
	} else
		swap_storage = persistant_swap_storage;

	swap_space = swap_storage->f_path.dentry->d_inode->i_mapping;

	for (i = 0; i < ttm->num_pages; ++i) {
		from_page = ttm->pages[i];
		if (unlikely(from_page == NULL))
			continue;
		to_page = read_mapping_page(swap_space, i, NULL);
		if (unlikely(IS_ERR(to_page))) {
			ret = PTR_ERR(to_page);
			goto out_err;
		}
		preempt_disable();
		from_virtual = kmap_atomic(from_page, KM_USER0);
		to_virtual = kmap_atomic(to_page, KM_USER1);
		memcpy(to_virtual, from_virtual, PAGE_SIZE);
		kunmap_atomic(to_virtual, KM_USER1);
		kunmap_atomic(from_virtual, KM_USER0);
		preempt_enable();
		set_page_dirty(to_page);
		mark_page_accessed(to_page);
		page_cache_release(to_page);
	}

	ttm_tt_free_alloced_pages(ttm);
	ttm->swap_storage = swap_storage;
	ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED;
	if (persistant_swap_storage)
		ttm->page_flags |= TTM_PAGE_FLAG_PERSISTANT_SWAP;

	return 0;
out_err:
	if (!persistant_swap_storage)
		fput(swap_storage);

	return ret;
}
Commit	Line	Data
ba4e7d97 TH	1	/**************************************************************************
	2	*
	3	* Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
	4	* All Rights Reserved.
	5	*
	6	* Permission is hereby granted, free of charge, to any person obtaining a
	7	* copy of this software and associated documentation files (the
	8	* "Software"), to deal in the Software without restriction, including
	9	* without limitation the rights to use, copy, modify, merge, publish,
	10	* distribute, sub license, and/or sell copies of the Software, and to
	11	* permit persons to whom the Software is furnished to do so, subject to
	12	* the following conditions:
	13	*
	14	* The above copyright notice and this permission notice (including the
	15	* next paragraph) shall be included in all copies or substantial portions
	16	* of the Software.
	17	*
	18	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	19	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	20	* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
	21	* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
	22	* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
	23	* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
	24	* USE OR OTHER DEALINGS IN THE SOFTWARE.
	25	*
	26	**************************************************************************/
	27	/*
	28	* Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
	29	*/
	30
ba4e7d97 TH	31	#include <linux/vmalloc.h>
	32	#include <linux/sched.h>
	33	#include <linux/highmem.h>
	34	#include <linux/pagemap.h>
	35	#include <linux/file.h>
	36	#include <linux/swap.h>
5a0e3ad6	37	#include <linux/slab.h>
c9c97b8c	38	#include "drm_cache.h"
ba4e7d97 TH	39	#include "ttm/ttm_module.h"
	40	#include "ttm/ttm_bo_driver.h"
	41	#include "ttm/ttm_placement.h"
	42
	43	static int ttm_tt_swapin(struct ttm_tt *ttm);
	44
ba4e7d97 TH	45	/**
	46	* Allocates storage for pointers to the pages that back the ttm.
	47	*
	48	* Uses kmalloc if possible. Otherwise falls back to vmalloc.
	49	*/
	50	static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
	51	{
	52	unsigned long size = ttm->num_pages * sizeof(*ttm->pages);
	53	ttm->pages = NULL;
	54
	55	if (size <= PAGE_SIZE)
	56	ttm->pages = kzalloc(size, GFP_KERNEL);
	57
	58	if (!ttm->pages) {
	59	ttm->pages = vmalloc_user(size);
	60	if (ttm->pages)
	61	ttm->page_flags \|= TTM_PAGE_FLAG_VMALLOC;
	62	}
	63	}
	64
	65	static void ttm_tt_free_page_directory(struct ttm_tt *ttm)
	66	{
	67	if (ttm->page_flags & TTM_PAGE_FLAG_VMALLOC) {
	68	vfree(ttm->pages);
	69	ttm->page_flags &= ~TTM_PAGE_FLAG_VMALLOC;
	70	} else {
	71	kfree(ttm->pages);
	72	}
	73	ttm->pages = NULL;
	74	}
	75
	76	static struct page *ttm_tt_alloc_page(unsigned page_flags)
	77	{
b42db2b1	78	gfp_t gfp_flags = GFP_USER;
ad49f501	79
ba4e7d97	80	if (page_flags & TTM_PAGE_FLAG_ZERO_ALLOC)
ad49f501 DA	81	gfp_flags \|= __GFP_ZERO;
	82
	83	if (page_flags & TTM_PAGE_FLAG_DMA32)
	84	gfp_flags \|= __GFP_DMA32;
b42db2b1 DA	85	else
b42db2b1 DA	86	gfp_flags \|= __GFP_HIGHMEM;
ba4e7d97	87
ad49f501	88	return alloc_page(gfp_flags);
ba4e7d97 TH	89	}
	90
	91	static void ttm_tt_free_user_pages(struct ttm_tt *ttm)
	92	{
	93	int write;
	94	int dirty;
	95	struct page *page;
	96	int i;
	97	struct ttm_backend *be = ttm->be;
	98
	99	BUG_ON(!(ttm->page_flags & TTM_PAGE_FLAG_USER));
	100	write = ((ttm->page_flags & TTM_PAGE_FLAG_WRITE) != 0);
	101	dirty = ((ttm->page_flags & TTM_PAGE_FLAG_USER_DIRTY) != 0);
	102
	103	if (be)
	104	be->func->clear(be);
	105
	106	for (i = 0; i < ttm->num_pages; ++i) {
	107	page = ttm->pages[i];
	108	if (page == NULL)
	109	continue;
	110
	111	if (page == ttm->dummy_read_page) {
	112	BUG_ON(write);
	113	continue;
	114	}
	115
	116	if (write && dirty && !PageReserved(page))
	117	set_page_dirty_lock(page);
	118
	119	ttm->pages[i] = NULL;
a987fcaa	120	ttm_mem_global_free(ttm->glob->mem_glob, PAGE_SIZE);
ba4e7d97 TH	121	put_page(page);
	122	}
	123	ttm->state = tt_unpopulated;
	124	ttm->first_himem_page = ttm->num_pages;
	125	ttm->last_lomem_page = -1;
	126	}
	127
	128	static struct page __ttm_tt_get_page(struct ttm_tt ttm, int index)
	129	{
	130	struct page *p;
a987fcaa	131	struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
ba4e7d97 TH	132	int ret;
	133
	134	while (NULL == (p = ttm->pages[index])) {
	135	p = ttm_tt_alloc_page(ttm->page_flags);
	136
	137	if (!p)
	138	return NULL;
	139
5fd9cbad TH	140	ret = ttm_mem_global_alloc_page(mem_glob, p, false, false);
	141	if (unlikely(ret != 0))
	142	goto out_err;
	143
	144	if (PageHighMem(p))
ba4e7d97	145	ttm->pages[--ttm->first_himem_page] = p;
5fd9cbad	146	else
ba4e7d97	147	ttm->pages[++ttm->last_lomem_page] = p;
ba4e7d97 TH	148	}
	149	return p;
	150	out_err:
	151	put_page(p);
	152	return NULL;
	153	}
	154
	155	struct page ttm_tt_get_page(struct ttm_tt ttm, int index)
	156	{
	157	int ret;
	158
	159	if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
	160	ret = ttm_tt_swapin(ttm);
	161	if (unlikely(ret != 0))
	162	return NULL;
	163	}
	164	return __ttm_tt_get_page(ttm, index);
	165	}
	166
	167	int ttm_tt_populate(struct ttm_tt *ttm)
	168	{
	169	struct page *page;
	170	unsigned long i;
	171	struct ttm_backend *be;
	172	int ret;
	173
	174	if (ttm->state != tt_unpopulated)
	175	return 0;
	176
	177	if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
	178	ret = ttm_tt_swapin(ttm);
	179	if (unlikely(ret != 0))
	180	return ret;
	181	}
	182
	183	be = ttm->be;
	184
	185	for (i = 0; i < ttm->num_pages; ++i) {
	186	page = __ttm_tt_get_page(ttm, i);
	187	if (!page)
	188	return -ENOMEM;
	189	}
	190
	191	be->func->populate(be, ttm->num_pages, ttm->pages,
	192	ttm->dummy_read_page);
	193	ttm->state = tt_unbound;
	194	return 0;
	195	}
4bfd75cb	196	EXPORT_SYMBOL(ttm_tt_populate);
ba4e7d97 TH	197
	198	#ifdef CONFIG_X86
	199	static inline int ttm_tt_set_page_caching(struct page *p,
f0e2f38b FJ	200	enum ttm_caching_state c_old,
f0e2f38b FJ	201	enum ttm_caching_state c_new)
ba4e7d97	202	{
db78e27d FJ	203	int ret = 0;
db78e27d FJ	204
ba4e7d97 TH	205	if (PageHighMem(p))
	206	return 0;
	207
f0e2f38b	208	if (c_old != tt_cached) {
db78e27d FJ	209	/* p isn't in the default caching state, set it to
	210	* writeback first to free its current memtype. */
	211
	212	ret = set_pages_wb(p, 1);
	213	if (ret)
	214	return ret;
ba4e7d97	215	}
db78e27d	216
f0e2f38b	217	if (c_new == tt_wc)
db78e27d	218	ret = set_memory_wc((unsigned long) page_address(p), 1);
f0e2f38b	219	else if (c_new == tt_uncached)
db78e27d FJ	220	ret = set_pages_uc(p, 1);
	221
	222	return ret;
ba4e7d97 TH	223	}
	224	#else /* CONFIG_X86 */
	225	static inline int ttm_tt_set_page_caching(struct page *p,
f0e2f38b FJ	226	enum ttm_caching_state c_old,
f0e2f38b FJ	227	enum ttm_caching_state c_new)
ba4e7d97 TH	228	{
	229	return 0;
	230	}
	231	#endif /* CONFIG_X86 */
	232
	233	/*
	234	* Change caching policy for the linear kernel map
	235	* for range of pages in a ttm.
	236	*/
	237
	238	static int ttm_tt_set_caching(struct ttm_tt *ttm,
	239	enum ttm_caching_state c_state)
	240	{
	241	int i, j;
	242	struct page *cur_page;
	243	int ret;
	244
	245	if (ttm->caching_state == c_state)
	246	return 0;
	247
	248	if (c_state != tt_cached) {
	249	ret = ttm_tt_populate(ttm);
	250	if (unlikely(ret != 0))
	251	return ret;
	252	}
	253
	254	if (ttm->caching_state == tt_cached)
c9c97b8c	255	drm_clflush_pages(ttm->pages, ttm->num_pages);
ba4e7d97 TH	256
	257	for (i = 0; i < ttm->num_pages; ++i) {
	258	cur_page = ttm->pages[i];
	259	if (likely(cur_page != NULL)) {
f0e2f38b FJ	260	ret = ttm_tt_set_page_caching(cur_page,
	261	ttm->caching_state,
	262	c_state);
ba4e7d97 TH	263	if (unlikely(ret != 0))
	264	goto out_err;
	265	}
	266	}
	267
	268	ttm->caching_state = c_state;
	269
	270	return 0;
	271
	272	out_err:
	273	for (j = 0; j < i; ++j) {
	274	cur_page = ttm->pages[j];
	275	if (likely(cur_page != NULL)) {
f0e2f38b	276	(void)ttm_tt_set_page_caching(cur_page, c_state,
ba4e7d97 TH	277	ttm->caching_state);
	278	}
	279	}
	280
	281	return ret;
	282	}
	283
	284	int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement)
	285	{
	286	enum ttm_caching_state state;
	287
	288	if (placement & TTM_PL_FLAG_WC)
	289	state = tt_wc;
	290	else if (placement & TTM_PL_FLAG_UNCACHED)
	291	state = tt_uncached;
	292	else
	293	state = tt_cached;
	294
	295	return ttm_tt_set_caching(ttm, state);
	296	}
df67bed9	297	EXPORT_SYMBOL(ttm_tt_set_placement_caching);
ba4e7d97 TH	298
	299	static void ttm_tt_free_alloced_pages(struct ttm_tt *ttm)
	300	{
	301	int i;
	302	struct page *cur_page;
	303	struct ttm_backend *be = ttm->be;
	304
	305	if (be)
	306	be->func->clear(be);
	307	(void)ttm_tt_set_caching(ttm, tt_cached);
	308	for (i = 0; i < ttm->num_pages; ++i) {
	309	cur_page = ttm->pages[i];
	310	ttm->pages[i] = NULL;
	311	if (cur_page) {
	312	if (page_count(cur_page) != 1)
	313	printk(KERN_ERR TTM_PFX
	314	"Erroneous page count. "
	315	"Leaking pages.\n");
a987fcaa	316	ttm_mem_global_free_page(ttm->glob->mem_glob,
5fd9cbad	317	cur_page);
ba4e7d97 TH	318	__free_page(cur_page);
	319	}
	320	}
	321	ttm->state = tt_unpopulated;
	322	ttm->first_himem_page = ttm->num_pages;
	323	ttm->last_lomem_page = -1;
	324	}
	325
	326	void ttm_tt_destroy(struct ttm_tt *ttm)
	327	{
	328	struct ttm_backend *be;
	329
	330	if (unlikely(ttm == NULL))
	331	return;
	332
	333	be = ttm->be;
	334	if (likely(be != NULL)) {
	335	be->func->destroy(be);
	336	ttm->be = NULL;
	337	}
	338
	339	if (likely(ttm->pages != NULL)) {
	340	if (ttm->page_flags & TTM_PAGE_FLAG_USER)
	341	ttm_tt_free_user_pages(ttm);
	342	else
	343	ttm_tt_free_alloced_pages(ttm);
	344
	345	ttm_tt_free_page_directory(ttm);
	346	}
	347
	348	if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTANT_SWAP) &&
	349	ttm->swap_storage)
	350	fput(ttm->swap_storage);
	351
	352	kfree(ttm);
	353	}
	354
	355	int ttm_tt_set_user(struct ttm_tt *ttm,
	356	struct task_struct *tsk,
	357	unsigned long start, unsigned long num_pages)
	358	{
	359	struct mm_struct *mm = tsk->mm;
	360	int ret;
	361	int write = (ttm->page_flags & TTM_PAGE_FLAG_WRITE) != 0;
a987fcaa	362	struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
ba4e7d97 TH	363
	364	BUG_ON(num_pages != ttm->num_pages);
	365	BUG_ON((ttm->page_flags & TTM_PAGE_FLAG_USER) == 0);
	366
	367	/**
	368	* Account user pages as lowmem pages for now.
	369	*/
	370
	371	ret = ttm_mem_global_alloc(mem_glob, num_pages * PAGE_SIZE,
5fd9cbad	372	false, false);
ba4e7d97 TH	373	if (unlikely(ret != 0))
	374	return ret;
	375
	376	down_read(&mm->mmap_sem);
	377	ret = get_user_pages(tsk, mm, start, num_pages,
	378	write, 0, ttm->pages, NULL);
	379	up_read(&mm->mmap_sem);
	380
	381	if (ret != num_pages && write) {
	382	ttm_tt_free_user_pages(ttm);
5fd9cbad	383	ttm_mem_global_free(mem_glob, num_pages * PAGE_SIZE);
ba4e7d97 TH	384	return -ENOMEM;
	385	}
	386
	387	ttm->tsk = tsk;
	388	ttm->start = start;
	389	ttm->state = tt_unbound;
	390
	391	return 0;
	392	}
	393
	394	struct ttm_tt ttm_tt_create(struct ttm_bo_device bdev, unsigned long size,
	395	uint32_t page_flags, struct page *dummy_read_page)
	396	{
	397	struct ttm_bo_driver *bo_driver = bdev->driver;
	398	struct ttm_tt *ttm;
	399
	400	if (!bo_driver)
	401	return NULL;
	402
	403	ttm = kzalloc(sizeof(*ttm), GFP_KERNEL);
	404	if (!ttm)
	405	return NULL;
	406
a987fcaa	407	ttm->glob = bdev->glob;
ba4e7d97 TH	408	ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
	409	ttm->first_himem_page = ttm->num_pages;
	410	ttm->last_lomem_page = -1;
	411	ttm->caching_state = tt_cached;
	412	ttm->page_flags = page_flags;
	413
	414	ttm->dummy_read_page = dummy_read_page;
	415
	416	ttm_tt_alloc_page_directory(ttm);
	417	if (!ttm->pages) {
	418	ttm_tt_destroy(ttm);
	419	printk(KERN_ERR TTM_PFX "Failed allocating page table\n");
	420	return NULL;
	421	}
	422	ttm->be = bo_driver->create_ttm_backend_entry(bdev);
	423	if (!ttm->be) {
	424	ttm_tt_destroy(ttm);
	425	printk(KERN_ERR TTM_PFX "Failed creating ttm backend entry\n");
	426	return NULL;
	427	}
	428	ttm->state = tt_unpopulated;
	429	return ttm;
	430	}
	431
	432	void ttm_tt_unbind(struct ttm_tt *ttm)
	433	{
	434	int ret;
	435	struct ttm_backend *be = ttm->be;
	436
	437	if (ttm->state == tt_bound) {
	438	ret = be->func->unbind(be);
	439	BUG_ON(ret);
	440	ttm->state = tt_unbound;
	441	}
	442	}
	443
	444	int ttm_tt_bind(struct ttm_tt ttm, struct ttm_mem_reg bo_mem)
	445	{
	446	int ret = 0;
	447	struct ttm_backend *be;
	448
	449	if (!ttm)
	450	return -EINVAL;
	451
	452	if (ttm->state == tt_bound)
	453	return 0;
	454
	455	be = ttm->be;
	456
	457	ret = ttm_tt_populate(ttm);
	458	if (ret)
	459	return ret;
	460
	461	ret = be->func->bind(be, bo_mem);
	462	if (ret) {
	463	printk(KERN_ERR TTM_PFX "Couldn't bind backend.\n");
	464	return ret;
	465	}
	466
	467	ttm->state = tt_bound;
	468
	469	if (ttm->page_flags & TTM_PAGE_FLAG_USER)
	470	ttm->page_flags \|= TTM_PAGE_FLAG_USER_DIRTY;
	471	return 0;
472	}
473	EXPORT_SYMBOL(ttm_tt_bind);
474
475	static int ttm_tt_swapin(struct ttm_tt *ttm)
476	{
477	struct address_space *swap_space;
478	struct file *swap_storage;
479	struct page *from_page;
480	struct page *to_page;
481	void *from_virtual;
482	void *to_virtual;
483	int i;
290e5505	484	int ret = -ENOMEM;
ba4e7d97 TH	485
	486	if (ttm->page_flags & TTM_PAGE_FLAG_USER) {
	487	ret = ttm_tt_set_user(ttm, ttm->tsk, ttm->start,
	488	ttm->num_pages);
	489	if (unlikely(ret != 0))
	490	return ret;
	491
	492	ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED;
	493	return 0;
	494	}
	495
	496	swap_storage = ttm->swap_storage;
	497	BUG_ON(swap_storage == NULL);
	498
	499	swap_space = swap_storage->f_path.dentry->d_inode->i_mapping;
	500
	501	for (i = 0; i < ttm->num_pages; ++i) {
	502	from_page = read_mapping_page(swap_space, i, NULL);
290e5505 MM	503	if (IS_ERR(from_page)) {
290e5505 MM	504	ret = PTR_ERR(from_page);
ba4e7d97	505	goto out_err;
290e5505	506	}
ba4e7d97 TH	507	to_page = __ttm_tt_get_page(ttm, i);
	508	if (unlikely(to_page == NULL))
	509	goto out_err;
	510
	511	preempt_disable();
	512	from_virtual = kmap_atomic(from_page, KM_USER0);
	513	to_virtual = kmap_atomic(to_page, KM_USER1);
	514	memcpy(to_virtual, from_virtual, PAGE_SIZE);
	515	kunmap_atomic(to_virtual, KM_USER1);
	516	kunmap_atomic(from_virtual, KM_USER0);
	517	preempt_enable();
	518	page_cache_release(from_page);
	519	}
	520
	521	if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTANT_SWAP))
	522	fput(swap_storage);
	523	ttm->swap_storage = NULL;
	524	ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED;
	525
	526	return 0;
	527	out_err:
	528	ttm_tt_free_alloced_pages(ttm);
290e5505	529	return ret;
ba4e7d97 TH	530	}
	531
	532	int ttm_tt_swapout(struct ttm_tt ttm, struct file persistant_swap_storage)
	533	{
	534	struct address_space *swap_space;
	535	struct file *swap_storage;
	536	struct page *from_page;
	537	struct page *to_page;
	538	void *from_virtual;
	539	void *to_virtual;
	540	int i;
290e5505	541	int ret = -ENOMEM;
ba4e7d97 TH	542
	543	BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated);
	544	BUG_ON(ttm->caching_state != tt_cached);
	545
	546	/*
	547	* For user buffers, just unpin the pages, as there should be
	548	* vma references.
	549	*/
	550
	551	if (ttm->page_flags & TTM_PAGE_FLAG_USER) {
	552	ttm_tt_free_user_pages(ttm);
	553	ttm->page_flags \|= TTM_PAGE_FLAG_SWAPPED;
	554	ttm->swap_storage = NULL;
	555	return 0;
	556	}
	557
	558	if (!persistant_swap_storage) {
	559	swap_storage = shmem_file_setup("ttm swap",
	560	ttm->num_pages << PAGE_SHIFT,
	561	0);
	562	if (unlikely(IS_ERR(swap_storage))) {
	563	printk(KERN_ERR "Failed allocating swap storage.\n");
290e5505	564	return PTR_ERR(swap_storage);
ba4e7d97 TH	565	}
	566	} else
	567	swap_storage = persistant_swap_storage;
	568
	569	swap_space = swap_storage->f_path.dentry->d_inode->i_mapping;
	570
	571	for (i = 0; i < ttm->num_pages; ++i) {
	572	from_page = ttm->pages[i];
	573	if (unlikely(from_page == NULL))
	574	continue;
	575	to_page = read_mapping_page(swap_space, i, NULL);
290e5505 MM	576	if (unlikely(IS_ERR(to_page))) {
290e5505 MM	577	ret = PTR_ERR(to_page);
ba4e7d97	578	goto out_err;
290e5505	579	}
ba4e7d97 TH	580	preempt_disable();
	581	from_virtual = kmap_atomic(from_page, KM_USER0);
	582	to_virtual = kmap_atomic(to_page, KM_USER1);
	583	memcpy(to_virtual, from_virtual, PAGE_SIZE);
	584	kunmap_atomic(to_virtual, KM_USER1);
	585	kunmap_atomic(from_virtual, KM_USER0);
	586	preempt_enable();
	587	set_page_dirty(to_page);
	588	mark_page_accessed(to_page);
	589	page_cache_release(to_page);
	590	}
	591
	592	ttm_tt_free_alloced_pages(ttm);
	593	ttm->swap_storage = swap_storage;
	594	ttm->page_flags \|= TTM_PAGE_FLAG_SWAPPED;
	595	if (persistant_swap_storage)
	596	ttm->page_flags \|= TTM_PAGE_FLAG_PERSISTANT_SWAP;
	597
	598	return 0;
	599	out_err:
	600	if (!persistant_swap_storage)
	601	fput(swap_storage);
	602
290e5505	603	return ret;
ba4e7d97	604	}