From e5281ccd2e0049e2b9e8ce82449630d25082372d Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 28 Oct 2010 13:45:36 +0100 Subject: [PATCH] drm/i915: Eliminate nested get/put pages By using read_cache_page() for individual pages during pwrite/pread we can eliminate an unnecessary large allocation (and immediate free) of obj->pages. Also this eliminates any potential nesting of get/put pages, simplifying the code and preparing the path for greater things. Signed-off-by: Chris Wilson --- drivers/gpu/drm/i915/i915_drv.h | 9 - drivers/gpu/drm/i915/i915_gem.c | 358 +++++++++++++++----------------- 2 files changed, 163 insertions(+), 204 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 1f7e8b3c8df0..3df8a624ddc9 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -747,15 +747,6 @@ struct drm_i915_gem_object { */ unsigned int madv : 2; - /** - * Refcount for the pages array. With the current locking scheme, there - * are at most two concurrent users: Binding a bo to the gtt and - * pwrite/pread using physical addresses. So two bits for a maximum - * of two users are enough. - */ - unsigned int pages_refcount : 2; -#define DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT 0x3 - /** * Current tiling mode for the object. */ diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 4ade4e243379..abe6d901f95b 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -58,13 +58,6 @@ static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *o struct drm_file *file_priv); static void i915_gem_free_object_tail(struct drm_gem_object *obj); -static int -i915_gem_object_get_pages(struct drm_gem_object *obj, - gfp_t gfpmask); - -static void -i915_gem_object_put_pages(struct drm_gem_object *obj); - static int i915_gem_inactive_shrink(struct shrinker *shrinker, int nr_to_scan, gfp_t gfp_mask); @@ -326,22 +319,6 @@ i915_gem_object_cpu_accessible(struct drm_i915_gem_object *obj) obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; } -static inline int -fast_shmem_read(struct page **pages, - loff_t page_base, int page_offset, - char __user *data, - int length) -{ - char *vaddr; - int ret; - - vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT]); - ret = __copy_to_user_inatomic(data, vaddr + page_offset, length); - kunmap_atomic(vaddr); - - return ret; -} - static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj) { drm_i915_private_t *dev_priv = obj->dev->dev_private; @@ -429,8 +406,9 @@ i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj, struct drm_file *file_priv) { struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); + struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping; ssize_t remain; - loff_t offset, page_base; + loff_t offset; char __user *user_data; int page_offset, page_length; @@ -441,21 +419,34 @@ i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj, offset = args->offset; while (remain > 0) { + struct page *page; + char *vaddr; + int ret; + /* Operation in this page * - * page_base = page offset within aperture * page_offset = offset within page * page_length = bytes to copy for this page */ - page_base = (offset & ~(PAGE_SIZE-1)); page_offset = offset & (PAGE_SIZE-1); page_length = remain; if ((page_offset + remain) > PAGE_SIZE) page_length = PAGE_SIZE - page_offset; - if (fast_shmem_read(obj_priv->pages, - page_base, page_offset, - user_data, page_length)) + page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT, + GFP_HIGHUSER | __GFP_RECLAIMABLE); + if (IS_ERR(page)) + return PTR_ERR(page); + + vaddr = kmap_atomic(page); + ret = __copy_to_user_inatomic(user_data, + vaddr + page_offset, + page_length); + kunmap_atomic(vaddr); + + mark_page_accessed(page); + page_cache_release(page); + if (ret) return -EFAULT; remain -= page_length; @@ -466,31 +457,6 @@ i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj, return 0; } -static int -i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj) -{ - int ret; - - ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN); - - /* If we've insufficient memory to map in the pages, attempt - * to make some space by throwing out some old buffers. - */ - if (ret == -ENOMEM) { - struct drm_device *dev = obj->dev; - - ret = i915_gem_evict_something(dev, obj->size, - i915_gem_get_gtt_alignment(obj), - false); - if (ret) - return ret; - - ret = i915_gem_object_get_pages(obj, 0); - } - - return ret; -} - /** * This is the fallback shmem pread path, which allocates temporary storage * in kernel space to copy_to_user into outside of the struct_mutex, so we @@ -502,14 +468,15 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj, struct drm_i915_gem_pread *args, struct drm_file *file_priv) { + struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping; struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); struct mm_struct *mm = current->mm; struct page **user_pages; ssize_t remain; loff_t offset, pinned_pages, i; loff_t first_data_page, last_data_page, num_pages; - int shmem_page_index, shmem_page_offset; - int data_page_index, data_page_offset; + int shmem_page_offset; + int data_page_index, data_page_offset; int page_length; int ret; uint64_t data_ptr = args->data_ptr; @@ -552,15 +519,15 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj, offset = args->offset; while (remain > 0) { + struct page *page; + /* Operation in this page * - * shmem_page_index = page number within shmem file * shmem_page_offset = offset within page in shmem file * data_page_index = page number in get_user_pages return * data_page_offset = offset with data_page_index page. * page_length = bytes to copy for this page */ - shmem_page_index = offset / PAGE_SIZE; shmem_page_offset = offset & ~PAGE_MASK; data_page_index = data_ptr / PAGE_SIZE - first_data_page; data_page_offset = data_ptr & ~PAGE_MASK; @@ -571,8 +538,13 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj, if ((data_page_offset + page_length) > PAGE_SIZE) page_length = PAGE_SIZE - data_page_offset; + page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT, + GFP_HIGHUSER | __GFP_RECLAIMABLE); + if (IS_ERR(page)) + return PTR_ERR(page); + if (do_bit17_swizzling) { - slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index], + slow_shmem_bit17_copy(page, shmem_page_offset, user_pages[data_page_index], data_page_offset, @@ -581,11 +553,14 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj, } else { slow_shmem_copy(user_pages[data_page_index], data_page_offset, - obj_priv->pages[shmem_page_index], + page, shmem_page_offset, page_length); } + mark_page_accessed(page); + page_cache_release(page); + remain -= page_length; data_ptr += page_length; offset += page_length; @@ -594,6 +569,7 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj, out: for (i = 0; i < pinned_pages; i++) { SetPageDirty(user_pages[i]); + mark_page_accessed(user_pages[i]); page_cache_release(user_pages[i]); } drm_free_large(user_pages); @@ -649,15 +625,11 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, goto out; } - ret = i915_gem_object_get_pages_or_evict(obj); - if (ret) - goto out; - ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset, args->size); if (ret) - goto out_put; + goto out; ret = -EFAULT; if (!i915_gem_object_needs_bit17_swizzle(obj)) @@ -665,8 +637,6 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, if (ret == -EFAULT) ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv); -out_put: - i915_gem_object_put_pages(obj); out: drm_gem_object_unreference(obj); unlock: @@ -718,22 +688,6 @@ slow_kernel_write(struct io_mapping *mapping, io_mapping_unmap(dst_vaddr); } -static inline int -fast_shmem_write(struct page **pages, - loff_t page_base, int page_offset, - char __user *data, - int length) -{ - char *vaddr; - int ret; - - vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT]); - ret = __copy_from_user_inatomic(vaddr + page_offset, data, length); - kunmap_atomic(vaddr); - - return ret; -} - /** * This is the fast pwrite path, where we copy the data directly from the * user into the GTT, uncached. @@ -890,9 +844,10 @@ i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, struct drm_i915_gem_pwrite *args, struct drm_file *file_priv) { + struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping; struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); ssize_t remain; - loff_t offset, page_base; + loff_t offset; char __user *user_data; int page_offset, page_length; @@ -904,21 +859,40 @@ i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, obj_priv->dirty = 1; while (remain > 0) { + struct page *page; + char *vaddr; + int ret; + /* Operation in this page * - * page_base = page offset within aperture * page_offset = offset within page * page_length = bytes to copy for this page */ - page_base = (offset & ~(PAGE_SIZE-1)); page_offset = offset & (PAGE_SIZE-1); page_length = remain; if ((page_offset + remain) > PAGE_SIZE) page_length = PAGE_SIZE - page_offset; - if (fast_shmem_write(obj_priv->pages, - page_base, page_offset, - user_data, page_length)) + page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT, + GFP_HIGHUSER | __GFP_RECLAIMABLE); + if (IS_ERR(page)) + return PTR_ERR(page); + + vaddr = kmap_atomic(page, KM_USER0); + ret = __copy_from_user_inatomic(vaddr + page_offset, + user_data, + page_length); + kunmap_atomic(vaddr, KM_USER0); + + set_page_dirty(page); + mark_page_accessed(page); + page_cache_release(page); + + /* If we get a fault while copying data, then (presumably) our + * source page isn't available. Return the error and we'll + * retry in the slow path. + */ + if (ret) return -EFAULT; remain -= page_length; @@ -941,13 +915,14 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, struct drm_i915_gem_pwrite *args, struct drm_file *file_priv) { + struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping; struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); struct mm_struct *mm = current->mm; struct page **user_pages; ssize_t remain; loff_t offset, pinned_pages, i; loff_t first_data_page, last_data_page, num_pages; - int shmem_page_index, shmem_page_offset; + int shmem_page_offset; int data_page_index, data_page_offset; int page_length; int ret; @@ -990,15 +965,15 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, obj_priv->dirty = 1; while (remain > 0) { + struct page *page; + /* Operation in this page * - * shmem_page_index = page number within shmem file * shmem_page_offset = offset within page in shmem file * data_page_index = page number in get_user_pages return * data_page_offset = offset with data_page_index page. * page_length = bytes to copy for this page */ - shmem_page_index = offset / PAGE_SIZE; shmem_page_offset = offset & ~PAGE_MASK; data_page_index = data_ptr / PAGE_SIZE - first_data_page; data_page_offset = data_ptr & ~PAGE_MASK; @@ -1009,21 +984,32 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, if ((data_page_offset + page_length) > PAGE_SIZE) page_length = PAGE_SIZE - data_page_offset; + page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT, + GFP_HIGHUSER | __GFP_RECLAIMABLE); + if (IS_ERR(page)) { + ret = PTR_ERR(page); + goto out; + } + if (do_bit17_swizzling) { - slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index], + slow_shmem_bit17_copy(page, shmem_page_offset, user_pages[data_page_index], data_page_offset, page_length, 0); } else { - slow_shmem_copy(obj_priv->pages[shmem_page_index], + slow_shmem_copy(page, shmem_page_offset, user_pages[data_page_index], data_page_offset, page_length); } + set_page_dirty(page); + mark_page_accessed(page); + page_cache_release(page); + remain -= page_length; data_ptr += page_length; offset += page_length; @@ -1112,22 +1098,15 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, out_unpin: i915_gem_object_unpin(obj); } else { - ret = i915_gem_object_get_pages_or_evict(obj); - if (ret) - goto out; - ret = i915_gem_object_set_to_cpu_domain(obj, 1); if (ret) - goto out_put; + goto out; ret = -EFAULT; if (!i915_gem_object_needs_bit17_swizzle(obj)) ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file); if (ret == -EFAULT) ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file); - -out_put: - i915_gem_object_put_pages(obj); } out: @@ -1587,19 +1566,62 @@ unlock: return ret; } +static int +i915_gem_object_get_pages_gtt(struct drm_gem_object *obj, + gfp_t gfpmask) +{ + struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); + int page_count, i; + struct address_space *mapping; + struct inode *inode; + struct page *page; + + /* Get the list of pages out of our struct file. They'll be pinned + * at this point until we release them. + */ + page_count = obj->size / PAGE_SIZE; + BUG_ON(obj_priv->pages != NULL); + obj_priv->pages = drm_malloc_ab(page_count, sizeof(struct page *)); + if (obj_priv->pages == NULL) + return -ENOMEM; + + inode = obj->filp->f_path.dentry->d_inode; + mapping = inode->i_mapping; + for (i = 0; i < page_count; i++) { + page = read_cache_page_gfp(mapping, i, + GFP_HIGHUSER | + __GFP_COLD | + __GFP_RECLAIMABLE | + gfpmask); + if (IS_ERR(page)) + goto err_pages; + + obj_priv->pages[i] = page; + } + + if (obj_priv->tiling_mode != I915_TILING_NONE) + i915_gem_object_do_bit_17_swizzle(obj); + + return 0; + +err_pages: + while (i--) + page_cache_release(obj_priv->pages[i]); + + drm_free_large(obj_priv->pages); + obj_priv->pages = NULL; + return PTR_ERR(page); +} + static void -i915_gem_object_put_pages(struct drm_gem_object *obj) +i915_gem_object_put_pages_gtt(struct drm_gem_object *obj) { struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); int page_count = obj->size / PAGE_SIZE; int i; - BUG_ON(obj_priv->pages_refcount == 0); BUG_ON(obj_priv->madv == __I915_MADV_PURGED); - if (--obj_priv->pages_refcount != 0) - return; - if (obj_priv->tiling_mode != I915_TILING_NONE) i915_gem_object_save_bit_17_swizzle(obj); @@ -2229,8 +2251,7 @@ i915_gem_object_unbind(struct drm_gem_object *obj) drm_unbind_agp(obj_priv->agp_mem); drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE); - i915_gem_object_put_pages(obj); - BUG_ON(obj_priv->pages_refcount); + i915_gem_object_put_pages_gtt(obj); i915_gem_info_remove_gtt(dev_priv, obj); list_del_init(&obj_priv->mm_list); @@ -2290,62 +2311,6 @@ i915_gpu_idle(struct drm_device *dev) return 0; } -static int -i915_gem_object_get_pages(struct drm_gem_object *obj, - gfp_t gfpmask) -{ - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - int page_count, i; - struct address_space *mapping; - struct inode *inode; - struct page *page; - - BUG_ON(obj_priv->pages_refcount - == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT); - - if (obj_priv->pages_refcount++ != 0) - return 0; - - /* Get the list of pages out of our struct file. They'll be pinned - * at this point until we release them. - */ - page_count = obj->size / PAGE_SIZE; - BUG_ON(obj_priv->pages != NULL); - obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *)); - if (obj_priv->pages == NULL) { - obj_priv->pages_refcount--; - return -ENOMEM; - } - - inode = obj->filp->f_path.dentry->d_inode; - mapping = inode->i_mapping; - for (i = 0; i < page_count; i++) { - page = read_cache_page_gfp(mapping, i, - GFP_HIGHUSER | - __GFP_COLD | - __GFP_RECLAIMABLE | - gfpmask); - if (IS_ERR(page)) - goto err_pages; - - obj_priv->pages[i] = page; - } - - if (obj_priv->tiling_mode != I915_TILING_NONE) - i915_gem_object_do_bit_17_swizzle(obj); - - return 0; - -err_pages: - while (i--) - page_cache_release(obj_priv->pages[i]); - - drm_free_large(obj_priv->pages); - obj_priv->pages = NULL; - obj_priv->pages_refcount--; - return PTR_ERR(page); -} - static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg) { struct drm_gem_object *obj = reg->obj; @@ -2772,7 +2737,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, goto search_free; } - ret = i915_gem_object_get_pages(obj, gfpmask); + ret = i915_gem_object_get_pages_gtt(obj, gfpmask); if (ret) { drm_mm_put_block(obj_priv->gtt_space); obj_priv->gtt_space = NULL; @@ -2806,7 +2771,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, obj_priv->gtt_space->start, obj_priv->agp_type); if (obj_priv->agp_mem == NULL) { - i915_gem_object_put_pages(obj); + i915_gem_object_put_pages_gtt(obj); drm_mm_put_block(obj_priv->gtt_space); obj_priv->gtt_space = NULL; @@ -4860,33 +4825,35 @@ void i915_gem_free_all_phys_object(struct drm_device *dev) void i915_gem_detach_phys_object(struct drm_device *dev, struct drm_gem_object *obj) { - struct drm_i915_gem_object *obj_priv; + struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping; + struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); + char *vaddr; int i; - int ret; int page_count; - obj_priv = to_intel_bo(obj); if (!obj_priv->phys_obj) return; - - ret = i915_gem_object_get_pages(obj, 0); - if (ret) - goto out; + vaddr = obj_priv->phys_obj->handle->vaddr; page_count = obj->size / PAGE_SIZE; for (i = 0; i < page_count; i++) { - char *dst = kmap_atomic(obj_priv->pages[i]); - char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE); - - memcpy(dst, src, PAGE_SIZE); - kunmap_atomic(dst); + struct page *page = read_cache_page_gfp(mapping, i, + GFP_HIGHUSER | __GFP_RECLAIMABLE); + if (!IS_ERR(page)) { + char *dst = kmap_atomic(page); + memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE); + kunmap_atomic(dst); + + drm_clflush_pages(&page, 1); + + set_page_dirty(page); + mark_page_accessed(page); + page_cache_release(page); + } } - drm_clflush_pages(obj_priv->pages, page_count); drm_agp_chipset_flush(dev); - i915_gem_object_put_pages(obj); -out: obj_priv->phys_obj->cur_obj = NULL; obj_priv->phys_obj = NULL; } @@ -4897,6 +4864,7 @@ i915_gem_attach_phys_object(struct drm_device *dev, int id, int align) { + struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping; drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj_priv; int ret = 0; @@ -4920,7 +4888,7 @@ i915_gem_attach_phys_object(struct drm_device *dev, obj->size, align); if (ret) { DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size); - goto out; + return ret; } } @@ -4928,27 +4896,27 @@ i915_gem_attach_phys_object(struct drm_device *dev, obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1]; obj_priv->phys_obj->cur_obj = obj; - ret = i915_gem_object_get_pages(obj, 0); - if (ret) { - DRM_ERROR("failed to get page list\n"); - goto out; - } - page_count = obj->size / PAGE_SIZE; for (i = 0; i < page_count; i++) { - char *src = kmap_atomic(obj_priv->pages[i]); - char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE); + struct page *page; + char *dst, *src; + + page = read_cache_page_gfp(mapping, i, + GFP_HIGHUSER | __GFP_RECLAIMABLE); + if (IS_ERR(page)) + return PTR_ERR(page); + src = kmap_atomic(obj_priv->pages[i]); + dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE); memcpy(dst, src, PAGE_SIZE); kunmap_atomic(src); - } - i915_gem_object_put_pages(obj); + mark_page_accessed(page); + page_cache_release(page); + } return 0; -out: - return ret; } static int -- 2.20.1