mm: extend prefault helpers to fault in more than PAGE_SIZE

drm/i915 wants to read/write more than one page in its fastpath
and hence needs to prefault more than PAGE_SIZE bytes.

Add new functions in filemap.h to make that possible.

Also kill a copy&pasted spurious space in both functions while at it.

v2: As suggested by Andrew Morton, add a multipage parameter to both
functions to avoid the additional branch for the pagemap.c hotpath.
My gcc 4.6 here seems to dtrt and indeed reap these branches where not
needed.

v3: Becaus I couldn't find a way around adding a uaddr += PAGE_SIZE to
the filemap.c hotpaths (that the compiler couldn't remove again),
let's go with separate new functions for the multipage use-case.

v4: Adjust comment to CodingStlye and fix spelling.

Acked-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index e9cac478cced309d2dcb687ce4b056aad8aac2cf..6dc832902f53fe26318788f9bed6522b457f5097 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -416,7 +416,7 @@ i915_gem_shmem_pread(struct drm_device *dev,
                mutex_unlock(&dev->struct_mutex);
 
                if (!prefaulted) {
-                       ret = fault_in_pages_writeable(user_data, remain);
+                       ret = fault_in_multipages_writeable(user_data, remain);
                        /* Userspace is tricking us, but we've already clobbered
                         * its pages with the prefault and promised to write the
                         * data up to the first fault. Hence ignore any errors
@@ -809,8 +809,8 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
                       args->size))
                return -EFAULT;
 
-       ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
-                                     args->size);
+       ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
+                                          args->size);
        if (ret)
                return -EFAULT;
 

diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index eb85860001ec16eefffb9cd1aa4fc331341bae22..8e0b686d3afbe61ede1aeade08def2b0b1684709 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -997,7 +997,7 @@ validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
                if (!access_ok(VERIFY_WRITE, ptr, length))
                        return -EFAULT;
 
-               if (fault_in_pages_readable(ptr, length))
+               if (fault_in_multipages_readable(ptr, length))
                        return -EFAULT;
        }
 

diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index cfaaa6949b8b03687b4be0074fc35b1165b245ed..c93a9a9bcd358565ec44a71cd223aa161c750eb2 100644 (file)
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -426,7 +426,7 @@ static inline int fault_in_pages_writeable(char __user *uaddr, int size)
                 */
                if (((unsigned long)uaddr & PAGE_MASK) !=
                                ((unsigned long)end & PAGE_MASK))
-                       ret = __put_user(0, end);
+                       ret = __put_user(0, end);
        }
        return ret;
 }
@@ -445,13 +445,73 @@ static inline int fault_in_pages_readable(const char __user *uaddr, int size)
 
                if (((unsigned long)uaddr & PAGE_MASK) !=
                                ((unsigned long)end & PAGE_MASK)) {
-                       ret = __get_user(c, end);
+                       ret = __get_user(c, end);
                        (void)c;
                }
        }
        return ret;
 }
 
+/*
+ * Multipage variants of the above prefault helpers, useful if more than
+ * PAGE_SIZE of data needs to be prefaulted. These are separate from the above
+ * functions (which only handle up to PAGE_SIZE) to avoid clobbering the
+ * filemap.c hotpaths.
+ */
+static inline int fault_in_multipages_writeable(char __user *uaddr, int size)
+{
+       int ret;
+       const char __user *end = uaddr + size - 1;
+
+       if (unlikely(size == 0))
+               return 0;
+
+       /*
+        * Writing zeroes into userspace here is OK, because we know that if
+        * the zero gets there, we'll be overwriting it.
+        */
+       while (uaddr <= end) {
+               ret = __put_user(0, uaddr);
+               if (ret != 0)
+                       return ret;
+               uaddr += PAGE_SIZE;
+       }
+
+       /* Check whether the range spilled into the next page. */
+       if (((unsigned long)uaddr & PAGE_MASK) ==
+                       ((unsigned long)end & PAGE_MASK))
+               ret = __put_user(0, end);
+
+       return ret;
+}
+
+static inline int fault_in_multipages_readable(const char __user *uaddr,
+                                              int size)
+{
+       volatile char c;
+       int ret;
+       const char __user *end = uaddr + size - 1;
+
+       if (unlikely(size == 0))
+               return 0;
+
+       while (uaddr <= end) {
+               ret = __get_user(c, uaddr);
+               if (ret != 0)
+                       return ret;
+               uaddr += PAGE_SIZE;
+       }
+
+       /* Check whether the range spilled into the next page. */
+       if (((unsigned long)uaddr & PAGE_MASK) ==
+                       ((unsigned long)end & PAGE_MASK)) {
+               ret = __get_user(c, end);
+               (void)c;
+       }
+
+       return ret;
+}
+
 int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
                                pgoff_t index, gfp_t gfp_mask);
 int add_to_page_cache_lru(struct page *page, struct address_space *mapping,