projects / GitHub / LineageOS / android_kernel_motorola_exynos9610.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: try another possible DDC bus for the SDVO device with multiple outputs
[GitHub/LineageOS/android_kernel_motorola_exynos9610.git] / drivers / gpu / drm / i915 / i915_gem.c
CommitLineData
673a394b
EA		 1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
1c5d22f7 32#include "i915_trace.h"
652c393a 33#include "intel_drv.h"
673a394b 34#include <linux/swap.h>
79e53945 35#include <linux/pci.h>
673a394b 36
28dfe52a
EA		 37#define I915_GEM_GPU_DOMAINS (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
38
e47c68e9
EA		 39static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
40static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
41static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
e47c68e9
EA		 42static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
43 int write);
44static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
45 uint64_t offset,
46 uint64_t size);
47static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
673a394b 48static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
de151cf6
JB		 49static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
50 unsigned alignment);
de151cf6 51static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
07f73f69 52static int i915_gem_evict_something(struct drm_device *dev, int min_size);
ab5ee576 53static int i915_gem_evict_from_inactive_list(struct drm_device *dev);
71acb5eb
DA		 54static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
55 struct drm_i915_gem_pwrite *args,
56 struct drm_file *file_priv);
673a394b 57
31169714
CW		 58static LIST_HEAD(shrink_list);
59static DEFINE_SPINLOCK(shrink_list_lock);
60
79e53945
JB		 61int i915_gem_do_init(struct drm_device *dev, unsigned long start,
62 unsigned long end)
673a394b
EA		 63{
64 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 65
79e53945
JB		 66 if (start >= end ||
67 (start & (PAGE_SIZE - 1)) != 0 ||
68 (end & (PAGE_SIZE - 1)) != 0) {
673a394b
EA		 69 return -EINVAL;
70 }
71
79e53945
JB		 72 drm_mm_init(&dev_priv->mm.gtt_space, start,
73 end - start);
673a394b 74
79e53945
JB		 75 dev->gtt_total = (uint32_t) (end - start);
76
77 return 0;
78}
673a394b 79
79e53945
JB		 80int
81i915_gem_init_ioctl(struct drm_device *dev, void *data,
82 struct drm_file *file_priv)
83{
84 struct drm_i915_gem_init *args = data;
85 int ret;
86
87 mutex_lock(&dev->struct_mutex);
88 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
673a394b
EA		 89 mutex_unlock(&dev->struct_mutex);
90
79e53945 91 return ret;
673a394b
EA		 92}
93
5a125c3c
EA		 94int
95i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
96 struct drm_file *file_priv)
97{
5a125c3c 98 struct drm_i915_gem_get_aperture *args = data;
5a125c3c
EA		 99
100 if (!(dev->driver->driver_features & DRIVER_GEM))
101 return -ENODEV;
102
103 args->aper_size = dev->gtt_total;
2678d9d6
KP		 104 args->aper_available_size = (args->aper_size -
105 atomic_read(&dev->pin_memory));
5a125c3c
EA		 106
107 return 0;
108}
109
673a394b
EA		 110
111/**
112 * Creates a new mm object and returns a handle to it.
113 */
114int
115i915_gem_create_ioctl(struct drm_device *dev, void *data,
116 struct drm_file *file_priv)
117{
118 struct drm_i915_gem_create *args = data;
119 struct drm_gem_object *obj;
a1a2d1d3
PP		 120 int ret;
121 u32 handle;
673a394b
EA		 122
123 args->size = roundup(args->size, PAGE_SIZE);
124
125 /* Allocate the new object */
126 obj = drm_gem_object_alloc(dev, args->size);
127 if (obj == NULL)
128 return -ENOMEM;
129
130 ret = drm_gem_handle_create(file_priv, obj, &handle);
131 mutex_lock(&dev->struct_mutex);
132 drm_gem_object_handle_unreference(obj);
133 mutex_unlock(&dev->struct_mutex);
134
135 if (ret)
136 return ret;
137
138 args->handle = handle;
139
140 return 0;
141}
142
eb01459f
EA		 143static inline int
144fast_shmem_read(struct page **pages,
145 loff_t page_base, int page_offset,
146 char __user *data,
147 int length)
148{
149 char __iomem *vaddr;
2bc43b5c 150 int unwritten;
eb01459f
EA		 151
152 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
153 if (vaddr == NULL)
154 return -ENOMEM;
2bc43b5c 155 unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
eb01459f
EA		 156 kunmap_atomic(vaddr, KM_USER0);
157
2bc43b5c
FM		 158 if (unwritten)
159 return -EFAULT;
160
161 return 0;
eb01459f
EA		 162}
163
280b713b
EA		 164static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
165{
166 drm_i915_private_t *dev_priv = obj->dev->dev_private;
167 struct drm_i915_gem_object *obj_priv = obj->driver_private;
168
169 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
170 obj_priv->tiling_mode != I915_TILING_NONE;
171}
172
40123c1f
EA		 173static inline int
174slow_shmem_copy(struct page *dst_page,
175 int dst_offset,
176 struct page *src_page,
177 int src_offset,
178 int length)
179{
180 char *dst_vaddr, *src_vaddr;
181
182 dst_vaddr = kmap_atomic(dst_page, KM_USER0);
183 if (dst_vaddr == NULL)
184 return -ENOMEM;
185
186 src_vaddr = kmap_atomic(src_page, KM_USER1);
187 if (src_vaddr == NULL) {
188 kunmap_atomic(dst_vaddr, KM_USER0);
189 return -ENOMEM;
190 }
191
192 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
193
194 kunmap_atomic(src_vaddr, KM_USER1);
195 kunmap_atomic(dst_vaddr, KM_USER0);
196
197 return 0;
198}
199
280b713b
EA		 200static inline int
201slow_shmem_bit17_copy(struct page *gpu_page,
202 int gpu_offset,
203 struct page *cpu_page,
204 int cpu_offset,
205 int length,
206 int is_read)
207{
208 char *gpu_vaddr, *cpu_vaddr;
209
210 /* Use the unswizzled path if this page isn't affected. */
211 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
212 if (is_read)
213 return slow_shmem_copy(cpu_page, cpu_offset,
214 gpu_page, gpu_offset, length);
215 else
216 return slow_shmem_copy(gpu_page, gpu_offset,
217 cpu_page, cpu_offset, length);
218 }
219
220 gpu_vaddr = kmap_atomic(gpu_page, KM_USER0);
221 if (gpu_vaddr == NULL)
222 return -ENOMEM;
223
224 cpu_vaddr = kmap_atomic(cpu_page, KM_USER1);
225 if (cpu_vaddr == NULL) {
226 kunmap_atomic(gpu_vaddr, KM_USER0);
227 return -ENOMEM;
228 }
229
230 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
231 * XORing with the other bits (A9 for Y, A9 and A10 for X)
232 */
233 while (length > 0) {
234 int cacheline_end = ALIGN(gpu_offset + 1, 64);
235 int this_length = min(cacheline_end - gpu_offset, length);
236 int swizzled_gpu_offset = gpu_offset ^ 64;
237
238 if (is_read) {
239 memcpy(cpu_vaddr + cpu_offset,
240 gpu_vaddr + swizzled_gpu_offset,
241 this_length);
242 } else {
243 memcpy(gpu_vaddr + swizzled_gpu_offset,
244 cpu_vaddr + cpu_offset,
245 this_length);
246 }
247 cpu_offset += this_length;
248 gpu_offset += this_length;
249 length -= this_length;
250 }
251
252 kunmap_atomic(cpu_vaddr, KM_USER1);
253 kunmap_atomic(gpu_vaddr, KM_USER0);
254
255 return 0;
256}
257
eb01459f
EA		 258/**
259 * This is the fast shmem pread path, which attempts to copy_from_user directly
260 * from the backing pages of the object to the user's address space. On a
261 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
262 */
263static int
264i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
265 struct drm_i915_gem_pread *args,
266 struct drm_file *file_priv)
267{
268 struct drm_i915_gem_object *obj_priv = obj->driver_private;
269 ssize_t remain;
270 loff_t offset, page_base;
271 char __user *user_data;
272 int page_offset, page_length;
273 int ret;
274
275 user_data = (char __user *) (uintptr_t) args->data_ptr;
276 remain = args->size;
277
278 mutex_lock(&dev->struct_mutex);
279
280 ret = i915_gem_object_get_pages(obj);
281 if (ret != 0)
282 goto fail_unlock;
283
284 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
285 args->size);
286 if (ret != 0)
287 goto fail_put_pages;
288
289 obj_priv = obj->driver_private;
290 offset = args->offset;
291
292 while (remain > 0) {
293 /* Operation in this page
294 *
295 * page_base = page offset within aperture
296 * page_offset = offset within page
297 * page_length = bytes to copy for this page
298 */
299 page_base = (offset & ~(PAGE_SIZE-1));
300 page_offset = offset & (PAGE_SIZE-1);
301 page_length = remain;
302 if ((page_offset + remain) > PAGE_SIZE)
303 page_length = PAGE_SIZE - page_offset;
304
305 ret = fast_shmem_read(obj_priv->pages,
306 page_base, page_offset,
307 user_data, page_length);
308 if (ret)
309 goto fail_put_pages;
310
311 remain -= page_length;
312 user_data += page_length;
313 offset += page_length;
314 }
315
316fail_put_pages:
317 i915_gem_object_put_pages(obj);
318fail_unlock:
319 mutex_unlock(&dev->struct_mutex);
320
321 return ret;
322}
323
07f73f69
CW		 324static inline gfp_t
325i915_gem_object_get_page_gfp_mask (struct drm_gem_object *obj)
326{
327 return mapping_gfp_mask(obj->filp->f_path.dentry->d_inode->i_mapping);
328}
329
330static inline void
331i915_gem_object_set_page_gfp_mask (struct drm_gem_object *obj, gfp_t gfp)
332{
333 mapping_set_gfp_mask(obj->filp->f_path.dentry->d_inode->i_mapping, gfp);
334}
335
336static int
337i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
338{
339 int ret;
340
341 ret = i915_gem_object_get_pages(obj);
342
343 /* If we've insufficient memory to map in the pages, attempt
344 * to make some space by throwing out some old buffers.
345 */
346 if (ret == -ENOMEM) {
347 struct drm_device *dev = obj->dev;
348 gfp_t gfp;
349
350 ret = i915_gem_evict_something(dev, obj->size);
351 if (ret)
352 return ret;
353
354 gfp = i915_gem_object_get_page_gfp_mask(obj);
355 i915_gem_object_set_page_gfp_mask(obj, gfp & ~__GFP_NORETRY);
356 ret = i915_gem_object_get_pages(obj);
357 i915_gem_object_set_page_gfp_mask (obj, gfp);
358 }
359
360 return ret;
361}
362
eb01459f
EA		 363/**
364 * This is the fallback shmem pread path, which allocates temporary storage
365 * in kernel space to copy_to_user into outside of the struct_mutex, so we
366 * can copy out of the object's backing pages while holding the struct mutex
367 * and not take page faults.
368 */
369static int
370i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
371 struct drm_i915_gem_pread *args,
372 struct drm_file *file_priv)
373{
374 struct drm_i915_gem_object *obj_priv = obj->driver_private;
375 struct mm_struct *mm = current->mm;
376 struct page **user_pages;
377 ssize_t remain;
378 loff_t offset, pinned_pages, i;
379 loff_t first_data_page, last_data_page, num_pages;
380 int shmem_page_index, shmem_page_offset;
381 int data_page_index, data_page_offset;
382 int page_length;
383 int ret;
384 uint64_t data_ptr = args->data_ptr;
280b713b 385 int do_bit17_swizzling;
eb01459f
EA		 386
387 remain = args->size;
388
389 /* Pin the user pages containing the data. We can't fault while
390 * holding the struct mutex, yet we want to hold it while
391 * dereferencing the user data.
392 */
393 first_data_page = data_ptr / PAGE_SIZE;
394 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
395 num_pages = last_data_page - first_data_page + 1;
396
8e7d2b2c 397 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
eb01459f
EA		 398 if (user_pages == NULL)
399 return -ENOMEM;
400
401 down_read(&mm->mmap_sem);
402 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
e5e9ecde 403 num_pages, 1, 0, user_pages, NULL);
eb01459f
EA		 404 up_read(&mm->mmap_sem);
405 if (pinned_pages < num_pages) {
406 ret = -EFAULT;
407 goto fail_put_user_pages;
408 }
409
280b713b
EA		 410 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
411
eb01459f
EA		 412 mutex_lock(&dev->struct_mutex);
413
07f73f69
CW		 414 ret = i915_gem_object_get_pages_or_evict(obj);
415 if (ret)
eb01459f
EA		 416 goto fail_unlock;
417
418 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
419 args->size);
420 if (ret != 0)
421 goto fail_put_pages;
422
423 obj_priv = obj->driver_private;
424 offset = args->offset;
425
426 while (remain > 0) {
427 /* Operation in this page
428 *
429 * shmem_page_index = page number within shmem file
430 * shmem_page_offset = offset within page in shmem file
431 * data_page_index = page number in get_user_pages return
432 * data_page_offset = offset with data_page_index page.
433 * page_length = bytes to copy for this page
434 */
435 shmem_page_index = offset / PAGE_SIZE;
436 shmem_page_offset = offset & ~PAGE_MASK;
437 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
438 data_page_offset = data_ptr & ~PAGE_MASK;
439
440 page_length = remain;
441 if ((shmem_page_offset + page_length) > PAGE_SIZE)
442 page_length = PAGE_SIZE - shmem_page_offset;
443 if ((data_page_offset + page_length) > PAGE_SIZE)
444 page_length = PAGE_SIZE - data_page_offset;
445
280b713b
EA		 446 if (do_bit17_swizzling) {
447 ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
448 shmem_page_offset,
449 user_pages[data_page_index],
450 data_page_offset,
451 page_length,
452 1);
453 } else {
454 ret = slow_shmem_copy(user_pages[data_page_index],
455 data_page_offset,
456 obj_priv->pages[shmem_page_index],
457 shmem_page_offset,
458 page_length);
459 }
eb01459f
EA		 460 if (ret)
461 goto fail_put_pages;
462
463 remain -= page_length;
464 data_ptr += page_length;
465 offset += page_length;
466 }
467
468fail_put_pages:
469 i915_gem_object_put_pages(obj);
470fail_unlock:
471 mutex_unlock(&dev->struct_mutex);
472fail_put_user_pages:
473 for (i = 0; i < pinned_pages; i++) {
474 SetPageDirty(user_pages[i]);
475 page_cache_release(user_pages[i]);
476 }
8e7d2b2c 477 drm_free_large(user_pages);
eb01459f
EA		 478
479 return ret;
480}
481
673a394b
EA		 482/**
483 * Reads data from the object referenced by handle.
484 *
485 * On error, the contents of *data are undefined.
486 */
487int
488i915_gem_pread_ioctl(struct drm_device *dev, void *data,
489 struct drm_file *file_priv)
490{
491 struct drm_i915_gem_pread *args = data;
492 struct drm_gem_object *obj;
493 struct drm_i915_gem_object *obj_priv;
673a394b
EA		 494 int ret;
495
496 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
497 if (obj == NULL)
498 return -EBADF;
499 obj_priv = obj->driver_private;
500
501 /* Bounds check source.
502 *
503 * XXX: This could use review for overflow issues...
504 */
505 if (args->offset > obj->size || args->size > obj->size ||
506 args->offset + args->size > obj->size) {
507 drm_gem_object_unreference(obj);
508 return -EINVAL;
509 }
510
280b713b 511 if (i915_gem_object_needs_bit17_swizzle(obj)) {
eb01459f 512 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
280b713b
EA		 513 } else {
514 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
515 if (ret != 0)
516 ret = i915_gem_shmem_pread_slow(dev, obj, args,
517 file_priv);
518 }
673a394b
EA		 519
520 drm_gem_object_unreference(obj);
673a394b 521
eb01459f 522 return ret;
673a394b
EA		 523}
524
0839ccb8
KP		 525/* This is the fast write path which cannot handle
526 * page faults in the source data
9b7530cc 527 */
0839ccb8
KP		 528
529static inline int
530fast_user_write(struct io_mapping *mapping,
531 loff_t page_base, int page_offset,
532 char __user *user_data,
533 int length)
9b7530cc 534{
9b7530cc 535 char *vaddr_atomic;
0839ccb8 536 unsigned long unwritten;
9b7530cc 537
0839ccb8
KP		 538 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
539 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
540 user_data, length);
541 io_mapping_unmap_atomic(vaddr_atomic);
542 if (unwritten)
543 return -EFAULT;
544 return 0;
545}
546
547/* Here's the write path which can sleep for
548 * page faults
549 */
550
551static inline int
3de09aa3
EA		 552slow_kernel_write(struct io_mapping *mapping,
553 loff_t gtt_base, int gtt_offset,
554 struct page *user_page, int user_offset,
555 int length)
0839ccb8 556{
3de09aa3 557 char *src_vaddr, *dst_vaddr;
0839ccb8
KP		 558 unsigned long unwritten;
559
3de09aa3
EA		 560 dst_vaddr = io_mapping_map_atomic_wc(mapping, gtt_base);
561 src_vaddr = kmap_atomic(user_page, KM_USER1);
562 unwritten = __copy_from_user_inatomic_nocache(dst_vaddr + gtt_offset,
563 src_vaddr + user_offset,
564 length);
565 kunmap_atomic(src_vaddr, KM_USER1);
566 io_mapping_unmap_atomic(dst_vaddr);
0839ccb8
KP		 567 if (unwritten)
568 return -EFAULT;
9b7530cc 569 return 0;
9b7530cc
LT		 570}
571
40123c1f
EA		 572static inline int
573fast_shmem_write(struct page **pages,
574 loff_t page_base, int page_offset,
575 char __user *data,
576 int length)
577{
578 char __iomem *vaddr;
d0088775 579 unsigned long unwritten;
40123c1f
EA		 580
581 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
582 if (vaddr == NULL)
583 return -ENOMEM;
d0088775 584 unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
40123c1f
EA		 585 kunmap_atomic(vaddr, KM_USER0);
586
d0088775
DA		 587 if (unwritten)
588 return -EFAULT;
40123c1f
EA		 589 return 0;
590}
591
3de09aa3
EA		 592/**
593 * This is the fast pwrite path, where we copy the data directly from the
594 * user into the GTT, uncached.
595 */
673a394b 596static int
3de09aa3
EA		 597i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
598 struct drm_i915_gem_pwrite *args,
599 struct drm_file *file_priv)
673a394b
EA		 600{
601 struct drm_i915_gem_object *obj_priv = obj->driver_private;
0839ccb8 602 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 603 ssize_t remain;
0839ccb8 604 loff_t offset, page_base;
673a394b 605 char __user *user_data;
0839ccb8
KP		 606 int page_offset, page_length;
607 int ret;
673a394b
EA		 608
609 user_data = (char __user *) (uintptr_t) args->data_ptr;
610 remain = args->size;
611 if (!access_ok(VERIFY_READ, user_data, remain))
612 return -EFAULT;
613
614
615 mutex_lock(&dev->struct_mutex);
616 ret = i915_gem_object_pin(obj, 0);
617 if (ret) {
618 mutex_unlock(&dev->struct_mutex);
619 return ret;
620 }
2ef7eeaa 621 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
673a394b
EA		 622 if (ret)
623 goto fail;
624
625 obj_priv = obj->driver_private;
626 offset = obj_priv->gtt_offset + args->offset;
673a394b
EA		 627
628 while (remain > 0) {
629 /* Operation in this page
630 *
0839ccb8
KP		 631 * page_base = page offset within aperture
632 * page_offset = offset within page
633 * page_length = bytes to copy for this page
673a394b 634 */
0839ccb8
KP		 635 page_base = (offset & ~(PAGE_SIZE-1));
636 page_offset = offset & (PAGE_SIZE-1);
637 page_length = remain;
638 if ((page_offset + remain) > PAGE_SIZE)
639 page_length = PAGE_SIZE - page_offset;
640
641 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
642 page_offset, user_data, page_length);
643
644 /* If we get a fault while copying data, then (presumably) our
3de09aa3
EA		 645 * source page isn't available. Return the error and we'll
646 * retry in the slow path.
0839ccb8 647 */
3de09aa3
EA		 648 if (ret)
649 goto fail;
673a394b 650
0839ccb8
KP		 651 remain -= page_length;
652 user_data += page_length;
653 offset += page_length;
673a394b 654 }
673a394b
EA		 655
656fail:
657 i915_gem_object_unpin(obj);
658 mutex_unlock(&dev->struct_mutex);
659
660 return ret;
661}
662
3de09aa3
EA		 663/**
664 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
665 * the memory and maps it using kmap_atomic for copying.
666 *
667 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
668 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
669 */
3043c60c 670static int
3de09aa3
EA		 671i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
672 struct drm_i915_gem_pwrite *args,
673 struct drm_file *file_priv)
673a394b 674{
3de09aa3
EA		 675 struct drm_i915_gem_object *obj_priv = obj->driver_private;
676 drm_i915_private_t *dev_priv = dev->dev_private;
677 ssize_t remain;
678 loff_t gtt_page_base, offset;
679 loff_t first_data_page, last_data_page, num_pages;
680 loff_t pinned_pages, i;
681 struct page **user_pages;
682 struct mm_struct *mm = current->mm;
683 int gtt_page_offset, data_page_offset, data_page_index, page_length;
673a394b 684 int ret;
3de09aa3
EA		 685 uint64_t data_ptr = args->data_ptr;
686
687 remain = args->size;
688
689 /* Pin the user pages containing the data. We can't fault while
690 * holding the struct mutex, and all of the pwrite implementations
691 * want to hold it while dereferencing the user data.
692 */
693 first_data_page = data_ptr / PAGE_SIZE;
694 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
695 num_pages = last_data_page - first_data_page + 1;
696
8e7d2b2c 697 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
3de09aa3
EA		 698 if (user_pages == NULL)
699 return -ENOMEM;
700
701 down_read(&mm->mmap_sem);
702 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
703 num_pages, 0, 0, user_pages, NULL);
704 up_read(&mm->mmap_sem);
705 if (pinned_pages < num_pages) {
706 ret = -EFAULT;
707 goto out_unpin_pages;
708 }
673a394b
EA		 709
710 mutex_lock(&dev->struct_mutex);
3de09aa3
EA		 711 ret = i915_gem_object_pin(obj, 0);
712 if (ret)
713 goto out_unlock;
714
715 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
716 if (ret)
717 goto out_unpin_object;
718
719 obj_priv = obj->driver_private;
720 offset = obj_priv->gtt_offset + args->offset;
721
722 while (remain > 0) {
723 /* Operation in this page
724 *
725 * gtt_page_base = page offset within aperture
726 * gtt_page_offset = offset within page in aperture
727 * data_page_index = page number in get_user_pages return
728 * data_page_offset = offset with data_page_index page.
729 * page_length = bytes to copy for this page
730 */
731 gtt_page_base = offset & PAGE_MASK;
732 gtt_page_offset = offset & ~PAGE_MASK;
733 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
734 data_page_offset = data_ptr & ~PAGE_MASK;
735
736 page_length = remain;
737 if ((gtt_page_offset + page_length) > PAGE_SIZE)
738 page_length = PAGE_SIZE - gtt_page_offset;
739 if ((data_page_offset + page_length) > PAGE_SIZE)
740 page_length = PAGE_SIZE - data_page_offset;
741
742 ret = slow_kernel_write(dev_priv->mm.gtt_mapping,
743 gtt_page_base, gtt_page_offset,
744 user_pages[data_page_index],
745 data_page_offset,
746 page_length);
747
748 /* If we get a fault while copying data, then (presumably) our
749 * source page isn't available. Return the error and we'll
750 * retry in the slow path.
751 */
752 if (ret)
753 goto out_unpin_object;
754
755 remain -= page_length;
756 offset += page_length;
757 data_ptr += page_length;
758 }
759
760out_unpin_object:
761 i915_gem_object_unpin(obj);
762out_unlock:
763 mutex_unlock(&dev->struct_mutex);
764out_unpin_pages:
765 for (i = 0; i < pinned_pages; i++)
766 page_cache_release(user_pages[i]);
8e7d2b2c 767 drm_free_large(user_pages);
3de09aa3
EA		 768
769 return ret;
770}
771
40123c1f
EA		 772/**
773 * This is the fast shmem pwrite path, which attempts to directly
774 * copy_from_user into the kmapped pages backing the object.
775 */
3043c60c 776static int
40123c1f
EA		 777i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
778 struct drm_i915_gem_pwrite *args,
779 struct drm_file *file_priv)
673a394b 780{
40123c1f
EA		 781 struct drm_i915_gem_object *obj_priv = obj->driver_private;
782 ssize_t remain;
783 loff_t offset, page_base;
784 char __user *user_data;
785 int page_offset, page_length;
673a394b 786 int ret;
40123c1f
EA		 787
788 user_data = (char __user *) (uintptr_t) args->data_ptr;
789 remain = args->size;
673a394b
EA		 790
791 mutex_lock(&dev->struct_mutex);
792
40123c1f
EA		 793 ret = i915_gem_object_get_pages(obj);
794 if (ret != 0)
795 goto fail_unlock;
673a394b 796
e47c68e9 797 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
40123c1f
EA		 798 if (ret != 0)
799 goto fail_put_pages;
800
801 obj_priv = obj->driver_private;
802 offset = args->offset;
803 obj_priv->dirty = 1;
804
805 while (remain > 0) {
806 /* Operation in this page
807 *
808 * page_base = page offset within aperture
809 * page_offset = offset within page
810 * page_length = bytes to copy for this page
811 */
812 page_base = (offset & ~(PAGE_SIZE-1));
813 page_offset = offset & (PAGE_SIZE-1);
814 page_length = remain;
815 if ((page_offset + remain) > PAGE_SIZE)
816 page_length = PAGE_SIZE - page_offset;
817
818 ret = fast_shmem_write(obj_priv->pages,
819 page_base, page_offset,
820 user_data, page_length);
821 if (ret)
822 goto fail_put_pages;
823
824 remain -= page_length;
825 user_data += page_length;
826 offset += page_length;
827 }
828
829fail_put_pages:
830 i915_gem_object_put_pages(obj);
831fail_unlock:
832 mutex_unlock(&dev->struct_mutex);
833
834 return ret;
835}
836
837/**
838 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
839 * the memory and maps it using kmap_atomic for copying.
840 *
841 * This avoids taking mmap_sem for faulting on the user's address while the
842 * struct_mutex is held.
843 */
844static int
845i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
846 struct drm_i915_gem_pwrite *args,
847 struct drm_file *file_priv)
848{
849 struct drm_i915_gem_object *obj_priv = obj->driver_private;
850 struct mm_struct *mm = current->mm;
851 struct page **user_pages;
852 ssize_t remain;
853 loff_t offset, pinned_pages, i;
854 loff_t first_data_page, last_data_page, num_pages;
855 int shmem_page_index, shmem_page_offset;
856 int data_page_index, data_page_offset;
857 int page_length;
858 int ret;
859 uint64_t data_ptr = args->data_ptr;
280b713b 860 int do_bit17_swizzling;
40123c1f
EA		 861
862 remain = args->size;
863
864 /* Pin the user pages containing the data. We can't fault while
865 * holding the struct mutex, and all of the pwrite implementations
866 * want to hold it while dereferencing the user data.
867 */
868 first_data_page = data_ptr / PAGE_SIZE;
869 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
870 num_pages = last_data_page - first_data_page + 1;
871
8e7d2b2c 872 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
40123c1f
EA		 873 if (user_pages == NULL)
874 return -ENOMEM;
875
876 down_read(&mm->mmap_sem);
877 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
878 num_pages, 0, 0, user_pages, NULL);
879 up_read(&mm->mmap_sem);
880 if (pinned_pages < num_pages) {
881 ret = -EFAULT;
882 goto fail_put_user_pages;
673a394b
EA		 883 }
884
280b713b
EA		 885 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
886
40123c1f
EA		 887 mutex_lock(&dev->struct_mutex);
888
07f73f69
CW		 889 ret = i915_gem_object_get_pages_or_evict(obj);
890 if (ret)
40123c1f
EA		 891 goto fail_unlock;
892
893 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
894 if (ret != 0)
895 goto fail_put_pages;
896
897 obj_priv = obj->driver_private;
673a394b 898 offset = args->offset;
40123c1f 899 obj_priv->dirty = 1;
673a394b 900
40123c1f
EA		 901 while (remain > 0) {
902 /* Operation in this page
903 *
904 * shmem_page_index = page number within shmem file
905 * shmem_page_offset = offset within page in shmem file
906 * data_page_index = page number in get_user_pages return
907 * data_page_offset = offset with data_page_index page.
908 * page_length = bytes to copy for this page
909 */
910 shmem_page_index = offset / PAGE_SIZE;
911 shmem_page_offset = offset & ~PAGE_MASK;
912 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
913 data_page_offset = data_ptr & ~PAGE_MASK;
914
915 page_length = remain;
916 if ((shmem_page_offset + page_length) > PAGE_SIZE)
917 page_length = PAGE_SIZE - shmem_page_offset;
918 if ((data_page_offset + page_length) > PAGE_SIZE)
919 page_length = PAGE_SIZE - data_page_offset;
920
280b713b
EA		 921 if (do_bit17_swizzling) {
922 ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
923 shmem_page_offset,
924 user_pages[data_page_index],
925 data_page_offset,
926 page_length,
927 0);
928 } else {
929 ret = slow_shmem_copy(obj_priv->pages[shmem_page_index],
930 shmem_page_offset,
931 user_pages[data_page_index],
932 data_page_offset,
933 page_length);
934 }
40123c1f
EA		 935 if (ret)
936 goto fail_put_pages;
937
938 remain -= page_length;
939 data_ptr += page_length;
940 offset += page_length;
673a394b
EA		 941 }
942
40123c1f
EA		 943fail_put_pages:
944 i915_gem_object_put_pages(obj);
945fail_unlock:
673a394b 946 mutex_unlock(&dev->struct_mutex);
40123c1f
EA		 947fail_put_user_pages:
948 for (i = 0; i < pinned_pages; i++)
949 page_cache_release(user_pages[i]);
8e7d2b2c 950 drm_free_large(user_pages);
673a394b 951
40123c1f 952 return ret;
673a394b
EA		 953}
954
955/**
956 * Writes data to the object referenced by handle.
957 *
958 * On error, the contents of the buffer that were to be modified are undefined.
959 */
960int
961i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
962 struct drm_file *file_priv)
963{
964 struct drm_i915_gem_pwrite *args = data;
965 struct drm_gem_object *obj;
966 struct drm_i915_gem_object *obj_priv;
967 int ret = 0;
968
969 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
970 if (obj == NULL)
971 return -EBADF;
972 obj_priv = obj->driver_private;
973
974 /* Bounds check destination.
975 *
976 * XXX: This could use review for overflow issues...
977 */
978 if (args->offset > obj->size || args->size > obj->size ||
979 args->offset + args->size > obj->size) {
980 drm_gem_object_unreference(obj);
981 return -EINVAL;
982 }
983
984 /* We can only do the GTT pwrite on untiled buffers, as otherwise
985 * it would end up going through the fenced access, and we'll get
986 * different detiling behavior between reading and writing.
987 * pread/pwrite currently are reading and writing from the CPU
988 * perspective, requiring manual detiling by the client.
989 */
71acb5eb
DA		 990 if (obj_priv->phys_obj)
991 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
992 else if (obj_priv->tiling_mode == I915_TILING_NONE &&
3de09aa3
EA		 993 dev->gtt_total != 0) {
994 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
995 if (ret == -EFAULT) {
996 ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
997 file_priv);
998 }
280b713b
EA		 999 } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
1000 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
40123c1f
EA		 1001 } else {
1002 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
1003 if (ret == -EFAULT) {
1004 ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
1005 file_priv);
1006 }
1007 }
673a394b
EA		 1008
1009#if WATCH_PWRITE
1010 if (ret)
1011 DRM_INFO("pwrite failed %d\n", ret);
1012#endif
1013
1014 drm_gem_object_unreference(obj);
1015
1016 return ret;
1017}
1018
1019/**
2ef7eeaa
EA		 1020 * Called when user space prepares to use an object with the CPU, either
1021 * through the mmap ioctl's mapping or a GTT mapping.
673a394b
EA		 1022 */
1023int
1024i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1025 struct drm_file *file_priv)
1026{
a09ba7fa 1027 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b
EA		 1028 struct drm_i915_gem_set_domain *args = data;
1029 struct drm_gem_object *obj;
652c393a 1030 struct drm_i915_gem_object *obj_priv;
2ef7eeaa
EA		 1031 uint32_t read_domains = args->read_domains;
1032 uint32_t write_domain = args->write_domain;
673a394b
EA		 1033 int ret;
1034
1035 if (!(dev->driver->driver_features & DRIVER_GEM))
1036 return -ENODEV;
1037
2ef7eeaa 1038 /* Only handle setting domains to types used by the CPU. */
21d509e3 1039 if (write_domain & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1040 return -EINVAL;
1041
21d509e3 1042 if (read_domains & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1043 return -EINVAL;
1044
1045 /* Having something in the write domain implies it's in the read
1046 * domain, and only that read domain. Enforce that in the request.
1047 */
1048 if (write_domain != 0 && read_domains != write_domain)
1049 return -EINVAL;
1050
673a394b
EA		 1051 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1052 if (obj == NULL)
1053 return -EBADF;
652c393a 1054 obj_priv = obj->driver_private;
673a394b
EA		 1055
1056 mutex_lock(&dev->struct_mutex);
652c393a
JB		 1057
1058 intel_mark_busy(dev, obj);
1059
673a394b 1060#if WATCH_BUF
cfd43c02 1061 DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
2ef7eeaa 1062 obj, obj->size, read_domains, write_domain);
673a394b 1063#endif
2ef7eeaa
EA		 1064 if (read_domains & I915_GEM_DOMAIN_GTT) {
1065 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
02354392 1066
a09ba7fa
EA		 1067 /* Update the LRU on the fence for the CPU access that's
1068 * about to occur.
1069 */
1070 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1071 list_move_tail(&obj_priv->fence_list,
1072 &dev_priv->mm.fence_list);
1073 }
1074
02354392
EA		 1075 /* Silently promote "you're not bound, there was nothing to do"
1076 * to success, since the client was just asking us to
1077 * make sure everything was done.
1078 */
1079 if (ret == -EINVAL)
1080 ret = 0;
2ef7eeaa 1081 } else {
e47c68e9 1082 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
2ef7eeaa
EA		 1083 }
1084
673a394b
EA		 1085 drm_gem_object_unreference(obj);
1086 mutex_unlock(&dev->struct_mutex);
1087 return ret;
1088}
1089
1090/**
1091 * Called when user space has done writes to this buffer
1092 */
1093int
1094i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1095 struct drm_file *file_priv)
1096{
1097 struct drm_i915_gem_sw_finish *args = data;
1098 struct drm_gem_object *obj;
1099 struct drm_i915_gem_object *obj_priv;
1100 int ret = 0;
1101
1102 if (!(dev->driver->driver_features & DRIVER_GEM))
1103 return -ENODEV;
1104
1105 mutex_lock(&dev->struct_mutex);
1106 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1107 if (obj == NULL) {
1108 mutex_unlock(&dev->struct_mutex);
1109 return -EBADF;
1110 }
1111
1112#if WATCH_BUF
cfd43c02 1113 DRM_INFO("%s: sw_finish %d (%p %zd)\n",
673a394b
EA		 1114 __func__, args->handle, obj, obj->size);
1115#endif
1116 obj_priv = obj->driver_private;
1117
1118 /* Pinned buffers may be scanout, so flush the cache */
e47c68e9
EA		 1119 if (obj_priv->pin_count)
1120 i915_gem_object_flush_cpu_write_domain(obj);
1121
673a394b
EA		 1122 drm_gem_object_unreference(obj);
1123 mutex_unlock(&dev->struct_mutex);
1124 return ret;
1125}
1126
1127/**
1128 * Maps the contents of an object, returning the address it is mapped
1129 * into.
1130 *
1131 * While the mapping holds a reference on the contents of the object, it doesn't
1132 * imply a ref on the object itself.
1133 */
1134int
1135i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1136 struct drm_file *file_priv)
1137{
1138 struct drm_i915_gem_mmap *args = data;
1139 struct drm_gem_object *obj;
1140 loff_t offset;
1141 unsigned long addr;
1142
1143 if (!(dev->driver->driver_features & DRIVER_GEM))
1144 return -ENODEV;
1145
1146 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1147 if (obj == NULL)
1148 return -EBADF;
1149
1150 offset = args->offset;
1151
1152 down_write(&current->mm->mmap_sem);
1153 addr = do_mmap(obj->filp, 0, args->size,
1154 PROT_READ | PROT_WRITE, MAP_SHARED,
1155 args->offset);
1156 up_write(&current->mm->mmap_sem);
1157 mutex_lock(&dev->struct_mutex);
1158 drm_gem_object_unreference(obj);
1159 mutex_unlock(&dev->struct_mutex);
1160 if (IS_ERR((void *)addr))
1161 return addr;
1162
1163 args->addr_ptr = (uint64_t) addr;
1164
1165 return 0;
1166}
1167
de151cf6
JB		 1168/**
1169 * i915_gem_fault - fault a page into the GTT
1170 * vma: VMA in question
1171 * vmf: fault info
1172 *
1173 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1174 * from userspace. The fault handler takes care of binding the object to
1175 * the GTT (if needed), allocating and programming a fence register (again,
1176 * only if needed based on whether the old reg is still valid or the object
1177 * is tiled) and inserting a new PTE into the faulting process.
1178 *
1179 * Note that the faulting process may involve evicting existing objects
1180 * from the GTT and/or fence registers to make room. So performance may
1181 * suffer if the GTT working set is large or there are few fence registers
1182 * left.
1183 */
1184int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1185{
1186 struct drm_gem_object *obj = vma->vm_private_data;
1187 struct drm_device *dev = obj->dev;
1188 struct drm_i915_private *dev_priv = dev->dev_private;
1189 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1190 pgoff_t page_offset;
1191 unsigned long pfn;
1192 int ret = 0;
0f973f27 1193 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
de151cf6
JB		 1194
1195 /* We don't use vmf->pgoff since that has the fake offset */
1196 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1197 PAGE_SHIFT;
1198
1199 /* Now bind it into the GTT if needed */
1200 mutex_lock(&dev->struct_mutex);
1201 if (!obj_priv->gtt_space) {
e67b8ce1 1202 ret = i915_gem_object_bind_to_gtt(obj, 0);
c715089f
CW		 1203 if (ret)
1204 goto unlock;
07f4f3e8 1205
14b60391 1206 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
07f4f3e8
KH		 1207
1208 ret = i915_gem_object_set_to_gtt_domain(obj, write);
c715089f
CW		 1209 if (ret)
1210 goto unlock;
de151cf6
JB		 1211 }
1212
1213 /* Need a new fence register? */
a09ba7fa 1214 if (obj_priv->tiling_mode != I915_TILING_NONE) {
8c4b8c3f 1215 ret = i915_gem_object_get_fence_reg(obj);
c715089f
CW		 1216 if (ret)
1217 goto unlock;
d9ddcb96 1218 }
de151cf6
JB		 1219
1220 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1221 page_offset;
1222
1223 /* Finally, remap it using the new GTT offset */
1224 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
c715089f 1225unlock:
de151cf6
JB		 1226 mutex_unlock(&dev->struct_mutex);
1227
1228 switch (ret) {
c715089f
CW		 1229 case 0:
1230 case -ERESTARTSYS:
1231 return VM_FAULT_NOPAGE;
de151cf6
JB		 1232 case -ENOMEM:
1233 case -EAGAIN:
1234 return VM_FAULT_OOM;
de151cf6 1235 default:
c715089f 1236 return VM_FAULT_SIGBUS;
de151cf6
JB		 1237 }
1238}
1239
1240/**
1241 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1242 * @obj: obj in question
1243 *
1244 * GEM memory mapping works by handing back to userspace a fake mmap offset
1245 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1246 * up the object based on the offset and sets up the various memory mapping
1247 * structures.
1248 *
1249 * This routine allocates and attaches a fake offset for @obj.
1250 */
1251static int
1252i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1253{
1254 struct drm_device *dev = obj->dev;
1255 struct drm_gem_mm *mm = dev->mm_private;
1256 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1257 struct drm_map_list *list;
f77d390c 1258 struct drm_local_map *map;
de151cf6
JB		 1259 int ret = 0;
1260
1261 /* Set the object up for mmap'ing */
1262 list = &obj->map_list;
9a298b2a 1263 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
de151cf6
JB		 1264 if (!list->map)
1265 return -ENOMEM;
1266
1267 map = list->map;
1268 map->type = _DRM_GEM;
1269 map->size = obj->size;
1270 map->handle = obj;
1271
1272 /* Get a DRM GEM mmap offset allocated... */
1273 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1274 obj->size / PAGE_SIZE, 0, 0);
1275 if (!list->file_offset_node) {
1276 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1277 ret = -ENOMEM;
1278 goto out_free_list;
1279 }
1280
1281 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1282 obj->size / PAGE_SIZE, 0);
1283 if (!list->file_offset_node) {
1284 ret = -ENOMEM;
1285 goto out_free_list;
1286 }
1287
1288 list->hash.key = list->file_offset_node->start;
1289 if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1290 DRM_ERROR("failed to add to map hash\n");
5618ca6a 1291 ret = -ENOMEM;
de151cf6
JB		 1292 goto out_free_mm;
1293 }
1294
1295 /* By now we should be all set, any drm_mmap request on the offset
1296 * below will get to our mmap & fault handler */
1297 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1298
1299 return 0;
1300
1301out_free_mm:
1302 drm_mm_put_block(list->file_offset_node);
1303out_free_list:
9a298b2a 1304 kfree(list->map);
de151cf6
JB		 1305
1306 return ret;
1307}
1308
901782b2
CW		 1309/**
1310 * i915_gem_release_mmap - remove physical page mappings
1311 * @obj: obj in question
1312 *
af901ca1 1313 * Preserve the reservation of the mmapping with the DRM core code, but
901782b2
CW		 1314 * relinquish ownership of the pages back to the system.
1315 *
1316 * It is vital that we remove the page mapping if we have mapped a tiled
1317 * object through the GTT and then lose the fence register due to
1318 * resource pressure. Similarly if the object has been moved out of the
1319 * aperture, than pages mapped into userspace must be revoked. Removing the
1320 * mapping will then trigger a page fault on the next user access, allowing
1321 * fixup by i915_gem_fault().
1322 */
d05ca301 1323void
901782b2
CW		 1324i915_gem_release_mmap(struct drm_gem_object *obj)
1325{
1326 struct drm_device *dev = obj->dev;
1327 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1328
1329 if (dev->dev_mapping)
1330 unmap_mapping_range(dev->dev_mapping,
1331 obj_priv->mmap_offset, obj->size, 1);
1332}
1333
ab00b3e5
JB		 1334static void
1335i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1336{
1337 struct drm_device *dev = obj->dev;
1338 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1339 struct drm_gem_mm *mm = dev->mm_private;
1340 struct drm_map_list *list;
1341
1342 list = &obj->map_list;
1343 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1344
1345 if (list->file_offset_node) {
1346 drm_mm_put_block(list->file_offset_node);
1347 list->file_offset_node = NULL;
1348 }
1349
1350 if (list->map) {
9a298b2a 1351 kfree(list->map);
ab00b3e5
JB		 1352 list->map = NULL;
1353 }
1354
1355 obj_priv->mmap_offset = 0;
1356}
1357
de151cf6
JB		 1358/**
1359 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1360 * @obj: object to check
1361 *
1362 * Return the required GTT alignment for an object, taking into account
1363 * potential fence register mapping if needed.
1364 */
1365static uint32_t
1366i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1367{
1368 struct drm_device *dev = obj->dev;
1369 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1370 int start, i;
1371
1372 /*
1373 * Minimum alignment is 4k (GTT page size), but might be greater
1374 * if a fence register is needed for the object.
1375 */
1376 if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1377 return 4096;
1378
1379 /*
1380 * Previous chips need to be aligned to the size of the smallest
1381 * fence register that can contain the object.
1382 */
1383 if (IS_I9XX(dev))
1384 start = 1024*1024;
1385 else
1386 start = 512*1024;
1387
1388 for (i = start; i < obj->size; i <<= 1)
1389 ;
1390
1391 return i;
1392}
1393
1394/**
1395 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1396 * @dev: DRM device
1397 * @data: GTT mapping ioctl data
1398 * @file_priv: GEM object info
1399 *
1400 * Simply returns the fake offset to userspace so it can mmap it.
1401 * The mmap call will end up in drm_gem_mmap(), which will set things
1402 * up so we can get faults in the handler above.
1403 *
1404 * The fault handler will take care of binding the object into the GTT
1405 * (since it may have been evicted to make room for something), allocating
1406 * a fence register, and mapping the appropriate aperture address into
1407 * userspace.
1408 */
1409int
1410i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1411 struct drm_file *file_priv)
1412{
1413 struct drm_i915_gem_mmap_gtt *args = data;
1414 struct drm_i915_private *dev_priv = dev->dev_private;
1415 struct drm_gem_object *obj;
1416 struct drm_i915_gem_object *obj_priv;
1417 int ret;
1418
1419 if (!(dev->driver->driver_features & DRIVER_GEM))
1420 return -ENODEV;
1421
1422 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1423 if (obj == NULL)
1424 return -EBADF;
1425
1426 mutex_lock(&dev->struct_mutex);
1427
1428 obj_priv = obj->driver_private;
1429
ab18282d
CW		 1430 if (obj_priv->madv != I915_MADV_WILLNEED) {
1431 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1432 drm_gem_object_unreference(obj);
1433 mutex_unlock(&dev->struct_mutex);
1434 return -EINVAL;
1435 }
1436
1437
de151cf6
JB		 1438 if (!obj_priv->mmap_offset) {
1439 ret = i915_gem_create_mmap_offset(obj);
13af1062
CW		 1440 if (ret) {
1441 drm_gem_object_unreference(obj);
1442 mutex_unlock(&dev->struct_mutex);
de151cf6 1443 return ret;
13af1062 1444 }
de151cf6
JB		 1445 }
1446
1447 args->offset = obj_priv->mmap_offset;
1448
de151cf6
JB		 1449 /*
1450 * Pull it into the GTT so that we have a page list (makes the
1451 * initial fault faster and any subsequent flushing possible).
1452 */
1453 if (!obj_priv->agp_mem) {
e67b8ce1 1454 ret = i915_gem_object_bind_to_gtt(obj, 0);
de151cf6
JB		 1455 if (ret) {
1456 drm_gem_object_unreference(obj);
1457 mutex_unlock(&dev->struct_mutex);
1458 return ret;
1459 }
14b60391 1460 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
de151cf6
JB		 1461 }
1462
1463 drm_gem_object_unreference(obj);
1464 mutex_unlock(&dev->struct_mutex);
1465
1466 return 0;
1467}
1468
6911a9b8 1469void
856fa198 1470i915_gem_object_put_pages(struct drm_gem_object *obj)
673a394b
EA		 1471{
1472 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1473 int page_count = obj->size / PAGE_SIZE;
1474 int i;
1475
856fa198 1476 BUG_ON(obj_priv->pages_refcount == 0);
bb6baf76 1477 BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
673a394b 1478
856fa198
EA		 1479 if (--obj_priv->pages_refcount != 0)
1480 return;
673a394b 1481
280b713b
EA		 1482 if (obj_priv->tiling_mode != I915_TILING_NONE)
1483 i915_gem_object_save_bit_17_swizzle(obj);
1484
3ef94daa 1485 if (obj_priv->madv == I915_MADV_DONTNEED)
13a05fd9 1486 obj_priv->dirty = 0;
3ef94daa
CW		 1487
1488 for (i = 0; i < page_count; i++) {
1489 if (obj_priv->pages[i] == NULL)
1490 break;
1491
1492 if (obj_priv->dirty)
1493 set_page_dirty(obj_priv->pages[i]);
1494
1495 if (obj_priv->madv == I915_MADV_WILLNEED)
856fa198 1496 mark_page_accessed(obj_priv->pages[i]);
3ef94daa
CW		 1497
1498 page_cache_release(obj_priv->pages[i]);
1499 }
673a394b
EA		 1500 obj_priv->dirty = 0;
1501
8e7d2b2c 1502 drm_free_large(obj_priv->pages);
856fa198 1503 obj_priv->pages = NULL;
673a394b
EA		 1504}
1505
1506static void
ce44b0ea 1507i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno)
673a394b
EA		 1508{
1509 struct drm_device *dev = obj->dev;
1510 drm_i915_private_t *dev_priv = dev->dev_private;
1511 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1512
1513 /* Add a reference if we're newly entering the active list. */
1514 if (!obj_priv->active) {
1515 drm_gem_object_reference(obj);
1516 obj_priv->active = 1;
1517 }
1518 /* Move from whatever list we were on to the tail of execution. */
5e118f41 1519 spin_lock(&dev_priv->mm.active_list_lock);
673a394b
EA		 1520 list_move_tail(&obj_priv->list,
1521 &dev_priv->mm.active_list);
5e118f41 1522 spin_unlock(&dev_priv->mm.active_list_lock);
ce44b0ea 1523 obj_priv->last_rendering_seqno = seqno;
673a394b
EA		 1524}
1525
ce44b0ea
EA		 1526static void
1527i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1528{
1529 struct drm_device *dev = obj->dev;
1530 drm_i915_private_t *dev_priv = dev->dev_private;
1531 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1532
1533 BUG_ON(!obj_priv->active);
1534 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1535 obj_priv->last_rendering_seqno = 0;
1536}
673a394b 1537
963b4836
CW		 1538/* Immediately discard the backing storage */
1539static void
1540i915_gem_object_truncate(struct drm_gem_object *obj)
1541{
bb6baf76
CW		 1542 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1543 struct inode *inode;
963b4836 1544
bb6baf76
CW		 1545 inode = obj->filp->f_path.dentry->d_inode;
1546 if (inode->i_op->truncate)
1547 inode->i_op->truncate (inode);
1548
1549 obj_priv->madv = __I915_MADV_PURGED;
963b4836
CW		 1550}
1551
1552static inline int
1553i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1554{
1555 return obj_priv->madv == I915_MADV_DONTNEED;
1556}
1557
673a394b
EA		 1558static void
1559i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1560{
1561 struct drm_device *dev = obj->dev;
1562 drm_i915_private_t *dev_priv = dev->dev_private;
1563 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1564
1565 i915_verify_inactive(dev, __FILE__, __LINE__);
1566 if (obj_priv->pin_count != 0)
1567 list_del_init(&obj_priv->list);
1568 else
1569 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1570
ce44b0ea 1571 obj_priv->last_rendering_seqno = 0;
673a394b
EA		 1572 if (obj_priv->active) {
1573 obj_priv->active = 0;
1574 drm_gem_object_unreference(obj);
1575 }
1576 i915_verify_inactive(dev, __FILE__, __LINE__);
1577}
1578
1579/**
1580 * Creates a new sequence number, emitting a write of it to the status page
1581 * plus an interrupt, which will trigger i915_user_interrupt_handler.
1582 *
1583 * Must be called with struct_lock held.
1584 *
1585 * Returned sequence numbers are nonzero on success.
1586 */
5a5a0c64 1587uint32_t
b962442e
EA		 1588i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
1589 uint32_t flush_domains)
673a394b
EA		 1590{
1591 drm_i915_private_t *dev_priv = dev->dev_private;
b962442e 1592 struct drm_i915_file_private *i915_file_priv = NULL;
673a394b
EA		 1593 struct drm_i915_gem_request *request;
1594 uint32_t seqno;
1595 int was_empty;
1596 RING_LOCALS;
1597
b962442e
EA		 1598 if (file_priv != NULL)
1599 i915_file_priv = file_priv->driver_priv;
1600
9a298b2a 1601 request = kzalloc(sizeof(*request), GFP_KERNEL);
673a394b
EA		 1602 if (request == NULL)
1603 return 0;
1604
1605 /* Grab the seqno we're going to make this request be, and bump the
1606 * next (skipping 0 so it can be the reserved no-seqno value).
1607 */
1608 seqno = dev_priv->mm.next_gem_seqno;
1609 dev_priv->mm.next_gem_seqno++;
1610 if (dev_priv->mm.next_gem_seqno == 0)
1611 dev_priv->mm.next_gem_seqno++;
1612
1613 BEGIN_LP_RING(4);
1614 OUT_RING(MI_STORE_DWORD_INDEX);
1615 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1616 OUT_RING(seqno);
1617
1618 OUT_RING(MI_USER_INTERRUPT);
1619 ADVANCE_LP_RING();
1620
44d98a61 1621 DRM_DEBUG_DRIVER("%d\n", seqno);
673a394b
EA		 1622
1623 request->seqno = seqno;
1624 request->emitted_jiffies = jiffies;
673a394b
EA		 1625 was_empty = list_empty(&dev_priv->mm.request_list);
1626 list_add_tail(&request->list, &dev_priv->mm.request_list);
b962442e
EA		 1627 if (i915_file_priv) {
1628 list_add_tail(&request->client_list,
1629 &i915_file_priv->mm.request_list);
1630 } else {
1631 INIT_LIST_HEAD(&request->client_list);
1632 }
673a394b 1633
ce44b0ea
EA		 1634 /* Associate any objects on the flushing list matching the write
1635 * domain we're flushing with our flush.
1636 */
1637 if (flush_domains != 0) {
1638 struct drm_i915_gem_object *obj_priv, *next;
1639
1640 list_for_each_entry_safe(obj_priv, next,
1641 &dev_priv->mm.flushing_list, list) {
1642 struct drm_gem_object *obj = obj_priv->obj;
1643
1644 if ((obj->write_domain & flush_domains) ==
1645 obj->write_domain) {
1c5d22f7
CW		 1646 uint32_t old_write_domain = obj->write_domain;
1647
ce44b0ea
EA		 1648 obj->write_domain = 0;
1649 i915_gem_object_move_to_active(obj, seqno);
1c5d22f7
CW		 1650
1651 trace_i915_gem_object_change_domain(obj,
1652 obj->read_domains,
1653 old_write_domain);
ce44b0ea
EA		 1654 }
1655 }
1656
1657 }
1658
f65d9421
BG		 1659 if (!dev_priv->mm.suspended) {
1660 mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
1661 if (was_empty)
1662 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1663 }
673a394b
EA		 1664 return seqno;
1665}
1666
1667/**
1668 * Command execution barrier
1669 *
1670 * Ensures that all commands in the ring are finished
1671 * before signalling the CPU
1672 */
3043c60c 1673static uint32_t
673a394b
EA		 1674i915_retire_commands(struct drm_device *dev)
1675{
1676 drm_i915_private_t *dev_priv = dev->dev_private;
1677 uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1678 uint32_t flush_domains = 0;
1679 RING_LOCALS;
1680
1681 /* The sampler always gets flushed on i965 (sigh) */
1682 if (IS_I965G(dev))
1683 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1684 BEGIN_LP_RING(2);
1685 OUT_RING(cmd);
1686 OUT_RING(0); /* noop */
1687 ADVANCE_LP_RING();
1688 return flush_domains;
1689}
1690
1691/**
1692 * Moves buffers associated only with the given active seqno from the active
1693 * to inactive list, potentially freeing them.
1694 */
1695static void
1696i915_gem_retire_request(struct drm_device *dev,
1697 struct drm_i915_gem_request *request)
1698{
1699 drm_i915_private_t *dev_priv = dev->dev_private;
1700
1c5d22f7
CW		 1701 trace_i915_gem_request_retire(dev, request->seqno);
1702
673a394b
EA		 1703 /* Move any buffers on the active list that are no longer referenced
1704 * by the ringbuffer to the flushing/inactive lists as appropriate.
1705 */
5e118f41 1706 spin_lock(&dev_priv->mm.active_list_lock);
673a394b
EA		 1707 while (!list_empty(&dev_priv->mm.active_list)) {
1708 struct drm_gem_object *obj;
1709 struct drm_i915_gem_object *obj_priv;
1710
1711 obj_priv = list_first_entry(&dev_priv->mm.active_list,
1712 struct drm_i915_gem_object,
1713 list);
1714 obj = obj_priv->obj;
1715
1716 /* If the seqno being retired doesn't match the oldest in the
1717 * list, then the oldest in the list must still be newer than
1718 * this seqno.
1719 */
1720 if (obj_priv->last_rendering_seqno != request->seqno)
5e118f41 1721 goto out;
de151cf6 1722
673a394b
EA		 1723#if WATCH_LRU
1724 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1725 __func__, request->seqno, obj);
1726#endif
1727
ce44b0ea
EA		 1728 if (obj->write_domain != 0)
1729 i915_gem_object_move_to_flushing(obj);
68c84342
SL		 1730 else {
1731 /* Take a reference on the object so it won't be
1732 * freed while the spinlock is held. The list
1733 * protection for this spinlock is safe when breaking
1734 * the lock like this since the next thing we do
1735 * is just get the head of the list again.
1736 */
1737 drm_gem_object_reference(obj);
673a394b 1738 i915_gem_object_move_to_inactive(obj);
68c84342
SL		 1739 spin_unlock(&dev_priv->mm.active_list_lock);
1740 drm_gem_object_unreference(obj);
1741 spin_lock(&dev_priv->mm.active_list_lock);
1742 }
673a394b 1743 }
5e118f41
CW		 1744out:
1745 spin_unlock(&dev_priv->mm.active_list_lock);
673a394b
EA		 1746}
1747
1748/**
1749 * Returns true if seq1 is later than seq2.
1750 */
22be1724 1751bool
673a394b
EA		 1752i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1753{
1754 return (int32_t)(seq1 - seq2) >= 0;
1755}
1756
1757uint32_t
1758i915_get_gem_seqno(struct drm_device *dev)
1759{
1760 drm_i915_private_t *dev_priv = dev->dev_private;
1761
1762 return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
1763}
1764
1765/**
1766 * This function clears the request list as sequence numbers are passed.
1767 */
1768void
1769i915_gem_retire_requests(struct drm_device *dev)
1770{
1771 drm_i915_private_t *dev_priv = dev->dev_private;
1772 uint32_t seqno;
1773
9d34e5db 1774 if (!dev_priv->hw_status_page || list_empty(&dev_priv->mm.request_list))
6c0594a3
KW		 1775 return;
1776
673a394b
EA		 1777 seqno = i915_get_gem_seqno(dev);
1778
1779 while (!list_empty(&dev_priv->mm.request_list)) {
1780 struct drm_i915_gem_request *request;
1781 uint32_t retiring_seqno;
1782
1783 request = list_first_entry(&dev_priv->mm.request_list,
1784 struct drm_i915_gem_request,
1785 list);
1786 retiring_seqno = request->seqno;
1787
1788 if (i915_seqno_passed(seqno, retiring_seqno) ||
ba1234d1 1789 atomic_read(&dev_priv->mm.wedged)) {
673a394b
EA		 1790 i915_gem_retire_request(dev, request);
1791
1792 list_del(&request->list);
b962442e 1793 list_del(&request->client_list);
9a298b2a 1794 kfree(request);
673a394b
EA		 1795 } else
1796 break;
1797 }
9d34e5db
CW		 1798
1799 if (unlikely (dev_priv->trace_irq_seqno &&
1800 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1801 i915_user_irq_put(dev);
1802 dev_priv->trace_irq_seqno = 0;
1803 }
673a394b
EA		 1804}
1805
1806void
1807i915_gem_retire_work_handler(struct work_struct *work)
1808{
1809 drm_i915_private_t *dev_priv;
1810 struct drm_device *dev;
1811
1812 dev_priv = container_of(work, drm_i915_private_t,
1813 mm.retire_work.work);
1814 dev = dev_priv->dev;
1815
1816 mutex_lock(&dev->struct_mutex);
1817 i915_gem_retire_requests(dev);
6dbe2772
KP		 1818 if (!dev_priv->mm.suspended &&
1819 !list_empty(&dev_priv->mm.request_list))
9c9fe1f8 1820 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
673a394b
EA		 1821 mutex_unlock(&dev->struct_mutex);
1822}
1823
5a5a0c64 1824int
48764bf4 1825i915_do_wait_request(struct drm_device *dev, uint32_t seqno, int interruptible)
673a394b
EA		 1826{
1827 drm_i915_private_t *dev_priv = dev->dev_private;
802c7eb6 1828 u32 ier;
673a394b
EA		 1829 int ret = 0;
1830
1831 BUG_ON(seqno == 0);
1832
ba1234d1 1833 if (atomic_read(&dev_priv->mm.wedged))
ffed1d09
BG		 1834 return -EIO;
1835
673a394b 1836 if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
f2b115e6 1837 if (IS_IRONLAKE(dev))
036a4a7d
ZW		 1838 ier = I915_READ(DEIER) | I915_READ(GTIER);
1839 else
1840 ier = I915_READ(IER);
802c7eb6
JB		 1841 if (!ier) {
1842 DRM_ERROR("something (likely vbetool) disabled "
1843 "interrupts, re-enabling\n");
1844 i915_driver_irq_preinstall(dev);
1845 i915_driver_irq_postinstall(dev);
1846 }
1847
1c5d22f7
CW		 1848 trace_i915_gem_request_wait_begin(dev, seqno);
1849
673a394b
EA		 1850 dev_priv->mm.waiting_gem_seqno = seqno;
1851 i915_user_irq_get(dev);
48764bf4
DV		 1852 if (interruptible)
1853 ret = wait_event_interruptible(dev_priv->irq_queue,
1854 i915_seqno_passed(i915_get_gem_seqno(dev), seqno) ||
1855 atomic_read(&dev_priv->mm.wedged));
1856 else
1857 wait_event(dev_priv->irq_queue,
1858 i915_seqno_passed(i915_get_gem_seqno(dev), seqno) ||
1859 atomic_read(&dev_priv->mm.wedged));
1860
673a394b
EA		 1861 i915_user_irq_put(dev);
1862 dev_priv->mm.waiting_gem_seqno = 0;
1c5d22f7
CW		 1863
1864 trace_i915_gem_request_wait_end(dev, seqno);
673a394b 1865 }
ba1234d1 1866 if (atomic_read(&dev_priv->mm.wedged))
673a394b
EA		 1867 ret = -EIO;
1868
1869 if (ret && ret != -ERESTARTSYS)
1870 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
1871 __func__, ret, seqno, i915_get_gem_seqno(dev));
1872
1873 /* Directly dispatch request retiring. While we have the work queue
1874 * to handle this, the waiter on a request often wants an associated
1875 * buffer to have made it to the inactive list, and we would need
1876 * a separate wait queue to handle that.
1877 */
1878 if (ret == 0)
1879 i915_gem_retire_requests(dev);
1880
1881 return ret;
1882}
1883
48764bf4
DV		 1884/**
1885 * Waits for a sequence number to be signaled, and cleans up the
1886 * request and object lists appropriately for that event.
1887 */
1888static int
1889i915_wait_request(struct drm_device *dev, uint32_t seqno)
1890{
1891 return i915_do_wait_request(dev, seqno, 1);
1892}
1893
673a394b
EA		 1894static void
1895i915_gem_flush(struct drm_device *dev,
1896 uint32_t invalidate_domains,
1897 uint32_t flush_domains)
1898{
1899 drm_i915_private_t *dev_priv = dev->dev_private;
1900 uint32_t cmd;
1901 RING_LOCALS;
1902
1903#if WATCH_EXEC
1904 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
1905 invalidate_domains, flush_domains);
1906#endif
1c5d22f7
CW		 1907 trace_i915_gem_request_flush(dev, dev_priv->mm.next_gem_seqno,
1908 invalidate_domains, flush_domains);
673a394b
EA		 1909
1910 if (flush_domains & I915_GEM_DOMAIN_CPU)
1911 drm_agp_chipset_flush(dev);
1912
21d509e3 1913 if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
673a394b
EA		 1914 /*
1915 * read/write caches:
1916 *
1917 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
1918 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
1919 * also flushed at 2d versus 3d pipeline switches.
1920 *
1921 * read-only caches:
1922 *
1923 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
1924 * MI_READ_FLUSH is set, and is always flushed on 965.
1925 *
1926 * I915_GEM_DOMAIN_COMMAND may not exist?
1927 *
1928 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
1929 * invalidated when MI_EXE_FLUSH is set.
1930 *
1931 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
1932 * invalidated with every MI_FLUSH.
1933 *
1934 * TLBs:
1935 *
1936 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
1937 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
1938 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
1939 * are flushed at any MI_FLUSH.
1940 */
1941
1942 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1943 if ((invalidate_domains|flush_domains) &
1944 I915_GEM_DOMAIN_RENDER)
1945 cmd &= ~MI_NO_WRITE_FLUSH;
1946 if (!IS_I965G(dev)) {
1947 /*
1948 * On the 965, the sampler cache always gets flushed
1949 * and this bit is reserved.
1950 */
1951 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
1952 cmd |= MI_READ_FLUSH;
1953 }
1954 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
1955 cmd |= MI_EXE_FLUSH;
1956
1957#if WATCH_EXEC
1958 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
1959#endif
1960 BEGIN_LP_RING(2);
1961 OUT_RING(cmd);
48764bf4 1962 OUT_RING(MI_NOOP);
673a394b
EA		 1963 ADVANCE_LP_RING();
1964 }
1965}
1966
1967/**
1968 * Ensures that all rendering to the object has completed and the object is
1969 * safe to unbind from the GTT or access from the CPU.
1970 */
1971static int
1972i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1973{
1974 struct drm_device *dev = obj->dev;
1975 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1976 int ret;
1977
e47c68e9
EA		 1978 /* This function only exists to support waiting for existing rendering,
1979 * not for emitting required flushes.
673a394b 1980 */
e47c68e9 1981 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
673a394b
EA		 1982
1983 /* If there is rendering queued on the buffer being evicted, wait for
1984 * it.
1985 */
1986 if (obj_priv->active) {
1987#if WATCH_BUF
1988 DRM_INFO("%s: object %p wait for seqno %08x\n",
1989 __func__, obj, obj_priv->last_rendering_seqno);
1990#endif
1991 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
1992 if (ret != 0)
1993 return ret;
1994 }
1995
1996 return 0;
1997}
1998
1999/**
2000 * Unbinds an object from the GTT aperture.
2001 */
0f973f27 2002int
673a394b
EA		 2003i915_gem_object_unbind(struct drm_gem_object *obj)
2004{
2005 struct drm_device *dev = obj->dev;
2006 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2007 int ret = 0;
2008
2009#if WATCH_BUF
2010 DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
2011 DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
2012#endif
2013 if (obj_priv->gtt_space == NULL)
2014 return 0;
2015
2016 if (obj_priv->pin_count != 0) {
2017 DRM_ERROR("Attempting to unbind pinned buffer\n");
2018 return -EINVAL;
2019 }
2020
5323fd04
EA		 2021 /* blow away mappings if mapped through GTT */
2022 i915_gem_release_mmap(obj);
2023
673a394b
EA		 2024 /* Move the object to the CPU domain to ensure that
2025 * any possible CPU writes while it's not in the GTT
2026 * are flushed when we go to remap it. This will
2027 * also ensure that all pending GPU writes are finished
2028 * before we unbind.
2029 */
e47c68e9 2030 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
673a394b 2031 if (ret) {
e47c68e9
EA		 2032 if (ret != -ERESTARTSYS)
2033 DRM_ERROR("set_domain failed: %d\n", ret);
673a394b
EA		 2034 return ret;
2035 }
2036
5323fd04
EA		 2037 BUG_ON(obj_priv->active);
2038
96b47b65
DV		 2039 /* release the fence reg _after_ flushing */
2040 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2041 i915_gem_clear_fence_reg(obj);
2042
673a394b
EA		 2043 if (obj_priv->agp_mem != NULL) {
2044 drm_unbind_agp(obj_priv->agp_mem);
2045 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2046 obj_priv->agp_mem = NULL;
2047 }
2048
856fa198 2049 i915_gem_object_put_pages(obj);
a32808c0 2050 BUG_ON(obj_priv->pages_refcount);
673a394b
EA		 2051
2052 if (obj_priv->gtt_space) {
2053 atomic_dec(&dev->gtt_count);
2054 atomic_sub(obj->size, &dev->gtt_memory);
2055
2056 drm_mm_put_block(obj_priv->gtt_space);
2057 obj_priv->gtt_space = NULL;
2058 }
2059
2060 /* Remove ourselves from the LRU list if present. */
2061 if (!list_empty(&obj_priv->list))
2062 list_del_init(&obj_priv->list);
2063
963b4836
CW		 2064 if (i915_gem_object_is_purgeable(obj_priv))
2065 i915_gem_object_truncate(obj);
2066
1c5d22f7
CW		 2067 trace_i915_gem_object_unbind(obj);
2068
673a394b
EA		 2069 return 0;
2070}
2071
07f73f69
CW		 2072static struct drm_gem_object *
2073i915_gem_find_inactive_object(struct drm_device *dev, int min_size)
2074{
2075 drm_i915_private_t *dev_priv = dev->dev_private;
2076 struct drm_i915_gem_object *obj_priv;
2077 struct drm_gem_object *best = NULL;
2078 struct drm_gem_object *first = NULL;
2079
2080 /* Try to find the smallest clean object */
2081 list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
2082 struct drm_gem_object *obj = obj_priv->obj;
2083 if (obj->size >= min_size) {
963b4836
CW		 2084 if ((!obj_priv->dirty ||
2085 i915_gem_object_is_purgeable(obj_priv)) &&
07f73f69
CW		 2086 (!best || obj->size < best->size)) {
2087 best = obj;
2088 if (best->size == min_size)
2089 return best;
2090 }
2091 if (!first)
2092 first = obj;
2093 }
2094 }
2095
2096 return best ? best : first;
2097}
2098
673a394b 2099static int
07f73f69
CW		 2100i915_gem_evict_everything(struct drm_device *dev)
2101{
2102 drm_i915_private_t *dev_priv = dev->dev_private;
2103 uint32_t seqno;
2104 int ret;
2105 bool lists_empty;
2106
07f73f69
CW		 2107 spin_lock(&dev_priv->mm.active_list_lock);
2108 lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2109 list_empty(&dev_priv->mm.flushing_list) &&
2110 list_empty(&dev_priv->mm.active_list));
2111 spin_unlock(&dev_priv->mm.active_list_lock);
2112
9731129c 2113 if (lists_empty)
07f73f69 2114 return -ENOSPC;
07f73f69
CW		 2115
2116 /* Flush everything (on to the inactive lists) and evict */
2117 i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2118 seqno = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS);
2119 if (seqno == 0)
2120 return -ENOMEM;
2121
2122 ret = i915_wait_request(dev, seqno);
2123 if (ret)
2124 return ret;
2125
ab5ee576 2126 ret = i915_gem_evict_from_inactive_list(dev);
07f73f69
CW		 2127 if (ret)
2128 return ret;
2129
2130 spin_lock(&dev_priv->mm.active_list_lock);
2131 lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2132 list_empty(&dev_priv->mm.flushing_list) &&
2133 list_empty(&dev_priv->mm.active_list));
2134 spin_unlock(&dev_priv->mm.active_list_lock);
2135 BUG_ON(!lists_empty);
2136
2137 return 0;
2138}
2139
673a394b 2140static int
07f73f69 2141i915_gem_evict_something(struct drm_device *dev, int min_size)
673a394b
EA		 2142{
2143 drm_i915_private_t *dev_priv = dev->dev_private;
2144 struct drm_gem_object *obj;
07f73f69 2145 int ret;
673a394b
EA		 2146
2147 for (;;) {
07f73f69
CW		 2148 i915_gem_retire_requests(dev);
2149
673a394b
EA		 2150 /* If there's an inactive buffer available now, grab it
2151 * and be done.
2152 */
07f73f69
CW		 2153 obj = i915_gem_find_inactive_object(dev, min_size);
2154 if (obj) {
2155 struct drm_i915_gem_object *obj_priv;
2156
673a394b
EA		 2157#if WATCH_LRU
2158 DRM_INFO("%s: evicting %p\n", __func__, obj);
2159#endif
07f73f69
CW		 2160 obj_priv = obj->driver_private;
2161 BUG_ON(obj_priv->pin_count != 0);
673a394b
EA		 2162 BUG_ON(obj_priv->active);
2163
2164 /* Wait on the rendering and unbind the buffer. */
07f73f69 2165 return i915_gem_object_unbind(obj);
673a394b
EA		 2166 }
2167
2168 /* If we didn't get anything, but the ring is still processing
07f73f69
CW		 2169 * things, wait for the next to finish and hopefully leave us
2170 * a buffer to evict.
673a394b
EA		 2171 */
2172 if (!list_empty(&dev_priv->mm.request_list)) {
2173 struct drm_i915_gem_request *request;
2174
2175 request = list_first_entry(&dev_priv->mm.request_list,
2176 struct drm_i915_gem_request,
2177 list);
2178
2179 ret = i915_wait_request(dev, request->seqno);
2180 if (ret)
07f73f69 2181 return ret;
673a394b 2182
07f73f69 2183 continue;
673a394b
EA		 2184 }
2185
2186 /* If we didn't have anything on the request list but there
2187 * are buffers awaiting a flush, emit one and try again.
2188 * When we wait on it, those buffers waiting for that flush
2189 * will get moved to inactive.
2190 */
2191 if (!list_empty(&dev_priv->mm.flushing_list)) {
07f73f69 2192 struct drm_i915_gem_object *obj_priv;
673a394b 2193
9a1e2582
CW		 2194 /* Find an object that we can immediately reuse */
2195 list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
2196 obj = obj_priv->obj;
2197 if (obj->size >= min_size)
2198 break;
673a394b 2199
9a1e2582
CW		 2200 obj = NULL;
2201 }
673a394b 2202
9a1e2582
CW		 2203 if (obj != NULL) {
2204 uint32_t seqno;
673a394b 2205
9a1e2582
CW		 2206 i915_gem_flush(dev,
2207 obj->write_domain,
2208 obj->write_domain);
2209 seqno = i915_add_request(dev, NULL, obj->write_domain);
2210 if (seqno == 0)
2211 return -ENOMEM;
ac94a962 2212
9a1e2582
CW		 2213 ret = i915_wait_request(dev, seqno);
2214 if (ret)
2215 return ret;
2216
2217 continue;
2218 }
673a394b
EA		 2219 }
2220
07f73f69
CW		 2221 /* If we didn't do any of the above, there's no single buffer
2222 * large enough to swap out for the new one, so just evict
2223 * everything and start again. (This should be rare.)
673a394b 2224 */
9731129c 2225 if (!list_empty (&dev_priv->mm.inactive_list))
ab5ee576 2226 return i915_gem_evict_from_inactive_list(dev);
9731129c 2227 else
07f73f69 2228 return i915_gem_evict_everything(dev);
ac94a962 2229 }
ac94a962
KP		 2230}
2231
6911a9b8 2232int
856fa198 2233i915_gem_object_get_pages(struct drm_gem_object *obj)
673a394b
EA		 2234{
2235 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2236 int page_count, i;
2237 struct address_space *mapping;
2238 struct inode *inode;
2239 struct page *page;
2240 int ret;
2241
856fa198 2242 if (obj_priv->pages_refcount++ != 0)
673a394b
EA		 2243 return 0;
2244
2245 /* Get the list of pages out of our struct file. They'll be pinned
2246 * at this point until we release them.
2247 */
2248 page_count = obj->size / PAGE_SIZE;
856fa198 2249 BUG_ON(obj_priv->pages != NULL);
8e7d2b2c 2250 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
856fa198 2251 if (obj_priv->pages == NULL) {
856fa198 2252 obj_priv->pages_refcount--;
673a394b
EA		 2253 return -ENOMEM;
2254 }
2255
2256 inode = obj->filp->f_path.dentry->d_inode;
2257 mapping = inode->i_mapping;
2258 for (i = 0; i < page_count; i++) {
2259 page = read_mapping_page(mapping, i, NULL);
2260 if (IS_ERR(page)) {
2261 ret = PTR_ERR(page);
856fa198 2262 i915_gem_object_put_pages(obj);
673a394b
EA		 2263 return ret;
2264 }
856fa198 2265 obj_priv->pages[i] = page;
673a394b 2266 }
280b713b
EA		 2267
2268 if (obj_priv->tiling_mode != I915_TILING_NONE)
2269 i915_gem_object_do_bit_17_swizzle(obj);
2270
673a394b
EA		 2271 return 0;
2272}
2273
de151cf6
JB		 2274static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2275{
2276 struct drm_gem_object *obj = reg->obj;
2277 struct drm_device *dev = obj->dev;
2278 drm_i915_private_t *dev_priv = dev->dev_private;
2279 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2280 int regnum = obj_priv->fence_reg;
2281 uint64_t val;
2282
2283 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2284 0xfffff000) << 32;
2285 val |= obj_priv->gtt_offset & 0xfffff000;
2286 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2287 if (obj_priv->tiling_mode == I915_TILING_Y)
2288 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2289 val |= I965_FENCE_REG_VALID;
2290
2291 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2292}
2293
2294static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2295{
2296 struct drm_gem_object *obj = reg->obj;
2297 struct drm_device *dev = obj->dev;
2298 drm_i915_private_t *dev_priv = dev->dev_private;
2299 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2300 int regnum = obj_priv->fence_reg;
0f973f27 2301 int tile_width;
dc529a4f 2302 uint32_t fence_reg, val;
de151cf6
JB		 2303 uint32_t pitch_val;
2304
2305 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2306 (obj_priv->gtt_offset & (obj->size - 1))) {
f06da264 2307 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
0f973f27 2308 __func__, obj_priv->gtt_offset, obj->size);
de151cf6
JB		 2309 return;
2310 }
2311
0f973f27
JB		 2312 if (obj_priv->tiling_mode == I915_TILING_Y &&
2313 HAS_128_BYTE_Y_TILING(dev))
2314 tile_width = 128;
de151cf6 2315 else
0f973f27
JB		 2316 tile_width = 512;
2317
2318 /* Note: pitch better be a power of two tile widths */
2319 pitch_val = obj_priv->stride / tile_width;
2320 pitch_val = ffs(pitch_val) - 1;
de151cf6
JB		 2321
2322 val = obj_priv->gtt_offset;
2323 if (obj_priv->tiling_mode == I915_TILING_Y)
2324 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2325 val |= I915_FENCE_SIZE_BITS(obj->size);
2326 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2327 val |= I830_FENCE_REG_VALID;
2328
dc529a4f
EA		 2329 if (regnum < 8)
2330 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2331 else
2332 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2333 I915_WRITE(fence_reg, val);
de151cf6
JB		 2334}
2335
2336static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2337{
2338 struct drm_gem_object *obj = reg->obj;
2339 struct drm_device *dev = obj->dev;
2340 drm_i915_private_t *dev_priv = dev->dev_private;
2341 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2342 int regnum = obj_priv->fence_reg;
2343 uint32_t val;
2344 uint32_t pitch_val;
8d7773a3 2345 uint32_t fence_size_bits;
de151cf6 2346
8d7773a3 2347 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
de151cf6 2348 (obj_priv->gtt_offset & (obj->size - 1))) {
8d7773a3 2349 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
0f973f27 2350 __func__, obj_priv->gtt_offset);
de151cf6
JB		 2351 return;
2352 }
2353
e76a16de
EA		 2354 pitch_val = obj_priv->stride / 128;
2355 pitch_val = ffs(pitch_val) - 1;
2356 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2357
de151cf6
JB		 2358 val = obj_priv->gtt_offset;
2359 if (obj_priv->tiling_mode == I915_TILING_Y)
2360 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
8d7773a3
DV		 2361 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2362 WARN_ON(fence_size_bits & ~0x00000f00);
2363 val |= fence_size_bits;
de151cf6
JB		 2364 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2365 val |= I830_FENCE_REG_VALID;
2366
2367 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
de151cf6
JB		 2368}
2369
2370/**
2371 * i915_gem_object_get_fence_reg - set up a fence reg for an object
2372 * @obj: object to map through a fence reg
2373 *
2374 * When mapping objects through the GTT, userspace wants to be able to write
2375 * to them without having to worry about swizzling if the object is tiled.
2376 *
2377 * This function walks the fence regs looking for a free one for @obj,
2378 * stealing one if it can't find any.
2379 *
2380 * It then sets up the reg based on the object's properties: address, pitch
2381 * and tiling format.
2382 */
8c4b8c3f
CW		 2383int
2384i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
de151cf6
JB		 2385{
2386 struct drm_device *dev = obj->dev;
79e53945 2387 struct drm_i915_private *dev_priv = dev->dev_private;
de151cf6
JB		 2388 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2389 struct drm_i915_fence_reg *reg = NULL;
fc7170ba
CW		 2390 struct drm_i915_gem_object *old_obj_priv = NULL;
2391 int i, ret, avail;
de151cf6 2392
a09ba7fa
EA		 2393 /* Just update our place in the LRU if our fence is getting used. */
2394 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2395 list_move_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list);
2396 return 0;
2397 }
2398
de151cf6
JB		 2399 switch (obj_priv->tiling_mode) {
2400 case I915_TILING_NONE:
2401 WARN(1, "allocating a fence for non-tiled object?\n");
2402 break;
2403 case I915_TILING_X:
0f973f27
JB		 2404 if (!obj_priv->stride)
2405 return -EINVAL;
2406 WARN((obj_priv->stride & (512 - 1)),
2407 "object 0x%08x is X tiled but has non-512B pitch\n",
2408 obj_priv->gtt_offset);
de151cf6
JB		 2409 break;
2410 case I915_TILING_Y:
0f973f27
JB		 2411 if (!obj_priv->stride)
2412 return -EINVAL;
2413 WARN((obj_priv->stride & (128 - 1)),
2414 "object 0x%08x is Y tiled but has non-128B pitch\n",
2415 obj_priv->gtt_offset);
de151cf6
JB		 2416 break;
2417 }
2418
2419 /* First try to find a free reg */
fc7170ba 2420 avail = 0;
de151cf6
JB		 2421 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2422 reg = &dev_priv->fence_regs[i];
2423 if (!reg->obj)
2424 break;
fc7170ba
CW		 2425
2426 old_obj_priv = reg->obj->driver_private;
2427 if (!old_obj_priv->pin_count)
2428 avail++;
de151cf6
JB		 2429 }
2430
2431 /* None available, try to steal one or wait for a user to finish */
2432 if (i == dev_priv->num_fence_regs) {
a09ba7fa 2433 struct drm_gem_object *old_obj = NULL;
de151cf6 2434
fc7170ba 2435 if (avail == 0)
2939e1f5 2436 return -ENOSPC;
fc7170ba 2437
a09ba7fa
EA		 2438 list_for_each_entry(old_obj_priv, &dev_priv->mm.fence_list,
2439 fence_list) {
2440 old_obj = old_obj_priv->obj;
d7619c4b 2441
d7619c4b
CW		 2442 if (old_obj_priv->pin_count)
2443 continue;
2444
a09ba7fa
EA		 2445 /* Take a reference, as otherwise the wait_rendering
2446 * below may cause the object to get freed out from
2447 * under us.
2448 */
2449 drm_gem_object_reference(old_obj);
2450
d7619c4b
CW		 2451 /* i915 uses fences for GPU access to tiled buffers */
2452 if (IS_I965G(dev) || !old_obj_priv->active)
de151cf6 2453 break;
d7619c4b 2454
a09ba7fa
EA		 2455 /* This brings the object to the head of the LRU if it
2456 * had been written to. The only way this should
2457 * result in us waiting longer than the expected
2458 * optimal amount of time is if there was a
2459 * fence-using buffer later that was read-only.
2460 */
2461 i915_gem_object_flush_gpu_write_domain(old_obj);
2462 ret = i915_gem_object_wait_rendering(old_obj);
58c2fb64
CW		 2463 if (ret != 0) {
2464 drm_gem_object_unreference(old_obj);
d7619c4b 2465 return ret;
58c2fb64
CW		 2466 }
2467
a09ba7fa 2468 break;
de151cf6
JB		 2469 }
2470
2471 /*
2472 * Zap this virtual mapping so we can set up a fence again
2473 * for this object next time we need it.
2474 */
58c2fb64
CW		 2475 i915_gem_release_mmap(old_obj);
2476
a09ba7fa 2477 i = old_obj_priv->fence_reg;
58c2fb64
CW		 2478 reg = &dev_priv->fence_regs[i];
2479
de151cf6 2480 old_obj_priv->fence_reg = I915_FENCE_REG_NONE;
a09ba7fa 2481 list_del_init(&old_obj_priv->fence_list);
58c2fb64 2482
a09ba7fa 2483 drm_gem_object_unreference(old_obj);
de151cf6
JB		 2484 }
2485
2486 obj_priv->fence_reg = i;
a09ba7fa
EA		 2487 list_add_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list);
2488
de151cf6
JB		 2489 reg->obj = obj;
2490
2491 if (IS_I965G(dev))
2492 i965_write_fence_reg(reg);
2493 else if (IS_I9XX(dev))
2494 i915_write_fence_reg(reg);
2495 else
2496 i830_write_fence_reg(reg);
d9ddcb96 2497
1c5d22f7
CW		 2498 trace_i915_gem_object_get_fence(obj, i, obj_priv->tiling_mode);
2499
d9ddcb96 2500 return 0;
de151cf6
JB		 2501}
2502
2503/**
2504 * i915_gem_clear_fence_reg - clear out fence register info
2505 * @obj: object to clear
2506 *
2507 * Zeroes out the fence register itself and clears out the associated
2508 * data structures in dev_priv and obj_priv.
2509 */
2510static void
2511i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2512{
2513 struct drm_device *dev = obj->dev;
79e53945 2514 drm_i915_private_t *dev_priv = dev->dev_private;
de151cf6
JB		 2515 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2516
2517 if (IS_I965G(dev))
2518 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
dc529a4f
EA		 2519 else {
2520 uint32_t fence_reg;
2521
2522 if (obj_priv->fence_reg < 8)
2523 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2524 else
2525 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
2526 8) * 4;
2527
2528 I915_WRITE(fence_reg, 0);
2529 }
de151cf6
JB		 2530
2531 dev_priv->fence_regs[obj_priv->fence_reg].obj = NULL;
2532 obj_priv->fence_reg = I915_FENCE_REG_NONE;
a09ba7fa 2533 list_del_init(&obj_priv->fence_list);
de151cf6
JB		 2534}
2535
52dc7d32
CW		 2536/**
2537 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2538 * to the buffer to finish, and then resets the fence register.
2539 * @obj: tiled object holding a fence register.
2540 *
2541 * Zeroes out the fence register itself and clears out the associated
2542 * data structures in dev_priv and obj_priv.
2543 */
2544int
2545i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2546{
2547 struct drm_device *dev = obj->dev;
2548 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2549
2550 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2551 return 0;
2552
2553 /* On the i915, GPU access to tiled buffers is via a fence,
2554 * therefore we must wait for any outstanding access to complete
2555 * before clearing the fence.
2556 */
2557 if (!IS_I965G(dev)) {
2558 int ret;
2559
2560 i915_gem_object_flush_gpu_write_domain(obj);
2561 i915_gem_object_flush_gtt_write_domain(obj);
2562 ret = i915_gem_object_wait_rendering(obj);
2563 if (ret != 0)
2564 return ret;
2565 }
2566
2567 i915_gem_clear_fence_reg (obj);
2568
2569 return 0;
2570}
2571
673a394b
EA		 2572/**
2573 * Finds free space in the GTT aperture and binds the object there.
2574 */
2575static int
2576i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2577{
2578 struct drm_device *dev = obj->dev;
2579 drm_i915_private_t *dev_priv = dev->dev_private;
2580 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2581 struct drm_mm_node *free_space;
07f73f69
CW		 2582 bool retry_alloc = false;
2583 int ret;
673a394b 2584
bb6baf76 2585 if (obj_priv->madv != I915_MADV_WILLNEED) {
3ef94daa
CW		 2586 DRM_ERROR("Attempting to bind a purgeable object\n");
2587 return -EINVAL;
2588 }
2589
673a394b 2590 if (alignment == 0)
0f973f27 2591 alignment = i915_gem_get_gtt_alignment(obj);
8d7773a3 2592 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
673a394b
EA		 2593 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2594 return -EINVAL;
2595 }
2596
2597 search_free:
2598 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2599 obj->size, alignment, 0);
2600 if (free_space != NULL) {
2601 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2602 alignment);
2603 if (obj_priv->gtt_space != NULL) {
2604 obj_priv->gtt_space->private = obj;
2605 obj_priv->gtt_offset = obj_priv->gtt_space->start;
2606 }
2607 }
2608 if (obj_priv->gtt_space == NULL) {
2609 /* If the gtt is empty and we're still having trouble
2610 * fitting our object in, we're out of memory.
2611 */
2612#if WATCH_LRU
2613 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2614#endif
07f73f69 2615 ret = i915_gem_evict_something(dev, obj->size);
9731129c 2616 if (ret)
673a394b 2617 return ret;
9731129c 2618
673a394b
EA		 2619 goto search_free;
2620 }
2621
2622#if WATCH_BUF
cfd43c02 2623 DRM_INFO("Binding object of size %zd at 0x%08x\n",
673a394b
EA		 2624 obj->size, obj_priv->gtt_offset);
2625#endif
07f73f69
CW		 2626 if (retry_alloc) {
2627 i915_gem_object_set_page_gfp_mask (obj,
2628 i915_gem_object_get_page_gfp_mask (obj) & ~__GFP_NORETRY);
2629 }
856fa198 2630 ret = i915_gem_object_get_pages(obj);
07f73f69
CW		 2631 if (retry_alloc) {
2632 i915_gem_object_set_page_gfp_mask (obj,
2633 i915_gem_object_get_page_gfp_mask (obj) | __GFP_NORETRY);
2634 }
673a394b
EA		 2635 if (ret) {
2636 drm_mm_put_block(obj_priv->gtt_space);
2637 obj_priv->gtt_space = NULL;
07f73f69
CW		 2638
2639 if (ret == -ENOMEM) {
2640 /* first try to clear up some space from the GTT */
2641 ret = i915_gem_evict_something(dev, obj->size);
2642 if (ret) {
07f73f69
CW		 2643 /* now try to shrink everyone else */
2644 if (! retry_alloc) {
2645 retry_alloc = true;
2646 goto search_free;
2647 }
2648
2649 return ret;
2650 }
2651
2652 goto search_free;
2653 }
2654
673a394b
EA		 2655 return ret;
2656 }
2657
673a394b
EA		 2658 /* Create an AGP memory structure pointing at our pages, and bind it
2659 * into the GTT.
2660 */
2661 obj_priv->agp_mem = drm_agp_bind_pages(dev,
856fa198 2662 obj_priv->pages,
07f73f69 2663 obj->size >> PAGE_SHIFT,
ba1eb1d8
KP		 2664 obj_priv->gtt_offset,
2665 obj_priv->agp_type);
673a394b 2666 if (obj_priv->agp_mem == NULL) {
856fa198 2667 i915_gem_object_put_pages(obj);
673a394b
EA		 2668 drm_mm_put_block(obj_priv->gtt_space);
2669 obj_priv->gtt_space = NULL;
07f73f69
CW		 2670
2671 ret = i915_gem_evict_something(dev, obj->size);
9731129c 2672 if (ret)
07f73f69 2673 return ret;
07f73f69
CW		 2674
2675 goto search_free;
673a394b
EA		 2676 }
2677 atomic_inc(&dev->gtt_count);
2678 atomic_add(obj->size, &dev->gtt_memory);
2679
2680 /* Assert that the object is not currently in any GPU domain. As it
2681 * wasn't in the GTT, there shouldn't be any way it could have been in
2682 * a GPU cache
2683 */
21d509e3
CW		 2684 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2685 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
673a394b 2686
1c5d22f7
CW		 2687 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2688
673a394b
EA		 2689 return 0;
2690}
2691
2692void
2693i915_gem_clflush_object(struct drm_gem_object *obj)
2694{
2695 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2696
2697 /* If we don't have a page list set up, then we're not pinned
2698 * to GPU, and we can ignore the cache flush because it'll happen
2699 * again at bind time.
2700 */
856fa198 2701 if (obj_priv->pages == NULL)
673a394b
EA		 2702 return;
2703
1c5d22f7 2704 trace_i915_gem_object_clflush(obj);
cfa16a0d 2705
856fa198 2706 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
673a394b
EA		 2707}
2708
e47c68e9
EA		 2709/** Flushes any GPU write domain for the object if it's dirty. */
2710static void
2711i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2712{
2713 struct drm_device *dev = obj->dev;
2714 uint32_t seqno;
1c5d22f7 2715 uint32_t old_write_domain;
e47c68e9
EA		 2716
2717 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2718 return;
2719
2720 /* Queue the GPU write cache flushing we need. */
1c5d22f7 2721 old_write_domain = obj->write_domain;
e47c68e9 2722 i915_gem_flush(dev, 0, obj->write_domain);
b962442e 2723 seqno = i915_add_request(dev, NULL, obj->write_domain);
e47c68e9
EA		 2724 obj->write_domain = 0;
2725 i915_gem_object_move_to_active(obj, seqno);
1c5d22f7
CW		 2726
2727 trace_i915_gem_object_change_domain(obj,
2728 obj->read_domains,
2729 old_write_domain);
e47c68e9
EA		 2730}
2731
2732/** Flushes the GTT write domain for the object if it's dirty. */
2733static void
2734i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2735{
1c5d22f7
CW		 2736 uint32_t old_write_domain;
2737
e47c68e9
EA		 2738 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2739 return;
2740
2741 /* No actual flushing is required for the GTT write domain. Writes
2742 * to it immediately go to main memory as far as we know, so there's
2743 * no chipset flush. It also doesn't land in render cache.
2744 */
1c5d22f7 2745 old_write_domain = obj->write_domain;
e47c68e9 2746 obj->write_domain = 0;
1c5d22f7
CW		 2747
2748 trace_i915_gem_object_change_domain(obj,
2749 obj->read_domains,
2750 old_write_domain);
e47c68e9
EA		 2751}
2752
2753/** Flushes the CPU write domain for the object if it's dirty. */
2754static void
2755i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2756{
2757 struct drm_device *dev = obj->dev;
1c5d22f7 2758 uint32_t old_write_domain;
e47c68e9
EA		 2759
2760 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2761 return;
2762
2763 i915_gem_clflush_object(obj);
2764 drm_agp_chipset_flush(dev);
1c5d22f7 2765 old_write_domain = obj->write_domain;
e47c68e9 2766 obj->write_domain = 0;
1c5d22f7
CW		 2767
2768 trace_i915_gem_object_change_domain(obj,
2769 obj->read_domains,
2770 old_write_domain);
e47c68e9
EA		 2771}
2772
6b95a207
KH		 2773void
2774i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
2775{
2776 switch (obj->write_domain) {
2777 case I915_GEM_DOMAIN_GTT:
2778 i915_gem_object_flush_gtt_write_domain(obj);
2779 break;
2780 case I915_GEM_DOMAIN_CPU:
2781 i915_gem_object_flush_cpu_write_domain(obj);
2782 break;
2783 default:
2784 i915_gem_object_flush_gpu_write_domain(obj);
2785 break;
2786 }
2787}
2788
2ef7eeaa
EA		 2789/**
2790 * Moves a single object to the GTT read, and possibly write domain.
2791 *
2792 * This function returns when the move is complete, including waiting on
2793 * flushes to occur.
2794 */
79e53945 2795int
2ef7eeaa
EA		 2796i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2797{
2ef7eeaa 2798 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1c5d22f7 2799 uint32_t old_write_domain, old_read_domains;
e47c68e9 2800 int ret;
2ef7eeaa 2801
02354392
EA		 2802 /* Not valid to be called on unbound objects. */
2803 if (obj_priv->gtt_space == NULL)
2804 return -EINVAL;
2805
e47c68e9
EA		 2806 i915_gem_object_flush_gpu_write_domain(obj);
2807 /* Wait on any GPU rendering and flushing to occur. */
2808 ret = i915_gem_object_wait_rendering(obj);
2809 if (ret != 0)
2810 return ret;
2811
1c5d22f7
CW		 2812 old_write_domain = obj->write_domain;
2813 old_read_domains = obj->read_domains;
2814
e47c68e9
EA		 2815 /* If we're writing through the GTT domain, then CPU and GPU caches
2816 * will need to be invalidated at next use.
2ef7eeaa 2817 */
e47c68e9
EA		 2818 if (write)
2819 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2ef7eeaa 2820
e47c68e9 2821 i915_gem_object_flush_cpu_write_domain(obj);
2ef7eeaa 2822
e47c68e9
EA		 2823 /* It should now be out of any other write domains, and we can update
2824 * the domain values for our changes.
2825 */
2826 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2827 obj->read_domains |= I915_GEM_DOMAIN_GTT;
2828 if (write) {
2829 obj->write_domain = I915_GEM_DOMAIN_GTT;
2830 obj_priv->dirty = 1;
2ef7eeaa
EA		 2831 }
2832
1c5d22f7
CW		 2833 trace_i915_gem_object_change_domain(obj,
2834 old_read_domains,
2835 old_write_domain);
2836
e47c68e9
EA		 2837 return 0;
2838}
2839
2840/**
2841 * Moves a single object to the CPU read, and possibly write domain.
2842 *
2843 * This function returns when the move is complete, including waiting on
2844 * flushes to occur.
2845 */
2846static int
2847i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2848{
1c5d22f7 2849 uint32_t old_write_domain, old_read_domains;
e47c68e9
EA		 2850 int ret;
2851
2852 i915_gem_object_flush_gpu_write_domain(obj);
2ef7eeaa 2853 /* Wait on any GPU rendering and flushing to occur. */
e47c68e9
EA		 2854 ret = i915_gem_object_wait_rendering(obj);
2855 if (ret != 0)
2856 return ret;
2ef7eeaa 2857
e47c68e9 2858 i915_gem_object_flush_gtt_write_domain(obj);
2ef7eeaa 2859
e47c68e9
EA		 2860 /* If we have a partially-valid cache of the object in the CPU,
2861 * finish invalidating it and free the per-page flags.
2ef7eeaa 2862 */
e47c68e9 2863 i915_gem_object_set_to_full_cpu_read_domain(obj);
2ef7eeaa 2864
1c5d22f7
CW		 2865 old_write_domain = obj->write_domain;
2866 old_read_domains = obj->read_domains;
2867
e47c68e9
EA		 2868 /* Flush the CPU cache if it's still invalid. */
2869 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2ef7eeaa 2870 i915_gem_clflush_object(obj);
2ef7eeaa 2871
e47c68e9 2872 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2ef7eeaa
EA		 2873 }
2874
2875 /* It should now be out of any other write domains, and we can update
2876 * the domain values for our changes.
2877 */
e47c68e9
EA		 2878 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2879
2880 /* If we're writing through the CPU, then the GPU read domains will
2881 * need to be invalidated at next use.
2882 */
2883 if (write) {
2884 obj->read_domains &= I915_GEM_DOMAIN_CPU;
2885 obj->write_domain = I915_GEM_DOMAIN_CPU;
2886 }
2ef7eeaa 2887
1c5d22f7
CW		 2888 trace_i915_gem_object_change_domain(obj,
2889 old_read_domains,
2890 old_write_domain);
2891
2ef7eeaa
EA		 2892 return 0;
2893}
2894
673a394b
EA		 2895/*
2896 * Set the next domain for the specified object. This
2897 * may not actually perform the necessary flushing/invaliding though,
2898 * as that may want to be batched with other set_domain operations
2899 *
2900 * This is (we hope) the only really tricky part of gem. The goal
2901 * is fairly simple -- track which caches hold bits of the object
2902 * and make sure they remain coherent. A few concrete examples may
2903 * help to explain how it works. For shorthand, we use the notation
2904 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2905 * a pair of read and write domain masks.
2906 *
2907 * Case 1: the batch buffer
2908 *
2909 * 1. Allocated
2910 * 2. Written by CPU
2911 * 3. Mapped to GTT
2912 * 4. Read by GPU
2913 * 5. Unmapped from GTT
2914 * 6. Freed
2915 *
2916 * Let's take these a step at a time
2917 *
2918 * 1. Allocated
2919 * Pages allocated from the kernel may still have
2920 * cache contents, so we set them to (CPU, CPU) always.
2921 * 2. Written by CPU (using pwrite)
2922 * The pwrite function calls set_domain (CPU, CPU) and
2923 * this function does nothing (as nothing changes)
2924 * 3. Mapped by GTT
2925 * This function asserts that the object is not
2926 * currently in any GPU-based read or write domains
2927 * 4. Read by GPU
2928 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
2929 * As write_domain is zero, this function adds in the
2930 * current read domains (CPU+COMMAND, 0).
2931 * flush_domains is set to CPU.
2932 * invalidate_domains is set to COMMAND
2933 * clflush is run to get data out of the CPU caches
2934 * then i915_dev_set_domain calls i915_gem_flush to
2935 * emit an MI_FLUSH and drm_agp_chipset_flush
2936 * 5. Unmapped from GTT
2937 * i915_gem_object_unbind calls set_domain (CPU, CPU)
2938 * flush_domains and invalidate_domains end up both zero
2939 * so no flushing/invalidating happens
2940 * 6. Freed
2941 * yay, done
2942 *
2943 * Case 2: The shared render buffer
2944 *
2945 * 1. Allocated
2946 * 2. Mapped to GTT
2947 * 3. Read/written by GPU
2948 * 4. set_domain to (CPU,CPU)
2949 * 5. Read/written by CPU
2950 * 6. Read/written by GPU
2951 *
2952 * 1. Allocated
2953 * Same as last example, (CPU, CPU)
2954 * 2. Mapped to GTT
2955 * Nothing changes (assertions find that it is not in the GPU)
2956 * 3. Read/written by GPU
2957 * execbuffer calls set_domain (RENDER, RENDER)
2958 * flush_domains gets CPU
2959 * invalidate_domains gets GPU
2960 * clflush (obj)
2961 * MI_FLUSH and drm_agp_chipset_flush
2962 * 4. set_domain (CPU, CPU)
2963 * flush_domains gets GPU
2964 * invalidate_domains gets CPU
2965 * wait_rendering (obj) to make sure all drawing is complete.
2966 * This will include an MI_FLUSH to get the data from GPU
2967 * to memory
2968 * clflush (obj) to invalidate the CPU cache
2969 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
2970 * 5. Read/written by CPU
2971 * cache lines are loaded and dirtied
2972 * 6. Read written by GPU
2973 * Same as last GPU access
2974 *
2975 * Case 3: The constant buffer
2976 *
2977 * 1. Allocated
2978 * 2. Written by CPU
2979 * 3. Read by GPU
2980 * 4. Updated (written) by CPU again
2981 * 5. Read by GPU
2982 *
2983 * 1. Allocated
2984 * (CPU, CPU)
2985 * 2. Written by CPU
2986 * (CPU, CPU)
2987 * 3. Read by GPU
2988 * (CPU+RENDER, 0)
2989 * flush_domains = CPU
2990 * invalidate_domains = RENDER
2991 * clflush (obj)
2992 * MI_FLUSH
2993 * drm_agp_chipset_flush
2994 * 4. Updated (written) by CPU again
2995 * (CPU, CPU)
2996 * flush_domains = 0 (no previous write domain)
2997 * invalidate_domains = 0 (no new read domains)
2998 * 5. Read by GPU
2999 * (CPU+RENDER, 0)
3000 * flush_domains = CPU
3001 * invalidate_domains = RENDER
3002 * clflush (obj)
3003 * MI_FLUSH
3004 * drm_agp_chipset_flush
3005 */
c0d90829 3006static void
8b0e378a 3007i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
673a394b
EA		 3008{
3009 struct drm_device *dev = obj->dev;
3010 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3011 uint32_t invalidate_domains = 0;
3012 uint32_t flush_domains = 0;
1c5d22f7 3013 uint32_t old_read_domains;
e47c68e9 3014
8b0e378a
EA		 3015 BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
3016 BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
673a394b 3017
652c393a
JB		 3018 intel_mark_busy(dev, obj);
3019
673a394b
EA		 3020#if WATCH_BUF
3021 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
3022 __func__, obj,
8b0e378a
EA		 3023 obj->read_domains, obj->pending_read_domains,
3024 obj->write_domain, obj->pending_write_domain);
673a394b
EA		 3025#endif
3026 /*
3027 * If the object isn't moving to a new write domain,
3028 * let the object stay in multiple read domains
3029 */
8b0e378a
EA		 3030 if (obj->pending_write_domain == 0)
3031 obj->pending_read_domains |= obj->read_domains;
673a394b
EA		 3032 else
3033 obj_priv->dirty = 1;
3034
3035 /*
3036 * Flush the current write domain if
3037 * the new read domains don't match. Invalidate
3038 * any read domains which differ from the old
3039 * write domain
3040 */
8b0e378a
EA		 3041 if (obj->write_domain &&
3042 obj->write_domain != obj->pending_read_domains) {
673a394b 3043 flush_domains |= obj->write_domain;
8b0e378a
EA		 3044 invalidate_domains |=
3045 obj->pending_read_domains & ~obj->write_domain;
673a394b
EA		 3046 }
3047 /*
3048 * Invalidate any read caches which may have
3049 * stale data. That is, any new read domains.
3050 */
8b0e378a 3051 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
673a394b
EA		 3052 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3053#if WATCH_BUF
3054 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3055 __func__, flush_domains, invalidate_domains);
3056#endif
673a394b
EA		 3057 i915_gem_clflush_object(obj);
3058 }
3059
1c5d22f7
CW		 3060 old_read_domains = obj->read_domains;
3061
efbeed96
EA		 3062 /* The actual obj->write_domain will be updated with
3063 * pending_write_domain after we emit the accumulated flush for all
3064 * of our domain changes in execbuffers (which clears objects'
3065 * write_domains). So if we have a current write domain that we
3066 * aren't changing, set pending_write_domain to that.
3067 */
3068 if (flush_domains == 0 && obj->pending_write_domain == 0)
3069 obj->pending_write_domain = obj->write_domain;
8b0e378a 3070 obj->read_domains = obj->pending_read_domains;
673a394b
EA		 3071
3072 dev->invalidate_domains |= invalidate_domains;
3073 dev->flush_domains |= flush_domains;
3074#if WATCH_BUF
3075 DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3076 __func__,
3077 obj->read_domains, obj->write_domain,
3078 dev->invalidate_domains, dev->flush_domains);
3079#endif
1c5d22f7
CW		 3080
3081 trace_i915_gem_object_change_domain(obj,
3082 old_read_domains,
3083 obj->write_domain);
673a394b
EA		 3084}
3085
3086/**
e47c68e9 3087 * Moves the object from a partially CPU read to a full one.
673a394b 3088 *
e47c68e9
EA		 3089 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3090 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
673a394b 3091 */
e47c68e9
EA		 3092static void
3093i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
673a394b
EA		 3094{
3095 struct drm_i915_gem_object *obj_priv = obj->driver_private;
673a394b 3096
e47c68e9
EA		 3097 if (!obj_priv->page_cpu_valid)
3098 return;
3099
3100 /* If we're partially in the CPU read domain, finish moving it in.
3101 */
3102 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3103 int i;
3104
3105 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3106 if (obj_priv->page_cpu_valid[i])
3107 continue;
856fa198 3108 drm_clflush_pages(obj_priv->pages + i, 1);
e47c68e9 3109 }
e47c68e9
EA		 3110 }
3111
3112 /* Free the page_cpu_valid mappings which are now stale, whether
3113 * or not we've got I915_GEM_DOMAIN_CPU.
3114 */
9a298b2a 3115 kfree(obj_priv->page_cpu_valid);
e47c68e9
EA		 3116 obj_priv->page_cpu_valid = NULL;
3117}
3118
3119/**
3120 * Set the CPU read domain on a range of the object.
3121 *
3122 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3123 * not entirely valid. The page_cpu_valid member of the object flags which
3124 * pages have been flushed, and will be respected by
3125 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3126 * of the whole object.
3127 *
3128 * This function returns when the move is complete, including waiting on
3129 * flushes to occur.
3130 */
3131static int
3132i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3133 uint64_t offset, uint64_t size)
3134{
3135 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1c5d22f7 3136 uint32_t old_read_domains;
e47c68e9 3137 int i, ret;
673a394b 3138
e47c68e9
EA		 3139 if (offset == 0 && size == obj->size)
3140 return i915_gem_object_set_to_cpu_domain(obj, 0);
673a394b 3141
e47c68e9
EA		 3142 i915_gem_object_flush_gpu_write_domain(obj);
3143 /* Wait on any GPU rendering and flushing to occur. */
6a47baa6 3144 ret = i915_gem_object_wait_rendering(obj);
e47c68e9 3145 if (ret != 0)
6a47baa6 3146 return ret;
e47c68e9
EA		 3147 i915_gem_object_flush_gtt_write_domain(obj);
3148
3149 /* If we're already fully in the CPU read domain, we're done. */
3150 if (obj_priv->page_cpu_valid == NULL &&
3151 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3152 return 0;
673a394b 3153
e47c68e9
EA		 3154 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3155 * newly adding I915_GEM_DOMAIN_CPU
3156 */
673a394b 3157 if (obj_priv->page_cpu_valid == NULL) {
9a298b2a
EA		 3158 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3159 GFP_KERNEL);
e47c68e9
EA		 3160 if (obj_priv->page_cpu_valid == NULL)
3161 return -ENOMEM;
3162 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3163 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
673a394b
EA		 3164
3165 /* Flush the cache on any pages that are still invalid from the CPU's
3166 * perspective.
3167 */
e47c68e9
EA		 3168 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3169 i++) {
673a394b
EA		 3170 if (obj_priv->page_cpu_valid[i])
3171 continue;
3172
856fa198 3173 drm_clflush_pages(obj_priv->pages + i, 1);
673a394b
EA		 3174
3175 obj_priv->page_cpu_valid[i] = 1;
3176 }
3177
e47c68e9
EA		 3178 /* It should now be out of any other write domains, and we can update
3179 * the domain values for our changes.
3180 */
3181 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3182
1c5d22f7 3183 old_read_domains = obj->read_domains;
e47c68e9
EA		 3184 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3185
1c5d22f7
CW		 3186 trace_i915_gem_object_change_domain(obj,
3187 old_read_domains,
3188 obj->write_domain);
3189
673a394b
EA		 3190 return 0;
3191}
3192
673a394b
EA		 3193/**
3194 * Pin an object to the GTT and evaluate the relocations landing in it.
3195 */
3196static int
3197i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3198 struct drm_file *file_priv,
76446cac 3199 struct drm_i915_gem_exec_object2 *entry,
40a5f0de 3200 struct drm_i915_gem_relocation_entry *relocs)
673a394b
EA		 3201{
3202 struct drm_device *dev = obj->dev;
0839ccb8 3203 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 3204 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3205 int i, ret;
0839ccb8 3206 void __iomem *reloc_page;
76446cac
JB		 3207 bool need_fence;
3208
3209 need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3210 obj_priv->tiling_mode != I915_TILING_NONE;
3211
3212 /* Check fence reg constraints and rebind if necessary */
3213 if (need_fence && !i915_obj_fenceable(dev, obj))
3214 i915_gem_object_unbind(obj);
673a394b
EA		 3215
3216 /* Choose the GTT offset for our buffer and put it there. */
3217 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3218 if (ret)
3219 return ret;
3220
76446cac
JB		 3221 /*
3222 * Pre-965 chips need a fence register set up in order to
3223 * properly handle blits to/from tiled surfaces.
3224 */
3225 if (need_fence) {
3226 ret = i915_gem_object_get_fence_reg(obj);
3227 if (ret != 0) {
3228 if (ret != -EBUSY && ret != -ERESTARTSYS)
3229 DRM_ERROR("Failure to install fence: %d\n",
3230 ret);
3231 i915_gem_object_unpin(obj);
3232 return ret;
3233 }
3234 }
3235
673a394b
EA		 3236 entry->offset = obj_priv->gtt_offset;
3237
673a394b
EA		 3238 /* Apply the relocations, using the GTT aperture to avoid cache
3239 * flushing requirements.
3240 */
3241 for (i = 0; i < entry->relocation_count; i++) {
40a5f0de 3242 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
673a394b
EA		 3243 struct drm_gem_object *target_obj;
3244 struct drm_i915_gem_object *target_obj_priv;
3043c60c
EA		 3245 uint32_t reloc_val, reloc_offset;
3246 uint32_t __iomem *reloc_entry;
673a394b 3247
673a394b 3248 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
40a5f0de 3249 reloc->target_handle);
673a394b
EA		 3250 if (target_obj == NULL) {
3251 i915_gem_object_unpin(obj);
3252 return -EBADF;
3253 }
3254 target_obj_priv = target_obj->driver_private;
3255
8542a0bb
CW		 3256#if WATCH_RELOC
3257 DRM_INFO("%s: obj %p offset %08x target %d "
3258 "read %08x write %08x gtt %08x "
3259 "presumed %08x delta %08x\n",
3260 __func__,
3261 obj,
3262 (int) reloc->offset,
3263 (int) reloc->target_handle,
3264 (int) reloc->read_domains,
3265 (int) reloc->write_domain,
3266 (int) target_obj_priv->gtt_offset,
3267 (int) reloc->presumed_offset,
3268 reloc->delta);
3269#endif
3270
673a394b
EA		 3271 /* The target buffer should have appeared before us in the
3272 * exec_object list, so it should have a GTT space bound by now.
3273 */
3274 if (target_obj_priv->gtt_space == NULL) {
3275 DRM_ERROR("No GTT space found for object %d\n",
40a5f0de 3276 reloc->target_handle);
673a394b
EA		 3277 drm_gem_object_unreference(target_obj);
3278 i915_gem_object_unpin(obj);
3279 return -EINVAL;
3280 }
3281
8542a0bb 3282 /* Validate that the target is in a valid r/w GPU domain */
40a5f0de
EA		 3283 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3284 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
e47c68e9
EA		 3285 DRM_ERROR("reloc with read/write CPU domains: "
3286 "obj %p target %d offset %d "
3287 "read %08x write %08x",
40a5f0de
EA		 3288 obj, reloc->target_handle,
3289 (int) reloc->offset,
3290 reloc->read_domains,
3291 reloc->write_domain);
491152b8
CW		 3292 drm_gem_object_unreference(target_obj);
3293 i915_gem_object_unpin(obj);
e47c68e9
EA		 3294 return -EINVAL;
3295 }
40a5f0de
EA		 3296 if (reloc->write_domain && target_obj->pending_write_domain &&
3297 reloc->write_domain != target_obj->pending_write_domain) {
673a394b
EA		 3298 DRM_ERROR("Write domain conflict: "
3299 "obj %p target %d offset %d "
3300 "new %08x old %08x\n",
40a5f0de
EA		 3301 obj, reloc->target_handle,
3302 (int) reloc->offset,
3303 reloc->write_domain,
673a394b
EA		 3304 target_obj->pending_write_domain);
3305 drm_gem_object_unreference(target_obj);
3306 i915_gem_object_unpin(obj);
3307 return -EINVAL;
3308 }
3309
40a5f0de
EA		 3310 target_obj->pending_read_domains |= reloc->read_domains;
3311 target_obj->pending_write_domain |= reloc->write_domain;
673a394b
EA		 3312
3313 /* If the relocation already has the right value in it, no
3314 * more work needs to be done.
3315 */
40a5f0de 3316 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
673a394b
EA		 3317 drm_gem_object_unreference(target_obj);
3318 continue;
3319 }
3320
8542a0bb
CW		 3321 /* Check that the relocation address is valid... */
3322 if (reloc->offset > obj->size - 4) {
3323 DRM_ERROR("Relocation beyond object bounds: "
3324 "obj %p target %d offset %d size %d.\n",
3325 obj, reloc->target_handle,
3326 (int) reloc->offset, (int) obj->size);
3327 drm_gem_object_unreference(target_obj);
3328 i915_gem_object_unpin(obj);
3329 return -EINVAL;
3330 }
3331 if (reloc->offset & 3) {
3332 DRM_ERROR("Relocation not 4-byte aligned: "
3333 "obj %p target %d offset %d.\n",
3334 obj, reloc->target_handle,
3335 (int) reloc->offset);
3336 drm_gem_object_unreference(target_obj);
3337 i915_gem_object_unpin(obj);
3338 return -EINVAL;
3339 }
3340
3341 /* and points to somewhere within the target object. */
3342 if (reloc->delta >= target_obj->size) {
3343 DRM_ERROR("Relocation beyond target object bounds: "
3344 "obj %p target %d delta %d size %d.\n",
3345 obj, reloc->target_handle,
3346 (int) reloc->delta, (int) target_obj->size);
3347 drm_gem_object_unreference(target_obj);
3348 i915_gem_object_unpin(obj);
3349 return -EINVAL;
3350 }
3351
2ef7eeaa
EA		 3352 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3353 if (ret != 0) {
3354 drm_gem_object_unreference(target_obj);
3355 i915_gem_object_unpin(obj);
3356 return -EINVAL;
673a394b
EA		 3357 }
3358
3359 /* Map the page containing the relocation we're going to
3360 * perform.
3361 */
40a5f0de 3362 reloc_offset = obj_priv->gtt_offset + reloc->offset;
0839ccb8
KP		 3363 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3364 (reloc_offset &
3365 ~(PAGE_SIZE - 1)));
3043c60c 3366 reloc_entry = (uint32_t __iomem *)(reloc_page +
0839ccb8 3367 (reloc_offset & (PAGE_SIZE - 1)));
40a5f0de 3368 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
673a394b
EA		 3369
3370#if WATCH_BUF
3371 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
40a5f0de 3372 obj, (unsigned int) reloc->offset,
673a394b
EA		 3373 readl(reloc_entry), reloc_val);
3374#endif
3375 writel(reloc_val, reloc_entry);
0839ccb8 3376 io_mapping_unmap_atomic(reloc_page);
673a394b 3377
40a5f0de
EA		 3378 /* The updated presumed offset for this entry will be
3379 * copied back out to the user.
673a394b 3380 */
40a5f0de 3381 reloc->presumed_offset = target_obj_priv->gtt_offset;
673a394b
EA		 3382
3383 drm_gem_object_unreference(target_obj);
3384 }
3385
673a394b
EA		 3386#if WATCH_BUF
3387 if (0)
3388 i915_gem_dump_object(obj, 128, __func__, ~0);
3389#endif
3390 return 0;
3391}
3392
3393/** Dispatch a batchbuffer to the ring
3394 */
3395static int
3396i915_dispatch_gem_execbuffer(struct drm_device *dev,
76446cac 3397 struct drm_i915_gem_execbuffer2 *exec,
201361a5 3398 struct drm_clip_rect *cliprects,
673a394b
EA		 3399 uint64_t exec_offset)
3400{
3401 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 3402 int nbox = exec->num_cliprects;
3403 int i = 0, count;
83d60795 3404 uint32_t exec_start, exec_len;
673a394b
EA		 3405 RING_LOCALS;
3406
3407 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3408 exec_len = (uint32_t) exec->batch_len;
3409
8f0dc5bf 3410 trace_i915_gem_request_submit(dev, dev_priv->mm.next_gem_seqno + 1);
1c5d22f7 3411
673a394b
EA		 3412 count = nbox ? nbox : 1;
3413
3414 for (i = 0; i < count; i++) {
3415 if (i < nbox) {
201361a5 3416 int ret = i915_emit_box(dev, cliprects, i,
673a394b
EA		 3417 exec->DR1, exec->DR4);
3418 if (ret)
3419 return ret;
3420 }
3421
3422 if (IS_I830(dev) || IS_845G(dev)) {
3423 BEGIN_LP_RING(4);
3424 OUT_RING(MI_BATCH_BUFFER);
3425 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3426 OUT_RING(exec_start + exec_len - 4);
3427 OUT_RING(0);
3428 ADVANCE_LP_RING();
3429 } else {
3430 BEGIN_LP_RING(2);
3431 if (IS_I965G(dev)) {
3432 OUT_RING(MI_BATCH_BUFFER_START |
3433 (2 << 6) |
3434 MI_BATCH_NON_SECURE_I965);
3435 OUT_RING(exec_start);
3436 } else {
3437 OUT_RING(MI_BATCH_BUFFER_START |
3438 (2 << 6));
3439 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3440 }
3441 ADVANCE_LP_RING();
3442 }
3443 }
3444
3445 /* XXX breadcrumb */
3446 return 0;
3447}
3448
3449/* Throttle our rendering by waiting until the ring has completed our requests
3450 * emitted over 20 msec ago.
3451 *
b962442e
EA		 3452 * Note that if we were to use the current jiffies each time around the loop,
3453 * we wouldn't escape the function with any frames outstanding if the time to
3454 * render a frame was over 20ms.
3455 *
673a394b
EA		 3456 * This should get us reasonable parallelism between CPU and GPU but also
3457 * relatively low latency when blocking on a particular request to finish.
3458 */
3459static int
3460i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3461{
3462 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3463 int ret = 0;
b962442e 3464 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
673a394b
EA		 3465
3466 mutex_lock(&dev->struct_mutex);
b962442e
EA		 3467 while (!list_empty(&i915_file_priv->mm.request_list)) {
3468 struct drm_i915_gem_request *request;
3469
3470 request = list_first_entry(&i915_file_priv->mm.request_list,
3471 struct drm_i915_gem_request,
3472 client_list);
3473
3474 if (time_after_eq(request->emitted_jiffies, recent_enough))
3475 break;
3476
3477 ret = i915_wait_request(dev, request->seqno);
3478 if (ret != 0)
3479 break;
3480 }
673a394b 3481 mutex_unlock(&dev->struct_mutex);
b962442e 3482
673a394b
EA		 3483 return ret;
3484}
3485
40a5f0de 3486static int
76446cac 3487i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
40a5f0de
EA		 3488 uint32_t buffer_count,
3489 struct drm_i915_gem_relocation_entry **relocs)
3490{
3491 uint32_t reloc_count = 0, reloc_index = 0, i;
3492 int ret;
3493
3494 *relocs = NULL;
3495 for (i = 0; i < buffer_count; i++) {
3496 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3497 return -EINVAL;
3498 reloc_count += exec_list[i].relocation_count;
3499 }
3500
8e7d2b2c 3501 *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
76446cac
JB		 3502 if (*relocs == NULL) {
3503 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
40a5f0de 3504 return -ENOMEM;
76446cac 3505 }
40a5f0de
EA		 3506
3507 for (i = 0; i < buffer_count; i++) {
3508 struct drm_i915_gem_relocation_entry __user *user_relocs;
3509
3510 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3511
3512 ret = copy_from_user(&(*relocs)[reloc_index],
3513 user_relocs,
3514 exec_list[i].relocation_count *
3515 sizeof(**relocs));
3516 if (ret != 0) {
8e7d2b2c 3517 drm_free_large(*relocs);
40a5f0de 3518 *relocs = NULL;
2bc43b5c 3519 return -EFAULT;
40a5f0de
EA		 3520 }
3521
3522 reloc_index += exec_list[i].relocation_count;
3523 }
3524
2bc43b5c 3525 return 0;
40a5f0de
EA		 3526}
3527
3528static int
76446cac 3529i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
40a5f0de
EA		 3530 uint32_t buffer_count,
3531 struct drm_i915_gem_relocation_entry *relocs)
3532{
3533 uint32_t reloc_count = 0, i;
2bc43b5c 3534 int ret = 0;
40a5f0de
EA		 3535
3536 for (i = 0; i < buffer_count; i++) {
3537 struct drm_i915_gem_relocation_entry __user *user_relocs;
2bc43b5c 3538 int unwritten;
40a5f0de
EA		 3539
3540 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3541
2bc43b5c
FM		 3542 unwritten = copy_to_user(user_relocs,
3543 &relocs[reloc_count],
3544 exec_list[i].relocation_count *
3545 sizeof(*relocs));
3546
3547 if (unwritten) {
3548 ret = -EFAULT;
3549 goto err;
40a5f0de
EA		 3550 }
3551
3552 reloc_count += exec_list[i].relocation_count;
3553 }
3554
2bc43b5c 3555err:
8e7d2b2c 3556 drm_free_large(relocs);
40a5f0de
EA		 3557
3558 return ret;
3559}
3560
83d60795 3561static int
76446cac 3562i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
83d60795
CW		 3563 uint64_t exec_offset)
3564{
3565 uint32_t exec_start, exec_len;
3566
3567 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3568 exec_len = (uint32_t) exec->batch_len;
3569
3570 if ((exec_start | exec_len) & 0x7)
3571 return -EINVAL;
3572
3573 if (!exec_start)
3574 return -EINVAL;
3575
3576 return 0;
3577}
3578
6b95a207
KH		 3579static int
3580i915_gem_wait_for_pending_flip(struct drm_device *dev,
3581 struct drm_gem_object **object_list,
3582 int count)
3583{
3584 drm_i915_private_t *dev_priv = dev->dev_private;
3585 struct drm_i915_gem_object *obj_priv;
3586 DEFINE_WAIT(wait);
3587 int i, ret = 0;
3588
3589 for (;;) {
3590 prepare_to_wait(&dev_priv->pending_flip_queue,
3591 &wait, TASK_INTERRUPTIBLE);
3592 for (i = 0; i < count; i++) {
3593 obj_priv = object_list[i]->driver_private;
3594 if (atomic_read(&obj_priv->pending_flip) > 0)
3595 break;
3596 }
3597 if (i == count)
3598 break;
3599
3600 if (!signal_pending(current)) {
3601 mutex_unlock(&dev->struct_mutex);
3602 schedule();
3603 mutex_lock(&dev->struct_mutex);
3604 continue;
3605 }
3606 ret = -ERESTARTSYS;
3607 break;
3608 }
3609 finish_wait(&dev_priv->pending_flip_queue, &wait);
3610
3611 return ret;
3612}
3613
673a394b 3614int
76446cac
JB		 3615i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3616 struct drm_file *file_priv,
3617 struct drm_i915_gem_execbuffer2 *args,
3618 struct drm_i915_gem_exec_object2 *exec_list)
673a394b
EA		 3619{
3620 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 3621 struct drm_gem_object **object_list = NULL;
3622 struct drm_gem_object *batch_obj;
b70d11da 3623 struct drm_i915_gem_object *obj_priv;
201361a5 3624 struct drm_clip_rect *cliprects = NULL;
40a5f0de 3625 struct drm_i915_gem_relocation_entry *relocs;
76446cac 3626 int ret = 0, ret2, i, pinned = 0;
673a394b 3627 uint64_t exec_offset;
40a5f0de 3628 uint32_t seqno, flush_domains, reloc_index;
6b95a207 3629 int pin_tries, flips;
673a394b
EA		 3630
3631#if WATCH_EXEC
3632 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3633 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3634#endif
3635
4f481ed2
EA		 3636 if (args->buffer_count < 1) {
3637 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3638 return -EINVAL;
3639 }
c8e0f93a 3640 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
76446cac
JB		 3641 if (object_list == NULL) {
3642 DRM_ERROR("Failed to allocate object list for %d buffers\n",
673a394b
EA		 3643 args->buffer_count);
3644 ret = -ENOMEM;
3645 goto pre_mutex_err;
3646 }
673a394b 3647
201361a5 3648 if (args->num_cliprects != 0) {
9a298b2a
EA		 3649 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3650 GFP_KERNEL);
201361a5
EA		 3651 if (cliprects == NULL)
3652 goto pre_mutex_err;
3653
3654 ret = copy_from_user(cliprects,
3655 (struct drm_clip_rect __user *)
3656 (uintptr_t) args->cliprects_ptr,
3657 sizeof(*cliprects) * args->num_cliprects);
3658 if (ret != 0) {
3659 DRM_ERROR("copy %d cliprects failed: %d\n",
3660 args->num_cliprects, ret);
3661 goto pre_mutex_err;
3662 }
3663 }
3664
40a5f0de
EA		 3665 ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3666 &relocs);
3667 if (ret != 0)
3668 goto pre_mutex_err;
3669
673a394b
EA		 3670 mutex_lock(&dev->struct_mutex);
3671
3672 i915_verify_inactive(dev, __FILE__, __LINE__);
3673
ba1234d1 3674 if (atomic_read(&dev_priv->mm.wedged)) {
673a394b 3675 mutex_unlock(&dev->struct_mutex);
a198bc80
CW		 3676 ret = -EIO;
3677 goto pre_mutex_err;
673a394b
EA		 3678 }
3679
3680 if (dev_priv->mm.suspended) {
673a394b 3681 mutex_unlock(&dev->struct_mutex);
a198bc80
CW		 3682 ret = -EBUSY;
3683 goto pre_mutex_err;
673a394b
EA		 3684 }
3685
ac94a962 3686 /* Look up object handles */
6b95a207 3687 flips = 0;
673a394b
EA		 3688 for (i = 0; i < args->buffer_count; i++) {
3689 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3690 exec_list[i].handle);
3691 if (object_list[i] == NULL) {
3692 DRM_ERROR("Invalid object handle %d at index %d\n",
3693 exec_list[i].handle, i);
3694 ret = -EBADF;
3695 goto err;
3696 }
b70d11da
KH		 3697
3698 obj_priv = object_list[i]->driver_private;
3699 if (obj_priv->in_execbuffer) {
3700 DRM_ERROR("Object %p appears more than once in object list\n",
3701 object_list[i]);
3702 ret = -EBADF;
3703 goto err;
3704 }
3705 obj_priv->in_execbuffer = true;
6b95a207
KH		 3706 flips += atomic_read(&obj_priv->pending_flip);
3707 }
3708
3709 if (flips > 0) {
3710 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3711 args->buffer_count);
3712 if (ret)
3713 goto err;
ac94a962 3714 }
673a394b 3715
ac94a962
KP		 3716 /* Pin and relocate */
3717 for (pin_tries = 0; ; pin_tries++) {
3718 ret = 0;
40a5f0de
EA		 3719 reloc_index = 0;
3720
ac94a962
KP		 3721 for (i = 0; i < args->buffer_count; i++) {
3722 object_list[i]->pending_read_domains = 0;
3723 object_list[i]->pending_write_domain = 0;
3724 ret = i915_gem_object_pin_and_relocate(object_list[i],
3725 file_priv,
40a5f0de
EA		 3726 &exec_list[i],
3727 &relocs[reloc_index]);
ac94a962
KP		 3728 if (ret)
3729 break;
3730 pinned = i + 1;
40a5f0de 3731 reloc_index += exec_list[i].relocation_count;
ac94a962
KP		 3732 }
3733 /* success */
3734 if (ret == 0)
3735 break;
3736
3737 /* error other than GTT full, or we've already tried again */
2939e1f5 3738 if (ret != -ENOSPC || pin_tries >= 1) {
07f73f69
CW		 3739 if (ret != -ERESTARTSYS) {
3740 unsigned long long total_size = 0;
3741 for (i = 0; i < args->buffer_count; i++)
3742 total_size += object_list[i]->size;
3743 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes: %d\n",
3744 pinned+1, args->buffer_count,
3745 total_size, ret);
3746 DRM_ERROR("%d objects [%d pinned], "
3747 "%d object bytes [%d pinned], "
3748 "%d/%d gtt bytes\n",
3749 atomic_read(&dev->object_count),
3750 atomic_read(&dev->pin_count),
3751 atomic_read(&dev->object_memory),
3752 atomic_read(&dev->pin_memory),
3753 atomic_read(&dev->gtt_memory),
3754 dev->gtt_total);
3755 }
673a394b
EA		 3756 goto err;
3757 }
ac94a962
KP		 3758
3759 /* unpin all of our buffers */
3760 for (i = 0; i < pinned; i++)
3761 i915_gem_object_unpin(object_list[i]);
b1177636 3762 pinned = 0;
ac94a962
KP		 3763
3764 /* evict everyone we can from the aperture */
3765 ret = i915_gem_evict_everything(dev);
07f73f69 3766 if (ret && ret != -ENOSPC)
ac94a962 3767 goto err;
673a394b
EA		 3768 }
3769
3770 /* Set the pending read domains for the batch buffer to COMMAND */
3771 batch_obj = object_list[args->buffer_count-1];
5f26a2c7
CW		 3772 if (batch_obj->pending_write_domain) {
3773 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3774 ret = -EINVAL;
3775 goto err;
3776 }
3777 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
673a394b 3778
83d60795
CW		 3779 /* Sanity check the batch buffer, prior to moving objects */
3780 exec_offset = exec_list[args->buffer_count - 1].offset;
3781 ret = i915_gem_check_execbuffer (args, exec_offset);
3782 if (ret != 0) {
3783 DRM_ERROR("execbuf with invalid offset/length\n");
3784 goto err;
3785 }
3786
673a394b
EA		 3787 i915_verify_inactive(dev, __FILE__, __LINE__);
3788
646f0f6e
KP		 3789 /* Zero the global flush/invalidate flags. These
3790 * will be modified as new domains are computed
3791 * for each object
3792 */
3793 dev->invalidate_domains = 0;
3794 dev->flush_domains = 0;
3795
673a394b
EA		 3796 for (i = 0; i < args->buffer_count; i++) {
3797 struct drm_gem_object *obj = object_list[i];
673a394b 3798
646f0f6e 3799 /* Compute new gpu domains and update invalidate/flush */
8b0e378a 3800 i915_gem_object_set_to_gpu_domain(obj);
673a394b
EA		 3801 }
3802
3803 i915_verify_inactive(dev, __FILE__, __LINE__);
3804
646f0f6e
KP		 3805 if (dev->invalidate_domains | dev->flush_domains) {
3806#if WATCH_EXEC
3807 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3808 __func__,
3809 dev->invalidate_domains,
3810 dev->flush_domains);
3811#endif
3812 i915_gem_flush(dev,
3813 dev->invalidate_domains,
3814 dev->flush_domains);
3815 if (dev->flush_domains)
b962442e
EA		 3816 (void)i915_add_request(dev, file_priv,
3817 dev->flush_domains);
646f0f6e 3818 }
673a394b 3819
efbeed96
EA		 3820 for (i = 0; i < args->buffer_count; i++) {
3821 struct drm_gem_object *obj = object_list[i];
1c5d22f7 3822 uint32_t old_write_domain = obj->write_domain;
efbeed96
EA		 3823
3824 obj->write_domain = obj->pending_write_domain;
1c5d22f7
CW		 3825 trace_i915_gem_object_change_domain(obj,
3826 obj->read_domains,
3827 old_write_domain);
efbeed96
EA		 3828 }
3829
673a394b
EA		 3830 i915_verify_inactive(dev, __FILE__, __LINE__);
3831
3832#if WATCH_COHERENCY
3833 for (i = 0; i < args->buffer_count; i++) {
3834 i915_gem_object_check_coherency(object_list[i],
3835 exec_list[i].handle);
3836 }
3837#endif
3838
673a394b 3839#if WATCH_EXEC
6911a9b8 3840 i915_gem_dump_object(batch_obj,
673a394b
EA		 3841 args->batch_len,
3842 __func__,
3843 ~0);
3844#endif
3845
673a394b 3846 /* Exec the batchbuffer */
201361a5 3847 ret = i915_dispatch_gem_execbuffer(dev, args, cliprects, exec_offset);
673a394b
EA		 3848 if (ret) {
3849 DRM_ERROR("dispatch failed %d\n", ret);
3850 goto err;
3851 }
3852
3853 /*
3854 * Ensure that the commands in the batch buffer are
3855 * finished before the interrupt fires
3856 */
3857 flush_domains = i915_retire_commands(dev);
3858
3859 i915_verify_inactive(dev, __FILE__, __LINE__);
3860
3861 /*
3862 * Get a seqno representing the execution of the current buffer,
3863 * which we can wait on. We would like to mitigate these interrupts,
3864 * likely by only creating seqnos occasionally (so that we have
3865 * *some* interrupts representing completion of buffers that we can
3866 * wait on when trying to clear up gtt space).
3867 */
b962442e 3868 seqno = i915_add_request(dev, file_priv, flush_domains);
673a394b 3869 BUG_ON(seqno == 0);
673a394b
EA		 3870 for (i = 0; i < args->buffer_count; i++) {
3871 struct drm_gem_object *obj = object_list[i];
673a394b 3872
ce44b0ea 3873 i915_gem_object_move_to_active(obj, seqno);
673a394b
EA		 3874#if WATCH_LRU
3875 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3876#endif
3877 }
3878#if WATCH_LRU
3879 i915_dump_lru(dev, __func__);
3880#endif
3881
3882 i915_verify_inactive(dev, __FILE__, __LINE__);
3883
673a394b 3884err:
aad87dff
JL		 3885 for (i = 0; i < pinned; i++)
3886 i915_gem_object_unpin(object_list[i]);
3887
b70d11da
KH		 3888 for (i = 0; i < args->buffer_count; i++) {
3889 if (object_list[i]) {
3890 obj_priv = object_list[i]->driver_private;
3891 obj_priv->in_execbuffer = false;
3892 }
aad87dff 3893 drm_gem_object_unreference(object_list[i]);
b70d11da 3894 }
673a394b 3895
673a394b
EA		 3896 mutex_unlock(&dev->struct_mutex);
3897
40a5f0de
EA		 3898 /* Copy the updated relocations out regardless of current error
3899 * state. Failure to update the relocs would mean that the next
3900 * time userland calls execbuf, it would do so with presumed offset
3901 * state that didn't match the actual object state.
3902 */
3903 ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3904 relocs);
3905 if (ret2 != 0) {
3906 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3907
3908 if (ret == 0)
3909 ret = ret2;
3910 }
3911
673a394b 3912pre_mutex_err:
8e7d2b2c 3913 drm_free_large(object_list);
9a298b2a 3914 kfree(cliprects);
673a394b
EA		 3915
3916 return ret;
3917}
3918
76446cac
JB		 3919/*
3920 * Legacy execbuffer just creates an exec2 list from the original exec object
3921 * list array and passes it to the real function.
3922 */
3923int
3924i915_gem_execbuffer(struct drm_device *dev, void *data,
3925 struct drm_file *file_priv)
3926{
3927 struct drm_i915_gem_execbuffer *args = data;
3928 struct drm_i915_gem_execbuffer2 exec2;
3929 struct drm_i915_gem_exec_object *exec_list = NULL;
3930 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3931 int ret, i;
3932
3933#if WATCH_EXEC
3934 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3935 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3936#endif
3937
3938 if (args->buffer_count < 1) {
3939 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3940 return -EINVAL;
3941 }
3942
3943 /* Copy in the exec list from userland */
3944 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3945 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3946 if (exec_list == NULL || exec2_list == NULL) {
3947 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3948 args->buffer_count);
3949 drm_free_large(exec_list);
3950 drm_free_large(exec2_list);
3951 return -ENOMEM;
3952 }
3953 ret = copy_from_user(exec_list,
3954 (struct drm_i915_relocation_entry __user *)
3955 (uintptr_t) args->buffers_ptr,
3956 sizeof(*exec_list) * args->buffer_count);
3957 if (ret != 0) {
3958 DRM_ERROR("copy %d exec entries failed %d\n",
3959 args->buffer_count, ret);
3960 drm_free_large(exec_list);
3961 drm_free_large(exec2_list);
3962 return -EFAULT;
3963 }
3964
3965 for (i = 0; i < args->buffer_count; i++) {
3966 exec2_list[i].handle = exec_list[i].handle;
3967 exec2_list[i].relocation_count = exec_list[i].relocation_count;
3968 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
3969 exec2_list[i].alignment = exec_list[i].alignment;
3970 exec2_list[i].offset = exec_list[i].offset;
3971 if (!IS_I965G(dev))
3972 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
3973 else
3974 exec2_list[i].flags = 0;
3975 }
3976
3977 exec2.buffers_ptr = args->buffers_ptr;
3978 exec2.buffer_count = args->buffer_count;
3979 exec2.batch_start_offset = args->batch_start_offset;
3980 exec2.batch_len = args->batch_len;
3981 exec2.DR1 = args->DR1;
3982 exec2.DR4 = args->DR4;
3983 exec2.num_cliprects = args->num_cliprects;
3984 exec2.cliprects_ptr = args->cliprects_ptr;
3985 exec2.flags = 0;
3986
3987 ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
3988 if (!ret) {
3989 /* Copy the new buffer offsets back to the user's exec list. */
3990 for (i = 0; i < args->buffer_count; i++)
3991 exec_list[i].offset = exec2_list[i].offset;
3992 /* ... and back out to userspace */
3993 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3994 (uintptr_t) args->buffers_ptr,
3995 exec_list,
3996 sizeof(*exec_list) * args->buffer_count);
3997 if (ret) {
3998 ret = -EFAULT;
3999 DRM_ERROR("failed to copy %d exec entries "
4000 "back to user (%d)\n",
4001 args->buffer_count, ret);
4002 }
4003 } else {
4004 DRM_ERROR("i915_gem_do_execbuffer returns %d\n", ret);
4005 }
4006
4007 drm_free_large(exec_list);
4008 drm_free_large(exec2_list);
4009 return ret;
4010}
4011
4012int
4013i915_gem_execbuffer2(struct drm_device *dev, void *data,
4014 struct drm_file *file_priv)
4015{
4016 struct drm_i915_gem_execbuffer2 *args = data;
4017 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4018 int ret;
4019
4020#if WATCH_EXEC
4021 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4022 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4023#endif
4024
4025 if (args->buffer_count < 1) {
4026 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4027 return -EINVAL;
4028 }
4029
4030 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4031 if (exec2_list == NULL) {
4032 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4033 args->buffer_count);
4034 return -ENOMEM;
4035 }
4036 ret = copy_from_user(exec2_list,
4037 (struct drm_i915_relocation_entry __user *)
4038 (uintptr_t) args->buffers_ptr,
4039 sizeof(*exec2_list) * args->buffer_count);
4040 if (ret != 0) {
4041 DRM_ERROR("copy %d exec entries failed %d\n",
4042 args->buffer_count, ret);
4043 drm_free_large(exec2_list);
4044 return -EFAULT;
4045 }
4046
4047 ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4048 if (!ret) {
4049 /* Copy the new buffer offsets back to the user's exec list. */
4050 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4051 (uintptr_t) args->buffers_ptr,
4052 exec2_list,
4053 sizeof(*exec2_list) * args->buffer_count);
4054 if (ret) {
4055 ret = -EFAULT;
4056 DRM_ERROR("failed to copy %d exec entries "
4057 "back to user (%d)\n",
4058 args->buffer_count, ret);
4059 }
4060 }
4061
4062 drm_free_large(exec2_list);
4063 return ret;
4064}
4065
673a394b
EA		 4066int
4067i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4068{
4069 struct drm_device *dev = obj->dev;
4070 struct drm_i915_gem_object *obj_priv = obj->driver_private;
4071 int ret;
4072
4073 i915_verify_inactive(dev, __FILE__, __LINE__);
4074 if (obj_priv->gtt_space == NULL) {
4075 ret = i915_gem_object_bind_to_gtt(obj, alignment);
9731129c 4076 if (ret)
673a394b 4077 return ret;
22c344e9 4078 }
76446cac 4079
673a394b
EA		 4080 obj_priv->pin_count++;
4081
4082 /* If the object is not active and not pending a flush,
4083 * remove it from the inactive list
4084 */
4085 if (obj_priv->pin_count == 1) {
4086 atomic_inc(&dev->pin_count);
4087 atomic_add(obj->size, &dev->pin_memory);
4088 if (!obj_priv->active &&
21d509e3 4089 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0 &&
673a394b
EA		 4090 !list_empty(&obj_priv->list))
4091 list_del_init(&obj_priv->list);
4092 }
4093 i915_verify_inactive(dev, __FILE__, __LINE__);
4094
4095 return 0;
4096}
4097
4098void
4099i915_gem_object_unpin(struct drm_gem_object *obj)
4100{
4101 struct drm_device *dev = obj->dev;
4102 drm_i915_private_t *dev_priv = dev->dev_private;
4103 struct drm_i915_gem_object *obj_priv = obj->driver_private;
4104
4105 i915_verify_inactive(dev, __FILE__, __LINE__);
4106 obj_priv->pin_count--;
4107 BUG_ON(obj_priv->pin_count < 0);
4108 BUG_ON(obj_priv->gtt_space == NULL);
4109
4110 /* If the object is no longer pinned, and is
4111 * neither active nor being flushed, then stick it on
4112 * the inactive list
4113 */
4114 if (obj_priv->pin_count == 0) {
4115 if (!obj_priv->active &&
21d509e3 4116 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
673a394b
EA		 4117 list_move_tail(&obj_priv->list,
4118 &dev_priv->mm.inactive_list);
4119 atomic_dec(&dev->pin_count);
4120 atomic_sub(obj->size, &dev->pin_memory);
4121 }
4122 i915_verify_inactive(dev, __FILE__, __LINE__);
4123}
4124
4125int
4126i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4127 struct drm_file *file_priv)
4128{
4129 struct drm_i915_gem_pin *args = data;
4130 struct drm_gem_object *obj;
4131 struct drm_i915_gem_object *obj_priv;
4132 int ret;
4133
4134 mutex_lock(&dev->struct_mutex);
4135
4136 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4137 if (obj == NULL) {
4138 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4139 args->handle);
4140 mutex_unlock(&dev->struct_mutex);
4141 return -EBADF;
4142 }
4143 obj_priv = obj->driver_private;
4144
bb6baf76
CW		 4145 if (obj_priv->madv != I915_MADV_WILLNEED) {
4146 DRM_ERROR("Attempting to pin a purgeable buffer\n");
3ef94daa
CW		 4147 drm_gem_object_unreference(obj);
4148 mutex_unlock(&dev->struct_mutex);
4149 return -EINVAL;
4150 }
4151
79e53945
JB		 4152 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4153 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4154 args->handle);
96dec61d 4155 drm_gem_object_unreference(obj);
673a394b 4156 mutex_unlock(&dev->struct_mutex);
79e53945
JB		 4157 return -EINVAL;
4158 }
4159
4160 obj_priv->user_pin_count++;
4161 obj_priv->pin_filp = file_priv;
4162 if (obj_priv->user_pin_count == 1) {
4163 ret = i915_gem_object_pin(obj, args->alignment);
4164 if (ret != 0) {
4165 drm_gem_object_unreference(obj);
4166 mutex_unlock(&dev->struct_mutex);
4167 return ret;
4168 }
673a394b
EA		 4169 }
4170
4171 /* XXX - flush the CPU caches for pinned objects
4172 * as the X server doesn't manage domains yet
4173 */
e47c68e9 4174 i915_gem_object_flush_cpu_write_domain(obj);
673a394b
EA		 4175 args->offset = obj_priv->gtt_offset;
4176 drm_gem_object_unreference(obj);
4177 mutex_unlock(&dev->struct_mutex);
4178
4179 return 0;
4180}
4181
4182int
4183i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4184 struct drm_file *file_priv)
4185{
4186 struct drm_i915_gem_pin *args = data;
4187 struct drm_gem_object *obj;
79e53945 4188 struct drm_i915_gem_object *obj_priv;
673a394b
EA		 4189
4190 mutex_lock(&dev->struct_mutex);
4191
4192 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4193 if (obj == NULL) {
4194 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4195 args->handle);
4196 mutex_unlock(&dev->struct_mutex);
4197 return -EBADF;
4198 }
4199
79e53945
JB		 4200 obj_priv = obj->driver_private;
4201 if (obj_priv->pin_filp != file_priv) {
4202 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4203 args->handle);
4204 drm_gem_object_unreference(obj);
4205 mutex_unlock(&dev->struct_mutex);
4206 return -EINVAL;
4207 }
4208 obj_priv->user_pin_count--;
4209 if (obj_priv->user_pin_count == 0) {
4210 obj_priv->pin_filp = NULL;
4211 i915_gem_object_unpin(obj);
4212 }
673a394b
EA		 4213
4214 drm_gem_object_unreference(obj);
4215 mutex_unlock(&dev->struct_mutex);
4216 return 0;
4217}
4218
4219int
4220i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4221 struct drm_file *file_priv)
4222{
4223 struct drm_i915_gem_busy *args = data;
4224 struct drm_gem_object *obj;
4225 struct drm_i915_gem_object *obj_priv;
4226
673a394b
EA		 4227 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4228 if (obj == NULL) {
4229 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4230 args->handle);
673a394b
EA		 4231 return -EBADF;
4232 }
4233
b1ce786c 4234 mutex_lock(&dev->struct_mutex);
f21289b3
EA		 4235 /* Update the active list for the hardware's current position.
4236 * Otherwise this only updates on a delayed timer or when irqs are
4237 * actually unmasked, and our working set ends up being larger than
4238 * required.
4239 */
4240 i915_gem_retire_requests(dev);
4241
673a394b 4242 obj_priv = obj->driver_private;
c4de0a5d
EA		 4243 /* Don't count being on the flushing list against the object being
4244 * done. Otherwise, a buffer left on the flushing list but not getting
4245 * flushed (because nobody's flushing that domain) won't ever return
4246 * unbusy and get reused by libdrm's bo cache. The other expected
4247 * consumer of this interface, OpenGL's occlusion queries, also specs
4248 * that the objects get unbusy "eventually" without any interference.
4249 */
4250 args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
673a394b
EA		 4251
4252 drm_gem_object_unreference(obj);
4253 mutex_unlock(&dev->struct_mutex);
4254 return 0;
4255}
4256
4257int
4258i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4259 struct drm_file *file_priv)
4260{
4261 return i915_gem_ring_throttle(dev, file_priv);
4262}
4263
3ef94daa
CW		 4264int
4265i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4266 struct drm_file *file_priv)
4267{
4268 struct drm_i915_gem_madvise *args = data;
4269 struct drm_gem_object *obj;
4270 struct drm_i915_gem_object *obj_priv;
4271
4272 switch (args->madv) {
4273 case I915_MADV_DONTNEED:
4274 case I915_MADV_WILLNEED:
4275 break;
4276 default:
4277 return -EINVAL;
4278 }
4279
4280 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4281 if (obj == NULL) {
4282 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4283 args->handle);
4284 return -EBADF;
4285 }
4286
4287 mutex_lock(&dev->struct_mutex);
4288 obj_priv = obj->driver_private;
4289
4290 if (obj_priv->pin_count) {
4291 drm_gem_object_unreference(obj);
4292 mutex_unlock(&dev->struct_mutex);
4293
4294 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4295 return -EINVAL;
4296 }
4297
bb6baf76
CW		 4298 if (obj_priv->madv != __I915_MADV_PURGED)
4299 obj_priv->madv = args->madv;
3ef94daa 4300
2d7ef395
CW		 4301 /* if the object is no longer bound, discard its backing storage */
4302 if (i915_gem_object_is_purgeable(obj_priv) &&
4303 obj_priv->gtt_space == NULL)
4304 i915_gem_object_truncate(obj);
4305
bb6baf76
CW		 4306 args->retained = obj_priv->madv != __I915_MADV_PURGED;
4307
3ef94daa
CW		 4308 drm_gem_object_unreference(obj);
4309 mutex_unlock(&dev->struct_mutex);
4310
4311 return 0;
4312}
4313
673a394b
EA		 4314int i915_gem_init_object(struct drm_gem_object *obj)
4315{
4316 struct drm_i915_gem_object *obj_priv;
4317
9a298b2a 4318 obj_priv = kzalloc(sizeof(*obj_priv), GFP_KERNEL);
673a394b
EA		 4319 if (obj_priv == NULL)
4320 return -ENOMEM;
4321
4322 /*
4323 * We've just allocated pages from the kernel,
4324 * so they've just been written by the CPU with
4325 * zeros. They'll need to be clflushed before we
4326 * use them with the GPU.
4327 */
4328 obj->write_domain = I915_GEM_DOMAIN_CPU;
4329 obj->read_domains = I915_GEM_DOMAIN_CPU;
4330
ba1eb1d8
KP		 4331 obj_priv->agp_type = AGP_USER_MEMORY;
4332
673a394b
EA		 4333 obj->driver_private = obj_priv;
4334 obj_priv->obj = obj;
de151cf6 4335 obj_priv->fence_reg = I915_FENCE_REG_NONE;
673a394b 4336 INIT_LIST_HEAD(&obj_priv->list);
a09ba7fa 4337 INIT_LIST_HEAD(&obj_priv->fence_list);
3ef94daa 4338 obj_priv->madv = I915_MADV_WILLNEED;
de151cf6 4339
1c5d22f7 4340 trace_i915_gem_object_create(obj);
de151cf6 4341
673a394b
EA		 4342 return 0;
4343}
4344
4345void i915_gem_free_object(struct drm_gem_object *obj)
4346{
de151cf6 4347 struct drm_device *dev = obj->dev;
673a394b
EA		 4348 struct drm_i915_gem_object *obj_priv = obj->driver_private;
4349
1c5d22f7
CW		 4350 trace_i915_gem_object_destroy(obj);
4351
673a394b
EA		 4352 while (obj_priv->pin_count > 0)
4353 i915_gem_object_unpin(obj);
4354
71acb5eb
DA		 4355 if (obj_priv->phys_obj)
4356 i915_gem_detach_phys_object(dev, obj);
4357
673a394b
EA		 4358 i915_gem_object_unbind(obj);
4359
7e616158
CW		 4360 if (obj_priv->mmap_offset)
4361 i915_gem_free_mmap_offset(obj);
de151cf6 4362
9a298b2a 4363 kfree(obj_priv->page_cpu_valid);
280b713b 4364 kfree(obj_priv->bit_17);
9a298b2a 4365 kfree(obj->driver_private);
673a394b
EA		 4366}
4367
ab5ee576 4368/** Unbinds all inactive objects. */
673a394b 4369static int
ab5ee576 4370i915_gem_evict_from_inactive_list(struct drm_device *dev)
673a394b 4371{
ab5ee576 4372 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 4373
ab5ee576
CW		 4374 while (!list_empty(&dev_priv->mm.inactive_list)) {
4375 struct drm_gem_object *obj;
4376 int ret;
673a394b 4377
ab5ee576
CW		 4378 obj = list_first_entry(&dev_priv->mm.inactive_list,
4379 struct drm_i915_gem_object,
4380 list)->obj;
673a394b
EA		 4381
4382 ret = i915_gem_object_unbind(obj);
4383 if (ret != 0) {
ab5ee576 4384 DRM_ERROR("Error unbinding object: %d\n", ret);
673a394b
EA		 4385 return ret;
4386 }
4387 }
4388
673a394b
EA		 4389 return 0;
4390}
4391
5669fcac 4392int
673a394b
EA		 4393i915_gem_idle(struct drm_device *dev)
4394{
4395 drm_i915_private_t *dev_priv = dev->dev_private;
4396 uint32_t seqno, cur_seqno, last_seqno;
4397 int stuck, ret;
4398
6dbe2772
KP		 4399 mutex_lock(&dev->struct_mutex);
4400
4401 if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) {
4402 mutex_unlock(&dev->struct_mutex);
673a394b 4403 return 0;
6dbe2772 4404 }
673a394b
EA		 4405
4406 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4407 * We need to replace this with a semaphore, or something.
4408 */
4409 dev_priv->mm.suspended = 1;
f65d9421 4410 del_timer(&dev_priv->hangcheck_timer);
673a394b 4411
6dbe2772
KP		 4412 /* Cancel the retire work handler, wait for it to finish if running
4413 */
4414 mutex_unlock(&dev->struct_mutex);
4415 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4416 mutex_lock(&dev->struct_mutex);
4417
673a394b
EA		 4418 i915_kernel_lost_context(dev);
4419
4420 /* Flush the GPU along with all non-CPU write domains
4421 */
21d509e3
CW		 4422 i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
4423 seqno = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS);
673a394b
EA		 4424
4425 if (seqno == 0) {
4426 mutex_unlock(&dev->struct_mutex);
4427 return -ENOMEM;
4428 }
4429
4430 dev_priv->mm.waiting_gem_seqno = seqno;
4431 last_seqno = 0;
4432 stuck = 0;
4433 for (;;) {
4434 cur_seqno = i915_get_gem_seqno(dev);
4435 if (i915_seqno_passed(cur_seqno, seqno))
4436 break;
4437 if (last_seqno == cur_seqno) {
4438 if (stuck++ > 100) {
4439 DRM_ERROR("hardware wedged\n");
ba1234d1 4440 atomic_set(&dev_priv->mm.wedged, 1);
673a394b
EA		 4441 DRM_WAKEUP(&dev_priv->irq_queue);
4442 break;
4443 }
4444 }
4445 msleep(10);
4446 last_seqno = cur_seqno;
4447 }
4448 dev_priv->mm.waiting_gem_seqno = 0;
4449
4450 i915_gem_retire_requests(dev);
4451
5e118f41 4452 spin_lock(&dev_priv->mm.active_list_lock);
ba1234d1 4453 if (!atomic_read(&dev_priv->mm.wedged)) {
28dfe52a
EA		 4454 /* Active and flushing should now be empty as we've
4455 * waited for a sequence higher than any pending execbuffer
4456 */
4457 WARN_ON(!list_empty(&dev_priv->mm.active_list));
4458 WARN_ON(!list_empty(&dev_priv->mm.flushing_list));
4459 /* Request should now be empty as we've also waited
4460 * for the last request in the list
4461 */
4462 WARN_ON(!list_empty(&dev_priv->mm.request_list));
4463 }
673a394b 4464
28dfe52a
EA		 4465 /* Empty the active and flushing lists to inactive. If there's
4466 * anything left at this point, it means that we're wedged and
4467 * nothing good's going to happen by leaving them there. So strip
4468 * the GPU domains and just stuff them onto inactive.
673a394b 4469 */
28dfe52a 4470 while (!list_empty(&dev_priv->mm.active_list)) {
1c5d22f7
CW		 4471 struct drm_gem_object *obj;
4472 uint32_t old_write_domain;
673a394b 4473
1c5d22f7
CW		 4474 obj = list_first_entry(&dev_priv->mm.active_list,
4475 struct drm_i915_gem_object,
4476 list)->obj;
4477 old_write_domain = obj->write_domain;
4478 obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
4479 i915_gem_object_move_to_inactive(obj);
4480
4481 trace_i915_gem_object_change_domain(obj,
4482 obj->read_domains,
4483 old_write_domain);
28dfe52a 4484 }
5e118f41 4485 spin_unlock(&dev_priv->mm.active_list_lock);
28dfe52a
EA		 4486
4487 while (!list_empty(&dev_priv->mm.flushing_list)) {
1c5d22f7
CW		 4488 struct drm_gem_object *obj;
4489 uint32_t old_write_domain;
28dfe52a 4490
1c5d22f7
CW		 4491 obj = list_first_entry(&dev_priv->mm.flushing_list,
4492 struct drm_i915_gem_object,
4493 list)->obj;
4494 old_write_domain = obj->write_domain;
4495 obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
4496 i915_gem_object_move_to_inactive(obj);
4497
4498 trace_i915_gem_object_change_domain(obj,
4499 obj->read_domains,
4500 old_write_domain);
28dfe52a
EA		 4501 }
4502
4503
4504 /* Move all inactive buffers out of the GTT. */
ab5ee576 4505 ret = i915_gem_evict_from_inactive_list(dev);
28dfe52a 4506 WARN_ON(!list_empty(&dev_priv->mm.inactive_list));
6dbe2772
KP		 4507 if (ret) {
4508 mutex_unlock(&dev->struct_mutex);
673a394b 4509 return ret;
6dbe2772 4510 }
673a394b 4511
6dbe2772
KP		 4512 i915_gem_cleanup_ringbuffer(dev);
4513 mutex_unlock(&dev->struct_mutex);
4514
673a394b
EA		 4515 return 0;
4516}
4517
4518static int
4519i915_gem_init_hws(struct drm_device *dev)
4520{
4521 drm_i915_private_t *dev_priv = dev->dev_private;
4522 struct drm_gem_object *obj;
4523 struct drm_i915_gem_object *obj_priv;
4524 int ret;
4525
4526 /* If we need a physical address for the status page, it's already
4527 * initialized at driver load time.
4528 */
4529 if (!I915_NEED_GFX_HWS(dev))
4530 return 0;
4531
4532 obj = drm_gem_object_alloc(dev, 4096);
4533 if (obj == NULL) {
4534 DRM_ERROR("Failed to allocate status page\n");
4535 return -ENOMEM;
4536 }
4537 obj_priv = obj->driver_private;
ba1eb1d8 4538 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
673a394b
EA		 4539
4540 ret = i915_gem_object_pin(obj, 4096);
4541 if (ret != 0) {
4542 drm_gem_object_unreference(obj);
4543 return ret;
4544 }
4545
4546 dev_priv->status_gfx_addr = obj_priv->gtt_offset;
673a394b 4547
856fa198 4548 dev_priv->hw_status_page = kmap(obj_priv->pages[0]);
ba1eb1d8 4549 if (dev_priv->hw_status_page == NULL) {
673a394b
EA		 4550 DRM_ERROR("Failed to map status page.\n");
4551 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
3eb2ee77 4552 i915_gem_object_unpin(obj);
673a394b
EA		 4553 drm_gem_object_unreference(obj);
4554 return -EINVAL;
4555 }
4556 dev_priv->hws_obj = obj;
673a394b
EA		 4557 memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
4558 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
ba1eb1d8 4559 I915_READ(HWS_PGA); /* posting read */
44d98a61 4560 DRM_DEBUG_DRIVER("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
673a394b
EA		 4561
4562 return 0;
4563}
4564
85a7bb98
CW		 4565static void
4566i915_gem_cleanup_hws(struct drm_device *dev)
4567{
4568 drm_i915_private_t *dev_priv = dev->dev_private;
bab2d1f6
CW		 4569 struct drm_gem_object *obj;
4570 struct drm_i915_gem_object *obj_priv;
85a7bb98
CW		 4571
4572 if (dev_priv->hws_obj == NULL)
4573 return;
4574
bab2d1f6
CW		 4575 obj = dev_priv->hws_obj;
4576 obj_priv = obj->driver_private;
4577
856fa198 4578 kunmap(obj_priv->pages[0]);
85a7bb98
CW		 4579 i915_gem_object_unpin(obj);
4580 drm_gem_object_unreference(obj);
4581 dev_priv->hws_obj = NULL;
bab2d1f6 4582
85a7bb98
CW		 4583 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
4584 dev_priv->hw_status_page = NULL;
4585
4586 /* Write high address into HWS_PGA when disabling. */
4587 I915_WRITE(HWS_PGA, 0x1ffff000);
4588}
4589
79e53945 4590int
673a394b
EA		 4591i915_gem_init_ringbuffer(struct drm_device *dev)
4592{
4593 drm_i915_private_t *dev_priv = dev->dev_private;
4594 struct drm_gem_object *obj;
4595 struct drm_i915_gem_object *obj_priv;
79e53945 4596 drm_i915_ring_buffer_t *ring = &dev_priv->ring;
673a394b 4597 int ret;
50aa253d 4598 u32 head;
673a394b
EA		 4599
4600 ret = i915_gem_init_hws(dev);
4601 if (ret != 0)
4602 return ret;
4603
4604 obj = drm_gem_object_alloc(dev, 128 * 1024);
4605 if (obj == NULL) {
4606 DRM_ERROR("Failed to allocate ringbuffer\n");
85a7bb98 4607 i915_gem_cleanup_hws(dev);
673a394b
EA		 4608 return -ENOMEM;
4609 }
4610 obj_priv = obj->driver_private;
4611
4612 ret = i915_gem_object_pin(obj, 4096);
4613 if (ret != 0) {
4614 drm_gem_object_unreference(obj);
85a7bb98 4615 i915_gem_cleanup_hws(dev);
673a394b
EA		 4616 return ret;
4617 }
4618
4619 /* Set up the kernel mapping for the ring. */
79e53945 4620 ring->Size = obj->size;
673a394b 4621
79e53945
JB		 4622 ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
4623 ring->map.size = obj->size;
4624 ring->map.type = 0;
4625 ring->map.flags = 0;
4626 ring->map.mtrr = 0;
673a394b 4627
79e53945
JB		 4628 drm_core_ioremap_wc(&ring->map, dev);
4629 if (ring->map.handle == NULL) {
673a394b
EA		 4630 DRM_ERROR("Failed to map ringbuffer.\n");
4631 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
47ed185a 4632 i915_gem_object_unpin(obj);
673a394b 4633 drm_gem_object_unreference(obj);
85a7bb98 4634 i915_gem_cleanup_hws(dev);
673a394b
EA		 4635 return -EINVAL;
4636 }
79e53945
JB		 4637 ring->ring_obj = obj;
4638 ring->virtual_start = ring->map.handle;
673a394b
EA		 4639
4640 /* Stop the ring if it's running. */
4641 I915_WRITE(PRB0_CTL, 0);
673a394b 4642 I915_WRITE(PRB0_TAIL, 0);
50aa253d 4643 I915_WRITE(PRB0_HEAD, 0);
673a394b
EA		 4644
4645 /* Initialize the ring. */
4646 I915_WRITE(PRB0_START, obj_priv->gtt_offset);
50aa253d
KP		 4647 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4648
4649 /* G45 ring initialization fails to reset head to zero */
4650 if (head != 0) {
4651 DRM_ERROR("Ring head not reset to zero "
4652 "ctl %08x head %08x tail %08x start %08x\n",
4653 I915_READ(PRB0_CTL),
4654 I915_READ(PRB0_HEAD),
4655 I915_READ(PRB0_TAIL),
4656 I915_READ(PRB0_START));
4657 I915_WRITE(PRB0_HEAD, 0);
4658
4659 DRM_ERROR("Ring head forced to zero "
4660 "ctl %08x head %08x tail %08x start %08x\n",
4661 I915_READ(PRB0_CTL),
4662 I915_READ(PRB0_HEAD),
4663 I915_READ(PRB0_TAIL),
4664 I915_READ(PRB0_START));
4665 }
4666
673a394b
EA		 4667 I915_WRITE(PRB0_CTL,
4668 ((obj->size - 4096) & RING_NR_PAGES) |
4669 RING_NO_REPORT |
4670 RING_VALID);
4671
50aa253d
KP		 4672 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4673
4674 /* If the head is still not zero, the ring is dead */
4675 if (head != 0) {
4676 DRM_ERROR("Ring initialization failed "
4677 "ctl %08x head %08x tail %08x start %08x\n",
4678 I915_READ(PRB0_CTL),
4679 I915_READ(PRB0_HEAD),
4680 I915_READ(PRB0_TAIL),
4681 I915_READ(PRB0_START));
4682 return -EIO;
4683 }
4684
673a394b 4685 /* Update our cache of the ring state */
79e53945
JB		 4686 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4687 i915_kernel_lost_context(dev);
4688 else {
4689 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4690 ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
4691 ring->space = ring->head - (ring->tail + 8);
4692 if (ring->space < 0)
4693 ring->space += ring->Size;
4694 }
673a394b
EA		 4695
4696 return 0;
4697}
4698
79e53945 4699void
673a394b
EA		 4700i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4701{
4702 drm_i915_private_t *dev_priv = dev->dev_private;
4703
4704 if (dev_priv->ring.ring_obj == NULL)
4705 return;
4706
4707 drm_core_ioremapfree(&dev_priv->ring.map, dev);
4708
4709 i915_gem_object_unpin(dev_priv->ring.ring_obj);
4710 drm_gem_object_unreference(dev_priv->ring.ring_obj);
4711 dev_priv->ring.ring_obj = NULL;
4712 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
4713
85a7bb98 4714 i915_gem_cleanup_hws(dev);
673a394b
EA		 4715}
4716
4717int
4718i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4719 struct drm_file *file_priv)
4720{
4721 drm_i915_private_t *dev_priv = dev->dev_private;
4722 int ret;
4723
79e53945
JB		 4724 if (drm_core_check_feature(dev, DRIVER_MODESET))
4725 return 0;
4726
ba1234d1 4727 if (atomic_read(&dev_priv->mm.wedged)) {
673a394b 4728 DRM_ERROR("Reenabling wedged hardware, good luck\n");
ba1234d1 4729 atomic_set(&dev_priv->mm.wedged, 0);
673a394b
EA		 4730 }
4731
673a394b 4732 mutex_lock(&dev->struct_mutex);
9bb2d6f9
EA		 4733 dev_priv->mm.suspended = 0;
4734
4735 ret = i915_gem_init_ringbuffer(dev);
d816f6ac
WF		 4736 if (ret != 0) {
4737 mutex_unlock(&dev->struct_mutex);
9bb2d6f9 4738 return ret;
d816f6ac 4739 }
9bb2d6f9 4740
5e118f41 4741 spin_lock(&dev_priv->mm.active_list_lock);
673a394b 4742 BUG_ON(!list_empty(&dev_priv->mm.active_list));
5e118f41
CW		 4743 spin_unlock(&dev_priv->mm.active_list_lock);
4744
673a394b
EA		 4745 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4746 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4747 BUG_ON(!list_empty(&dev_priv->mm.request_list));
673a394b 4748 mutex_unlock(&dev->struct_mutex);
dbb19d30
KH		 4749
4750 drm_irq_install(dev);
4751
673a394b
EA		 4752 return 0;
4753}
4754
4755int
4756i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4757 struct drm_file *file_priv)
4758{
79e53945
JB		 4759 if (drm_core_check_feature(dev, DRIVER_MODESET))
4760 return 0;
4761
dbb19d30 4762 drm_irq_uninstall(dev);
e6890f6f 4763 return i915_gem_idle(dev);
673a394b
EA		 4764}
4765
4766void
4767i915_gem_lastclose(struct drm_device *dev)
4768{
4769 int ret;
673a394b 4770
e806b495
EA		 4771 if (drm_core_check_feature(dev, DRIVER_MODESET))
4772 return;
4773
6dbe2772
KP		 4774 ret = i915_gem_idle(dev);
4775 if (ret)
4776 DRM_ERROR("failed to idle hardware: %d\n", ret);
673a394b
EA		 4777}
4778
4779void
4780i915_gem_load(struct drm_device *dev)
4781{
b5aa8a0f 4782 int i;
673a394b
EA		 4783 drm_i915_private_t *dev_priv = dev->dev_private;
4784
5e118f41 4785 spin_lock_init(&dev_priv->mm.active_list_lock);
673a394b
EA		 4786 INIT_LIST_HEAD(&dev_priv->mm.active_list);
4787 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4788 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4789 INIT_LIST_HEAD(&dev_priv->mm.request_list);
a09ba7fa 4790 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
673a394b
EA		 4791 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4792 i915_gem_retire_work_handler);
4793 dev_priv->mm.next_gem_seqno = 1;
4794
31169714
CW		 4795 spin_lock(&shrink_list_lock);
4796 list_add(&dev_priv->mm.shrink_list, &shrink_list);
4797 spin_unlock(&shrink_list_lock);
4798
de151cf6
JB		 4799 /* Old X drivers will take 0-2 for front, back, depth buffers */
4800 dev_priv->fence_reg_start = 3;
4801
0f973f27 4802 if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
de151cf6
JB		 4803 dev_priv->num_fence_regs = 16;
4804 else
4805 dev_priv->num_fence_regs = 8;
4806
b5aa8a0f
GH		 4807 /* Initialize fence registers to zero */
4808 if (IS_I965G(dev)) {
4809 for (i = 0; i < 16; i++)
4810 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4811 } else {
4812 for (i = 0; i < 8; i++)
4813 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4814 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4815 for (i = 0; i < 8; i++)
4816 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4817 }
673a394b 4818 i915_gem_detect_bit_6_swizzle(dev);
6b95a207 4819 init_waitqueue_head(&dev_priv->pending_flip_queue);
673a394b 4820}
71acb5eb
DA		 4821
4822/*
4823 * Create a physically contiguous memory object for this object
4824 * e.g. for cursor + overlay regs
4825 */
4826int i915_gem_init_phys_object(struct drm_device *dev,
4827 int id, int size)
4828{
4829 drm_i915_private_t *dev_priv = dev->dev_private;
4830 struct drm_i915_gem_phys_object *phys_obj;
4831 int ret;
4832
4833 if (dev_priv->mm.phys_objs[id - 1] || !size)
4834 return 0;
4835
9a298b2a 4836 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
71acb5eb
DA		 4837 if (!phys_obj)
4838 return -ENOMEM;
4839
4840 phys_obj->id = id;
4841
e6be8d9d 4842 phys_obj->handle = drm_pci_alloc(dev, size, 0);
71acb5eb
DA		 4843 if (!phys_obj->handle) {
4844 ret = -ENOMEM;
4845 goto kfree_obj;
4846 }
4847#ifdef CONFIG_X86
4848 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4849#endif
4850
4851 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4852
4853 return 0;
4854kfree_obj:
9a298b2a 4855 kfree(phys_obj);
71acb5eb
DA		 4856 return ret;
4857}
4858
4859void i915_gem_free_phys_object(struct drm_device *dev, int id)
4860{
4861 drm_i915_private_t *dev_priv = dev->dev_private;
4862 struct drm_i915_gem_phys_object *phys_obj;
4863
4864 if (!dev_priv->mm.phys_objs[id - 1])
4865 return;
4866
4867 phys_obj = dev_priv->mm.phys_objs[id - 1];
4868 if (phys_obj->cur_obj) {
4869 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4870 }
4871
4872#ifdef CONFIG_X86
4873 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4874#endif
4875 drm_pci_free(dev, phys_obj->handle);
4876 kfree(phys_obj);
4877 dev_priv->mm.phys_objs[id - 1] = NULL;
4878}
4879
4880void i915_gem_free_all_phys_object(struct drm_device *dev)
4881{
4882 int i;
4883
260883c8 4884 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
71acb5eb
DA		 4885 i915_gem_free_phys_object(dev, i);
4886}
4887
4888void i915_gem_detach_phys_object(struct drm_device *dev,
4889 struct drm_gem_object *obj)
4890{
4891 struct drm_i915_gem_object *obj_priv;
4892 int i;
4893 int ret;
4894 int page_count;
4895
4896 obj_priv = obj->driver_private;
4897 if (!obj_priv->phys_obj)
4898 return;
4899
856fa198 4900 ret = i915_gem_object_get_pages(obj);
71acb5eb
DA		 4901 if (ret)
4902 goto out;
4903
4904 page_count = obj->size / PAGE_SIZE;
4905
4906 for (i = 0; i < page_count; i++) {
856fa198 4907 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4908 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4909
4910 memcpy(dst, src, PAGE_SIZE);
4911 kunmap_atomic(dst, KM_USER0);
4912 }
856fa198 4913 drm_clflush_pages(obj_priv->pages, page_count);
71acb5eb 4914 drm_agp_chipset_flush(dev);
d78b47b9
CW		 4915
4916 i915_gem_object_put_pages(obj);
71acb5eb
DA		 4917out:
4918 obj_priv->phys_obj->cur_obj = NULL;
4919 obj_priv->phys_obj = NULL;
4920}
4921
4922int
4923i915_gem_attach_phys_object(struct drm_device *dev,
4924 struct drm_gem_object *obj, int id)
4925{
4926 drm_i915_private_t *dev_priv = dev->dev_private;
4927 struct drm_i915_gem_object *obj_priv;
4928 int ret = 0;
4929 int page_count;
4930 int i;
4931
4932 if (id > I915_MAX_PHYS_OBJECT)
4933 return -EINVAL;
4934
4935 obj_priv = obj->driver_private;
4936
4937 if (obj_priv->phys_obj) {
4938 if (obj_priv->phys_obj->id == id)
4939 return 0;
4940 i915_gem_detach_phys_object(dev, obj);
4941 }
4942
4943
4944 /* create a new object */
4945 if (!dev_priv->mm.phys_objs[id - 1]) {
4946 ret = i915_gem_init_phys_object(dev, id,
4947 obj->size);
4948 if (ret) {
aeb565df 4949 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
71acb5eb
DA		 4950 goto out;
4951 }
4952 }
4953
4954 /* bind to the object */
4955 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4956 obj_priv->phys_obj->cur_obj = obj;
4957
856fa198 4958 ret = i915_gem_object_get_pages(obj);
71acb5eb
DA		 4959 if (ret) {
4960 DRM_ERROR("failed to get page list\n");
4961 goto out;
4962 }
4963
4964 page_count = obj->size / PAGE_SIZE;
4965
4966 for (i = 0; i < page_count; i++) {
856fa198 4967 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4968 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4969
4970 memcpy(dst, src, PAGE_SIZE);
4971 kunmap_atomic(src, KM_USER0);
4972 }
4973
d78b47b9
CW		 4974 i915_gem_object_put_pages(obj);
4975
71acb5eb
DA		 4976 return 0;
4977out:
4978 return ret;
4979}
4980
4981static int
4982i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4983 struct drm_i915_gem_pwrite *args,
4984 struct drm_file *file_priv)
4985{
4986 struct drm_i915_gem_object *obj_priv = obj->driver_private;
4987 void *obj_addr;
4988 int ret;
4989 char __user *user_data;
4990
4991 user_data = (char __user *) (uintptr_t) args->data_ptr;
4992 obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4993
44d98a61 4994 DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
71acb5eb
DA		 4995 ret = copy_from_user(obj_addr, user_data, args->size);
4996 if (ret)
4997 return -EFAULT;
4998
4999 drm_agp_chipset_flush(dev);
5000 return 0;
5001}
b962442e
EA		 5002
5003void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
5004{
5005 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
5006
5007 /* Clean up our request list when the client is going away, so that
5008 * later retire_requests won't dereference our soon-to-be-gone
5009 * file_priv.
5010 */
5011 mutex_lock(&dev->struct_mutex);
5012 while (!list_empty(&i915_file_priv->mm.request_list))
5013 list_del_init(i915_file_priv->mm.request_list.next);
5014 mutex_unlock(&dev->struct_mutex);
5015}
31169714 5016
31169714
CW		 5017static int
5018i915_gem_shrink(int nr_to_scan, gfp_t gfp_mask)
5019{
5020 drm_i915_private_t *dev_priv, *next_dev;
5021 struct drm_i915_gem_object *obj_priv, *next_obj;
5022 int cnt = 0;
5023 int would_deadlock = 1;
5024
5025 /* "fast-path" to count number of available objects */
5026 if (nr_to_scan == 0) {
5027 spin_lock(&shrink_list_lock);
5028 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5029 struct drm_device *dev = dev_priv->dev;
5030
5031 if (mutex_trylock(&dev->struct_mutex)) {
5032 list_for_each_entry(obj_priv,
5033 &dev_priv->mm.inactive_list,
5034 list)
5035 cnt++;
5036 mutex_unlock(&dev->struct_mutex);
5037 }
5038 }
5039 spin_unlock(&shrink_list_lock);
5040
5041 return (cnt / 100) * sysctl_vfs_cache_pressure;
5042 }
5043
5044 spin_lock(&shrink_list_lock);
5045
5046 /* first scan for clean buffers */
5047 list_for_each_entry_safe(dev_priv, next_dev,
5048 &shrink_list, mm.shrink_list) {
5049 struct drm_device *dev = dev_priv->dev;
5050
5051 if (! mutex_trylock(&dev->struct_mutex))
5052 continue;
5053
5054 spin_unlock(&shrink_list_lock);
5055
5056 i915_gem_retire_requests(dev);
5057
5058 list_for_each_entry_safe(obj_priv, next_obj,
5059 &dev_priv->mm.inactive_list,
5060 list) {
5061 if (i915_gem_object_is_purgeable(obj_priv)) {
963b4836 5062 i915_gem_object_unbind(obj_priv->obj);
31169714
CW		 5063 if (--nr_to_scan <= 0)
5064 break;
5065 }
5066 }
5067
5068 spin_lock(&shrink_list_lock);
5069 mutex_unlock(&dev->struct_mutex);
5070
963b4836
CW		 5071 would_deadlock = 0;
5072
31169714
CW		 5073 if (nr_to_scan <= 0)
5074 break;
5075 }
5076
5077 /* second pass, evict/count anything still on the inactive list */
5078 list_for_each_entry_safe(dev_priv, next_dev,
5079 &shrink_list, mm.shrink_list) {
5080 struct drm_device *dev = dev_priv->dev;
5081
5082 if (! mutex_trylock(&dev->struct_mutex))
5083 continue;
5084
5085 spin_unlock(&shrink_list_lock);
5086
5087 list_for_each_entry_safe(obj_priv, next_obj,
5088 &dev_priv->mm.inactive_list,
5089 list) {
5090 if (nr_to_scan > 0) {
963b4836 5091 i915_gem_object_unbind(obj_priv->obj);
31169714
CW		 5092 nr_to_scan--;
5093 } else
5094 cnt++;
5095 }
5096
5097 spin_lock(&shrink_list_lock);
5098 mutex_unlock(&dev->struct_mutex);
5099
5100 would_deadlock = 0;
5101 }
5102
5103 spin_unlock(&shrink_list_lock);
5104
5105 if (would_deadlock)
5106 return -1;
5107 else if (cnt > 0)
5108 return (cnt / 100) * sysctl_vfs_cache_pressure;
5109 else
5110 return 0;
5111}
5112
5113static struct shrinker shrinker = {
5114 .shrink = i915_gem_shrink,
5115 .seeks = DEFAULT_SEEKS,
5116};
5117
5118__init void
5119i915_gem_shrinker_init(void)
5120{
5121 register_shrinker(&shrinker);
5122}
5123
5124__exit void
5125i915_gem_shrinker_exit(void)
5126{
5127 unregister_shrinker(&shrinker);
5128}