drm/i915: Store a direct lookup from object handle to vma
authorChris Wilson <chris@chris-wilson.co.uk>
Fri, 16 Jun 2017 14:05:16 +0000 (15:05 +0100)
committerChris Wilson <chris@chris-wilson.co.uk>
Fri, 16 Jun 2017 15:54:04 +0000 (16:54 +0100)
The advent of full-ppgtt lead to an extra indirection between the object
and its binding. That extra indirection has a noticeable impact on how
fast we can convert from the user handles to our internal vma for
execbuffer. In order to bypass the extra indirection, we use a
resizable hashtable to jump from the object to the per-ctx vma.
rhashtable was considered but we don't need the online resizing feature
and the extra complexity proved to undermine its usefulness. Instead, we
simply reallocate the hastable on demand in a background task and
serialize it before iterating.

In non-full-ppgtt modes, multiple files and multiple contexts can share
the same vma. This leads to having multiple possible handle->vma links,
so we only use the first to establish the fast path. The majority of
buffers are not shared and so we should still be able to realise
speedups with multiple clients.

v2: Prettier names, more magic.
v3: Many style tweaks, most notably hiding the misuse of execobj[].rsvd2

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
drivers/gpu/drm/i915/i915_debugfs.c
drivers/gpu/drm/i915/i915_drv.h
drivers/gpu/drm/i915/i915_gem.c
drivers/gpu/drm/i915/i915_gem_context.c
drivers/gpu/drm/i915/i915_gem_context.h
drivers/gpu/drm/i915/i915_gem_execbuffer.c
drivers/gpu/drm/i915/i915_gem_object.h
drivers/gpu/drm/i915/i915_utils.h
drivers/gpu/drm/i915/i915_vma.c
drivers/gpu/drm/i915/i915_vma.h
drivers/gpu/drm/i915/selftests/mock_context.c

index a6ba2100bb88ebb63a28242ee3055a95e1a20f82..4577b0af6886f8b3a9b4b12ba540c8e492142f0d 100644 (file)
@@ -1998,6 +1998,12 @@ static int i915_context_status(struct seq_file *m, void *unused)
                        seq_putc(m, '\n');
                }
 
+               seq_printf(m,
+                          "\tvma hashtable size=%u (actual %lu), count=%u\n",
+                          ctx->vma_lut.ht_size,
+                          BIT(ctx->vma_lut.ht_bits),
+                          ctx->vma_lut.ht_count);
+
                seq_putc(m, '\n');
        }
 
index a1b2672cfe568577951b966608f1081f66544417..af2a5467239654288c167105ad7aaee3fee80385 100644 (file)
@@ -37,7 +37,7 @@
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
 #include <linux/backlight.h>
-#include <linux/hashtable.h>
+#include <linux/hash.h>
 #include <linux/intel-iommu.h>
 #include <linux/kref.h>
 #include <linux/pm_qos.h>
index 4ae30f74c475ed955b93a2dc1f6d3533076f8bcf..fcdc452f28bb6cb81e0eab7c1db3cc2551c85a9b 100644 (file)
@@ -3261,6 +3261,10 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
                if (vma->vm->file == fpriv)
                        i915_vma_close(vma);
 
+       vma = obj->vma_hashed;
+       if (vma && vma->ctx->file_priv == fpriv)
+               i915_vma_unlink_ctx(vma);
+
        if (i915_gem_object_is_active(obj) &&
            !i915_gem_object_has_active_reference(obj)) {
                i915_gem_object_set_active_reference(obj);
@@ -4254,7 +4258,6 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
 
        INIT_LIST_HEAD(&obj->global_link);
        INIT_LIST_HEAD(&obj->userfault_link);
-       INIT_LIST_HEAD(&obj->obj_exec_link);
        INIT_LIST_HEAD(&obj->vma_list);
        INIT_LIST_HEAD(&obj->batch_pool_link);
 
index 81c73dee095f503f05a25f5778ee9e5fc8d5f2bc..23f74014e158b5cb01ff11944353bd0c141189fa 100644 (file)
@@ -85,6 +85,7 @@
  *
  */
 
+#include <linux/log2.h>
 #include <drm/drmP.h>
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
 
 #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1
 
+/* Initial size (as log2) to preallocate the handle->object hashtable */
+#define VMA_HT_BITS 2u /* 4 x 2 pointers, 64 bytes minimum */
+
+static void resize_vma_ht(struct work_struct *work)
+{
+       struct i915_gem_context_vma_lut *lut =
+               container_of(work, typeof(*lut), resize);
+       unsigned int bits, new_bits, size, i;
+       struct hlist_head *new_ht;
+
+       GEM_BUG_ON(!(lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS));
+
+       bits = 1 + ilog2(4*lut->ht_count/3 + 1);
+       new_bits = min_t(unsigned int,
+                        max(bits, VMA_HT_BITS),
+                        sizeof(unsigned int) * BITS_PER_BYTE - 1);
+       if (new_bits == lut->ht_bits)
+               goto out;
+
+       new_ht = kzalloc(sizeof(*new_ht)<<new_bits, GFP_KERNEL | __GFP_NOWARN);
+       if (!new_ht)
+               new_ht = vzalloc(sizeof(*new_ht)<<new_bits);
+       if (!new_ht)
+               /* Pretend resize succeeded and stop calling us for a bit! */
+               goto out;
+
+       size = BIT(lut->ht_bits);
+       for (i = 0; i < size; i++) {
+               struct i915_vma *vma;
+               struct hlist_node *tmp;
+
+               hlist_for_each_entry_safe(vma, tmp, &lut->ht[i], ctx_node)
+                       hlist_add_head(&vma->ctx_node,
+                                      &new_ht[hash_32(vma->ctx_handle,
+                                                      new_bits)]);
+       }
+       kvfree(lut->ht);
+       lut->ht = new_ht;
+       lut->ht_bits = new_bits;
+out:
+       smp_store_release(&lut->ht_size, BIT(bits));
+       GEM_BUG_ON(lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS);
+}
+
+static void vma_lut_free(struct i915_gem_context *ctx)
+{
+       struct i915_gem_context_vma_lut *lut = &ctx->vma_lut;
+       unsigned int i, size;
+
+       if (lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS)
+               cancel_work_sync(&lut->resize);
+
+       size = BIT(lut->ht_bits);
+       for (i = 0; i < size; i++) {
+               struct i915_vma *vma;
+
+               hlist_for_each_entry(vma, &lut->ht[i], ctx_node) {
+                       vma->obj->vma_hashed = NULL;
+                       vma->ctx = NULL;
+               }
+       }
+       kvfree(lut->ht);
+}
+
 void i915_gem_context_free(struct kref *ctx_ref)
 {
        struct i915_gem_context *ctx = container_of(ctx_ref, typeof(*ctx), ref);
@@ -101,6 +166,7 @@ void i915_gem_context_free(struct kref *ctx_ref)
        trace_i915_context_free(ctx);
        GEM_BUG_ON(!i915_gem_context_is_closed(ctx));
 
+       vma_lut_free(ctx);
        i915_ppgtt_put(ctx->ppgtt);
 
        for (i = 0; i < I915_NUM_ENGINES; i++) {
@@ -118,6 +184,7 @@ void i915_gem_context_free(struct kref *ctx_ref)
 
        kfree(ctx->name);
        put_pid(ctx->pid);
+
        list_del(&ctx->link);
 
        ida_simple_remove(&ctx->i915->context_hw_ida, ctx->hw_id);
@@ -201,13 +268,24 @@ __create_hw_context(struct drm_i915_private *dev_priv,
        ctx->i915 = dev_priv;
        ctx->priority = I915_PRIORITY_NORMAL;
 
+       ctx->vma_lut.ht_bits = VMA_HT_BITS;
+       ctx->vma_lut.ht_size = BIT(VMA_HT_BITS);
+       BUILD_BUG_ON(BIT(VMA_HT_BITS) == I915_CTX_RESIZE_IN_PROGRESS);
+       ctx->vma_lut.ht = kcalloc(ctx->vma_lut.ht_size,
+                                 sizeof(*ctx->vma_lut.ht),
+                                 GFP_KERNEL);
+       if (!ctx->vma_lut.ht)
+               goto err_out;
+
+       INIT_WORK(&ctx->vma_lut.resize, resize_vma_ht);
+
        /* Default context will never have a file_priv */
        ret = DEFAULT_CONTEXT_HANDLE;
        if (file_priv) {
                ret = idr_alloc(&file_priv->context_idr, ctx,
                                DEFAULT_CONTEXT_HANDLE, 0, GFP_KERNEL);
                if (ret < 0)
-                       goto err_out;
+                       goto err_lut;
        }
        ctx->user_handle = ret;
 
@@ -248,6 +326,8 @@ __create_hw_context(struct drm_i915_private *dev_priv,
 err_pid:
        put_pid(ctx->pid);
        idr_remove(&file_priv->context_idr, ctx->user_handle);
+err_lut:
+       kvfree(ctx->vma_lut.ht);
 err_out:
        context_close(ctx);
        return ERR_PTR(ret);
index 4af2ab94558bc4f28bf8a25c973accfe858f464c..82c99ba92ad386b3e1ad60a1b4e2966a2761d16d 100644 (file)
@@ -143,6 +143,32 @@ struct i915_gem_context {
        /** ggtt_offset_bias: placement restriction for context objects */
        u32 ggtt_offset_bias;
 
+       struct i915_gem_context_vma_lut {
+               /** ht_size: last request size to allocate the hashtable for. */
+               unsigned int ht_size;
+#define I915_CTX_RESIZE_IN_PROGRESS BIT(0)
+               /** ht_bits: real log2(size) of hashtable. */
+               unsigned int ht_bits;
+               /** ht_count: current number of entries inside the hashtable */
+               unsigned int ht_count;
+
+               /** ht: the array of buckets comprising the simple hashtable */
+               struct hlist_head *ht;
+
+               /**
+                * resize: After an execbuf completes, we check the load factor
+                * of the hashtable. If the hashtable is too full, or too empty,
+                * we schedule a task to resize the hashtable. During the
+                * resize, the entries are moved between different buckets and
+                * so we cannot simultaneously read the hashtable as it is
+                * being resized (unlike rhashtable). Therefore we treat the
+                * active work as a strong barrier, pausing a subsequent
+                * execbuf to wait for the resize worker to complete, if
+                * required.
+                */
+               struct work_struct resize;
+       } vma_lut;
+
        /** engine: per-engine logical HW state */
        struct intel_context {
                struct i915_vma *state;
index d6099d08474867959caabdea7d64d120569f500f..b06f561a268f93a198cad59bdcbfddd22c8c6e7a 100644 (file)
@@ -75,37 +75,42 @@ struct i915_execbuffer {
                unsigned int page;
                bool use_64bit_reloc : 1;
        } reloc_cache;
-       int and;
-       union {
-               struct i915_vma **lut;
-               struct hlist_head *buckets;
-       };
+       int lut_mask;
+       struct hlist_head *buckets;
 };
 
+/*
+ * As an alternative to creating a hashtable of handle-to-vma for a batch,
+ * we used the last available reserved field in the execobject[] and stash
+ * a link from the execobj to its vma.
+ */
+#define __exec_to_vma(ee) (ee)->rsvd2
+#define exec_to_vma(ee) u64_to_ptr(struct i915_vma, __exec_to_vma(ee))
+
 static int eb_create(struct i915_execbuffer *eb)
 {
-       eb->lut = NULL;
-       if (eb->args->flags & I915_EXEC_HANDLE_LUT) {
-               unsigned int size = eb->args->buffer_count;
-               size *= sizeof(struct i915_vma *);
-               eb->lut = kmalloc(size,
-                                 GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
-       }
-
-       if (!eb->lut) {
-               unsigned int size = eb->args->buffer_count;
-               unsigned int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
-               BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head));
-               while (count > 2*size)
-                       count >>= 1;
-               eb->lut = kzalloc(count * sizeof(struct hlist_head),
-                                 GFP_TEMPORARY);
-               if (!eb->lut)
-                       return -ENOMEM;
-
-               eb->and = count - 1;
+       if ((eb->args->flags & I915_EXEC_HANDLE_LUT) == 0) {
+               unsigned int size = 1 + ilog2(eb->args->buffer_count);
+
+               do {
+                       eb->buckets = kzalloc(sizeof(struct hlist_head) << size,
+                                             GFP_TEMPORARY |
+                                             __GFP_NORETRY |
+                                             __GFP_NOWARN);
+                       if (eb->buckets)
+                               break;
+               } while (--size);
+
+               if (unlikely(!eb->buckets)) {
+                       eb->buckets = kzalloc(sizeof(struct hlist_head),
+                                             GFP_TEMPORARY);
+                       if (unlikely(!eb->buckets))
+                               return -ENOMEM;
+               }
+
+               eb->lut_mask = size;
        } else {
-               eb->and = -eb->args->buffer_count;
+               eb->lut_mask = -eb->args->buffer_count;
        }
 
        return 0;
@@ -142,73 +147,112 @@ eb_reset(struct i915_execbuffer *eb)
                vma->exec_entry = NULL;
        }
 
-       if (eb->and >= 0)
-               memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
+       if (eb->lut_mask >= 0)
+               memset(eb->buckets, 0,
+                      sizeof(struct hlist_head) << eb->lut_mask);
 }
 
-static struct i915_vma *
-eb_get_batch(struct i915_execbuffer *eb)
+static bool
+eb_add_vma(struct i915_execbuffer *eb, struct i915_vma *vma, int i)
 {
-       struct i915_vma *vma = list_entry(eb->vmas.prev, typeof(*vma), exec_link);
+       if (unlikely(vma->exec_entry)) {
+               DRM_DEBUG("Object [handle %d, index %d] appears more than once in object list\n",
+                         eb->exec[i].handle, i);
+               return false;
+       }
+       list_add_tail(&vma->exec_link, &eb->vmas);
 
-       /*
-        * SNA is doing fancy tricks with compressing batch buffers, which leads
-        * to negative relocation deltas. Usually that works out ok since the
-        * relocate address is still positive, except when the batch is placed
-        * very low in the GTT. Ensure this doesn't happen.
-        *
-        * Note that actual hangs have only been observed on gen7, but for
-        * paranoia do it everywhere.
-        */
-       if ((vma->exec_entry->flags & EXEC_OBJECT_PINNED) == 0)
-               vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
+       vma->exec_entry = &eb->exec[i];
+       if (eb->lut_mask >= 0) {
+               vma->exec_handle = eb->exec[i].handle;
+               hlist_add_head(&vma->exec_node,
+                              &eb->buckets[hash_32(vma->exec_handle,
+                                                   eb->lut_mask)]);
+       }
 
-       return vma;
+       i915_vma_get(vma);
+       __exec_to_vma(&eb->exec[i]) = (uintptr_t)vma;
+       return true;
+}
+
+static inline struct hlist_head *
+ht_head(const struct i915_gem_context *ctx, u32 handle)
+{
+       return &ctx->vma_lut.ht[hash_32(handle, ctx->vma_lut.ht_bits)];
+}
+
+static inline bool
+ht_needs_resize(const struct i915_gem_context *ctx)
+{
+       return (4*ctx->vma_lut.ht_count > 3*ctx->vma_lut.ht_size ||
+               4*ctx->vma_lut.ht_count + 1 < ctx->vma_lut.ht_size);
 }
 
 static int
 eb_lookup_vmas(struct i915_execbuffer *eb)
 {
-       struct drm_i915_gem_object *obj;
-       struct list_head objects;
-       int i, ret;
+#define INTERMEDIATE BIT(0)
+       const int count = eb->args->buffer_count;
+       struct i915_vma *vma;
+       int slow_pass = -1;
+       int i;
 
        INIT_LIST_HEAD(&eb->vmas);
 
-       INIT_LIST_HEAD(&objects);
+       if (unlikely(eb->ctx->vma_lut.ht_size & I915_CTX_RESIZE_IN_PROGRESS))
+               flush_work(&eb->ctx->vma_lut.resize);
+       GEM_BUG_ON(eb->ctx->vma_lut.ht_size & I915_CTX_RESIZE_IN_PROGRESS);
+
+       for (i = 0; i < count; i++) {
+               __exec_to_vma(&eb->exec[i]) = 0;
+
+               hlist_for_each_entry(vma,
+                                    ht_head(eb->ctx, eb->exec[i].handle),
+                                    ctx_node) {
+                       if (vma->ctx_handle != eb->exec[i].handle)
+                               continue;
+
+                       if (!eb_add_vma(eb, vma, i))
+                               return -EINVAL;
+
+                       goto next_vma;
+               }
+
+               if (slow_pass < 0)
+                       slow_pass = i;
+next_vma: ;
+       }
+
+       if (slow_pass < 0)
+               return 0;
+
        spin_lock(&eb->file->table_lock);
        /* Grab a reference to the object and release the lock so we can lookup
         * or create the VMA without using GFP_ATOMIC */
-       for (i = 0; i < eb->args->buffer_count; i++) {
-               obj = to_intel_bo(idr_find(&eb->file->object_idr, eb->exec[i].handle));
-               if (obj == NULL) {
-                       spin_unlock(&eb->file->table_lock);
-                       DRM_DEBUG("Invalid object handle %d at index %d\n",
-                                  eb->exec[i].handle, i);
-                       ret = -ENOENT;
-                       goto err;
-               }
+       for (i = slow_pass; i < count; i++) {
+               struct drm_i915_gem_object *obj;
 
-               if (!list_empty(&obj->obj_exec_link)) {
+               if (__exec_to_vma(&eb->exec[i]))
+                       continue;
+
+               obj = to_intel_bo(idr_find(&eb->file->object_idr,
+                                          eb->exec[i].handle));
+               if (unlikely(!obj)) {
                        spin_unlock(&eb->file->table_lock);
-                       DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
-                                  obj, eb->exec[i].handle, i);
-                       ret = -EINVAL;
-                       goto err;
+                       DRM_DEBUG("Invalid object handle %d at index %d\n",
+                                 eb->exec[i].handle, i);
+                       return -ENOENT;
                }
 
-               i915_gem_object_get(obj);
-               list_add_tail(&obj->obj_exec_link, &objects);
+               __exec_to_vma(&eb->exec[i]) = INTERMEDIATE | (uintptr_t)obj;
        }
        spin_unlock(&eb->file->table_lock);
 
-       i = 0;
-       while (!list_empty(&objects)) {
-               struct i915_vma *vma;
+       for (i = slow_pass; i < count; i++) {
+               struct drm_i915_gem_object *obj;
 
-               obj = list_first_entry(&objects,
-                                      struct drm_i915_gem_object,
-                                      obj_exec_link);
+               if ((__exec_to_vma(&eb->exec[i]) & INTERMEDIATE) == 0)
+                       continue;
 
                /*
                 * NOTE: We can leak any vmas created here when something fails
@@ -218,61 +262,73 @@ eb_lookup_vmas(struct i915_execbuffer *eb)
                 * from the (obj, vm) we don't run the risk of creating
                 * duplicated vmas for the same vm.
                 */
+               obj = u64_to_ptr(struct drm_i915_gem_object,
+                                __exec_to_vma(&eb->exec[i]) & ~INTERMEDIATE);
                vma = i915_vma_instance(obj, eb->vm, NULL);
                if (unlikely(IS_ERR(vma))) {
                        DRM_DEBUG("Failed to lookup VMA\n");
-                       ret = PTR_ERR(vma);
-                       goto err;
+                       return PTR_ERR(vma);
                }
 
-               /* Transfer ownership from the objects list to the vmas list. */
-               list_add_tail(&vma->exec_link, &eb->vmas);
-               list_del_init(&obj->obj_exec_link);
-
-               vma->exec_entry = &eb->exec[i];
-               if (eb->and < 0) {
-                       eb->lut[i] = vma;
-               } else {
-                       u32 handle =
-                               eb->args->flags & I915_EXEC_HANDLE_LUT ?
-                               i : eb->exec[i].handle;
-                       vma->exec_handle = handle;
-                       hlist_add_head(&vma->exec_node,
-                                      &eb->buckets[handle & eb->and]);
+               /* First come, first served */
+               if (!vma->ctx) {
+                       vma->ctx = eb->ctx;
+                       vma->ctx_handle = eb->exec[i].handle;
+                       hlist_add_head(&vma->ctx_node,
+                                      ht_head(eb->ctx, eb->exec[i].handle));
+                       eb->ctx->vma_lut.ht_count++;
+                       if (i915_vma_is_ggtt(vma)) {
+                               GEM_BUG_ON(obj->vma_hashed);
+                               obj->vma_hashed = vma;
+                       }
                }
-               ++i;
+
+               if (!eb_add_vma(eb, vma, i))
+                       return -EINVAL;
+       }
+
+       if (ht_needs_resize(eb->ctx)) {
+               eb->ctx->vma_lut.ht_size |= I915_CTX_RESIZE_IN_PROGRESS;
+               queue_work(system_highpri_wq, &eb->ctx->vma_lut.resize);
        }
 
        return 0;
+#undef INTERMEDIATE
+}
 
+static struct i915_vma *
+eb_get_batch(struct i915_execbuffer *eb)
+{
+       struct i915_vma *vma =
+               exec_to_vma(&eb->exec[eb->args->buffer_count - 1]);
 
-err:
-       while (!list_empty(&objects)) {
-               obj = list_first_entry(&objects,
-                                      struct drm_i915_gem_object,
-                                      obj_exec_link);
-               list_del_init(&obj->obj_exec_link);
-               i915_gem_object_put(obj);
-       }
        /*
-        * Objects already transfered to the vmas list will be unreferenced by
-        * eb_destroy.
+        * SNA is doing fancy tricks with compressing batch buffers, which leads
+        * to negative relocation deltas. Usually that works out ok since the
+        * relocate address is still positive, except when the batch is placed
+        * very low in the GTT. Ensure this doesn't happen.
+        *
+        * Note that actual hangs have only been observed on gen7, but for
+        * paranoia do it everywhere.
         */
+       if ((vma->exec_entry->flags & EXEC_OBJECT_PINNED) == 0)
+               vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
 
-       return ret;
+       return vma;
 }
 
-static struct i915_vma *eb_get_vma(struct i915_execbuffer *eb, unsigned long handle)
+static struct i915_vma *
+eb_get_vma(struct i915_execbuffer *eb, unsigned long handle)
 {
-       if (eb->and < 0) {
-               if (handle >= -eb->and)
+       if (eb->lut_mask < 0) {
+               if (handle >= -eb->lut_mask)
                        return NULL;
-               return eb->lut[handle];
+               return exec_to_vma(&eb->exec[handle]);
        } else {
                struct hlist_head *head;
                struct i915_vma *vma;
 
-               head = &eb->buckets[handle & eb->and];
+               head = &eb->buckets[hash_32(handle, eb->lut_mask)];
                hlist_for_each_entry(vma, head, exec_node) {
                        if (vma->exec_handle == handle)
                                return vma;
@@ -296,7 +352,7 @@ static void eb_destroy(struct i915_execbuffer *eb)
 
        i915_gem_context_put(eb->ctx);
 
-       if (eb->buckets)
+       if (eb->lut_mask >= 0)
                kfree(eb->buckets);
 }
 
@@ -916,7 +972,7 @@ static int eb_reserve(struct i915_execbuffer *eb)
                need_fence =
                        (entry->flags & EXEC_OBJECT_NEEDS_FENCE ||
                         needs_unfenced_map) &&
-                       i915_gem_object_is_tiled(obj);
+                       i915_gem_object_is_tiled(vma->obj);
                need_mappable = need_fence || need_reloc_mappable(vma);
 
                if (entry->flags & EXEC_OBJECT_PINNED)
index adb482b00271df454d12825fab5063e22e544cdd..5b19a4916a4d71cc74768c265fb16dc8ca9ac4c3 100644 (file)
@@ -86,6 +86,7 @@ struct drm_i915_gem_object {
         * They are also added to @vma_list for easy iteration.
         */
        struct rb_root vma_tree;
+       struct i915_vma *vma_hashed;
 
        /** Stolen memory for this object, instead of being backed by shmem. */
        struct drm_mm_node *stolen;
@@ -100,9 +101,6 @@ struct drm_i915_gem_object {
         */
        struct list_head userfault_link;
 
-       /** Used in execbuf to temporarily hold a ref */
-       struct list_head obj_exec_link;
-
        struct list_head batch_pool_link;
        I915_SELFTEST_DECLARE(struct list_head st_link);
 
index 16ecd1ab108d48f6d7b036801a32f40c2e9ccf5d..12fc250b47b963a24217cfbc90bf5c1d4fe92e57 100644 (file)
        __T;                                                            \
 })
 
+#define u64_to_ptr(T, x) ({                                            \
+       typecheck(u64, x);                                              \
+       (T *)(uintptr_t)(x);                                            \
+})
+
 #define __mask_next_bit(mask) ({                                       \
        int __idx = ffs(mask) - 1;                                      \
        mask &= ~BIT(__idx);                                            \
index bbc8309743a0a6c55c155b874ff3a625c2b7f624..ce68194ebff6f32291f1eedbc91428438ee480bf 100644 (file)
@@ -590,11 +590,31 @@ static void i915_vma_destroy(struct i915_vma *vma)
        kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
 }
 
+void i915_vma_unlink_ctx(struct i915_vma *vma)
+{
+       struct i915_gem_context *ctx = vma->ctx;
+
+       if (ctx->vma_lut.ht_size & I915_CTX_RESIZE_IN_PROGRESS) {
+               cancel_work_sync(&ctx->vma_lut.resize);
+               ctx->vma_lut.ht_size &= ~I915_CTX_RESIZE_IN_PROGRESS;
+       }
+
+       __hlist_del(&vma->ctx_node);
+       ctx->vma_lut.ht_count--;
+
+       if (i915_vma_is_ggtt(vma))
+               vma->obj->vma_hashed = NULL;
+       vma->ctx = NULL;
+}
+
 void i915_vma_close(struct i915_vma *vma)
 {
        GEM_BUG_ON(i915_vma_is_closed(vma));
        vma->flags |= I915_VMA_CLOSED;
 
+       if (vma->ctx)
+               i915_vma_unlink_ctx(vma);
+
        list_del(&vma->obj_link);
        rb_erase(&vma->obj_node, &vma->obj->vma_tree);
 
index 11ce83a8adf03bf565e1b157a5b3e1e27a800134..ea98e6e4262f354284ead306b3fcac03ad95a8cf 100644 (file)
@@ -99,6 +99,7 @@ struct i915_vma {
 
        struct list_head obj_link; /* Link in the object's VMA list */
        struct rb_node obj_node;
+       struct hlist_node obj_hash;
 
        /** This vma's place in the execbuf reservation list */
        struct list_head exec_link;
@@ -110,8 +111,12 @@ struct i915_vma {
         * Used for performing relocations during execbuffer insertion.
         */
        struct hlist_node exec_node;
-       unsigned long exec_handle;
        struct drm_i915_gem_exec_object2 *exec_entry;
+       u32 exec_handle;
+
+       struct i915_gem_context *ctx;
+       struct hlist_node ctx_node;
+       u32 ctx_handle;
 };
 
 struct i915_vma *
@@ -235,6 +240,7 @@ bool i915_vma_misplaced(const struct i915_vma *vma,
                        u64 size, u64 alignment, u64 flags);
 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);
 int __must_check i915_vma_unbind(struct i915_vma *vma);
+void i915_vma_unlink_ctx(struct i915_vma *vma);
 void i915_vma_close(struct i915_vma *vma);
 
 int __i915_vma_do_pin(struct i915_vma *vma,
index 8d3a90c3f8ac20305ce4c02136f235eb3b39f392..f8b9cc212b02df33cf06ee9c0d87ae433ebc6a6c 100644 (file)
@@ -40,10 +40,18 @@ mock_context(struct drm_i915_private *i915,
        INIT_LIST_HEAD(&ctx->link);
        ctx->i915 = i915;
 
+       ctx->vma_lut.ht_bits = VMA_HT_BITS;
+       ctx->vma_lut.ht_size = BIT(VMA_HT_BITS);
+       ctx->vma_lut.ht = kcalloc(ctx->vma_lut.ht_size,
+                                 sizeof(*ctx->vma_lut.ht),
+                                 GFP_KERNEL);
+       if (!ctx->vma_lut.ht)
+               goto err_free;
+
        ret = ida_simple_get(&i915->context_hw_ida,
                             0, MAX_CONTEXT_HW_ID, GFP_KERNEL);
        if (ret < 0)
-               goto err_free;
+               goto err_vma_ht;
        ctx->hw_id = ret;
 
        if (name) {
@@ -58,6 +66,8 @@ mock_context(struct drm_i915_private *i915,
 
        return ctx;
 
+err_vma_ht:
+       kvfree(ctx->vma_lut.ht);
 err_free:
        kfree(ctx);
        return NULL;