drivers/gpu/drm/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <drm/drmP.h>
  26 #include <drm/i915_drm.h>
  27 #include "i915_drv.h"
  28 #include "i915_trace.h"
  29 #include "intel_drv.h"
  30
  31 /* PPGTT support for Sandybdrige/Gen6 and later */
  32 static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
  33                                    unsigned first_entry,
  34                                    unsigned num_entries)
  35 {
  36         uint32_t *pt_vaddr;
  37         uint32_t scratch_pte;
  38         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
  39         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
  40         unsigned last_pte, i;
  41
  42         scratch_pte = GEN6_PTE_ADDR_ENCODE(ppgtt->scratch_page_dma_addr);
  43         scratch_pte |= GEN6_PTE_VALID | GEN6_PTE_CACHE_LLC;
  44
  45         while (num_entries) {
  46                 last_pte = first_pte + num_entries;
  47                 if (last_pte > I915_PPGTT_PT_ENTRIES)
  48                         last_pte = I915_PPGTT_PT_ENTRIES;
  49
  50                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
  51
  52                 for (i = first_pte; i < last_pte; i++)
  53                         pt_vaddr[i] = scratch_pte;
  54
  55                 kunmap_atomic(pt_vaddr);
  56
  57                 num_entries -= last_pte - first_pte;
  58                 first_pte = 0;
  59                 act_pd++;
  60         }
  61 }
  62
  63 int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
  64 {
  65         struct drm_i915_private *dev_priv = dev->dev_private;
  66         struct i915_hw_ppgtt *ppgtt;
  67         unsigned first_pd_entry_in_global_pt;
  68         int i;
  69         int ret = -ENOMEM;
  70
  71         /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
  72          * entries. For aliasing ppgtt support we just steal them at the end for
  73          * now. */
  74         first_pd_entry_in_global_pt = dev_priv->mm.gtt->gtt_total_entries - I915_PPGTT_PD_ENTRIES;
  75
  76         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
  77         if (!ppgtt)
  78                 return ret;
  79
  80         ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
  81         ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
  82                                   GFP_KERNEL);
  83         if (!ppgtt->pt_pages)
  84                 goto err_ppgtt;
  85
  86         for (i = 0; i < ppgtt->num_pd_entries; i++) {
  87                 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
  88                 if (!ppgtt->pt_pages[i])
  89                         goto err_pt_alloc;
  90         }
  91
  92         if (dev_priv->mm.gtt->needs_dmar) {
  93                 ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t)
  94                                                 *ppgtt->num_pd_entries,
  95                                              GFP_KERNEL);
  96                 if (!ppgtt->pt_dma_addr)
  97                         goto err_pt_alloc;
  98
  99                 for (i = 0; i < ppgtt->num_pd_entries; i++) {
 100                         dma_addr_t pt_addr;
 101
 102                         pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i],
 103                                                0, 4096,
 104                                                PCI_DMA_BIDIRECTIONAL);
 105
 106                         if (pci_dma_mapping_error(dev->pdev,
 107                                                   pt_addr)) {
 108                                 ret = -EIO;
 109                                 goto err_pd_pin;
 110
 111                         }
 112                         ppgtt->pt_dma_addr[i] = pt_addr;
 113                 }
 114         }
 115
 116         ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt->scratch_page_dma;
 117
 118         i915_ppgtt_clear_range(ppgtt, 0,
 119                                ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
 120
 121         ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(uint32_t);
 122
 123         dev_priv->mm.aliasing_ppgtt = ppgtt;
 124
 125         return 0;
 126
 127 err_pd_pin:
 128         if (ppgtt->pt_dma_addr) {
 129                 for (i--; i >= 0; i--)
 130                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
 131                                        4096, PCI_DMA_BIDIRECTIONAL);
 132         }
 133 err_pt_alloc:
 134         kfree(ppgtt->pt_dma_addr);
 135         for (i = 0; i < ppgtt->num_pd_entries; i++) {
 136                 if (ppgtt->pt_pages[i])
 137                         __free_page(ppgtt->pt_pages[i]);
 138         }
 139         kfree(ppgtt->pt_pages);
 140 err_ppgtt:
 141         kfree(ppgtt);
 142
 143         return ret;
 144 }
 145
 146 void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
 147 {
 148         struct drm_i915_private *dev_priv = dev->dev_private;
 149         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
 150         int i;
 151
 152         if (!ppgtt)
 153                 return;
 154
 155         if (ppgtt->pt_dma_addr) {
 156                 for (i = 0; i < ppgtt->num_pd_entries; i++)
 157                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
 158                                        4096, PCI_DMA_BIDIRECTIONAL);
 159         }
 160
 161         kfree(ppgtt->pt_dma_addr);
 162         for (i = 0; i < ppgtt->num_pd_entries; i++)
 163                 __free_page(ppgtt->pt_pages[i]);
 164         kfree(ppgtt->pt_pages);
 165         kfree(ppgtt);
 166 }
 167
 168 static void i915_ppgtt_insert_sg_entries(struct i915_hw_ppgtt *ppgtt,
 169                                          struct scatterlist *sg_list,
 170                                          unsigned sg_len,
 171                                          unsigned first_entry,
 172                                          uint32_t pte_flags)
 173 {
 174         uint32_t *pt_vaddr, pte;
 175         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
 176         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 177         unsigned i, j, m, segment_len;
 178         dma_addr_t page_addr;
 179         struct scatterlist *sg;
 180
 181         /* init sg walking */
 182         sg = sg_list;
 183         i = 0;
 184         segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
 185         m = 0;
 186
 187         while (i < sg_len) {
 188                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
 189
 190                 for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) {
 191                         page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
 192                         pte = GEN6_PTE_ADDR_ENCODE(page_addr);
 193                         pt_vaddr[j] = pte | pte_flags;
 194
 195                         /* grab the next page */
 196                         m++;
 197                         if (m == segment_len) {
 198                                 sg = sg_next(sg);
 199                                 i++;
 200                                 if (i == sg_len)
 201                                         break;
 202
 203                                 segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
 204                                 m = 0;
 205                         }
 206                 }
 207
 208                 kunmap_atomic(pt_vaddr);
 209
 210                 first_pte = 0;
 211                 act_pd++;
 212         }
 213 }
 214
 215 static void i915_ppgtt_insert_pages(struct i915_hw_ppgtt *ppgtt,
 216                                     unsigned first_entry, unsigned num_entries,
 217                                     struct page **pages, uint32_t pte_flags)
 218 {
 219         uint32_t *pt_vaddr, pte;
 220         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
 221         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 222         unsigned last_pte, i;
 223         dma_addr_t page_addr;
 224
 225         while (num_entries) {
 226                 last_pte = first_pte + num_entries;
 227                 last_pte = min_t(unsigned, last_pte, I915_PPGTT_PT_ENTRIES);
 228
 229                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
 230
 231                 for (i = first_pte; i < last_pte; i++) {
 232                         page_addr = page_to_phys(*pages);
 233                         pte = GEN6_PTE_ADDR_ENCODE(page_addr);
 234                         pt_vaddr[i] = pte | pte_flags;
 235
 236                         pages++;
 237                 }
 238
 239                 kunmap_atomic(pt_vaddr);
 240
 241                 num_entries -= last_pte - first_pte;
 242                 first_pte = 0;
 243                 act_pd++;
 244         }
 245 }
 246
 247 void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
 248                             struct drm_i915_gem_object *obj,
 249                             enum i915_cache_level cache_level)
 250 {
 251         struct drm_device *dev = obj->base.dev;
 252         struct drm_i915_private *dev_priv = dev->dev_private;
 253         uint32_t pte_flags = GEN6_PTE_VALID;
 254
 255         switch (cache_level) {
 256         case I915_CACHE_LLC_MLC:
 257                 pte_flags |= GEN6_PTE_CACHE_LLC_MLC;
 258                 break;
 259         case I915_CACHE_LLC:
 260                 pte_flags |= GEN6_PTE_CACHE_LLC;
 261                 break;
 262         case I915_CACHE_NONE:
 263                 if (IS_HASWELL(dev))
 264                         pte_flags |= HSW_PTE_UNCACHED;
 265                 else
 266                         pte_flags |= GEN6_PTE_UNCACHED;
 267                 break;
 268         default:
 269                 BUG();
 270         }
 271
 272         if (obj->sg_table) {
 273                 i915_ppgtt_insert_sg_entries(ppgtt,
 274                                              obj->sg_table->sgl,
 275                                              obj->sg_table->nents,
 276                                              obj->gtt_space->start >> PAGE_SHIFT,
 277                                              pte_flags);
 278         } else if (dev_priv->mm.gtt->needs_dmar) {
 279                 BUG_ON(!obj->sg_list);
 280
 281                 i915_ppgtt_insert_sg_entries(ppgtt,
 282                                              obj->sg_list,
 283                                              obj->num_sg,
 284                                              obj->gtt_space->start >> PAGE_SHIFT,
 285                                              pte_flags);
 286         } else
 287                 i915_ppgtt_insert_pages(ppgtt,
 288                                         obj->gtt_space->start >> PAGE_SHIFT,
 289                                         obj->base.size >> PAGE_SHIFT,
 290                                         obj->pages,
 291                                         pte_flags);
 292 }
 293
 294 void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
 295                               struct drm_i915_gem_object *obj)
 296 {
 297         i915_ppgtt_clear_range(ppgtt,
 298                                obj->gtt_space->start >> PAGE_SHIFT,
 299                                obj->base.size >> PAGE_SHIFT);
 300 }
 301
 302 /* XXX kill agp_type! */
 303 static unsigned int cache_level_to_agp_type(struct drm_device *dev,
 304                                             enum i915_cache_level cache_level)
 305 {
 306         switch (cache_level) {
 307         case I915_CACHE_LLC_MLC:
 308                 if (INTEL_INFO(dev)->gen >= 6)
 309                         return AGP_USER_CACHED_MEMORY_LLC_MLC;
 310                 /* Older chipsets do not have this extra level of CPU
 311                  * cacheing, so fallthrough and request the PTE simply
 312                  * as cached.
 313                  */
 314         case I915_CACHE_LLC:
 315                 return AGP_USER_CACHED_MEMORY;
 316         default:
 317         case I915_CACHE_NONE:
 318                 return AGP_USER_MEMORY;
 319         }
 320 }
 321
 322 static bool do_idling(struct drm_i915_private *dev_priv)
 323 {
 324         bool ret = dev_priv->mm.interruptible;
 325
 326         if (unlikely(dev_priv->mm.gtt->do_idle_maps)) {
 327                 dev_priv->mm.interruptible = false;
 328                 if (i915_gpu_idle(dev_priv->dev)) {
 329                         DRM_ERROR("Couldn't idle GPU\n");
 330                         /* Wait a bit, in hopes it avoids the hang */
 331                         udelay(10);
 332                 }
 333         }
 334
 335         return ret;
 336 }
 337
 338 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
 339 {
 340         if (unlikely(dev_priv->mm.gtt->do_idle_maps))
 341                 dev_priv->mm.interruptible = interruptible;
 342 }
 343
 344 void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 345 {
 346         struct drm_i915_private *dev_priv = dev->dev_private;
 347         struct drm_i915_gem_object *obj;
 348
 349         /* First fill our portion of the GTT with scratch pages */
 350         intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE,
 351                               (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
 352
 353         list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
 354                 i915_gem_clflush_object(obj);
 355                 i915_gem_gtt_bind_object(obj, obj->cache_level);
 356         }
 357
 358         intel_gtt_chipset_flush();
 359 }
 360
 361 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
 362 {
 363         struct drm_device *dev = obj->base.dev;
 364         struct drm_i915_private *dev_priv = dev->dev_private;
 365
 366         /* don't map imported dma buf objects */
 367         if (dev_priv->mm.gtt->needs_dmar && !obj->sg_table)
 368                 return intel_gtt_map_memory(obj->pages,
 369                                             obj->base.size >> PAGE_SHIFT,
 370                                             &obj->sg_list,
 371                                             &obj->num_sg);
 372         else
 373                 return 0;
 374 }
 375
 376 void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
 377                               enum i915_cache_level cache_level)
 378 {
 379         struct drm_device *dev = obj->base.dev;
 380         struct drm_i915_private *dev_priv = dev->dev_private;
 381         unsigned int agp_type = cache_level_to_agp_type(dev, cache_level);
 382
 383         if (obj->sg_table) {
 384                 intel_gtt_insert_sg_entries(obj->sg_table->sgl,
 385                                             obj->sg_table->nents,
 386                                             obj->gtt_space->start >> PAGE_SHIFT,
 387                                             agp_type);
 388         } else if (dev_priv->mm.gtt->needs_dmar) {
 389                 BUG_ON(!obj->sg_list);
 390
 391                 intel_gtt_insert_sg_entries(obj->sg_list,
 392                                             obj->num_sg,
 393                                             obj->gtt_space->start >> PAGE_SHIFT,
 394                                             agp_type);
 395         } else
 396                 intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT,
 397                                        obj->base.size >> PAGE_SHIFT,
 398                                        obj->pages,
 399                                        agp_type);
 400
 401         obj->has_global_gtt_mapping = 1;
 402 }
 403
 404 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
 405 {
 406         intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT,
 407                               obj->base.size >> PAGE_SHIFT);
 408
 409         obj->has_global_gtt_mapping = 0;
 410 }
 411
 412 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
 413 {
 414         struct drm_device *dev = obj->base.dev;
 415         struct drm_i915_private *dev_priv = dev->dev_private;
 416         bool interruptible;
 417
 418         interruptible = do_idling(dev_priv);
 419
 420         if (obj->sg_list) {
 421                 intel_gtt_unmap_memory(obj->sg_list, obj->num_sg);
 422                 obj->sg_list = NULL;
 423         }
 424
 425         undo_idling(dev_priv, interruptible);
 426 }
 427
 428 void i915_gem_init_global_gtt(struct drm_device *dev,
 429                               unsigned long start,
 430                               unsigned long mappable_end,
 431                               unsigned long end)
 432 {
 433         drm_i915_private_t *dev_priv = dev->dev_private;
 434
 435         /* Substract the guard page ... */
 436         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE);
 437
 438         dev_priv->mm.gtt_start = start;
 439         dev_priv->mm.gtt_mappable_end = mappable_end;
 440         dev_priv->mm.gtt_end = end;
 441         dev_priv->mm.gtt_total = end - start;
 442         dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
 443
 444         /* ... but ensure that we clear the entire range. */
 445         intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
 446 }