2 * drivers/gpu/drm/omapdrm/omap_gem.c
4 * Copyright (C) 2011 Texas Instruments
5 * Author: Rob Clark <rob.clark@linaro.org>
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 as published by
9 * the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * You should have received a copy of the GNU General Public License along with
17 * this program. If not, see <http://www.gnu.org/licenses/>.
20 #include <linux/seq_file.h>
21 #include <linux/shmem_fs.h>
22 #include <linux/spinlock.h>
23 #include <linux/pfn_t.h>
25 #include <drm/drm_vma_manager.h>
28 #include "omap_dmm_tiler.h"
31 * GEM buffer object implementation.
34 /* note: we use upper 8 bits of flags for driver-internal flags: */
35 #define OMAP_BO_MEM_DMA_API 0x01000000 /* memory allocated with the dma_alloc_* API */
36 #define OMAP_BO_MEM_SHMEM 0x02000000 /* memory allocated through shmem backing */
37 #define OMAP_BO_MEM_DMABUF 0x08000000 /* memory imported from a dmabuf */
39 struct omap_gem_object
{
40 struct drm_gem_object base
;
42 struct list_head mm_list
;
46 /** width/height for tiled formats (rounded up to slot boundaries) */
47 uint16_t width
, height
;
49 /** roll applied when mapping to DMM */
53 * paddr contains the buffer DMA address. It is valid for
55 * - buffers allocated through the DMA mapping API (with the
56 * OMAP_BO_MEM_DMA_API flag set)
58 * - buffers imported from dmabuf (with the OMAP_BO_MEM_DMABUF flag set)
59 * if they are physically contiguous (when sgt->orig_nents == 1)
61 * - buffers mapped through the TILER when paddr_cnt is not zero, in
62 * which case the DMA address points to the TILER aperture
64 * Physically contiguous buffers have their DMA address equal to the
65 * physical address as we don't remap those buffers through the TILER.
67 * Buffers mapped to the TILER have their DMA address pointing to the
68 * TILER aperture. As TILER mappings are refcounted (through paddr_cnt)
69 * the DMA address must be accessed through omap_get_get_paddr() to
70 * ensure that the mapping won't disappear unexpectedly. References must
71 * be released with omap_gem_put_paddr().
81 * If the buffer has been imported from a dmabuf the OMAP_DB_DMABUF flag
82 * is set and the sgt field is valid.
87 * tiler block used when buffer is remapped in DMM/TILER.
89 struct tiler_block
*block
;
92 * Array of backing pages, if allocated. Note that pages are never
93 * allocated for buffers originally allocated from contiguous memory
97 /** addresses corresponding to pages in above array */
101 * Virtual address, if mapped.
106 * sync-object allocated on demand (if needed)
108 * Per-buffer sync-object for tracking pending and completed hw/dma
109 * read and write operations.
112 uint32_t write_pending
;
113 uint32_t write_complete
;
114 uint32_t read_pending
;
115 uint32_t read_complete
;
119 #define to_omap_bo(x) container_of(x, struct omap_gem_object, base)
121 /* To deal with userspace mmap'ings of 2d tiled buffers, which (a) are
122 * not necessarily pinned in TILER all the time, and (b) when they are
123 * they are not necessarily page aligned, we reserve one or more small
124 * regions in each of the 2d containers to use as a user-GART where we
125 * can create a second page-aligned mapping of parts of the buffer
126 * being accessed from userspace.
128 * Note that we could optimize slightly when we know that multiple
129 * tiler containers are backed by the same PAT.. but I'll leave that
132 #define NUM_USERGART_ENTRIES 2
133 struct omap_drm_usergart_entry
{
134 struct tiler_block
*block
; /* the reserved tiler block */
136 struct drm_gem_object
*obj
; /* the current pinned obj */
137 pgoff_t obj_pgoff
; /* page offset of obj currently
141 struct omap_drm_usergart
{
142 struct omap_drm_usergart_entry entry
[NUM_USERGART_ENTRIES
];
143 int height
; /* height in rows */
144 int height_shift
; /* ilog2(height in rows) */
145 int slot_shift
; /* ilog2(width per slot) */
146 int stride_pfn
; /* stride in pages */
147 int last
; /* index of last used entry */
150 /* -----------------------------------------------------------------------------
154 /** get mmap offset */
155 static uint64_t mmap_offset(struct drm_gem_object
*obj
)
157 struct drm_device
*dev
= obj
->dev
;
161 WARN_ON(!mutex_is_locked(&dev
->struct_mutex
));
163 /* Make it mmapable */
164 size
= omap_gem_mmap_size(obj
);
165 ret
= drm_gem_create_mmap_offset_size(obj
, size
);
167 dev_err(dev
->dev
, "could not allocate mmap offset\n");
171 return drm_vma_node_offset_addr(&obj
->vma_node
);
174 static bool is_contiguous(struct omap_gem_object
*omap_obj
)
176 if (omap_obj
->flags
& OMAP_BO_MEM_DMA_API
)
179 if ((omap_obj
->flags
& OMAP_BO_MEM_DMABUF
) && omap_obj
->sgt
->nents
== 1)
185 /* -----------------------------------------------------------------------------
189 static void evict_entry(struct drm_gem_object
*obj
,
190 enum tiler_fmt fmt
, struct omap_drm_usergart_entry
*entry
)
192 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
193 struct omap_drm_private
*priv
= obj
->dev
->dev_private
;
194 int n
= priv
->usergart
[fmt
].height
;
195 size_t size
= PAGE_SIZE
* n
;
196 loff_t off
= mmap_offset(obj
) +
197 (entry
->obj_pgoff
<< PAGE_SHIFT
);
198 const int m
= 1 + ((omap_obj
->width
<< fmt
) / PAGE_SIZE
);
202 /* if stride > than PAGE_SIZE then sparse mapping: */
203 for (i
= n
; i
> 0; i
--) {
204 unmap_mapping_range(obj
->dev
->anon_inode
->i_mapping
,
206 off
+= PAGE_SIZE
* m
;
209 unmap_mapping_range(obj
->dev
->anon_inode
->i_mapping
,
216 /* Evict a buffer from usergart, if it is mapped there */
217 static void evict(struct drm_gem_object
*obj
)
219 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
220 struct omap_drm_private
*priv
= obj
->dev
->dev_private
;
222 if (omap_obj
->flags
& OMAP_BO_TILED
) {
223 enum tiler_fmt fmt
= gem2fmt(omap_obj
->flags
);
226 for (i
= 0; i
< NUM_USERGART_ENTRIES
; i
++) {
227 struct omap_drm_usergart_entry
*entry
=
228 &priv
->usergart
[fmt
].entry
[i
];
230 if (entry
->obj
== obj
)
231 evict_entry(obj
, fmt
, entry
);
236 /* -----------------------------------------------------------------------------
240 /** ensure backing pages are allocated */
241 static int omap_gem_attach_pages(struct drm_gem_object
*obj
)
243 struct drm_device
*dev
= obj
->dev
;
244 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
246 int npages
= obj
->size
>> PAGE_SHIFT
;
250 WARN_ON(omap_obj
->pages
);
252 pages
= drm_gem_get_pages(obj
);
254 dev_err(obj
->dev
->dev
, "could not get pages: %ld\n", PTR_ERR(pages
));
255 return PTR_ERR(pages
);
258 /* for non-cached buffers, ensure the new pages are clean because
259 * DSS, GPU, etc. are not cache coherent:
261 if (omap_obj
->flags
& (OMAP_BO_WC
|OMAP_BO_UNCACHED
)) {
262 addrs
= kmalloc(npages
* sizeof(*addrs
), GFP_KERNEL
);
268 for (i
= 0; i
< npages
; i
++) {
269 addrs
[i
] = dma_map_page(dev
->dev
, pages
[i
],
270 0, PAGE_SIZE
, DMA_BIDIRECTIONAL
);
272 if (dma_mapping_error(dev
->dev
, addrs
[i
])) {
274 "%s: failed to map page\n", __func__
);
276 for (i
= i
- 1; i
>= 0; --i
) {
277 dma_unmap_page(dev
->dev
, addrs
[i
],
278 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
286 addrs
= kzalloc(npages
* sizeof(*addrs
), GFP_KERNEL
);
293 omap_obj
->addrs
= addrs
;
294 omap_obj
->pages
= pages
;
301 drm_gem_put_pages(obj
, pages
, true, false);
306 /* acquire pages when needed (for example, for DMA where physically
307 * contiguous buffer is not required
309 static int get_pages(struct drm_gem_object
*obj
, struct page
***pages
)
311 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
314 if ((omap_obj
->flags
& OMAP_BO_MEM_SHMEM
) && !omap_obj
->pages
) {
315 ret
= omap_gem_attach_pages(obj
);
317 dev_err(obj
->dev
->dev
, "could not attach pages\n");
322 /* TODO: even phys-contig.. we should have a list of pages? */
323 *pages
= omap_obj
->pages
;
328 /** release backing pages */
329 static void omap_gem_detach_pages(struct drm_gem_object
*obj
)
331 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
333 /* for non-cached buffers, ensure the new pages are clean because
334 * DSS, GPU, etc. are not cache coherent:
336 if (omap_obj
->flags
& (OMAP_BO_WC
|OMAP_BO_UNCACHED
)) {
337 int i
, npages
= obj
->size
>> PAGE_SHIFT
;
338 for (i
= 0; i
< npages
; i
++) {
339 if (omap_obj
->addrs
[i
])
340 dma_unmap_page(obj
->dev
->dev
,
342 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
346 kfree(omap_obj
->addrs
);
347 omap_obj
->addrs
= NULL
;
349 drm_gem_put_pages(obj
, omap_obj
->pages
, true, false);
350 omap_obj
->pages
= NULL
;
353 /* get buffer flags */
354 uint32_t omap_gem_flags(struct drm_gem_object
*obj
)
356 return to_omap_bo(obj
)->flags
;
359 uint64_t omap_gem_mmap_offset(struct drm_gem_object
*obj
)
362 mutex_lock(&obj
->dev
->struct_mutex
);
363 offset
= mmap_offset(obj
);
364 mutex_unlock(&obj
->dev
->struct_mutex
);
369 size_t omap_gem_mmap_size(struct drm_gem_object
*obj
)
371 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
372 size_t size
= obj
->size
;
374 if (omap_obj
->flags
& OMAP_BO_TILED
) {
375 /* for tiled buffers, the virtual size has stride rounded up
376 * to 4kb.. (to hide the fact that row n+1 might start 16kb or
377 * 32kb later!). But we don't back the entire buffer with
378 * pages, only the valid picture part.. so need to adjust for
379 * this in the size used to mmap and generate mmap offset
381 size
= tiler_vsize(gem2fmt(omap_obj
->flags
),
382 omap_obj
->width
, omap_obj
->height
);
388 /* -----------------------------------------------------------------------------
392 /* Normal handling for the case of faulting in non-tiled buffers */
393 static int fault_1d(struct drm_gem_object
*obj
,
394 struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
396 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
400 /* We don't use vmf->pgoff since that has the fake offset: */
401 pgoff
= (vmf
->address
- vma
->vm_start
) >> PAGE_SHIFT
;
403 if (omap_obj
->pages
) {
404 omap_gem_cpu_sync(obj
, pgoff
);
405 pfn
= page_to_pfn(omap_obj
->pages
[pgoff
]);
407 BUG_ON(!is_contiguous(omap_obj
));
408 pfn
= (omap_obj
->paddr
>> PAGE_SHIFT
) + pgoff
;
411 VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf
->address
,
412 pfn
, pfn
<< PAGE_SHIFT
);
414 return vm_insert_mixed(vma
, vmf
->address
, __pfn_to_pfn_t(pfn
, PFN_DEV
));
417 /* Special handling for the case of faulting in 2d tiled buffers */
418 static int fault_2d(struct drm_gem_object
*obj
,
419 struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
421 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
422 struct omap_drm_private
*priv
= obj
->dev
->dev_private
;
423 struct omap_drm_usergart_entry
*entry
;
424 enum tiler_fmt fmt
= gem2fmt(omap_obj
->flags
);
425 struct page
*pages
[64]; /* XXX is this too much to have on stack? */
427 pgoff_t pgoff
, base_pgoff
;
432 * Note the height of the slot is also equal to the number of pages
433 * that need to be mapped in to fill 4kb wide CPU page. If the slot
434 * height is 64, then 64 pages fill a 4kb wide by 64 row region.
436 const int n
= priv
->usergart
[fmt
].height
;
437 const int n_shift
= priv
->usergart
[fmt
].height_shift
;
440 * If buffer width in bytes > PAGE_SIZE then the virtual stride is
441 * rounded up to next multiple of PAGE_SIZE.. this need to be taken
442 * into account in some of the math, so figure out virtual stride
445 const int m
= 1 + ((omap_obj
->width
<< fmt
) / PAGE_SIZE
);
447 /* We don't use vmf->pgoff since that has the fake offset: */
448 pgoff
= (vmf
->address
- vma
->vm_start
) >> PAGE_SHIFT
;
451 * Actual address we start mapping at is rounded down to previous slot
452 * boundary in the y direction:
454 base_pgoff
= round_down(pgoff
, m
<< n_shift
);
456 /* figure out buffer width in slots */
457 slots
= omap_obj
->width
>> priv
->usergart
[fmt
].slot_shift
;
459 vaddr
= vmf
->address
- ((pgoff
- base_pgoff
) << PAGE_SHIFT
);
461 entry
= &priv
->usergart
[fmt
].entry
[priv
->usergart
[fmt
].last
];
463 /* evict previous buffer using this usergart entry, if any: */
465 evict_entry(entry
->obj
, fmt
, entry
);
468 entry
->obj_pgoff
= base_pgoff
;
470 /* now convert base_pgoff to phys offset from virt offset: */
471 base_pgoff
= (base_pgoff
>> n_shift
) * slots
;
473 /* for wider-than 4k.. figure out which part of the slot-row we want: */
476 entry
->obj_pgoff
+= off
;
478 slots
= min(slots
- (off
<< n_shift
), n
);
479 base_pgoff
+= off
<< n_shift
;
480 vaddr
+= off
<< PAGE_SHIFT
;
484 * Map in pages. Beyond the valid pixel part of the buffer, we set
485 * pages[i] to NULL to get a dummy page mapped in.. if someone
486 * reads/writes it they will get random/undefined content, but at
487 * least it won't be corrupting whatever other random page used to
488 * be mapped in, or other undefined behavior.
490 memcpy(pages
, &omap_obj
->pages
[base_pgoff
],
491 sizeof(struct page
*) * slots
);
492 memset(pages
+ slots
, 0,
493 sizeof(struct page
*) * (n
- slots
));
495 ret
= tiler_pin(entry
->block
, pages
, ARRAY_SIZE(pages
), 0, true);
497 dev_err(obj
->dev
->dev
, "failed to pin: %d\n", ret
);
501 pfn
= entry
->paddr
>> PAGE_SHIFT
;
503 VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf
->address
,
504 pfn
, pfn
<< PAGE_SHIFT
);
506 for (i
= n
; i
> 0; i
--) {
507 vm_insert_mixed(vma
, vaddr
, __pfn_to_pfn_t(pfn
, PFN_DEV
));
508 pfn
+= priv
->usergart
[fmt
].stride_pfn
;
509 vaddr
+= PAGE_SIZE
* m
;
512 /* simple round-robin: */
513 priv
->usergart
[fmt
].last
= (priv
->usergart
[fmt
].last
+ 1)
514 % NUM_USERGART_ENTRIES
;
520 * omap_gem_fault - pagefault handler for GEM objects
523 * Invoked when a fault occurs on an mmap of a GEM managed area. GEM
524 * does most of the work for us including the actual map/unmap calls
525 * but we need to do the actual page work.
527 * The VMA was set up by GEM. In doing so it also ensured that the
528 * vma->vm_private_data points to the GEM object that is backing this
531 int omap_gem_fault(struct vm_fault
*vmf
)
533 struct vm_area_struct
*vma
= vmf
->vma
;
534 struct drm_gem_object
*obj
= vma
->vm_private_data
;
535 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
536 struct drm_device
*dev
= obj
->dev
;
540 /* Make sure we don't parallel update on a fault, nor move or remove
541 * something from beneath our feet
543 mutex_lock(&dev
->struct_mutex
);
545 /* if a shmem backed object, make sure we have pages attached now */
546 ret
= get_pages(obj
, &pages
);
550 /* where should we do corresponding put_pages().. we are mapping
551 * the original page, rather than thru a GART, so we can't rely
552 * on eviction to trigger this. But munmap() or all mappings should
553 * probably trigger put_pages()?
556 if (omap_obj
->flags
& OMAP_BO_TILED
)
557 ret
= fault_2d(obj
, vma
, vmf
);
559 ret
= fault_1d(obj
, vma
, vmf
);
563 mutex_unlock(&dev
->struct_mutex
);
570 * EBUSY is ok: this just means that another thread
571 * already did the job.
573 return VM_FAULT_NOPAGE
;
577 return VM_FAULT_SIGBUS
;
581 /** We override mainly to fix up some of the vm mapping flags.. */
582 int omap_gem_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
586 ret
= drm_gem_mmap(filp
, vma
);
588 DBG("mmap failed: %d", ret
);
592 return omap_gem_mmap_obj(vma
->vm_private_data
, vma
);
595 int omap_gem_mmap_obj(struct drm_gem_object
*obj
,
596 struct vm_area_struct
*vma
)
598 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
600 vma
->vm_flags
&= ~VM_PFNMAP
;
601 vma
->vm_flags
|= VM_MIXEDMAP
;
603 if (omap_obj
->flags
& OMAP_BO_WC
) {
604 vma
->vm_page_prot
= pgprot_writecombine(vm_get_page_prot(vma
->vm_flags
));
605 } else if (omap_obj
->flags
& OMAP_BO_UNCACHED
) {
606 vma
->vm_page_prot
= pgprot_noncached(vm_get_page_prot(vma
->vm_flags
));
609 * We do have some private objects, at least for scanout buffers
610 * on hardware without DMM/TILER. But these are allocated write-
613 if (WARN_ON(!obj
->filp
))
617 * Shunt off cached objs to shmem file so they have their own
618 * address_space (so unmap_mapping_range does what we want,
619 * in particular in the case of mmap'd dmabufs)
623 vma
->vm_file
= get_file(obj
->filp
);
625 vma
->vm_page_prot
= vm_get_page_prot(vma
->vm_flags
);
631 /* -----------------------------------------------------------------------------
636 * omap_gem_dumb_create - create a dumb buffer
637 * @drm_file: our client file
639 * @args: the requested arguments copied from userspace
641 * Allocate a buffer suitable for use for a frame buffer of the
642 * form described by user space. Give userspace a handle by which
645 int omap_gem_dumb_create(struct drm_file
*file
, struct drm_device
*dev
,
646 struct drm_mode_create_dumb
*args
)
648 union omap_gem_size gsize
;
650 args
->pitch
= DIV_ROUND_UP(args
->width
* args
->bpp
, 8);
652 args
->size
= PAGE_ALIGN(args
->pitch
* args
->height
);
654 gsize
= (union omap_gem_size
){
658 return omap_gem_new_handle(dev
, file
, gsize
,
659 OMAP_BO_SCANOUT
| OMAP_BO_WC
, &args
->handle
);
663 * omap_gem_dumb_map - buffer mapping for dumb interface
664 * @file: our drm client file
666 * @handle: GEM handle to the object (from dumb_create)
668 * Do the necessary setup to allow the mapping of the frame buffer
669 * into user memory. We don't have to do much here at the moment.
671 int omap_gem_dumb_map_offset(struct drm_file
*file
, struct drm_device
*dev
,
672 uint32_t handle
, uint64_t *offset
)
674 struct drm_gem_object
*obj
;
677 /* GEM does all our handle to object mapping */
678 obj
= drm_gem_object_lookup(file
, handle
);
684 *offset
= omap_gem_mmap_offset(obj
);
686 drm_gem_object_unreference_unlocked(obj
);
692 #ifdef CONFIG_DRM_FBDEV_EMULATION
693 /* Set scrolling position. This allows us to implement fast scrolling
696 * Call only from non-atomic contexts.
698 int omap_gem_roll(struct drm_gem_object
*obj
, uint32_t roll
)
700 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
701 uint32_t npages
= obj
->size
>> PAGE_SHIFT
;
705 dev_err(obj
->dev
->dev
, "invalid roll: %d\n", roll
);
709 omap_obj
->roll
= roll
;
711 mutex_lock(&obj
->dev
->struct_mutex
);
713 /* if we aren't mapped yet, we don't need to do anything */
714 if (omap_obj
->block
) {
716 ret
= get_pages(obj
, &pages
);
719 ret
= tiler_pin(omap_obj
->block
, pages
, npages
, roll
, true);
721 dev_err(obj
->dev
->dev
, "could not repin: %d\n", ret
);
725 mutex_unlock(&obj
->dev
->struct_mutex
);
731 /* -----------------------------------------------------------------------------
732 * Memory Management & DMA Sync
736 * shmem buffers that are mapped cached can simulate coherency via using
737 * page faulting to keep track of dirty pages
739 static inline bool is_cached_coherent(struct drm_gem_object
*obj
)
741 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
743 return (omap_obj
->flags
& OMAP_BO_MEM_SHMEM
) &&
744 ((omap_obj
->flags
& OMAP_BO_CACHE_MASK
) == OMAP_BO_CACHED
);
747 /* Sync the buffer for CPU access.. note pages should already be
748 * attached, ie. omap_gem_get_pages()
750 void omap_gem_cpu_sync(struct drm_gem_object
*obj
, int pgoff
)
752 struct drm_device
*dev
= obj
->dev
;
753 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
755 if (is_cached_coherent(obj
) && omap_obj
->addrs
[pgoff
]) {
756 dma_unmap_page(dev
->dev
, omap_obj
->addrs
[pgoff
],
757 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
758 omap_obj
->addrs
[pgoff
] = 0;
762 /* sync the buffer for DMA access */
763 void omap_gem_dma_sync(struct drm_gem_object
*obj
,
764 enum dma_data_direction dir
)
766 struct drm_device
*dev
= obj
->dev
;
767 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
769 if (is_cached_coherent(obj
)) {
770 int i
, npages
= obj
->size
>> PAGE_SHIFT
;
771 struct page
**pages
= omap_obj
->pages
;
774 for (i
= 0; i
< npages
; i
++) {
775 if (!omap_obj
->addrs
[i
]) {
778 addr
= dma_map_page(dev
->dev
, pages
[i
], 0,
779 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
781 if (dma_mapping_error(dev
->dev
, addr
)) {
783 "%s: failed to map page\n",
789 omap_obj
->addrs
[i
] = addr
;
794 unmap_mapping_range(obj
->filp
->f_mapping
, 0,
795 omap_gem_mmap_size(obj
), 1);
800 /* Get physical address for DMA.. if 'remap' is true, and the buffer is not
801 * already contiguous, remap it to pin in physically contiguous memory.. (ie.
804 int omap_gem_get_paddr(struct drm_gem_object
*obj
,
805 dma_addr_t
*paddr
, bool remap
)
807 struct omap_drm_private
*priv
= obj
->dev
->dev_private
;
808 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
811 mutex_lock(&obj
->dev
->struct_mutex
);
813 if (!is_contiguous(omap_obj
) && remap
&& priv
->has_dmm
) {
814 if (omap_obj
->paddr_cnt
== 0) {
816 uint32_t npages
= obj
->size
>> PAGE_SHIFT
;
817 enum tiler_fmt fmt
= gem2fmt(omap_obj
->flags
);
818 struct tiler_block
*block
;
820 BUG_ON(omap_obj
->block
);
822 ret
= get_pages(obj
, &pages
);
826 if (omap_obj
->flags
& OMAP_BO_TILED
) {
827 block
= tiler_reserve_2d(fmt
,
829 omap_obj
->height
, 0);
831 block
= tiler_reserve_1d(obj
->size
);
835 ret
= PTR_ERR(block
);
836 dev_err(obj
->dev
->dev
,
837 "could not remap: %d (%d)\n", ret
, fmt
);
841 /* TODO: enable async refill.. */
842 ret
= tiler_pin(block
, pages
, npages
,
843 omap_obj
->roll
, true);
845 tiler_release(block
);
846 dev_err(obj
->dev
->dev
,
847 "could not pin: %d\n", ret
);
851 omap_obj
->paddr
= tiler_ssptr(block
);
852 omap_obj
->block
= block
;
854 DBG("got paddr: %pad", &omap_obj
->paddr
);
857 omap_obj
->paddr_cnt
++;
859 *paddr
= omap_obj
->paddr
;
860 } else if (is_contiguous(omap_obj
)) {
861 *paddr
= omap_obj
->paddr
;
868 mutex_unlock(&obj
->dev
->struct_mutex
);
873 /* Release physical address, when DMA is no longer being performed.. this
874 * could potentially unpin and unmap buffers from TILER
876 void omap_gem_put_paddr(struct drm_gem_object
*obj
)
878 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
881 mutex_lock(&obj
->dev
->struct_mutex
);
882 if (omap_obj
->paddr_cnt
> 0) {
883 omap_obj
->paddr_cnt
--;
884 if (omap_obj
->paddr_cnt
== 0) {
885 ret
= tiler_unpin(omap_obj
->block
);
887 dev_err(obj
->dev
->dev
,
888 "could not unpin pages: %d\n", ret
);
890 ret
= tiler_release(omap_obj
->block
);
892 dev_err(obj
->dev
->dev
,
893 "could not release unmap: %d\n", ret
);
896 omap_obj
->block
= NULL
;
900 mutex_unlock(&obj
->dev
->struct_mutex
);
903 /* Get rotated scanout address (only valid if already pinned), at the
904 * specified orientation and x,y offset from top-left corner of buffer
905 * (only valid for tiled 2d buffers)
907 int omap_gem_rotated_paddr(struct drm_gem_object
*obj
, uint32_t orient
,
908 int x
, int y
, dma_addr_t
*paddr
)
910 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
913 mutex_lock(&obj
->dev
->struct_mutex
);
914 if ((omap_obj
->paddr_cnt
> 0) && omap_obj
->block
&&
915 (omap_obj
->flags
& OMAP_BO_TILED
)) {
916 *paddr
= tiler_tsptr(omap_obj
->block
, orient
, x
, y
);
919 mutex_unlock(&obj
->dev
->struct_mutex
);
923 /* Get tiler stride for the buffer (only valid for 2d tiled buffers) */
924 int omap_gem_tiled_stride(struct drm_gem_object
*obj
, uint32_t orient
)
926 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
928 if (omap_obj
->flags
& OMAP_BO_TILED
)
929 ret
= tiler_stride(gem2fmt(omap_obj
->flags
), orient
);
933 /* if !remap, and we don't have pages backing, then fail, rather than
934 * increasing the pin count (which we don't really do yet anyways,
935 * because we don't support swapping pages back out). And 'remap'
936 * might not be quite the right name, but I wanted to keep it working
937 * similarly to omap_gem_get_paddr(). Note though that mutex is not
938 * aquired if !remap (because this can be called in atomic ctxt),
939 * but probably omap_gem_get_paddr() should be changed to work in the
940 * same way. If !remap, a matching omap_gem_put_pages() call is not
941 * required (and should not be made).
943 int omap_gem_get_pages(struct drm_gem_object
*obj
, struct page
***pages
,
948 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
949 if (!omap_obj
->pages
)
951 *pages
= omap_obj
->pages
;
954 mutex_lock(&obj
->dev
->struct_mutex
);
955 ret
= get_pages(obj
, pages
);
956 mutex_unlock(&obj
->dev
->struct_mutex
);
960 /* release pages when DMA no longer being performed */
961 int omap_gem_put_pages(struct drm_gem_object
*obj
)
963 /* do something here if we dynamically attach/detach pages.. at
964 * least they would no longer need to be pinned if everyone has
965 * released the pages..
970 #ifdef CONFIG_DRM_FBDEV_EMULATION
971 /* Get kernel virtual address for CPU access.. this more or less only
972 * exists for omap_fbdev. This should be called with struct_mutex
975 void *omap_gem_vaddr(struct drm_gem_object
*obj
)
977 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
978 WARN_ON(!mutex_is_locked(&obj
->dev
->struct_mutex
));
979 if (!omap_obj
->vaddr
) {
981 int ret
= get_pages(obj
, &pages
);
984 omap_obj
->vaddr
= vmap(pages
, obj
->size
>> PAGE_SHIFT
,
985 VM_MAP
, pgprot_writecombine(PAGE_KERNEL
));
987 return omap_obj
->vaddr
;
991 /* -----------------------------------------------------------------------------
996 /* re-pin objects in DMM in resume path: */
997 int omap_gem_resume(struct device
*dev
)
999 struct drm_device
*drm_dev
= dev_get_drvdata(dev
);
1000 struct omap_drm_private
*priv
= drm_dev
->dev_private
;
1001 struct omap_gem_object
*omap_obj
;
1004 list_for_each_entry(omap_obj
, &priv
->obj_list
, mm_list
) {
1005 if (omap_obj
->block
) {
1006 struct drm_gem_object
*obj
= &omap_obj
->base
;
1007 uint32_t npages
= obj
->size
>> PAGE_SHIFT
;
1008 WARN_ON(!omap_obj
->pages
); /* this can't happen */
1009 ret
= tiler_pin(omap_obj
->block
,
1010 omap_obj
->pages
, npages
,
1011 omap_obj
->roll
, true);
1013 dev_err(dev
, "could not repin: %d\n", ret
);
1023 /* -----------------------------------------------------------------------------
1027 #ifdef CONFIG_DEBUG_FS
1028 void omap_gem_describe(struct drm_gem_object
*obj
, struct seq_file
*m
)
1030 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1033 off
= drm_vma_node_start(&obj
->vma_node
);
1035 seq_printf(m
, "%08x: %2d (%2d) %08llx %pad (%2d) %p %4d",
1036 omap_obj
->flags
, obj
->name
, kref_read(&obj
->refcount
),
1037 off
, &omap_obj
->paddr
, omap_obj
->paddr_cnt
,
1038 omap_obj
->vaddr
, omap_obj
->roll
);
1040 if (omap_obj
->flags
& OMAP_BO_TILED
) {
1041 seq_printf(m
, " %dx%d", omap_obj
->width
, omap_obj
->height
);
1042 if (omap_obj
->block
) {
1043 struct tcm_area
*area
= &omap_obj
->block
->area
;
1044 seq_printf(m
, " (%dx%d, %dx%d)",
1045 area
->p0
.x
, area
->p0
.y
,
1046 area
->p1
.x
, area
->p1
.y
);
1049 seq_printf(m
, " %d", obj
->size
);
1052 seq_printf(m
, "\n");
1055 void omap_gem_describe_objects(struct list_head
*list
, struct seq_file
*m
)
1057 struct omap_gem_object
*omap_obj
;
1061 list_for_each_entry(omap_obj
, list
, mm_list
) {
1062 struct drm_gem_object
*obj
= &omap_obj
->base
;
1064 omap_gem_describe(obj
, m
);
1069 seq_printf(m
, "Total %d objects, %zu bytes\n", count
, size
);
1073 /* -----------------------------------------------------------------------------
1074 * Buffer Synchronization
1077 static DEFINE_SPINLOCK(sync_lock
);
1079 struct omap_gem_sync_waiter
{
1080 struct list_head list
;
1081 struct omap_gem_object
*omap_obj
;
1082 enum omap_gem_op op
;
1083 uint32_t read_target
, write_target
;
1084 /* notify called w/ sync_lock held */
1085 void (*notify
)(void *arg
);
1089 /* list of omap_gem_sync_waiter.. the notify fxn gets called back when
1090 * the read and/or write target count is achieved which can call a user
1091 * callback (ex. to kick 3d and/or 2d), wakeup blocked task (prep for
1094 static LIST_HEAD(waiters
);
1096 static inline bool is_waiting(struct omap_gem_sync_waiter
*waiter
)
1098 struct omap_gem_object
*omap_obj
= waiter
->omap_obj
;
1099 if ((waiter
->op
& OMAP_GEM_READ
) &&
1100 (omap_obj
->sync
->write_complete
< waiter
->write_target
))
1102 if ((waiter
->op
& OMAP_GEM_WRITE
) &&
1103 (omap_obj
->sync
->read_complete
< waiter
->read_target
))
1108 /* macro for sync debug.. */
1110 #define SYNC(fmt, ...) do { if (SYNCDBG) \
1111 pr_err("%s:%d: " fmt "\n", __func__, __LINE__, ##__VA_ARGS__); \
1115 static void sync_op_update(void)
1117 struct omap_gem_sync_waiter
*waiter
, *n
;
1118 list_for_each_entry_safe(waiter
, n
, &waiters
, list
) {
1119 if (!is_waiting(waiter
)) {
1120 list_del(&waiter
->list
);
1121 SYNC("notify: %p", waiter
);
1122 waiter
->notify(waiter
->arg
);
1128 static inline int sync_op(struct drm_gem_object
*obj
,
1129 enum omap_gem_op op
, bool start
)
1131 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1134 spin_lock(&sync_lock
);
1136 if (!omap_obj
->sync
) {
1137 omap_obj
->sync
= kzalloc(sizeof(*omap_obj
->sync
), GFP_ATOMIC
);
1138 if (!omap_obj
->sync
) {
1145 if (op
& OMAP_GEM_READ
)
1146 omap_obj
->sync
->read_pending
++;
1147 if (op
& OMAP_GEM_WRITE
)
1148 omap_obj
->sync
->write_pending
++;
1150 if (op
& OMAP_GEM_READ
)
1151 omap_obj
->sync
->read_complete
++;
1152 if (op
& OMAP_GEM_WRITE
)
1153 omap_obj
->sync
->write_complete
++;
1158 spin_unlock(&sync_lock
);
1163 /* mark the start of read and/or write operation */
1164 int omap_gem_op_start(struct drm_gem_object
*obj
, enum omap_gem_op op
)
1166 return sync_op(obj
, op
, true);
1169 int omap_gem_op_finish(struct drm_gem_object
*obj
, enum omap_gem_op op
)
1171 return sync_op(obj
, op
, false);
1174 static DECLARE_WAIT_QUEUE_HEAD(sync_event
);
1176 static void sync_notify(void *arg
)
1178 struct task_struct
**waiter_task
= arg
;
1179 *waiter_task
= NULL
;
1180 wake_up_all(&sync_event
);
1183 int omap_gem_op_sync(struct drm_gem_object
*obj
, enum omap_gem_op op
)
1185 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1187 if (omap_obj
->sync
) {
1188 struct task_struct
*waiter_task
= current
;
1189 struct omap_gem_sync_waiter
*waiter
=
1190 kzalloc(sizeof(*waiter
), GFP_KERNEL
);
1195 waiter
->omap_obj
= omap_obj
;
1197 waiter
->read_target
= omap_obj
->sync
->read_pending
;
1198 waiter
->write_target
= omap_obj
->sync
->write_pending
;
1199 waiter
->notify
= sync_notify
;
1200 waiter
->arg
= &waiter_task
;
1202 spin_lock(&sync_lock
);
1203 if (is_waiting(waiter
)) {
1204 SYNC("waited: %p", waiter
);
1205 list_add_tail(&waiter
->list
, &waiters
);
1206 spin_unlock(&sync_lock
);
1207 ret
= wait_event_interruptible(sync_event
,
1208 (waiter_task
== NULL
));
1209 spin_lock(&sync_lock
);
1211 SYNC("interrupted: %p", waiter
);
1212 /* we were interrupted */
1213 list_del(&waiter
->list
);
1216 /* freed in sync_op_update() */
1220 spin_unlock(&sync_lock
);
1226 /* call fxn(arg), either synchronously or asynchronously if the op
1227 * is currently blocked.. fxn() can be called from any context
1229 * (TODO for now fxn is called back from whichever context calls
1230 * omap_gem_op_finish().. but this could be better defined later
1233 * TODO more code in common w/ _sync()..
1235 int omap_gem_op_async(struct drm_gem_object
*obj
, enum omap_gem_op op
,
1236 void (*fxn
)(void *arg
), void *arg
)
1238 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1239 if (omap_obj
->sync
) {
1240 struct omap_gem_sync_waiter
*waiter
=
1241 kzalloc(sizeof(*waiter
), GFP_ATOMIC
);
1246 waiter
->omap_obj
= omap_obj
;
1248 waiter
->read_target
= omap_obj
->sync
->read_pending
;
1249 waiter
->write_target
= omap_obj
->sync
->write_pending
;
1250 waiter
->notify
= fxn
;
1253 spin_lock(&sync_lock
);
1254 if (is_waiting(waiter
)) {
1255 SYNC("waited: %p", waiter
);
1256 list_add_tail(&waiter
->list
, &waiters
);
1257 spin_unlock(&sync_lock
);
1261 spin_unlock(&sync_lock
);
1272 /* -----------------------------------------------------------------------------
1273 * Constructor & Destructor
1276 void omap_gem_free_object(struct drm_gem_object
*obj
)
1278 struct drm_device
*dev
= obj
->dev
;
1279 struct omap_drm_private
*priv
= dev
->dev_private
;
1280 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1284 WARN_ON(!mutex_is_locked(&dev
->struct_mutex
));
1286 spin_lock(&priv
->list_lock
);
1287 list_del(&omap_obj
->mm_list
);
1288 spin_unlock(&priv
->list_lock
);
1290 /* this means the object is still pinned.. which really should
1291 * not happen. I think..
1293 WARN_ON(omap_obj
->paddr_cnt
> 0);
1295 if (omap_obj
->pages
) {
1296 if (omap_obj
->flags
& OMAP_BO_MEM_DMABUF
)
1297 kfree(omap_obj
->pages
);
1299 omap_gem_detach_pages(obj
);
1302 if (omap_obj
->flags
& OMAP_BO_MEM_DMA_API
) {
1303 dma_free_wc(dev
->dev
, obj
->size
, omap_obj
->vaddr
,
1305 } else if (omap_obj
->vaddr
) {
1306 vunmap(omap_obj
->vaddr
);
1307 } else if (obj
->import_attach
) {
1308 drm_prime_gem_destroy(obj
, omap_obj
->sgt
);
1311 kfree(omap_obj
->sync
);
1313 drm_gem_object_release(obj
);
1318 /* GEM buffer object constructor */
1319 struct drm_gem_object
*omap_gem_new(struct drm_device
*dev
,
1320 union omap_gem_size gsize
, uint32_t flags
)
1322 struct omap_drm_private
*priv
= dev
->dev_private
;
1323 struct omap_gem_object
*omap_obj
;
1324 struct drm_gem_object
*obj
;
1325 struct address_space
*mapping
;
1329 /* Validate the flags and compute the memory and cache flags. */
1330 if (flags
& OMAP_BO_TILED
) {
1331 if (!priv
->usergart
) {
1332 dev_err(dev
->dev
, "Tiled buffers require DMM\n");
1337 * Tiled buffers are always shmem paged backed. When they are
1338 * scanned out, they are remapped into DMM/TILER.
1340 flags
&= ~OMAP_BO_SCANOUT
;
1341 flags
|= OMAP_BO_MEM_SHMEM
;
1344 * Currently don't allow cached buffers. There is some caching
1345 * stuff that needs to be handled better.
1347 flags
&= ~(OMAP_BO_CACHED
|OMAP_BO_WC
|OMAP_BO_UNCACHED
);
1348 flags
|= tiler_get_cpu_cache_flags();
1349 } else if ((flags
& OMAP_BO_SCANOUT
) && !priv
->has_dmm
) {
1351 * OMAP_BO_SCANOUT hints that the buffer doesn't need to be
1352 * tiled. However, to lower the pressure on memory allocation,
1353 * use contiguous memory only if no TILER is available.
1355 flags
|= OMAP_BO_MEM_DMA_API
;
1356 } else if (!(flags
& OMAP_BO_MEM_DMABUF
)) {
1358 * All other buffers not backed by dma_buf are shmem-backed.
1360 flags
|= OMAP_BO_MEM_SHMEM
;
1363 /* Allocate the initialize the OMAP GEM object. */
1364 omap_obj
= kzalloc(sizeof(*omap_obj
), GFP_KERNEL
);
1368 obj
= &omap_obj
->base
;
1369 omap_obj
->flags
= flags
;
1371 if (flags
& OMAP_BO_TILED
) {
1373 * For tiled buffers align dimensions to slot boundaries and
1374 * calculate size based on aligned dimensions.
1376 tiler_align(gem2fmt(flags
), &gsize
.tiled
.width
,
1377 &gsize
.tiled
.height
);
1379 size
= tiler_size(gem2fmt(flags
), gsize
.tiled
.width
,
1380 gsize
.tiled
.height
);
1382 omap_obj
->width
= gsize
.tiled
.width
;
1383 omap_obj
->height
= gsize
.tiled
.height
;
1385 size
= PAGE_ALIGN(gsize
.bytes
);
1388 /* Initialize the GEM object. */
1389 if (!(flags
& OMAP_BO_MEM_SHMEM
)) {
1390 drm_gem_private_object_init(dev
, obj
, size
);
1392 ret
= drm_gem_object_init(dev
, obj
, size
);
1396 mapping
= obj
->filp
->f_mapping
;
1397 mapping_set_gfp_mask(mapping
, GFP_USER
| __GFP_DMA32
);
1400 /* Allocate memory if needed. */
1401 if (flags
& OMAP_BO_MEM_DMA_API
) {
1402 omap_obj
->vaddr
= dma_alloc_wc(dev
->dev
, size
,
1405 if (!omap_obj
->vaddr
)
1409 spin_lock(&priv
->list_lock
);
1410 list_add(&omap_obj
->mm_list
, &priv
->obj_list
);
1411 spin_unlock(&priv
->list_lock
);
1416 drm_gem_object_release(obj
);
1422 struct drm_gem_object
*omap_gem_new_dmabuf(struct drm_device
*dev
, size_t size
,
1423 struct sg_table
*sgt
)
1425 struct omap_drm_private
*priv
= dev
->dev_private
;
1426 struct omap_gem_object
*omap_obj
;
1427 struct drm_gem_object
*obj
;
1428 union omap_gem_size gsize
;
1430 /* Without a DMM only physically contiguous buffers can be supported. */
1431 if (sgt
->orig_nents
!= 1 && !priv
->has_dmm
)
1432 return ERR_PTR(-EINVAL
);
1434 mutex_lock(&dev
->struct_mutex
);
1436 gsize
.bytes
= PAGE_ALIGN(size
);
1437 obj
= omap_gem_new(dev
, gsize
, OMAP_BO_MEM_DMABUF
| OMAP_BO_WC
);
1439 obj
= ERR_PTR(-ENOMEM
);
1443 omap_obj
= to_omap_bo(obj
);
1444 omap_obj
->sgt
= sgt
;
1446 if (sgt
->orig_nents
== 1) {
1447 omap_obj
->paddr
= sg_dma_address(sgt
->sgl
);
1449 /* Create pages list from sgt */
1450 struct sg_page_iter iter
;
1451 struct page
**pages
;
1452 unsigned int npages
;
1455 npages
= DIV_ROUND_UP(size
, PAGE_SIZE
);
1456 pages
= kcalloc(npages
, sizeof(*pages
), GFP_KERNEL
);
1458 omap_gem_free_object(obj
);
1459 obj
= ERR_PTR(-ENOMEM
);
1463 omap_obj
->pages
= pages
;
1465 for_each_sg_page(sgt
->sgl
, &iter
, sgt
->orig_nents
, 0) {
1466 pages
[i
++] = sg_page_iter_page(&iter
);
1471 if (WARN_ON(i
!= npages
)) {
1472 omap_gem_free_object(obj
);
1473 obj
= ERR_PTR(-ENOMEM
);
1479 mutex_unlock(&dev
->struct_mutex
);
1483 /* convenience method to construct a GEM buffer object, and userspace handle */
1484 int omap_gem_new_handle(struct drm_device
*dev
, struct drm_file
*file
,
1485 union omap_gem_size gsize
, uint32_t flags
, uint32_t *handle
)
1487 struct drm_gem_object
*obj
;
1490 obj
= omap_gem_new(dev
, gsize
, flags
);
1494 ret
= drm_gem_handle_create(file
, obj
, handle
);
1496 omap_gem_free_object(obj
);
1500 /* drop reference from allocate - handle holds it now */
1501 drm_gem_object_unreference_unlocked(obj
);
1506 /* -----------------------------------------------------------------------------
1510 /* If DMM is used, we need to set some stuff up.. */
1511 void omap_gem_init(struct drm_device
*dev
)
1513 struct omap_drm_private
*priv
= dev
->dev_private
;
1514 struct omap_drm_usergart
*usergart
;
1515 const enum tiler_fmt fmts
[] = {
1516 TILFMT_8BIT
, TILFMT_16BIT
, TILFMT_32BIT
1520 if (!dmm_is_available()) {
1521 /* DMM only supported on OMAP4 and later, so this isn't fatal */
1522 dev_warn(dev
->dev
, "DMM not available, disable DMM support\n");
1526 usergart
= kcalloc(3, sizeof(*usergart
), GFP_KERNEL
);
1530 /* reserve 4k aligned/wide regions for userspace mappings: */
1531 for (i
= 0; i
< ARRAY_SIZE(fmts
); i
++) {
1532 uint16_t h
= 1, w
= PAGE_SIZE
>> i
;
1533 tiler_align(fmts
[i
], &w
, &h
);
1534 /* note: since each region is 1 4kb page wide, and minimum
1535 * number of rows, the height ends up being the same as the
1536 * # of pages in the region
1538 usergart
[i
].height
= h
;
1539 usergart
[i
].height_shift
= ilog2(h
);
1540 usergart
[i
].stride_pfn
= tiler_stride(fmts
[i
], 0) >> PAGE_SHIFT
;
1541 usergart
[i
].slot_shift
= ilog2((PAGE_SIZE
/ h
) >> i
);
1542 for (j
= 0; j
< NUM_USERGART_ENTRIES
; j
++) {
1543 struct omap_drm_usergart_entry
*entry
;
1544 struct tiler_block
*block
;
1546 entry
= &usergart
[i
].entry
[j
];
1547 block
= tiler_reserve_2d(fmts
[i
], w
, h
, PAGE_SIZE
);
1548 if (IS_ERR(block
)) {
1550 "reserve failed: %d, %d, %ld\n",
1551 i
, j
, PTR_ERR(block
));
1554 entry
->paddr
= tiler_ssptr(block
);
1555 entry
->block
= block
;
1557 DBG("%d:%d: %dx%d: paddr=%pad stride=%d", i
, j
, w
, h
,
1559 usergart
[i
].stride_pfn
<< PAGE_SHIFT
);
1563 priv
->usergart
= usergart
;
1564 priv
->has_dmm
= true;
1567 void omap_gem_deinit(struct drm_device
*dev
)
1569 struct omap_drm_private
*priv
= dev
->dev_private
;
1571 /* I believe we can rely on there being no more outstanding GEM
1572 * objects which could depend on usergart/dmm at this point.
1574 kfree(priv
->usergart
);