Commit | Line | Data |
---|---|---|
6fa3eb70 S |
1 | /* |
2 | * drivers/gpu/ion/ion_priv.h | |
3 | * | |
4 | * Copyright (C) 2011 Google, Inc. | |
5 | * | |
6 | * This software is licensed under the terms of the GNU General Public | |
7 | * License version 2, as published by the Free Software Foundation, and | |
8 | * may be copied, distributed, and modified under those terms. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, | |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | * GNU General Public License for more details. | |
14 | * | |
15 | */ | |
16 | ||
17 | #ifndef _ION_PRIV_H | |
18 | #define _ION_PRIV_H | |
19 | ||
20 | #include <linux/device.h> | |
21 | #include <linux/dma-direction.h> | |
22 | #include <linux/kref.h> | |
23 | #include <linux/mm_types.h> | |
24 | #include <linux/mutex.h> | |
25 | #include <linux/rbtree.h> | |
26 | #include <linux/sched.h> | |
27 | #include <linux/shrinker.h> | |
28 | #include <linux/types.h> | |
29 | #include <linux/miscdevice.h> | |
30 | #include <linux/idr.h> | |
31 | #include <linux/ion_drv.h> | |
32 | ||
33 | struct ion_buffer *ion_handle_buffer(struct ion_handle *handle); | |
34 | ||
35 | /** | |
36 | * struct ion_device - the metadata of the ion device node | |
37 | * @dev: the actual misc device | |
38 | * @buffers: an rb tree of all the existing buffers | |
39 | * @buffer_lock: lock protecting the tree of buffers | |
40 | * @lock: rwsem protecting the tree of heaps and clients | |
41 | * @heaps: list of all the heaps in the system | |
42 | * @user_clients: list of all the clients created from userspace | |
43 | */ | |
44 | struct ion_device { | |
45 | struct miscdevice dev; | |
46 | struct rb_root buffers; | |
47 | struct mutex buffer_lock; | |
48 | struct rw_semaphore lock; | |
49 | struct plist_head heaps; | |
50 | long (*custom_ioctl) (struct ion_client *client, unsigned int cmd, | |
51 | unsigned long arg); | |
52 | struct rb_root clients; | |
53 | struct dentry *debug_root; | |
54 | struct dentry *heaps_debug_root; | |
55 | struct dentry *clients_debug_root; | |
56 | }; | |
57 | ||
58 | /** | |
59 | * struct ion_client - a process/hw block local address space | |
60 | * @node: node in the tree of all clients | |
61 | * @dev: backpointer to ion device | |
62 | * @handles: an rb tree of all the handles in this client | |
63 | * @idr: an idr space for allocating handle ids | |
64 | * @lock: lock protecting the tree of handles | |
65 | * @name: used for debugging | |
66 | * @display_name: used for debugging (unique version of @name) | |
67 | * @display_serial: used for debugging (to make display_name unique) | |
68 | * @task: used for debugging | |
69 | * | |
70 | * A client represents a list of buffers this client may access. | |
71 | * The mutex stored here is used to protect both handles tree | |
72 | * as well as the handles themselves, and should be held while modifying either. | |
73 | */ | |
74 | struct ion_client { | |
75 | struct rb_node node; | |
76 | struct ion_device *dev; | |
77 | struct rb_root handles; | |
78 | struct idr idr; | |
79 | struct mutex lock; | |
80 | const char *name; | |
81 | char *display_name; | |
82 | int display_serial; | |
83 | struct task_struct *task; | |
84 | pid_t pid; | |
85 | struct dentry *debug_root; | |
86 | char dbg_name[ION_MM_DBG_NAME_LEN]; //add by K for debug! | |
87 | }; | |
88 | ||
89 | struct ion_handle_debug { | |
90 | pid_t pid; | |
91 | pid_t tgid; | |
92 | unsigned int backtrace[BACKTRACE_SIZE]; | |
93 | unsigned int backtrace_num; | |
94 | }; | |
95 | ||
96 | /** | |
97 | * ion_handle - a client local reference to a buffer | |
98 | * @ref: reference count | |
99 | * @client: back pointer to the client the buffer resides in | |
100 | * @buffer: pointer to the buffer | |
101 | * @node: node in the client's handle rbtree | |
102 | * @kmap_cnt: count of times this client has mapped to kernel | |
103 | * @id: client-unique id allocated by client->idr | |
104 | * | |
105 | * Modifications to node, map_cnt or mapping should be protected by the | |
106 | * lock in the client. Other fields are never changed after initialization. | |
107 | */ | |
108 | struct ion_handle { | |
109 | struct kref ref; | |
4b9e9796 | 110 | unsigned int user_ref_count; |
6fa3eb70 S |
111 | struct ion_client *client; |
112 | struct ion_buffer *buffer; | |
113 | struct rb_node node; | |
114 | unsigned int kmap_cnt; | |
115 | int id; | |
116 | #if ION_RUNTIME_DEBUGGER | |
117 | struct ion_handle_debug dbg; //add by K for debug | |
118 | #endif | |
119 | }; | |
120 | ||
121 | ||
122 | /** | |
123 | * struct ion_buffer - metadata for a particular buffer | |
124 | * @ref: refernce count | |
125 | * @node: node in the ion_device buffers tree | |
126 | * @dev: back pointer to the ion_device | |
127 | * @heap: back pointer to the heap the buffer came from | |
128 | * @flags: buffer specific flags | |
129 | * @private_flags: internal buffer specific flags | |
130 | * @size: size of the buffer | |
131 | * @priv_virt: private data to the buffer representable as | |
132 | * a void * | |
133 | * @priv_phys: private data to the buffer representable as | |
134 | * an ion_phys_addr_t (and someday a phys_addr_t) | |
135 | * @lock: protects the buffers cnt fields | |
136 | * @kmap_cnt: number of times the buffer is mapped to the kernel | |
137 | * @vaddr: the kenrel mapping if kmap_cnt is not zero | |
138 | * @dmap_cnt: number of times the buffer is mapped for dma | |
139 | * @sg_table: the sg table for the buffer if dmap_cnt is not zero | |
140 | * @pages: flat array of pages in the buffer -- used by fault | |
141 | * handler and only valid for buffers that are faulted in | |
142 | * @vmas: list of vma's mapping this buffer | |
143 | * @handle_count: count of handles referencing this buffer | |
144 | * @task_comm: taskcomm of last client to reference this buffer in a | |
145 | * handle, used for debugging | |
146 | * @pid: pid of last client to reference this buffer in a | |
147 | * handle, used for debugging | |
148 | */ | |
149 | struct ion_buffer { | |
150 | struct kref ref; | |
151 | union { | |
152 | struct rb_node node; | |
153 | struct list_head list; | |
154 | }; | |
155 | struct ion_device *dev; | |
156 | struct ion_heap *heap; | |
157 | unsigned long flags; | |
158 | unsigned long private_flags; | |
159 | size_t size; | |
160 | union { | |
161 | void *priv_virt; | |
162 | ion_phys_addr_t priv_phys; | |
163 | }; | |
164 | struct mutex lock; | |
165 | int kmap_cnt; | |
166 | void *vaddr; | |
167 | int dmap_cnt; | |
168 | struct sg_table *sg_table; | |
169 | struct page **pages; | |
170 | struct list_head vmas; | |
171 | /* used to track orphaned buffers */ | |
172 | int handle_count; | |
173 | char task_comm[TASK_COMM_LEN]; | |
174 | pid_t pid; | |
175 | }; | |
176 | ||
177 | void ion_buffer_destroy(struct ion_buffer *buffer); | |
178 | ||
179 | /** | |
180 | * struct ion_heap_ops - ops to operate on a given heap | |
181 | * @allocate: allocate memory | |
182 | * @free: free memory | |
183 | * @phys get physical address of a buffer (only define on | |
184 | * physically contiguous heaps) | |
185 | * @map_dma map the memory for dma to a scatterlist | |
186 | * @unmap_dma unmap the memory for dma | |
187 | * @map_kernel map memory to the kernel | |
188 | * @unmap_kernel unmap memory to the kernel | |
189 | * @map_user map memory to userspace | |
190 | * | |
191 | * allocate, phys, and map_user return 0 on success, -errno on error. | |
192 | * map_dma and map_kernel return pointer on success, ERR_PTR on | |
193 | * error. @free will be called with ION_PRIV_FLAG_SHRINKER_FREE set in | |
194 | * the buffer's private_flags when called from a shrinker. In that | |
195 | * case, the pages being free'd must be truly free'd back to the | |
196 | * system, not put in a page pool or otherwise cached. | |
197 | */ | |
198 | struct ion_heap_ops { | |
199 | int (*allocate) (struct ion_heap *heap, | |
200 | struct ion_buffer *buffer, unsigned long len, | |
201 | unsigned long align, unsigned long flags); | |
202 | void (*free) (struct ion_buffer *buffer); | |
203 | int (*phys) (struct ion_heap *heap, struct ion_buffer *buffer, | |
204 | ion_phys_addr_t *addr, size_t *len); | |
205 | struct sg_table *(*map_dma) (struct ion_heap *heap, | |
206 | struct ion_buffer *buffer); | |
207 | void (*unmap_dma) (struct ion_heap *heap, struct ion_buffer *buffer); | |
208 | void * (*map_kernel) (struct ion_heap *heap, struct ion_buffer *buffer); | |
209 | void (*unmap_kernel) (struct ion_heap *heap, struct ion_buffer *buffer); | |
210 | int (*map_user) (struct ion_heap *mapper, struct ion_buffer *buffer, | |
211 | struct vm_area_struct *vma); | |
212 | int (*shrink)(struct ion_heap *heap, gfp_t gfp_mask, int nr_to_scan); | |
213 | void (*add_freelist) (struct ion_buffer *buffer); | |
214 | }; | |
215 | ||
216 | /** | |
217 | * heap flags - flags between the heaps and core ion code | |
218 | */ | |
219 | #define ION_HEAP_FLAG_DEFER_FREE (1 << 0) | |
220 | ||
221 | /** | |
222 | * private flags - flags internal to ion | |
223 | */ | |
224 | /* | |
225 | * Buffer is being freed from a shrinker function. Skip any possible | |
226 | * heap-specific caching mechanism (e.g. page pools). Guarantees that | |
227 | * any buffer storage that came from the system allocator will be | |
228 | * returned to the system allocator. | |
229 | */ | |
230 | #define ION_PRIV_FLAG_SHRINKER_FREE (1 << 0) | |
231 | ||
232 | /** | |
233 | * struct ion_heap - represents a heap in the system | |
234 | * @node: rb node to put the heap on the device's tree of heaps | |
235 | * @dev: back pointer to the ion_device | |
236 | * @type: type of heap | |
237 | * @ops: ops struct as above | |
238 | * @flags: flags | |
239 | * @id: id of heap, also indicates priority of this heap when | |
240 | * allocating. These are specified by platform data and | |
241 | * MUST be unique | |
242 | * @name: used for debugging | |
243 | * @shrinker: a shrinker for the heap | |
244 | * @free_list: free list head if deferred free is used | |
245 | * @free_list_size size of the deferred free list in bytes | |
246 | * @lock: protects the free list | |
247 | * @waitqueue: queue to wait on from deferred free thread | |
248 | * @task: task struct of deferred free thread | |
249 | * @debug_show: called when heap debug file is read to add any | |
250 | * heap specific debug info to output | |
251 | * | |
252 | * Represents a pool of memory from which buffers can be made. In some | |
253 | * systems the only heap is regular system memory allocated via vmalloc. | |
254 | * On others, some blocks might require large physically contiguous buffers | |
255 | * that are allocated from a specially reserved heap. | |
256 | */ | |
257 | struct ion_heap { | |
258 | struct plist_node node; | |
259 | struct ion_device *dev; | |
260 | enum ion_heap_type type; | |
261 | struct ion_heap_ops *ops; | |
262 | unsigned long flags; | |
263 | unsigned int id; | |
264 | const char *name; | |
265 | struct shrinker shrinker; | |
266 | struct list_head free_list; | |
267 | size_t free_list_size; | |
268 | spinlock_t free_lock; | |
269 | wait_queue_head_t waitqueue; | |
270 | struct task_struct *task; | |
271 | int (*debug_show)(struct ion_heap *heap, struct seq_file *, void *); | |
272 | }; | |
273 | ||
274 | /** | |
275 | * ion_buffer_cached - this ion buffer is cached | |
276 | * @buffer: buffer | |
277 | * | |
278 | * indicates whether this ion buffer is cached | |
279 | */ | |
280 | bool ion_buffer_cached(struct ion_buffer *buffer); | |
281 | ||
282 | /** | |
283 | * ion_buffer_fault_user_mappings - fault in user mappings of this buffer | |
284 | * @buffer: buffer | |
285 | * | |
286 | * indicates whether userspace mappings of this buffer will be faulted | |
287 | * in, this can affect how buffers are allocated from the heap. | |
288 | */ | |
289 | bool ion_buffer_fault_user_mappings(struct ion_buffer *buffer); | |
290 | ||
291 | /** | |
292 | * ion_device_create - allocates and returns an ion device | |
293 | * @custom_ioctl: arch specific ioctl function if applicable | |
294 | * | |
295 | * returns a valid device or -PTR_ERR | |
296 | */ | |
297 | struct ion_device *ion_device_create(long (*custom_ioctl) | |
298 | (struct ion_client *client, | |
299 | unsigned int cmd, | |
300 | unsigned long arg)); | |
301 | ||
302 | /** | |
303 | * ion_device_destroy - free and device and it's resource | |
304 | * @dev: the device | |
305 | */ | |
306 | void ion_device_destroy(struct ion_device *dev); | |
307 | ||
308 | /** | |
309 | * ion_device_add_heap - adds a heap to the ion device | |
310 | * @dev: the device | |
311 | * @heap: the heap to add | |
312 | */ | |
313 | void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap); | |
314 | ||
315 | /** | |
316 | * some helpers for common operations on buffers using the sg_table | |
317 | * and vaddr fields | |
318 | */ | |
319 | void *ion_heap_map_kernel(struct ion_heap *, struct ion_buffer *); | |
320 | void ion_heap_unmap_kernel(struct ion_heap *, struct ion_buffer *); | |
321 | int ion_heap_map_user(struct ion_heap *, struct ion_buffer *, | |
322 | struct vm_area_struct *); | |
323 | int ion_heap_buffer_zero(struct ion_buffer *buffer); | |
324 | int ion_heap_pages_zero(struct page *page, size_t size, pgprot_t pgprot); | |
325 | ||
326 | /** | |
327 | * ion_heap_init_shrinker | |
328 | * @heap: the heap | |
329 | * | |
330 | * If a heap sets the ION_HEAP_FLAG_DEFER_FREE flag or defines the shrink op | |
331 | * this function will be called to setup a shrinker to shrink the freelists | |
332 | * and call the heap's shrink op. | |
333 | */ | |
334 | void ion_heap_init_shrinker(struct ion_heap *heap); | |
335 | ||
336 | /** | |
337 | * ion_heap_init_deferred_free -- initialize deferred free functionality | |
338 | * @heap: the heap | |
339 | * | |
340 | * If a heap sets the ION_HEAP_FLAG_DEFER_FREE flag this function will | |
341 | * be called to setup deferred frees. Calls to free the buffer will | |
342 | * return immediately and the actual free will occur some time later | |
343 | */ | |
344 | int ion_heap_init_deferred_free(struct ion_heap *heap); | |
345 | ||
346 | /** | |
347 | * ion_heap_freelist_add - add a buffer to the deferred free list | |
348 | * @heap: the heap | |
349 | * @buffer: the buffer | |
350 | * | |
351 | * Adds an item to the deferred freelist. | |
352 | */ | |
353 | void ion_heap_freelist_add(struct ion_heap *heap, struct ion_buffer *buffer); | |
354 | ||
355 | /** | |
356 | * ion_heap_freelist_drain - drain the deferred free list | |
357 | * @heap: the heap | |
358 | * @size: ammount of memory to drain in bytes | |
359 | * | |
360 | * Drains the indicated amount of memory from the deferred freelist immediately. | |
361 | * Returns the total amount freed. The total freed may be higher depending | |
362 | * on the size of the items in the list, or lower if there is insufficient | |
363 | * total memory on the freelist. | |
364 | */ | |
365 | size_t ion_heap_freelist_drain(struct ion_heap *heap, size_t size); | |
366 | ||
367 | /** | |
368 | * ion_heap_freelist_shrink - drain the deferred free | |
369 | * list, skipping any heap-specific | |
370 | * pooling or caching mechanisms | |
371 | * | |
372 | * @heap: the heap | |
373 | * @size: amount of memory to drain in bytes | |
374 | * | |
375 | * Drains the indicated amount of memory from the deferred freelist immediately. | |
376 | * Returns the total amount freed. The total freed may be higher depending | |
377 | * on the size of the items in the list, or lower if there is insufficient | |
378 | * total memory on the freelist. | |
379 | * | |
380 | * Unlike with @ion_heap_freelist_drain, don't put any pages back into | |
381 | * page pools or otherwise cache the pages. Everything must be | |
382 | * genuinely free'd back to the system. If you're free'ing from a | |
383 | * shrinker you probably want to use this. Note that this relies on | |
384 | * the heap.ops.free callback honoring the ION_PRIV_FLAG_SHRINKER_FREE | |
385 | * flag. | |
386 | */ | |
387 | size_t ion_heap_freelist_shrink(struct ion_heap *heap, | |
388 | size_t size); | |
389 | ||
390 | /** | |
391 | * ion_heap_freelist_size - returns the size of the freelist in bytes | |
392 | * @heap: the heap | |
393 | */ | |
394 | size_t ion_heap_freelist_size(struct ion_heap *heap); | |
395 | ||
396 | ||
397 | /** | |
398 | * functions for creating and destroying the built in ion heaps. | |
399 | * architectures can add their own custom architecture specific | |
400 | * heaps as appropriate. | |
401 | */ | |
402 | ||
403 | struct ion_heap *ion_heap_create(struct ion_platform_heap *); | |
404 | void ion_heap_destroy(struct ion_heap *); | |
405 | struct ion_heap *ion_system_heap_create(struct ion_platform_heap *); | |
406 | void ion_system_heap_destroy(struct ion_heap *); | |
407 | ||
408 | struct ion_heap *ion_system_contig_heap_create(struct ion_platform_heap *); | |
409 | void ion_system_contig_heap_destroy(struct ion_heap *); | |
410 | ||
411 | struct ion_heap *ion_carveout_heap_create(struct ion_platform_heap *); | |
412 | void ion_carveout_heap_destroy(struct ion_heap *); | |
413 | ||
414 | struct ion_heap *ion_chunk_heap_create(struct ion_platform_heap *); | |
415 | void ion_chunk_heap_destroy(struct ion_heap *); | |
416 | ||
417 | struct ion_heap *ion_mm_heap_create(struct ion_platform_heap *); | |
418 | void ion_mm_heap_destroy(struct ion_heap *); | |
419 | ||
420 | struct ion_heap *ion_cma_heap_create(struct ion_platform_heap *); | |
421 | void ion_cma_heap_destroy(struct ion_heap *); | |
422 | ||
423 | /** | |
424 | * kernel api to allocate/free from carveout -- used when carveout is | |
425 | * used to back an architecture specific custom heap | |
426 | */ | |
427 | ion_phys_addr_t ion_carveout_allocate(struct ion_heap *heap, unsigned long size, | |
428 | unsigned long align); | |
429 | void ion_carveout_free(struct ion_heap *heap, ion_phys_addr_t addr, | |
430 | unsigned long size); | |
431 | /** | |
432 | * The carveout heap returns physical addresses, since 0 may be a valid | |
433 | * physical address, this is used to indicate allocation failed | |
434 | */ | |
435 | #define ION_CARVEOUT_ALLOCATE_FAIL -1 | |
436 | ||
437 | /** | |
438 | * functions for creating and destroying a heap pool -- allows you | |
439 | * to keep a pool of pre allocated memory to use from your heap. Keeping | |
440 | * a pool of memory that is ready for dma, ie any cached mapping have been | |
441 | * invalidated from the cache, provides a significant peformance benefit on | |
442 | * many systems */ | |
443 | ||
444 | /** | |
445 | * struct ion_page_pool - pagepool struct | |
446 | * @high_count: number of highmem items in the pool | |
447 | * @low_count: number of lowmem items in the pool | |
448 | * @high_items: list of highmem items | |
449 | * @low_items: list of lowmem items | |
450 | * @mutex: lock protecting this struct and especially the count | |
451 | * item list | |
452 | * @gfp_mask: gfp_mask to use from alloc | |
453 | * @order: order of pages in the pool | |
454 | * @list: plist node for list of pools | |
455 | * | |
456 | * Allows you to keep a pool of pre allocated pages to use from your heap. | |
457 | * Keeping a pool of pages that is ready for dma, ie any cached mapping have | |
458 | * been invalidated from the cache, provides a significant peformance benefit | |
459 | * on many systems | |
460 | */ | |
461 | struct ion_page_pool { | |
462 | int high_count; | |
463 | int low_count; | |
464 | struct list_head high_items; | |
465 | struct list_head low_items; | |
466 | struct mutex mutex; | |
467 | gfp_t gfp_mask; | |
468 | unsigned int order; | |
469 | struct plist_node list; | |
470 | }; | |
471 | ||
472 | struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order); | |
473 | void ion_page_pool_destroy(struct ion_page_pool *); | |
474 | void *ion_page_pool_alloc(struct ion_page_pool *); | |
475 | void ion_page_pool_free(struct ion_page_pool *, struct page *); | |
476 | ||
477 | /** ion_page_pool_shrink - shrinks the size of the memory cached in the pool | |
478 | * @pool: the pool | |
479 | * @gfp_mask: the memory type to reclaim | |
480 | * @nr_to_scan: number of items to shrink in pages | |
481 | * | |
482 | * returns the number of items freed in pages | |
483 | */ | |
484 | int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, | |
485 | int nr_to_scan); | |
486 | ||
487 | /** | |
488 | * ion_pages_sync_for_device - cache flush pages for use with the specified | |
489 | * device | |
490 | * @dev: the device the pages will be used with | |
491 | * @page: the first page to be flushed | |
492 | * @size: size in bytes of region to be flushed | |
493 | * @dir: direction of dma transfer | |
494 | */ | |
495 | void ion_pages_sync_for_device(struct device *dev, struct page *page, | |
496 | size_t size, enum dma_data_direction dir); | |
497 | ||
498 | #endif /* _ION_PRIV_H */ |