3 * drivers/staging/android/ion/ion.c
5 * Copyright (C) 2011 Google, Inc.
7 * This software is licensed under the terms of the GNU General Public
8 * License version 2, as published by the Free Software Foundation, and
9 * may be copied, distributed, and modified under those terms.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
18 #include <linux/device.h>
19 #include <linux/err.h>
20 #include <linux/file.h>
21 #include <linux/freezer.h>
23 #include <linux/anon_inodes.h>
24 #include <linux/kthread.h>
25 #include <linux/list.h>
26 #include <linux/memblock.h>
27 #include <linux/miscdevice.h>
28 #include <linux/export.h>
30 #include <linux/mm_types.h>
31 #include <linux/rbtree.h>
32 #include <linux/slab.h>
33 #include <linux/seq_file.h>
34 #include <linux/uaccess.h>
35 #include <linux/vmalloc.h>
36 #include <linux/debugfs.h>
37 #include <linux/dma-buf.h>
38 #include <linux/idr.h>
42 #include "compat_ion.h"
45 * struct ion_device - the metadata of the ion device node
46 * @dev: the actual misc device
47 * @buffers: an rb tree of all the existing buffers
48 * @buffer_lock: lock protecting the tree of buffers
49 * @lock: rwsem protecting the tree of heaps and clients
50 * @heaps: list of all the heaps in the system
51 * @user_clients: list of all the clients created from userspace
54 struct miscdevice dev
;
55 struct rb_root buffers
;
56 struct mutex buffer_lock
;
57 struct rw_semaphore lock
;
58 struct plist_head heaps
;
59 long (*custom_ioctl
)(struct ion_client
*client
, unsigned int cmd
,
61 struct rb_root clients
;
62 struct dentry
*debug_root
;
63 struct dentry
*heaps_debug_root
;
64 struct dentry
*clients_debug_root
;
68 * struct ion_client - a process/hw block local address space
69 * @node: node in the tree of all clients
70 * @dev: backpointer to ion device
71 * @handles: an rb tree of all the handles in this client
72 * @idr: an idr space for allocating handle ids
73 * @lock: lock protecting the tree of handles
74 * @name: used for debugging
75 * @display_name: used for debugging (unique version of @name)
76 * @display_serial: used for debugging (to make display_name unique)
77 * @task: used for debugging
79 * A client represents a list of buffers this client may access.
80 * The mutex stored here is used to protect both handles tree
81 * as well as the handles themselves, and should be held while modifying either.
85 struct ion_device
*dev
;
86 struct rb_root handles
;
92 struct task_struct
*task
;
94 struct dentry
*debug_root
;
98 * ion_handle - a client local reference to a buffer
99 * @ref: reference count
100 * @client: back pointer to the client the buffer resides in
101 * @buffer: pointer to the buffer
102 * @node: node in the client's handle rbtree
103 * @kmap_cnt: count of times this client has mapped to kernel
104 * @id: client-unique id allocated by client->idr
106 * Modifications to node, map_cnt or mapping should be protected by the
107 * lock in the client. Other fields are never changed after initialization.
111 struct ion_client
*client
;
112 struct ion_buffer
*buffer
;
114 unsigned int kmap_cnt
;
118 bool ion_buffer_fault_user_mappings(struct ion_buffer
*buffer
)
120 return (buffer
->flags
& ION_FLAG_CACHED
) &&
121 !(buffer
->flags
& ION_FLAG_CACHED_NEEDS_SYNC
);
124 bool ion_buffer_cached(struct ion_buffer
*buffer
)
126 return !!(buffer
->flags
& ION_FLAG_CACHED
);
129 static inline struct page
*ion_buffer_page(struct page
*page
)
131 return (struct page
*)((unsigned long)page
& ~(1UL));
134 static inline bool ion_buffer_page_is_dirty(struct page
*page
)
136 return !!((unsigned long)page
& 1UL);
139 static inline void ion_buffer_page_dirty(struct page
**page
)
141 *page
= (struct page
*)((unsigned long)(*page
) | 1UL);
144 static inline void ion_buffer_page_clean(struct page
**page
)
146 *page
= (struct page
*)((unsigned long)(*page
) & ~(1UL));
149 /* this function should only be called while dev->lock is held */
150 static void ion_buffer_add(struct ion_device
*dev
,
151 struct ion_buffer
*buffer
)
153 struct rb_node
**p
= &dev
->buffers
.rb_node
;
154 struct rb_node
*parent
= NULL
;
155 struct ion_buffer
*entry
;
159 entry
= rb_entry(parent
, struct ion_buffer
, node
);
161 if (buffer
< entry
) {
163 } else if (buffer
> entry
) {
166 pr_err("%s: buffer already found.", __func__
);
171 rb_link_node(&buffer
->node
, parent
, p
);
172 rb_insert_color(&buffer
->node
, &dev
->buffers
);
175 /* this function should only be called while dev->lock is held */
176 static struct ion_buffer
*ion_buffer_create(struct ion_heap
*heap
,
177 struct ion_device
*dev
,
182 struct ion_buffer
*buffer
;
183 struct sg_table
*table
;
184 struct scatterlist
*sg
;
187 buffer
= kzalloc(sizeof(struct ion_buffer
), GFP_KERNEL
);
189 return ERR_PTR(-ENOMEM
);
192 buffer
->flags
= flags
;
193 kref_init(&buffer
->ref
);
195 ret
= heap
->ops
->allocate(heap
, buffer
, len
, align
, flags
);
198 if (!(heap
->flags
& ION_HEAP_FLAG_DEFER_FREE
))
201 ion_heap_freelist_drain(heap
, 0);
202 ret
= heap
->ops
->allocate(heap
, buffer
, len
, align
,
211 table
= heap
->ops
->map_dma(heap
, buffer
);
212 if (WARN_ONCE(table
== NULL
,
213 "heap->ops->map_dma should return ERR_PTR on error"))
214 table
= ERR_PTR(-EINVAL
);
220 buffer
->sg_table
= table
;
221 if (ion_buffer_fault_user_mappings(buffer
)) {
222 int num_pages
= PAGE_ALIGN(buffer
->size
) / PAGE_SIZE
;
223 struct scatterlist
*sg
;
226 buffer
->pages
= vmalloc(sizeof(struct page
*) * num_pages
);
227 if (!buffer
->pages
) {
232 for_each_sg(table
->sgl
, sg
, table
->nents
, i
) {
233 struct page
*page
= sg_page(sg
);
235 for (j
= 0; j
< sg
->length
/ PAGE_SIZE
; j
++)
236 buffer
->pages
[k
++] = page
++;
242 INIT_LIST_HEAD(&buffer
->vmas
);
243 mutex_init(&buffer
->lock
);
245 * this will set up dma addresses for the sglist -- it is not
246 * technically correct as per the dma api -- a specific
247 * device isn't really taking ownership here. However, in practice on
248 * our systems the only dma_address space is physical addresses.
249 * Additionally, we can't afford the overhead of invalidating every
250 * allocation via dma_map_sg. The implicit contract here is that
251 * memory coming from the heaps is ready for dma, ie if it has a
252 * cached mapping that mapping has been invalidated
254 for_each_sg(buffer
->sg_table
->sgl
, sg
, buffer
->sg_table
->nents
, i
) {
255 sg_dma_address(sg
) = sg_phys(sg
);
256 sg_dma_len(sg
) = sg
->length
;
258 mutex_lock(&dev
->buffer_lock
);
259 ion_buffer_add(dev
, buffer
);
260 mutex_unlock(&dev
->buffer_lock
);
264 heap
->ops
->unmap_dma(heap
, buffer
);
266 heap
->ops
->free(buffer
);
272 void ion_buffer_destroy(struct ion_buffer
*buffer
)
274 if (WARN_ON(buffer
->kmap_cnt
> 0))
275 buffer
->heap
->ops
->unmap_kernel(buffer
->heap
, buffer
);
276 buffer
->heap
->ops
->unmap_dma(buffer
->heap
, buffer
);
277 buffer
->heap
->ops
->free(buffer
);
278 vfree(buffer
->pages
);
282 static void _ion_buffer_destroy(struct kref
*kref
)
284 struct ion_buffer
*buffer
= container_of(kref
, struct ion_buffer
, ref
);
285 struct ion_heap
*heap
= buffer
->heap
;
286 struct ion_device
*dev
= buffer
->dev
;
288 mutex_lock(&dev
->buffer_lock
);
289 rb_erase(&buffer
->node
, &dev
->buffers
);
290 mutex_unlock(&dev
->buffer_lock
);
292 if (heap
->flags
& ION_HEAP_FLAG_DEFER_FREE
)
293 ion_heap_freelist_add(heap
, buffer
);
295 ion_buffer_destroy(buffer
);
298 static void ion_buffer_get(struct ion_buffer
*buffer
)
300 kref_get(&buffer
->ref
);
303 static int ion_buffer_put(struct ion_buffer
*buffer
)
305 return kref_put(&buffer
->ref
, _ion_buffer_destroy
);
308 static void ion_buffer_add_to_handle(struct ion_buffer
*buffer
)
310 mutex_lock(&buffer
->lock
);
311 buffer
->handle_count
++;
312 mutex_unlock(&buffer
->lock
);
315 static void ion_buffer_remove_from_handle(struct ion_buffer
*buffer
)
318 * when a buffer is removed from a handle, if it is not in
319 * any other handles, copy the taskcomm and the pid of the
320 * process it's being removed from into the buffer. At this
321 * point there will be no way to track what processes this buffer is
322 * being used by, it only exists as a dma_buf file descriptor.
323 * The taskcomm and pid can provide a debug hint as to where this fd
326 mutex_lock(&buffer
->lock
);
327 buffer
->handle_count
--;
328 BUG_ON(buffer
->handle_count
< 0);
329 if (!buffer
->handle_count
) {
330 struct task_struct
*task
;
332 task
= current
->group_leader
;
333 get_task_comm(buffer
->task_comm
, task
);
334 buffer
->pid
= task_pid_nr(task
);
336 mutex_unlock(&buffer
->lock
);
339 static struct ion_handle
*ion_handle_create(struct ion_client
*client
,
340 struct ion_buffer
*buffer
)
342 struct ion_handle
*handle
;
344 handle
= kzalloc(sizeof(struct ion_handle
), GFP_KERNEL
);
346 return ERR_PTR(-ENOMEM
);
347 kref_init(&handle
->ref
);
348 RB_CLEAR_NODE(&handle
->node
);
349 handle
->client
= client
;
350 ion_buffer_get(buffer
);
351 ion_buffer_add_to_handle(buffer
);
352 handle
->buffer
= buffer
;
357 static void ion_handle_kmap_put(struct ion_handle
*);
359 static void ion_handle_destroy(struct kref
*kref
)
361 struct ion_handle
*handle
= container_of(kref
, struct ion_handle
, ref
);
362 struct ion_client
*client
= handle
->client
;
363 struct ion_buffer
*buffer
= handle
->buffer
;
365 mutex_lock(&buffer
->lock
);
366 while (handle
->kmap_cnt
)
367 ion_handle_kmap_put(handle
);
368 mutex_unlock(&buffer
->lock
);
370 idr_remove(&client
->idr
, handle
->id
);
371 if (!RB_EMPTY_NODE(&handle
->node
))
372 rb_erase(&handle
->node
, &client
->handles
);
374 ion_buffer_remove_from_handle(buffer
);
375 ion_buffer_put(buffer
);
380 struct ion_buffer
*ion_handle_buffer(struct ion_handle
*handle
)
382 return handle
->buffer
;
385 static void ion_handle_get(struct ion_handle
*handle
)
387 kref_get(&handle
->ref
);
390 static int ion_handle_put_nolock(struct ion_handle
*handle
)
394 ret
= kref_put(&handle
->ref
, ion_handle_destroy
);
399 int ion_handle_put(struct ion_handle
*handle
)
401 struct ion_client
*client
= handle
->client
;
404 mutex_lock(&client
->lock
);
405 ret
= ion_handle_put_nolock(handle
);
406 mutex_unlock(&client
->lock
);
411 static struct ion_handle
*ion_handle_lookup(struct ion_client
*client
,
412 struct ion_buffer
*buffer
)
414 struct rb_node
*n
= client
->handles
.rb_node
;
417 struct ion_handle
*entry
= rb_entry(n
, struct ion_handle
, node
);
419 if (buffer
< entry
->buffer
)
421 else if (buffer
> entry
->buffer
)
426 return ERR_PTR(-EINVAL
);
429 static struct ion_handle
*ion_handle_get_by_id_nolock(struct ion_client
*client
,
432 struct ion_handle
*handle
;
434 handle
= idr_find(&client
->idr
, id
);
436 ion_handle_get(handle
);
438 return handle
? handle
: ERR_PTR(-EINVAL
);
441 struct ion_handle
*ion_handle_get_by_id(struct ion_client
*client
,
444 struct ion_handle
*handle
;
446 mutex_lock(&client
->lock
);
447 handle
= ion_handle_get_by_id_nolock(client
, id
);
448 mutex_unlock(&client
->lock
);
453 static bool ion_handle_validate(struct ion_client
*client
,
454 struct ion_handle
*handle
)
456 WARN_ON(!mutex_is_locked(&client
->lock
));
457 return idr_find(&client
->idr
, handle
->id
) == handle
;
460 static int ion_handle_add(struct ion_client
*client
, struct ion_handle
*handle
)
463 struct rb_node
**p
= &client
->handles
.rb_node
;
464 struct rb_node
*parent
= NULL
;
465 struct ion_handle
*entry
;
467 id
= idr_alloc(&client
->idr
, handle
, 1, 0, GFP_KERNEL
);
475 entry
= rb_entry(parent
, struct ion_handle
, node
);
477 if (handle
->buffer
< entry
->buffer
)
479 else if (handle
->buffer
> entry
->buffer
)
482 WARN(1, "%s: buffer already found.", __func__
);
485 rb_link_node(&handle
->node
, parent
, p
);
486 rb_insert_color(&handle
->node
, &client
->handles
);
491 struct ion_handle
*ion_alloc(struct ion_client
*client
, size_t len
,
492 size_t align
, unsigned int heap_id_mask
,
495 struct ion_handle
*handle
;
496 struct ion_device
*dev
= client
->dev
;
497 struct ion_buffer
*buffer
= NULL
;
498 struct ion_heap
*heap
;
501 pr_debug("%s: len %zu align %zu heap_id_mask %u flags %x\n", __func__
,
502 len
, align
, heap_id_mask
, flags
);
504 * traverse the list of heaps available in this system in priority
505 * order. If the heap type is supported by the client, and matches the
506 * request of the caller allocate from it. Repeat until allocate has
507 * succeeded or all heaps have been tried
509 len
= PAGE_ALIGN(len
);
512 return ERR_PTR(-EINVAL
);
514 down_read(&dev
->lock
);
515 plist_for_each_entry(heap
, &dev
->heaps
, node
) {
516 /* if the caller didn't specify this heap id */
517 if (!((1 << heap
->id
) & heap_id_mask
))
519 buffer
= ion_buffer_create(heap
, dev
, len
, align
, flags
);
526 return ERR_PTR(-ENODEV
);
529 return ERR_CAST(buffer
);
531 handle
= ion_handle_create(client
, buffer
);
534 * ion_buffer_create will create a buffer with a ref_cnt of 1,
535 * and ion_handle_create will take a second reference, drop one here
537 ion_buffer_put(buffer
);
542 mutex_lock(&client
->lock
);
543 ret
= ion_handle_add(client
, handle
);
544 mutex_unlock(&client
->lock
);
546 ion_handle_put(handle
);
547 handle
= ERR_PTR(ret
);
552 EXPORT_SYMBOL(ion_alloc
);
554 static void ion_free_nolock(struct ion_client
*client
, struct ion_handle
*handle
)
558 BUG_ON(client
!= handle
->client
);
560 valid_handle
= ion_handle_validate(client
, handle
);
563 WARN(1, "%s: invalid handle passed to free.\n", __func__
);
566 ion_handle_put_nolock(handle
);
569 void ion_free(struct ion_client
*client
, struct ion_handle
*handle
)
571 BUG_ON(client
!= handle
->client
);
573 mutex_lock(&client
->lock
);
574 ion_free_nolock(client
, handle
);
575 mutex_unlock(&client
->lock
);
577 EXPORT_SYMBOL(ion_free
);
579 int ion_phys(struct ion_client
*client
, struct ion_handle
*handle
,
580 ion_phys_addr_t
*addr
, size_t *len
)
582 struct ion_buffer
*buffer
;
585 mutex_lock(&client
->lock
);
586 if (!ion_handle_validate(client
, handle
)) {
587 mutex_unlock(&client
->lock
);
591 buffer
= handle
->buffer
;
593 if (!buffer
->heap
->ops
->phys
) {
594 pr_err("%s: ion_phys is not implemented by this heap (name=%s, type=%d).\n",
595 __func__
, buffer
->heap
->name
, buffer
->heap
->type
);
596 mutex_unlock(&client
->lock
);
599 mutex_unlock(&client
->lock
);
600 ret
= buffer
->heap
->ops
->phys(buffer
->heap
, buffer
, addr
, len
);
603 EXPORT_SYMBOL(ion_phys
);
605 static void *ion_buffer_kmap_get(struct ion_buffer
*buffer
)
609 if (buffer
->kmap_cnt
) {
611 return buffer
->vaddr
;
613 vaddr
= buffer
->heap
->ops
->map_kernel(buffer
->heap
, buffer
);
614 if (WARN_ONCE(vaddr
== NULL
,
615 "heap->ops->map_kernel should return ERR_PTR on error"))
616 return ERR_PTR(-EINVAL
);
619 buffer
->vaddr
= vaddr
;
624 static void *ion_handle_kmap_get(struct ion_handle
*handle
)
626 struct ion_buffer
*buffer
= handle
->buffer
;
629 if (handle
->kmap_cnt
) {
631 return buffer
->vaddr
;
633 vaddr
= ion_buffer_kmap_get(buffer
);
640 static void ion_buffer_kmap_put(struct ion_buffer
*buffer
)
643 if (!buffer
->kmap_cnt
) {
644 buffer
->heap
->ops
->unmap_kernel(buffer
->heap
, buffer
);
645 buffer
->vaddr
= NULL
;
649 static void ion_handle_kmap_put(struct ion_handle
*handle
)
651 struct ion_buffer
*buffer
= handle
->buffer
;
653 if (!handle
->kmap_cnt
) {
654 WARN(1, "%s: Double unmap detected! bailing...\n", __func__
);
658 if (!handle
->kmap_cnt
)
659 ion_buffer_kmap_put(buffer
);
662 void *ion_map_kernel(struct ion_client
*client
, struct ion_handle
*handle
)
664 struct ion_buffer
*buffer
;
667 mutex_lock(&client
->lock
);
668 if (!ion_handle_validate(client
, handle
)) {
669 pr_err("%s: invalid handle passed to map_kernel.\n",
671 mutex_unlock(&client
->lock
);
672 return ERR_PTR(-EINVAL
);
675 buffer
= handle
->buffer
;
677 if (!handle
->buffer
->heap
->ops
->map_kernel
) {
678 pr_err("%s: map_kernel is not implemented by this heap.\n",
680 mutex_unlock(&client
->lock
);
681 return ERR_PTR(-ENODEV
);
684 mutex_lock(&buffer
->lock
);
685 vaddr
= ion_handle_kmap_get(handle
);
686 mutex_unlock(&buffer
->lock
);
687 mutex_unlock(&client
->lock
);
690 EXPORT_SYMBOL(ion_map_kernel
);
692 void ion_unmap_kernel(struct ion_client
*client
, struct ion_handle
*handle
)
694 struct ion_buffer
*buffer
;
696 mutex_lock(&client
->lock
);
697 buffer
= handle
->buffer
;
698 mutex_lock(&buffer
->lock
);
699 ion_handle_kmap_put(handle
);
700 mutex_unlock(&buffer
->lock
);
701 mutex_unlock(&client
->lock
);
703 EXPORT_SYMBOL(ion_unmap_kernel
);
705 static int ion_debug_client_show(struct seq_file
*s
, void *unused
)
707 struct ion_client
*client
= s
->private;
709 size_t sizes
[ION_NUM_HEAP_IDS
] = {0};
710 const char *names
[ION_NUM_HEAP_IDS
] = {NULL
};
713 mutex_lock(&client
->lock
);
714 for (n
= rb_first(&client
->handles
); n
; n
= rb_next(n
)) {
715 struct ion_handle
*handle
= rb_entry(n
, struct ion_handle
,
717 unsigned int id
= handle
->buffer
->heap
->id
;
720 names
[id
] = handle
->buffer
->heap
->name
;
721 sizes
[id
] += handle
->buffer
->size
;
723 mutex_unlock(&client
->lock
);
725 seq_printf(s
, "%16.16s: %16.16s\n", "heap_name", "size_in_bytes");
726 for (i
= 0; i
< ION_NUM_HEAP_IDS
; i
++) {
729 seq_printf(s
, "%16.16s: %16zu\n", names
[i
], sizes
[i
]);
734 static int ion_debug_client_open(struct inode
*inode
, struct file
*file
)
736 return single_open(file
, ion_debug_client_show
, inode
->i_private
);
739 static const struct file_operations debug_client_fops
= {
740 .open
= ion_debug_client_open
,
743 .release
= single_release
,
746 static int ion_get_client_serial(const struct rb_root
*root
,
747 const unsigned char *name
)
750 struct rb_node
*node
;
752 for (node
= rb_first(root
); node
; node
= rb_next(node
)) {
753 struct ion_client
*client
= rb_entry(node
, struct ion_client
,
756 if (strcmp(client
->name
, name
))
758 serial
= max(serial
, client
->display_serial
);
763 struct ion_client
*ion_client_create(struct ion_device
*dev
,
766 struct ion_client
*client
;
767 struct task_struct
*task
;
769 struct rb_node
*parent
= NULL
;
770 struct ion_client
*entry
;
774 pr_err("%s: Name cannot be null\n", __func__
);
775 return ERR_PTR(-EINVAL
);
778 get_task_struct(current
->group_leader
);
779 task_lock(current
->group_leader
);
780 pid
= task_pid_nr(current
->group_leader
);
782 * don't bother to store task struct for kernel threads,
783 * they can't be killed anyway
785 if (current
->group_leader
->flags
& PF_KTHREAD
) {
786 put_task_struct(current
->group_leader
);
789 task
= current
->group_leader
;
791 task_unlock(current
->group_leader
);
793 client
= kzalloc(sizeof(struct ion_client
), GFP_KERNEL
);
795 goto err_put_task_struct
;
798 client
->handles
= RB_ROOT
;
799 idr_init(&client
->idr
);
800 mutex_init(&client
->lock
);
803 client
->name
= kstrdup(name
, GFP_KERNEL
);
805 goto err_free_client
;
807 down_write(&dev
->lock
);
808 client
->display_serial
= ion_get_client_serial(&dev
->clients
, name
);
809 client
->display_name
= kasprintf(
810 GFP_KERNEL
, "%s-%d", name
, client
->display_serial
);
811 if (!client
->display_name
) {
812 up_write(&dev
->lock
);
813 goto err_free_client_name
;
815 p
= &dev
->clients
.rb_node
;
818 entry
= rb_entry(parent
, struct ion_client
, node
);
822 else if (client
> entry
)
825 rb_link_node(&client
->node
, parent
, p
);
826 rb_insert_color(&client
->node
, &dev
->clients
);
828 client
->debug_root
= debugfs_create_file(client
->display_name
, 0664,
829 dev
->clients_debug_root
,
830 client
, &debug_client_fops
);
831 if (!client
->debug_root
) {
832 char buf
[256], *path
;
834 path
= dentry_path(dev
->clients_debug_root
, buf
, 256);
835 pr_err("Failed to create client debugfs at %s/%s\n",
836 path
, client
->display_name
);
839 up_write(&dev
->lock
);
843 err_free_client_name
:
849 put_task_struct(current
->group_leader
);
850 return ERR_PTR(-ENOMEM
);
852 EXPORT_SYMBOL(ion_client_create
);
854 void ion_client_destroy(struct ion_client
*client
)
856 struct ion_device
*dev
= client
->dev
;
859 pr_debug("%s: %d\n", __func__
, __LINE__
);
860 while ((n
= rb_first(&client
->handles
))) {
861 struct ion_handle
*handle
= rb_entry(n
, struct ion_handle
,
863 ion_handle_destroy(&handle
->ref
);
866 idr_destroy(&client
->idr
);
868 down_write(&dev
->lock
);
870 put_task_struct(client
->task
);
871 rb_erase(&client
->node
, &dev
->clients
);
872 debugfs_remove_recursive(client
->debug_root
);
873 up_write(&dev
->lock
);
875 kfree(client
->display_name
);
879 EXPORT_SYMBOL(ion_client_destroy
);
881 struct sg_table
*ion_sg_table(struct ion_client
*client
,
882 struct ion_handle
*handle
)
884 struct ion_buffer
*buffer
;
885 struct sg_table
*table
;
887 mutex_lock(&client
->lock
);
888 if (!ion_handle_validate(client
, handle
)) {
889 pr_err("%s: invalid handle passed to map_dma.\n",
891 mutex_unlock(&client
->lock
);
892 return ERR_PTR(-EINVAL
);
894 buffer
= handle
->buffer
;
895 table
= buffer
->sg_table
;
896 mutex_unlock(&client
->lock
);
899 EXPORT_SYMBOL(ion_sg_table
);
901 static void ion_buffer_sync_for_device(struct ion_buffer
*buffer
,
903 enum dma_data_direction direction
);
905 static struct sg_table
*ion_map_dma_buf(struct dma_buf_attachment
*attachment
,
906 enum dma_data_direction direction
)
908 struct dma_buf
*dmabuf
= attachment
->dmabuf
;
909 struct ion_buffer
*buffer
= dmabuf
->priv
;
911 ion_buffer_sync_for_device(buffer
, attachment
->dev
, direction
);
912 return buffer
->sg_table
;
915 static void ion_unmap_dma_buf(struct dma_buf_attachment
*attachment
,
916 struct sg_table
*table
,
917 enum dma_data_direction direction
)
921 void ion_pages_sync_for_device(struct device
*dev
, struct page
*page
,
922 size_t size
, enum dma_data_direction dir
)
924 struct scatterlist sg
;
926 sg_init_table(&sg
, 1);
927 sg_set_page(&sg
, page
, size
, 0);
929 * This is not correct - sg_dma_address needs a dma_addr_t that is valid
930 * for the targeted device, but this works on the currently targeted
933 sg_dma_address(&sg
) = page_to_phys(page
);
934 dma_sync_sg_for_device(dev
, &sg
, 1, dir
);
937 struct ion_vma_list
{
938 struct list_head list
;
939 struct vm_area_struct
*vma
;
942 static void ion_buffer_sync_for_device(struct ion_buffer
*buffer
,
944 enum dma_data_direction dir
)
946 struct ion_vma_list
*vma_list
;
947 int pages
= PAGE_ALIGN(buffer
->size
) / PAGE_SIZE
;
950 pr_debug("%s: syncing for device %s\n", __func__
,
951 dev
? dev_name(dev
) : "null");
953 if (!ion_buffer_fault_user_mappings(buffer
))
956 mutex_lock(&buffer
->lock
);
957 for (i
= 0; i
< pages
; i
++) {
958 struct page
*page
= buffer
->pages
[i
];
960 if (ion_buffer_page_is_dirty(page
))
961 ion_pages_sync_for_device(dev
, ion_buffer_page(page
),
964 ion_buffer_page_clean(buffer
->pages
+ i
);
966 list_for_each_entry(vma_list
, &buffer
->vmas
, list
) {
967 struct vm_area_struct
*vma
= vma_list
->vma
;
969 zap_page_range(vma
, vma
->vm_start
, vma
->vm_end
- vma
->vm_start
,
972 mutex_unlock(&buffer
->lock
);
975 static int ion_vm_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
977 struct ion_buffer
*buffer
= vma
->vm_private_data
;
981 mutex_lock(&buffer
->lock
);
982 ion_buffer_page_dirty(buffer
->pages
+ vmf
->pgoff
);
983 BUG_ON(!buffer
->pages
|| !buffer
->pages
[vmf
->pgoff
]);
985 pfn
= page_to_pfn(ion_buffer_page(buffer
->pages
[vmf
->pgoff
]));
986 ret
= vm_insert_pfn(vma
, (unsigned long)vmf
->virtual_address
, pfn
);
987 mutex_unlock(&buffer
->lock
);
989 return VM_FAULT_ERROR
;
991 return VM_FAULT_NOPAGE
;
994 static void ion_vm_open(struct vm_area_struct
*vma
)
996 struct ion_buffer
*buffer
= vma
->vm_private_data
;
997 struct ion_vma_list
*vma_list
;
999 vma_list
= kmalloc(sizeof(struct ion_vma_list
), GFP_KERNEL
);
1002 vma_list
->vma
= vma
;
1003 mutex_lock(&buffer
->lock
);
1004 list_add(&vma_list
->list
, &buffer
->vmas
);
1005 mutex_unlock(&buffer
->lock
);
1006 pr_debug("%s: adding %p\n", __func__
, vma
);
1009 static void ion_vm_close(struct vm_area_struct
*vma
)
1011 struct ion_buffer
*buffer
= vma
->vm_private_data
;
1012 struct ion_vma_list
*vma_list
, *tmp
;
1014 pr_debug("%s\n", __func__
);
1015 mutex_lock(&buffer
->lock
);
1016 list_for_each_entry_safe(vma_list
, tmp
, &buffer
->vmas
, list
) {
1017 if (vma_list
->vma
!= vma
)
1019 list_del(&vma_list
->list
);
1021 pr_debug("%s: deleting %p\n", __func__
, vma
);
1024 mutex_unlock(&buffer
->lock
);
1027 static const struct vm_operations_struct ion_vma_ops
= {
1028 .open
= ion_vm_open
,
1029 .close
= ion_vm_close
,
1030 .fault
= ion_vm_fault
,
1033 static int ion_mmap(struct dma_buf
*dmabuf
, struct vm_area_struct
*vma
)
1035 struct ion_buffer
*buffer
= dmabuf
->priv
;
1038 if (!buffer
->heap
->ops
->map_user
) {
1039 pr_err("%s: this heap does not define a method for mapping to userspace\n",
1044 if (ion_buffer_fault_user_mappings(buffer
)) {
1045 vma
->vm_flags
|= VM_IO
| VM_PFNMAP
| VM_DONTEXPAND
|
1047 vma
->vm_private_data
= buffer
;
1048 vma
->vm_ops
= &ion_vma_ops
;
1053 if (!(buffer
->flags
& ION_FLAG_CACHED
))
1054 vma
->vm_page_prot
= pgprot_writecombine(vma
->vm_page_prot
);
1056 mutex_lock(&buffer
->lock
);
1057 /* now map it to userspace */
1058 ret
= buffer
->heap
->ops
->map_user(buffer
->heap
, buffer
, vma
);
1059 mutex_unlock(&buffer
->lock
);
1062 pr_err("%s: failure mapping buffer to userspace\n",
1068 static void ion_dma_buf_release(struct dma_buf
*dmabuf
)
1070 struct ion_buffer
*buffer
= dmabuf
->priv
;
1072 ion_buffer_put(buffer
);
1075 static void *ion_dma_buf_kmap(struct dma_buf
*dmabuf
, unsigned long offset
)
1077 struct ion_buffer
*buffer
= dmabuf
->priv
;
1079 return buffer
->vaddr
+ offset
* PAGE_SIZE
;
1082 static void ion_dma_buf_kunmap(struct dma_buf
*dmabuf
, unsigned long offset
,
1087 static int ion_dma_buf_begin_cpu_access(struct dma_buf
*dmabuf
, size_t start
,
1089 enum dma_data_direction direction
)
1091 struct ion_buffer
*buffer
= dmabuf
->priv
;
1094 if (!buffer
->heap
->ops
->map_kernel
) {
1095 pr_err("%s: map kernel is not implemented by this heap.\n",
1100 mutex_lock(&buffer
->lock
);
1101 vaddr
= ion_buffer_kmap_get(buffer
);
1102 mutex_unlock(&buffer
->lock
);
1103 return PTR_ERR_OR_ZERO(vaddr
);
1106 static void ion_dma_buf_end_cpu_access(struct dma_buf
*dmabuf
, size_t start
,
1108 enum dma_data_direction direction
)
1110 struct ion_buffer
*buffer
= dmabuf
->priv
;
1112 mutex_lock(&buffer
->lock
);
1113 ion_buffer_kmap_put(buffer
);
1114 mutex_unlock(&buffer
->lock
);
1117 static struct dma_buf_ops dma_buf_ops
= {
1118 .map_dma_buf
= ion_map_dma_buf
,
1119 .unmap_dma_buf
= ion_unmap_dma_buf
,
1121 .release
= ion_dma_buf_release
,
1122 .begin_cpu_access
= ion_dma_buf_begin_cpu_access
,
1123 .end_cpu_access
= ion_dma_buf_end_cpu_access
,
1124 .kmap_atomic
= ion_dma_buf_kmap
,
1125 .kunmap_atomic
= ion_dma_buf_kunmap
,
1126 .kmap
= ion_dma_buf_kmap
,
1127 .kunmap
= ion_dma_buf_kunmap
,
1130 struct dma_buf
*ion_share_dma_buf(struct ion_client
*client
,
1131 struct ion_handle
*handle
)
1133 DEFINE_DMA_BUF_EXPORT_INFO(exp_info
);
1134 struct ion_buffer
*buffer
;
1135 struct dma_buf
*dmabuf
;
1138 mutex_lock(&client
->lock
);
1139 valid_handle
= ion_handle_validate(client
, handle
);
1140 if (!valid_handle
) {
1141 WARN(1, "%s: invalid handle passed to share.\n", __func__
);
1142 mutex_unlock(&client
->lock
);
1143 return ERR_PTR(-EINVAL
);
1145 buffer
= handle
->buffer
;
1146 ion_buffer_get(buffer
);
1147 mutex_unlock(&client
->lock
);
1149 exp_info
.ops
= &dma_buf_ops
;
1150 exp_info
.size
= buffer
->size
;
1151 exp_info
.flags
= O_RDWR
;
1152 exp_info
.priv
= buffer
;
1154 dmabuf
= dma_buf_export(&exp_info
);
1155 if (IS_ERR(dmabuf
)) {
1156 ion_buffer_put(buffer
);
1162 EXPORT_SYMBOL(ion_share_dma_buf
);
1164 int ion_share_dma_buf_fd(struct ion_client
*client
, struct ion_handle
*handle
)
1166 struct dma_buf
*dmabuf
;
1169 dmabuf
= ion_share_dma_buf(client
, handle
);
1171 return PTR_ERR(dmabuf
);
1173 fd
= dma_buf_fd(dmabuf
, O_CLOEXEC
);
1175 dma_buf_put(dmabuf
);
1179 EXPORT_SYMBOL(ion_share_dma_buf_fd
);
1181 struct ion_handle
*ion_import_dma_buf(struct ion_client
*client
, int fd
)
1183 struct dma_buf
*dmabuf
;
1184 struct ion_buffer
*buffer
;
1185 struct ion_handle
*handle
;
1188 dmabuf
= dma_buf_get(fd
);
1190 return ERR_CAST(dmabuf
);
1191 /* if this memory came from ion */
1193 if (dmabuf
->ops
!= &dma_buf_ops
) {
1194 pr_err("%s: can not import dmabuf from another exporter\n",
1196 dma_buf_put(dmabuf
);
1197 return ERR_PTR(-EINVAL
);
1199 buffer
= dmabuf
->priv
;
1201 mutex_lock(&client
->lock
);
1202 /* if a handle exists for this buffer just take a reference to it */
1203 handle
= ion_handle_lookup(client
, buffer
);
1204 if (!IS_ERR(handle
)) {
1205 ion_handle_get(handle
);
1206 mutex_unlock(&client
->lock
);
1210 handle
= ion_handle_create(client
, buffer
);
1211 if (IS_ERR(handle
)) {
1212 mutex_unlock(&client
->lock
);
1216 ret
= ion_handle_add(client
, handle
);
1217 mutex_unlock(&client
->lock
);
1219 ion_handle_put(handle
);
1220 handle
= ERR_PTR(ret
);
1224 dma_buf_put(dmabuf
);
1227 EXPORT_SYMBOL(ion_import_dma_buf
);
1229 static int ion_sync_for_device(struct ion_client
*client
, int fd
)
1231 struct dma_buf
*dmabuf
;
1232 struct ion_buffer
*buffer
;
1234 dmabuf
= dma_buf_get(fd
);
1236 return PTR_ERR(dmabuf
);
1238 /* if this memory came from ion */
1239 if (dmabuf
->ops
!= &dma_buf_ops
) {
1240 pr_err("%s: can not sync dmabuf from another exporter\n",
1242 dma_buf_put(dmabuf
);
1245 buffer
= dmabuf
->priv
;
1247 dma_sync_sg_for_device(NULL
, buffer
->sg_table
->sgl
,
1248 buffer
->sg_table
->nents
, DMA_BIDIRECTIONAL
);
1249 dma_buf_put(dmabuf
);
1253 /* fix up the cases where the ioctl direction bits are incorrect */
1254 static unsigned int ion_ioctl_dir(unsigned int cmd
)
1259 case ION_IOC_CUSTOM
:
1262 return _IOC_DIR(cmd
);
1266 static long ion_ioctl(struct file
*filp
, unsigned int cmd
, unsigned long arg
)
1268 struct ion_client
*client
= filp
->private_data
;
1269 struct ion_device
*dev
= client
->dev
;
1270 struct ion_handle
*cleanup_handle
= NULL
;
1275 struct ion_fd_data fd
;
1276 struct ion_allocation_data allocation
;
1277 struct ion_handle_data handle
;
1278 struct ion_custom_data custom
;
1281 dir
= ion_ioctl_dir(cmd
);
1283 if (_IOC_SIZE(cmd
) > sizeof(data
))
1286 if (dir
& _IOC_WRITE
)
1287 if (copy_from_user(&data
, (void __user
*)arg
, _IOC_SIZE(cmd
)))
1293 struct ion_handle
*handle
;
1295 handle
= ion_alloc(client
, data
.allocation
.len
,
1296 data
.allocation
.align
,
1297 data
.allocation
.heap_id_mask
,
1298 data
.allocation
.flags
);
1300 return PTR_ERR(handle
);
1302 data
.allocation
.handle
= handle
->id
;
1304 cleanup_handle
= handle
;
1309 struct ion_handle
*handle
;
1311 mutex_lock(&client
->lock
);
1312 handle
= ion_handle_get_by_id_nolock(client
, data
.handle
.handle
);
1313 if (IS_ERR(handle
)) {
1314 mutex_unlock(&client
->lock
);
1315 return PTR_ERR(handle
);
1317 ion_free_nolock(client
, handle
);
1318 ion_handle_put_nolock(handle
);
1319 mutex_unlock(&client
->lock
);
1325 struct ion_handle
*handle
;
1327 handle
= ion_handle_get_by_id(client
, data
.handle
.handle
);
1329 return PTR_ERR(handle
);
1330 data
.fd
.fd
= ion_share_dma_buf_fd(client
, handle
);
1331 ion_handle_put(handle
);
1336 case ION_IOC_IMPORT
:
1338 struct ion_handle
*handle
;
1340 handle
= ion_import_dma_buf(client
, data
.fd
.fd
);
1342 ret
= PTR_ERR(handle
);
1344 data
.handle
.handle
= handle
->id
;
1349 ret
= ion_sync_for_device(client
, data
.fd
.fd
);
1352 case ION_IOC_CUSTOM
:
1354 if (!dev
->custom_ioctl
)
1356 ret
= dev
->custom_ioctl(client
, data
.custom
.cmd
,
1364 if (dir
& _IOC_READ
) {
1365 if (copy_to_user((void __user
*)arg
, &data
, _IOC_SIZE(cmd
))) {
1367 ion_free(client
, cleanup_handle
);
1374 static int ion_release(struct inode
*inode
, struct file
*file
)
1376 struct ion_client
*client
= file
->private_data
;
1378 pr_debug("%s: %d\n", __func__
, __LINE__
);
1379 ion_client_destroy(client
);
1383 static int ion_open(struct inode
*inode
, struct file
*file
)
1385 struct miscdevice
*miscdev
= file
->private_data
;
1386 struct ion_device
*dev
= container_of(miscdev
, struct ion_device
, dev
);
1387 struct ion_client
*client
;
1388 char debug_name
[64];
1390 pr_debug("%s: %d\n", __func__
, __LINE__
);
1391 snprintf(debug_name
, 64, "%u", task_pid_nr(current
->group_leader
));
1392 client
= ion_client_create(dev
, debug_name
);
1394 return PTR_ERR(client
);
1395 file
->private_data
= client
;
1400 static const struct file_operations ion_fops
= {
1401 .owner
= THIS_MODULE
,
1403 .release
= ion_release
,
1404 .unlocked_ioctl
= ion_ioctl
,
1405 .compat_ioctl
= compat_ion_ioctl
,
1408 static size_t ion_debug_heap_total(struct ion_client
*client
,
1414 mutex_lock(&client
->lock
);
1415 for (n
= rb_first(&client
->handles
); n
; n
= rb_next(n
)) {
1416 struct ion_handle
*handle
= rb_entry(n
,
1419 if (handle
->buffer
->heap
->id
== id
)
1420 size
+= handle
->buffer
->size
;
1422 mutex_unlock(&client
->lock
);
1426 static int ion_debug_heap_show(struct seq_file
*s
, void *unused
)
1428 struct ion_heap
*heap
= s
->private;
1429 struct ion_device
*dev
= heap
->dev
;
1431 size_t total_size
= 0;
1432 size_t total_orphaned_size
= 0;
1434 seq_printf(s
, "%16s %16s %16s\n", "client", "pid", "size");
1435 seq_puts(s
, "----------------------------------------------------\n");
1437 for (n
= rb_first(&dev
->clients
); n
; n
= rb_next(n
)) {
1438 struct ion_client
*client
= rb_entry(n
, struct ion_client
,
1440 size_t size
= ion_debug_heap_total(client
, heap
->id
);
1445 char task_comm
[TASK_COMM_LEN
];
1447 get_task_comm(task_comm
, client
->task
);
1448 seq_printf(s
, "%16s %16u %16zu\n", task_comm
,
1451 seq_printf(s
, "%16s %16u %16zu\n", client
->name
,
1455 seq_puts(s
, "----------------------------------------------------\n");
1456 seq_puts(s
, "orphaned allocations (info is from last known client):\n");
1457 mutex_lock(&dev
->buffer_lock
);
1458 for (n
= rb_first(&dev
->buffers
); n
; n
= rb_next(n
)) {
1459 struct ion_buffer
*buffer
= rb_entry(n
, struct ion_buffer
,
1461 if (buffer
->heap
->id
!= heap
->id
)
1463 total_size
+= buffer
->size
;
1464 if (!buffer
->handle_count
) {
1465 seq_printf(s
, "%16s %16u %16zu %d %d\n",
1466 buffer
->task_comm
, buffer
->pid
,
1467 buffer
->size
, buffer
->kmap_cnt
,
1468 atomic_read(&buffer
->ref
.refcount
));
1469 total_orphaned_size
+= buffer
->size
;
1472 mutex_unlock(&dev
->buffer_lock
);
1473 seq_puts(s
, "----------------------------------------------------\n");
1474 seq_printf(s
, "%16s %16zu\n", "total orphaned",
1475 total_orphaned_size
);
1476 seq_printf(s
, "%16s %16zu\n", "total ", total_size
);
1477 if (heap
->flags
& ION_HEAP_FLAG_DEFER_FREE
)
1478 seq_printf(s
, "%16s %16zu\n", "deferred free",
1479 heap
->free_list_size
);
1480 seq_puts(s
, "----------------------------------------------------\n");
1482 if (heap
->debug_show
)
1483 heap
->debug_show(heap
, s
, unused
);
1488 static int ion_debug_heap_open(struct inode
*inode
, struct file
*file
)
1490 return single_open(file
, ion_debug_heap_show
, inode
->i_private
);
1493 static const struct file_operations debug_heap_fops
= {
1494 .open
= ion_debug_heap_open
,
1496 .llseek
= seq_lseek
,
1497 .release
= single_release
,
1500 static int debug_shrink_set(void *data
, u64 val
)
1502 struct ion_heap
*heap
= data
;
1503 struct shrink_control sc
;
1507 sc
.nr_to_scan
= val
;
1510 objs
= heap
->shrinker
.count_objects(&heap
->shrinker
, &sc
);
1511 sc
.nr_to_scan
= objs
;
1514 heap
->shrinker
.scan_objects(&heap
->shrinker
, &sc
);
1518 static int debug_shrink_get(void *data
, u64
*val
)
1520 struct ion_heap
*heap
= data
;
1521 struct shrink_control sc
;
1527 objs
= heap
->shrinker
.count_objects(&heap
->shrinker
, &sc
);
1532 DEFINE_SIMPLE_ATTRIBUTE(debug_shrink_fops
, debug_shrink_get
,
1533 debug_shrink_set
, "%llu\n");
1535 void ion_device_add_heap(struct ion_device
*dev
, struct ion_heap
*heap
)
1537 struct dentry
*debug_file
;
1539 if (!heap
->ops
->allocate
|| !heap
->ops
->free
|| !heap
->ops
->map_dma
||
1540 !heap
->ops
->unmap_dma
)
1541 pr_err("%s: can not add heap with invalid ops struct.\n",
1544 spin_lock_init(&heap
->free_lock
);
1545 heap
->free_list_size
= 0;
1547 if (heap
->flags
& ION_HEAP_FLAG_DEFER_FREE
)
1548 ion_heap_init_deferred_free(heap
);
1550 if ((heap
->flags
& ION_HEAP_FLAG_DEFER_FREE
) || heap
->ops
->shrink
)
1551 ion_heap_init_shrinker(heap
);
1554 down_write(&dev
->lock
);
1556 * use negative heap->id to reverse the priority -- when traversing
1557 * the list later attempt higher id numbers first
1559 plist_node_init(&heap
->node
, -heap
->id
);
1560 plist_add(&heap
->node
, &dev
->heaps
);
1561 debug_file
= debugfs_create_file(heap
->name
, 0664,
1562 dev
->heaps_debug_root
, heap
,
1566 char buf
[256], *path
;
1568 path
= dentry_path(dev
->heaps_debug_root
, buf
, 256);
1569 pr_err("Failed to create heap debugfs at %s/%s\n",
1573 if (heap
->shrinker
.count_objects
&& heap
->shrinker
.scan_objects
) {
1574 char debug_name
[64];
1576 snprintf(debug_name
, 64, "%s_shrink", heap
->name
);
1577 debug_file
= debugfs_create_file(
1578 debug_name
, 0644, dev
->heaps_debug_root
, heap
,
1579 &debug_shrink_fops
);
1581 char buf
[256], *path
;
1583 path
= dentry_path(dev
->heaps_debug_root
, buf
, 256);
1584 pr_err("Failed to create heap shrinker debugfs at %s/%s\n",
1589 up_write(&dev
->lock
);
1591 EXPORT_SYMBOL(ion_device_add_heap
);
1593 struct ion_device
*ion_device_create(long (*custom_ioctl
)
1594 (struct ion_client
*client
,
1598 struct ion_device
*idev
;
1601 idev
= kzalloc(sizeof(struct ion_device
), GFP_KERNEL
);
1603 return ERR_PTR(-ENOMEM
);
1605 idev
->dev
.minor
= MISC_DYNAMIC_MINOR
;
1606 idev
->dev
.name
= "ion";
1607 idev
->dev
.fops
= &ion_fops
;
1608 idev
->dev
.parent
= NULL
;
1609 ret
= misc_register(&idev
->dev
);
1611 pr_err("ion: failed to register misc device.\n");
1613 return ERR_PTR(ret
);
1616 idev
->debug_root
= debugfs_create_dir("ion", NULL
);
1617 if (!idev
->debug_root
) {
1618 pr_err("ion: failed to create debugfs root directory.\n");
1621 idev
->heaps_debug_root
= debugfs_create_dir("heaps", idev
->debug_root
);
1622 if (!idev
->heaps_debug_root
) {
1623 pr_err("ion: failed to create debugfs heaps directory.\n");
1626 idev
->clients_debug_root
= debugfs_create_dir("clients",
1628 if (!idev
->clients_debug_root
)
1629 pr_err("ion: failed to create debugfs clients directory.\n");
1633 idev
->custom_ioctl
= custom_ioctl
;
1634 idev
->buffers
= RB_ROOT
;
1635 mutex_init(&idev
->buffer_lock
);
1636 init_rwsem(&idev
->lock
);
1637 plist_head_init(&idev
->heaps
);
1638 idev
->clients
= RB_ROOT
;
1641 EXPORT_SYMBOL(ion_device_create
);
1643 void ion_device_destroy(struct ion_device
*dev
)
1645 misc_deregister(&dev
->dev
);
1646 debugfs_remove_recursive(dev
->debug_root
);
1647 /* XXX need to free the heaps and clients ? */
1650 EXPORT_SYMBOL(ion_device_destroy
);
1652 void __init
ion_reserve(struct ion_platform_data
*data
)
1656 for (i
= 0; i
< data
->nr
; i
++) {
1657 if (data
->heaps
[i
].size
== 0)
1660 if (data
->heaps
[i
].base
== 0) {
1663 paddr
= memblock_alloc_base(data
->heaps
[i
].size
,
1664 data
->heaps
[i
].align
,
1665 MEMBLOCK_ALLOC_ANYWHERE
);
1667 pr_err("%s: error allocating memblock for heap %d\n",
1671 data
->heaps
[i
].base
= paddr
;
1673 int ret
= memblock_reserve(data
->heaps
[i
].base
,
1674 data
->heaps
[i
].size
);
1676 pr_err("memblock reserve of %zx@%lx failed\n",
1677 data
->heaps
[i
].size
,
1678 data
->heaps
[i
].base
);
1680 pr_info("%s: %s reserved base %lx size %zu\n", __func__
,
1681 data
->heaps
[i
].name
,
1682 data
->heaps
[i
].base
,
1683 data
->heaps
[i
].size
);