2 * This confidential and proprietary software may be used only as
3 * authorised by a licensing agreement from ARM Limited
4 * (C) COPYRIGHT 2012-2015 ARM Limited
6 * The entire notice above must be reproduced on all authorised
7 * copies and copies may only be made to the extent permitted
8 * by a licensing agreement from ARM Limited.
11 #include "mali_sync.h"
14 #include "mali_kernel_common.h"
15 #include "mali_timeline.h"
16 #include "mali_executor.h"
18 #include <linux/file.h>
19 #include <linux/seq_file.h>
20 #include <linux/module.h>
23 struct sync_pt sync_pt
;
24 struct mali_sync_flag
*flag
;
25 struct sync_timeline
*sync_tl
; /**< Sync timeline this pt is connected to. */
29 * The sync flag is used to connect sync fences to the Mali Timeline system. Sync fences can be
30 * created from a sync flag, and when the flag is signaled, the sync fences will also be signaled.
32 struct mali_sync_flag
{
33 struct sync_timeline
*sync_tl
; /**< Sync timeline this flag is connected to. */
34 u32 point
; /**< Point on timeline. */
35 int status
; /**< 0 if unsignaled, 1 if signaled without error or negative if signaled with error. */
36 struct kref refcount
; /**< Reference count. */
40 * Mali sync timeline is used to connect mali timeline to sync_timeline.
41 * When fence timeout can print more detailed mali timeline system info.
43 struct mali_sync_timeline_container
{
44 struct sync_timeline sync_timeline
;
45 struct mali_timeline
*timeline
;
48 MALI_STATIC_INLINE
struct mali_sync_pt
*to_mali_sync_pt(struct sync_pt
*pt
)
50 return container_of(pt
, struct mali_sync_pt
, sync_pt
);
53 MALI_STATIC_INLINE
struct mali_sync_timeline_container
*to_mali_sync_tl_container(struct sync_timeline
*sync_tl
)
55 return container_of(sync_tl
, struct mali_sync_timeline_container
, sync_timeline
);
58 static struct sync_pt
*timeline_dup(struct sync_pt
*pt
)
60 struct mali_sync_pt
*mpt
, *new_mpt
;
61 struct sync_pt
*new_pt
;
63 MALI_DEBUG_ASSERT_POINTER(pt
);
64 mpt
= to_mali_sync_pt(pt
);
66 new_pt
= sync_pt_create(mpt
->sync_tl
, sizeof(struct mali_sync_pt
));
67 if (NULL
== new_pt
) return NULL
;
69 new_mpt
= to_mali_sync_pt(new_pt
);
71 mali_sync_flag_get(mpt
->flag
);
72 new_mpt
->flag
= mpt
->flag
;
73 new_mpt
->sync_tl
= mpt
->sync_tl
;
78 static int timeline_has_signaled(struct sync_pt
*pt
)
80 struct mali_sync_pt
*mpt
;
82 MALI_DEBUG_ASSERT_POINTER(pt
);
83 mpt
= to_mali_sync_pt(pt
);
85 MALI_DEBUG_ASSERT_POINTER(mpt
->flag
);
87 return mpt
->flag
->status
;
90 static int timeline_compare(struct sync_pt
*pta
, struct sync_pt
*ptb
)
92 struct mali_sync_pt
*mpta
;
93 struct mali_sync_pt
*mptb
;
96 MALI_DEBUG_ASSERT_POINTER(pta
);
97 MALI_DEBUG_ASSERT_POINTER(ptb
);
98 mpta
= to_mali_sync_pt(pta
);
99 mptb
= to_mali_sync_pt(ptb
);
101 MALI_DEBUG_ASSERT_POINTER(mpta
->flag
);
102 MALI_DEBUG_ASSERT_POINTER(mptb
->flag
);
104 a
= mpta
->flag
->point
;
105 b
= mptb
->flag
->point
;
107 if (a
== b
) return 0;
109 return ((b
- a
) < (a
- b
) ? -1 : 1);
112 static void timeline_free_pt(struct sync_pt
*pt
)
114 struct mali_sync_pt
*mpt
;
116 MALI_DEBUG_ASSERT_POINTER(pt
);
117 mpt
= to_mali_sync_pt(pt
);
119 mali_sync_flag_put(mpt
->flag
);
122 static void timeline_release(struct sync_timeline
*sync_timeline
)
124 struct mali_sync_timeline_container
*mali_sync_tl
= NULL
;
125 struct mali_timeline
*mali_tl
= NULL
;
127 MALI_DEBUG_ASSERT_POINTER(sync_timeline
);
129 mali_sync_tl
= to_mali_sync_tl_container(sync_timeline
);
130 MALI_DEBUG_ASSERT_POINTER(mali_sync_tl
);
132 mali_tl
= mali_sync_tl
->timeline
;
134 /* always signaled timeline didn't have mali container */
136 if (NULL
!= mali_tl
->spinlock
) {
137 mali_spinlock_reentrant_term(mali_tl
->spinlock
);
139 _mali_osk_free(mali_tl
);
142 module_put(THIS_MODULE
);
145 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
146 static void timeline_print_pt(struct seq_file
*s
, struct sync_pt
*sync_pt
)
148 struct mali_sync_pt
*mpt
;
150 MALI_DEBUG_ASSERT_POINTER(s
);
151 MALI_DEBUG_ASSERT_POINTER(sync_pt
);
153 mpt
= to_mali_sync_pt(sync_pt
);
155 /* It is possible this sync point is just under construct,
156 * make sure the flag is valid before accessing it
159 seq_printf(s
, "%u", mpt
->flag
->point
);
161 seq_printf(s
, "uninitialized");
165 static void timeline_print_obj(struct seq_file
*s
, struct sync_timeline
*sync_tl
)
167 struct mali_sync_timeline_container
*mali_sync_tl
= NULL
;
168 struct mali_timeline
*mali_tl
= NULL
;
170 MALI_DEBUG_ASSERT_POINTER(sync_tl
);
172 mali_sync_tl
= to_mali_sync_tl_container(sync_tl
);
173 MALI_DEBUG_ASSERT_POINTER(mali_sync_tl
);
175 mali_tl
= mali_sync_tl
->timeline
;
177 if (NULL
!= mali_tl
) {
178 seq_printf(s
, "oldest (%u) ", mali_tl
->point_oldest
);
179 seq_printf(s
, "next (%u)", mali_tl
->point_next
);
182 #if defined(MALI_TIMELINE_DEBUG_FUNCTIONS)
184 u32 tid
= _mali_osk_get_tid();
185 struct mali_timeline_system
*system
= mali_tl
->system
;
187 mali_spinlock_reentrant_wait(mali_tl
->spinlock
, tid
);
188 if (!mali_tl
->destroyed
) {
189 mali_spinlock_reentrant_wait(system
->spinlock
, tid
);
190 mali_timeline_debug_print_timeline(mali_tl
, s
);
191 mali_spinlock_reentrant_signal(system
->spinlock
, tid
);
193 mali_spinlock_reentrant_signal(mali_tl
->spinlock
, tid
);
195 /* dump job queue status and group running status */
196 mali_executor_status_dump();
202 static void timeline_pt_value_str(struct sync_pt
*pt
, char *str
, int size
)
204 struct mali_sync_pt
*mpt
;
206 MALI_DEBUG_ASSERT_POINTER(str
);
207 MALI_DEBUG_ASSERT_POINTER(pt
);
209 mpt
= to_mali_sync_pt(pt
);
211 /* It is possible this sync point is just under construct,
212 * make sure the flag is valid before accessing it
215 _mali_osk_snprintf(str
, size
, "%u", mpt
->flag
->point
);
217 _mali_osk_snprintf(str
, size
, "uninitialized");
221 static void timeline_value_str(struct sync_timeline
*timeline
, char *str
, int size
)
223 struct mali_sync_timeline_container
*mali_sync_tl
= NULL
;
224 struct mali_timeline
*mali_tl
= NULL
;
226 MALI_DEBUG_ASSERT_POINTER(timeline
);
228 mali_sync_tl
= to_mali_sync_tl_container(timeline
);
229 MALI_DEBUG_ASSERT_POINTER(mali_sync_tl
);
231 mali_tl
= mali_sync_tl
->timeline
;
233 if (NULL
!= mali_tl
) {
234 _mali_osk_snprintf(str
, size
, "oldest (%u) ", mali_tl
->point_oldest
);
235 _mali_osk_snprintf(str
, size
, "next (%u)", mali_tl
->point_next
);
236 _mali_osk_snprintf(str
, size
, "\n");
238 #if defined(MALI_TIMELINE_DEBUG_FUNCTIONS)
240 u32 tid
= _mali_osk_get_tid();
241 struct mali_timeline_system
*system
= mali_tl
->system
;
243 mali_spinlock_reentrant_wait(mali_tl
->spinlock
, tid
);
244 if (!mali_tl
->destroyed
) {
245 mali_spinlock_reentrant_wait(system
->spinlock
, tid
);
246 mali_timeline_debug_direct_print_timeline(mali_tl
);
247 mali_spinlock_reentrant_signal(system
->spinlock
, tid
);
249 mali_spinlock_reentrant_signal(mali_tl
->spinlock
, tid
);
251 /* dump job queue status and group running status */
252 mali_executor_status_dump();
260 static struct sync_timeline_ops mali_timeline_ops
= {
261 .driver_name
= "Mali",
263 .has_signaled
= timeline_has_signaled
,
264 .compare
= timeline_compare
,
265 .free_pt
= timeline_free_pt
,
266 .release_obj
= timeline_release
,
267 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
268 .print_pt
= timeline_print_pt
,
269 .print_obj
= timeline_print_obj
,
271 .pt_value_str
= timeline_pt_value_str
,
272 .timeline_value_str
= timeline_value_str
,
276 struct sync_timeline
*mali_sync_timeline_create(struct mali_timeline
*timeline
, const char *name
)
278 struct sync_timeline
*sync_tl
;
279 struct mali_sync_timeline_container
*mali_sync_tl
;
281 sync_tl
= sync_timeline_create(&mali_timeline_ops
, sizeof(struct mali_sync_timeline_container
), name
);
282 if (NULL
== sync_tl
) return NULL
;
284 mali_sync_tl
= to_mali_sync_tl_container(sync_tl
);
285 mali_sync_tl
->timeline
= timeline
;
287 /* Grab a reference on the module to ensure the callbacks are present
288 * as long some timeline exists. The reference is released when the
290 * Since this function is called from a ioctl on an open file we know
291 * we already have a reference, so using __module_get is safe. */
292 __module_get(THIS_MODULE
);
297 s32
mali_sync_fence_fd_alloc(struct sync_fence
*sync_fence
)
301 fd
= get_unused_fd();
303 sync_fence_put(sync_fence
);
306 sync_fence_install(sync_fence
, fd
);
311 struct sync_fence
*mali_sync_fence_merge(struct sync_fence
*sync_fence1
, struct sync_fence
*sync_fence2
)
313 struct sync_fence
*sync_fence
;
315 MALI_DEBUG_ASSERT_POINTER(sync_fence1
);
316 MALI_DEBUG_ASSERT_POINTER(sync_fence1
);
318 sync_fence
= sync_fence_merge("mali_merge_fence", sync_fence1
, sync_fence2
);
319 sync_fence_put(sync_fence1
);
320 sync_fence_put(sync_fence2
);
325 struct sync_fence
*mali_sync_timeline_create_signaled_fence(struct sync_timeline
*sync_tl
)
327 struct mali_sync_flag
*flag
;
328 struct sync_fence
*sync_fence
;
330 MALI_DEBUG_ASSERT_POINTER(sync_tl
);
332 flag
= mali_sync_flag_create(sync_tl
, 0);
333 if (NULL
== flag
) return NULL
;
335 sync_fence
= mali_sync_flag_create_fence(flag
);
337 mali_sync_flag_signal(flag
, 0);
338 mali_sync_flag_put(flag
);
343 struct mali_sync_flag
*mali_sync_flag_create(struct sync_timeline
*sync_tl
, mali_timeline_point point
)
345 struct mali_sync_flag
*flag
;
347 if (NULL
== sync_tl
) return NULL
;
349 flag
= _mali_osk_calloc(1, sizeof(*flag
));
350 if (NULL
== flag
) return NULL
;
352 flag
->sync_tl
= sync_tl
;
356 kref_init(&flag
->refcount
);
361 void mali_sync_flag_get(struct mali_sync_flag
*flag
)
363 MALI_DEBUG_ASSERT_POINTER(flag
);
364 kref_get(&flag
->refcount
);
370 * @param ref kref object embedded in sync flag that should be freed.
372 static void mali_sync_flag_free(struct kref
*ref
)
374 struct mali_sync_flag
*flag
;
376 MALI_DEBUG_ASSERT_POINTER(ref
);
377 flag
= container_of(ref
, struct mali_sync_flag
, refcount
);
379 _mali_osk_free(flag
);
382 void mali_sync_flag_put(struct mali_sync_flag
*flag
)
384 MALI_DEBUG_ASSERT_POINTER(flag
);
385 kref_put(&flag
->refcount
, mali_sync_flag_free
);
388 void mali_sync_flag_signal(struct mali_sync_flag
*flag
, int error
)
390 MALI_DEBUG_ASSERT_POINTER(flag
);
392 MALI_DEBUG_ASSERT(0 == flag
->status
);
393 flag
->status
= (0 > error
) ? error
: 1;
395 _mali_osk_write_mem_barrier();
397 sync_timeline_signal(flag
->sync_tl
);
401 * Create a sync point attached to given sync flag.
403 * @note Sync points must be triggered in *exactly* the same order as they are created.
405 * @param flag Sync flag.
406 * @return New sync point if successful, NULL if not.
408 static struct sync_pt
*mali_sync_flag_create_pt(struct mali_sync_flag
*flag
)
411 struct mali_sync_pt
*mpt
;
413 MALI_DEBUG_ASSERT_POINTER(flag
);
414 MALI_DEBUG_ASSERT_POINTER(flag
->sync_tl
);
416 pt
= sync_pt_create(flag
->sync_tl
, sizeof(struct mali_sync_pt
));
417 if (NULL
== pt
) return NULL
;
419 mali_sync_flag_get(flag
);
421 mpt
= to_mali_sync_pt(pt
);
423 mpt
->sync_tl
= flag
->sync_tl
;
428 struct sync_fence
*mali_sync_flag_create_fence(struct mali_sync_flag
*flag
)
430 struct sync_pt
*sync_pt
;
431 struct sync_fence
*sync_fence
;
433 MALI_DEBUG_ASSERT_POINTER(flag
);
434 MALI_DEBUG_ASSERT_POINTER(flag
->sync_tl
);
436 sync_pt
= mali_sync_flag_create_pt(flag
);
437 if (NULL
== sync_pt
) return NULL
;
439 sync_fence
= sync_fence_create("mali_flag_fence", sync_pt
);
440 if (NULL
== sync_fence
) {
441 sync_pt_free(sync_pt
);