Commit | Line | Data |
---|---|---|
6fa3eb70 S |
1 | /* |
2 | * This confidential and proprietary software may be used only as | |
3 | * authorised by a licensing agreement from ARM Limited | |
bdc132d7 | 4 | * (C) COPYRIGHT 2012-2015 ARM Limited |
6fa3eb70 S |
5 | * ALL RIGHTS RESERVED |
6 | * The entire notice above must be reproduced on all authorised | |
7 | * copies and copies may only be made to the extent permitted | |
8 | * by a licensing agreement from ARM Limited. | |
9 | */ | |
10 | ||
11 | #include "mali_sync.h" | |
12 | ||
13 | #include "mali_osk.h" | |
14 | #include "mali_kernel_common.h" | |
15 | #include "mali_timeline.h" | |
bdc132d7 | 16 | #include "mali_executor.h" |
6fa3eb70 S |
17 | |
18 | #include <linux/file.h> | |
19 | #include <linux/seq_file.h> | |
20 | #include <linux/module.h> | |
21 | ||
22 | struct mali_sync_pt { | |
23 | struct sync_pt sync_pt; | |
24 | struct mali_sync_flag *flag; | |
bdc132d7 | 25 | struct sync_timeline *sync_tl; /**< Sync timeline this pt is connected to. */ |
6fa3eb70 S |
26 | }; |
27 | ||
28 | /** | |
29 | * The sync flag is used to connect sync fences to the Mali Timeline system. Sync fences can be | |
30 | * created from a sync flag, and when the flag is signaled, the sync fences will also be signaled. | |
31 | */ | |
32 | struct mali_sync_flag { | |
33 | struct sync_timeline *sync_tl; /**< Sync timeline this flag is connected to. */ | |
34 | u32 point; /**< Point on timeline. */ | |
35 | int status; /**< 0 if unsignaled, 1 if signaled without error or negative if signaled with error. */ | |
36 | struct kref refcount; /**< Reference count. */ | |
37 | }; | |
38 | ||
bdc132d7 S |
39 | /** |
40 | * Mali sync timeline is used to connect mali timeline to sync_timeline. | |
41 | * When fence timeout can print more detailed mali timeline system info. | |
42 | */ | |
43 | struct mali_sync_timeline_container { | |
44 | struct sync_timeline sync_timeline; | |
45 | struct mali_timeline *timeline; | |
46 | }; | |
47 | ||
6fa3eb70 S |
48 | MALI_STATIC_INLINE struct mali_sync_pt *to_mali_sync_pt(struct sync_pt *pt) |
49 | { | |
50 | return container_of(pt, struct mali_sync_pt, sync_pt); | |
51 | } | |
52 | ||
bdc132d7 S |
53 | MALI_STATIC_INLINE struct mali_sync_timeline_container *to_mali_sync_tl_container(struct sync_timeline *sync_tl) |
54 | { | |
55 | return container_of(sync_tl, struct mali_sync_timeline_container, sync_timeline); | |
56 | } | |
57 | ||
6fa3eb70 S |
58 | static struct sync_pt *timeline_dup(struct sync_pt *pt) |
59 | { | |
60 | struct mali_sync_pt *mpt, *new_mpt; | |
61 | struct sync_pt *new_pt; | |
62 | ||
63 | MALI_DEBUG_ASSERT_POINTER(pt); | |
64 | mpt = to_mali_sync_pt(pt); | |
65 | ||
bdc132d7 | 66 | new_pt = sync_pt_create(mpt->sync_tl, sizeof(struct mali_sync_pt)); |
6fa3eb70 S |
67 | if (NULL == new_pt) return NULL; |
68 | ||
69 | new_mpt = to_mali_sync_pt(new_pt); | |
70 | ||
71 | mali_sync_flag_get(mpt->flag); | |
72 | new_mpt->flag = mpt->flag; | |
bdc132d7 | 73 | new_mpt->sync_tl = mpt->sync_tl; |
6fa3eb70 S |
74 | |
75 | return new_pt; | |
76 | } | |
77 | ||
78 | static int timeline_has_signaled(struct sync_pt *pt) | |
79 | { | |
80 | struct mali_sync_pt *mpt; | |
81 | ||
82 | MALI_DEBUG_ASSERT_POINTER(pt); | |
83 | mpt = to_mali_sync_pt(pt); | |
84 | ||
85 | MALI_DEBUG_ASSERT_POINTER(mpt->flag); | |
86 | ||
87 | return mpt->flag->status; | |
88 | } | |
89 | ||
90 | static int timeline_compare(struct sync_pt *pta, struct sync_pt *ptb) | |
91 | { | |
92 | struct mali_sync_pt *mpta; | |
93 | struct mali_sync_pt *mptb; | |
94 | u32 a, b; | |
95 | ||
96 | MALI_DEBUG_ASSERT_POINTER(pta); | |
97 | MALI_DEBUG_ASSERT_POINTER(ptb); | |
98 | mpta = to_mali_sync_pt(pta); | |
99 | mptb = to_mali_sync_pt(ptb); | |
100 | ||
101 | MALI_DEBUG_ASSERT_POINTER(mpta->flag); | |
102 | MALI_DEBUG_ASSERT_POINTER(mptb->flag); | |
103 | ||
104 | a = mpta->flag->point; | |
bdc132d7 | 105 | b = mptb->flag->point; |
6fa3eb70 S |
106 | |
107 | if (a == b) return 0; | |
108 | ||
109 | return ((b - a) < (a - b) ? -1 : 1); | |
110 | } | |
111 | ||
112 | static void timeline_free_pt(struct sync_pt *pt) | |
113 | { | |
114 | struct mali_sync_pt *mpt; | |
115 | ||
116 | MALI_DEBUG_ASSERT_POINTER(pt); | |
117 | mpt = to_mali_sync_pt(pt); | |
118 | ||
119 | mali_sync_flag_put(mpt->flag); | |
120 | } | |
121 | ||
122 | static void timeline_release(struct sync_timeline *sync_timeline) | |
123 | { | |
bdc132d7 S |
124 | struct mali_sync_timeline_container *mali_sync_tl = NULL; |
125 | struct mali_timeline *mali_tl = NULL; | |
126 | ||
127 | MALI_DEBUG_ASSERT_POINTER(sync_timeline); | |
128 | ||
129 | mali_sync_tl = to_mali_sync_tl_container(sync_timeline); | |
130 | MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); | |
131 | ||
132 | mali_tl = mali_sync_tl->timeline; | |
133 | ||
134 | /* always signaled timeline didn't have mali container */ | |
135 | if (mali_tl) { | |
136 | if (NULL != mali_tl->spinlock) { | |
137 | mali_spinlock_reentrant_term(mali_tl->spinlock); | |
138 | } | |
139 | _mali_osk_free(mali_tl); | |
140 | } | |
141 | ||
6fa3eb70 S |
142 | module_put(THIS_MODULE); |
143 | } | |
144 | ||
bdc132d7 | 145 | #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) |
6fa3eb70 S |
146 | static void timeline_print_pt(struct seq_file *s, struct sync_pt *sync_pt) |
147 | { | |
148 | struct mali_sync_pt *mpt; | |
149 | ||
150 | MALI_DEBUG_ASSERT_POINTER(s); | |
151 | MALI_DEBUG_ASSERT_POINTER(sync_pt); | |
152 | ||
153 | mpt = to_mali_sync_pt(sync_pt); | |
6fa3eb70 | 154 | |
bdc132d7 S |
155 | /* It is possible this sync point is just under construct, |
156 | * make sure the flag is valid before accessing it | |
157 | */ | |
158 | if (mpt->flag) { | |
159 | seq_printf(s, "%u", mpt->flag->point); | |
160 | } else { | |
161 | seq_printf(s, "uninitialized"); | |
162 | } | |
6fa3eb70 S |
163 | } |
164 | ||
bdc132d7 S |
165 | static void timeline_print_obj(struct seq_file *s, struct sync_timeline *sync_tl) |
166 | { | |
167 | struct mali_sync_timeline_container *mali_sync_tl = NULL; | |
168 | struct mali_timeline *mali_tl = NULL; | |
169 | ||
170 | MALI_DEBUG_ASSERT_POINTER(sync_tl); | |
171 | ||
172 | mali_sync_tl = to_mali_sync_tl_container(sync_tl); | |
173 | MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); | |
174 | ||
175 | mali_tl = mali_sync_tl->timeline; | |
176 | ||
177 | if (NULL != mali_tl) { | |
178 | seq_printf(s, "oldest (%u) ", mali_tl->point_oldest); | |
179 | seq_printf(s, "next (%u)", mali_tl->point_next); | |
180 | seq_printf(s, "\n"); | |
181 | ||
182 | #if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) | |
183 | { | |
184 | u32 tid = _mali_osk_get_tid(); | |
185 | struct mali_timeline_system *system = mali_tl->system; | |
186 | ||
187 | mali_spinlock_reentrant_wait(mali_tl->spinlock, tid); | |
188 | if (!mali_tl->destroyed) { | |
189 | mali_spinlock_reentrant_wait(system->spinlock, tid); | |
190 | mali_timeline_debug_print_timeline(mali_tl, s); | |
191 | mali_spinlock_reentrant_signal(system->spinlock, tid); | |
192 | } | |
193 | mali_spinlock_reentrant_signal(mali_tl->spinlock, tid); | |
194 | ||
195 | /* dump job queue status and group running status */ | |
196 | mali_executor_status_dump(); | |
197 | } | |
198 | #endif | |
199 | } | |
200 | } | |
201 | #else | |
202 | static void timeline_pt_value_str(struct sync_pt *pt, char *str, int size) | |
203 | { | |
204 | struct mali_sync_pt *mpt; | |
205 | ||
206 | MALI_DEBUG_ASSERT_POINTER(str); | |
207 | MALI_DEBUG_ASSERT_POINTER(pt); | |
208 | ||
209 | mpt = to_mali_sync_pt(pt); | |
210 | ||
211 | /* It is possible this sync point is just under construct, | |
212 | * make sure the flag is valid before accessing it | |
213 | */ | |
214 | if (mpt->flag) { | |
215 | _mali_osk_snprintf(str, size, "%u", mpt->flag->point); | |
216 | } else { | |
217 | _mali_osk_snprintf(str, size, "uninitialized"); | |
218 | } | |
219 | } | |
220 | ||
221 | static void timeline_value_str(struct sync_timeline *timeline, char *str, int size) | |
222 | { | |
223 | struct mali_sync_timeline_container *mali_sync_tl = NULL; | |
224 | struct mali_timeline *mali_tl = NULL; | |
225 | ||
226 | MALI_DEBUG_ASSERT_POINTER(timeline); | |
227 | ||
228 | mali_sync_tl = to_mali_sync_tl_container(timeline); | |
229 | MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); | |
230 | ||
231 | mali_tl = mali_sync_tl->timeline; | |
232 | ||
233 | if (NULL != mali_tl) { | |
234 | _mali_osk_snprintf(str, size, "oldest (%u) ", mali_tl->point_oldest); | |
235 | _mali_osk_snprintf(str, size, "next (%u)", mali_tl->point_next); | |
236 | _mali_osk_snprintf(str, size, "\n"); | |
237 | ||
238 | #if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) | |
239 | { | |
240 | u32 tid = _mali_osk_get_tid(); | |
241 | struct mali_timeline_system *system = mali_tl->system; | |
242 | ||
243 | mali_spinlock_reentrant_wait(mali_tl->spinlock, tid); | |
244 | if (!mali_tl->destroyed) { | |
245 | mali_spinlock_reentrant_wait(system->spinlock, tid); | |
246 | mali_timeline_debug_direct_print_timeline(mali_tl); | |
247 | mali_spinlock_reentrant_signal(system->spinlock, tid); | |
248 | } | |
249 | mali_spinlock_reentrant_signal(mali_tl->spinlock, tid); | |
250 | ||
251 | /* dump job queue status and group running status */ | |
252 | mali_executor_status_dump(); | |
253 | } | |
254 | #endif | |
255 | } | |
256 | } | |
257 | #endif | |
258 | ||
259 | ||
6fa3eb70 S |
260 | static struct sync_timeline_ops mali_timeline_ops = { |
261 | .driver_name = "Mali", | |
262 | .dup = timeline_dup, | |
263 | .has_signaled = timeline_has_signaled, | |
264 | .compare = timeline_compare, | |
265 | .free_pt = timeline_free_pt, | |
266 | .release_obj = timeline_release, | |
bdc132d7 | 267 | #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) |
6fa3eb70 | 268 | .print_pt = timeline_print_pt, |
bdc132d7 S |
269 | .print_obj = timeline_print_obj, |
270 | #else | |
271 | .pt_value_str = timeline_pt_value_str, | |
272 | .timeline_value_str = timeline_value_str, | |
273 | #endif | |
6fa3eb70 S |
274 | }; |
275 | ||
bdc132d7 | 276 | struct sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name) |
6fa3eb70 S |
277 | { |
278 | struct sync_timeline *sync_tl; | |
bdc132d7 | 279 | struct mali_sync_timeline_container *mali_sync_tl; |
6fa3eb70 | 280 | |
bdc132d7 | 281 | sync_tl = sync_timeline_create(&mali_timeline_ops, sizeof(struct mali_sync_timeline_container), name); |
6fa3eb70 S |
282 | if (NULL == sync_tl) return NULL; |
283 | ||
bdc132d7 S |
284 | mali_sync_tl = to_mali_sync_tl_container(sync_tl); |
285 | mali_sync_tl->timeline = timeline; | |
286 | ||
6fa3eb70 S |
287 | /* Grab a reference on the module to ensure the callbacks are present |
288 | * as long some timeline exists. The reference is released when the | |
289 | * timeline is freed. | |
290 | * Since this function is called from a ioctl on an open file we know | |
291 | * we already have a reference, so using __module_get is safe. */ | |
292 | __module_get(THIS_MODULE); | |
293 | ||
294 | return sync_tl; | |
295 | } | |
296 | ||
6fa3eb70 S |
297 | s32 mali_sync_fence_fd_alloc(struct sync_fence *sync_fence) |
298 | { | |
299 | s32 fd = -1; | |
300 | ||
301 | fd = get_unused_fd(); | |
302 | if (fd < 0) { | |
303 | sync_fence_put(sync_fence); | |
6fa3eb70 S |
304 | return -1; |
305 | } | |
306 | sync_fence_install(sync_fence, fd); | |
307 | ||
308 | return fd; | |
309 | } | |
310 | ||
311 | struct sync_fence *mali_sync_fence_merge(struct sync_fence *sync_fence1, struct sync_fence *sync_fence2) | |
312 | { | |
313 | struct sync_fence *sync_fence; | |
314 | ||
315 | MALI_DEBUG_ASSERT_POINTER(sync_fence1); | |
316 | MALI_DEBUG_ASSERT_POINTER(sync_fence1); | |
317 | ||
318 | sync_fence = sync_fence_merge("mali_merge_fence", sync_fence1, sync_fence2); | |
319 | sync_fence_put(sync_fence1); | |
320 | sync_fence_put(sync_fence2); | |
321 | ||
322 | return sync_fence; | |
323 | } | |
324 | ||
325 | struct sync_fence *mali_sync_timeline_create_signaled_fence(struct sync_timeline *sync_tl) | |
326 | { | |
327 | struct mali_sync_flag *flag; | |
328 | struct sync_fence *sync_fence; | |
329 | ||
330 | MALI_DEBUG_ASSERT_POINTER(sync_tl); | |
331 | ||
332 | flag = mali_sync_flag_create(sync_tl, 0); | |
333 | if (NULL == flag) return NULL; | |
334 | ||
335 | sync_fence = mali_sync_flag_create_fence(flag); | |
336 | ||
337 | mali_sync_flag_signal(flag, 0); | |
338 | mali_sync_flag_put(flag); | |
339 | ||
340 | return sync_fence; | |
341 | } | |
342 | ||
343 | struct mali_sync_flag *mali_sync_flag_create(struct sync_timeline *sync_tl, mali_timeline_point point) | |
344 | { | |
345 | struct mali_sync_flag *flag; | |
346 | ||
347 | if (NULL == sync_tl) return NULL; | |
348 | ||
349 | flag = _mali_osk_calloc(1, sizeof(*flag)); | |
350 | if (NULL == flag) return NULL; | |
351 | ||
352 | flag->sync_tl = sync_tl; | |
353 | flag->point = point; | |
354 | ||
355 | flag->status = 0; | |
356 | kref_init(&flag->refcount); | |
357 | ||
358 | return flag; | |
359 | } | |
360 | ||
361 | void mali_sync_flag_get(struct mali_sync_flag *flag) | |
362 | { | |
363 | MALI_DEBUG_ASSERT_POINTER(flag); | |
364 | kref_get(&flag->refcount); | |
365 | } | |
366 | ||
367 | /** | |
368 | * Free sync flag. | |
369 | * | |
370 | * @param ref kref object embedded in sync flag that should be freed. | |
371 | */ | |
372 | static void mali_sync_flag_free(struct kref *ref) | |
373 | { | |
374 | struct mali_sync_flag *flag; | |
375 | ||
376 | MALI_DEBUG_ASSERT_POINTER(ref); | |
377 | flag = container_of(ref, struct mali_sync_flag, refcount); | |
378 | ||
379 | _mali_osk_free(flag); | |
380 | } | |
381 | ||
382 | void mali_sync_flag_put(struct mali_sync_flag *flag) | |
383 | { | |
384 | MALI_DEBUG_ASSERT_POINTER(flag); | |
385 | kref_put(&flag->refcount, mali_sync_flag_free); | |
386 | } | |
387 | ||
388 | void mali_sync_flag_signal(struct mali_sync_flag *flag, int error) | |
389 | { | |
390 | MALI_DEBUG_ASSERT_POINTER(flag); | |
391 | ||
392 | MALI_DEBUG_ASSERT(0 == flag->status); | |
393 | flag->status = (0 > error) ? error : 1; | |
394 | ||
395 | _mali_osk_write_mem_barrier(); | |
396 | ||
397 | sync_timeline_signal(flag->sync_tl); | |
398 | } | |
399 | ||
400 | /** | |
401 | * Create a sync point attached to given sync flag. | |
402 | * | |
403 | * @note Sync points must be triggered in *exactly* the same order as they are created. | |
404 | * | |
405 | * @param flag Sync flag. | |
406 | * @return New sync point if successful, NULL if not. | |
407 | */ | |
408 | static struct sync_pt *mali_sync_flag_create_pt(struct mali_sync_flag *flag) | |
409 | { | |
410 | struct sync_pt *pt; | |
411 | struct mali_sync_pt *mpt; | |
412 | ||
413 | MALI_DEBUG_ASSERT_POINTER(flag); | |
414 | MALI_DEBUG_ASSERT_POINTER(flag->sync_tl); | |
415 | ||
416 | pt = sync_pt_create(flag->sync_tl, sizeof(struct mali_sync_pt)); | |
417 | if (NULL == pt) return NULL; | |
418 | ||
419 | mali_sync_flag_get(flag); | |
420 | ||
421 | mpt = to_mali_sync_pt(pt); | |
422 | mpt->flag = flag; | |
bdc132d7 | 423 | mpt->sync_tl = flag->sync_tl; |
6fa3eb70 S |
424 | |
425 | return pt; | |
426 | } | |
427 | ||
428 | struct sync_fence *mali_sync_flag_create_fence(struct mali_sync_flag *flag) | |
429 | { | |
430 | struct sync_pt *sync_pt; | |
431 | struct sync_fence *sync_fence; | |
432 | ||
433 | MALI_DEBUG_ASSERT_POINTER(flag); | |
434 | MALI_DEBUG_ASSERT_POINTER(flag->sync_tl); | |
435 | ||
436 | sync_pt = mali_sync_flag_create_pt(flag); | |
437 | if (NULL == sync_pt) return NULL; | |
438 | ||
439 | sync_fence = sync_fence_create("mali_flag_fence", sync_pt); | |
440 | if (NULL == sync_fence) { | |
441 | sync_pt_free(sync_pt); | |
442 | return NULL; | |
443 | } | |
444 | ||
445 | return sync_fence; | |
446 | } |