projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
mali mess
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / misc / mediatek / gpu / mt8127 / mali / mali / common / mali_timeline.c
CommitLineData
6fa3eb70
S		 1/*
2 * This confidential and proprietary software may be used only as
3 * authorised by a licensing agreement from ARM Limited
02af6beb 4 * (C) COPYRIGHT 2013-2015 ARM Limited
6fa3eb70
S		 5 * ALL RIGHTS RESERVED
6 * The entire notice above must be reproduced on all authorised
7 * copies and copies may only be made to the extent permitted
8 * by a licensing agreement from ARM Limited.
9 */
10
11#include "mali_timeline.h"
12#include "mali_kernel_common.h"
6fa3eb70
S		 13#include "mali_scheduler.h"
14#include "mali_soft_job.h"
15#include "mali_timeline_fence_wait.h"
16#include "mali_timeline_sync_fence.h"
02af6beb
S		 17#include "mali_executor.h"
18#include "mali_pp_job.h"
6fa3eb70
S		 19
20#define MALI_TIMELINE_SYSTEM_LOCKED(system) (mali_spinlock_reentrant_is_held((system)->spinlock, _mali_osk_get_tid()))
21
02af6beb
S		 22/*
23 * Following three elements are used to record how many
24 * gp, physical pp or virtual pp jobs are delayed in the whole
25 * timeline system, we can use these three value to decide
26 * if need to deactivate idle group.
27 */
28_mali_osk_atomic_t gp_tracker_count;
29_mali_osk_atomic_t phy_pp_tracker_count;
30_mali_osk_atomic_t virt_pp_tracker_count;
6fa3eb70 31
02af6beb
S		 32static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system,
33 struct mali_timeline_waiter *waiter);
6fa3eb70
S		 34
35#if defined(CONFIG_SYNC)
6fa3eb70
S		 36#include <linux/version.h>
37#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0)
02af6beb 38#include <linux/list.h>
6fa3eb70 39#include <linux/workqueue.h>
02af6beb 40#include <linux/spinlock.h>
6fa3eb70
S		 41
42struct mali_deferred_fence_put_entry {
02af6beb 43 struct hlist_node list;
6fa3eb70
S		 44 struct sync_fence *fence;
45};
46
02af6beb
S		 47static HLIST_HEAD(mali_timeline_sync_fence_to_free_list);
48static DEFINE_SPINLOCK(mali_timeline_sync_fence_to_free_lock);
6fa3eb70
S		 49
50static void put_sync_fences(struct work_struct *ignore)
51{
02af6beb
S		 52 struct hlist_head list;
53 struct hlist_node *tmp, *pos;
54 unsigned long flags;
55 struct mali_deferred_fence_put_entry *o;
6fa3eb70 56
02af6beb
S		 57 spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags);
58 hlist_move_list(&mali_timeline_sync_fence_to_free_list, &list);
59 spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags);
6fa3eb70 60
02af6beb 61 hlist_for_each_entry_safe(o, pos, tmp, &list, list) {
6fa3eb70
S		 62 sync_fence_put(o->fence);
63 kfree(o);
64 }
65}
66
67static DECLARE_DELAYED_WORK(delayed_sync_fence_put, put_sync_fences);
68#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) */
69
6fa3eb70
S		 70/* Callback that is called when a sync fence a tracker is waiting on is signaled. */
71static void mali_timeline_sync_fence_callback(struct sync_fence *sync_fence, struct sync_fence_waiter *sync_fence_waiter)
72{
73 struct mali_timeline_system *system;
74 struct mali_timeline_waiter *waiter;
75 struct mali_timeline_tracker *tracker;
76 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
77 u32 tid = _mali_osk_get_tid();
78 mali_bool is_aborting = MALI_FALSE;
02af6beb 79#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
6fa3eb70 80 int fence_status = sync_fence->status;
02af6beb
S		 81#else
82 int fence_status = atomic_read(&sync_fence->status);
83#endif
6fa3eb70
S		 84
85 MALI_DEBUG_ASSERT_POINTER(sync_fence);
86 MALI_DEBUG_ASSERT_POINTER(sync_fence_waiter);
87
88 tracker = _MALI_OSK_CONTAINER_OF(sync_fence_waiter, struct mali_timeline_tracker, sync_fence_waiter);
89 MALI_DEBUG_ASSERT_POINTER(tracker);
90
91 system = tracker->system;
92 MALI_DEBUG_ASSERT_POINTER(system);
93 MALI_DEBUG_ASSERT_POINTER(system->session);
94
95 mali_spinlock_reentrant_wait(system->spinlock, tid);
96
97 is_aborting = system->session->is_aborting;
98 if (!is_aborting && (0 > fence_status)) {
99 MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, fence_status));
100 tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT;
101 }
102
103 waiter = tracker->waiter_sync;
104 MALI_DEBUG_ASSERT_POINTER(waiter);
105
106 tracker->sync_fence = NULL;
02af6beb
S		 107 tracker->fence.sync_fd = -1;
108
6fa3eb70
S		 109 schedule_mask |= mali_timeline_system_release_waiter(system, waiter);
110
111 /* If aborting, wake up sleepers that are waiting for sync fence callbacks to complete. */
112 if (is_aborting) {
113 _mali_osk_wait_queue_wake_up(system->wait_queue);
114 }
115
116 mali_spinlock_reentrant_signal(system->spinlock, tid);
02af6beb
S		 117
118 /*
119 * Older versions of Linux, before 3.5, doesn't support fput() in interrupt
120 * context. For those older kernels, allocate a list object and put the
121 * fence object on that and defer the call to sync_fence_put() to a workqueue.
122 */
123#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0)
6fa3eb70
S		 124 {
125 struct mali_deferred_fence_put_entry *obj;
126
127 obj = kzalloc(sizeof(struct mali_deferred_fence_put_entry), GFP_ATOMIC);
128 if (obj) {
02af6beb
S		 129 unsigned long flags;
130 mali_bool schedule = MALI_FALSE;
131
6fa3eb70 132 obj->fence = sync_fence;
02af6beb
S		 133
134 spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags);
135 if (hlist_empty(&mali_timeline_sync_fence_to_free_list))
136 schedule = MALI_TRUE;
137 hlist_add_head(&obj->list, &mali_timeline_sync_fence_to_free_list);
138 spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags);
139
140 if (schedule)
6fa3eb70 141 schedule_delayed_work(&delayed_sync_fence_put, 0);
6fa3eb70
S		 142 }
143 }
02af6beb
S		 144#else
145 sync_fence_put(sync_fence);
146#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) */
6fa3eb70
S		 147
148 if (!is_aborting) {
02af6beb 149 mali_executor_schedule_from_mask(schedule_mask, MALI_TRUE);
6fa3eb70
S		 150 }
151}
152#endif /* defined(CONFIG_SYNC) */
153
154static mali_scheduler_mask mali_timeline_tracker_time_out(struct mali_timeline_tracker *tracker)
155{
156 MALI_DEBUG_ASSERT_POINTER(tracker);
157 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_SOFT == tracker->type);
158
159 return mali_soft_job_system_timeout_job((struct mali_soft_job *) tracker->job);
160}
161
162static void mali_timeline_timer_callback(void *data)
163{
164 struct mali_timeline_system *system;
165 struct mali_timeline_tracker *tracker;
166 struct mali_timeline *timeline;
167 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
168 u32 tid = _mali_osk_get_tid();
169
170 timeline = (struct mali_timeline *) data;
171 MALI_DEBUG_ASSERT_POINTER(timeline);
172
173 system = timeline->system;
174 MALI_DEBUG_ASSERT_POINTER(system);
175
176 mali_spinlock_reentrant_wait(system->spinlock, tid);
177
178 if (!system->timer_enabled) {
179 mali_spinlock_reentrant_signal(system->spinlock, tid);
180 return;
181 }
182
183 tracker = timeline->tracker_tail;
184 timeline->timer_active = MALI_FALSE;
185
186 if (NULL != tracker && MALI_TRUE == tracker->timer_active) {
187 /* This is likely the delayed work that has been schedule out before cancelled. */
188 if (MALI_TIMELINE_TIMEOUT_HZ > (_mali_osk_time_tickcount() - tracker->os_tick_activate)) {
189 mali_spinlock_reentrant_signal(system->spinlock, tid);
190 return;
191 }
192
193 schedule_mask = mali_timeline_tracker_time_out(tracker);
194 tracker->timer_active = MALI_FALSE;
195 } else {
196 MALI_PRINT_ERROR(("Mali Timeline: Soft job timer callback without a waiting tracker.\n"));
197 }
198
199 mali_spinlock_reentrant_signal(system->spinlock, tid);
200
02af6beb 201 mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
6fa3eb70
S		 202}
203
204void mali_timeline_system_stop_timer(struct mali_timeline_system *system)
205{
206 u32 i;
207 u32 tid = _mali_osk_get_tid();
208
209 MALI_DEBUG_ASSERT_POINTER(system);
210
211 mali_spinlock_reentrant_wait(system->spinlock, tid);
212 system->timer_enabled = MALI_FALSE;
213 mali_spinlock_reentrant_signal(system->spinlock, tid);
214
215 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
216 struct mali_timeline *timeline = system->timelines[i];
217
218 MALI_DEBUG_ASSERT_POINTER(timeline);
219
220 if (NULL != timeline->delayed_work) {
221 _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work);
222 timeline->timer_active = MALI_FALSE;
223 }
224 }
225}
226
227static void mali_timeline_destroy(struct mali_timeline *timeline)
228{
229 MALI_DEBUG_ASSERT_POINTER(timeline);
230 if (NULL != timeline) {
231 /* Assert that the timeline object has been properly cleaned up before destroying it. */
232 MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next);
233 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head);
234 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
235 MALI_DEBUG_ASSERT(NULL == timeline->waiter_head);
236 MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail);
237 MALI_DEBUG_ASSERT(NULL != timeline->system);
238 MALI_DEBUG_ASSERT(MALI_TIMELINE_MAX > timeline->id);
239
240#if defined(CONFIG_SYNC)
241 if (NULL != timeline->sync_tl) {
242 sync_timeline_destroy(timeline->sync_tl);
243 }
244#endif /* defined(CONFIG_SYNC) */
245
246 if (NULL != timeline->delayed_work) {
247 _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work);
248 _mali_osk_wq_delayed_delete_work_nonflush(timeline->delayed_work);
249 }
250
02af6beb 251#ifndef CONFIG_SYNC
6fa3eb70 252 _mali_osk_free(timeline);
02af6beb 253#endif
6fa3eb70
S		 254 }
255}
256
257static struct mali_timeline *mali_timeline_create(struct mali_timeline_system *system, enum mali_timeline_id id)
258{
259 struct mali_timeline *timeline;
260
261 MALI_DEBUG_ASSERT_POINTER(system);
262 MALI_DEBUG_ASSERT(id < MALI_TIMELINE_MAX);
263
264 timeline = (struct mali_timeline *) _mali_osk_calloc(1, sizeof(struct mali_timeline));
265 if (NULL == timeline) {
266 return NULL;
267 }
268
269 /* Initially the timeline is empty. */
270#if defined(MALI_TIMELINE_DEBUG_START_POINT)
271 /* Start the timeline a bit before wrapping when debugging. */
272 timeline->point_next = UINT_MAX - MALI_TIMELINE_MAX_POINT_SPAN - 128;
273#else
274 timeline->point_next = 1;
275#endif
276 timeline->point_oldest = timeline->point_next;
277
278 /* The tracker and waiter lists will initially be empty. */
279
280 timeline->system = system;
281 timeline->id = id;
282
283 timeline->delayed_work = _mali_osk_wq_delayed_create_work(mali_timeline_timer_callback, timeline);
284 if (NULL == timeline->delayed_work) {
285 mali_timeline_destroy(timeline);
286 return NULL;
287 }
288
289 timeline->timer_active = MALI_FALSE;
290
291#if defined(CONFIG_SYNC)
292 {
293 char timeline_name[32];
294
295 switch (id) {
296 case MALI_TIMELINE_GP:
297 _mali_osk_snprintf(timeline_name, 32, "mali-%u-gp", _mali_osk_get_pid());
298 break;
299 case MALI_TIMELINE_PP:
300 _mali_osk_snprintf(timeline_name, 32, "mali-%u-pp", _mali_osk_get_pid());
301 break;
302 case MALI_TIMELINE_SOFT:
303 _mali_osk_snprintf(timeline_name, 32, "mali-%u-soft", _mali_osk_get_pid());
304 break;
305 default:
306 MALI_PRINT_ERROR(("Mali Timeline: Invalid timeline id %d\n", id));
307 mali_timeline_destroy(timeline);
308 return NULL;
309 }
310
02af6beb
S		 311 timeline->destroyed = MALI_FALSE;
312
313 timeline->sync_tl = mali_sync_timeline_create(timeline, timeline_name);
6fa3eb70
S		 314 if (NULL == timeline->sync_tl) {
315 mali_timeline_destroy(timeline);
316 return NULL;
317 }
02af6beb
S		 318
319 timeline->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM);
320 if (NULL == timeline->spinlock) {
321 mali_timeline_destroy(timeline);
322 return NULL;
323 }
6fa3eb70
S		 324 }
325#endif /* defined(CONFIG_SYNC) */
326
327 return timeline;
328}
329
330static void mali_timeline_insert_tracker(struct mali_timeline *timeline, struct mali_timeline_tracker *tracker)
331{
332 MALI_DEBUG_ASSERT_POINTER(timeline);
333 MALI_DEBUG_ASSERT_POINTER(tracker);
334
335 if (mali_timeline_is_full(timeline)) {
336 /* Don't add tracker if timeline is full. */
337 tracker->point = MALI_TIMELINE_NO_POINT;
338 return;
339 }
340
341 tracker->timeline = timeline;
342 tracker->point = timeline->point_next;
343
344 /* Find next available point. */
345 timeline->point_next++;
346 if (MALI_TIMELINE_NO_POINT == timeline->point_next) {
347 timeline->point_next++;
348 }
349
350 MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
351
02af6beb
S		 352 if (MALI_TIMELINE_TRACKER_GP == tracker->type) {
353 _mali_osk_atomic_inc(&gp_tracker_count);
354 } else if (MALI_TIMELINE_TRACKER_PP == tracker->type) {
355 if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) {
356 _mali_osk_atomic_inc(&virt_pp_tracker_count);
357 } else {
358 _mali_osk_atomic_inc(&phy_pp_tracker_count);
359 }
360 }
361
6fa3eb70
S		 362 /* Add tracker as new head on timeline's tracker list. */
363 if (NULL == timeline->tracker_head) {
364 /* Tracker list is empty. */
365 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
366
367 timeline->tracker_tail = tracker;
368
369 MALI_DEBUG_ASSERT(NULL == tracker->timeline_next);
370 MALI_DEBUG_ASSERT(NULL == tracker->timeline_prev);
371 } else {
372 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next);
373
374 tracker->timeline_prev = timeline->tracker_head;
375 timeline->tracker_head->timeline_next = tracker;
376
377 MALI_DEBUG_ASSERT(NULL == tracker->timeline_next);
378 }
379 timeline->tracker_head = tracker;
380
381 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next);
382 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail->timeline_prev);
383}
384
385/* Inserting the waiter object into the given timeline */
386static void mali_timeline_insert_waiter(struct mali_timeline *timeline, struct mali_timeline_waiter *waiter_new)
387{
388 struct mali_timeline_waiter *waiter_prev;
389 struct mali_timeline_waiter *waiter_next;
390
391 /* Waiter time must be between timeline head and tail, and there must
392 * be less than MALI_TIMELINE_MAX_POINT_SPAN elements between */
02af6beb 393 MALI_DEBUG_ASSERT((waiter_new->point - timeline->point_oldest) < MALI_TIMELINE_MAX_POINT_SPAN);
6fa3eb70
S		 394 MALI_DEBUG_ASSERT((-waiter_new->point + timeline->point_next) < MALI_TIMELINE_MAX_POINT_SPAN);
395
396 /* Finding out where to put this waiter, in the linked waiter list of the given timeline **/
397 waiter_prev = timeline->waiter_head; /* Insert new after waiter_prev */
398 waiter_next = NULL; /* Insert new before waiter_next */
399
400 /* Iterating backwards from head (newest) to tail (oldest) until we
401 * find the correct spot to insert the new waiter */
402 while (waiter_prev && mali_timeline_point_after(waiter_prev->point, waiter_new->point)) {
403 waiter_next = waiter_prev;
404 waiter_prev = waiter_prev->timeline_prev;
405 }
406
407 if (NULL == waiter_prev && NULL == waiter_next) {
408 /* list is empty */
409 timeline->waiter_head = waiter_new;
410 timeline->waiter_tail = waiter_new;
411 } else if (NULL == waiter_next) {
412 /* insert at head */
413 waiter_new->timeline_prev = timeline->waiter_head;
414 timeline->waiter_head->timeline_next = waiter_new;
415 timeline->waiter_head = waiter_new;
416 } else if (NULL == waiter_prev) {
417 /* insert at tail */
418 waiter_new->timeline_next = timeline->waiter_tail;
419 timeline->waiter_tail->timeline_prev = waiter_new;
420 timeline->waiter_tail = waiter_new;
421 } else {
422 /* insert between */
423 waiter_new->timeline_next = waiter_next;
424 waiter_new->timeline_prev = waiter_prev;
425 waiter_next->timeline_prev = waiter_new;
426 waiter_prev->timeline_next = waiter_new;
427 }
428}
429
430static void mali_timeline_update_delayed_work(struct mali_timeline *timeline)
431{
432 struct mali_timeline_system *system;
433 struct mali_timeline_tracker *oldest_tracker;
434
435 MALI_DEBUG_ASSERT_POINTER(timeline);
436 MALI_DEBUG_ASSERT(MALI_TIMELINE_SOFT == timeline->id);
437
438 system = timeline->system;
439 MALI_DEBUG_ASSERT_POINTER(system);
440
441 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
442
443 /* Timer is disabled, early out. */
444 if (!system->timer_enabled) return;
445
446 oldest_tracker = timeline->tracker_tail;
447 if (NULL != oldest_tracker && 0 == oldest_tracker->trigger_ref_count) {
448 if (MALI_FALSE == oldest_tracker->timer_active) {
449 if (MALI_TRUE == timeline->timer_active) {
450 _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work);
451 }
452 _mali_osk_wq_delayed_schedule_work(timeline->delayed_work, MALI_TIMELINE_TIMEOUT_HZ);
453 oldest_tracker->timer_active = MALI_TRUE;
454 timeline->timer_active = MALI_TRUE;
455 }
456 } else if (MALI_TRUE == timeline->timer_active) {
457 _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work);
458 timeline->timer_active = MALI_FALSE;
459 }
460}
461
462static mali_scheduler_mask mali_timeline_update_oldest_point(struct mali_timeline *timeline)
463{
464 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
465
466 MALI_DEBUG_ASSERT_POINTER(timeline);
467
468 MALI_DEBUG_CODE({
469 struct mali_timeline_system *system = timeline->system;
470 MALI_DEBUG_ASSERT_POINTER(system);
471
472 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
473 });
474
475 if (NULL != timeline->tracker_tail) {
476 /* Set oldest point to oldest tracker's point */
477 timeline->point_oldest = timeline->tracker_tail->point;
478 } else {
479 /* No trackers, mark point list as empty */
480 timeline->point_oldest = timeline->point_next;
481 }
482
483 /* Release all waiters no longer on the timeline's point list.
484 * Releasing a waiter can trigger this function to be called again, so
485 * we do not store any pointers on stack. */
486 while (NULL != timeline->waiter_tail) {
487 u32 waiter_time_relative;
488 u32 time_head_relative;
489 struct mali_timeline_waiter *waiter = timeline->waiter_tail;
490
491 time_head_relative = timeline->point_next - timeline->point_oldest;
492 waiter_time_relative = waiter->point - timeline->point_oldest;
493
494 if (waiter_time_relative < time_head_relative) {
495 /* This and all following waiters are on the point list, so we are done. */
496 break;
497 }
498
499 /* Remove waiter from timeline's waiter list. */
500 if (NULL != waiter->timeline_next) {
501 waiter->timeline_next->timeline_prev = NULL;
502 } else {
503 /* This was the last waiter */
504 timeline->waiter_head = NULL;
505 }
506 timeline->waiter_tail = waiter->timeline_next;
507
508 /* Release waiter. This could activate a tracker, if this was
509 * the last waiter for the tracker. */
510 schedule_mask |= mali_timeline_system_release_waiter(timeline->system, waiter);
511 }
512
513 return schedule_mask;
514}
515
516void mali_timeline_tracker_init(struct mali_timeline_tracker *tracker,
02af6beb
S		 517 mali_timeline_tracker_type type,
518 struct mali_timeline_fence *fence,
519 void *job)
6fa3eb70
S		 520{
521 MALI_DEBUG_ASSERT_POINTER(tracker);
522 MALI_DEBUG_ASSERT_POINTER(job);
523
524 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > type);
525
526 /* Zero out all tracker members. */
527 _mali_osk_memset(tracker, 0, sizeof(*tracker));
528
529 tracker->type = type;
530 tracker->job = job;
531 tracker->trigger_ref_count = 1; /* Prevents any callback from trigging while adding it */
532 tracker->os_tick_create = _mali_osk_time_tickcount();
533 MALI_DEBUG_CODE(tracker->magic = MALI_TIMELINE_TRACKER_MAGIC);
534
535 tracker->activation_error = MALI_TIMELINE_ACTIVATION_ERROR_NONE;
536
537 /* Copy fence. */
538 if (NULL != fence) {
539 _mali_osk_memcpy(&tracker->fence, fence, sizeof(struct mali_timeline_fence));
540 }
541}
542
543mali_scheduler_mask mali_timeline_tracker_release(struct mali_timeline_tracker *tracker)
544{
545 struct mali_timeline *timeline;
546 struct mali_timeline_system *system;
547 struct mali_timeline_tracker *tracker_next, *tracker_prev;
548 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
549 u32 tid = _mali_osk_get_tid();
550
551 /* Upon entry a group lock will be held, but not a scheduler lock. */
552 MALI_DEBUG_ASSERT_POINTER(tracker);
553 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
554
555 /* Tracker should have been triggered */
556 MALI_DEBUG_ASSERT(0 == tracker->trigger_ref_count);
557
558 /* All waiters should have been released at this point */
559 MALI_DEBUG_ASSERT(NULL == tracker->waiter_head);
560 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
561
562 MALI_DEBUG_PRINT(3, ("Mali Timeline: releasing tracker for job 0x%08X\n", tracker->job));
563
564 timeline = tracker->timeline;
565 if (NULL == timeline) {
566 /* Tracker was not on a timeline, there is nothing to release. */
567 return MALI_SCHEDULER_MASK_EMPTY;
568 }
569
570 system = timeline->system;
571 MALI_DEBUG_ASSERT_POINTER(system);
572
573 mali_spinlock_reentrant_wait(system->spinlock, tid);
574
575 /* Tracker should still be on timeline */
576 MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
02af6beb 577 MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, tracker->point));
6fa3eb70
S		 578
579 /* Tracker is no longer valid. */
580 MALI_DEBUG_CODE(tracker->magic = 0);
581
582 tracker_next = tracker->timeline_next;
583 tracker_prev = tracker->timeline_prev;
584 tracker->timeline_next = NULL;
585 tracker->timeline_prev = NULL;
586
587 /* Removing tracker from timeline's tracker list */
588 if (NULL == tracker_next) {
589 /* This tracker was the head */
590 timeline->tracker_head = tracker_prev;
591 } else {
592 tracker_next->timeline_prev = tracker_prev;
593 }
594
595 if (NULL == tracker_prev) {
596 /* This tracker was the tail */
597 timeline->tracker_tail = tracker_next;
598 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
599 /* Update the timeline's oldest time and release any waiters */
600 schedule_mask |= mali_timeline_update_oldest_point(timeline);
601 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
602 } else {
603 tracker_prev->timeline_next = tracker_next;
604 }
605
606 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
607
608 /* Update delayed work only when it is the soft job timeline */
609 if (MALI_TIMELINE_SOFT == tracker->timeline->id) {
610 mali_timeline_update_delayed_work(tracker->timeline);
611 }
612
613 mali_spinlock_reentrant_signal(system->spinlock, tid);
614
615 return schedule_mask;
616}
617
618void mali_timeline_system_release_waiter_list(struct mali_timeline_system *system,
02af6beb
S		 619 struct mali_timeline_waiter *tail,
620 struct mali_timeline_waiter *head)
6fa3eb70
S		 621{
622 MALI_DEBUG_ASSERT_POINTER(system);
623 MALI_DEBUG_ASSERT_POINTER(head);
624 MALI_DEBUG_ASSERT_POINTER(tail);
625 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
626
627 head->tracker_next = system->waiter_empty_list;
628 system->waiter_empty_list = tail;
629}
630
631static mali_scheduler_mask mali_timeline_tracker_activate(struct mali_timeline_tracker *tracker)
632{
633 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
634 struct mali_timeline_system *system;
635 struct mali_timeline *timeline;
636 u32 tid = _mali_osk_get_tid();
637
638 MALI_DEBUG_ASSERT_POINTER(tracker);
639 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
640
641 system = tracker->system;
642 MALI_DEBUG_ASSERT_POINTER(system);
643 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
644
645 tracker->os_tick_activate = _mali_osk_time_tickcount();
646
647 if (NULL != tracker->waiter_head) {
648 mali_timeline_system_release_waiter_list(system, tracker->waiter_tail, tracker->waiter_head);
649 tracker->waiter_head = NULL;
650 tracker->waiter_tail = NULL;
651 }
652
653 switch (tracker->type) {
654 case MALI_TIMELINE_TRACKER_GP:
02af6beb
S		 655 schedule_mask = mali_scheduler_activate_gp_job((struct mali_gp_job *) tracker->job);
656
657 _mali_osk_atomic_dec(&gp_tracker_count);
6fa3eb70
S		 658 break;
659 case MALI_TIMELINE_TRACKER_PP:
02af6beb
S		 660 if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) {
661 _mali_osk_atomic_dec(&virt_pp_tracker_count);
662 } else {
663 _mali_osk_atomic_dec(&phy_pp_tracker_count);
664 }
665 schedule_mask = mali_scheduler_activate_pp_job((struct mali_pp_job *) tracker->job);
6fa3eb70
S		 666 break;
667 case MALI_TIMELINE_TRACKER_SOFT:
668 timeline = tracker->timeline;
669 MALI_DEBUG_ASSERT_POINTER(timeline);
670
02af6beb 671 schedule_mask |= mali_soft_job_system_activate_job((struct mali_soft_job *) tracker->job);
6fa3eb70
S		 672
673 /* Start a soft timer to make sure the soft job be released in a limited time */
674 mali_spinlock_reentrant_wait(system->spinlock, tid);
675 mali_timeline_update_delayed_work(timeline);
676 mali_spinlock_reentrant_signal(system->spinlock, tid);
677 break;
678 case MALI_TIMELINE_TRACKER_WAIT:
679 mali_timeline_fence_wait_activate((struct mali_timeline_fence_wait_tracker *) tracker->job);
680 break;
681 case MALI_TIMELINE_TRACKER_SYNC:
682#if defined(CONFIG_SYNC)
683 mali_timeline_sync_fence_activate((struct mali_timeline_sync_fence_tracker *) tracker->job);
684#else
685 MALI_PRINT_ERROR(("Mali Timeline: sync tracker not supported\n", tracker->type));
686#endif /* defined(CONFIG_SYNC) */
687 break;
688 default:
689 MALI_PRINT_ERROR(("Mali Timeline - Illegal tracker type: %d\n", tracker->type));
690 break;
691 }
692
693 return schedule_mask;
694}
695
696void mali_timeline_system_tracker_get(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker)
697{
698 u32 tid = _mali_osk_get_tid();
699
700 MALI_DEBUG_ASSERT_POINTER(tracker);
701 MALI_DEBUG_ASSERT_POINTER(system);
702
703 mali_spinlock_reentrant_wait(system->spinlock, tid);
704
705 MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
706 tracker->trigger_ref_count++;
707
708 mali_spinlock_reentrant_signal(system->spinlock, tid);
709}
710
711mali_scheduler_mask mali_timeline_system_tracker_put(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker, mali_timeline_activation_error activation_error)
712{
713 u32 tid = _mali_osk_get_tid();
714 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
715
716 MALI_DEBUG_ASSERT_POINTER(tracker);
717 MALI_DEBUG_ASSERT_POINTER(system);
718
719 mali_spinlock_reentrant_wait(system->spinlock, tid);
720
721 MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
722 tracker->trigger_ref_count--;
723
724 tracker->activation_error |= activation_error;
725
726 if (0 == tracker->trigger_ref_count) {
727 schedule_mask |= mali_timeline_tracker_activate(tracker);
728 tracker = NULL;
729 }
730
731 mali_spinlock_reentrant_signal(system->spinlock, tid);
732
733 return schedule_mask;
734}
735
736void mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence *fence, _mali_uk_fence_t *uk_fence)
737{
738 u32 i;
739
740 MALI_DEBUG_ASSERT_POINTER(fence);
741 MALI_DEBUG_ASSERT_POINTER(uk_fence);
742
743 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
744 fence->points[i] = uk_fence->points[i];
745 }
746
747 fence->sync_fd = uk_fence->sync_fd;
748}
749
750struct mali_timeline_system *mali_timeline_system_create(struct mali_session_data *session)
751{
752 u32 i;
753 struct mali_timeline_system *system;
754
755 MALI_DEBUG_ASSERT_POINTER(session);
756 MALI_DEBUG_PRINT(4, ("Mali Timeline: creating timeline system\n"));
757
758 system = (struct mali_timeline_system *) _mali_osk_calloc(1, sizeof(struct mali_timeline_system));
759 if (NULL == system) {
760 return NULL;
761 }
762
763 system->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM);
764 if (NULL == system->spinlock) {
765 mali_timeline_system_destroy(system);
766 return NULL;
767 }
768
769 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
770 system->timelines[i] = mali_timeline_create(system, (enum mali_timeline_id)i);
771 if (NULL == system->timelines[i]) {
772 mali_timeline_system_destroy(system);
773 return NULL;
774 }
775 }
776
777#if defined(CONFIG_SYNC)
02af6beb 778 system->signaled_sync_tl = mali_sync_timeline_create(NULL, "mali-always-signaled");
6fa3eb70
S		 779 if (NULL == system->signaled_sync_tl) {
780 mali_timeline_system_destroy(system);
781 return NULL;
782 }
783#endif /* defined(CONFIG_SYNC) */
784
785 system->waiter_empty_list = NULL;
786 system->session = session;
787 system->timer_enabled = MALI_TRUE;
788
789 system->wait_queue = _mali_osk_wait_queue_init();
790 if (NULL == system->wait_queue) {
791 mali_timeline_system_destroy(system);
792 return NULL;
793 }
794
795 return system;
796}
797
798#if defined(CONFIG_SYNC)
799
800/**
801 * Check if there are any trackers left on timeline.
802 *
803 * Used as a wait queue conditional.
804 *
805 * @param data Timeline.
806 * @return MALI_TRUE if there are no trackers on timeline, MALI_FALSE if not.
807 */
808static mali_bool mali_timeline_has_no_trackers(void *data)
809{
810 struct mali_timeline *timeline = (struct mali_timeline *) data;
811
812 MALI_DEBUG_ASSERT_POINTER(timeline);
813
814 return mali_timeline_is_empty(timeline);
815}
816
817/**
818 * Cancel sync fence waiters waited upon by trackers on all timelines.
819 *
820 * Will return after all timelines have no trackers left.
821 *
822 * @param system Timeline system.
823 */
824static void mali_timeline_cancel_sync_fence_waiters(struct mali_timeline_system *system)
825{
826 u32 i;
827 u32 tid = _mali_osk_get_tid();
828 struct mali_timeline_tracker *tracker, *tracker_next;
829 _MALI_OSK_LIST_HEAD_STATIC_INIT(tracker_list);
830
831 MALI_DEBUG_ASSERT_POINTER(system);
832 MALI_DEBUG_ASSERT_POINTER(system->session);
833 MALI_DEBUG_ASSERT(system->session->is_aborting);
834
835 mali_spinlock_reentrant_wait(system->spinlock, tid);
836
837 /* Cancel sync fence waiters. */
838 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
839 struct mali_timeline *timeline = system->timelines[i];
840
841 MALI_DEBUG_ASSERT_POINTER(timeline);
842
843 tracker_next = timeline->tracker_tail;
844 while (NULL != tracker_next) {
845 tracker = tracker_next;
846 tracker_next = tracker->timeline_next;
847
848 if (NULL == tracker->sync_fence) continue;
849
850 MALI_DEBUG_PRINT(3, ("Mali Timeline: Cancelling sync fence wait for tracker 0x%08X.\n", tracker));
851
852 /* Cancel sync fence waiter. */
853 if (0 == sync_fence_cancel_async(tracker->sync_fence, &tracker->sync_fence_waiter)) {
854 /* Callback was not called, move tracker to local list. */
855 _mali_osk_list_add(&tracker->sync_fence_cancel_list, &tracker_list);
856 }
857 }
858 }
859
860 mali_spinlock_reentrant_signal(system->spinlock, tid);
861
862 /* Manually call sync fence callback in order to release waiter and trigger activation of tracker. */
863 _MALI_OSK_LIST_FOREACHENTRY(tracker, tracker_next, &tracker_list, struct mali_timeline_tracker, sync_fence_cancel_list) {
864 mali_timeline_sync_fence_callback(tracker->sync_fence, &tracker->sync_fence_waiter);
865 }
866
867 /* Sleep until all sync fence callbacks are done and all timelines are empty. */
868 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
869 struct mali_timeline *timeline = system->timelines[i];
870
871 MALI_DEBUG_ASSERT_POINTER(timeline);
872
873 _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_has_no_trackers, (void *) timeline);
874 }
875}
876
877#endif /* defined(CONFIG_SYNC) */
878
879void mali_timeline_system_abort(struct mali_timeline_system *system)
880{
881 MALI_DEBUG_CODE(u32 tid = _mali_osk_get_tid(););
882
883 MALI_DEBUG_ASSERT_POINTER(system);
884 MALI_DEBUG_ASSERT_POINTER(system->session);
885 MALI_DEBUG_ASSERT(system->session->is_aborting);
886
887 MALI_DEBUG_PRINT(3, ("Mali Timeline: Aborting timeline system for session 0x%08X.\n", system->session));
888
889#if defined(CONFIG_SYNC)
890 mali_timeline_cancel_sync_fence_waiters(system);
891#endif /* defined(CONFIG_SYNC) */
892
893 /* Should not be any waiters or trackers left at this point. */
02af6beb 894 MALI_DEBUG_CODE({
6fa3eb70
S		 895 u32 i;
896 mali_spinlock_reentrant_wait(system->spinlock, tid);
897 for (i = 0; i < MALI_TIMELINE_MAX; ++i)
898 {
899 struct mali_timeline *timeline = system->timelines[i];
900 MALI_DEBUG_ASSERT_POINTER(timeline);
901 MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next);
902 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head);
903 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
904 MALI_DEBUG_ASSERT(NULL == timeline->waiter_head);
905 MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail);
906 }
907 mali_spinlock_reentrant_signal(system->spinlock, tid);
908 });
909}
910
911void mali_timeline_system_destroy(struct mali_timeline_system *system)
912{
913 u32 i;
914 struct mali_timeline_waiter *waiter, *next;
02af6beb
S		 915#if defined(CONFIG_SYNC)
916 u32 tid = _mali_osk_get_tid();
917#endif
6fa3eb70
S		 918
919 MALI_DEBUG_ASSERT_POINTER(system);
920 MALI_DEBUG_ASSERT_POINTER(system->session);
921
922 MALI_DEBUG_PRINT(4, ("Mali Timeline: destroying timeline system\n"));
923
924 if (NULL != system) {
02af6beb 925
6fa3eb70
S		 926 /* There should be no waiters left on this queue. */
927 if (NULL != system->wait_queue) {
928 _mali_osk_wait_queue_term(system->wait_queue);
929 system->wait_queue = NULL;
930 }
931
932 /* Free all waiters in empty list */
933 waiter = system->waiter_empty_list;
934 while (NULL != waiter) {
935 next = waiter->tracker_next;
936 _mali_osk_free(waiter);
937 waiter = next;
938 }
939
940#if defined(CONFIG_SYNC)
941 if (NULL != system->signaled_sync_tl) {
942 sync_timeline_destroy(system->signaled_sync_tl);
943 }
02af6beb
S		 944
945 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
946 if ((NULL != system->timelines[i]) && (NULL != system->timelines[i]->spinlock)) {
947 mali_spinlock_reentrant_wait(system->timelines[i]->spinlock, tid);
948 system->timelines[i]->destroyed = MALI_TRUE;
949 mali_spinlock_reentrant_signal(system->timelines[i]->spinlock, tid);
950 }
951 }
6fa3eb70
S		 952#endif /* defined(CONFIG_SYNC) */
953
954 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
955 if (NULL != system->timelines[i]) {
956 mali_timeline_destroy(system->timelines[i]);
957 }
958 }
02af6beb 959
6fa3eb70
S		 960 if (NULL != system->spinlock) {
961 mali_spinlock_reentrant_term(system->spinlock);
962 }
963
964 _mali_osk_free(system);
965 }
966}
967
968/**
969 * Find how many waiters are needed for a given fence.
970 *
971 * @param fence The fence to check.
972 * @return Number of waiters needed for fence.
973 */
974static u32 mali_timeline_fence_num_waiters(struct mali_timeline_fence *fence)
975{
976 u32 i, num_waiters = 0;
977
978 MALI_DEBUG_ASSERT_POINTER(fence);
979
980 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
981 if (MALI_TIMELINE_NO_POINT != fence->points[i]) {
982 ++num_waiters;
983 }
984 }
985
986#if defined(CONFIG_SYNC)
987 if (-1 != fence->sync_fd) ++num_waiters;
988#endif /* defined(CONFIG_SYNC) */
989
990 return num_waiters;
991}
992
993static struct mali_timeline_waiter *mali_timeline_system_get_zeroed_waiter(struct mali_timeline_system *system)
994{
995 struct mali_timeline_waiter *waiter;
996
997 MALI_DEBUG_ASSERT_POINTER(system);
998 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
999
1000 waiter = system->waiter_empty_list;
1001 if (NULL != waiter) {
1002 /* Remove waiter from empty list and zero it */
1003 system->waiter_empty_list = waiter->tracker_next;
1004 _mali_osk_memset(waiter, 0, sizeof(*waiter));
1005 }
1006
1007 /* Return NULL if list was empty. */
1008 return waiter;
1009}
1010
1011static void mali_timeline_system_allocate_waiters(struct mali_timeline_system *system,
02af6beb
S		 1012 struct mali_timeline_waiter **tail,
1013 struct mali_timeline_waiter **head,
1014 int max_num_waiters)
6fa3eb70
S		 1015{
1016 u32 i, tid = _mali_osk_get_tid();
1017 mali_bool do_alloc;
1018 struct mali_timeline_waiter *waiter;
1019
1020 MALI_DEBUG_ASSERT_POINTER(system);
1021 MALI_DEBUG_ASSERT_POINTER(tail);
1022 MALI_DEBUG_ASSERT_POINTER(head);
1023
1024 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1025
1026 *head = *tail = NULL;
1027 do_alloc = MALI_FALSE;
1028 i = 0;
1029 while (i < max_num_waiters) {
1030 if (MALI_FALSE == do_alloc) {
1031 waiter = mali_timeline_system_get_zeroed_waiter(system);
1032 if (NULL == waiter) {
1033 do_alloc = MALI_TRUE;
1034 mali_spinlock_reentrant_signal(system->spinlock, tid);
1035 continue;
1036 }
1037 } else {
1038 waiter = _mali_osk_calloc(1, sizeof(struct mali_timeline_waiter));
1039 if (NULL == waiter) break;
1040 }
1041 ++i;
1042 if (NULL == *tail) {
1043 *tail = waiter;
1044 *head = waiter;
1045 } else {
1046 (*head)->tracker_next = waiter;
1047 *head = waiter;
1048 }
1049 }
1050 if (MALI_TRUE == do_alloc) {
1051 mali_spinlock_reentrant_wait(system->spinlock, tid);
1052 }
1053}
1054
1055/**
1056 * Create waiters for the given tracker. The tracker is activated when all waiters are release.
1057 *
1058 * @note Tracker can potentially be activated before this function returns.
1059 *
1060 * @param system Timeline system.
1061 * @param tracker Tracker we will create waiters for.
1062 * @param waiter_tail List of pre-allocated waiters.
1063 * @param waiter_head List of pre-allocated waiters.
1064 */
1065static void mali_timeline_system_create_waiters_and_unlock(struct mali_timeline_system *system,
02af6beb
S		 1066 struct mali_timeline_tracker *tracker,
1067 struct mali_timeline_waiter *waiter_tail,
1068 struct mali_timeline_waiter *waiter_head)
6fa3eb70
S		 1069{
1070 int i;
1071 u32 tid = _mali_osk_get_tid();
1072 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1073#if defined(CONFIG_SYNC)
1074 struct sync_fence *sync_fence = NULL;
1075#endif /* defined(CONFIG_SYNC) */
1076
1077 MALI_DEBUG_ASSERT_POINTER(system);
1078 MALI_DEBUG_ASSERT_POINTER(tracker);
1079
1080 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1081
1082 MALI_DEBUG_ASSERT(NULL == tracker->waiter_head);
1083 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1084 MALI_DEBUG_ASSERT(NULL != tracker->job);
1085
1086 /* Creating waiter object for all the timelines the fence is put on. Inserting this waiter
1087 * into the timelines sorted list of waiters */
1088 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1089 mali_timeline_point point;
1090 struct mali_timeline *timeline;
1091 struct mali_timeline_waiter *waiter;
1092
1093 /* Get point on current timeline from tracker's fence. */
1094 point = tracker->fence.points[i];
1095
1096 if (likely(MALI_TIMELINE_NO_POINT == point)) {
1097 /* Fence contains no point on this timeline so we don't need a waiter. */
1098 continue;
1099 }
1100
1101 timeline = system->timelines[i];
1102 MALI_DEBUG_ASSERT_POINTER(timeline);
1103
1104 if (unlikely(!mali_timeline_is_point_valid(timeline, point))) {
1105 MALI_PRINT_ERROR(("Mali Timeline: point %d is not valid (oldest=%d, next=%d)\n",
02af6beb 1106 point, timeline->point_oldest, timeline->point_next));
6fa3eb70
S		 1107 continue;
1108 }
1109
1110 if (likely(mali_timeline_is_point_released(timeline, point))) {
1111 /* Tracker representing the point has been released so we don't need a
1112 * waiter. */
1113 continue;
1114 }
1115
1116 /* The point is on timeline. */
1117 MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, point));
1118
1119 /* Get a new zeroed waiter object. */
1120 if (likely(NULL != waiter_tail)) {
1121 waiter = waiter_tail;
1122 waiter_tail = waiter_tail->tracker_next;
1123 } else {
1124 MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
1125 continue;
1126 }
1127
1128 /* Yanking the trigger ref count of the tracker. */
1129 tracker->trigger_ref_count++;
1130
1131 waiter->point = point;
1132 waiter->tracker = tracker;
1133
1134 /* Insert waiter on tracker's singly-linked waiter list. */
1135 if (NULL == tracker->waiter_head) {
1136 /* list is empty */
1137 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1138 tracker->waiter_tail = waiter;
1139 } else {
1140 tracker->waiter_head->tracker_next = waiter;
1141 }
1142 tracker->waiter_head = waiter;
1143
1144 /* Add waiter to timeline. */
1145 mali_timeline_insert_waiter(timeline, waiter);
1146 }
1147#if defined(CONFIG_SYNC)
1148 if (-1 != tracker->fence.sync_fd) {
1149 int ret;
1150 struct mali_timeline_waiter *waiter;
1151
1152 sync_fence = sync_fence_fdget(tracker->fence.sync_fd);
1153 if (unlikely(NULL == sync_fence)) {
1154 MALI_PRINT_ERROR(("Mali Timeline: failed to get sync fence from fd %d\n", tracker->fence.sync_fd));
1155 goto exit;
1156 }
1157
1158 /* Check if we have a zeroed waiter object available. */
1159 if (unlikely(NULL == waiter_tail)) {
1160 MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
1161 goto exit;
1162 }
1163
1164 /* Start asynchronous wait that will release waiter when the fence is signaled. */
1165 sync_fence_waiter_init(&tracker->sync_fence_waiter, mali_timeline_sync_fence_callback);
1166 ret = sync_fence_wait_async(sync_fence, &tracker->sync_fence_waiter);
1167 if (1 == ret) {
1168 /* Fence already signaled, no waiter needed. */
02af6beb 1169 tracker->fence.sync_fd = -1;
6fa3eb70
S		 1170 goto exit;
1171 } else if (0 != ret) {
1172 MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, ret));
1173 tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT;
1174 goto exit;
1175 }
1176
1177 /* Grab new zeroed waiter object. */
1178 waiter = waiter_tail;
1179 waiter_tail = waiter_tail->tracker_next;
1180
1181 /* Increase the trigger ref count of the tracker. */
1182 tracker->trigger_ref_count++;
1183
1184 waiter->point = MALI_TIMELINE_NO_POINT;
1185 waiter->tracker = tracker;
1186
1187 /* Insert waiter on tracker's singly-linked waiter list. */
1188 if (NULL == tracker->waiter_head) {
1189 /* list is empty */
1190 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1191 tracker->waiter_tail = waiter;
1192 } else {
1193 tracker->waiter_head->tracker_next = waiter;
1194 }
1195 tracker->waiter_head = waiter;
1196
1197 /* Also store waiter in separate field for easy access by sync callback. */
1198 tracker->waiter_sync = waiter;
1199
1200 /* Store the sync fence in tracker so we can retrieve in abort session, if needed. */
1201 tracker->sync_fence = sync_fence;
1202
1203 sync_fence = NULL;
1204 }
1205exit:
1206#endif /* defined(CONFIG_SYNC) */
1207
1208 if (NULL != waiter_tail) {
1209 mali_timeline_system_release_waiter_list(system, waiter_tail, waiter_head);
1210 }
1211
1212 /* Release the initial trigger ref count. */
1213 tracker->trigger_ref_count--;
1214
1215 /* If there were no waiters added to this tracker we activate immediately. */
1216 if (0 == tracker->trigger_ref_count) {
1217 schedule_mask |= mali_timeline_tracker_activate(tracker);
1218 }
1219
1220 mali_spinlock_reentrant_signal(system->spinlock, tid);
1221
1222#if defined(CONFIG_SYNC)
1223 if (NULL != sync_fence) {
1224 sync_fence_put(sync_fence);
1225 }
1226#endif /* defined(CONFIG_SYNC) */
1227
02af6beb 1228 mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
6fa3eb70
S		 1229}
1230
1231mali_timeline_point mali_timeline_system_add_tracker(struct mali_timeline_system *system,
02af6beb
S		 1232 struct mali_timeline_tracker *tracker,
1233 enum mali_timeline_id timeline_id)
6fa3eb70
S		 1234{
1235 int num_waiters = 0;
1236 struct mali_timeline_waiter *waiter_tail, *waiter_head;
1237 u32 tid = _mali_osk_get_tid();
1238 mali_timeline_point point = MALI_TIMELINE_NO_POINT;
1239
1240 MALI_DEBUG_ASSERT_POINTER(system);
1241 MALI_DEBUG_ASSERT_POINTER(system->session);
1242 MALI_DEBUG_ASSERT_POINTER(tracker);
1243
1244 MALI_DEBUG_ASSERT(MALI_FALSE == system->session->is_aborting);
1245 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > tracker->type);
1246 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
1247
1248 MALI_DEBUG_PRINT(4, ("Mali Timeline: adding tracker for job %p, timeline: %d\n", tracker->job, timeline_id));
1249
1250 MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
1251 tracker->system = system;
1252
1253 mali_spinlock_reentrant_wait(system->spinlock, tid);
1254
1255 num_waiters = mali_timeline_fence_num_waiters(&tracker->fence);
1256
1257 /* Allocate waiters. */
1258 mali_timeline_system_allocate_waiters(system, &waiter_tail, &waiter_head, num_waiters);
1259 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1260
1261 /* Add tracker to timeline. This will allocate a point for the tracker on the timeline. If
1262 * timeline ID is MALI_TIMELINE_NONE the tracker will NOT be added to a timeline and the
1263 * point will be MALI_TIMELINE_NO_POINT.
1264 *
1265 * NOTE: the tracker can fail to be added if the timeline is full. If this happens, the
1266 * point will be MALI_TIMELINE_NO_POINT. */
1267 MALI_DEBUG_ASSERT(timeline_id < MALI_TIMELINE_MAX || timeline_id == MALI_TIMELINE_NONE);
1268 if (likely(timeline_id < MALI_TIMELINE_MAX)) {
1269 struct mali_timeline *timeline = system->timelines[timeline_id];
1270 mali_timeline_insert_tracker(timeline, tracker);
1271 MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
1272 }
1273
1274 point = tracker->point;
1275
1276 /* Create waiters for tracker based on supplied fence. Each waiter will increase the
1277 * trigger ref count. */
1278 mali_timeline_system_create_waiters_and_unlock(system, tracker, waiter_tail, waiter_head);
1279 tracker = NULL;
1280
1281 /* At this point the tracker object might have been freed so we should no longer
1282 * access it. */
1283
1284
1285 /* The tracker will always be activated after calling add_tracker, even if NO_POINT is
1286 * returned. */
1287 return point;
1288}
1289
1290static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system,
02af6beb 1291 struct mali_timeline_waiter *waiter)
6fa3eb70
S		 1292{
1293 struct mali_timeline_tracker *tracker;
1294 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1295
1296 MALI_DEBUG_ASSERT_POINTER(system);
1297 MALI_DEBUG_ASSERT_POINTER(waiter);
1298
1299 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1300
1301 tracker = waiter->tracker;
1302 MALI_DEBUG_ASSERT_POINTER(tracker);
1303
1304 /* At this point the waiter has been removed from the timeline's waiter list, but it is
1305 * still on the tracker's waiter list. All of the tracker's waiters will be released when
1306 * the tracker is activated. */
1307
1308 waiter->point = MALI_TIMELINE_NO_POINT;
1309 waiter->tracker = NULL;
1310
1311 tracker->trigger_ref_count--;
1312 if (0 == tracker->trigger_ref_count) {
1313 /* This was the last waiter; activate tracker */
1314 schedule_mask |= mali_timeline_tracker_activate(tracker);
1315 tracker = NULL;
1316 }
1317
1318 return schedule_mask;
1319}
1320
1321mali_timeline_point mali_timeline_system_get_latest_point(struct mali_timeline_system *system,
02af6beb 1322 enum mali_timeline_id timeline_id)
6fa3eb70
S		 1323{
1324 mali_timeline_point point;
1325 struct mali_timeline *timeline;
1326 u32 tid = _mali_osk_get_tid();
1327
1328 MALI_DEBUG_ASSERT_POINTER(system);
1329
1330 if (MALI_TIMELINE_MAX <= timeline_id) {
1331 return MALI_TIMELINE_NO_POINT;
1332 }
1333
1334 mali_spinlock_reentrant_wait(system->spinlock, tid);
1335
1336 timeline = system->timelines[timeline_id];
1337 MALI_DEBUG_ASSERT_POINTER(timeline);
1338
1339 point = MALI_TIMELINE_NO_POINT;
1340 if (timeline->point_oldest != timeline->point_next) {
1341 point = timeline->point_next - 1;
1342 if (MALI_TIMELINE_NO_POINT == point) point--;
1343 }
1344
1345 mali_spinlock_reentrant_signal(system->spinlock, tid);
1346
1347 return point;
1348}
1349
02af6beb
S		 1350void mali_timeline_initialize(void)
1351{
1352 _mali_osk_atomic_init(&gp_tracker_count, 0);
1353 _mali_osk_atomic_init(&phy_pp_tracker_count, 0);
1354 _mali_osk_atomic_init(&virt_pp_tracker_count, 0);
1355}
1356
1357void mali_timeline_terminate(void)
1358{
1359 _mali_osk_atomic_term(&gp_tracker_count);
1360 _mali_osk_atomic_term(&phy_pp_tracker_count);
1361 _mali_osk_atomic_term(&virt_pp_tracker_count);
1362}
1363
6fa3eb70
S		 1364#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS)
1365
1366static mali_bool is_waiting_on_timeline(struct mali_timeline_tracker *tracker, enum mali_timeline_id id)
1367{
1368 struct mali_timeline *timeline;
1369 struct mali_timeline_system *system;
1370
1371 MALI_DEBUG_ASSERT_POINTER(tracker);
1372
1373 MALI_DEBUG_ASSERT_POINTER(tracker->timeline);
1374 timeline = tracker->timeline;
1375
1376 MALI_DEBUG_ASSERT_POINTER(timeline->system);
1377 system = timeline->system;
1378
1379 if (MALI_TIMELINE_MAX > id) {
02af6beb
S		 1380 if (MALI_TIMELINE_NO_POINT != tracker->fence.points[id]) {
1381 return mali_timeline_is_point_on(system->timelines[id], tracker->fence.points[id]);
1382 } else {
1383 return MALI_FALSE;
1384 }
6fa3eb70
S		 1385 } else {
1386 MALI_DEBUG_ASSERT(MALI_TIMELINE_NONE == id);
1387 return MALI_FALSE;
1388 }
1389}
1390
1391static const char *timeline_id_to_string(enum mali_timeline_id id)
1392{
1393 switch (id) {
1394 case MALI_TIMELINE_GP:
02af6beb 1395 return "GP";
6fa3eb70 1396 case MALI_TIMELINE_PP:
02af6beb 1397 return "PP";
6fa3eb70
S		 1398 case MALI_TIMELINE_SOFT:
1399 return "SOFT";
1400 default:
1401 return "NONE";
1402 }
1403}
1404
1405static const char *timeline_tracker_type_to_string(enum mali_timeline_tracker_type type)
1406{
1407 switch (type) {
1408 case MALI_TIMELINE_TRACKER_GP:
02af6beb 1409 return "GP";
6fa3eb70 1410 case MALI_TIMELINE_TRACKER_PP:
02af6beb 1411 return "PP";
6fa3eb70
S		 1412 case MALI_TIMELINE_TRACKER_SOFT:
1413 return "SOFT";
1414 case MALI_TIMELINE_TRACKER_WAIT:
1415 return "WAIT";
1416 case MALI_TIMELINE_TRACKER_SYNC:
1417 return "SYNC";
1418 default:
1419 return "INVALID";
1420 }
1421}
1422
1423mali_timeline_tracker_state mali_timeline_debug_get_tracker_state(struct mali_timeline_tracker *tracker)
1424{
1425 struct mali_timeline *timeline = NULL;
1426
1427 MALI_DEBUG_ASSERT_POINTER(tracker);
1428 timeline = tracker->timeline;
1429
1430 if (0 != tracker->trigger_ref_count) {
1431 return MALI_TIMELINE_TS_WAITING;
1432 }
1433
1434 if (timeline && (timeline->tracker_tail == tracker || NULL != tracker->timeline_prev)) {
1435 return MALI_TIMELINE_TS_ACTIVE;
1436 }
1437
1438 if (timeline && (MALI_TIMELINE_NO_POINT == tracker->point)) {
1439 return MALI_TIMELINE_TS_INIT;
1440 }
1441
1442 return MALI_TIMELINE_TS_FINISH;
1443}
1444
02af6beb 1445void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker, _mali_osk_print_ctx *print_ctx)
6fa3eb70
S		 1446{
1447 const char *tracker_state = "IWAF";
02af6beb
S		 1448 char state_char = 'I';
1449 char tracker_type[32] = {0};
6fa3eb70
S		 1450
1451 MALI_DEBUG_ASSERT_POINTER(tracker);
1452
02af6beb
S		 1453 state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker));
1454 _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type));
1455
1456#if defined(CONFIG_SYNC)
1457 if (0 != tracker->trigger_ref_count) {
1458 _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n",
1459 tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1460 is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1461 is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1462 is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1463 tracker->fence.sync_fd, tracker->sync_fence, tracker->job);
1464 } else {
1465 _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n",
1466 tracker_type, tracker->point, state_char,
1467 tracker->fence.sync_fd, tracker->sync_fence, tracker->job);
1468 }
1469#else
6fa3eb70 1470 if (0 != tracker->trigger_ref_count) {
02af6beb
S		 1471 _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n",
1472 tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1473 is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1474 is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1475 is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1476 tracker->job);
6fa3eb70 1477 } else {
02af6beb
S		 1478 _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c job:(0x%08X)\n",
1479 tracker_type, tracker->point, state_char,
1480 tracker->job);
6fa3eb70 1481 }
02af6beb 1482#endif
6fa3eb70
S		 1483}
1484
02af6beb 1485void mali_timeline_debug_print_timeline(struct mali_timeline *timeline, _mali_osk_print_ctx *print_ctx)
6fa3eb70
S		 1486{
1487 struct mali_timeline_tracker *tracker = NULL;
6fa3eb70
S		 1488
1489 MALI_DEBUG_ASSERT_POINTER(timeline);
1490
1491 tracker = timeline->tracker_tail;
02af6beb
S		 1492 while (NULL != tracker) {
1493 mali_timeline_debug_print_tracker(tracker, print_ctx);
6fa3eb70
S		 1494 tracker = tracker->timeline_next;
1495 }
02af6beb
S		 1496}
1497
1498#if !(LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0))
1499void mali_timeline_debug_direct_print_tracker(struct mali_timeline_tracker *tracker)
1500{
1501 const char *tracker_state = "IWAF";
1502 char state_char = 'I';
1503 char tracker_type[32] = {0};
1504
1505 MALI_DEBUG_ASSERT_POINTER(tracker);
1506
1507 state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker));
1508 _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type));
1509
1510#if defined(CONFIG_SYNC)
1511 if (0 != tracker->trigger_ref_count) {
1512 MALI_PRINT(("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n",
1513 tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1514 is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1515 is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1516 is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1517 tracker->fence.sync_fd, tracker->sync_fence, tracker->job));
1518 } else {
1519 MALI_PRINT(("TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n",
1520 tracker_type, tracker->point, state_char,
1521 tracker->fence.sync_fd, tracker->sync_fence, tracker->job));
1522 }
1523#else
1524 if (0 != tracker->trigger_ref_count) {
1525 MALI_PRINT(("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n",
1526 tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1527 is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1528 is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1529 is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1530 tracker->job));
1531 } else {
1532 MALI_PRINT(("TL: %s %u %c job:(0x%08X)\n",
1533 tracker_type, tracker->point, state_char,
1534 tracker->job));
1535 }
1536#endif
1537}
1538
1539void mali_timeline_debug_direct_print_timeline(struct mali_timeline *timeline)
1540{
1541 struct mali_timeline_tracker *tracker = NULL;
6fa3eb70 1542
02af6beb
S		 1543 MALI_DEBUG_ASSERT_POINTER(timeline);
1544
1545 tracker = timeline->tracker_tail;
1546 while (NULL != tracker) {
1547 mali_timeline_debug_direct_print_tracker(tracker);
1548 tracker = tracker->timeline_next;
6fa3eb70
S		 1549 }
1550}
1551
02af6beb
S		 1552#endif
1553
1554void mali_timeline_debug_print_system(struct mali_timeline_system *system, _mali_osk_print_ctx *print_ctx)
6fa3eb70
S		 1555{
1556 int i;
1557 int num_printed = 0;
02af6beb 1558 u32 tid = _mali_osk_get_tid();
6fa3eb70
S		 1559
1560 MALI_DEBUG_ASSERT_POINTER(system);
1561
02af6beb
S		 1562 mali_spinlock_reentrant_wait(system->spinlock, tid);
1563
6fa3eb70
S		 1564 /* Print all timelines */
1565 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1566 struct mali_timeline *timeline = system->timelines[i];
1567
1568 MALI_DEBUG_ASSERT_POINTER(timeline);
1569
1570 if (NULL == timeline->tracker_head) continue;
1571
02af6beb
S		 1572 _mali_osk_ctxprintf(print_ctx, "TL: Timeline %s:\n",
1573 timeline_id_to_string((enum mali_timeline_id)i));
1574
1575 mali_timeline_debug_print_timeline(timeline, print_ctx);
6fa3eb70
S		 1576 num_printed++;
1577 }
1578
1579 if (0 == num_printed) {
02af6beb 1580 _mali_osk_ctxprintf(print_ctx, "TL: All timelines empty\n");
6fa3eb70 1581 }
02af6beb
S		 1582
1583 mali_spinlock_reentrant_signal(system->spinlock, tid);
6fa3eb70
S		 1584}
1585
1586#endif /* defined(MALI_TIMELINE_DEBUG_FUNCTIONS) */
kernel source for telekom puls (alcatel/mediatek)