Commit | Line | Data |
---|---|---|
6fa3eb70 S |
1 | /* |
2 | * This confidential and proprietary software may be used only as | |
3 | * authorised by a licensing agreement from ARM Limited | |
4 | * (C) COPYRIGHT 2013 ARM Limited | |
5 | * ALL RIGHTS RESERVED | |
6 | * The entire notice above must be reproduced on all authorised | |
7 | * copies and copies may only be made to the extent permitted | |
8 | * by a licensing agreement from ARM Limited. | |
9 | */ | |
10 | ||
11 | #ifndef __MALI_TIMELINE_H__ | |
12 | #define __MALI_TIMELINE_H__ | |
13 | ||
14 | #include "mali_osk.h" | |
15 | #include "mali_ukk.h" | |
16 | #include "mali_session.h" | |
17 | #include "mali_kernel_common.h" | |
18 | #include "mali_spinlock_reentrant.h" | |
19 | #include "mali_sync.h" | |
20 | #include "mali_scheduler_types.h" | |
21 | ||
22 | /** | |
23 | * Soft job timeout. | |
24 | * | |
25 | * Soft jobs have to be signaled as complete after activation. Normally this is done by user space, | |
26 | * but in order to guarantee that every soft job is completed, we also have a timer. | |
27 | */ | |
28 | #define MALI_TIMELINE_TIMEOUT_HZ ((u32) (HZ * 3 / 2)) /* 1500 ms. */ | |
29 | ||
30 | /** | |
31 | * Timeline type. | |
32 | */ | |
33 | typedef enum mali_timeline_id { | |
34 | MALI_TIMELINE_GP = MALI_UK_TIMELINE_GP, /**< GP job timeline. */ | |
35 | MALI_TIMELINE_PP = MALI_UK_TIMELINE_PP, /**< PP job timeline. */ | |
36 | MALI_TIMELINE_SOFT = MALI_UK_TIMELINE_SOFT, /**< Soft job timeline. */ | |
37 | MALI_TIMELINE_MAX = MALI_UK_TIMELINE_MAX | |
38 | } mali_timeline_id; | |
39 | ||
40 | /** | |
41 | * Used by trackers that should not be added to a timeline (@ref mali_timeline_system_add_tracker). | |
42 | */ | |
43 | #define MALI_TIMELINE_NONE MALI_TIMELINE_MAX | |
44 | ||
45 | /** | |
46 | * Tracker type. | |
47 | */ | |
48 | typedef enum mali_timeline_tracker_type { | |
49 | MALI_TIMELINE_TRACKER_GP = 0, /**< Tracker used by GP jobs. */ | |
50 | MALI_TIMELINE_TRACKER_PP = 1, /**< Tracker used by PP jobs. */ | |
51 | MALI_TIMELINE_TRACKER_SOFT = 2, /**< Tracker used by soft jobs. */ | |
52 | MALI_TIMELINE_TRACKER_WAIT = 3, /**< Tracker used for fence wait. */ | |
53 | MALI_TIMELINE_TRACKER_SYNC = 4, /**< Tracker used for sync fence. */ | |
54 | MALI_TIMELINE_TRACKER_MAX = 5, | |
55 | } mali_timeline_tracker_type; | |
56 | ||
57 | /** | |
58 | * Tracker activation error. | |
59 | */ | |
60 | typedef u32 mali_timeline_activation_error; | |
61 | #define MALI_TIMELINE_ACTIVATION_ERROR_NONE 0 | |
62 | #define MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT (1<<1) | |
63 | #define MALI_TIMELINE_ACTIVATION_ERROR_FATAL_BIT (1<<0) | |
64 | ||
65 | /** | |
66 | * Type used to represent a point on a timeline. | |
67 | */ | |
68 | typedef u32 mali_timeline_point; | |
69 | ||
70 | /** | |
71 | * Used to represent that no point on a timeline. | |
72 | */ | |
73 | #define MALI_TIMELINE_NO_POINT ((mali_timeline_point) 0) | |
74 | ||
75 | /** | |
76 | * The maximum span of points on a timeline. A timeline will be considered full if the difference | |
77 | * between the oldest and newest points is equal or larger to this value. | |
78 | */ | |
79 | #define MALI_TIMELINE_MAX_POINT_SPAN 65536 | |
80 | ||
81 | /** | |
82 | * Magic value used to assert on validity of trackers. | |
83 | */ | |
84 | #define MALI_TIMELINE_TRACKER_MAGIC 0xabcdabcd | |
85 | ||
86 | struct mali_timeline; | |
87 | struct mali_timeline_waiter; | |
88 | struct mali_timeline_tracker; | |
89 | ||
90 | /** | |
91 | * Timeline fence. | |
92 | */ | |
93 | struct mali_timeline_fence { | |
94 | mali_timeline_point points[MALI_TIMELINE_MAX]; /**< For each timeline, a point or MALI_TIMELINE_NO_POINT. */ | |
95 | s32 sync_fd; /**< A file descriptor representing a sync fence, or -1. */ | |
96 | }; | |
97 | ||
98 | /** | |
99 | * Timeline system. | |
100 | * | |
101 | * The Timeline system has a set of timelines associated with a session. | |
102 | */ | |
103 | struct mali_timeline_system { | |
104 | struct mali_spinlock_reentrant *spinlock; /**< Spin lock protecting the timeline system */ | |
105 | struct mali_timeline *timelines[MALI_TIMELINE_MAX]; /**< The timelines in this system */ | |
106 | ||
107 | /* Single-linked list of unused waiter objects. Uses the tracker_next field in tracker. */ | |
108 | struct mali_timeline_waiter *waiter_empty_list; | |
109 | ||
110 | struct mali_session_data *session; /**< Session that owns this system. */ | |
111 | ||
112 | mali_bool timer_enabled; /**< Set to MALI_TRUE if soft job timer should be enabled, MALI_FALSE if not. */ | |
113 | ||
114 | _mali_osk_wait_queue_t *wait_queue; /**< Wait queue. */ | |
115 | ||
116 | #if defined(CONFIG_SYNC) | |
117 | struct sync_timeline *signaled_sync_tl; /**< Special sync timeline used to create pre-signaled sync fences */ | |
118 | #endif /* defined(CONFIG_SYNC) */ | |
119 | }; | |
120 | ||
121 | /** | |
122 | * Timeline. Each Timeline system will have MALI_TIMELINE_MAX timelines. | |
123 | */ | |
124 | struct mali_timeline { | |
125 | mali_timeline_point point_next; /**< The next available point. */ | |
126 | mali_timeline_point point_oldest; /**< The oldest point not released. */ | |
127 | ||
128 | /* Double-linked list of trackers. Sorted in ascending order by tracker->time_number with | |
129 | * tail pointing to the tracker with the oldest time. */ | |
130 | struct mali_timeline_tracker *tracker_head; | |
131 | struct mali_timeline_tracker *tracker_tail; | |
132 | ||
133 | /* Double-linked list of waiters. Sorted in ascending order by waiter->time_number_wait | |
134 | * with tail pointing to the waiter with oldest wait time. */ | |
135 | struct mali_timeline_waiter *waiter_head; | |
136 | struct mali_timeline_waiter *waiter_tail; | |
137 | ||
138 | struct mali_timeline_system *system; /**< Timeline system this timeline belongs to. */ | |
139 | enum mali_timeline_id id; /**< Timeline type. */ | |
140 | ||
141 | #if defined(CONFIG_SYNC) | |
142 | struct sync_timeline *sync_tl; /**< Sync timeline that corresponds to this timeline. */ | |
143 | #endif /* defined(CONFIG_SYNC) */ | |
144 | ||
145 | /* The following fields are used to time out soft job trackers. */ | |
146 | _mali_osk_wq_delayed_work_t *delayed_work; | |
147 | mali_bool timer_active; | |
148 | }; | |
149 | ||
150 | /** | |
151 | * Timeline waiter. | |
152 | */ | |
153 | struct mali_timeline_waiter { | |
154 | mali_timeline_point point; /**< Point on timeline we are waiting for to be released. */ | |
155 | struct mali_timeline_tracker *tracker; /**< Tracker that is waiting. */ | |
156 | ||
157 | struct mali_timeline_waiter *timeline_next; /**< Next waiter on timeline's waiter list. */ | |
158 | struct mali_timeline_waiter *timeline_prev; /**< Previous waiter on timeline's waiter list. */ | |
159 | ||
160 | struct mali_timeline_waiter *tracker_next; /**< Next waiter on tracker's waiter list. */ | |
161 | }; | |
162 | ||
163 | /** | |
164 | * Timeline tracker. | |
165 | */ | |
166 | struct mali_timeline_tracker { | |
167 | MALI_DEBUG_CODE(u32 magic); /**< Should always be MALI_TIMELINE_TRACKER_MAGIC for a valid tracker. */ | |
168 | ||
169 | mali_timeline_point point; /**< Point on timeline for this tracker */ | |
170 | ||
171 | struct mali_timeline_tracker *timeline_next; /**< Next tracker on timeline's tracker list */ | |
172 | struct mali_timeline_tracker *timeline_prev; /**< Previous tracker on timeline's tracker list */ | |
173 | ||
174 | u32 trigger_ref_count; /**< When zero tracker will be activated */ | |
175 | mali_timeline_activation_error activation_error; /**< Activation error. */ | |
176 | struct mali_timeline_fence fence; /**< Fence used to create this tracker */ | |
177 | ||
178 | /* Single-linked list of waiters. Sorted in order of insertions with | |
179 | * tail pointing to first waiter. */ | |
180 | struct mali_timeline_waiter *waiter_head; | |
181 | struct mali_timeline_waiter *waiter_tail; | |
182 | ||
183 | #if defined(CONFIG_SYNC) | |
184 | /* These are only used if the tracker is waiting on a sync fence. */ | |
185 | struct mali_timeline_waiter *waiter_sync; /**< A direct pointer to timeline waiter representing sync fence. */ | |
186 | struct sync_fence_waiter sync_fence_waiter; /**< Used to connect sync fence and tracker in sync fence wait callback. */ | |
187 | struct sync_fence *sync_fence; /**< The sync fence this tracker is waiting on. */ | |
188 | _mali_osk_list_t sync_fence_cancel_list; /**< List node used to cancel sync fence waiters. */ | |
189 | #endif /* defined(CONFIG_SYNC) */ | |
190 | ||
191 | struct mali_timeline_system *system; /**< Timeline system. */ | |
192 | struct mali_timeline *timeline; /**< Timeline, or NULL if not on a timeline. */ | |
193 | enum mali_timeline_tracker_type type; /**< Type of tracker. */ | |
194 | void *job; /**< Owner of tracker. */ | |
195 | ||
196 | /* The following fields are used to time out soft job trackers. */ | |
197 | u32 os_tick_create; | |
198 | u32 os_tick_activate; | |
199 | mali_bool timer_active; | |
200 | }; | |
201 | ||
202 | /** | |
203 | * What follows is a set of functions to check the state of a timeline and to determine where on a | |
204 | * timeline a given point is. Most of these checks will translate the timeline so the oldest point | |
205 | * on the timeline is aligned with zero. Remember that all of these calculation are done on | |
206 | * unsigned integers. | |
207 | * | |
208 | * The following example illustrates the three different states a point can be in. The timeline has | |
209 | * been translated to put the oldest point at zero: | |
210 | * | |
211 | * | |
212 | * | |
213 | * [ point is in forbidden zone ] | |
214 | * 64k wide | |
215 | * MALI_TIMELINE_MAX_POINT_SPAN | |
216 | * | |
217 | * [ point is on timeline ) ( point is released ] | |
218 | * | |
219 | * 0--------------------------##############################--------------------2^32 - 1 | |
220 | * ^ ^ | |
221 | * \ | | |
222 | * oldest point on timeline | | |
223 | * \ | |
224 | * next point on timeline | |
225 | */ | |
226 | ||
227 | /** | |
228 | * Compare two timeline points | |
229 | * | |
230 | * Returns true if a is after b, false if a is before or equal to b. | |
231 | * | |
232 | * This funcion ignores MALI_TIMELINE_MAX_POINT_SPAN. Wrapping is supported and | |
233 | * the result will be correct if the points is less then UINT_MAX/2 apart. | |
234 | * | |
235 | * @param a Point on timeline | |
236 | * @param b Point on timeline | |
237 | * @return MALI_TRUE if a is after b | |
238 | */ | |
239 | MALI_STATIC_INLINE mali_bool mali_timeline_point_after(mali_timeline_point a, mali_timeline_point b) | |
240 | { | |
241 | return 0 > ((s32)b) - ((s32)a); | |
242 | } | |
243 | ||
244 | /** | |
245 | * Check if a point is on timeline. A point is on a timeline if it is greater than, or equal to, | |
246 | * the oldest point, and less than the next point. | |
247 | * | |
248 | * @param timeline Timeline. | |
249 | * @param point Point on timeline. | |
250 | * @return MALI_TRUE if point is on timeline, MALI_FALSE if not. | |
251 | */ | |
252 | MALI_STATIC_INLINE mali_bool mali_timeline_is_point_on(struct mali_timeline *timeline, mali_timeline_point point) | |
253 | { | |
254 | MALI_DEBUG_ASSERT_POINTER(timeline); | |
255 | MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); | |
256 | ||
257 | return (point - timeline->point_oldest) < (timeline->point_next - timeline->point_oldest); | |
258 | } | |
259 | ||
260 | /** | |
261 | * Check if a point has been released. A point is released if it is older than the oldest point on | |
262 | * the timeline, newer than the next point, and also not in the forbidden zone. | |
263 | * | |
264 | * @param timeline Timeline. | |
265 | * @param point Point on timeline. | |
266 | * @return MALI_TRUE if point has been release, MALI_FALSE if not. | |
267 | */ | |
268 | MALI_STATIC_INLINE mali_bool mali_timeline_is_point_released(struct mali_timeline *timeline, mali_timeline_point point) | |
269 | { | |
270 | mali_timeline_point point_normalized; | |
271 | mali_timeline_point next_normalized; | |
272 | ||
273 | MALI_DEBUG_ASSERT_POINTER(timeline); | |
274 | MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); | |
275 | ||
276 | point_normalized = point - timeline->point_oldest; | |
277 | next_normalized = timeline->point_next - timeline->point_oldest; | |
278 | ||
279 | return point_normalized > (next_normalized + MALI_TIMELINE_MAX_POINT_SPAN); | |
280 | } | |
281 | ||
282 | /** | |
283 | * Check if a point is valid. A point is valid if is on the timeline or has been released. | |
284 | * | |
285 | * @param timeline Timeline. | |
286 | * @param point Point on timeline. | |
287 | * @return MALI_TRUE if point is valid, MALI_FALSE if not. | |
288 | */ | |
289 | MALI_STATIC_INLINE mali_bool mali_timeline_is_point_valid(struct mali_timeline *timeline, mali_timeline_point point) | |
290 | { | |
291 | MALI_DEBUG_ASSERT_POINTER(timeline); | |
292 | return mali_timeline_is_point_on(timeline, point) || mali_timeline_is_point_released(timeline, point); | |
293 | } | |
294 | ||
295 | /** | |
296 | * Check if timeline is empty (has no points on it). A timeline is empty if next == oldest. | |
297 | * | |
298 | * @param timeline Timeline. | |
299 | * @return MALI_TRUE if timeline is empty, MALI_FALSE if not. | |
300 | */ | |
301 | MALI_STATIC_INLINE mali_bool mali_timeline_is_empty(struct mali_timeline *timeline) | |
302 | { | |
303 | MALI_DEBUG_ASSERT_POINTER(timeline); | |
304 | return timeline->point_next == timeline->point_oldest; | |
305 | } | |
306 | ||
307 | /** | |
308 | * Check if timeline is full. A valid timeline cannot span more than 64k points (@ref | |
309 | * MALI_TIMELINE_MAX_POINT_SPAN). | |
310 | * | |
311 | * @param timeline Timeline. | |
312 | * @return MALI_TRUE if timeline is full, MALI_FALSE if not. | |
313 | */ | |
314 | MALI_STATIC_INLINE mali_bool mali_timeline_is_full(struct mali_timeline *timeline) | |
315 | { | |
316 | MALI_DEBUG_ASSERT_POINTER(timeline); | |
317 | return MALI_TIMELINE_MAX_POINT_SPAN <= (timeline->point_next - timeline->point_oldest); | |
318 | } | |
319 | ||
320 | /** | |
321 | * Create a new timeline system. | |
322 | * | |
323 | * @param session The session this timeline system will belong to. | |
324 | * @return New timeline system. | |
325 | */ | |
326 | struct mali_timeline_system *mali_timeline_system_create(struct mali_session_data *session); | |
327 | ||
328 | /** | |
329 | * Abort timeline system. | |
330 | * | |
331 | * This will release all pending waiters in the timeline system causing all trackers to be | |
332 | * activated. | |
333 | * | |
334 | * @param system Timeline system to abort all jobs from. | |
335 | */ | |
336 | void mali_timeline_system_abort(struct mali_timeline_system *system); | |
337 | ||
338 | /** | |
339 | * Destroy an empty timeline system. | |
340 | * | |
341 | * @note @ref mali_timeline_system_abort() should be called prior to this function. | |
342 | * | |
343 | * @param system Timeline system to destroy. | |
344 | */ | |
345 | void mali_timeline_system_destroy(struct mali_timeline_system *system); | |
346 | ||
347 | /** | |
348 | * Stop the soft job timer. | |
349 | * | |
350 | * @param system Timeline system | |
351 | */ | |
352 | void mali_timeline_system_stop_timer(struct mali_timeline_system *system); | |
353 | ||
354 | /** | |
355 | * Add a tracker to a timeline system and optionally also on a timeline. | |
356 | * | |
357 | * Once added to the timeline system, the tracker is guaranteed to be activated. The tracker can be | |
358 | * activated before this function returns. Thus, it is also possible that the tracker is released | |
359 | * before this function returns, depending on the tracker type. | |
360 | * | |
361 | * @note Tracker must be initialized (@ref mali_timeline_tracker_init) before being added to the | |
362 | * timeline system. | |
363 | * | |
364 | * @param system Timeline system the tracker will be added to. | |
365 | * @param tracker The tracker to be added. | |
366 | * @param timeline_id Id of the timeline the tracker will be added to, or | |
367 | * MALI_TIMELINE_NONE if it should not be added on a timeline. | |
368 | * @return Point on timeline identifying this tracker, or MALI_TIMELINE_NO_POINT if not on timeline. | |
369 | */ | |
370 | mali_timeline_point mali_timeline_system_add_tracker(struct mali_timeline_system *system, | |
371 | struct mali_timeline_tracker *tracker, | |
372 | enum mali_timeline_id timeline_id); | |
373 | ||
374 | /** | |
375 | * Get latest point on timeline. | |
376 | * | |
377 | * @param system Timeline system. | |
378 | * @param timeline_id Id of timeline to get latest point from. | |
379 | * @return Latest point on timeline, or MALI_TIMELINE_NO_POINT if the timeline is empty. | |
380 | */ | |
381 | mali_timeline_point mali_timeline_system_get_latest_point(struct mali_timeline_system *system, | |
382 | enum mali_timeline_id timeline_id); | |
383 | ||
384 | /** | |
385 | * Initialize tracker. | |
386 | * | |
387 | * Must be called before tracker is added to timeline system (@ref mali_timeline_system_add_tracker). | |
388 | * | |
389 | * @param tracker Tracker to initialize. | |
390 | * @param type Type of tracker. | |
391 | * @param fence Fence used to set up dependencies for tracker. | |
392 | * @param job Pointer to job struct this tracker is associated with. | |
393 | */ | |
394 | void mali_timeline_tracker_init(struct mali_timeline_tracker *tracker, | |
395 | mali_timeline_tracker_type type, | |
396 | struct mali_timeline_fence *fence, | |
397 | void *job); | |
398 | ||
399 | /** | |
400 | * Grab trigger ref count on tracker. | |
401 | * | |
402 | * This will prevent tracker from being activated until the trigger ref count reaches zero. | |
403 | * | |
404 | * @note Tracker must have been initialized (@ref mali_timeline_tracker_init). | |
405 | * | |
406 | * @param system Timeline system. | |
407 | * @param tracker Tracker. | |
408 | */ | |
409 | void mali_timeline_system_tracker_get(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker); | |
410 | ||
411 | /** | |
412 | * Release trigger ref count on tracker. | |
413 | * | |
414 | * If the trigger ref count reaches zero, the tracker will be activated. | |
415 | * | |
416 | * @param system Timeline system. | |
417 | * @param tracker Tracker. | |
418 | * @param activation_error Error bitmask if activated with error, or MALI_TIMELINE_ACTIVATION_ERROR_NONE if no error. | |
419 | * @return Scheduling bitmask. | |
420 | */ | |
421 | mali_scheduler_mask mali_timeline_system_tracker_put(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker, mali_timeline_activation_error activation_error); | |
422 | ||
423 | /** | |
424 | * Release a tracker from the timeline system. | |
425 | * | |
426 | * This is used to signal that the job being tracker is finished, either due to normal circumstances | |
427 | * (job complete/abort) or due to a timeout. | |
428 | * | |
429 | * We may need to schedule some subsystems after a tracker has been released and the returned | |
430 | * bitmask will tell us if it is necessary. If the return value is non-zero, this value needs to be | |
431 | * sent as an input parameter to @ref mali_scheduler_schedule_from_mask() to do the scheduling. | |
432 | * | |
433 | * @note Tracker must have been activated before being released. | |
434 | * @warning Not calling @ref mali_scheduler_schedule_from_mask() after releasing a tracker can lead | |
435 | * to a deadlock. | |
436 | * | |
437 | * @param tracker Tracker being released. | |
438 | * @return Scheduling bitmask. | |
439 | */ | |
440 | mali_scheduler_mask mali_timeline_tracker_release(struct mali_timeline_tracker *tracker); | |
441 | ||
442 | /** | |
443 | * Copy data from a UK fence to a Timeline fence. | |
444 | * | |
445 | * @param fence Timeline fence. | |
446 | * @param uk_fence UK fence. | |
447 | */ | |
448 | void mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence *fence, _mali_uk_fence_t *uk_fence); | |
449 | ||
450 | #define MALI_TIMELINE_DEBUG_FUNCTIONS | |
451 | #if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) | |
452 | ||
453 | /** | |
454 | * Tracker state. Used for debug printing. | |
455 | */ | |
456 | typedef enum mali_timeline_tracker_state { | |
457 | MALI_TIMELINE_TS_INIT = 0, | |
458 | MALI_TIMELINE_TS_WAITING = 1, | |
459 | MALI_TIMELINE_TS_ACTIVE = 2, | |
460 | MALI_TIMELINE_TS_FINISH = 3, | |
461 | } mali_timeline_tracker_state; | |
462 | ||
463 | /** | |
464 | * Get tracker state. | |
465 | * | |
466 | * @param tracker Tracker to check. | |
467 | * @return State of tracker. | |
468 | */ | |
469 | mali_timeline_tracker_state mali_timeline_debug_get_tracker_state(struct mali_timeline_tracker *tracker); | |
470 | ||
471 | /** | |
472 | * Print debug information about tracker. | |
473 | * | |
474 | * @param tracker Tracker to print. | |
475 | */ | |
476 | void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker); | |
477 | ||
478 | /** | |
479 | * Print debug information about timeline. | |
480 | * | |
481 | * @param timeline Timeline to print. | |
482 | */ | |
483 | void mali_timeline_debug_print_timeline(struct mali_timeline *timeline); | |
484 | ||
485 | /** | |
486 | * Print debug information about timeline system. | |
487 | * | |
488 | * @param system Timeline system to print. | |
489 | */ | |
490 | void mali_timeline_debug_print_system(struct mali_timeline_system *system); | |
491 | ||
492 | #endif /* defined(MALI_TIMELINE_DEBUG_FUNCTIONS) */ | |
493 | ||
494 | #endif /* __MALI_TIMELINE_H__ */ |