3 * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
21 * @file mali_kbase_js.h
28 #include "mali_kbase_js_defs.h"
29 #include "mali_kbase_context.h"
30 #include "mali_kbase_defs.h"
31 #include "mali_kbase_debug.h"
33 #include "mali_kbase_js_ctx_attr.h"
36 * @addtogroup base_api
41 * @addtogroup base_kbase_api
46 * @addtogroup kbase_js Job Scheduler Internal APIs
49 * These APIs are Internal to KBase.
53 * @brief Initialize the Job Scheduler
55 * The struct kbasep_js_device_data sub-structure of \a kbdev must be zero
56 * initialized before passing to the kbasep_js_devdata_init() function. This is
57 * to give efficient error path code.
59 int kbasep_js_devdata_init(struct kbase_device
* const kbdev
);
62 * @brief Halt the Job Scheduler.
64 * It is safe to call this on \a kbdev even if it the kbasep_js_device_data
65 * sub-structure was never initialized/failed initialization, to give efficient
68 * For this to work, the struct kbasep_js_device_data sub-structure of \a kbdev must
69 * be zero initialized before passing to the kbasep_js_devdata_init()
70 * function. This is to give efficient error path code.
72 * It is a Programming Error to call this whilst there are still kbase_context
73 * structures registered with this scheduler.
76 void kbasep_js_devdata_halt(struct kbase_device
*kbdev
);
79 * @brief Terminate the Job Scheduler
81 * It is safe to call this on \a kbdev even if it the kbasep_js_device_data
82 * sub-structure was never initialized/failed initialization, to give efficient
85 * For this to work, the struct kbasep_js_device_data sub-structure of \a kbdev must
86 * be zero initialized before passing to the kbasep_js_devdata_init()
87 * function. This is to give efficient error path code.
89 * It is a Programming Error to call this whilst there are still kbase_context
90 * structures registered with this scheduler.
92 void kbasep_js_devdata_term(struct kbase_device
*kbdev
);
95 * @brief Initialize the Scheduling Component of a struct kbase_context on the Job Scheduler.
97 * This effectively registers a struct kbase_context with a Job Scheduler.
99 * It does not register any jobs owned by the struct kbase_context with the scheduler.
100 * Those must be separately registered by kbasep_js_add_job().
102 * The struct kbase_context must be zero intitialized before passing to the
103 * kbase_js_init() function. This is to give efficient error path code.
105 int kbasep_js_kctx_init(struct kbase_context
* const kctx
);
108 * @brief Terminate the Scheduling Component of a struct kbase_context on the Job Scheduler
110 * This effectively de-registers a struct kbase_context from its Job Scheduler
112 * It is safe to call this on a struct kbase_context that has never had or failed
113 * initialization of its jctx.sched_info member, to give efficient error-path
116 * For this to work, the struct kbase_context must be zero intitialized before passing
117 * to the kbase_js_init() function.
119 * It is a Programming Error to call this whilst there are still jobs
120 * registered with this context.
122 void kbasep_js_kctx_term(struct kbase_context
*kctx
);
125 * @brief Add a job chain to the Job Scheduler, and take necessary actions to
126 * schedule the context/run the job.
128 * This atomically does the following:
129 * - Update the numbers of jobs information
130 * - Add the job to the run pool if necessary (part of init_job)
132 * Once this is done, then an appropriate action is taken:
133 * - If the ctx is scheduled, it attempts to start the next job (which might be
135 * - Otherwise, and if this is the first job on the context, it enqueues it on
138 * The Policy's Queue can be updated by this in the following ways:
139 * - In the above case that this is the first job on the context
140 * - If the context is high priority and the context is not scheduled, then it
141 * could cause the Policy to schedule out a low-priority context, allowing
142 * this context to be scheduled in.
144 * If the context is already scheduled on the RunPool, then adding a job to it
145 * is guarenteed not to update the Policy Queue. And so, the caller is
146 * guarenteed to not need to try scheduling a context from the Run Pool - it
147 * can safely assert that the result is false.
149 * It is a programming error to have more than U32_MAX jobs in flight at a time.
151 * The following locking conditions are made on the caller:
152 * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex.
153 * - it must \em not hold hwaccess_lock (as this will be obtained internally)
154 * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be
155 * obtained internally)
156 * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally).
158 * @return true indicates that the Policy Queue was updated, and so the
159 * caller will need to try scheduling a context onto the Run Pool.
160 * @return false indicates that no updates were made to the Policy Queue,
161 * so no further action is required from the caller. This is \b always returned
162 * when the context is currently scheduled.
164 bool kbasep_js_add_job(struct kbase_context
*kctx
, struct kbase_jd_atom
*atom
);
167 * @brief Remove a job chain from the Job Scheduler, except for its 'retained state'.
169 * Completely removing a job requires several calls:
170 * - kbasep_js_copy_atom_retained_state(), to capture the 'retained state' of
172 * - kbasep_js_remove_job(), to partially remove the atom from the Job Scheduler
173 * - kbasep_js_runpool_release_ctx_and_katom_retained_state(), to release the
174 * remaining state held as part of the job having been run.
176 * In the common case of atoms completing normally, this set of actions is more optimal for spinlock purposes than having kbasep_js_remove_job() handle all of the actions.
178 * In the case of cancelling atoms, it is easier to call kbasep_js_remove_cancelled_job(), which handles all the necessary actions.
180 * It is a programming error to call this when:
181 * - \a atom is not a job belonging to kctx.
182 * - \a atom has already been removed from the Job Scheduler.
183 * - \a atom is still in the runpool
185 * Do not use this for removing jobs being killed by kbase_jd_cancel() - use
186 * kbasep_js_remove_cancelled_job() instead.
188 * The following locking conditions are made on the caller:
189 * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex.
192 void kbasep_js_remove_job(struct kbase_device
*kbdev
, struct kbase_context
*kctx
, struct kbase_jd_atom
*atom
);
195 * @brief Completely remove a job chain from the Job Scheduler, in the case
196 * where the job chain was cancelled.
198 * This is a variant of kbasep_js_remove_job() that takes care of removing all
199 * of the retained state too. This is generally useful for cancelled atoms,
200 * which need not be handled in an optimal way.
202 * It is a programming error to call this when:
203 * - \a atom is not a job belonging to kctx.
204 * - \a atom has already been removed from the Job Scheduler.
205 * - \a atom is still in the runpool:
206 * - it is not being killed with kbasep_jd_cancel()
208 * The following locking conditions are made on the caller:
209 * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex.
210 * - it must \em not hold the hwaccess_lock, (as this will be obtained
212 * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this could be
213 * obtained internally)
215 * @return true indicates that ctx attributes have changed and the caller
216 * should call kbase_js_sched_all() to try to run more jobs
217 * @return false otherwise
219 bool kbasep_js_remove_cancelled_job(struct kbase_device
*kbdev
,
220 struct kbase_context
*kctx
,
221 struct kbase_jd_atom
*katom
);
224 * @brief Refcount a context as being busy, preventing it from being scheduled
227 * @note This function can safely be called from IRQ context.
229 * The following locking conditions are made on the caller:
230 * - it must \em not hold the hwaccess_lock, because it will be used internally.
232 * @return value != false if the retain succeeded, and the context will not be scheduled out.
233 * @return false if the retain failed (because the context is being/has been scheduled out).
235 bool kbasep_js_runpool_retain_ctx(struct kbase_device
*kbdev
, struct kbase_context
*kctx
);
238 * @brief Refcount a context as being busy, preventing it from being scheduled
241 * @note This function can safely be called from IRQ context.
243 * The following locks must be held by the caller:
246 * @return value != false if the retain succeeded, and the context will not be scheduled out.
247 * @return false if the retain failed (because the context is being/has been scheduled out).
249 bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device
*kbdev
, struct kbase_context
*kctx
);
252 * @brief Lookup a context in the Run Pool based upon its current address space
253 * and ensure that is stays scheduled in.
255 * The context is refcounted as being busy to prevent it from scheduling
256 * out. It must be released with kbasep_js_runpool_release_ctx() when it is no
257 * longer required to stay scheduled in.
259 * @note This function can safely be called from IRQ context.
261 * The following locking conditions are made on the caller:
262 * - it must \em not hold the hwaccess_lock, because it will be used internally.
263 * If the hwaccess_lock is already held, then the caller should use
264 * kbasep_js_runpool_lookup_ctx_nolock() instead.
266 * @return a valid struct kbase_context on success, which has been refcounted as being busy.
267 * @return NULL on failure, indicating that no context was found in \a as_nr
269 struct kbase_context
*kbasep_js_runpool_lookup_ctx(struct kbase_device
*kbdev
, int as_nr
);
272 * kbasep_js_runpool_lookup_ctx_nolock - Lookup a context in the Run Pool based
273 * upon its current address space and ensure that is stays scheduled in.
274 * @kbdev: Device pointer
275 * @as_nr: Address space to lookup
277 * The context is refcounted as being busy to prevent it from scheduling
278 * out. It must be released with kbasep_js_runpool_release_ctx() when it is no
279 * longer required to stay scheduled in.
281 * Note: This function can safely be called from IRQ context.
283 * The following locking conditions are made on the caller:
284 * - it must the hold the hwaccess_lock
286 * Return: a valid struct kbase_context on success, which has been refcounted as
288 * NULL on failure, indicating that no context was found in \a as_nr
290 struct kbase_context
*kbasep_js_runpool_lookup_ctx_nolock(
291 struct kbase_device
*kbdev
, int as_nr
);
294 * @brief Handling the requeuing/killing of a context that was evicted from the
295 * policy queue or runpool.
297 * This should be used whenever handing off a context that has been evicted
298 * from the policy queue or the runpool:
299 * - If the context is not dying and has jobs, it gets re-added to the policy
301 * - Otherwise, it is not added
303 * In addition, if the context is dying the jobs are killed asynchronously.
305 * In all cases, the Power Manager active reference is released
306 * (kbase_pm_context_idle()) whenever the has_pm_ref parameter is true. \a
307 * has_pm_ref must be set to false whenever the context was not previously in
308 * the runpool and does not hold a Power Manager active refcount. Note that
309 * contexts in a rollback of kbasep_js_try_schedule_head_ctx() might have an
310 * active refcount even though they weren't in the runpool.
312 * The following locking conditions are made on the caller:
313 * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex.
314 * - it must \em not hold kbasep_jd_device_data::queue_mutex (as this will be
315 * obtained internally)
317 void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device
*kbdev
, struct kbase_context
*kctx
, bool has_pm_ref
);
320 * @brief Release a refcount of a context being busy, allowing it to be
323 * When the refcount reaches zero and the context \em might be scheduled out
324 * (depending on whether the Scheudling Policy has deemed it so, or if it has run
327 * If the context does get scheduled out, then The following actions will be
328 * taken as part of deschduling a context:
329 * - For the context being descheduled:
330 * - If the context is in the processing of dying (all the jobs are being
331 * removed from it), then descheduling also kills off any jobs remaining in the
333 * - If the context is not dying, and any jobs remain after descheduling the
334 * context then it is re-enqueued to the Policy's Queue.
335 * - Otherwise, the context is still known to the scheduler, but remains absent
336 * from the Policy Queue until a job is next added to it.
337 * - In all descheduling cases, the Power Manager active reference (obtained
338 * during kbasep_js_try_schedule_head_ctx()) is released (kbase_pm_context_idle()).
340 * Whilst the context is being descheduled, this also handles actions that
341 * cause more atoms to be run:
342 * - Attempt submitting atoms when the Context Attributes on the Runpool have
343 * changed. This is because the context being scheduled out could mean that
344 * there are more opportunities to run atoms.
345 * - Attempt submitting to a slot that was previously blocked due to affinity
346 * restrictions. This is usually only necessary when releasing a context
347 * happens as part of completing a previous job, but is harmless nonetheless.
348 * - Attempt scheduling in a new context (if one is available), and if necessary,
349 * running a job from that new context.
351 * Unlike retaining a context in the runpool, this function \b cannot be called
354 * It is a programming error to call this on a \a kctx that is not currently
355 * scheduled, or that already has a zero refcount.
357 * The following locking conditions are made on the caller:
358 * - it must \em not hold the hwaccess_lock, because it will be used internally.
359 * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex.
360 * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be
361 * obtained internally)
362 * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be
363 * obtained internally)
364 * - it must \em not hold kbasep_jd_device_data::queue_mutex (as this will be
365 * obtained internally)
368 void kbasep_js_runpool_release_ctx(struct kbase_device
*kbdev
, struct kbase_context
*kctx
);
371 * @brief Variant of kbasep_js_runpool_release_ctx() that handles additional
372 * actions from completing an atom.
374 * This is usually called as part of completing an atom and releasing the
375 * refcount on the context held by the atom.
377 * Therefore, the extra actions carried out are part of handling actions queued
378 * on a completed atom, namely:
379 * - Releasing the atom's context attributes
380 * - Retrying the submission on a particular slot, because we couldn't submit
381 * on that slot from an IRQ handler.
383 * The locking conditions of this function are the same as those for
384 * kbasep_js_runpool_release_ctx()
386 void kbasep_js_runpool_release_ctx_and_katom_retained_state(struct kbase_device
*kbdev
, struct kbase_context
*kctx
, struct kbasep_js_atom_retained_state
*katom_retained_state
);
389 * @brief Variant of kbase_js_runpool_release_ctx() that assumes that
390 * kbasep_js_device_data::runpool_mutex and
391 * kbasep_js_kctx_info::ctx::jsctx_mutex are held by the caller, and does not
392 * attempt to schedule new contexts.
394 void kbasep_js_runpool_release_ctx_nolock(struct kbase_device
*kbdev
,
395 struct kbase_context
*kctx
);
398 * @brief Schedule in a privileged context
400 * This schedules a context in regardless of the context priority.
401 * If the runpool is full, a context will be forced out of the runpool and the function will wait
402 * for the new context to be scheduled in.
403 * The context will be kept scheduled in (and the corresponding address space reserved) until
404 * kbasep_js_release_privileged_ctx is called).
406 * The following locking conditions are made on the caller:
407 * - it must \em not hold the hwaccess_lock, because it will be used internally.
408 * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be
409 * obtained internally)
410 * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be
411 * obtained internally)
412 * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally).
413 * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex, because it will
414 * be used internally.
417 void kbasep_js_schedule_privileged_ctx(struct kbase_device
*kbdev
, struct kbase_context
*kctx
);
420 * @brief Release a privileged context, allowing it to be scheduled out.
422 * See kbasep_js_runpool_release_ctx for potential side effects.
424 * The following locking conditions are made on the caller:
425 * - it must \em not hold the hwaccess_lock, because it will be used internally.
426 * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex.
427 * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be
428 * obtained internally)
429 * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be
430 * obtained internally)
433 void kbasep_js_release_privileged_ctx(struct kbase_device
*kbdev
, struct kbase_context
*kctx
);
436 * @brief Try to submit the next job on each slot
438 * The following locks may be used:
439 * - kbasep_js_device_data::runpool_mutex
442 void kbase_js_try_run_jobs(struct kbase_device
*kbdev
);
445 * @brief Suspend the job scheduler during a Power Management Suspend event.
447 * Causes all contexts to be removed from the runpool, and prevents any
448 * contexts from (re)entering the runpool.
450 * This does not handle suspending the one privileged context: the caller must
451 * instead do this by by suspending the GPU HW Counter Instrumentation.
453 * This will eventually cause all Power Management active references held by
454 * contexts on the runpool to be released, without running any more atoms.
456 * The caller must then wait for all Power Mangement active refcount to become
457 * zero before completing the suspend.
459 * The emptying mechanism may take some time to complete, since it can wait for
460 * jobs to complete naturally instead of forcing them to end quickly. However,
461 * this is bounded by the Job Scheduler's Job Timeouts. Hence, this
462 * function is guaranteed to complete in a finite time.
464 void kbasep_js_suspend(struct kbase_device
*kbdev
);
467 * @brief Resume the Job Scheduler after a Power Management Resume event.
469 * This restores the actions from kbasep_js_suspend():
470 * - Schedules contexts back into the runpool
471 * - Resumes running atoms on the GPU
473 void kbasep_js_resume(struct kbase_device
*kbdev
);
476 * @brief Submit an atom to the job scheduler.
478 * The atom is enqueued on the context's ringbuffer. The caller must have
479 * ensured that all dependencies can be represented in the ringbuffer.
481 * Caller must hold jctx->lock
483 * @param[in] kctx Context pointer
484 * @param[in] atom Pointer to the atom to submit
486 * @return Whether the context requires to be enqueued. */
487 bool kbase_js_dep_resolved_submit(struct kbase_context
*kctx
,
488 struct kbase_jd_atom
*katom
);
491 * jsctx_ll_flush_to_rb() - Pushes atoms from the linked list to ringbuffer.
492 * @kctx: Context Pointer
493 * @prio: Priority (specifies the queue together with js).
494 * @js: Job slot (specifies the queue together with prio).
496 * Pushes all possible atoms from the linked list to the ringbuffer.
497 * Number of atoms are limited to free space in the ringbuffer and
498 * number of available atoms in the linked list.
501 void jsctx_ll_flush_to_rb(struct kbase_context
*kctx
, int prio
, int js
);
503 * @brief Pull an atom from a context in the job scheduler for execution.
505 * The atom will not be removed from the ringbuffer at this stage.
507 * The HW access lock must be held when calling this function.
509 * @param[in] kctx Context to pull from
510 * @param[in] js Job slot to pull from
511 * @return Pointer to an atom, or NULL if there are no atoms for this
512 * slot that can be currently run.
514 struct kbase_jd_atom
*kbase_js_pull(struct kbase_context
*kctx
, int js
);
517 * @brief Return an atom to the job scheduler ringbuffer.
519 * An atom is 'unpulled' if execution is stopped but intended to be returned to
520 * later. The most common reason for this is that the atom has been
523 * Note that if multiple atoms are to be 'unpulled', they must be returned in
524 * the reverse order to which they were originally pulled. It is a programming
525 * error to return atoms in any other order.
527 * The HW access lock must be held when calling this function.
529 * @param[in] kctx Context pointer
530 * @param[in] atom Pointer to the atom to unpull
532 void kbase_js_unpull(struct kbase_context
*kctx
, struct kbase_jd_atom
*katom
);
535 * @brief Complete an atom from jd_done_worker(), removing it from the job
536 * scheduler ringbuffer.
538 * If the atom failed then all dependee atoms marked for failure propagation
541 * @param[in] kctx Context pointer
542 * @param[in] katom Pointer to the atom to complete
543 * @return true if the context is now idle (no jobs pulled)
546 bool kbase_js_complete_atom_wq(struct kbase_context
*kctx
,
547 struct kbase_jd_atom
*katom
);
550 * @brief Complete an atom.
552 * Most of the work required to complete an atom will be performed by
555 * The HW access lock must be held when calling this function.
557 * @param[in] katom Pointer to the atom to complete
558 * @param[in] end_timestamp The time that the atom completed (may be NULL)
560 * Return: Atom that has now been unblocked and can now be run, or NULL if none
562 struct kbase_jd_atom
*kbase_js_complete_atom(struct kbase_jd_atom
*katom
,
563 ktime_t
*end_timestamp
);
566 * @brief Submit atoms from all available contexts.
568 * This will attempt to submit as many jobs as possible to the provided job
569 * slots. It will exit when either all job slots are full, or all contexts have
572 * @param[in] kbdev Device pointer
573 * @param[in] js_mask Mask of job slots to submit to
575 void kbase_js_sched(struct kbase_device
*kbdev
, int js_mask
);
578 * kbase_jd_zap_context - Attempt to deschedule a context that is being
580 * @kctx: Context pointer
582 * This will attempt to remove a context from any internal job scheduler queues
583 * and perform any other actions to ensure a context will not be submitted
586 * If the context is currently scheduled, then the caller must wait for all
587 * pending jobs to complete before taking any further action.
589 void kbase_js_zap_context(struct kbase_context
*kctx
);
592 * @brief Validate an atom
594 * This will determine whether the atom can be scheduled onto the GPU. Atoms
595 * with invalid combinations of core requirements will be rejected.
597 * @param[in] kbdev Device pointer
598 * @param[in] katom Atom to validate
599 * @return true if atom is valid
602 bool kbase_js_is_atom_valid(struct kbase_device
*kbdev
,
603 struct kbase_jd_atom
*katom
);
606 * kbase_js_set_timeouts - update all JS timeouts with user specified data
607 * @kbdev: Device pointer
609 * Timeouts are specified through the 'js_timeouts' sysfs file. If a timeout is
610 * set to a positive number then that becomes the new value used, if a timeout
611 * is negative then the default is set.
613 void kbase_js_set_timeouts(struct kbase_device
*kbdev
);
620 * @brief Check that a context is allowed to submit jobs on this policy
622 * The purpose of this abstraction is to hide the underlying data size, and wrap up
623 * the long repeated line of code.
625 * As with any bool, never test the return value with true.
627 * The caller must hold hwaccess_lock.
629 static inline bool kbasep_js_is_submit_allowed(struct kbasep_js_device_data
*js_devdata
, struct kbase_context
*kctx
)
633 /* Ensure context really is scheduled in */
634 KBASE_DEBUG_ASSERT(kctx
->as_nr
!= KBASEP_AS_NR_INVALID
);
635 KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx
, KCTX_SCHEDULED
));
637 test_bit
= (u16
) (1u << kctx
->as_nr
);
639 return (bool) (js_devdata
->runpool_irq
.submit_allowed
& test_bit
);
643 * @brief Allow a context to submit jobs on this policy
645 * The purpose of this abstraction is to hide the underlying data size, and wrap up
646 * the long repeated line of code.
648 * The caller must hold hwaccess_lock.
650 static inline void kbasep_js_set_submit_allowed(struct kbasep_js_device_data
*js_devdata
, struct kbase_context
*kctx
)
654 /* Ensure context really is scheduled in */
655 KBASE_DEBUG_ASSERT(kctx
->as_nr
!= KBASEP_AS_NR_INVALID
);
656 KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx
, KCTX_SCHEDULED
));
658 set_bit
= (u16
) (1u << kctx
->as_nr
);
660 dev_dbg(kctx
->kbdev
->dev
, "JS: Setting Submit Allowed on %p (as=%d)", kctx
, kctx
->as_nr
);
662 js_devdata
->runpool_irq
.submit_allowed
|= set_bit
;
666 * @brief Prevent a context from submitting more jobs on this policy
668 * The purpose of this abstraction is to hide the underlying data size, and wrap up
669 * the long repeated line of code.
671 * The caller must hold hwaccess_lock.
673 static inline void kbasep_js_clear_submit_allowed(struct kbasep_js_device_data
*js_devdata
, struct kbase_context
*kctx
)
678 /* Ensure context really is scheduled in */
679 KBASE_DEBUG_ASSERT(kctx
->as_nr
!= KBASEP_AS_NR_INVALID
);
680 KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx
, KCTX_SCHEDULED
));
682 clear_bit
= (u16
) (1u << kctx
->as_nr
);
683 clear_mask
= ~clear_bit
;
685 dev_dbg(kctx
->kbdev
->dev
, "JS: Clearing Submit Allowed on %p (as=%d)", kctx
, kctx
->as_nr
);
687 js_devdata
->runpool_irq
.submit_allowed
&= clear_mask
;
691 * @brief Manage the 'retry_submit_on_slot' part of a kbase_jd_atom
693 static inline void kbasep_js_clear_job_retry_submit(struct kbase_jd_atom
*atom
)
695 atom
->retry_submit_on_slot
= KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID
;
699 * Mark a slot as requiring resubmission by carrying that information on a
702 * @note This can ASSERT in debug builds if the submit slot has been set to
703 * something other than the current value for @a js. This is because you might
704 * be unintentionally stopping more jobs being submitted on the old submit
705 * slot, and that might cause a scheduling-hang.
707 * @note If you can guarantee that the atoms for the original slot will be
708 * submitted on some other slot, then call kbasep_js_clear_job_retry_submit()
709 * first to silence the ASSERT.
711 static inline void kbasep_js_set_job_retry_submit_slot(struct kbase_jd_atom
*atom
, int js
)
713 KBASE_DEBUG_ASSERT(0 <= js
&& js
<= BASE_JM_MAX_NR_SLOTS
);
714 KBASE_DEBUG_ASSERT((atom
->retry_submit_on_slot
==
715 KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID
)
716 || (atom
->retry_submit_on_slot
== js
));
718 atom
->retry_submit_on_slot
= js
;
722 * Create an initial 'invalid' atom retained state, that requires no
723 * atom-related work to be done on releasing with
724 * kbasep_js_runpool_release_ctx_and_katom_retained_state()
726 static inline void kbasep_js_atom_retained_state_init_invalid(struct kbasep_js_atom_retained_state
*retained_state
)
728 retained_state
->event_code
= BASE_JD_EVENT_NOT_STARTED
;
729 retained_state
->core_req
= KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID
;
730 retained_state
->retry_submit_on_slot
= KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID
;
734 * Copy atom state that can be made available after jd_done_nolock() is called
737 static inline void kbasep_js_atom_retained_state_copy(struct kbasep_js_atom_retained_state
*retained_state
, const struct kbase_jd_atom
*katom
)
739 retained_state
->event_code
= katom
->event_code
;
740 retained_state
->core_req
= katom
->core_req
;
741 retained_state
->retry_submit_on_slot
= katom
->retry_submit_on_slot
;
742 retained_state
->sched_priority
= katom
->sched_priority
;
743 retained_state
->device_nr
= katom
->device_nr
;
747 * @brief Determine whether an atom has finished (given its retained state),
748 * and so should be given back to userspace/removed from the system.
750 * Reasons for an atom not finishing include:
751 * - Being soft-stopped (and so, the atom should be resubmitted sometime later)
753 * @param[in] katom_retained_state the retained state of the atom to check
754 * @return false if the atom has not finished
755 * @return !=false if the atom has finished
757 static inline bool kbasep_js_has_atom_finished(const struct kbasep_js_atom_retained_state
*katom_retained_state
)
759 return (bool) (katom_retained_state
->event_code
!= BASE_JD_EVENT_STOPPED
&& katom_retained_state
->event_code
!= BASE_JD_EVENT_REMOVED_FROM_NEXT
);
763 * @brief Determine whether a struct kbasep_js_atom_retained_state is valid
765 * An invalid struct kbasep_js_atom_retained_state is allowed, and indicates that the
766 * code should just ignore it.
768 * @param[in] katom_retained_state the atom's retained state to check
769 * @return false if the retained state is invalid, and can be ignored
770 * @return !=false if the retained state is valid
772 static inline bool kbasep_js_atom_retained_state_is_valid(const struct kbasep_js_atom_retained_state
*katom_retained_state
)
774 return (bool) (katom_retained_state
->core_req
!= KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID
);
777 static inline bool kbasep_js_get_atom_retry_submit_slot(const struct kbasep_js_atom_retained_state
*katom_retained_state
, int *res
)
779 int js
= katom_retained_state
->retry_submit_on_slot
;
782 return (bool) (js
>= 0);
785 #if KBASE_DEBUG_DISABLE_ASSERTS == 0
787 * Debug Check the refcount of a context. Only use within ASSERTs
789 * Obtains hwaccess_lock
791 * @return negative value if the context is not scheduled in
792 * @return current refcount of the context if it is scheduled in. The refcount
793 * is not guarenteed to be kept constant.
795 static inline int kbasep_js_debug_check_ctx_refcount(struct kbase_device
*kbdev
, struct kbase_context
*kctx
)
798 struct kbasep_js_device_data
*js_devdata
;
802 KBASE_DEBUG_ASSERT(kbdev
!= NULL
);
803 KBASE_DEBUG_ASSERT(kctx
!= NULL
);
804 js_devdata
= &kbdev
->js_data
;
806 spin_lock_irqsave(&kbdev
->hwaccess_lock
, flags
);
808 if (as_nr
!= KBASEP_AS_NR_INVALID
)
809 result
= js_devdata
->runpool_irq
.per_as_data
[as_nr
].as_busy_refcount
;
811 spin_unlock_irqrestore(&kbdev
->hwaccess_lock
, flags
);
815 #endif /* KBASE_DEBUG_DISABLE_ASSERTS == 0 */
818 * @brief Variant of kbasep_js_runpool_lookup_ctx() that can be used when the
819 * context is guarenteed to be already previously retained.
821 * It is a programming error to supply the \a as_nr of a context that has not
822 * been previously retained/has a busy refcount of zero. The only exception is
823 * when there is no ctx in \a as_nr (NULL returned).
825 * The following locking conditions are made on the caller:
826 * - it must \em not hold the hwaccess_lock, because it will be used internally.
828 * @return a valid struct kbase_context on success, with a refcount that is guarenteed
829 * to be non-zero and unmodified by this function.
830 * @return NULL on failure, indicating that no context was found in \a as_nr
832 static inline struct kbase_context
*kbasep_js_runpool_lookup_ctx_noretain(struct kbase_device
*kbdev
, int as_nr
)
835 struct kbasep_js_device_data
*js_devdata
;
836 struct kbase_context
*found_kctx
;
837 struct kbasep_js_per_as_data
*js_per_as_data
;
839 KBASE_DEBUG_ASSERT(kbdev
!= NULL
);
840 KBASE_DEBUG_ASSERT(0 <= as_nr
&& as_nr
< BASE_MAX_NR_AS
);
841 js_devdata
= &kbdev
->js_data
;
842 js_per_as_data
= &js_devdata
->runpool_irq
.per_as_data
[as_nr
];
844 spin_lock_irqsave(&kbdev
->hwaccess_lock
, flags
);
846 found_kctx
= js_per_as_data
->kctx
;
847 KBASE_DEBUG_ASSERT(found_kctx
== NULL
|| js_per_as_data
->as_busy_refcount
> 0);
849 spin_unlock_irqrestore(&kbdev
->hwaccess_lock
, flags
);
855 * This will provide a conversion from time (us) to ticks of the gpu clock
856 * based on the minimum available gpu frequency.
857 * This is usually good to compute best/worst case (where the use of current
858 * frequency is not valid due to DVFS).
859 * e.g.: when you need the number of cycles to guarantee you won't wait for
860 * longer than 'us' time (you might have a shorter wait).
862 static inline u32
kbasep_js_convert_us_to_gpu_ticks_min_freq(struct kbase_device
*kbdev
, u32 us
)
864 u32 gpu_freq
= kbdev
->gpu_props
.props
.core_props
.gpu_freq_khz_min
;
866 KBASE_DEBUG_ASSERT(0 != gpu_freq
);
867 return us
* (gpu_freq
/ 1000);
871 * This will provide a conversion from time (us) to ticks of the gpu clock
872 * based on the maximum available gpu frequency.
873 * This is usually good to compute best/worst case (where the use of current
874 * frequency is not valid due to DVFS).
875 * e.g.: When you need the number of cycles to guarantee you'll wait at least
876 * 'us' amount of time (but you might wait longer).
878 static inline u32
kbasep_js_convert_us_to_gpu_ticks_max_freq(struct kbase_device
*kbdev
, u32 us
)
880 u32 gpu_freq
= kbdev
->gpu_props
.props
.core_props
.gpu_freq_khz_max
;
882 KBASE_DEBUG_ASSERT(0 != gpu_freq
);
883 return us
* (u32
) (gpu_freq
/ 1000);
887 * This will provide a conversion from ticks of the gpu clock to time (us)
888 * based on the minimum available gpu frequency.
889 * This is usually good to compute best/worst case (where the use of current
890 * frequency is not valid due to DVFS).
891 * e.g.: When you need to know the worst-case wait that 'ticks' cycles will
892 * take (you guarantee that you won't wait any longer than this, but it may
895 static inline u32
kbasep_js_convert_gpu_ticks_to_us_min_freq(struct kbase_device
*kbdev
, u32 ticks
)
897 u32 gpu_freq
= kbdev
->gpu_props
.props
.core_props
.gpu_freq_khz_min
;
899 KBASE_DEBUG_ASSERT(0 != gpu_freq
);
900 return ticks
/ gpu_freq
* 1000;
904 * This will provide a conversion from ticks of the gpu clock to time (us)
905 * based on the maximum available gpu frequency.
906 * This is usually good to compute best/worst case (where the use of current
907 * frequency is not valid due to DVFS).
908 * e.g.: When you need to know the best-case wait for 'tick' cycles (you
909 * guarantee to be waiting for at least this long, but it may be longer).
911 static inline u32
kbasep_js_convert_gpu_ticks_to_us_max_freq(struct kbase_device
*kbdev
, u32 ticks
)
913 u32 gpu_freq
= kbdev
->gpu_props
.props
.core_props
.gpu_freq_khz_max
;
915 KBASE_DEBUG_ASSERT(0 != gpu_freq
);
916 return ticks
/ gpu_freq
* 1000;
920 * The following locking conditions are made on the caller:
921 * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex.
922 * - The caller must hold the kbasep_js_device_data::runpool_mutex
924 static inline void kbase_js_runpool_inc_context_count(
925 struct kbase_device
*kbdev
,
926 struct kbase_context
*kctx
)
928 struct kbasep_js_device_data
*js_devdata
;
929 struct kbasep_js_kctx_info
*js_kctx_info
;
931 KBASE_DEBUG_ASSERT(kbdev
!= NULL
);
932 KBASE_DEBUG_ASSERT(kctx
!= NULL
);
934 js_devdata
= &kbdev
->js_data
;
935 js_kctx_info
= &kctx
->jctx
.sched_info
;
937 lockdep_assert_held(&js_kctx_info
->ctx
.jsctx_mutex
);
938 lockdep_assert_held(&js_devdata
->runpool_mutex
);
940 /* Track total contexts */
941 KBASE_DEBUG_ASSERT(js_devdata
->nr_all_contexts_running
< S8_MAX
);
942 ++(js_devdata
->nr_all_contexts_running
);
944 if (!kbase_ctx_flag(kctx
, KCTX_SUBMIT_DISABLED
)) {
945 /* Track contexts that can submit jobs */
946 KBASE_DEBUG_ASSERT(js_devdata
->nr_user_contexts_running
<
948 ++(js_devdata
->nr_user_contexts_running
);
953 * The following locking conditions are made on the caller:
954 * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex.
955 * - The caller must hold the kbasep_js_device_data::runpool_mutex
957 static inline void kbase_js_runpool_dec_context_count(
958 struct kbase_device
*kbdev
,
959 struct kbase_context
*kctx
)
961 struct kbasep_js_device_data
*js_devdata
;
962 struct kbasep_js_kctx_info
*js_kctx_info
;
964 KBASE_DEBUG_ASSERT(kbdev
!= NULL
);
965 KBASE_DEBUG_ASSERT(kctx
!= NULL
);
967 js_devdata
= &kbdev
->js_data
;
968 js_kctx_info
= &kctx
->jctx
.sched_info
;
970 lockdep_assert_held(&js_kctx_info
->ctx
.jsctx_mutex
);
971 lockdep_assert_held(&js_devdata
->runpool_mutex
);
973 /* Track total contexts */
974 --(js_devdata
->nr_all_contexts_running
);
975 KBASE_DEBUG_ASSERT(js_devdata
->nr_all_contexts_running
>= 0);
977 if (!kbase_ctx_flag(kctx
, KCTX_SUBMIT_DISABLED
)) {
978 /* Track contexts that can submit jobs */
979 --(js_devdata
->nr_user_contexts_running
);
980 KBASE_DEBUG_ASSERT(js_devdata
->nr_user_contexts_running
>= 0);
986 * @brief Submit atoms from all available contexts to all job slots.
988 * This will attempt to submit as many jobs as possible. It will exit when
989 * either all job slots are full, or all contexts have been used.
991 * @param[in] kbdev Device pointer
993 static inline void kbase_js_sched_all(struct kbase_device
*kbdev
)
995 kbase_js_sched(kbdev
, (1 << kbdev
->gpu_props
.num_job_slots
) - 1);
999 kbasep_js_atom_priority_to_relative
[BASE_JD_NR_PRIO_LEVELS
];
1001 extern const base_jd_prio
1002 kbasep_js_relative_priority_to_atom
[KBASE_JS_ATOM_SCHED_PRIO_COUNT
];
1005 * kbasep_js_atom_prio_to_sched_prio(): - Convert atom priority (base_jd_prio)
1006 * to relative ordering
1007 * @atom_prio: Priority ID to translate.
1009 * Atom priority values for @ref base_jd_prio cannot be compared directly to
1010 * find out which are higher or lower.
1012 * This function will convert base_jd_prio values for successively lower
1013 * priorities into a monotonically increasing sequence. That is, the lower the
1014 * base_jd_prio priority, the higher the value produced by this function. This
1015 * is in accordance with how the rest of the kernel treates priority.
1017 * The mapping is 1:1 and the size of the valid input range is the same as the
1018 * size of the valid output range, i.e.
1019 * KBASE_JS_ATOM_SCHED_PRIO_COUNT == BASE_JD_NR_PRIO_LEVELS
1021 * Note This must be kept in sync with BASE_JD_PRIO_<...> definitions
1023 * Return: On success: a value in the inclusive range
1024 * 0..KBASE_JS_ATOM_SCHED_PRIO_COUNT-1. On failure:
1025 * KBASE_JS_ATOM_SCHED_PRIO_INVALID
1027 static inline int kbasep_js_atom_prio_to_sched_prio(base_jd_prio atom_prio
)
1029 if (atom_prio
>= BASE_JD_NR_PRIO_LEVELS
)
1030 return KBASE_JS_ATOM_SCHED_PRIO_INVALID
;
1032 return kbasep_js_atom_priority_to_relative
[atom_prio
];
1035 static inline base_jd_prio
kbasep_js_sched_prio_to_atom_prio(int sched_prio
)
1037 unsigned int prio_idx
;
1039 KBASE_DEBUG_ASSERT(0 <= sched_prio
1040 && sched_prio
< KBASE_JS_ATOM_SCHED_PRIO_COUNT
);
1042 prio_idx
= (unsigned int)sched_prio
;
1044 return kbasep_js_relative_priority_to_atom
[prio_idx
];
1047 /** @} *//* end group kbase_js */
1048 /** @} *//* end group base_kbase_api */
1049 /** @} *//* end group base_api */
1051 #endif /* _KBASE_JS_H_ */