projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
import OT_8063_20170412 mali driver
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / misc / mediatek / gpu / mt8127 / mali / mali / common / mali_scheduler.c
CommitLineData
6fa3eb70
S		 1/*
2 * This confidential and proprietary software may be used only as
3 * authorised by a licensing agreement from ARM Limited
bdc132d7 4 * (C) COPYRIGHT 2012-2015 ARM Limited
6fa3eb70
S		 5 * ALL RIGHTS RESERVED
6 * The entire notice above must be reproduced on all authorised
7 * copies and copies may only be made to the extent permitted
8 * by a licensing agreement from ARM Limited.
9 */
10
11#include "mali_scheduler.h"
6fa3eb70
S		 12#include "mali_kernel_common.h"
13#include "mali_osk.h"
bdc132d7
S		 14#include "mali_osk_profiling.h"
15#include "mali_kernel_utilization.h"
16#include "mali_timeline.h"
17#include "mali_gp_job.h"
18#include "mali_pp_job.h"
19#include "mali_executor.h"
20#include "mali_group.h"
6fa3eb70 21
bdc132d7
S		 22#if defined(CONFIG_DMA_SHARED_BUFFER)
23#include "mali_memory_dma_buf.h"
24#endif
6fa3eb70 25
bdc132d7
S		 26#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
27#include <linux/sched.h>
28#include <trace/events/gpu.h>
29#endif
30/*
31 * ---------- static defines/constants ----------
32 */
6fa3eb70 33
bdc132d7
S		 34/*
35 * If dma_buf with map on demand is used, we defer job deletion and job queue
36 * if in atomic context, since both might sleep.
37 */
38#if defined(CONFIG_DMA_SHARED_BUFFER)
39#if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
40#define MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE 1
41#define MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE 1
42#endif
43#endif
6fa3eb70 44
bdc132d7
S		 45/*
46 * ---------- global variables (exported due to inline functions) ----------
47 */
6fa3eb70 48
bdc132d7
S		 49/* Lock protecting this module */
50_mali_osk_spinlock_irq_t *mali_scheduler_lock_obj = NULL;
6fa3eb70 51
bdc132d7
S		 52/* Queue of jobs to be executed on the GP group */
53struct mali_scheduler_job_queue job_queue_gp;
6fa3eb70 54
bdc132d7
S		 55/* Queue of PP jobs */
56struct mali_scheduler_job_queue job_queue_pp;
57
58_mali_osk_atomic_t mali_job_id_autonumber;
59_mali_osk_atomic_t mali_job_cache_order_autonumber;
60/*
61 * ---------- static variables ----------
62 */
63
64#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE)
65static _mali_osk_wq_work_t *scheduler_wq_pp_job_delete = NULL;
66static _mali_osk_spinlock_irq_t *scheduler_pp_job_delete_lock = NULL;
67static _MALI_OSK_LIST_HEAD_STATIC_INIT(scheduler_pp_job_deletion_queue);
68#endif
69
70#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
71static _mali_osk_wq_work_t *scheduler_wq_pp_job_queue = NULL;
72static _mali_osk_spinlock_irq_t *scheduler_pp_job_queue_lock = NULL;
73static _MALI_OSK_LIST_HEAD_STATIC_INIT(scheduler_pp_job_queue_list);
74#endif
75
76/*
77 * ---------- Forward declaration of static functions ----------
78 */
79
80static mali_timeline_point mali_scheduler_submit_gp_job(
81 struct mali_session_data *session, struct mali_gp_job *job);
82static mali_timeline_point mali_scheduler_submit_pp_job(
83 struct mali_session_data *session, struct mali_pp_job *job);
84
85static mali_bool mali_scheduler_queue_gp_job(struct mali_gp_job *job);
86static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job);
87
88static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job,
89 mali_bool success);
90static void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job,
91 u32 num_cores_in_virtual);
92
93#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE)
94static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job);
95static void mali_scheduler_do_pp_job_delete(void *arg);
96#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) */
97
98#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
99static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job);
100static void mali_scheduler_do_pp_job_queue(void *arg);
101#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
102
103/*
104 * ---------- Actual implementation ----------
105 */
6fa3eb70
S		 106
107_mali_osk_errcode_t mali_scheduler_initialize(void)
108{
bdc132d7
S		 109 _mali_osk_atomic_init(&mali_job_id_autonumber, 0);
110 _mali_osk_atomic_init(&mali_job_cache_order_autonumber, 0);
111
112 _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.normal_pri);
113 _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.high_pri);
114 job_queue_gp.depth = 0;
115
116 _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.normal_pri);
117 _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.high_pri);
118 job_queue_pp.depth = 0;
119
120 mali_scheduler_lock_obj = _mali_osk_spinlock_irq_init(
121 _MALI_OSK_LOCKFLAG_ORDERED,
122 _MALI_OSK_LOCK_ORDER_SCHEDULER);
123 if (NULL == mali_scheduler_lock_obj) {
124 mali_scheduler_terminate();
125 }
126
127#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE)
128 scheduler_wq_pp_job_delete = _mali_osk_wq_create_work(
129 mali_scheduler_do_pp_job_delete, NULL);
130 if (NULL == scheduler_wq_pp_job_delete) {
131 mali_scheduler_terminate();
6fa3eb70
S		 132 return _MALI_OSK_ERR_FAULT;
133 }
134
bdc132d7
S		 135 scheduler_pp_job_delete_lock = _mali_osk_spinlock_irq_init(
136 _MALI_OSK_LOCKFLAG_ORDERED,
137 _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED);
138 if (NULL == scheduler_pp_job_delete_lock) {
139 mali_scheduler_terminate();
6fa3eb70
S		 140 return _MALI_OSK_ERR_FAULT;
141 }
bdc132d7 142#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) */
6fa3eb70 143
bdc132d7
S		 144#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
145 scheduler_wq_pp_job_queue = _mali_osk_wq_create_work(
146 mali_scheduler_do_pp_job_queue, NULL);
147 if (NULL == scheduler_wq_pp_job_queue) {
148 mali_scheduler_terminate();
149 return _MALI_OSK_ERR_FAULT;
6fa3eb70
S		 150 }
151
bdc132d7
S		 152 scheduler_pp_job_queue_lock = _mali_osk_spinlock_irq_init(
153 _MALI_OSK_LOCKFLAG_ORDERED,
154 _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED);
155 if (NULL == scheduler_pp_job_queue_lock) {
156 mali_scheduler_terminate();
157 return _MALI_OSK_ERR_FAULT;
6fa3eb70 158 }
bdc132d7 159#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
6fa3eb70
S		 160
161 return _MALI_OSK_ERR_OK;
162}
163
164void mali_scheduler_terminate(void)
165{
bdc132d7
S		 166#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
167 if (NULL != scheduler_pp_job_queue_lock) {
168 _mali_osk_spinlock_irq_term(scheduler_pp_job_queue_lock);
169 scheduler_pp_job_queue_lock = NULL;
170 }
171
172 if (NULL != scheduler_wq_pp_job_queue) {
173 _mali_osk_wq_delete_work(scheduler_wq_pp_job_queue);
174 scheduler_wq_pp_job_queue = NULL;
175 }
176#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
177
178#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE)
179 if (NULL != scheduler_pp_job_delete_lock) {
180 _mali_osk_spinlock_irq_term(scheduler_pp_job_delete_lock);
181 scheduler_pp_job_delete_lock = NULL;
182 }
183
184 if (NULL != scheduler_wq_pp_job_delete) {
185 _mali_osk_wq_delete_work(scheduler_wq_pp_job_delete);
186 scheduler_wq_pp_job_delete = NULL;
187 }
188#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) */
189
190 if (NULL != mali_scheduler_lock_obj) {
191 _mali_osk_spinlock_irq_term(mali_scheduler_lock_obj);
192 mali_scheduler_lock_obj = NULL;
193 }
194
6fa3eb70
S		 195 _mali_osk_atomic_term(&mali_job_cache_order_autonumber);
196 _mali_osk_atomic_term(&mali_job_id_autonumber);
197}
198
bdc132d7 199u32 mali_scheduler_job_physical_head_count(void)
6fa3eb70 200{
bdc132d7
S		 201 /*
202 * Count how many physical sub jobs are present from the head of queue
203 * until the first virtual job is present.
204 * Early out when we have reached maximum number of PP cores (8)
205 */
206 u32 count = 0;
207 struct mali_pp_job *job;
208 struct mali_pp_job *temp;
209
210 /* Check for partially started normal pri jobs */
211 if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
212 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
213
214 job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next,
215 struct mali_pp_job, list);
216
217 MALI_DEBUG_ASSERT_POINTER(job);
218
219 if (MALI_TRUE == mali_pp_job_has_started_sub_jobs(job)) {
220 /*
221 * Remember; virtual jobs can't be queued and started
222 * at the same time, so this must be a physical job
223 */
224 count += mali_pp_job_unstarted_sub_job_count(job);
225 if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) {
226 return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS;
227 }
228 }
229 }
230
231 _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.high_pri,
232 struct mali_pp_job, list) {
233 if (MALI_FALSE == mali_pp_job_is_virtual(job)) {
234 count += mali_pp_job_unstarted_sub_job_count(job);
235 if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) {
236 return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS;
237 }
238 } else {
239 /* Came across a virtual job, so stop counting */
240 return count;
241 }
242 }
243
244 _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.normal_pri,
245 struct mali_pp_job, list) {
246 if (MALI_FALSE == mali_pp_job_is_virtual(job)) {
247 /* any partially started is already counted */
248 if (MALI_FALSE == mali_pp_job_has_started_sub_jobs(job)) {
249 count += mali_pp_job_unstarted_sub_job_count(job);
250 if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <=
251 count) {
252 return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS;
253 }
254 }
255 } else {
256 /* Came across a virtual job, so stop counting */
257 return count;
258 }
259 }
260
261 return count;
262}
263
264mali_bool mali_scheduler_job_next_is_virtual(void)
265{
266 struct mali_pp_job *job;
267
268 job = mali_scheduler_job_pp_virtual_peek();
269 if (NULL != job) {
270 MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job));
271
272 return MALI_TRUE;
273 }
274
275 return MALI_FALSE;
6fa3eb70
S		 276}
277
bdc132d7 278struct mali_gp_job *mali_scheduler_job_gp_get(void)
6fa3eb70 279{
bdc132d7
S		 280 _mali_osk_list_t *queue;
281 struct mali_gp_job *job = NULL;
282
283 MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
284 MALI_DEBUG_ASSERT(0 < job_queue_gp.depth);
285
286 if (!_mali_osk_list_empty(&job_queue_gp.high_pri)) {
287 queue = &job_queue_gp.high_pri;
288 } else {
289 queue = &job_queue_gp.normal_pri;
290 MALI_DEBUG_ASSERT(!_mali_osk_list_empty(queue));
291 }
292
293 job = _MALI_OSK_LIST_ENTRY(queue->next, struct mali_gp_job, list);
294
295 MALI_DEBUG_ASSERT_POINTER(job);
296
297 mali_gp_job_list_remove(job);
298 job_queue_gp.depth--;
299
300 return job;
6fa3eb70
S		 301}
302
bdc132d7 303struct mali_pp_job *mali_scheduler_job_pp_physical_peek(void)
6fa3eb70 304{
bdc132d7
S		 305 struct mali_pp_job *job = NULL;
306 struct mali_pp_job *tmp_job = NULL;
307
308 MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
309
310 /*
311 * For PP jobs we favour partially started jobs in normal
312 * priority queue over unstarted jobs in high priority queue
313 */
314
315 if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
316 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
317
318 tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next,
319 struct mali_pp_job, list);
320 MALI_DEBUG_ASSERT(NULL != tmp_job);
321
322 if (MALI_FALSE == mali_pp_job_is_virtual(tmp_job)) {
323 job = tmp_job;
324 }
325 }
326
327 if (NULL == job ||
328 MALI_FALSE == mali_pp_job_has_started_sub_jobs(job)) {
329 /*
330 * There isn't a partially started job in normal queue, so
331 * look in high priority queue.
332 */
333 if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) {
334 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
335
336 tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.high_pri.next,
337 struct mali_pp_job, list);
338 MALI_DEBUG_ASSERT(NULL != tmp_job);
339
340 if (MALI_FALSE == mali_pp_job_is_virtual(tmp_job)) {
341 job = tmp_job;
342 }
343 }
344 }
345
346 return job;
347}
348
349struct mali_pp_job *mali_scheduler_job_pp_virtual_peek(void)
350{
351 struct mali_pp_job *job = NULL;
352 struct mali_pp_job *tmp_job = NULL;
353
354 MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
355
356 if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) {
357 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
358
359 tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.high_pri.next,
360 struct mali_pp_job, list);
361
362 if (MALI_TRUE == mali_pp_job_is_virtual(tmp_job)) {
363 job = tmp_job;
364 }
365 }
366
367 if (NULL == job) {
368 if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
369 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
370
371 tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next,
372 struct mali_pp_job, list);
373
374 if (MALI_TRUE == mali_pp_job_is_virtual(tmp_job)) {
375 job = tmp_job;
376 }
377 }
378 }
379
380 return job;
381}
382
383struct mali_pp_job *mali_scheduler_job_pp_physical_get(u32 *sub_job)
384{
385 struct mali_pp_job *job = mali_scheduler_job_pp_physical_peek();
386
387 MALI_DEBUG_ASSERT(MALI_FALSE == mali_pp_job_is_virtual(job));
388
389 if (NULL != job) {
390 *sub_job = mali_pp_job_get_first_unstarted_sub_job(job);
391
392 mali_pp_job_mark_sub_job_started(job, *sub_job);
393 if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(job)) {
394 /* Remove from queue when last sub job has been retrieved */
395 mali_pp_job_list_remove(job);
396 }
397
398 job_queue_pp.depth--;
399
400 /*
401 * Job about to start so it is no longer be
402 * possible to discard WB
403 */
404 mali_pp_job_fb_lookup_remove(job);
405 }
406
407 return job;
408}
409
410struct mali_pp_job *mali_scheduler_job_pp_virtual_get(void)
411{
412 struct mali_pp_job *job = mali_scheduler_job_pp_virtual_peek();
413
414 MALI_DEBUG_ASSERT(MALI_TRUE == mali_pp_job_is_virtual(job));
415
416 if (NULL != job) {
417 MALI_DEBUG_ASSERT(0 ==
418 mali_pp_job_get_first_unstarted_sub_job(job));
419 MALI_DEBUG_ASSERT(1 ==
420 mali_pp_job_get_sub_job_count(job));
421
422 mali_pp_job_mark_sub_job_started(job, 0);
423
424 mali_pp_job_list_remove(job);
425
426 job_queue_pp.depth--;
427
428 /*
429 * Job about to start so it is no longer be
430 * possible to discard WB
431 */
432 mali_pp_job_fb_lookup_remove(job);
433 }
434
435 return job;
436}
437
438mali_scheduler_mask mali_scheduler_activate_gp_job(struct mali_gp_job *job)
439{
440 MALI_DEBUG_ASSERT_POINTER(job);
441
442 MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Timeline activation for job %u (0x%08X).\n",
443 mali_gp_job_get_id(job), job));
444
445 mali_scheduler_lock();
446
447 if (!mali_scheduler_queue_gp_job(job)) {
448 /* Failed to enqueue job, release job (with error) */
449
450 mali_scheduler_unlock();
451
452 mali_timeline_tracker_release(mali_gp_job_get_tracker(job));
453 mali_gp_job_signal_pp_tracker(job, MALI_FALSE);
454
455 /* This will notify user space and close the job object */
456 mali_scheduler_complete_gp_job(job, MALI_FALSE,
457 MALI_TRUE, MALI_FALSE);
458
459 return MALI_SCHEDULER_MASK_EMPTY;
460 }
461
462 mali_scheduler_unlock();
463
464 return MALI_SCHEDULER_MASK_GP;
465}
466
467mali_scheduler_mask mali_scheduler_activate_pp_job(struct mali_pp_job *job)
468{
469 MALI_DEBUG_ASSERT_POINTER(job);
470
471 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Timeline activation for job %u (0x%08X).\n",
472 mali_pp_job_get_id(job), job));
473
474 if (MALI_TRUE == mali_timeline_tracker_activation_error(
475 mali_pp_job_get_tracker(job))) {
476 MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Job %u (0x%08X) activated with error, aborting.\n",
477 mali_pp_job_get_id(job), job));
478
479 mali_scheduler_lock();
480 mali_pp_job_fb_lookup_remove(job);
481 mali_pp_job_mark_unstarted_failed(job);
482 mali_scheduler_unlock();
483
484 mali_timeline_tracker_release(mali_pp_job_get_tracker(job));
485
486 /* This will notify user space and close the job object */
487 mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, MALI_FALSE);
488
489 return MALI_SCHEDULER_MASK_EMPTY;
490 }
491
492#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
493 if (mali_pp_job_needs_dma_buf_mapping(job)) {
494 mali_scheduler_deferred_pp_job_queue(job);
495 return MALI_SCHEDULER_MASK_EMPTY;
496 }
497#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
498
499 mali_scheduler_lock();
500
501 if (!mali_scheduler_queue_pp_job(job)) {
502 /* Failed to enqueue job, release job (with error) */
503 mali_pp_job_fb_lookup_remove(job);
504 mali_pp_job_mark_unstarted_failed(job);
505 mali_scheduler_unlock();
506
507 mali_timeline_tracker_release(mali_pp_job_get_tracker(job));
508
509 /* This will notify user space and close the job object */
510 mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, MALI_FALSE);
511
512 return MALI_SCHEDULER_MASK_EMPTY;
513 }
514
515 mali_scheduler_unlock();
516 return MALI_SCHEDULER_MASK_PP;
517}
518
519void mali_scheduler_complete_gp_job(struct mali_gp_job *job,
520 mali_bool success,
521 mali_bool user_notification,
522 mali_bool dequeued)
523{
524 if (user_notification) {
525 mali_scheduler_return_gp_job_to_user(job, success);
526 }
527
528 if (dequeued) {
529 _mali_osk_pm_dev_ref_put();
530
531 if (mali_utilization_enabled()) {
532 mali_utilization_gp_end();
533 }
534 }
535
536 mali_gp_job_delete(job);
537}
538
539void mali_scheduler_complete_pp_job(struct mali_pp_job *job,
540 u32 num_cores_in_virtual,
541 mali_bool user_notification,
542 mali_bool dequeued)
543{
544 if (user_notification) {
545 mali_scheduler_return_pp_job_to_user(job,
546 num_cores_in_virtual);
547 }
548
549 if (dequeued) {
550#if defined(CONFIG_MALI_DVFS)
551 if (mali_pp_job_is_window_surface(job)) {
552 struct mali_session_data *session;
553 session = mali_pp_job_get_session(job);
554 mali_session_inc_num_window_jobs(session);
555 }
556#endif
557
558 _mali_osk_pm_dev_ref_put();
559
560 if (mali_utilization_enabled()) {
561 mali_utilization_pp_end();
562 }
563 }
564
565#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE)
566 /*
567 * The deletion of the job object (releasing sync refs etc)
568 * must be done in a different context
569 */
570 mali_scheduler_deferred_pp_job_delete(job);
571#else
572 /* no use cases need this in this configuration */
573 mali_pp_job_delete(job);
574#endif
575}
576
577void mali_scheduler_abort_session(struct mali_session_data *session)
578{
579 struct mali_gp_job *gp_job;
580 struct mali_gp_job *gp_tmp;
581 struct mali_pp_job *pp_job;
582 struct mali_pp_job *pp_tmp;
583 _MALI_OSK_LIST_HEAD_STATIC_INIT(removed_jobs_gp);
584 _MALI_OSK_LIST_HEAD_STATIC_INIT(removed_jobs_pp);
585
586 MALI_DEBUG_ASSERT_POINTER(session);
587 MALI_DEBUG_ASSERT(session->is_aborting);
588
589 MALI_DEBUG_PRINT(3, ("Mali scheduler: Aborting all queued jobs from session 0x%08X.\n",
590 session));
591
592 mali_scheduler_lock();
593
594 /* Remove from GP normal priority queue */
595 _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &job_queue_gp.normal_pri,
596 struct mali_gp_job, list) {
597 if (mali_gp_job_get_session(gp_job) == session) {
598 mali_gp_job_list_move(gp_job, &removed_jobs_gp);
599 job_queue_gp.depth--;
600 }
601 }
602
603 /* Remove from GP high priority queue */
604 _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &job_queue_gp.high_pri,
605 struct mali_gp_job, list) {
606 if (mali_gp_job_get_session(gp_job) == session) {
607 mali_gp_job_list_move(gp_job, &removed_jobs_gp);
608 job_queue_gp.depth--;
609 }
610 }
611
612 /* Remove from PP normal priority queue */
613 _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp,
614 &job_queue_pp.normal_pri,
615 struct mali_pp_job, list) {
616 if (mali_pp_job_get_session(pp_job) == session) {
617 mali_pp_job_fb_lookup_remove(pp_job);
618
619 job_queue_pp.depth -=
620 mali_pp_job_unstarted_sub_job_count(
621 pp_job);
622 mali_pp_job_mark_unstarted_failed(pp_job);
623
624 if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(pp_job)) {
625 if (mali_pp_job_is_complete(pp_job)) {
626 mali_pp_job_list_move(pp_job,
627 &removed_jobs_pp);
628 } else {
629 mali_pp_job_list_remove(pp_job);
630 }
631 }
632 }
633 }
634
635 /* Remove from PP high priority queue */
636 _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp,
637 &job_queue_pp.high_pri,
638 struct mali_pp_job, list) {
639 if (mali_pp_job_get_session(pp_job) == session) {
640 mali_pp_job_fb_lookup_remove(pp_job);
641
642 job_queue_pp.depth -=
643 mali_pp_job_unstarted_sub_job_count(
644 pp_job);
645 mali_pp_job_mark_unstarted_failed(pp_job);
646
647 if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(pp_job)) {
648 if (mali_pp_job_is_complete(pp_job)) {
649 mali_pp_job_list_move(pp_job,
650 &removed_jobs_pp);
651 } else {
652 mali_pp_job_list_remove(pp_job);
653 }
654 }
655 }
656 }
657
658 /*
659 * Release scheduler lock so we can release trackers
660 * (which will potentially queue new jobs)
661 */
662 mali_scheduler_unlock();
663
664 /* Release and complete all (non-running) found GP jobs */
665 _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &removed_jobs_gp,
666 struct mali_gp_job, list) {
667 mali_timeline_tracker_release(mali_gp_job_get_tracker(gp_job));
668 mali_gp_job_signal_pp_tracker(gp_job, MALI_FALSE);
669 _mali_osk_list_delinit(&gp_job->list);
670 mali_scheduler_complete_gp_job(gp_job,
671 MALI_FALSE, MALI_FALSE, MALI_TRUE);
672 }
673
674 /* Release and complete non-running PP jobs */
675 _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp, &removed_jobs_pp,
676 struct mali_pp_job, list) {
677 mali_timeline_tracker_release(mali_pp_job_get_tracker(pp_job));
678 _mali_osk_list_delinit(&pp_job->list);
679 mali_scheduler_complete_pp_job(pp_job, 0,
680 MALI_FALSE, MALI_TRUE);
681 }
682}
683
684_mali_osk_errcode_t _mali_ukk_gp_start_job(void *ctx,
685 _mali_uk_gp_start_job_s *uargs)
686{
687 struct mali_session_data *session;
688 struct mali_gp_job *job;
689 mali_timeline_point point;
690 u32 __user *point_ptr = NULL;
691
692 MALI_DEBUG_ASSERT_POINTER(uargs);
693 MALI_DEBUG_ASSERT_POINTER(ctx);
694
695 session = (struct mali_session_data *)(uintptr_t)ctx;
696
697 job = mali_gp_job_create(session, uargs, mali_scheduler_get_new_id(),
698 NULL);
699 if (NULL == job) {
700 MALI_PRINT_ERROR(("Failed to create GP job.\n"));
701 return _MALI_OSK_ERR_NOMEM;
702 }
703
704 point_ptr = (u32 __user *)(uintptr_t)mali_gp_job_get_timeline_point_ptr(job);
705
706 point = mali_scheduler_submit_gp_job(session, job);
707
708 if (0 != _mali_osk_put_user(((u32) point), point_ptr)) {
709 /*
710 * Let user space know that something failed
711 * after the job was started.
712 */
713 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
714 }
715
716 return _MALI_OSK_ERR_OK;
717}
718
719_mali_osk_errcode_t _mali_ukk_pp_start_job(void *ctx,
720 _mali_uk_pp_start_job_s *uargs)
721{
722 struct mali_session_data *session;
723 struct mali_pp_job *job;
724 mali_timeline_point point;
725 u32 __user *point_ptr = NULL;
726
727 MALI_DEBUG_ASSERT_POINTER(uargs);
728 MALI_DEBUG_ASSERT_POINTER(ctx);
729
730 session = (struct mali_session_data *)(uintptr_t)ctx;
731
732 job = mali_pp_job_create(session, uargs, mali_scheduler_get_new_id());
733 if (NULL == job) {
734 MALI_PRINT_ERROR(("Failed to create PP job.\n"));
735 return _MALI_OSK_ERR_NOMEM;
736 }
737
738 point_ptr = (u32 __user *)(uintptr_t)mali_pp_job_get_timeline_point_ptr(job);
739
740 point = mali_scheduler_submit_pp_job(session, job);
741 job = NULL;
742
743 if (0 != _mali_osk_put_user(((u32) point), point_ptr)) {
744 /*
745 * Let user space know that something failed
746 * after the job was started.
747 */
748 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
749 }
750
751 return _MALI_OSK_ERR_OK;
752}
753
754_mali_osk_errcode_t _mali_ukk_pp_and_gp_start_job(void *ctx,
755 _mali_uk_pp_and_gp_start_job_s *uargs)
756{
757 struct mali_session_data *session;
758 _mali_uk_pp_and_gp_start_job_s kargs;
759 struct mali_pp_job *pp_job;
760 struct mali_gp_job *gp_job;
761 u32 __user *point_ptr = NULL;
762 mali_timeline_point point;
763 _mali_uk_pp_start_job_s __user *pp_args;
764 _mali_uk_gp_start_job_s __user *gp_args;
765
766 MALI_DEBUG_ASSERT_POINTER(ctx);
767 MALI_DEBUG_ASSERT_POINTER(uargs);
768
769 session = (struct mali_session_data *) ctx;
770
771 if (0 != _mali_osk_copy_from_user(&kargs, uargs,
772 sizeof(_mali_uk_pp_and_gp_start_job_s))) {
773 return _MALI_OSK_ERR_NOMEM;
774 }
775
776 pp_args = (_mali_uk_pp_start_job_s __user *)(uintptr_t)kargs.pp_args;
777 gp_args = (_mali_uk_gp_start_job_s __user *)(uintptr_t)kargs.gp_args;
778
779 pp_job = mali_pp_job_create(session, pp_args,
780 mali_scheduler_get_new_id());
781 if (NULL == pp_job) {
782 MALI_PRINT_ERROR(("Failed to create PP job.\n"));
783 return _MALI_OSK_ERR_NOMEM;
784 }
785
786 gp_job = mali_gp_job_create(session, gp_args,
787 mali_scheduler_get_new_id(),
788 mali_pp_job_get_tracker(pp_job));
789 if (NULL == gp_job) {
790 MALI_PRINT_ERROR(("Failed to create GP job.\n"));
791 mali_pp_job_delete(pp_job);
792 return _MALI_OSK_ERR_NOMEM;
793 }
794
795 point_ptr = (u32 __user *)(uintptr_t)mali_pp_job_get_timeline_point_ptr(pp_job);
796
797 /* Submit GP job. */
798 mali_scheduler_submit_gp_job(session, gp_job);
799 gp_job = NULL;
800
801 /* Submit PP job. */
802 point = mali_scheduler_submit_pp_job(session, pp_job);
803 pp_job = NULL;
804
805 if (0 != _mali_osk_put_user(((u32) point), point_ptr)) {
806 /*
807 * Let user space know that something failed
808 * after the jobs were started.
809 */
810 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
811 }
812
813 return _MALI_OSK_ERR_OK;
814}
815
816void _mali_ukk_pp_job_disable_wb(_mali_uk_pp_disable_wb_s *args)
817{
818 struct mali_session_data *session;
819 struct mali_pp_job *job;
820 struct mali_pp_job *tmp;
821 u32 fb_lookup_id;
822
823 MALI_DEBUG_ASSERT_POINTER(args);
824 MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
825
826 session = (struct mali_session_data *)(uintptr_t)args->ctx;
827
828 fb_lookup_id = args->fb_id & MALI_PP_JOB_FB_LOOKUP_LIST_MASK;
829
830 mali_scheduler_lock();
831
832 /* Iterate over all jobs for given frame builder_id. */
833 _MALI_OSK_LIST_FOREACHENTRY(job, tmp,
834 &session->pp_job_fb_lookup_list[fb_lookup_id],
835 struct mali_pp_job, session_fb_lookup_list) {
836 MALI_DEBUG_CODE(u32 disable_mask = 0);
837
838 if (mali_pp_job_get_frame_builder_id(job) !=
839 (u32) args->fb_id) {
840 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Disable WB mismatching FB.\n"));
841 continue;
842 }
843
844 MALI_DEBUG_CODE(disable_mask |= 0xD << (4 * 3));
845
846 if (mali_pp_job_get_wb0_source_addr(job) == args->wb0_memory) {
847 MALI_DEBUG_CODE(disable_mask |= 0x1 << (4 * 1));
848 mali_pp_job_disable_wb0(job);
849 }
850
851 if (mali_pp_job_get_wb1_source_addr(job) == args->wb1_memory) {
852 MALI_DEBUG_CODE(disable_mask |= 0x2 << (4 * 2));
853 mali_pp_job_disable_wb1(job);
854 }
855
856 if (mali_pp_job_get_wb2_source_addr(job) == args->wb2_memory) {
857 MALI_DEBUG_CODE(disable_mask |= 0x3 << (4 * 3));
858 mali_pp_job_disable_wb2(job);
859 }
860 MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Disable WB: 0x%X.\n",
861 disable_mask));
862 }
863
864 mali_scheduler_unlock();
865}
866
867#if MALI_STATE_TRACKING
868u32 mali_scheduler_dump_state(char *buf, u32 size)
869{
870 int n = 0;
871
872 n += _mali_osk_snprintf(buf + n, size - n, "GP queues\n");
873 n += _mali_osk_snprintf(buf + n, size - n,
874 "\tQueue depth: %u\n", job_queue_gp.depth);
875 n += _mali_osk_snprintf(buf + n, size - n,
876 "\tNormal priority queue is %s\n",
877 _mali_osk_list_empty(&job_queue_gp.normal_pri) ?
878 "empty" : "not empty");
879 n += _mali_osk_snprintf(buf + n, size - n,
880 "\tHigh priority queue is %s\n",
881 _mali_osk_list_empty(&job_queue_gp.high_pri) ?
882 "empty" : "not empty");
883
884 n += _mali_osk_snprintf(buf + n, size - n,
885 "PP queues\n");
886 n += _mali_osk_snprintf(buf + n, size - n,
887 "\tQueue depth: %u\n", job_queue_pp.depth);
888 n += _mali_osk_snprintf(buf + n, size - n,
889 "\tNormal priority queue is %s\n",
890 _mali_osk_list_empty(&job_queue_pp.normal_pri)
891 ? "empty" : "not empty");
892 n += _mali_osk_snprintf(buf + n, size - n,
893 "\tHigh priority queue is %s\n",
894 _mali_osk_list_empty(&job_queue_pp.high_pri)
895 ? "empty" : "not empty");
896
897 n += _mali_osk_snprintf(buf + n, size - n, "\n");
898
899 return n;
900}
901#endif
902
903/*
904 * ---------- Implementation of static functions ----------
905 */
906
907static mali_timeline_point mali_scheduler_submit_gp_job(
908 struct mali_session_data *session, struct mali_gp_job *job)
909{
910 mali_timeline_point point;
911
912 MALI_DEBUG_ASSERT_POINTER(session);
913 MALI_DEBUG_ASSERT_POINTER(job);
914
915 /* Add job to Timeline system. */
916 point = mali_timeline_system_add_tracker(session->timeline_system,
917 mali_gp_job_get_tracker(job), MALI_TIMELINE_GP);
918
919 return point;
920}
921
922static mali_timeline_point mali_scheduler_submit_pp_job(
923 struct mali_session_data *session, struct mali_pp_job *job)
924{
925 mali_timeline_point point;
926
927 MALI_DEBUG_ASSERT_POINTER(session);
928 MALI_DEBUG_ASSERT_POINTER(job);
929
930 mali_scheduler_lock();
931 /*
932 * Adding job to the lookup list used to quickly discard
933 * writeback units of queued jobs.
934 */
935 mali_pp_job_fb_lookup_add(job);
936 mali_scheduler_unlock();
937
938 /* Add job to Timeline system. */
939 point = mali_timeline_system_add_tracker(session->timeline_system,
940 mali_pp_job_get_tracker(job), MALI_TIMELINE_PP);
941
942 return point;
943}
944
945static mali_bool mali_scheduler_queue_gp_job(struct mali_gp_job *job)
946{
947 struct mali_session_data *session;
948 _mali_osk_list_t *queue;
949
950 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
951 MALI_DEBUG_ASSERT_POINTER(job);
952
953 session = mali_gp_job_get_session(job);
954 MALI_DEBUG_ASSERT_POINTER(session);
955
956 if (unlikely(session->is_aborting)) {
957 MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Job %u (0x%08X) queued while session is aborting.\n",
958 mali_gp_job_get_id(job), job));
959 return MALI_FALSE; /* job not queued */
960 }
961
962 mali_gp_job_set_cache_order(job, mali_scheduler_get_new_cache_order());
963
964 /* Determine which queue the job should be added to. */
965 if (session->use_high_priority_job_queue) {
966 queue = &job_queue_gp.high_pri;
967 } else {
968 queue = &job_queue_gp.normal_pri;
969 }
970
971 job_queue_gp.depth += 1;
972
973 /* Add job to queue (mali_gp_job_queue_add find correct place). */
974 mali_gp_job_list_add(job, queue);
975
976 /*
977 * We hold a PM reference for every job we hold queued (and running)
978 * It is important that we take this reference after job has been
979 * added the the queue so that any runtime resume could schedule this
980 * job right there and then.
981 */
982 _mali_osk_pm_dev_ref_get_async();
983
984 if (mali_utilization_enabled()) {
985 /*
986 * We cheat a little bit by counting the GP as busy from the
987 * time a GP job is queued. This will be fine because we only
988 * loose the tiny idle gap between jobs, but we will instead
989 * get less utilization work to do (less locks taken)
990 */
991 mali_utilization_gp_start();
992 }
993
994 /* Add profiling events for job enqueued */
995 _mali_osk_profiling_add_event(
996 MALI_PROFILING_EVENT_TYPE_SINGLE |
997 MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
998 MALI_PROFILING_EVENT_REASON_SINGLE_SW_GP_ENQUEUE,
999 mali_gp_job_get_pid(job),
1000 mali_gp_job_get_tid(job),
1001 mali_gp_job_get_frame_builder_id(job),
1002 mali_gp_job_get_flush_id(job),
1003 0);
1004
1005#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
1006 trace_gpu_job_enqueue(mali_gp_job_get_tid(job),
1007 mali_gp_job_get_id(job), "GP");
1008#endif
1009
1010 MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Job %u (0x%08X) queued\n",
1011 mali_gp_job_get_id(job), job));
1012
1013 return MALI_TRUE; /* job queued */
1014}
1015
1016static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job)
1017{
1018 struct mali_session_data *session;
1019 _mali_osk_list_t *queue = NULL;
1020
1021 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
1022 MALI_DEBUG_ASSERT_POINTER(job);
1023
1024 session = mali_pp_job_get_session(job);
1025 MALI_DEBUG_ASSERT_POINTER(session);
1026
1027 if (unlikely(session->is_aborting)) {
1028 MALI_DEBUG_PRINT(2, ("Mali PP scheduler: Job %u (0x%08X) queued while session is aborting.\n",
1029 mali_pp_job_get_id(job), job));
1030 return MALI_FALSE; /* job not queued */
1031 }
1032
1033 mali_pp_job_set_cache_order(job, mali_scheduler_get_new_cache_order());
1034
1035 if (session->use_high_priority_job_queue) {
1036 queue = &job_queue_pp.high_pri;
1037 } else {
1038 queue = &job_queue_pp.normal_pri;
1039 }
1040
1041 job_queue_pp.depth +=
1042 mali_pp_job_get_sub_job_count(job);
1043
1044 /* Add job to queue (mali_gp_job_queue_add find correct place). */
1045 mali_pp_job_list_add(job, queue);
1046
1047 /*
1048 * We hold a PM reference for every job we hold queued (and running)
1049 * It is important that we take this reference after job has been
1050 * added the the queue so that any runtime resume could schedule this
1051 * job right there and then.
1052 */
1053 _mali_osk_pm_dev_ref_get_async();
1054
1055 if (mali_utilization_enabled()) {
1056 /*
1057 * We cheat a little bit by counting the PP as busy from the
1058 * time a PP job is queued. This will be fine because we only
1059 * loose the tiny idle gap between jobs, but we will instead
1060 * get less utilization work to do (less locks taken)
1061 */
1062 mali_utilization_pp_start();
1063 }
1064
1065 /* Add profiling events for job enqueued */
1066
1067 _mali_osk_profiling_add_event(
1068 MALI_PROFILING_EVENT_TYPE_SINGLE |
1069 MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1070 MALI_PROFILING_EVENT_REASON_SINGLE_SW_PP_ENQUEUE,
1071 mali_pp_job_get_pid(job),
1072 mali_pp_job_get_tid(job),
1073 mali_pp_job_get_frame_builder_id(job),
1074 mali_pp_job_get_flush_id(job),
1075 0);
1076
1077#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
1078 trace_gpu_job_enqueue(mali_pp_job_get_tid(job),
1079 mali_pp_job_get_id(job), "PP");
1080#endif
1081
1082 MALI_DEBUG_PRINT(3, ("Mali PP scheduler: %s job %u (0x%08X) with %u parts queued.\n",
1083 mali_pp_job_is_virtual(job)
1084 ? "Virtual" : "Physical",
1085 mali_pp_job_get_id(job), job,
1086 mali_pp_job_get_sub_job_count(job)));
1087
1088 return MALI_TRUE; /* job queued */
1089}
1090
1091static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job,
1092 mali_bool success)
1093{
1094 _mali_uk_gp_job_finished_s *jobres;
1095 struct mali_session_data *session;
1096 _mali_osk_notification_t *notification;
1097
1098 MALI_DEBUG_ASSERT_POINTER(job);
1099
1100 session = mali_gp_job_get_session(job);
1101 MALI_DEBUG_ASSERT_POINTER(session);
1102
1103 notification = mali_gp_job_get_finished_notification(job);
1104 MALI_DEBUG_ASSERT_POINTER(notification);
1105
1106 jobres = notification->result_buffer;
1107 MALI_DEBUG_ASSERT_POINTER(jobres);
1108
1109 jobres->user_job_ptr = mali_gp_job_get_user_id(job);
1110 if (MALI_TRUE == success) {
1111 jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS;
1112 } else {
1113 jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR;
1114 }
1115 jobres->heap_current_addr = mali_gp_job_get_current_heap_addr(job);
1116 jobres->perf_counter0 = mali_gp_job_get_perf_counter_value0(job);
1117 jobres->perf_counter1 = mali_gp_job_get_perf_counter_value1(job);
1118
1119 mali_session_send_notification(session, notification);
1120}
1121
1122static void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job,
1123 u32 num_cores_in_virtual)
1124{
1125 u32 i;
1126 u32 num_counters_to_copy;
1127 _mali_uk_pp_job_finished_s *jobres;
1128 struct mali_session_data *session;
1129 _mali_osk_notification_t *notification;
1130
1131 if (MALI_TRUE == mali_pp_job_use_no_notification(job)) {
1132 return;
1133 }
1134
1135 MALI_DEBUG_ASSERT_POINTER(job);
1136
1137 session = mali_pp_job_get_session(job);
1138 MALI_DEBUG_ASSERT_POINTER(session);
1139
1140 notification = mali_pp_job_get_finished_notification(job);
1141 MALI_DEBUG_ASSERT_POINTER(notification);
1142
1143 jobres = notification->result_buffer;
1144 MALI_DEBUG_ASSERT_POINTER(jobres);
1145
1146 jobres->user_job_ptr = mali_pp_job_get_user_id(job);
1147 if (MALI_TRUE == mali_pp_job_was_success(job)) {
1148 jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS;
1149 } else {
1150 jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR;
1151 }
1152
1153 if (mali_pp_job_is_virtual(job)) {
1154 num_counters_to_copy = num_cores_in_virtual;
1155 } else {
1156 num_counters_to_copy = mali_pp_job_get_sub_job_count(job);
1157 }
1158
1159 for (i = 0; i < num_counters_to_copy; i++) {
1160 jobres->perf_counter0[i] =
1161 mali_pp_job_get_perf_counter_value0(job, i);
1162 jobres->perf_counter1[i] =
1163 mali_pp_job_get_perf_counter_value1(job, i);
1164 jobres->perf_counter_src0 =
1165 mali_pp_job_get_pp_counter_global_src0();
1166 jobres->perf_counter_src1 =
1167 mali_pp_job_get_pp_counter_global_src1();
1168 }
1169
1170 mali_session_send_notification(session, notification);
1171}
1172
1173#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE)
1174
1175static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job)
1176{
1177 MALI_DEBUG_ASSERT_POINTER(job);
1178
1179 _mali_osk_spinlock_irq_lock(scheduler_pp_job_delete_lock);
1180 mali_pp_job_list_addtail(job, &scheduler_pp_job_deletion_queue);
1181 _mali_osk_spinlock_irq_unlock(scheduler_pp_job_delete_lock);
1182
1183 _mali_osk_wq_schedule_work(scheduler_wq_pp_job_delete);
1184}
1185
1186static void mali_scheduler_do_pp_job_delete(void *arg)
1187{
1188 _MALI_OSK_LIST_HEAD_STATIC_INIT(list);
1189 struct mali_pp_job *job;
1190 struct mali_pp_job *tmp;
1191
1192 MALI_IGNORE(arg);
1193
1194 /*
1195 * Quickly "unhook" the jobs pending to be deleted, so we can release
1196 * the lock before we start deleting the job objects
1197 * (without any locks held)
1198 */
1199 _mali_osk_spinlock_irq_lock(scheduler_pp_job_delete_lock);
1200 _mali_osk_list_move_list(&scheduler_pp_job_deletion_queue, &list);
1201 _mali_osk_spinlock_irq_unlock(scheduler_pp_job_delete_lock);
1202
1203 _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list,
1204 struct mali_pp_job, list) {
1205
1206 _mali_osk_list_delinit(&job->list);
1207 mali_pp_job_delete(job); /* delete the job object itself */
1208 }
1209}
1210
1211#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) */
1212
1213#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
1214
1215static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job)
1216{
1217 MALI_DEBUG_ASSERT_POINTER(job);
1218
1219 _mali_osk_spinlock_irq_lock(scheduler_pp_job_queue_lock);
1220 mali_pp_job_list_addtail(job, &scheduler_pp_job_queue_list);
1221 _mali_osk_spinlock_irq_unlock(scheduler_pp_job_queue_lock);
1222
1223 _mali_osk_wq_schedule_work(scheduler_wq_pp_job_queue);
1224}
1225
1226static void mali_scheduler_do_pp_job_queue(void *arg)
1227{
1228 _MALI_OSK_LIST_HEAD_STATIC_INIT(list);
1229 struct mali_pp_job *job;
1230 struct mali_pp_job *tmp;
1231 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1232
1233 MALI_IGNORE(arg);
1234
1235 /*
1236 * Quickly "unhook" the jobs pending to be queued, so we can release
1237 * the lock before we start queueing the job objects
1238 * (without any locks held)
1239 */
1240 _mali_osk_spinlock_irq_lock(scheduler_pp_job_queue_lock);
1241 _mali_osk_list_move_list(&scheduler_pp_job_queue_list, &list);
1242 _mali_osk_spinlock_irq_unlock(scheduler_pp_job_queue_lock);
1243
1244 /* First loop through all jobs and do the pre-work (no locks needed) */
1245 _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list,
1246 struct mali_pp_job, list) {
1247 if (mali_pp_job_needs_dma_buf_mapping(job)) {
1248 /*
1249 * This operation could fail, but we continue anyway,
1250 * because the worst that could happen is that this
1251 * job will fail due to a Mali page fault.
1252 */
1253 mali_dma_buf_map_job(job);
6fa3eb70
S		 1254 }
1255 }
1256
bdc132d7
S		 1257 mali_scheduler_lock();
1258
1259 /* Then loop through all jobs again to queue them (lock needed) */
1260 _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list,
1261 struct mali_pp_job, list) {
1262
1263 /* Remove from scheduler_pp_job_queue_list before queueing */
1264 mali_pp_job_list_remove(job);
1265
1266 if (mali_scheduler_queue_pp_job(job)) {
1267 /* Job queued successfully */
1268 schedule_mask |= MALI_SCHEDULER_MASK_PP;
6fa3eb70 1269 } else {
bdc132d7
S		 1270 /* Failed to enqueue job, release job (with error) */
1271 mali_pp_job_fb_lookup_remove(job);
1272 mali_pp_job_mark_unstarted_failed(job);
1273
1274 /* unlock scheduler in this uncommon case */
1275 mali_scheduler_unlock();
1276
1277 schedule_mask |= mali_timeline_tracker_release(
1278 mali_pp_job_get_tracker(job));
1279
1280 /* Notify user space and close the job object */
1281 mali_scheduler_complete_pp_job(job, 0, MALI_TRUE,
1282 MALI_FALSE);
1283
1284 mali_scheduler_lock();
1285 }
1286 }
1287
1288 mali_scheduler_unlock();
1289
1290 /* Trigger scheduling of jobs */
1291 mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
1292}
1293
1294#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
1295
1296void mali_scheduler_gp_pp_job_queue_print(void)
1297{
1298 struct mali_gp_job *gp_job = NULL;
1299 struct mali_gp_job *tmp_gp_job = NULL;
1300 struct mali_pp_job *pp_job = NULL;
1301 struct mali_pp_job *tmp_pp_job = NULL;
1302
1303 MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
1304 MALI_DEBUG_ASSERT_LOCK_HELD(mali_executor_lock_obj);
1305
1306 /* dump job queup status */
1307 if ((0 == job_queue_gp.depth) && (0 == job_queue_pp.depth)) {
1308 MALI_PRINT(("No GP&PP job in the job queue.\n"));
1309 return;
1310 }
1311
1312 MALI_PRINT(("Total (%d) GP job in the job queue.\n", job_queue_gp.depth));
1313 if (job_queue_gp.depth > 0) {
1314 if (!_mali_osk_list_empty(&job_queue_gp.high_pri)) {
1315 _MALI_OSK_LIST_FOREACHENTRY(gp_job, tmp_gp_job, &job_queue_gp.high_pri,
1316 struct mali_gp_job, list) {
1317 MALI_PRINT(("GP job(%p) id = %d tid = %d pid = %d in the gp job high_pri queue\n", gp_job, gp_job->id, gp_job->tid, gp_job->pid));
1318 }
1319 }
1320
1321 if (!_mali_osk_list_empty(&job_queue_gp.normal_pri)) {
1322 _MALI_OSK_LIST_FOREACHENTRY(gp_job, tmp_gp_job, &job_queue_gp.normal_pri,
1323 struct mali_gp_job, list) {
1324 MALI_PRINT(("GP job(%p) id = %d tid = %d pid = %d in the gp job normal_pri queue\n", gp_job, gp_job->id, gp_job->tid, gp_job->pid));
1325 }
6fa3eb70
S		 1326 }
1327 }
bdc132d7
S		 1328
1329 MALI_PRINT(("Total (%d) PP job in the job queue.\n", job_queue_pp.depth));
1330 if (job_queue_pp.depth > 0) {
1331 if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) {
1332 _MALI_OSK_LIST_FOREACHENTRY(pp_job, tmp_pp_job, &job_queue_pp.high_pri,
1333 struct mali_pp_job, list) {
1334 if (mali_pp_job_is_virtual(pp_job)) {
1335 MALI_PRINT(("PP Virtual job(%p) id = %d tid = %d pid = %d in the pp job high_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid));
1336 } else {
1337 MALI_PRINT(("PP Physical job(%p) id = %d tid = %d pid = %d in the pp job high_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid));
1338 }
1339 }
1340 }
1341
1342 if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
1343 _MALI_OSK_LIST_FOREACHENTRY(pp_job, tmp_pp_job, &job_queue_pp.normal_pri,
1344 struct mali_pp_job, list) {
1345 if (mali_pp_job_is_virtual(pp_job)) {
1346 MALI_PRINT(("PP Virtual job(%p) id = %d tid = %d pid = %d in the pp job normal_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid));
1347 } else {
1348 MALI_PRINT(("PP Physical job(%p) id = %d tid = %d pid = %d in the pp job normal_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid));
1349 }
1350 }
1351 }
1352 }
1353
1354 /* dump group running job status */
1355 mali_executor_running_status_print();
6fa3eb70 1356}
kernel source for telekom puls (alcatel/mediatek)