2 * This confidential and proprietary software may be used only as
3 * authorised by a licensing agreement from ARM Limited
4 * (C) COPYRIGHT 2008-2015 ARM Limited
6 * The entire notice above must be reproduced on all authorised
7 * copies and copies may only be made to the extent permitted
8 * by a licensing agreement from ARM Limited.
10 #include "mali_kernel_common.h"
12 #include "mali_l2_cache.h"
13 #include "mali_hw_core.h"
14 #include "mali_scheduler.h"
16 #include "mali_pm_domain.h"
19 * Size of the Mali L2 cache registers in bytes
21 #define MALI400_L2_CACHE_REGISTERS_SIZE 0x30
24 * Mali L2 cache register numbers
25 * Used in the register read/write routines.
26 * See the hardware documentation for more information about each register
28 typedef enum mali_l2_cache_register
{
29 MALI400_L2_CACHE_REGISTER_SIZE
= 0x0004,
30 MALI400_L2_CACHE_REGISTER_STATUS
= 0x0008,
32 MALI400_L2_CACHE_REGISTER_COMMAND
= 0x0010,
33 MALI400_L2_CACHE_REGISTER_CLEAR_PAGE
= 0x0014,
34 MALI400_L2_CACHE_REGISTER_MAX_READS
= 0x0018,
35 MALI400_L2_CACHE_REGISTER_ENABLE
= 0x001C,
36 MALI400_L2_CACHE_REGISTER_PERFCNT_SRC0
= 0x0020,
37 MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0
= 0x0024,
38 MALI400_L2_CACHE_REGISTER_PERFCNT_SRC1
= 0x0028,
39 MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1
= 0x002C,
40 } mali_l2_cache_register
;
43 * Mali L2 cache commands
44 * These are the commands that can be sent to the Mali L2 cache unit
46 typedef enum mali_l2_cache_command
{
47 MALI400_L2_CACHE_COMMAND_CLEAR_ALL
= 0x01,
48 } mali_l2_cache_command
;
51 * Mali L2 cache commands
52 * These are the commands that can be sent to the Mali L2 cache unit
54 typedef enum mali_l2_cache_enable
{
55 MALI400_L2_CACHE_ENABLE_DEFAULT
= 0x0, /* Default */
56 MALI400_L2_CACHE_ENABLE_ACCESS
= 0x01,
57 MALI400_L2_CACHE_ENABLE_READ_ALLOCATE
= 0x02,
58 } mali_l2_cache_enable
;
61 * Mali L2 cache status bits
63 typedef enum mali_l2_cache_status
{
64 MALI400_L2_CACHE_STATUS_COMMAND_BUSY
= 0x01,
65 MALI400_L2_CACHE_STATUS_DATA_BUSY
= 0x02,
66 } mali_l2_cache_status
;
68 #define MALI400_L2_MAX_READS_NOT_SET -1
70 static struct mali_l2_cache_core
*
71 mali_global_l2s
[MALI_MAX_NUMBER_OF_L2_CACHE_CORES
] = { NULL
, };
72 static u32 mali_global_num_l2s
= 0;
74 int mali_l2_max_reads
= MALI400_L2_MAX_READS_NOT_SET
;
77 /* Local helper functions */
79 static void mali_l2_cache_reset(struct mali_l2_cache_core
*cache
);
81 static _mali_osk_errcode_t
mali_l2_cache_send_command(
82 struct mali_l2_cache_core
*cache
, u32 reg
, u32 val
);
84 static void mali_l2_cache_lock(struct mali_l2_cache_core
*cache
)
86 MALI_DEBUG_ASSERT_POINTER(cache
);
87 _mali_osk_spinlock_irq_lock(cache
->lock
);
90 static void mali_l2_cache_unlock(struct mali_l2_cache_core
*cache
)
92 MALI_DEBUG_ASSERT_POINTER(cache
);
93 _mali_osk_spinlock_irq_unlock(cache
->lock
);
96 /* Implementation of the L2 cache interface */
98 struct mali_l2_cache_core
*mali_l2_cache_create(
99 _mali_osk_resource_t
*resource
, u32 domain_index
)
101 struct mali_l2_cache_core
*cache
= NULL
;
106 MALI_DEBUG_PRINT(4, ("Mali L2 cache: Creating Mali L2 cache: %s\n",
107 resource
->description
));
109 if (mali_global_num_l2s
>= MALI_MAX_NUMBER_OF_L2_CACHE_CORES
) {
110 MALI_PRINT_ERROR(("Mali L2 cache: Too many L2 caches\n"));
114 cache
= _mali_osk_malloc(sizeof(struct mali_l2_cache_core
));
116 MALI_PRINT_ERROR(("Mali L2 cache: Failed to allocate memory for L2 cache core\n"));
120 cache
->core_id
= mali_global_num_l2s
;
121 cache
->counter_src0
= MALI_HW_CORE_NO_COUNTER
;
122 cache
->counter_src1
= MALI_HW_CORE_NO_COUNTER
;
123 cache
->counter_value0_base
= 0;
124 cache
->counter_value1_base
= 0;
125 cache
->pm_domain
= NULL
;
126 cache
->power_is_on
= MALI_FALSE
;
127 cache
->last_invalidated_id
= 0;
129 if (_MALI_OSK_ERR_OK
!= mali_hw_core_create(&cache
->hw_core
,
130 resource
, MALI400_L2_CACHE_REGISTERS_SIZE
)) {
131 _mali_osk_free(cache
);
136 cache_size
= mali_hw_core_register_read(&cache
->hw_core
,
137 MALI400_L2_CACHE_REGISTER_SIZE
);
138 MALI_DEBUG_PRINT(2, ("Mali L2 cache: Created %s: % 3uK, %u-way, % 2ubyte cache line, % 3ubit external bus\n",
139 resource
->description
,
140 1 << (((cache_size
>> 16) & 0xff) - 10),
141 1 << ((cache_size
>> 8) & 0xff),
142 1 << (cache_size
& 0xff),
143 1 << ((cache_size
>> 24) & 0xff)));
146 cache
->lock
= _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED
,
147 _MALI_OSK_LOCK_ORDER_L2
);
148 if (NULL
== cache
->lock
) {
149 MALI_PRINT_ERROR(("Mali L2 cache: Failed to create counter lock for L2 cache core %s\n",
150 cache
->hw_core
.description
));
151 mali_hw_core_delete(&cache
->hw_core
);
152 _mali_osk_free(cache
);
156 /* register with correct power domain */
157 cache
->pm_domain
= mali_pm_register_l2_cache(
158 domain_index
, cache
);
160 mali_global_l2s
[mali_global_num_l2s
] = cache
;
161 mali_global_num_l2s
++;
166 void mali_l2_cache_delete(struct mali_l2_cache_core
*cache
)
169 for (i
= 0; i
< mali_global_num_l2s
; i
++) {
170 if (mali_global_l2s
[i
] != cache
) {
174 mali_global_l2s
[i
] = NULL
;
175 mali_global_num_l2s
--;
177 if (i
== mali_global_num_l2s
) {
178 /* Removed last element, nothing more to do */
183 * We removed a l2 cache from the middle of the array,
184 * so move the last l2 cache to current position
186 mali_global_l2s
[i
] = mali_global_l2s
[mali_global_num_l2s
];
187 mali_global_l2s
[mali_global_num_l2s
] = NULL
;
193 _mali_osk_spinlock_irq_term(cache
->lock
);
194 mali_hw_core_delete(&cache
->hw_core
);
195 _mali_osk_free(cache
);
198 void mali_l2_cache_power_up(struct mali_l2_cache_core
*cache
)
200 MALI_DEBUG_ASSERT_POINTER(cache
);
202 mali_l2_cache_lock(cache
);
204 mali_l2_cache_reset(cache
);
206 MALI_DEBUG_ASSERT(MALI_FALSE
== cache
->power_is_on
);
207 cache
->power_is_on
= MALI_TRUE
;
209 mali_l2_cache_unlock(cache
);
212 void mali_l2_cache_power_down(struct mali_l2_cache_core
*cache
)
214 MALI_DEBUG_ASSERT_POINTER(cache
);
216 mali_l2_cache_lock(cache
);
218 MALI_DEBUG_ASSERT(MALI_TRUE
== cache
->power_is_on
);
221 * The HW counters will start from zero again when we resume,
222 * but we should report counters as always increasing.
223 * Take a copy of the HW values now in order to add this to
224 * the values we report after being powered up.
226 * The physical power off of the L2 cache might be outside our
227 * own control (e.g. runtime PM). That is why we must manually
228 * set set the counter value to zero as well.
231 if (cache
->counter_src0
!= MALI_HW_CORE_NO_COUNTER
) {
232 cache
->counter_value0_base
+= mali_hw_core_register_read(
234 MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0
);
235 mali_hw_core_register_write(&cache
->hw_core
,
236 MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0
, 0);
239 if (cache
->counter_src1
!= MALI_HW_CORE_NO_COUNTER
) {
240 cache
->counter_value1_base
+= mali_hw_core_register_read(
242 MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1
);
243 mali_hw_core_register_write(&cache
->hw_core
,
244 MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1
, 0);
248 cache
->power_is_on
= MALI_FALSE
;
250 mali_l2_cache_unlock(cache
);
253 void mali_l2_cache_core_set_counter_src(
254 struct mali_l2_cache_core
*cache
, u32 source_id
, u32 counter
)
259 MALI_DEBUG_ASSERT_POINTER(cache
);
260 MALI_DEBUG_ASSERT(source_id
>= 0 && source_id
<= 1);
262 mali_l2_cache_lock(cache
);
264 if (0 == source_id
) {
265 /* start counting from 0 */
266 cache
->counter_value0_base
= 0;
267 cache
->counter_src0
= counter
;
268 reg_offset_src
= MALI400_L2_CACHE_REGISTER_PERFCNT_SRC0
;
269 reg_offset_val
= MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0
;
271 /* start counting from 0 */
272 cache
->counter_value1_base
= 0;
273 cache
->counter_src1
= counter
;
274 reg_offset_src
= MALI400_L2_CACHE_REGISTER_PERFCNT_SRC1
;
275 reg_offset_val
= MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1
;
278 if (cache
->power_is_on
) {
281 if (MALI_HW_CORE_NO_COUNTER
!= counter
) {
284 hw_src
= 0; /* disable value for HW */
287 /* Set counter src */
288 mali_hw_core_register_write(&cache
->hw_core
,
289 reg_offset_src
, hw_src
);
291 /* Make sure the HW starts counting from 0 again */
292 mali_hw_core_register_write(&cache
->hw_core
,
296 mali_l2_cache_unlock(cache
);
299 void mali_l2_cache_core_get_counter_values(
300 struct mali_l2_cache_core
*cache
,
301 u32
*src0
, u32
*value0
, u32
*src1
, u32
*value1
)
303 MALI_DEBUG_ASSERT_POINTER(cache
);
304 MALI_DEBUG_ASSERT(NULL
!= src0
);
305 MALI_DEBUG_ASSERT(NULL
!= value0
);
306 MALI_DEBUG_ASSERT(NULL
!= src1
);
307 MALI_DEBUG_ASSERT(NULL
!= value1
);
309 mali_l2_cache_lock(cache
);
311 *src0
= cache
->counter_src0
;
312 *src1
= cache
->counter_src1
;
314 if (cache
->counter_src0
!= MALI_HW_CORE_NO_COUNTER
) {
315 if (MALI_TRUE
== cache
->power_is_on
) {
316 *value0
= mali_hw_core_register_read(&cache
->hw_core
,
317 MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0
);
322 /* Add base offset value (in case we have been power off) */
323 *value0
+= cache
->counter_value0_base
;
326 if (cache
->counter_src1
!= MALI_HW_CORE_NO_COUNTER
) {
327 if (MALI_TRUE
== cache
->power_is_on
) {
328 *value1
= mali_hw_core_register_read(&cache
->hw_core
,
329 MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1
);
334 /* Add base offset value (in case we have been power off) */
335 *value1
+= cache
->counter_value1_base
;
338 mali_l2_cache_unlock(cache
);
341 struct mali_l2_cache_core
*mali_l2_cache_core_get_glob_l2_core(u32 index
)
343 if (mali_global_num_l2s
> index
) {
344 return mali_global_l2s
[index
];
350 u32
mali_l2_cache_core_get_glob_num_l2_cores(void)
352 return mali_global_num_l2s
;
355 void mali_l2_cache_invalidate(struct mali_l2_cache_core
*cache
)
357 MALI_DEBUG_ASSERT_POINTER(cache
);
363 mali_l2_cache_lock(cache
);
365 cache
->last_invalidated_id
= mali_scheduler_get_new_cache_order();
366 mali_l2_cache_send_command(cache
, MALI400_L2_CACHE_REGISTER_COMMAND
,
367 MALI400_L2_CACHE_COMMAND_CLEAR_ALL
);
369 mali_l2_cache_unlock(cache
);
372 void mali_l2_cache_invalidate_conditional(
373 struct mali_l2_cache_core
*cache
, u32 id
)
375 MALI_DEBUG_ASSERT_POINTER(cache
);
382 * If the last cache invalidation was done by a job with a higher id we
383 * don't have to flush. Since user space will store jobs w/ their
384 * corresponding memory in sequence (first job #0, then job #1, ...),
385 * we don't have to flush for job n-1 if job n has already invalidated
386 * the cache since we know for sure that job n-1's memory was already
387 * written when job n was started.
390 mali_l2_cache_lock(cache
);
392 if (((s32
)id
) > ((s32
)cache
->last_invalidated_id
)) {
393 /* Set latest invalidated id to current "point in time" */
394 cache
->last_invalidated_id
=
395 mali_scheduler_get_new_cache_order();
396 mali_l2_cache_send_command(cache
,
397 MALI400_L2_CACHE_REGISTER_COMMAND
,
398 MALI400_L2_CACHE_COMMAND_CLEAR_ALL
);
401 mali_l2_cache_unlock(cache
);
404 void mali_l2_cache_invalidate_all(void)
407 for (i
= 0; i
< mali_global_num_l2s
; i
++) {
408 struct mali_l2_cache_core
*cache
= mali_global_l2s
[i
];
409 _mali_osk_errcode_t ret
;
411 MALI_DEBUG_ASSERT_POINTER(cache
);
413 mali_l2_cache_lock(cache
);
415 if (MALI_TRUE
!= cache
->power_is_on
) {
416 mali_l2_cache_unlock(cache
);
420 cache
->last_invalidated_id
=
421 mali_scheduler_get_new_cache_order();
423 ret
= mali_l2_cache_send_command(cache
,
424 MALI400_L2_CACHE_REGISTER_COMMAND
,
425 MALI400_L2_CACHE_COMMAND_CLEAR_ALL
);
426 if (_MALI_OSK_ERR_OK
!= ret
) {
427 MALI_PRINT_ERROR(("Failed to invalidate cache\n"));
430 mali_l2_cache_unlock(cache
);
434 void mali_l2_cache_invalidate_all_pages(u32
*pages
, u32 num_pages
)
437 for (i
= 0; i
< mali_global_num_l2s
; i
++) {
438 struct mali_l2_cache_core
*cache
= mali_global_l2s
[i
];
441 MALI_DEBUG_ASSERT_POINTER(cache
);
443 mali_l2_cache_lock(cache
);
445 if (MALI_TRUE
!= cache
->power_is_on
) {
446 mali_l2_cache_unlock(cache
);
450 for (j
= 0; j
< num_pages
; j
++) {
451 _mali_osk_errcode_t ret
;
453 ret
= mali_l2_cache_send_command(cache
,
454 MALI400_L2_CACHE_REGISTER_CLEAR_PAGE
,
456 if (_MALI_OSK_ERR_OK
!= ret
) {
457 MALI_PRINT_ERROR(("Failed to invalidate cache (page)\n"));
461 mali_l2_cache_unlock(cache
);
465 /* -------- local helper functions below -------- */
467 static void mali_l2_cache_reset(struct mali_l2_cache_core
*cache
)
469 MALI_DEBUG_ASSERT_POINTER(cache
);
470 MALI_DEBUG_ASSERT_LOCK_HELD(cache
->lock
);
472 /* Invalidate cache (just to keep it in a known state at startup) */
473 mali_l2_cache_send_command(cache
, MALI400_L2_CACHE_REGISTER_COMMAND
,
474 MALI400_L2_CACHE_COMMAND_CLEAR_ALL
);
477 mali_hw_core_register_write(&cache
->hw_core
,
478 MALI400_L2_CACHE_REGISTER_ENABLE
,
479 (u32
)MALI400_L2_CACHE_ENABLE_ACCESS
|
480 (u32
)MALI400_L2_CACHE_ENABLE_READ_ALLOCATE
);
482 if (MALI400_L2_MAX_READS_NOT_SET
!= mali_l2_max_reads
) {
483 mali_hw_core_register_write(&cache
->hw_core
,
484 MALI400_L2_CACHE_REGISTER_MAX_READS
,
485 (u32
)mali_l2_max_reads
);
488 /* Restart any performance counters (if enabled) */
489 if (cache
->counter_src0
!= MALI_HW_CORE_NO_COUNTER
) {
491 mali_hw_core_register_write(&cache
->hw_core
,
492 MALI400_L2_CACHE_REGISTER_PERFCNT_SRC0
,
493 cache
->counter_src0
);
496 if (cache
->counter_src1
!= MALI_HW_CORE_NO_COUNTER
) {
497 mali_hw_core_register_write(&cache
->hw_core
,
498 MALI400_L2_CACHE_REGISTER_PERFCNT_SRC1
,
499 cache
->counter_src1
);
503 static _mali_osk_errcode_t
mali_l2_cache_send_command(
504 struct mali_l2_cache_core
*cache
, u32 reg
, u32 val
)
507 const int loop_count
= 100000;
509 MALI_DEBUG_ASSERT_POINTER(cache
);
510 MALI_DEBUG_ASSERT_LOCK_HELD(cache
->lock
);
513 * First, wait for L2 cache command handler to go idle.
514 * (Commands received while processing another command will be ignored)
516 for (i
= 0; i
< loop_count
; i
++) {
517 if (!(mali_hw_core_register_read(&cache
->hw_core
,
518 MALI400_L2_CACHE_REGISTER_STATUS
) &
519 (u32
)MALI400_L2_CACHE_STATUS_COMMAND_BUSY
)) {
524 if (i
== loop_count
) {
525 MALI_DEBUG_PRINT(1, ("Mali L2 cache: aborting wait for command interface to go idle\n"));
526 return _MALI_OSK_ERR_FAULT
;
529 /* then issue the command */
530 mali_hw_core_register_write(&cache
->hw_core
, reg
, val
);
532 return _MALI_OSK_ERR_OK
;