drivers/misc/mediatek/gpu/mt8127/mali/mali/common/mali_pp_job.c

   1 /*
   2  * This confidential and proprietary software may be used only as
   3  * authorised by a licensing agreement from ARM Limited
   4  * (C) COPYRIGHT 2011-2013 ARM Limited
   5  * ALL RIGHTS RESERVED
   6  * The entire notice above must be reproduced on all authorised
   7  * copies and copies may only be made to the extent permitted
   8  * by a licensing agreement from ARM Limited.
   9  */
  10
  11 #include "mali_pp.h"
  12 #include "mali_pp_job.h"
  13 #include "mali_dma.h"
  14 #include "mali_osk.h"
  15 #include "mali_osk_list.h"
  16 #include "mali_kernel_common.h"
  17 #include "mali_uk_types.h"
  18 #include "mali_pp_scheduler.h"
  19 #if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
  20 #include "linux/mali_memory_dma_buf.h"
  21 #endif
  22 #include "mali_kernel_utilization.h"
  23
  24 /// static u32 pp_counter_src0 = MALI_HW_CORE_NO_COUNTER;      /**< Performance counter 0, MALI_HW_CORE_NO_COUNTER for disabled */
  25 /// static u32 pp_counter_src1 = MALI_HW_CORE_NO_COUNTER;      /**< Performance counter 1, MALI_HW_CORE_NO_COUNTER for disabled */
  26 static u32 pp_counter_src0 = MALI_UTILIZATION_BW_CTR_SRC0;      /**< Performance counter 0, MALI_HW_CORE_NO_COUNTER for disabled */
  27 static u32 pp_counter_src1 = MALI_UTILIZATION_BW_CTR_SRC1;      /**< Performance counter 1, MALI_HW_CORE_NO_COUNTER for disabled */
  28
  29 static _mali_osk_atomic_t pp_counter_per_sub_job_count; /**< Number of values in the two arrays which is != MALI_HW_CORE_NO_COUNTER */
  30 static u32 pp_counter_per_sub_job_src0[_MALI_PP_MAX_SUB_JOBS] = { MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER };
  31 static u32 pp_counter_per_sub_job_src1[_MALI_PP_MAX_SUB_JOBS] = { MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER };
  32
  33 void mali_pp_job_initialize(void)
  34 {
  35         _mali_osk_atomic_init(&pp_counter_per_sub_job_count, 0);
  36 }
  37
  38 void mali_pp_job_terminate(void)
  39 {
  40         _mali_osk_atomic_term(&pp_counter_per_sub_job_count);
  41 }
  42
  43 struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session, _mali_uk_pp_start_job_s *uargs, u32 id)
  44 {
  45         struct mali_pp_job *job;
  46         u32 perf_counter_flag;
  47
  48         job = _mali_osk_calloc(1, sizeof(struct mali_pp_job));
  49         if (NULL != job) {
  50                 if (0 != _mali_osk_copy_from_user(&job->uargs, uargs, sizeof(_mali_uk_pp_start_job_s))) {
  51                         goto fail;
  52                 }
  53
  54                 if (job->uargs.num_cores > _MALI_PP_MAX_SUB_JOBS) {
  55                         MALI_PRINT_ERROR(("Mali PP job: Too many sub jobs specified in job object\n"));
  56                         goto fail;
  57                 }
  58
  59                 if (!mali_pp_job_use_no_notification(job)) {
  60                         job->finished_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_PP_FINISHED, sizeof(_mali_uk_pp_job_finished_s));
  61                         if (NULL == job->finished_notification) goto fail;
  62                 }
  63
  64                 perf_counter_flag = mali_pp_job_get_perf_counter_flag(job);
  65
  66                 /* case when no counters came from user space
  67                  * so pass the debugfs / DS-5 provided global ones to the job object */
  68                 if (!((perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE) ||
  69                       (perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE))) {
  70                         u32 sub_job_count = _mali_osk_atomic_read(&pp_counter_per_sub_job_count);
  71
  72                         /* These counters apply for all virtual jobs, and where no per sub job counter is specified */
  73                         job->uargs.perf_counter_src0 = pp_counter_src0;
  74                         job->uargs.perf_counter_src1 = pp_counter_src1;
  75
  76                         /* We only copy the per sub job array if it is enabled with at least one counter */
  77                         if (0 < sub_job_count) {
  78                                 job->perf_counter_per_sub_job_count = sub_job_count;
  79                                 _mali_osk_memcpy(job->perf_counter_per_sub_job_src0, pp_counter_per_sub_job_src0, sizeof(pp_counter_per_sub_job_src0));
  80                                 _mali_osk_memcpy(job->perf_counter_per_sub_job_src1, pp_counter_per_sub_job_src1, sizeof(pp_counter_per_sub_job_src1));
  81                         }
  82                 }
  83
  84                 _mali_osk_list_init(&job->list);
  85                 job->session = session;
  86                 _mali_osk_list_init(&job->session_list);
  87                 job->id = id;
  88
  89                 job->sub_jobs_num = job->uargs.num_cores ? job->uargs.num_cores : 1;
  90                 job->pid = _mali_osk_get_pid();
  91                 job->tid = _mali_osk_get_tid();
  92
  93                 job->num_memory_cookies = job->uargs.num_memory_cookies;
  94                 if (job->num_memory_cookies > 0) {
  95                         u32 size;
  96
  97                         if (job->uargs.num_memory_cookies > session->descriptor_mapping->current_nr_mappings) {
  98                                 MALI_PRINT_ERROR(("Mali PP job: Too many memory cookies specified in job object\n"));
  99                                 goto fail;
 100                         }
 101
 102                         size = sizeof(*job->uargs.memory_cookies) * job->num_memory_cookies;
 103
 104                         job->memory_cookies = _mali_osk_malloc(size);
 105                         if (NULL == job->memory_cookies) {
 106                                 MALI_PRINT_ERROR(("Mali PP job: Failed to allocate %d bytes of memory cookies!\n", size));
 107                                 goto fail;
 108                         }
 109
 110                         if (0 != _mali_osk_copy_from_user(job->memory_cookies, job->uargs.memory_cookies, size)) {
 111                                 MALI_PRINT_ERROR(("Mali PP job: Failed to copy %d bytes of memory cookies from user!\n", size));
 112                                 goto fail;
 113                         }
 114
 115 #if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
 116                         job->num_dma_bufs = job->num_memory_cookies;
 117                         job->dma_bufs = _mali_osk_calloc(job->num_dma_bufs, sizeof(struct mali_dma_buf_attachment *));
 118                         if (NULL == job->dma_bufs) {
 119                                 MALI_PRINT_ERROR(("Mali PP job: Failed to allocate dma_bufs array!\n"));
 120                                 goto fail;
 121                         }
 122 #endif
 123                 }
 124
 125                 /* Prepare DMA command buffer to start job, if it is virtual. */
 126                 if (mali_pp_job_is_virtual(job)) {
 127                         struct mali_pp_core *core;
 128                         _mali_osk_errcode_t err =  mali_dma_get_cmd_buf(&job->dma_cmd_buf);
 129
 130                         if (_MALI_OSK_ERR_OK != err) {
 131                                 MALI_PRINT_ERROR(("Mali PP job: Failed to allocate DMA command buffer\n"));
 132                                 goto fail;
 133                         }
 134
 135                         core = mali_pp_scheduler_get_virtual_pp();
 136                         MALI_DEBUG_ASSERT_POINTER(core);
 137
 138                         mali_pp_job_dma_cmd_prepare(core, job, 0, MALI_FALSE, &job->dma_cmd_buf);
 139                 }
 140
 141                 if (_MALI_OSK_ERR_OK != mali_pp_job_check(job)) {
 142                         /* Not a valid job. */
 143                         goto fail;
 144                 }
 145
 146                 mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_PP, NULL, job);
 147                 mali_timeline_fence_copy_uk_fence(&(job->tracker.fence), &(job->uargs.fence));
 148
 149                 return job;
 150         }
 151
 152 fail:
 153         if (NULL != job) {
 154                 mali_pp_job_delete(job);
 155         }
 156
 157         return NULL;
 158 }
 159
 160 void mali_pp_job_delete(struct mali_pp_job *job)
 161 {
 162         mali_dma_put_cmd_buf(&job->dma_cmd_buf);
 163         if (NULL != job->finished_notification) {
 164                 _mali_osk_notification_delete(job->finished_notification);
 165         }
 166
 167         _mali_osk_free(job->memory_cookies);
 168
 169 #if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
 170         /* Unmap buffers attached to job */
 171         if (0 < job->num_dma_bufs) {
 172                 mali_dma_buf_unmap_job(job);
 173         }
 174
 175         _mali_osk_free(job->dma_bufs);
 176 #endif /* CONFIG_DMA_SHARED_BUFFER */
 177
 178         _mali_osk_free(job);
 179 }
 180
 181 u32 mali_pp_job_get_perf_counter_src0(struct mali_pp_job *job, u32 sub_job)
 182 {
 183         /* Virtual jobs always use the global job counter (or if there are per sub job counters at all) */
 184         if (mali_pp_job_is_virtual(job) || 0 == job->perf_counter_per_sub_job_count) {
 185                 return job->uargs.perf_counter_src0;
 186         }
 187
 188         /* Use per sub job counter if enabled... */
 189         if (MALI_HW_CORE_NO_COUNTER != job->perf_counter_per_sub_job_src0[sub_job]) {
 190                 return job->perf_counter_per_sub_job_src0[sub_job];
 191         }
 192
 193         /* ...else default to global job counter */
 194         return job->uargs.perf_counter_src0;
 195 }
 196
 197 u32 mali_pp_job_get_perf_counter_src1(struct mali_pp_job *job, u32 sub_job)
 198 {
 199         /* Virtual jobs always use the global job counter (or if there are per sub job counters at all) */
 200         if (mali_pp_job_is_virtual(job) || 0 == job->perf_counter_per_sub_job_count) {
 201                 /* Virtual jobs always use the global job counter */
 202                 return job->uargs.perf_counter_src1;
 203         }
 204
 205         /* Use per sub job counter if enabled... */
 206         if (MALI_HW_CORE_NO_COUNTER != job->perf_counter_per_sub_job_src1[sub_job]) {
 207                 return job->perf_counter_per_sub_job_src1[sub_job];
 208         }
 209
 210         /* ...else default to global job counter */
 211         return job->uargs.perf_counter_src1;
 212 }
 213
 214 void mali_pp_job_set_pp_counter_global_src0(u32 counter)
 215 {
 216         if (MALI_HW_CORE_NO_COUNTER == counter)
 217         {
 218                 counter = MALI_UTILIZATION_BW_CTR_SRC0;
 219         }
 220         pp_counter_src0 = counter;
 221 }
 222
 223 void mali_pp_job_set_pp_counter_global_src1(u32 counter)
 224 {
 225         if (MALI_HW_CORE_NO_COUNTER == counter)
 226         {
 227                 counter = MALI_UTILIZATION_BW_CTR_SRC1;
 228         }
 229         pp_counter_src1 = counter;
 230 }
 231
 232 void mali_pp_job_set_pp_counter_sub_job_src0(u32 sub_job, u32 counter)
 233 {
 234         MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS);
 235
 236         if (MALI_HW_CORE_NO_COUNTER == pp_counter_per_sub_job_src0[sub_job]) {
 237                 /* increment count since existing counter was disabled */
 238                 _mali_osk_atomic_inc(&pp_counter_per_sub_job_count);
 239         }
 240
 241         if (MALI_HW_CORE_NO_COUNTER == counter) {
 242                 /* decrement count since new counter is disabled */
 243                 _mali_osk_atomic_dec(&pp_counter_per_sub_job_count);
 244         }
 245
 246         /* PS: A change from MALI_HW_CORE_NO_COUNTER to MALI_HW_CORE_NO_COUNTER will inc and dec, result will be 0 change */
 247         if (MALI_HW_CORE_NO_COUNTER == counter)
 248         {
 249                 counter = MALI_UTILIZATION_BW_CTR_SRC0;
 250         }
 251         pp_counter_per_sub_job_src0[sub_job] = counter;
 252 }
 253
 254 void mali_pp_job_set_pp_counter_sub_job_src1(u32 sub_job, u32 counter)
 255 {
 256         MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS);
 257
 258         if (MALI_HW_CORE_NO_COUNTER == pp_counter_per_sub_job_src1[sub_job]) {
 259                 /* increment count since existing counter was disabled */
 260                 _mali_osk_atomic_inc(&pp_counter_per_sub_job_count);
 261         }
 262
 263         if (MALI_HW_CORE_NO_COUNTER == counter) {
 264                 /* decrement count since new counter is disabled */
 265                 _mali_osk_atomic_dec(&pp_counter_per_sub_job_count);
 266         }
 267
 268         /* PS: A change from MALI_HW_CORE_NO_COUNTER to MALI_HW_CORE_NO_COUNTER will inc and dec, result will be 0 change */
 269         if (MALI_HW_CORE_NO_COUNTER == counter)
 270         {
 271                 counter = MALI_UTILIZATION_BW_CTR_SRC1;
 272         }
 273         pp_counter_per_sub_job_src1[sub_job] = counter;
 274 }
 275
 276 u32 mali_pp_job_get_pp_counter_global_src0(void)
 277 {
 278         return pp_counter_src0;
 279 }
 280
 281 u32 mali_pp_job_get_pp_counter_global_src1(void)
 282 {
 283         return pp_counter_src1;
 284 }
 285
 286 u32 mali_pp_job_get_pp_counter_sub_job_src0(u32 sub_job)
 287 {
 288         MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS);
 289         return pp_counter_per_sub_job_src0[sub_job];
 290 }
 291
 292 u32 mali_pp_job_get_pp_counter_sub_job_src1(u32 sub_job)
 293 {
 294         MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS);
 295         return pp_counter_per_sub_job_src1[sub_job];
 296 }