From 8614e713b0cce8634fa8d2cd22655a453ea15938 Mon Sep 17 00:00:00 2001 From: sky zhou Date: Mon, 28 May 2018 15:01:37 +0800 Subject: [PATCH] mem: remove NORETRY flag, and fix possible deadlock. PD#168336 For low ram device, gpu may failed to alloc memory with NORETRY flag. Without NORETRY flag, allocation may be meet deadlock when OOM killer triggered. Change-Id: Ia93c16d387d524e4f9f078df1156da96800cc49a --- .../gpu/arm/midgard/mali_kbase_context.c | 2 + .../drivers/gpu/arm/midgard/mali_kbase_defs.h | 33 +- .../drivers/gpu/arm/midgard/mali_kbase_mem.c | 479 ++++++++++++++++-- .../drivers/gpu/arm/midgard/mali_kbase_mem.h | 209 +++++++- .../gpu/arm/midgard/mali_kbase_mem_pool.c | 236 ++++++++- .../drivers/gpu/arm/midgard/mali_kbase_mmu.c | 83 ++- 6 files changed, 933 insertions(+), 109 deletions(-) diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_context.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_context.c index 868442a..b29a82c 100644 --- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_context.c +++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_context.c @@ -252,6 +252,8 @@ void kbase_destroy_context(struct kbase_context *kctx) * thread. */ kbase_pm_context_active(kbdev); + kbase_mem_pool_mark_dying(&kctx->mem_pool); + kbase_jd_zap_context(kctx); #ifdef CONFIG_DEBUG_FS diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_defs.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_defs.h index 4bb3442..6672677 100644 --- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_defs.h +++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_defs.h @@ -904,19 +904,23 @@ struct kbase_pm_device_data { /** * struct kbase_mem_pool - Page based memory pool for kctx/kbdev - * @kbdev: Kbase device where memory is used - * @cur_size: Number of free pages currently in the pool (may exceed @max_size - * in some corner cases) - * @max_size: Maximum number of free pages in the pool - * @order: order = 0 refers to a pool of 4 KB pages - * order = 9 refers to a pool of 2 MB pages (2^9 * 4KB = 2 MB) - * @pool_lock: Lock protecting the pool - must be held when modifying @cur_size - * and @page_list - * @page_list: List of free pages in the pool - * @reclaim: Shrinker for kernel reclaim of free pages - * @next_pool: Pointer to next pool where pages can be allocated when this pool - * is empty. Pages will spill over to the next pool when this pool - * is full. Can be NULL if there is no next pool. + * @kbdev: Kbase device where memory is used + * @cur_size: Number of free pages currently in the pool (may exceed + * @max_size in some corner cases) + * @max_size: Maximum number of free pages in the pool + * @order: order = 0 refers to a pool of 4 KB pages + * order = 9 refers to a pool of 2 MB pages (2^9 * 4KB = 2 MB) + * @pool_lock: Lock protecting the pool - must be held when modifying + * @cur_size and @page_list + * @page_list: List of free pages in the pool + * @reclaim: Shrinker for kernel reclaim of free pages + * @next_pool: Pointer to next pool where pages can be allocated when this + * pool is empty. Pages will spill over to the next pool when + * this pool is full. Can be NULL if there is no next pool. + * @dying: true if the pool is being terminated, and any ongoing + * operations should be abandoned + * @dont_reclaim: true if the shrinker is forbidden from reclaiming memory from + * this pool, eg during a grow operation */ struct kbase_mem_pool { struct kbase_device *kbdev; @@ -928,6 +932,9 @@ struct kbase_mem_pool { struct shrinker reclaim; struct kbase_mem_pool *next_pool; + + bool dying; + bool dont_reclaim; }; /** diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.c index a0897fb..0b63132 100644 --- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.c +++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.c @@ -1497,9 +1497,8 @@ int kbase_update_region_flags(struct kbase_context *kctx, return 0; } -int kbase_alloc_phy_pages_helper( - struct kbase_mem_phy_alloc *alloc, - size_t nr_pages_requested) +int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, + size_t nr_pages_requested) { int new_page_count __maybe_unused; size_t nr_left = nr_pages_requested; @@ -1649,12 +1648,18 @@ done: alloc_failed: /* rollback needed if got one or more 2MB but failed later */ - if (nr_left != nr_pages_requested) - kbase_mem_pool_free_pages(&kctx->lp_mem_pool, - nr_pages_requested - nr_left, - alloc->pages + alloc->nents, - false, - false); + if (nr_left != nr_pages_requested) { + size_t nr_pages_to_free = nr_pages_requested - nr_left; + + alloc->nents += nr_pages_to_free; + + kbase_process_page_usage_inc(kctx, nr_pages_to_free); + kbase_atomic_add_pages(nr_pages_to_free, &kctx->used_pages); + kbase_atomic_add_pages(nr_pages_to_free, + &kctx->kbdev->memdev.used_pages); + + kbase_free_phy_pages_helper(alloc, nr_pages_to_free); + } kbase_process_page_usage_dec(kctx, nr_pages_requested); kbase_atomic_sub_pages(nr_pages_requested, &kctx->used_pages); @@ -1665,6 +1670,183 @@ invalid_request: return -ENOMEM; } +struct tagged_addr *kbase_alloc_phy_pages_helper_locked( + struct kbase_mem_phy_alloc *alloc, struct kbase_mem_pool *pool, + size_t nr_pages_requested) +{ + int new_page_count __maybe_unused; + size_t nr_left = nr_pages_requested; + int res; + struct kbase_context *kctx; + struct tagged_addr *tp; + struct tagged_addr *new_pages = NULL; + + KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); + KBASE_DEBUG_ASSERT(alloc->imported.kctx); + + lockdep_assert_held(&pool->pool_lock); + +#if !defined(CONFIG_MALI_2MB_ALLOC) + WARN_ON(pool->order); +#endif + + if (alloc->reg) { + if (nr_pages_requested > alloc->reg->nr_pages - alloc->nents) + goto invalid_request; + } + + kctx = alloc->imported.kctx; + + lockdep_assert_held(&kctx->mem_partials_lock); + + if (nr_pages_requested == 0) + goto done; /*nothing to do*/ + + new_page_count = kbase_atomic_add_pages( + nr_pages_requested, &kctx->used_pages); + kbase_atomic_add_pages(nr_pages_requested, + &kctx->kbdev->memdev.used_pages); + + /* Increase mm counters before we allocate pages so that this + * allocation is visible to the OOM killer + */ + kbase_process_page_usage_inc(kctx, nr_pages_requested); + + tp = alloc->pages + alloc->nents; + new_pages = tp; + +#ifdef CONFIG_MALI_2MB_ALLOC + if (pool->order) { + int nr_lp = nr_left / (SZ_2M / SZ_4K); + + res = kbase_mem_pool_alloc_pages_locked(pool, + nr_lp * (SZ_2M / SZ_4K), + tp); + + if (res > 0) { + nr_left -= res; + tp += res; + } + + if (nr_left) { + struct kbase_sub_alloc *sa, *temp_sa; + + list_for_each_entry_safe(sa, temp_sa, + &kctx->mem_partials, link) { + int pidx = 0; + + while (nr_left) { + pidx = find_next_zero_bit(sa->sub_pages, + SZ_2M / SZ_4K, + pidx); + bitmap_set(sa->sub_pages, pidx, 1); + *tp++ = as_tagged_tag(page_to_phys( + sa->page + pidx), + FROM_PARTIAL); + nr_left--; + + if (bitmap_full(sa->sub_pages, + SZ_2M / SZ_4K)) { + /* unlink from partial list when + * full + */ + list_del_init(&sa->link); + break; + } + } + } + } + + /* only if we actually have a chunk left <512. If more it + * indicates that we couldn't allocate a 2MB above, so no point + * to retry here. + */ + if (nr_left > 0 && nr_left < (SZ_2M / SZ_4K)) { + /* create a new partial and suballocate the rest from it + */ + struct page *np = NULL; + + np = kbase_mem_pool_alloc_locked(pool); + + if (np) { + int i; + struct kbase_sub_alloc *sa; + struct page *p; + + sa = kmalloc(sizeof(*sa), GFP_KERNEL); + if (!sa) { + kbase_mem_pool_free_locked(pool, np, + false); + goto alloc_failed; + } + + /* store pointers back to the control struct */ + np->lru.next = (void *)sa; + for (p = np; p < np + SZ_2M / SZ_4K; p++) + p->lru.prev = (void *)np; + INIT_LIST_HEAD(&sa->link); + bitmap_zero(sa->sub_pages, SZ_2M / SZ_4K); + sa->page = np; + + for (i = 0; i < nr_left; i++) + *tp++ = as_tagged_tag( + page_to_phys(np + i), + FROM_PARTIAL); + + bitmap_set(sa->sub_pages, 0, nr_left); + nr_left = 0; + + /* expose for later use */ + list_add(&sa->link, &kctx->mem_partials); + } + } + if (nr_left) + goto alloc_failed; + } else { +#endif + res = kbase_mem_pool_alloc_pages_locked(pool, + nr_left, + tp); + if (res <= 0) + goto alloc_failed; + + nr_left = 0; +#ifdef CONFIG_MALI_2MB_ALLOC + } +#endif + + KBASE_TLSTREAM_AUX_PAGESALLOC( + kctx->id, + (u64)new_page_count); + + alloc->nents += nr_pages_requested; +done: + return new_pages; + +alloc_failed: + /* rollback needed if got one or more 2MB but failed later */ + if (nr_left != nr_pages_requested) { + size_t nr_pages_to_free = nr_pages_requested - nr_left; + + alloc->nents += nr_pages_to_free; + + kbase_process_page_usage_inc(kctx, nr_pages_to_free); + kbase_atomic_add_pages(nr_pages_to_free, &kctx->used_pages); + kbase_atomic_add_pages(nr_pages_to_free, + &kctx->kbdev->memdev.used_pages); + + kbase_free_phy_pages_helper(alloc, nr_pages_to_free); + } + + kbase_process_page_usage_dec(kctx, nr_pages_requested); + kbase_atomic_sub_pages(nr_pages_requested, &kctx->used_pages); + kbase_atomic_sub_pages(nr_pages_requested, + &kctx->kbdev->memdev.used_pages); + +invalid_request: + return NULL; +} + static void free_partial(struct kbase_context *kctx, struct tagged_addr tp) { struct page *p, *head_page; @@ -1776,6 +1958,124 @@ int kbase_free_phy_pages_helper( return 0; } +static void free_partial_locked(struct kbase_context *kctx, + struct kbase_mem_pool *pool, struct tagged_addr tp) +{ + struct page *p, *head_page; + struct kbase_sub_alloc *sa; + + lockdep_assert_held(&pool->pool_lock); + lockdep_assert_held(&kctx->mem_partials_lock); + + p = phys_to_page(as_phys_addr_t(tp)); + head_page = (struct page *)p->lru.prev; + sa = (struct kbase_sub_alloc *)head_page->lru.next; + clear_bit(p - head_page, sa->sub_pages); + if (bitmap_empty(sa->sub_pages, SZ_2M / SZ_4K)) { + list_del(&sa->link); + kbase_mem_pool_free(pool, head_page, true); + kfree(sa); + } else if (bitmap_weight(sa->sub_pages, SZ_2M / SZ_4K) == + SZ_2M / SZ_4K - 1) { + /* expose the partial again */ + list_add(&sa->link, &kctx->mem_partials); + } +} + +void kbase_free_phy_pages_helper_locked(struct kbase_mem_phy_alloc *alloc, + struct kbase_mem_pool *pool, struct tagged_addr *pages, + size_t nr_pages_to_free) +{ + struct kbase_context *kctx = alloc->imported.kctx; + bool syncback; + bool reclaimed = (alloc->evicted != 0); + struct tagged_addr *start_free; + size_t freed = 0; + + KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); + KBASE_DEBUG_ASSERT(alloc->imported.kctx); + KBASE_DEBUG_ASSERT(alloc->nents >= nr_pages_to_free); + + lockdep_assert_held(&pool->pool_lock); + lockdep_assert_held(&kctx->mem_partials_lock); + + /* early out if nothing to do */ + if (!nr_pages_to_free) + return; + + start_free = pages; + + syncback = alloc->properties & KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; + + /* pad start_free to a valid start location */ + while (nr_pages_to_free && is_huge(*start_free) && + !is_huge_head(*start_free)) { + nr_pages_to_free--; + start_free++; + } + + while (nr_pages_to_free) { + if (is_huge_head(*start_free)) { + /* This is a 2MB entry, so free all the 512 pages that + * it points to + */ + WARN_ON(!pool->order); + kbase_mem_pool_free_pages_locked(pool, + 512, + start_free, + syncback, + reclaimed); + nr_pages_to_free -= 512; + start_free += 512; + freed += 512; + } else if (is_partial(*start_free)) { + WARN_ON(!pool->order); + free_partial_locked(kctx, pool, *start_free); + nr_pages_to_free--; + start_free++; + freed++; + } else { + struct tagged_addr *local_end_free; + + WARN_ON(pool->order); + local_end_free = start_free; + while (nr_pages_to_free && + !is_huge(*local_end_free) && + !is_partial(*local_end_free)) { + local_end_free++; + nr_pages_to_free--; + } + kbase_mem_pool_free_pages_locked(pool, + local_end_free - start_free, + start_free, + syncback, + reclaimed); + freed += local_end_free - start_free; + start_free += local_end_free - start_free; + } + } + + alloc->nents -= freed; + + /* + * If the allocation was not evicted (i.e. evicted == 0) then + * the page accounting needs to be done. + */ + if (!reclaimed) { + int new_page_count; + + kbase_process_page_usage_dec(kctx, freed); + new_page_count = kbase_atomic_sub_pages(freed, + &kctx->used_pages); + kbase_atomic_sub_pages(freed, + &kctx->kbdev->memdev.used_pages); + + KBASE_TLSTREAM_AUX_PAGESALLOC( + kctx->id, + (u64)new_page_count); + } +} + void kbase_mem_kref_free(struct kref *kref) { struct kbase_mem_phy_alloc *alloc; @@ -2295,6 +2595,117 @@ int kbase_jit_init(struct kbase_context *kctx) return 0; } +static int kbase_jit_grow(struct kbase_context *kctx, + struct base_jit_alloc_info *info, struct kbase_va_region *reg) +{ + size_t delta; + size_t pages_required; + size_t old_size; + struct kbase_mem_pool *pool; + int ret = -ENOMEM; + struct tagged_addr *gpu_pages; + + kbase_gpu_vm_lock(kctx); + + /* Make the physical backing no longer reclaimable */ + if (!kbase_mem_evictable_unmake(reg->gpu_alloc)) + goto update_failed; + + if (reg->gpu_alloc->nents >= info->commit_pages) + goto done; + + /* Grow the backing */ + old_size = reg->gpu_alloc->nents; + + /* Allocate some more pages */ + delta = info->commit_pages - reg->gpu_alloc->nents; + pages_required = delta; + +#ifdef CONFIG_MALI_2MB_ALLOC + if (pages_required >= (SZ_2M / SZ_4K)) { + pool = &kctx->lp_mem_pool; + /* Round up to number of 2 MB pages required */ + pages_required += ((SZ_2M / SZ_4K) - 1); + pages_required /= (SZ_2M / SZ_4K); + } else { +#endif + pool = &kctx->mem_pool; +#ifdef CONFIG_MALI_2MB_ALLOC + } +#endif + + if (reg->cpu_alloc != reg->gpu_alloc) + pages_required *= 2; + + mutex_lock(&kctx->mem_partials_lock); + kbase_mem_pool_lock(pool); + + /* As we can not allocate memory from the kernel with the vm_lock held, + * grow the pool to the required size with the lock dropped. We hold the + * pool lock to prevent another thread from allocating from the pool + * between the grow and allocation. + */ + while (kbase_mem_pool_size(pool) < pages_required) { + int pool_delta = pages_required - kbase_mem_pool_size(pool); + + kbase_mem_pool_unlock(pool); + mutex_unlock(&kctx->mem_partials_lock); + kbase_gpu_vm_unlock(kctx); + + if (kbase_mem_pool_grow(pool, pool_delta)) + goto update_failed_unlocked; + + kbase_gpu_vm_lock(kctx); + mutex_lock(&kctx->mem_partials_lock); + kbase_mem_pool_lock(pool); + } + + gpu_pages = kbase_alloc_phy_pages_helper_locked(reg->gpu_alloc, pool, + delta); + if (!gpu_pages) { + kbase_mem_pool_unlock(pool); + mutex_unlock(&kctx->mem_partials_lock); + goto update_failed; + } + + if (reg->cpu_alloc != reg->gpu_alloc) { + struct tagged_addr *cpu_pages; + + cpu_pages = kbase_alloc_phy_pages_helper_locked(reg->cpu_alloc, + pool, delta); + if (!cpu_pages) { + kbase_free_phy_pages_helper_locked(reg->gpu_alloc, + pool, gpu_pages, delta); + kbase_mem_pool_unlock(pool); + mutex_unlock(&kctx->mem_partials_lock); + goto update_failed; + } + } + kbase_mem_pool_unlock(pool); + mutex_unlock(&kctx->mem_partials_lock); + + ret = kbase_mem_grow_gpu_mapping(kctx, reg, info->commit_pages, + old_size); + /* + * The grow failed so put the allocation back in the + * pool and return failure. + */ + if (ret) + goto update_failed; + +done: + ret = 0; + + /* Update attributes of JIT allocation taken from the pool */ + reg->initial_commit = info->commit_pages; + reg->extent = info->extent; + +update_failed: + kbase_gpu_vm_unlock(kctx); +update_failed_unlocked: + return ret; +} + struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, struct base_jit_alloc_info *info) { @@ -2303,9 +2714,8 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, struct kbase_va_region *temp; size_t current_diff = SIZE_MAX; - int ret; - mutex_lock(&kctx->jit_evict_lock); + /* * Scan the pool for an existing allocation which meets our * requirements and remove it. @@ -2352,42 +2762,15 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, list_del_init(®->gpu_alloc->evict_node); mutex_unlock(&kctx->jit_evict_lock); - kbase_gpu_vm_lock(kctx); - - /* Make the physical backing no longer reclaimable */ - if (!kbase_mem_evictable_unmake(reg->gpu_alloc)) - goto update_failed; - - /* Grow the backing if required */ - if (reg->gpu_alloc->nents < info->commit_pages) { - size_t delta; - size_t old_size = reg->gpu_alloc->nents; - - /* Allocate some more pages */ - delta = info->commit_pages - reg->gpu_alloc->nents; - if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, delta) - != 0) - goto update_failed; - - if (reg->cpu_alloc != reg->gpu_alloc) { - if (kbase_alloc_phy_pages_helper( - reg->cpu_alloc, delta) != 0) { - kbase_free_phy_pages_helper( - reg->gpu_alloc, delta); - goto update_failed; - } - } - - ret = kbase_mem_grow_gpu_mapping(kctx, reg, - info->commit_pages, old_size); + if (kbase_jit_grow(kctx, info, reg) < 0) { /* - * The grow failed so put the allocation back in the - * pool and return failure. + * An update to an allocation from the pool failed, + * chances are slim a new allocation would fair any + * better so return the allocation to the pool and + * return the function with failure. */ - if (ret) - goto update_failed; + goto update_failed_unlocked; } - kbase_gpu_vm_unlock(kctx); } else { /* No suitable JIT allocation was found so create a new one */ u64 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD | @@ -2411,13 +2794,7 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, return reg; -update_failed: - /* - * An update to an allocation from the pool failed, chances - * are slim a new allocation would fair any better so return - * the allocation to the pool and return the function with failure. - */ - kbase_gpu_vm_unlock(kctx); +update_failed_unlocked: mutex_lock(&kctx->jit_evict_lock); list_move(®->jit_node, &kctx->jit_pool_head); mutex_unlock(&kctx->jit_evict_lock); diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.h index 36de381..0dfa608 100644 --- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.h +++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.h @@ -539,9 +539,25 @@ void kbase_mem_pool_term(struct kbase_mem_pool *pool); * 3. Return NULL if no memory in the pool * * Return: Pointer to allocated page, or NULL if allocation failed. + * + * Note : This function should not be used if the pool lock is held. Use + * kbase_mem_pool_alloc_locked() instead. */ struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool); +/** + * kbase_mem_pool_alloc_locked - Allocate a page from memory pool + * @pool: Memory pool to allocate from + * + * If there are free pages in the pool, this function allocates a page from + * @pool. This function does not use @next_pool. + * + * Return: Pointer to allocated page, or NULL if allocation failed. + * + * Note : Caller must hold the pool lock. + */ +struct page *kbase_mem_pool_alloc_locked(struct kbase_mem_pool *pool); + /** * kbase_mem_pool_free - Free a page to memory pool * @pool: Memory pool where page should be freed @@ -553,10 +569,27 @@ struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool); * 2. Otherwise, if @next_pool is not NULL and not full, add @page to * @next_pool. * 3. Finally, free @page to the kernel. + * + * Note : This function should not be used if the pool lock is held. Use + * kbase_mem_pool_free_locked() instead. */ void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *page, bool dirty); +/** + * kbase_mem_pool_free_locked - Free a page to memory pool + * @pool: Memory pool where page should be freed + * @p: Page to free to the pool + * @dirty: Whether some of the page may be dirty in the cache. + * + * If @pool is not full, this function adds @page to @pool. Otherwise, @page is + * freed to the kernel. This function does not use @next_pool. + * + * Note : Caller must hold the pool lock. + */ +void kbase_mem_pool_free_locked(struct kbase_mem_pool *pool, struct page *p, + bool dirty); + /** * kbase_mem_pool_alloc_pages - Allocate pages from memory pool * @pool: Memory pool to allocate from @@ -571,10 +604,57 @@ void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *page, * On success number of pages allocated (could be less than nr_pages if * partial_allowed). * On error an error code. + * + * Note : This function should not be used if the pool lock is held. Use + * kbase_mem_pool_alloc_pages_locked() instead. + * + * The caller must not hold vm_lock, as this could cause a deadlock if + * the kernel OoM killer runs. If the caller must allocate pages while holding + * this lock, it should use kbase_mem_pool_alloc_pages_locked() instead. */ int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_pages, struct tagged_addr *pages, bool partial_allowed); +/** + * kbase_mem_pool_alloc_pages_locked - Allocate pages from memory pool + * @pool: Memory pool to allocate from + * @nr_4k_pages: Number of pages to allocate + * @pages: Pointer to array where the physical address of the allocated + * pages will be stored. + * + * Like kbase_mem_pool_alloc() but optimized for allocating many pages. This + * version does not allocate new pages from the kernel, and therefore will never + * trigger the OoM killer. Therefore, it can be run while the vm_lock is held. + * + * As new pages can not be allocated, the caller must ensure there are + * sufficient pages in the pool. Usage of this function should look like : + * + * kbase_gpu_vm_lock(kctx); + * kbase_mem_pool_lock(pool) + * while (kbase_mem_pool_size(pool) < pages_required) { + * kbase_mem_pool_unlock(pool) + * kbase_gpu_vm_unlock(kctx); + * kbase_mem_pool_grow(pool) + * kbase_gpu_vm_lock(kctx); + * kbase_mem_pool_lock(pool) + * } + * kbase_mem_pool_alloc_pages_locked(pool) + * kbase_mem_pool_unlock(pool) + * Perform other processing that requires vm_lock... + * kbase_gpu_vm_unlock(kctx); + * + * This ensures that the pool can be grown to the required size and that the + * allocation can complete without another thread using the newly grown pages. + * + * Return: + * On success number of pages allocated. + * On error an error code. + * + * Note : Caller must hold the pool lock. + */ +int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool, + size_t nr_4k_pages, struct tagged_addr *pages); + /** * kbase_mem_pool_free_pages - Free pages to memory pool * @pool: Memory pool where pages should be freed @@ -590,6 +670,22 @@ int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_pages, void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, struct tagged_addr *pages, bool dirty, bool reclaimed); +/** + * kbase_mem_pool_free_pages_locked - Free pages to memory pool + * @pool: Memory pool where pages should be freed + * @nr_pages: Number of pages to free + * @pages: Pointer to array holding the physical addresses of the pages to + * free. + * @dirty: Whether any pages may be dirty in the cache. + * @reclaimed: Whether the pages where reclaimable and thus should bypass + * the pool and go straight to the kernel. + * + * Like kbase_mem_pool_free() but optimized for freeing many pages. + */ +void kbase_mem_pool_free_pages_locked(struct kbase_mem_pool *pool, + size_t nr_pages, struct tagged_addr *pages, bool dirty, + bool reclaimed); + /** * kbase_mem_pool_size - Get number of free pages in memory pool * @pool: Memory pool to inspect @@ -648,6 +744,15 @@ int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow); */ void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size); +/** + * kbase_mem_pool_mark_dying - Mark that this pool is dying + * @pool: Memory pool + * + * This will cause any ongoing allocation operations (eg growing on page fault) + * to be terminated. + */ +void kbase_mem_pool_mark_dying(struct kbase_mem_pool *pool); + /** * kbase_mem_alloc_page - Allocate a new page for a device * @pool: Memory pool to allocate a page from @@ -922,16 +1027,62 @@ void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_ void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); /** -* @brief Allocates physical pages. -* -* Allocates \a nr_pages_requested and updates the alloc object. -* -* @param[in] alloc allocation object to add pages to -* @param[in] nr_pages_requested number of physical pages to allocate -* -* @return 0 if all pages have been successfully allocated. Error code otherwise -*/ -int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_requested); + * kbase_alloc_phy_pages_helper - Allocates physical pages. + * @alloc: allocation object to add pages to + * @nr_pages_requested: number of physical pages to allocate + * + * Allocates \a nr_pages_requested and updates the alloc object. + * + * Return: 0 if all pages have been successfully allocated. Error code otherwise + * + * Note : The caller must not hold vm_lock, as this could cause a deadlock if + * the kernel OoM killer runs. If the caller must allocate pages while holding + * this lock, it should use kbase_mem_pool_alloc_pages_locked() instead. + */ +int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, + size_t nr_pages_requested); + +/** + * kbase_alloc_phy_pages_helper_locked - Allocates physical pages. + * @alloc: allocation object to add pages to + * @pool: Memory pool to allocate from + * @nr_pages_requested: number of physical pages to allocate + * + * Allocates \a nr_pages_requested and updates the alloc object. This function + * does not allocate new pages from the kernel, and therefore will never trigger + * the OoM killer. Therefore, it can be run while the vm_lock is held. + * + * As new pages can not be allocated, the caller must ensure there are + * sufficient pages in the pool. Usage of this function should look like : + * + * kbase_gpu_vm_lock(kctx); + * kbase_mem_pool_lock(pool) + * while (kbase_mem_pool_size(pool) < pages_required) { + * kbase_mem_pool_unlock(pool) + * kbase_gpu_vm_unlock(kctx); + * kbase_mem_pool_grow(pool) + * kbase_gpu_vm_lock(kctx); + * kbase_mem_pool_lock(pool) + * } + * kbase_alloc_phy_pages_helper_locked(pool) + * kbase_mem_pool_unlock(pool) + * Perform other processing that requires vm_lock... + * kbase_gpu_vm_unlock(kctx); + * + * This ensures that the pool can be grown to the required size and that the + * allocation can complete without another thread using the newly grown pages. + * + * If CONFIG_MALI_2MB_ALLOC is defined and the allocation is >= 2MB, then + * @pool must be alloc->imported.kctx->lp_mem_pool. Otherwise it must be + * alloc->imported.kctx->mem_pool. + * + * Return: Pointer to array of allocated pages. NULL on failure. + * + * Note : Caller must hold pool->pool_lock + */ +struct tagged_addr *kbase_alloc_phy_pages_helper_locked( + struct kbase_mem_phy_alloc *alloc, struct kbase_mem_pool *pool, + size_t nr_pages_requested); /** * @brief Free physical pages. @@ -943,6 +1094,26 @@ int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pa */ int kbase_free_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_to_free); +/** + * kbase_free_phy_pages_helper_locked - Free pages allocated with + * kbase_alloc_phy_pages_helper_locked() + * @alloc: Allocation object to free pages from + * @pool: Memory pool to return freed pages to + * @pages: Pages allocated by kbase_alloc_phy_pages_helper_locked() + * @nr_pages_to_free: Number of physical pages to free + * + * This function atomically frees pages allocated with + * kbase_alloc_phy_pages_helper_locked(). @pages is the pointer to the page + * array that is returned by that function. @pool must be the pool that the + * pages were originally allocated from. + * + * If the mem_pool has been unlocked since the allocation then + * kbase_free_phy_pages_helper() should be used instead. + */ +void kbase_free_phy_pages_helper_locked(struct kbase_mem_phy_alloc *alloc, + struct kbase_mem_pool *pool, struct tagged_addr *pages, + size_t nr_pages_to_free); + static inline void kbase_set_dma_addr(struct page *p, dma_addr_t dma_addr) { SetPagePrivate(p); @@ -1150,4 +1321,22 @@ bool kbase_sticky_resource_release(struct kbase_context *kctx, */ void kbase_sticky_resource_term(struct kbase_context *kctx); +/** + * kbase_mem_pool_lock - Lock a memory pool + * @pool: Memory pool to lock + */ +static inline void kbase_mem_pool_lock(struct kbase_mem_pool *pool) +{ + spin_lock(&pool->pool_lock); +} + +/** + * kbase_mem_pool_lock - Release a memory pool + * @pool: Memory pool to lock + */ +static inline void kbase_mem_pool_unlock(struct kbase_mem_pool *pool) +{ + spin_unlock(&pool->pool_lock); +} + #endif /* _KBASE_MEM_H_ */ diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c index 574f1d5..1255df0 100644 --- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c +++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c @@ -39,16 +39,6 @@ #define NOT_DIRTY false #define NOT_RECLAIMED false -static inline void kbase_mem_pool_lock(struct kbase_mem_pool *pool) -{ - spin_lock(&pool->pool_lock); -} - -static inline void kbase_mem_pool_unlock(struct kbase_mem_pool *pool) -{ - spin_unlock(&pool->pool_lock); -} - static size_t kbase_mem_pool_capacity(struct kbase_mem_pool *pool) { ssize_t max_size = kbase_mem_pool_max_size(pool); @@ -177,12 +167,6 @@ struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool) gfp = GFP_HIGHUSER | __GFP_ZERO; #endif - if (current->flags & PF_KTHREAD) { - /* Don't trigger OOM killer from kernel threads, e.g. when - * growing memory on GPU page fault */ - gfp |= __GFP_NORETRY; - } - /* don't warn on higer order failures */ if (pool->order) gfp |= __GFP_NOWARN; @@ -255,12 +239,33 @@ int kbase_mem_pool_grow(struct kbase_mem_pool *pool, struct page *p; size_t i; + kbase_mem_pool_lock(pool); + + pool->dont_reclaim = true; for (i = 0; i < nr_to_grow; i++) { + if (pool->dying) { + pool->dont_reclaim = false; + kbase_mem_pool_shrink_locked(pool, nr_to_grow); + kbase_mem_pool_unlock(pool); + + return -ENOMEM; + } + kbase_mem_pool_unlock(pool); + p = kbase_mem_alloc_page(pool); - if (!p) + if (!p) { + kbase_mem_pool_lock(pool); + pool->dont_reclaim = false; + kbase_mem_pool_unlock(pool); + return -ENOMEM; - kbase_mem_pool_add(pool, p); + } + + kbase_mem_pool_lock(pool); + kbase_mem_pool_add_locked(pool, p); } + pool->dont_reclaim = false; + kbase_mem_pool_unlock(pool); return 0; } @@ -312,10 +317,19 @@ static unsigned long kbase_mem_pool_reclaim_count_objects(struct shrinker *s, struct shrink_control *sc) { struct kbase_mem_pool *pool; + size_t pool_size; pool = container_of(s, struct kbase_mem_pool, reclaim); - pool_dbg(pool, "reclaim count: %zu\n", kbase_mem_pool_size(pool)); - return kbase_mem_pool_size(pool); + + kbase_mem_pool_lock(pool); + if (pool->dont_reclaim && !pool->dying) { + kbase_mem_pool_unlock(pool); + return 0; + } + pool_size = kbase_mem_pool_size(pool); + kbase_mem_pool_unlock(pool); + + return pool_size; } static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s, @@ -326,9 +340,17 @@ static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s, pool = container_of(s, struct kbase_mem_pool, reclaim); + kbase_mem_pool_lock(pool); + if (pool->dont_reclaim && !pool->dying) { + kbase_mem_pool_unlock(pool); + return 0; + } + pool_dbg(pool, "reclaim scan %ld:\n", sc->nr_to_scan); - freed = kbase_mem_pool_shrink(pool, sc->nr_to_scan); + freed = kbase_mem_pool_shrink_locked(pool, sc->nr_to_scan); + + kbase_mem_pool_unlock(pool); pool_dbg(pool, "reclaim freed %ld pages\n", freed); @@ -357,6 +379,7 @@ int kbase_mem_pool_init(struct kbase_mem_pool *pool, pool->order = order; pool->kbdev = kbdev; pool->next_pool = next_pool; + pool->dying = false; spin_lock_init(&pool->pool_lock); INIT_LIST_HEAD(&pool->page_list); @@ -381,6 +404,13 @@ int kbase_mem_pool_init(struct kbase_mem_pool *pool, return 0; } +void kbase_mem_pool_mark_dying(struct kbase_mem_pool *pool) +{ + kbase_mem_pool_lock(pool); + pool->dying = true; + kbase_mem_pool_unlock(pool); +} + void kbase_mem_pool_term(struct kbase_mem_pool *pool) { struct kbase_mem_pool *next_pool = pool->next_pool; @@ -444,6 +474,21 @@ struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool) return NULL; } +struct page *kbase_mem_pool_alloc_locked(struct kbase_mem_pool *pool) +{ + struct page *p; + + lockdep_assert_held(&pool->pool_lock); + + pool_dbg(pool, "alloc_locked()\n"); + p = kbase_mem_pool_remove_locked(pool); + + if (p) + return p; + + return NULL; +} + void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *p, bool dirty) { @@ -466,6 +511,25 @@ void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *p, } } +void kbase_mem_pool_free_locked(struct kbase_mem_pool *pool, struct page *p, + bool dirty) +{ + pool_dbg(pool, "free_locked()\n"); + + lockdep_assert_held(&pool->pool_lock); + + if (!kbase_mem_pool_is_full(pool)) { + /* Add to our own pool */ + if (dirty) + kbase_mem_pool_sync_page(pool, p); + + kbase_mem_pool_add_locked(pool, p); + } else { + /* Free page */ + kbase_mem_pool_free_page(pool, p); + } +} + int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages, struct tagged_addr *pages, bool partial_allowed) { @@ -543,7 +607,6 @@ int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages, done: pool_dbg(pool, "alloc_pages(%zu) done\n", i); - return i; err_rollback: @@ -551,6 +614,49 @@ err_rollback: return err; } +int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool, + size_t nr_4k_pages, struct tagged_addr *pages) +{ + struct page *p; + size_t i; + size_t nr_pages_internal; + + lockdep_assert_held(&pool->pool_lock); + + nr_pages_internal = nr_4k_pages / (1u << (pool->order)); + + if (nr_pages_internal * (1u << pool->order) != nr_4k_pages) + return -EINVAL; + + pool_dbg(pool, "alloc_pages_locked(4k=%zu):\n", nr_4k_pages); + pool_dbg(pool, "alloc_pages_locked(internal=%zu):\n", + nr_pages_internal); + + if (kbase_mem_pool_size(pool) < nr_pages_internal) { + pool_dbg(pool, "Failed alloc\n"); + return -ENOMEM; + } + + for (i = 0; i < nr_pages_internal; i++) { + int j; + + p = kbase_mem_pool_remove_locked(pool); + if (pool->order) { + *pages++ = as_tagged_tag(page_to_phys(p), + HUGE_HEAD | HUGE_PAGE); + for (j = 1; j < (1u << pool->order); j++) { + *pages++ = as_tagged_tag(page_to_phys(p) + + PAGE_SIZE * j, + HUGE_PAGE); + } + } else { + *pages++ = as_tagged(page_to_phys(p)); + } + } + + return nr_4k_pages; +} + static void kbase_mem_pool_add_array(struct kbase_mem_pool *pool, size_t nr_pages, struct tagged_addr *pages, bool zero, bool sync) @@ -591,6 +697,48 @@ static void kbase_mem_pool_add_array(struct kbase_mem_pool *pool, nr_pages, nr_to_pool); } +static void kbase_mem_pool_add_array_locked(struct kbase_mem_pool *pool, + size_t nr_pages, struct tagged_addr *pages, + bool zero, bool sync) +{ + struct page *p; + size_t nr_to_pool = 0; + LIST_HEAD(new_page_list); + size_t i; + + lockdep_assert_held(&pool->pool_lock); + + if (!nr_pages) + return; + + pool_dbg(pool, "add_array_locked(%zu, zero=%d, sync=%d):\n", + nr_pages, zero, sync); + + /* Zero/sync pages first */ + for (i = 0; i < nr_pages; i++) { + if (unlikely(!as_phys_addr_t(pages[i]))) + continue; + + if (is_huge_head(pages[i]) || !is_huge(pages[i])) { + p = phys_to_page(as_phys_addr_t(pages[i])); + if (zero) + kbase_mem_pool_zero_page(pool, p); + else if (sync) + kbase_mem_pool_sync_page(pool, p); + + list_add(&p->lru, &new_page_list); + nr_to_pool++; + } + pages[i] = as_tagged(0); + } + + /* Add new page list to pool */ + kbase_mem_pool_add_list_locked(pool, &new_page_list, nr_to_pool); + + pool_dbg(pool, "add_array_locked(%zu) added %zu pages\n", + nr_pages, nr_to_pool); +} + void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, struct tagged_addr *pages, bool dirty, bool reclaimed) { @@ -640,3 +788,47 @@ void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, pool_dbg(pool, "free_pages(%zu) done\n", nr_pages); } + + +void kbase_mem_pool_free_pages_locked(struct kbase_mem_pool *pool, + size_t nr_pages, struct tagged_addr *pages, bool dirty, + bool reclaimed) +{ + struct page *p; + size_t nr_to_pool; + LIST_HEAD(to_pool_list); + size_t i = 0; + + lockdep_assert_held(&pool->pool_lock); + + pool_dbg(pool, "free_pages_locked(%zu):\n", nr_pages); + + if (!reclaimed) { + /* Add to this pool */ + nr_to_pool = kbase_mem_pool_capacity(pool); + nr_to_pool = min(nr_pages, nr_to_pool); + + kbase_mem_pool_add_array_locked(pool, nr_pages, pages, false, + dirty); + + i += nr_to_pool; + } + + /* Free any remaining pages to kernel */ + for (; i < nr_pages; i++) { + if (unlikely(!as_phys_addr_t(pages[i]))) + continue; + + if (is_huge(pages[i]) && !is_huge_head(pages[i])) { + pages[i] = as_tagged(0); + continue; + } + + p = phys_to_page(as_phys_addr_t(pages[i])); + + kbase_mem_pool_free_page(pool, p); + pages[i] = as_tagged(0); + } + + pool_dbg(pool, "free_pages_locked(%zu) done\n", nr_pages); +} diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mmu.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mmu.c index 65b7da0..3e54f10 100644 --- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mmu.c +++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mmu.c @@ -318,6 +318,10 @@ void page_fault_worker(struct work_struct *data) struct kbase_va_region *region; int err; bool grown = false; + size_t min_pool_size; + struct kbase_mem_pool *pool; + int pages_to_grow; + struct tagged_addr *gpu_pages, *cpu_pages; faulting_as = container_of(data, struct kbase_as, work_pagefault); fault_pfn = faulting_as->fault_addr >> PAGE_SHIFT; @@ -403,6 +407,7 @@ void page_fault_worker(struct work_struct *data) goto fault_done; } +page_fault_retry: /* so we have a translation fault, let's see if it is for growable * memory */ kbase_gpu_vm_lock(kctx); @@ -496,20 +501,59 @@ void page_fault_worker(struct work_struct *data) goto fault_done; } - if (kbase_alloc_phy_pages_helper(region->gpu_alloc, new_pages) == 0) { - if (region->gpu_alloc != region->cpu_alloc) { - if (kbase_alloc_phy_pages_helper( - region->cpu_alloc, new_pages) == 0) { - grown = true; - } else { - kbase_free_phy_pages_helper(region->gpu_alloc, +#ifdef CONFIG_MALI_2MB_ALLOC + if (new_pages >= (SZ_2M / SZ_4K)) { + pool = &kctx->lp_mem_pool; + /* Round up to number of 2 MB pages required */ + min_pool_size = new_pages + ((SZ_2M / SZ_4K) - 1); + min_pool_size /= (SZ_2M / SZ_4K); + } else { +#endif + pool = &kctx->mem_pool; + min_pool_size = new_pages; +#ifdef CONFIG_MALI_2MB_ALLOC + } +#endif + + if (region->gpu_alloc != region->cpu_alloc) + min_pool_size *= 2; + + pages_to_grow = 0; + + mutex_lock(&kctx->mem_partials_lock); + kbase_mem_pool_lock(pool); + /* We can not allocate memory from the kernel with the vm_lock held, so + * check that there is enough memory in the pool. If not then calculate + * how much it has to grow by, grow the pool when the vm_lock is + * dropped, and retry the allocation. + */ + if (kbase_mem_pool_size(pool) >= min_pool_size) { + gpu_pages = kbase_alloc_phy_pages_helper_locked( + region->gpu_alloc, pool, new_pages); + + if (gpu_pages) { + if (region->gpu_alloc != region->cpu_alloc) { + cpu_pages = kbase_alloc_phy_pages_helper_locked( + region->cpu_alloc, pool, new_pages); + + if (cpu_pages) { + grown = true; + } else { + kbase_free_phy_pages_helper_locked( + region->gpu_alloc, + pool, gpu_pages, + new_pages); + } + } else { + grown = true; } - } else { - grown = true; } + } else { + pages_to_grow = min_pool_size - kbase_mem_pool_size(pool); } - + kbase_mem_pool_unlock(pool); + mutex_unlock(&kctx->mem_partials_lock); if (grown) { u64 pfn_offset; @@ -600,10 +644,23 @@ void page_fault_worker(struct work_struct *data) #endif kbase_gpu_vm_unlock(kctx); } else { - /* failed to extend, handle as a normal PF */ + int ret = -ENOMEM; + kbase_gpu_vm_unlock(kctx); - kbase_mmu_report_fault_and_kill(kctx, faulting_as, - "Page allocation failure"); + + /* If the memory pool was insufficient then grow it and retry. + * Otherwise fail the allocation. + */ + if (pages_to_grow > 0) + ret = kbase_mem_pool_grow(pool, pages_to_grow); + + if (ret < 0) { + /* failed to extend, handle as a normal PF */ + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Page allocation failure"); + } else { + goto page_fault_retry; + } } fault_done: -- 2.20.1