From: Janghyuck Kim Date: Fri, 21 Jul 2017 08:03:45 +0000 (+0900) Subject: [COMMON] iommu/exynos: separate register access file X-Git-Url: https://git.stricted.de/?a=commitdiff_plain;h=808de0ab268337a12c14b3a606ccb8cee2782fd6;p=GitHub%2FLineageOS%2Fandroid_kernel_motorola_exynos9610.git [COMMON] iommu/exynos: separate register access file H/W access codes are separated into exynos-iommu-reg.h. Change-Id: Ifcb239ba84f2545ae9174adc531248843265e56f Signed-off-by: Janghyuck Kim --- diff --git a/drivers/iommu/exynos-iommu-reg.h b/drivers/iommu/exynos-iommu-reg.h new file mode 100644 index 000000000000..85c65c5b83c8 --- /dev/null +++ b/drivers/iommu/exynos-iommu-reg.h @@ -0,0 +1,578 @@ +/* linux/drivers/iommu/exynos_iommu-reg.h + * + * Copyright (c) 2017 Samsung Electronics Co., Ltd. + * http://www.samsung.com + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include "exynos-iommu.h" +#include + +#define is_secure_info_fail(x) ((((x) >> 16) & 0xffff) == 0xdead) +static inline u32 __secure_info_read(unsigned int addr) +{ + u32 ret; + + ret = exynos_smc(SMC_DRM_SEC_SMMU_INFO, (unsigned long)addr, + 0, SEC_SMMU_SFR_READ); + if (is_secure_info_fail(ret)) + pr_err("Invalid value returned, %#x\n", ret); + + return ret; +} + +static inline void __sysmmu_tlb_invalidate_all(void __iomem *sfrbase) +{ + writel(0x1, sfrbase + REG_MMU_FLUSH); +} + +static inline void __sysmmu_tlb_invalidate(struct sysmmu_drvdata *drvdata, + dma_addr_t iova, size_t size) +{ + void * __iomem sfrbase = drvdata->sfrbase; + + __raw_writel(iova, sfrbase + REG_FLUSH_RANGE_START); + __raw_writel(size - 1 + iova, sfrbase + REG_FLUSH_RANGE_END); + writel(0x1, sfrbase + REG_MMU_FLUSH_RANGE); + SYSMMU_EVENT_LOG_TLB_INV_RANGE(SYSMMU_DRVDATA_TO_LOG(drvdata), + iova, iova + size); +} + +static inline void __sysmmu_set_ptbase(struct sysmmu_drvdata *drvdata, + phys_addr_t pfn_pgtable) +{ + void * __iomem sfrbase = drvdata->sfrbase; + + writel_relaxed(pfn_pgtable, sfrbase + REG_PT_BASE_PPN); + + __sysmmu_tlb_invalidate_all(sfrbase); + SYSMMU_EVENT_LOG_TLB_INV_ALL( + SYSMMU_DRVDATA_TO_LOG(drvdata)); +} + +static inline unsigned int dump_tlb_entry_way_type(void __iomem *sfrbase, + int idx_way, int idx_set) +{ + if (MMU_TLB_ENTRY_VALID(__raw_readl(sfrbase + REG_TLB_ATTR))) { + pr_crit("[%02d][%02d] VPN: %#010x, PPN: %#010x, ATTR: %#010x\n", + idx_way, idx_set, + __raw_readl(sfrbase + REG_TLB_VPN), + __raw_readl(sfrbase + REG_TLB_PPN), + __raw_readl(sfrbase + REG_TLB_ATTR)); + return 1; + } + return 0; +} + +static inline unsigned int dump_tlb_entry_port_type(void __iomem *sfrbase, + int idx_way, int idx_set) +{ + if (MMU_TLB_ENTRY_VALID(__raw_readl(sfrbase + REG_CAPA1_TLB_ATTR))) { + pr_crit("[%02d][%02d] VPN: %#010x, PPN: %#010x, ATTR: %#010x\n", + idx_way, idx_set, + __raw_readl(sfrbase + REG_CAPA1_TLB_VPN), + __raw_readl(sfrbase + REG_CAPA1_TLB_PPN), + __raw_readl(sfrbase + REG_CAPA1_TLB_ATTR)); + return 1; + } + return 0; +} + +#define MMU_NUM_TLB_SUBLINE 4 +static inline void dump_sysmmu_tlb_way(struct sysmmu_drvdata *drvdata) +{ + int i, j, k; + u32 capa; + unsigned int cnt; + void __iomem *sfrbase = drvdata->sfrbase; + + capa = __raw_readl(sfrbase + REG_MMU_CAPA0_V7); + + pr_crit("TLB has %d way, %d set. SBB has %d entries.\n", + MMU_CAPA_NUM_TLB_WAY(capa), + (1 << MMU_CAPA_NUM_TLB_SET(capa)), + (1 << MMU_CAPA_NUM_SBB_ENTRY(capa))); + pr_crit("------------- TLB[WAY][SET][ENTRY] -------------\n"); + for (i = 0, cnt = 0; i < MMU_CAPA_NUM_TLB_WAY(capa); i++) { + for (j = 0; j < (1 << MMU_CAPA_NUM_TLB_SET(capa)); j++) { + for (k = 0; k < MMU_NUM_TLB_SUBLINE; k++) { + __raw_writel(MMU_SET_TLB_READ_ENTRY(j, i, k), + sfrbase + REG_TLB_READ); + cnt += dump_tlb_entry_way_type(sfrbase, i, j); + } + } + } + if (!cnt) + pr_crit(">> No Valid TLB Entries\n"); + + pr_crit("--- SBB(Second-Level Page Table Base Address Buffer ---\n"); + for (i = 0, cnt = 0; i < (1 << MMU_CAPA_NUM_SBB_ENTRY(capa)); i++) { + __raw_writel(i, sfrbase + REG_SBB_READ); + if (MMU_SBB_ENTRY_VALID(__raw_readl(sfrbase + REG_SBB_VPN))) { + pr_crit("[%02d] VPN: %#010x, PPN: %#010x, ATTR: %#010x\n", + i, __raw_readl(sfrbase + REG_SBB_VPN), + __raw_readl(sfrbase + REG_SBB_LINK), + __raw_readl(sfrbase + REG_SBB_ATTR)); + cnt++; + } + } + if (!cnt) + pr_crit(">> No Valid SBB Entries\n"); +} + +static inline void dump_sysmmu_tlb_port(struct sysmmu_drvdata *drvdata) +{ + int t, i, j, k; + u32 capa0, capa1, info; + unsigned int cnt; + int num_tlb, num_port, num_sbb; + void __iomem *sfrbase = drvdata->sfrbase; + + capa0 = __raw_readl(sfrbase + REG_MMU_CAPA0_V7); + capa1 = __raw_readl(sfrbase + REG_MMU_CAPA1_V7); + + num_tlb = MMU_CAPA1_NUM_TLB(capa1); + num_port = MMU_CAPA1_NUM_PORT(capa1); + num_sbb = 1 << MMU_CAPA_NUM_SBB_ENTRY(capa0); + + pr_crit("SysMMU has %d TLBs, %d ports, %d sbb entries\n", + num_tlb, num_port, num_sbb); + + for (t = 0; t < num_tlb; t++) { + int num_set, num_way; + + info = __raw_readl(sfrbase + MMU_TLB_INFO(t)); + num_way = MMU_CAPA1_NUM_TLB_WAY(info); + num_set = MMU_CAPA1_NUM_TLB_SET(info); + + pr_crit("TLB.%d has %d way, %d set.\n", t, num_way, num_set); + pr_crit("------------- TLB[WAY][SET][ENTRY] -------------\n"); + for (i = 0, cnt = 0; i < num_way; i++) { + for (j = 0; j < num_set; j++) { + for (k = 0; k < MMU_NUM_TLB_SUBLINE; k++) { + __raw_writel(MMU_CAPA1_SET_TLB_READ_ENTRY(t, j, i, k), + sfrbase + REG_CAPA1_TLB_READ); + cnt += dump_tlb_entry_port_type(sfrbase, i, j); + } + } + } + } + if (!cnt) + pr_crit(">> No Valid TLB Entries\n"); + + pr_crit("--- SBB(Second-Level Page Table Base Address Buffer ---\n"); + for (i = 0, cnt = 0; i < num_sbb; i++) { + __raw_writel(i, sfrbase + REG_CAPA1_SBB_READ); + if (MMU_SBB_ENTRY_VALID(__raw_readl(sfrbase + REG_CAPA1_SBB_VPN))) { + pr_crit("[%02d] VPN: %#010x, PPN: %#010x, ATTR: %#010x\n", + i, __raw_readl(sfrbase + REG_CAPA1_SBB_VPN), + __raw_readl(sfrbase + REG_CAPA1_SBB_LINK), + __raw_readl(sfrbase + REG_CAPA1_SBB_ATTR)); + cnt++; + } + } + if (!cnt) + pr_crit(">> No Valid SBB Entries\n"); +} + +static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = { + "PTW ACCESS FAULT", + "PAGE FAULT", + "L1TLB MULTI-HIT FAULT", + "ACCESS FAULT", + "SECURITY FAULT", + "UNKNOWN FAULT" +}; + +static inline void dump_sysmmu_status(struct sysmmu_drvdata *drvdata) +{ + int info; + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + phys_addr_t phys; + void __iomem *sfrbase = drvdata->sfrbase; + + pgd = pgd_offset_k((unsigned long)sfrbase); + if (!pgd) { + pr_crit("Invalid virtual address %p\n", sfrbase); + return; + } + + pud = pud_offset(pgd, (unsigned long)sfrbase); + if (!pud) { + pr_crit("Invalid virtual address %p\n", sfrbase); + return; + } + + pmd = pmd_offset(pud, (unsigned long)sfrbase); + if (!pmd) { + pr_crit("Invalid virtual address %p\n", sfrbase); + return; + } + + pte = pte_offset_kernel(pmd, (unsigned long)sfrbase); + if (!pte) { + pr_crit("Invalid virtual address %p\n", sfrbase); + return; + } + + info = MMU_RAW_VER(__raw_readl(sfrbase + REG_MMU_VERSION)); + + phys = pte_pfn(*pte) << PAGE_SHIFT; + pr_crit("ADDR: %pa(VA: %p), MMU_CTRL: %#010x, PT_BASE: %#010x\n", + &phys, sfrbase, + __raw_readl(sfrbase + REG_MMU_CTRL), + __raw_readl(sfrbase + REG_PT_BASE_PPN)); + pr_crit("VERSION %d.%d.%d, MMU_CFG: %#010x, MMU_STATUS: %#010x\n", + MMU_MAJ_VER(info), MMU_MIN_VER(info), MMU_REV_VER(info), + __raw_readl(sfrbase + REG_MMU_CFG), + __raw_readl(sfrbase + REG_MMU_STATUS)); + + if (IS_TLB_WAY_TYPE(drvdata)) + dump_sysmmu_tlb_way(drvdata); + else if(IS_TLB_PORT_TYPE(drvdata)) + dump_sysmmu_tlb_port(drvdata); +} + +static inline void show_secure_fault_information(struct sysmmu_drvdata *drvdata, + int flags, unsigned long fault_addr) +{ + unsigned int info; + phys_addr_t pgtable; + int fault_id = SYSMMU_FAULT_ID(flags); + unsigned int sfrbase = drvdata->securebase; + const char *port_name = NULL; + + pgtable = __secure_info_read(sfrbase + REG_PT_BASE_PPN); + pgtable <<= PAGE_SHIFT; + + info = __secure_info_read(sfrbase + REG_FAULT_TRANS_INFO); + + of_property_read_string(drvdata->sysmmu->of_node, + "port-name", &port_name); + + pr_crit("----------------------------------------------------------\n"); + pr_crit("From [%s], SysMMU %s %s at %#010lx (page table @ %pa)\n", + port_name ? port_name : dev_name(drvdata->sysmmu), + (flags & IOMMU_FAULT_WRITE) ? "WRITE" : "READ", + sysmmu_fault_name[fault_id], fault_addr, &pgtable); + + if (fault_id == SYSMMU_FAULT_UNKNOWN) { + pr_crit("The fault is not caused by this System MMU.\n"); + pr_crit("Please check IRQ and SFR base address.\n"); + goto finish; + } + + pr_crit("AxID: %#x, AxLEN: %#x\n", info & 0xFFFF, (info >> 16) & 0xF); + + if (fault_id == SYSMMU_FAULT_PTW_ACCESS) + pr_crit("System MMU has failed to access page table\n"); + + if (!pfn_valid(pgtable >> PAGE_SHIFT)) { + pr_crit("Page table base is not in a valid memory region\n"); + } else { + sysmmu_pte_t *ent; + ent = section_entry(phys_to_virt(pgtable), fault_addr); + pr_crit("Lv1 entry: %#010x\n", *ent); + + if (lv1ent_page(ent)) { + ent = page_entry(ent, fault_addr); + pr_crit("Lv2 entry: %#010x\n", *ent); + } + } + + info = MMU_RAW_VER(__secure_info_read(sfrbase + REG_MMU_VERSION)); + + pr_crit("ADDR: %#x, MMU_CTRL: %#010x, PT_BASE: %#010x\n", + sfrbase, + __secure_info_read(sfrbase + REG_MMU_CTRL), + __secure_info_read(sfrbase + REG_PT_BASE_PPN)); + pr_crit("VERSION %d.%d.%d, MMU_CFG: %#010x, MMU_STATUS: %#010x\n", + MMU_MAJ_VER(info), MMU_MIN_VER(info), MMU_REV_VER(info), + __secure_info_read(sfrbase + REG_MMU_CFG), + __secure_info_read(sfrbase + REG_MMU_STATUS)); + +finish: + pr_crit("----------------------------------------------------------\n"); +} + +static inline void show_fault_information(struct sysmmu_drvdata *drvdata, + int flags, unsigned long fault_addr) +{ + unsigned int info; + phys_addr_t pgtable; + int fault_id = SYSMMU_FAULT_ID(flags); + const char *port_name = NULL; + + pgtable = __raw_readl(drvdata->sfrbase + REG_PT_BASE_PPN); + pgtable <<= PAGE_SHIFT; + + info = __raw_readl(drvdata->sfrbase + REG_FAULT_TRANS_INFO); + + of_property_read_string(drvdata->sysmmu->of_node, + "port-name", &port_name); + + pr_crit("----------------------------------------------------------\n"); + pr_crit("From [%s], SysMMU %s %s at %#010lx (page table @ %pa)\n", + port_name ? port_name : dev_name(drvdata->sysmmu), + (flags & IOMMU_FAULT_WRITE) ? "WRITE" : "READ", + sysmmu_fault_name[fault_id], fault_addr, &pgtable); + + if (fault_id == SYSMMU_FAULT_UNKNOWN) { + pr_crit("The fault is not caused by this System MMU.\n"); + pr_crit("Please check IRQ and SFR base address.\n"); + goto finish; + } + + pr_crit("AxID: %#x, AxLEN: %#x\n", info & 0xFFFF, (info >> 16) & 0xF); + + if (pgtable != drvdata->pgtable) + pr_crit("Page table base of driver: %pa\n", + &drvdata->pgtable); + + if (fault_id == SYSMMU_FAULT_PTW_ACCESS) + pr_crit("System MMU has failed to access page table\n"); + + if (!pfn_valid(pgtable >> PAGE_SHIFT)) { + pr_crit("Page table base is not in a valid memory region\n"); + } else { + sysmmu_pte_t *ent; + ent = section_entry(phys_to_virt(pgtable), fault_addr); + pr_crit("Lv1 entry: %#010x\n", *ent); + + if (lv1ent_page(ent)) { + ent = page_entry(ent, fault_addr); + pr_crit("Lv2 entry: %#010x\n", *ent); + } + } + + dump_sysmmu_status(drvdata); + +finish: + pr_crit("----------------------------------------------------------\n"); +} + +static inline void __sysmmu_disable_nocount(struct sysmmu_drvdata *drvdata) +{ + writel_relaxed(0, drvdata->sfrbase + REG_MMU_CFG); + writel_relaxed(CTRL_BLOCK_DISABLE, drvdata->sfrbase + REG_MMU_CTRL); + BUG_ON(readl_relaxed(drvdata->sfrbase + REG_MMU_CTRL) != CTRL_BLOCK_DISABLE); + + clk_disable(drvdata->clk); + + SYSMMU_EVENT_LOG_DISABLE(SYSMMU_DRVDATA_TO_LOG(drvdata)); +} + +static inline void __sysmmu_set_public_way(struct sysmmu_drvdata *drvdata, + unsigned int public_cfg) +{ + u32 cfg = __raw_readl(drvdata->sfrbase + REG_PUBLIC_WAY_CFG); + cfg &= ~MMU_PUBLIC_WAY_MASK; + cfg |= public_cfg; + + writel_relaxed(cfg, drvdata->sfrbase + REG_PUBLIC_WAY_CFG); + + dev_dbg(drvdata->sysmmu, "public_cfg : %#x\n", cfg); +} + +static inline void __sysmmu_set_private_way_id(struct sysmmu_drvdata *drvdata, + unsigned int way_idx) +{ + struct tlb_priv_id *priv_cfg = drvdata->tlb_props.way_props.priv_id_cfg; + u32 cfg = __raw_readl(drvdata->sfrbase + REG_PRIVATE_WAY_CFG(way_idx)); + + cfg &= ~MMU_PRIVATE_WAY_MASK; + cfg |= MMU_WAY_CFG_ID_MATCHING | MMU_WAY_CFG_PRIVATE_ENABLE | + priv_cfg[way_idx].cfg; + + writel_relaxed(cfg, drvdata->sfrbase + REG_PRIVATE_WAY_CFG(way_idx)); + writel_relaxed(priv_cfg[way_idx].id, + drvdata->sfrbase + REG_PRIVATE_ID(way_idx)); + + dev_dbg(drvdata->sysmmu, "priv ID way[%d] cfg : %#x, id : %#x\n", + way_idx, cfg, priv_cfg[way_idx].id); +} + +static inline void __sysmmu_set_private_way_addr(struct sysmmu_drvdata *drvdata, + unsigned int priv_addr_idx) +{ + struct tlb_priv_addr *priv_cfg = + drvdata->tlb_props.way_props.priv_addr_cfg; + unsigned int way_idx = + drvdata->tlb_props.way_props.priv_id_cnt + priv_addr_idx; + u32 cfg = __raw_readl(drvdata->sfrbase + REG_PRIVATE_WAY_CFG(way_idx)); + + cfg &= ~MMU_PRIVATE_WAY_MASK; + cfg |= MMU_WAY_CFG_ADDR_MATCHING | MMU_WAY_CFG_PRIVATE_ENABLE | + priv_cfg[priv_addr_idx].cfg; + + writel_relaxed(cfg, drvdata->sfrbase + REG_PRIVATE_WAY_CFG(way_idx)); + + dev_dbg(drvdata->sysmmu, "priv ADDR way[%d] cfg : %#x\n", way_idx, cfg); +} + +static inline void __sysmmu_set_tlb_way_type(struct sysmmu_drvdata *drvdata) +{ + u32 cfg = __raw_readl(drvdata->sfrbase + REG_MMU_CAPA0_V7); + u32 tlb_way_num = MMU_CAPA_NUM_TLB_WAY(cfg); + u32 set_cnt = 0; + struct tlb_props *tlb_props = &drvdata->tlb_props; + unsigned int i; + int priv_id_cnt = tlb_props->way_props.priv_id_cnt; + int priv_addr_cnt = tlb_props->way_props.priv_addr_cnt; + + if (tlb_props->flags & TLB_WAY_PUBLIC) + __sysmmu_set_public_way(drvdata, + tlb_props->way_props.public_cfg); + + if (tlb_props->flags & TLB_WAY_PRIVATE_ID) { + for (i = 0; i < priv_id_cnt && + set_cnt < tlb_way_num; i++, set_cnt++) + __sysmmu_set_private_way_id(drvdata, i); + } + + if (tlb_props->flags & TLB_WAY_PRIVATE_ADDR) { + for (i = 0; i < priv_addr_cnt && + set_cnt < tlb_way_num; i++, set_cnt++) + __sysmmu_set_private_way_addr(drvdata, i); + } + + if (priv_id_cnt + priv_addr_cnt > tlb_way_num) { + dev_warn(drvdata->sysmmu, + "Too many values than TLB way count %d," + " so ignored!\n", tlb_way_num); + dev_warn(drvdata->sysmmu, + "Number of private way id/addr = %d/%d\n", + priv_id_cnt, priv_addr_cnt); + } +} + +static inline void __sysmmu_set_tlb_port(struct sysmmu_drvdata *drvdata, + unsigned int port_idx) +{ + struct tlb_port_cfg *port_cfg = drvdata->tlb_props.port_props.port_cfg; + + writel_relaxed(MMU_TLB_CFG_MASK(port_cfg[port_idx].cfg), + drvdata->sfrbase + REG_MMU_TLB_CFG(port_idx)); + + /* port_idx 0 is default port. */ + if (port_idx == 0) { + dev_dbg(drvdata->sysmmu, "port[%d] cfg : %#x for common\n", + port_idx, + MMU_TLB_CFG_MASK(port_cfg[port_idx].cfg)); + return; + } + + writel_relaxed(MMU_TLB_MATCH_CFG_MASK(port_cfg[port_idx].cfg), + drvdata->sfrbase + REG_MMU_TLB_MATCH_CFG(port_idx)); + writel_relaxed(port_cfg[port_idx].id, + drvdata->sfrbase + REG_MMU_TLB_MATCH_ID(port_idx)); + + dev_dbg(drvdata->sysmmu, "port[%d] cfg : %#x, match : %#x, id : %#x\n", + port_idx, + MMU_TLB_CFG_MASK(port_cfg[port_idx].cfg), + MMU_TLB_MATCH_CFG_MASK(port_cfg[port_idx].cfg), + port_cfg[port_idx].id); +} + +static inline void __sysmmu_set_tlb_port_type(struct sysmmu_drvdata *drvdata) +{ + u32 cfg = __raw_readl(drvdata->sfrbase + REG_MMU_CAPA1_V7); + u32 tlb_num = MMU_CAPA1_NUM_TLB(cfg); + struct tlb_props *tlb_props = &drvdata->tlb_props; + unsigned int i; + int port_id_cnt = tlb_props->port_props.port_id_cnt; + int slot_cnt = tlb_props->port_props.slot_cnt; + + if (port_id_cnt > tlb_num) { + dev_warn(drvdata->sysmmu, + "Too many values %d than TLB count %d," + " so ignored!\n", port_id_cnt, tlb_num); + port_id_cnt = tlb_num; + } + + for (i = 0; i < port_id_cnt; i++) + __sysmmu_set_tlb_port(drvdata, i); + + for (i = 0; i < slot_cnt; i++) + writel_relaxed(tlb_props->port_props.slot_cfg[i], + drvdata->sfrbase + REG_SLOT_RSV(i)); +} + +static inline void __sysmmu_init_config(struct sysmmu_drvdata *drvdata) +{ + unsigned long cfg = 0; + + writel_relaxed(CTRL_BLOCK, drvdata->sfrbase + REG_MMU_CTRL); + + if (IS_TLB_WAY_TYPE(drvdata)) + __sysmmu_set_tlb_way_type(drvdata); + else if (IS_TLB_PORT_TYPE(drvdata)) + __sysmmu_set_tlb_port_type(drvdata); + + if (drvdata->qos != DEFAULT_QOS_VALUE) + cfg |= CFG_QOS_OVRRIDE | CFG_QOS(drvdata->qos); + + cfg |= __raw_readl(drvdata->sfrbase + REG_MMU_CFG) & ~CFG_MASK; + writel_relaxed(cfg, drvdata->sfrbase + REG_MMU_CFG); +} + +static inline void __sysmmu_enable_nocount(struct sysmmu_drvdata *drvdata) +{ + clk_enable(drvdata->clk); + + __sysmmu_init_config(drvdata); + + __sysmmu_set_ptbase(drvdata, drvdata->pgtable / PAGE_SIZE); + + writel(CTRL_ENABLE, drvdata->sfrbase + REG_MMU_CTRL); + + SYSMMU_EVENT_LOG_ENABLE(SYSMMU_DRVDATA_TO_LOG(drvdata)); +} + +static inline u32 __sysmmu_get_intr_status(struct sysmmu_drvdata *drvdata, + bool is_secure) +{ + if (is_secure) + return __secure_info_read(drvdata->securebase + REG_INT_STATUS); + else + return __raw_readl(drvdata->sfrbase + REG_INT_STATUS); +} + +static inline u32 __sysmmu_get_fault_address(struct sysmmu_drvdata *drvdata, + bool is_secure) +{ + if (is_secure) + return __secure_info_read(drvdata->securebase + REG_FAULT_ADDR); + else + return __raw_readl(drvdata->sfrbase + REG_FAULT_ADDR); +} + +static inline u32 __sysmmu_get_fault_trans_info(struct sysmmu_drvdata *drvdata, + bool is_secure) +{ + if (is_secure) + return __secure_info_read( + drvdata->securebase + REG_FAULT_TRANS_INFO); + else + return __raw_readl(drvdata->sfrbase + REG_FAULT_TRANS_INFO); +} + +static inline u32 __sysmmu_get_hw_version(struct sysmmu_drvdata *data) +{ + return MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION)); +} + +static inline bool __sysmmu_has_capa1(struct sysmmu_drvdata *data) +{ + return MMU_CAPA1_EXIST(__raw_readl(data->sfrbase + REG_MMU_CAPA0_V7)); +} + +static inline u32 __sysmmu_get_capa_type(struct sysmmu_drvdata *data) +{ + return MMU_CAPA1_TYPE(__raw_readl(data->sfrbase + REG_MMU_CAPA1_V7)); +} diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 82ebcf4a53db..bfb4a4716720 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -33,6 +33,7 @@ #include #include "exynos-iommu.h" +#include "exynos-iommu-reg.h" /* Default IOVA region: [0x1000000, 0xD0000000) */ #define IOVA_START 0x10000000 @@ -96,78 +97,6 @@ static inline void pgtable_flush(void *vastart, void *vaend) __dma_flush_area(vastart, vaend - vastart); } -#define is_secure_info_fail(x) ((((x) >> 16) & 0xffff) == 0xdead) -static u32 __secure_info_read(unsigned int addr) -{ - u32 ret; - - ret = exynos_smc(SMC_DRM_SEC_SMMU_INFO, (unsigned long)addr, - 0, SEC_SMMU_SFR_READ); - if (is_secure_info_fail(ret)) - pr_err("Invalid value returned, %#x\n", ret); - - return ret; -} - -static bool has_sysmmu_capable_pbuf(void __iomem *sfrbase) -{ - unsigned long cfg = readl_relaxed(sfrbase + REG_MMU_CAPA); - - return MMU_HAVE_PB(cfg); -} - -static bool has_sysmmu_set_associative_tlb(void __iomem *sfrbase) -{ - u32 cfg = readl_relaxed(sfrbase + REG_MMU_CAPA_1); - - return MMU_IS_TLB_CONFIGURABLE(cfg); -} - -static void __sysmmu_set_tlb_line_size(void __iomem *sfrbase) -{ - u32 cfg = readl_relaxed(sfrbase + REG_L2TLB_CFG); - cfg &= ~MMU_MASK_LINE_SIZE; - cfg |= MMU_DEFAULT_LINE_SIZE; - writel_relaxed(cfg, sfrbase + REG_L2TLB_CFG); -} - -static void __exynos_sysmmu_set_prefbuf_axi_id(struct sysmmu_drvdata *drvdata) -{ - if (!has_sysmmu_capable_pbuf(drvdata->sfrbase)) - return; - - /* TODO: implement later */ -} - -static void __sysmmu_tlb_invalidate_all(void __iomem *sfrbase) -{ - writel(0x1, sfrbase + REG_MMU_FLUSH); -} - -static void __sysmmu_tlb_invalidate(struct sysmmu_drvdata *drvdata, - dma_addr_t iova, size_t size) -{ - void * __iomem sfrbase = drvdata->sfrbase; - - __raw_writel(iova, sfrbase + REG_FLUSH_RANGE_START); - __raw_writel(size - 1 + iova, sfrbase + REG_FLUSH_RANGE_END); - writel(0x1, sfrbase + REG_MMU_FLUSH_RANGE); - SYSMMU_EVENT_LOG_TLB_INV_RANGE(SYSMMU_DRVDATA_TO_LOG(drvdata), - iova, iova + size); -} - -static void __sysmmu_set_ptbase(struct sysmmu_drvdata *drvdata, - phys_addr_t pfn_pgtable) -{ - void * __iomem sfrbase = drvdata->sfrbase; - - writel_relaxed(pfn_pgtable, sfrbase + REG_PT_BASE_PPN); - - __sysmmu_tlb_invalidate_all(sfrbase); - SYSMMU_EVENT_LOG_TLB_INV_ALL( - SYSMMU_DRVDATA_TO_LOG(drvdata)); -} - void exynos_sysmmu_tlb_invalidate(struct iommu_domain *iommu_domain, dma_addr_t d_start, size_t size) { @@ -203,324 +132,27 @@ void exynos_sysmmu_tlb_invalidate(struct iommu_domain *iommu_domain, spin_unlock_irqrestore(&domain->lock, flags); } - -static unsigned int dump_tlb_entry_way_type(void __iomem *sfrbase, - int idx_way, int idx_set) -{ - if (MMU_TLB_ENTRY_VALID(__raw_readl(sfrbase + REG_TLB_ATTR))) { - pr_crit("[%02d][%02d] VPN: %#010x, PPN: %#010x, ATTR: %#010x\n", - idx_way, idx_set, - __raw_readl(sfrbase + REG_TLB_VPN), - __raw_readl(sfrbase + REG_TLB_PPN), - __raw_readl(sfrbase + REG_TLB_ATTR)); - return 1; - } - return 0; -} - -static unsigned int dump_tlb_entry_port_type(void __iomem *sfrbase, - int idx_way, int idx_set) -{ - if (MMU_TLB_ENTRY_VALID(__raw_readl(sfrbase + REG_CAPA1_TLB_ATTR))) { - pr_crit("[%02d][%02d] VPN: %#010x, PPN: %#010x, ATTR: %#010x\n", - idx_way, idx_set, - __raw_readl(sfrbase + REG_CAPA1_TLB_VPN), - __raw_readl(sfrbase + REG_CAPA1_TLB_PPN), - __raw_readl(sfrbase + REG_CAPA1_TLB_ATTR)); - return 1; - } - return 0; -} - -#define MMU_NUM_TLB_SUBLINE 4 -static void dump_sysmmu_tlb_way(struct sysmmu_drvdata *drvdata) -{ - int i, j, k; - u32 capa; - unsigned int cnt; - void __iomem *sfrbase = drvdata->sfrbase; - - capa = __raw_readl(sfrbase + REG_MMU_CAPA0_V7); - - pr_crit("TLB has %d way, %d set. SBB has %d entries.\n", - MMU_CAPA_NUM_TLB_WAY(capa), - (1 << MMU_CAPA_NUM_TLB_SET(capa)), - (1 << MMU_CAPA_NUM_SBB_ENTRY(capa))); - pr_crit("------------- TLB[WAY][SET][ENTRY] -------------\n"); - for (i = 0, cnt = 0; i < MMU_CAPA_NUM_TLB_WAY(capa); i++) { - for (j = 0; j < (1 << MMU_CAPA_NUM_TLB_SET(capa)); j++) { - for (k = 0; k < MMU_NUM_TLB_SUBLINE; k++) { - __raw_writel(MMU_SET_TLB_READ_ENTRY(j, i, k), - sfrbase + REG_TLB_READ); - cnt += dump_tlb_entry_way_type(sfrbase, i, j); - } - } - } - if (!cnt) - pr_crit(">> No Valid TLB Entries\n"); - - pr_crit("--- SBB(Second-Level Page Table Base Address Buffer ---\n"); - for (i = 0, cnt = 0; i < (1 << MMU_CAPA_NUM_SBB_ENTRY(capa)); i++) { - __raw_writel(i, sfrbase + REG_SBB_READ); - if (MMU_SBB_ENTRY_VALID(__raw_readl(sfrbase + REG_SBB_VPN))) { - pr_crit("[%02d] VPN: %#010x, PPN: %#010x, ATTR: %#010x\n", - i, __raw_readl(sfrbase + REG_SBB_VPN), - __raw_readl(sfrbase + REG_SBB_LINK), - __raw_readl(sfrbase + REG_SBB_ATTR)); - cnt++; - } - } - if (!cnt) - pr_crit(">> No Valid SBB Entries\n"); -} - -static void dump_sysmmu_tlb_port(struct sysmmu_drvdata *drvdata) -{ - int t, i, j, k; - u32 capa0, capa1, info; - unsigned int cnt; - int num_tlb, num_port, num_sbb; - void __iomem *sfrbase = drvdata->sfrbase; - - capa0 = __raw_readl(sfrbase + REG_MMU_CAPA0_V7); - capa1 = __raw_readl(sfrbase + REG_MMU_CAPA1_V7); - - num_tlb = MMU_CAPA1_NUM_TLB(capa1); - num_port = MMU_CAPA1_NUM_PORT(capa1); - num_sbb = 1 << MMU_CAPA_NUM_SBB_ENTRY(capa0); - - pr_crit("SysMMU has %d TLBs, %d ports, %d sbb entries\n", - num_tlb, num_port, num_sbb); - - for (t = 0; t < num_tlb; t++) { - int num_set, num_way; - - info = __raw_readl(sfrbase + MMU_TLB_INFO(t)); - num_way = MMU_CAPA1_NUM_TLB_WAY(info); - num_set = MMU_CAPA1_NUM_TLB_SET(info); - - pr_crit("TLB.%d has %d way, %d set.\n", t, num_way, num_set); - pr_crit("------------- TLB[WAY][SET][ENTRY] -------------\n"); - for (i = 0, cnt = 0; i < num_way; i++) { - for (j = 0; j < num_set; j++) { - for (k = 0; k < MMU_NUM_TLB_SUBLINE; k++) { - __raw_writel(MMU_CAPA1_SET_TLB_READ_ENTRY(t, j, i, k), - sfrbase + REG_CAPA1_TLB_READ); - cnt += dump_tlb_entry_port_type(sfrbase, i, j); - } - } - } - } - if (!cnt) - pr_crit(">> No Valid TLB Entries\n"); - - pr_crit("--- SBB(Second-Level Page Table Base Address Buffer ---\n"); - for (i = 0, cnt = 0; i < num_sbb; i++) { - __raw_writel(i, sfrbase + REG_CAPA1_SBB_READ); - if (MMU_SBB_ENTRY_VALID(__raw_readl(sfrbase + REG_CAPA1_SBB_VPN))) { - pr_crit("[%02d] VPN: %#010x, PPN: %#010x, ATTR: %#010x\n", - i, __raw_readl(sfrbase + REG_CAPA1_SBB_VPN), - __raw_readl(sfrbase + REG_CAPA1_SBB_LINK), - __raw_readl(sfrbase + REG_CAPA1_SBB_ATTR)); - cnt++; - } - } - if (!cnt) - pr_crit(">> No Valid SBB Entries\n"); -} - -static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = { - "PTW ACCESS FAULT", - "PAGE FAULT", - "L1TLB MULTI-HIT FAULT", - "ACCESS FAULT", - "SECURITY FAULT", - "UNKNOWN FAULT" -}; - -void dump_sysmmu_status(struct sysmmu_drvdata *drvdata) -{ - int info; - pgd_t *pgd; - pud_t *pud; - pmd_t *pmd; - pte_t *pte; - phys_addr_t phys; - void __iomem *sfrbase = drvdata->sfrbase; - - pgd = pgd_offset_k((unsigned long)sfrbase); - if (!pgd) { - pr_crit("Invalid virtual address %p\n", sfrbase); - return; - } - - pud = pud_offset(pgd, (unsigned long)sfrbase); - if (!pud) { - pr_crit("Invalid virtual address %p\n", sfrbase); - return; - } - - pmd = pmd_offset(pud, (unsigned long)sfrbase); - if (!pmd) { - pr_crit("Invalid virtual address %p\n", sfrbase); - return; - } - - pte = pte_offset_kernel(pmd, (unsigned long)sfrbase); - if (!pte) { - pr_crit("Invalid virtual address %p\n", sfrbase); - return; - } - - info = MMU_RAW_VER(__raw_readl(sfrbase + REG_MMU_VERSION)); - - phys = pte_pfn(*pte) << PAGE_SHIFT; - pr_crit("ADDR: %pa(VA: %p), MMU_CTRL: %#010x, PT_BASE: %#010x\n", - &phys, sfrbase, - __raw_readl(sfrbase + REG_MMU_CTRL), - __raw_readl(sfrbase + REG_PT_BASE_PPN)); - pr_crit("VERSION %d.%d.%d, MMU_CFG: %#010x, MMU_STATUS: %#010x\n", - MMU_MAJ_VER(info), MMU_MIN_VER(info), MMU_REV_VER(info), - __raw_readl(sfrbase + REG_MMU_CFG), - __raw_readl(sfrbase + REG_MMU_STATUS)); - - if (IS_TLB_WAY_TYPE(drvdata)) - dump_sysmmu_tlb_way(drvdata); - else if(IS_TLB_PORT_TYPE(drvdata)) - dump_sysmmu_tlb_port(drvdata); -} - -static void show_secure_fault_information(struct sysmmu_drvdata *drvdata, - int flags, unsigned long fault_addr) -{ - unsigned int info; - phys_addr_t pgtable; - int fault_id = SYSMMU_FAULT_ID(flags); - unsigned int sfrbase = drvdata->securebase; - const char *port_name = NULL; - - pgtable = __secure_info_read(sfrbase + REG_PT_BASE_PPN); - pgtable <<= PAGE_SHIFT; - - info = __secure_info_read(sfrbase + REG_FAULT_TRANS_INFO); - - of_property_read_string(drvdata->sysmmu->of_node, - "port-name", &port_name); - - pr_crit("----------------------------------------------------------\n"); - pr_crit("From [%s], SysMMU %s %s at %#010lx (page table @ %pa)\n", - port_name ? port_name : dev_name(drvdata->sysmmu), - (flags & IOMMU_FAULT_WRITE) ? "WRITE" : "READ", - sysmmu_fault_name[fault_id], fault_addr, &pgtable); - - if (fault_id == SYSMMU_FAULT_UNKNOWN) { - pr_crit("The fault is not caused by this System MMU.\n"); - pr_crit("Please check IRQ and SFR base address.\n"); - goto finish; - } - - pr_crit("AxID: %#x, AxLEN: %#x\n", info & 0xFFFF, (info >> 16) & 0xF); - - if (fault_id == SYSMMU_FAULT_PTW_ACCESS) - pr_crit("System MMU has failed to access page table\n"); - - if (!pfn_valid(pgtable >> PAGE_SHIFT)) { - pr_crit("Page table base is not in a valid memory region\n"); - } else { - sysmmu_pte_t *ent; - ent = section_entry(phys_to_virt(pgtable), fault_addr); - pr_crit("Lv1 entry: %#010x\n", *ent); - - if (lv1ent_page(ent)) { - ent = page_entry(ent, fault_addr); - pr_crit("Lv2 entry: %#010x\n", *ent); - } - } - - info = MMU_RAW_VER(__secure_info_read(sfrbase + REG_MMU_VERSION)); - - pr_crit("ADDR: %#x, MMU_CTRL: %#010x, PT_BASE: %#010x\n", - sfrbase, - __secure_info_read(sfrbase + REG_MMU_CTRL), - __secure_info_read(sfrbase + REG_PT_BASE_PPN)); - pr_crit("VERSION %d.%d.%d, MMU_CFG: %#010x, MMU_STATUS: %#010x\n", - MMU_MAJ_VER(info), MMU_MIN_VER(info), MMU_REV_VER(info), - __secure_info_read(sfrbase + REG_MMU_CFG), - __secure_info_read(sfrbase + REG_MMU_STATUS)); - -finish: - pr_crit("----------------------------------------------------------\n"); -} - -static void show_fault_information(struct sysmmu_drvdata *drvdata, - int flags, unsigned long fault_addr) +static void sysmmu_get_interrupt_info(struct sysmmu_drvdata *data, + int *flags, unsigned long *addr, bool is_secure) { - unsigned int info; - phys_addr_t pgtable; - int fault_id = SYSMMU_FAULT_ID(flags); - const char *port_name = NULL; - - pgtable = __raw_readl(drvdata->sfrbase + REG_PT_BASE_PPN); - pgtable <<= PAGE_SHIFT; - - if (MMU_MAJ_VER(drvdata->version) >= 7) { - info = __raw_readl(drvdata->sfrbase + REG_FAULT_TRANS_INFO); - } else { - /* TODO: Remove me later */ - info = __raw_readl(drvdata->sfrbase + - ((flags & IOMMU_FAULT_WRITE) ? - REG_FAULT_AW_TRANS_INFO : REG_FAULT_AR_TRANS_INFO)); - } - - of_property_read_string(drvdata->sysmmu->of_node, - "port-name", &port_name); - - pr_crit("----------------------------------------------------------\n"); - pr_crit("From [%s], SysMMU %s %s at %#010lx (page table @ %pa)\n", - port_name ? port_name : dev_name(drvdata->sysmmu), - (flags & IOMMU_FAULT_WRITE) ? "WRITE" : "READ", - sysmmu_fault_name[fault_id], fault_addr, &pgtable); - - if (fault_id == SYSMMU_FAULT_UNKNOWN) { - pr_crit("The fault is not caused by this System MMU.\n"); - pr_crit("Please check IRQ and SFR base address.\n"); - goto finish; - } - - pr_crit("AxID: %#x, AxLEN: %#x\n", info & 0xFFFF, (info >> 16) & 0xF); - - if (pgtable != drvdata->pgtable) - pr_crit("Page table base of driver: %pa\n", - &drvdata->pgtable); - - if (fault_id == SYSMMU_FAULT_PTW_ACCESS) - pr_crit("System MMU has failed to access page table\n"); - - if (!pfn_valid(pgtable >> PAGE_SHIFT)) { - pr_crit("Page table base is not in a valid memory region\n"); - } else { - sysmmu_pte_t *ent; - ent = section_entry(phys_to_virt(pgtable), fault_addr); - pr_crit("Lv1 entry: %#010x\n", *ent); - - if (lv1ent_page(ent)) { - ent = page_entry(ent, fault_addr); - pr_crit("Lv2 entry: %#010x\n", *ent); - } - } + unsigned int itype; + u32 info; - dump_sysmmu_status(drvdata); + itype = __ffs(__sysmmu_get_intr_status(data, is_secure)); + if (WARN_ON(!(itype < SYSMMU_FAULT_UNKNOWN))) + itype = SYSMMU_FAULT_UNKNOWN; + else + *addr = __sysmmu_get_fault_address(data, is_secure); -finish: - pr_crit("----------------------------------------------------------\n"); + info = __sysmmu_get_fault_trans_info(data, is_secure); + *flags = MMU_IS_READ_FAULT(info) ? + IOMMU_FAULT_READ : IOMMU_FAULT_WRITE; + *flags |= SYSMMU_FAULT_FLAG(itype); } -#define REG_INT_STATUS_WRITE_BIT 16 - irqreturn_t exynos_sysmmu_irq_secure(int irq, void *dev_id) { struct sysmmu_drvdata *drvdata = dev_id; - unsigned int itype = 0; unsigned long addr = -1; int flags = 0; @@ -533,17 +165,8 @@ irqreturn_t exynos_sysmmu_irq_secure(int irq, void *dev_id) (unsigned long)drvdata->securebase); } - itype = __ffs(__secure_info_read(drvdata->securebase + REG_INT_STATUS)); - if (itype >= REG_INT_STATUS_WRITE_BIT) { - itype -= REG_INT_STATUS_WRITE_BIT; - flags = IOMMU_FAULT_WRITE; - } - - addr = __secure_info_read(drvdata->securebase + REG_FAULT_ADDR); - flags |= SYSMMU_FAULT_FLAG(itype); - + sysmmu_get_interrupt_info(drvdata, &flags, &addr, true); show_secure_fault_information(drvdata, flags, addr); - atomic_notifier_call_chain(&drvdata->fault_notifiers, addr, &flags); BUG(); @@ -554,7 +177,6 @@ irqreturn_t exynos_sysmmu_irq_secure(int irq, void *dev_id) static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) { struct sysmmu_drvdata *drvdata = dev_id; - unsigned int itype; unsigned long addr = -1; int flags = 0; @@ -564,36 +186,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) "Fault occurred while System MMU %s is not enabled!\n", dev_name(drvdata->sysmmu)); - if (MMU_MAJ_VER(drvdata->version) >= 7) { - u32 info; - - itype = __ffs(__raw_readl(drvdata->sfrbase + REG_INT_STATUS)); - if (WARN_ON(!(itype < SYSMMU_FAULT_UNKNOWN))) - itype = SYSMMU_FAULT_UNKNOWN; - else - addr = __raw_readl(drvdata->sfrbase + REG_FAULT_ADDR); - - info = __raw_readl(drvdata->sfrbase + REG_FAULT_TRANS_INFO); - flags = MMU_IS_READ_FAULT(info) ? - IOMMU_FAULT_READ : IOMMU_FAULT_WRITE; - flags |= SYSMMU_FAULT_FLAG(itype); - } else { - /* TODO: Remove me later */ - itype = __ffs(__raw_readl(drvdata->sfrbase + REG_INT_STATUS)); - if (itype >= REG_INT_STATUS_WRITE_BIT) { - itype -= REG_INT_STATUS_WRITE_BIT; - flags = IOMMU_FAULT_WRITE; - } - - if (WARN_ON(!(itype < SYSMMU_FAULT_UNKNOWN))) - itype = SYSMMU_FAULT_UNKNOWN; - else - addr = __raw_readl(drvdata->sfrbase + - ((flags & IOMMU_FAULT_WRITE) ? - REG_FAULT_AW_ADDR : REG_FAULT_AR_ADDR)); - flags |= SYSMMU_FAULT_FLAG(itype); - } - + sysmmu_get_interrupt_info(drvdata, &flags, &addr, false); show_fault_information(drvdata, flags, addr); atomic_notifier_call_chain(&drvdata->fault_notifiers, addr, &flags); @@ -605,7 +198,6 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) static int sysmmu_get_hw_info(struct sysmmu_drvdata *data) { int ret; - void __iomem *sfrbase = data->sfrbase; struct tlb_props *tlb_props = &data->tlb_props; ret = pm_runtime_get_sync(data->sysmmu); @@ -614,15 +206,15 @@ static int sysmmu_get_hw_info(struct sysmmu_drvdata *data) return ret; } - data->version = MMU_RAW_VER(__raw_readl(sfrbase + REG_MMU_VERSION)); + data->version = __sysmmu_get_hw_version(data); /* * If CAPA1 doesn't exist, sysmmu uses TLB way dedication. * If CAPA1[31:28] is zero, sysmmu uses TLB port dedication. */ - if (!(__raw_readl(sfrbase + REG_MMU_CAPA0_V7) & (1 << 11))) + if (!__sysmmu_has_capa1(data)) tlb_props->flags |= TLB_TYPE_WAY; - else if (!(__raw_readl(sfrbase + REG_MMU_CAPA1_V7) & (0xF << 28))) + else if (__sysmmu_get_capa_type(data) == 0) tlb_props->flags |= TLB_TYPE_PORT; pm_runtime_put(data->sysmmu); @@ -992,17 +584,6 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) return 0; } -static void __sysmmu_disable_nocount(struct sysmmu_drvdata *drvdata) -{ - writel_relaxed(0, drvdata->sfrbase + REG_MMU_CFG); - writel_relaxed(CTRL_BLOCK_DISABLE, drvdata->sfrbase + REG_MMU_CTRL); - BUG_ON(readl_relaxed(drvdata->sfrbase + REG_MMU_CTRL) != CTRL_BLOCK_DISABLE); - - clk_disable(drvdata->clk); - - SYSMMU_EVENT_LOG_DISABLE(SYSMMU_DRVDATA_TO_LOG(drvdata)); -} - static bool __sysmmu_disable(struct sysmmu_drvdata *drvdata) { bool disabled; @@ -1046,177 +627,6 @@ static void sysmmu_disable_from_master(struct device *master) spin_unlock_irqrestore(&owner->lock, flags); } -static void __sysmmu_set_public_way(struct sysmmu_drvdata *drvdata, - unsigned int public_cfg) -{ - u32 cfg = __raw_readl(drvdata->sfrbase + REG_PUBLIC_WAY_CFG); - cfg &= ~MMU_PUBLIC_WAY_MASK; - cfg |= public_cfg; - - writel_relaxed(cfg, drvdata->sfrbase + REG_PUBLIC_WAY_CFG); - - dev_dbg(drvdata->sysmmu, "public_cfg : %#x\n", cfg); -} - -static void __sysmmu_set_private_way_id(struct sysmmu_drvdata *drvdata, - unsigned int way_idx) -{ - struct tlb_priv_id *priv_cfg = drvdata->tlb_props.way_props.priv_id_cfg; - u32 cfg = __raw_readl(drvdata->sfrbase + REG_PRIVATE_WAY_CFG(way_idx)); - - cfg &= ~MMU_PRIVATE_WAY_MASK; - cfg |= MMU_WAY_CFG_ID_MATCHING | MMU_WAY_CFG_PRIVATE_ENABLE | - priv_cfg[way_idx].cfg; - - writel_relaxed(cfg, drvdata->sfrbase + REG_PRIVATE_WAY_CFG(way_idx)); - writel_relaxed(priv_cfg[way_idx].id, - drvdata->sfrbase + REG_PRIVATE_ID(way_idx)); - - dev_dbg(drvdata->sysmmu, "priv ID way[%d] cfg : %#x, id : %#x\n", - way_idx, cfg, priv_cfg[way_idx].id); -} - -static void __sysmmu_set_private_way_addr(struct sysmmu_drvdata *drvdata, - unsigned int priv_addr_idx) -{ - struct tlb_priv_addr *priv_cfg = drvdata->tlb_props.way_props.priv_addr_cfg; - unsigned int way_idx = drvdata->tlb_props.way_props.priv_id_cnt + priv_addr_idx; - u32 cfg = __raw_readl(drvdata->sfrbase + REG_PRIVATE_WAY_CFG(way_idx)); - - cfg &= ~MMU_PRIVATE_WAY_MASK; - cfg |= MMU_WAY_CFG_ADDR_MATCHING | MMU_WAY_CFG_PRIVATE_ENABLE | - priv_cfg[priv_addr_idx].cfg; - - writel_relaxed(cfg, drvdata->sfrbase + REG_PRIVATE_WAY_CFG(way_idx)); - - dev_dbg(drvdata->sysmmu, "priv ADDR way[%d] cfg : %#x\n", way_idx, cfg); -} - -static void __sysmmu_set_tlb_way_dedication(struct sysmmu_drvdata *drvdata) -{ - u32 cfg = __raw_readl(drvdata->sfrbase + REG_MMU_CAPA0_V7); - u32 tlb_way_num = MMU_CAPA_NUM_TLB_WAY(cfg); - u32 set_cnt = 0; - struct tlb_props *tlb_props = &drvdata->tlb_props; - unsigned int i; - int priv_id_cnt = tlb_props->way_props.priv_id_cnt; - int priv_addr_cnt = tlb_props->way_props.priv_addr_cnt; - - if (tlb_props->flags & TLB_WAY_PUBLIC) - __sysmmu_set_public_way(drvdata, - tlb_props->way_props.public_cfg); - - if (tlb_props->flags & TLB_WAY_PRIVATE_ID) { - for (i = 0; i < priv_id_cnt && - set_cnt < tlb_way_num; i++, set_cnt++) - __sysmmu_set_private_way_id(drvdata, i); - } - - if (tlb_props->flags & TLB_WAY_PRIVATE_ADDR) { - for (i = 0; i < priv_addr_cnt && - set_cnt < tlb_way_num; i++, set_cnt++) - __sysmmu_set_private_way_addr(drvdata, i); - } - - if (priv_id_cnt + priv_addr_cnt > tlb_way_num) { - dev_warn(drvdata->sysmmu, - "Too many values than TLB way count %d," - " so ignored!\n", tlb_way_num); - dev_warn(drvdata->sysmmu, - "Number of private way id/addr = %d/%d\n", - priv_id_cnt, priv_addr_cnt); - } -} - -static void __sysmmu_set_tlb_port(struct sysmmu_drvdata *drvdata, - unsigned int port_idx) -{ - struct tlb_port_cfg *port_cfg = drvdata->tlb_props.port_props.port_cfg; - - writel_relaxed(MMU_TLB_CFG_MASK(port_cfg[port_idx].cfg), - drvdata->sfrbase + REG_MMU_TLB_CFG(port_idx)); - - /* port_idx 0 is default port. */ - if (port_idx == 0) { - dev_dbg(drvdata->sysmmu, "port[%d] cfg : %#x for common\n", - port_idx, - MMU_TLB_CFG_MASK(port_cfg[port_idx].cfg)); - return; - } - - writel_relaxed(MMU_TLB_MATCH_CFG_MASK(port_cfg[port_idx].cfg), - drvdata->sfrbase + REG_MMU_TLB_MATCH_CFG(port_idx)); - writel_relaxed(port_cfg[port_idx].id, - drvdata->sfrbase + REG_MMU_TLB_MATCH_ID(port_idx)); - - dev_dbg(drvdata->sysmmu, "port[%d] cfg : %#x, match : %#x, id : %#x\n", - port_idx, - MMU_TLB_CFG_MASK(port_cfg[port_idx].cfg), - MMU_TLB_MATCH_CFG_MASK(port_cfg[port_idx].cfg), - port_cfg[port_idx].id); -} - -static void __sysmmu_set_tlb_port_dedication(struct sysmmu_drvdata *drvdata) -{ - u32 cfg = __raw_readl(drvdata->sfrbase + REG_MMU_CAPA1_V7); - u32 tlb_num = MMU_CAPA1_NUM_TLB(cfg); - struct tlb_props *tlb_props = &drvdata->tlb_props; - unsigned int i; - int port_id_cnt = tlb_props->port_props.port_id_cnt; - int slot_cnt = tlb_props->port_props.slot_cnt; - - if (port_id_cnt > tlb_num) { - dev_warn(drvdata->sysmmu, - "Too many values %d than TLB count %d," - " so ignored!\n", port_id_cnt, tlb_num); - port_id_cnt = tlb_num; - } - - for (i = 0; i < port_id_cnt; i++) - __sysmmu_set_tlb_port(drvdata, i); - - for (i = 0; i < slot_cnt; i++) - writel_relaxed(tlb_props->port_props.slot_cfg[i], - drvdata->sfrbase + REG_SLOT_RSV(i)); -} - -static void __sysmmu_init_config(struct sysmmu_drvdata *drvdata) -{ - unsigned long cfg = 0; - - writel_relaxed(CTRL_BLOCK, drvdata->sfrbase + REG_MMU_CTRL); - - if (IS_TLB_WAY_TYPE(drvdata)) { - __sysmmu_set_tlb_way_dedication(drvdata); - } else if (IS_TLB_PORT_TYPE(drvdata)) { - __sysmmu_set_tlb_port_dedication(drvdata); - } else { - __exynos_sysmmu_set_prefbuf_axi_id(drvdata); - if (has_sysmmu_set_associative_tlb(drvdata->sfrbase)) - __sysmmu_set_tlb_line_size(drvdata->sfrbase); - cfg |= CFG_FLPDCACHE | CFG_ACGEN; - } - - if (drvdata->qos != DEFAULT_QOS_VALUE) - cfg |= CFG_QOS_OVRRIDE | CFG_QOS(drvdata->qos); - - cfg |= __raw_readl(drvdata->sfrbase + REG_MMU_CFG) & ~CFG_MASK; - writel_relaxed(cfg, drvdata->sfrbase + REG_MMU_CFG); -} - -static void __sysmmu_enable_nocount(struct sysmmu_drvdata *drvdata) -{ - clk_enable(drvdata->clk); - - __sysmmu_init_config(drvdata); - - __sysmmu_set_ptbase(drvdata, drvdata->pgtable / PAGE_SIZE); - - writel(CTRL_ENABLE, drvdata->sfrbase + REG_MMU_CTRL); - - SYSMMU_EVENT_LOG_ENABLE(SYSMMU_DRVDATA_TO_LOG(drvdata)); -} - static int __sysmmu_enable(struct sysmmu_drvdata *drvdata, phys_addr_t pgtable) { int ret = 0; diff --git a/drivers/iommu/exynos-iommu.h b/drivers/iommu/exynos-iommu.h index 5ea4c4b34439..f8bacc1154f1 100644 --- a/drivers/iommu/exynos-iommu.h +++ b/drivers/iommu/exynos-iommu.h @@ -166,6 +166,8 @@ typedef u32 sysmmu_pte_t; #define REG_SBB_ATTR 0x110C /* For SysMMU v7.1 */ +#define MMU_CAPA1_EXIST(reg) ((reg >> 11) & 0x1) +#define MMU_CAPA1_TYPE(reg) ((reg >> 28) & 0xF) #define MMU_CAPA1_NUM_TLB(reg) ((reg >> 4) & 0xFF) #define MMU_CAPA1_NUM_PORT(reg) ((reg) & 0xF) #define MMU_TLB_INFO(n) (0x2000 + ((n) * 0x20))