projects / GitHub / LineageOS / android_kernel_samsung_universal7580.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ARM: Fix build after memfd_create syscall
[GitHub/LineageOS/android_kernel_samsung_universal7580.git] / drivers / iommu / exynos5-iommu.c
CommitLineData
3c2a0909
S		 1/* linux/drivers/iommu/exynos_iommu.c
2 *
3 * Copyright (c) 2013-2014 Samsung Electronics Co., Ltd.
4 * http://www.samsung.com
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#ifdef CONFIG_EXYNOS_IOMMU_DEBUG
12#define DEBUG
13#endif
14
15#include <linux/kernel.h>
16#include <linux/io.h>
17#include <linux/interrupt.h>
18#include <linux/clk.h>
19#include <linux/err.h>
20#include <linux/errno.h>
21#include <linux/device.h>
22
23#include <asm/pgtable.h>
24
25#include "exynos-iommu.h"
26
27#define CFG_LRU 0x1
28#define CFG_MASK 0x0150FFFF /* Selecting bit 0-15, 20, 22 and 24 */
29#define CFG_SYSSEL (1 << 22) /* System MMU 3.2 only */
30
31#define PB_INFO_NUM(reg) ((reg) & 0xFF) /* System MMU 3.3 only */
32
33#define REG_MMU_FLUSH 0x00C
34#define REG_MMU_FLUSH_ENTRY 0x010
35#define REG_PT_BASE_ADDR 0x014
36#define REG_INT_STATUS 0x018
37#define REG_INT_CLEAR 0x01C
38#define REG_PB_INFO 0x400
39#define REG_PB_LMM 0x404
40#define REG_PB_INDICATE 0x408
41#define REG_PB_CFG 0x40C
42#define REG_PB_START_ADDR 0x410
43#define REG_PB_END_ADDR 0x414
44#define REG_SPB_BASE_VPN 0x418
45
46#define REG_PAGE_FAULT_ADDR 0x024
47#define REG_AW_FAULT_ADDR 0x028
48#define REG_AR_FAULT_ADDR 0x02C
49#define REG_DEFAULT_SLAVE_ADDR 0x030
50#define REG_FAULT_TRANS_INFO 0x04C
51#define REG_L1TLB_READ_ENTRY 0x040
52#define REG_L1TLB_ENTRY_PPN 0x044
53#define REG_L1TLB_ENTRY_VPN 0x048
54
55#define MAX_NUM_PBUF 6
56#define SINGLE_PB_SIZE 16
57
58#define NUM_MINOR_OF_SYSMMU_V3 4
59
60#define MMU_TLB_ENT_NUM(val) ((val) & 0x7F)
61
62enum exynos_sysmmu_inttype {
63 SYSMMU_PAGEFAULT,
64 SYSMMU_AR_MULTIHIT,
65 SYSMMU_AW_MULTIHIT,
66 SYSMMU_BUSERROR,
67 SYSMMU_AR_SECURITY,
68 SYSMMU_AR_ACCESS,
69 SYSMMU_AW_SECURITY,
70 SYSMMU_AW_PROTECTION, /* 7 */
71 SYSMMU_FAULT_UNDEF,
72 SYSMMU_FAULTS_NUM
73};
74
75static unsigned short fault_reg_offset[9] = {
76 REG_PAGE_FAULT_ADDR,
77 REG_AR_FAULT_ADDR,
78 REG_AW_FAULT_ADDR,
79 REG_PAGE_FAULT_ADDR,
80 REG_AR_FAULT_ADDR,
81 REG_AR_FAULT_ADDR,
82 REG_AW_FAULT_ADDR,
83 REG_AW_FAULT_ADDR
84};
85
86static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = {
87 "PAGE FAULT",
88 "AR MULTI-HIT FAULT",
89 "AW MULTI-HIT FAULT",
90 "BUS ERROR",
91 "AR SECURITY PROTECTION FAULT",
92 "AR ACCESS PROTECTION FAULT",
93 "AW SECURITY PROTECTION FAULT",
94 "AW ACCESS PROTECTION FAULT",
95 "UNDEFINED FAULT"
96};
97
98static bool has_sysmmu_capable_pbuf(void __iomem *sfrbase, int *min)
99{
100 unsigned int ver;
101
102 ver = __raw_sysmmu_version(sfrbase);
103 if (min)
104 *min = MMU_MIN_VER(ver);
105 return MMU_MAJ_VER(ver) == 3;
106}
107
108void __sysmmu_tlb_invalidate(struct sysmmu_drvdata *drvdata,
109 dma_addr_t iova, size_t size)
110{
111 void * __iomem sfrbase = drvdata->sfrbase;
112
113 if (!WARN_ON(!sysmmu_block(sfrbase))) {
114 __raw_writel(0x1, sfrbase + REG_MMU_FLUSH);
115 SYSMMU_EVENT_LOG_TLB_INV_ALL(SYSMMU_DRVDATA_TO_LOG(drvdata));
116 }
117 sysmmu_unblock(sfrbase);
118}
119
120void __sysmmu_tlb_invalidate_flpdcache(void __iomem *sfrbase, dma_addr_t iova)
121{
122 if (__raw_sysmmu_version(sfrbase) == MAKE_MMU_VER(3, 3))
123 __raw_writel(iova | 0x1, sfrbase + REG_MMU_FLUSH_ENTRY);
124}
125
126void __sysmmu_tlb_invalidate_entry(void __iomem *sfrbase, dma_addr_t iova)
127{
128 __raw_writel(iova | 0x1, sfrbase + REG_MMU_FLUSH_ENTRY);
129}
130
131void __sysmmu_set_ptbase(void __iomem *sfrbase, phys_addr_t pfn_pgtable)
132{
133 __raw_writel(pfn_pgtable * PAGE_SIZE, sfrbase + REG_PT_BASE_ADDR);
134
135 __raw_writel(0x1, sfrbase + REG_MMU_FLUSH);
136}
137
138static void __sysmmu_set_prefbuf(void __iomem *pbufbase, unsigned long base,
139 unsigned long size, int idx)
140{
141 __raw_writel(base, pbufbase + idx * 8);
142 __raw_writel(size - 1 + base, pbufbase + 4 + idx * 8);
143}
144
145/*
146 * Offset of prefetch buffer setting registers are different
147 * between SysMMU 3.1 and 3.2. 3.3 has a single prefetch buffer setting.
148 */
149static unsigned short
150 pbuf_offset[NUM_MINOR_OF_SYSMMU_V3] = {0x04C, 0x04C, 0x070, 0x410};
151
152
153static void __sysmmu_set_pbuf_ver31(struct sysmmu_drvdata *drvdata,
154 struct sysmmu_prefbuf prefbuf[], int num_bufs)
155{
156 unsigned long cfg =
157 __raw_readl(drvdata->sfrbase + REG_MMU_CFG) & CFG_MASK;
158
159 /* Only the first 2 buffers are set to PB */
160 if (num_bufs >= 2) {
161 /* Separate PB mode */
162 cfg |= 2 << 28;
163
164 if (prefbuf[1].size == 0)
165 prefbuf[1].size = 1;
166 __sysmmu_set_prefbuf(drvdata->sfrbase + pbuf_offset[1],
167 prefbuf[1].base, prefbuf[1].size, 1);
168 SYSMMU_EVENT_LOG_PBLMM(SYSMMU_DRVDATA_TO_LOG(drvdata),
169 cfg, num_bufs);
170 SYSMMU_EVENT_LOG_PBSET(SYSMMU_DRVDATA_TO_LOG(drvdata),
171 1, prefbuf[1].base,
172 prefbuf[1].base + prefbuf[1].size - 1);
173 } else {
174 /* Combined PB mode */
175 cfg |= 3 << 28;
176 drvdata->num_pbufs = 1;
177 drvdata->pbufs[0] = prefbuf[0];
178 SYSMMU_EVENT_LOG_PBLMM(SYSMMU_DRVDATA_TO_LOG(drvdata),
179 cfg, num_bufs);
180 }
181
182
183 __raw_writel(cfg, drvdata->sfrbase + REG_MMU_CFG);
184
185 if (prefbuf[0].size == 0)
186 prefbuf[0].size = 1;
187 __sysmmu_set_prefbuf(drvdata->sfrbase + pbuf_offset[1],
188 prefbuf[0].base, prefbuf[0].size, 0);
189 SYSMMU_EVENT_LOG_PBSET(SYSMMU_DRVDATA_TO_LOG(drvdata),
190 0, prefbuf[0].base,
191 prefbuf[0].base + prefbuf[0].size - 1);
192}
193
194static void __sysmmu_set_pbuf_ver32(struct sysmmu_drvdata *drvdata,
195 struct sysmmu_prefbuf prefbuf[], int num_bufs)
196{
197 static char pbidx[3][3] = { /* [numbufs][PB index] */
198 /* Index of Prefetch buffer entries */
199 {1, 0, 0}, {3, 1, 0}, {3, 2, 1},
200 };
201 int i;
202 unsigned long cfg =
203 __raw_readl(drvdata->sfrbase + REG_MMU_CFG) & CFG_MASK;
204
205 __raw_writel(0x1, drvdata->sfrbase + REG_MMU_FLUSH);
206
207 cfg |= 7 << 16; /* enabling PB0 ~ PB2 */
208
209 switch (num_bufs) {
210 case 1:
211 /* Combined PB mode (0 ~ 2) */
212 cfg |= 1 << 19;
213 break;
214 case 2:
215 /* Combined PB mode (0 ~ 1) */
216 cfg |= 1 << 21;
217 break;
218 case 3:
219 break;
220 default:
221 num_bufs = 3; /* Only the first 3 buffers are set to PB */
222 }
223
224 SYSMMU_EVENT_LOG_PBLMM(SYSMMU_DRVDATA_TO_LOG(drvdata), cfg, num_bufs);
225
226 for (i = 0; i < 3; i++) {
227 if (pbidx[num_bufs - 1][i]) {
228 if (prefbuf[i].size == 0) {
229 dev_err(drvdata->sysmmu,
230 "%s: Trying to init PB[%d/%d]with zero-size\n",
231 __func__, i, num_bufs);
232 prefbuf[i].size = 1;
233 }
234 __sysmmu_set_prefbuf(drvdata->sfrbase + pbuf_offset[2],
235 prefbuf[i].base, prefbuf[i].size,
236 pbidx[num_bufs - 1][i] - 1);
237
238 SYSMMU_EVENT_LOG_PBSET(SYSMMU_DRVDATA_TO_LOG(drvdata),
239 pbidx[num_bufs - 1][i] - 1,
240 prefbuf[i].base,
241 prefbuf[i].base + prefbuf[i].size - 1);
242 }
243 }
244
245 __raw_writel(cfg, drvdata->sfrbase + REG_MMU_CFG);
246}
247
248static unsigned int find_lmm_preset(unsigned int num_pb, unsigned int num_bufs)
249{
250 static char lmm_preset[4][6] = { /* [num of PB][num of buffers] */
251 /* 1, 2, 3, 4, 5, 6 */
252 { 1, 1, 0, -1, -1, -1}, /* num of pb: 3 */
253 { 3, 2, 1, 0, -1, -1}, /* num of pb: 4 */
254 {-1, -1, -1, -1, -1, -1},
255 { 5, 5, 4, 2, 1, 0}, /* num of pb: 6 */
256 };
257 unsigned int lmm;
258
259 BUG_ON(num_bufs > 6);
260 lmm = lmm_preset[num_pb - 3][num_bufs - 1];
261 BUG_ON(lmm == -1);
262 return lmm;
263}
264
265static unsigned int find_num_pb(unsigned int num_pb, unsigned int lmm)
266{
267 static char lmm_preset[6][6] = { /* [pb_num - 1][pb_lmm] */
268 {0, 0, 0, 0, 0, 0},
269 {0, 0, 0, 0, 0, 0},
270 {3, 2, 0, 0, 0, 0},
271 {4, 3, 2, 1, 0, 0},
272 {0, 0, 0, 0, 0, 0},
273 {6, 5, 4, 3, 3, 2},
274 };
275
276 num_pb = lmm_preset[num_pb - 1][lmm];
277 BUG_ON(num_pb == 0);
278 return num_pb;
279}
280
281static void __sysmmu_init_pb(void __iomem *sfrbase, unsigned int num_pb)
282{
283 unsigned int i = 0;
284
285 for (i = 0; i < num_pb; i++) {
286 __raw_writel(i, sfrbase + REG_PB_INDICATE);
287 __raw_writel(0, sfrbase + REG_PB_CFG);
288 }
289
290 __raw_writel(0x1, sfrbase + REG_MMU_FLUSH);
291}
292
293static void __sysmmu_set_pbuf_ver33(struct sysmmu_drvdata *drvdata,
294 struct sysmmu_prefbuf prefbuf[], int num_bufs)
295{
296 unsigned int i, num_pb, lmm;
297
298 num_pb = PB_INFO_NUM(__raw_readl(drvdata->sfrbase + REG_PB_INFO));
299
300 __sysmmu_init_pb(drvdata->sfrbase,
301 find_num_pb(num_pb,
302 __raw_readl(drvdata->sfrbase + REG_PB_LMM)));
303
304 lmm = find_lmm_preset(num_pb, (unsigned int)num_bufs);
305 num_pb = find_num_pb(num_pb, lmm);
306
307 __raw_writel(lmm, drvdata->sfrbase + REG_PB_LMM);
308
309 SYSMMU_EVENT_LOG_PBLMM(SYSMMU_DRVDATA_TO_LOG(drvdata), lmm, num_bufs);
310
311 for (i = 0; i < num_pb; i++) {
312 __raw_writel(i, drvdata->sfrbase + REG_PB_INDICATE);
313 if ((prefbuf[i].size > 0) && (i < num_bufs)) {
314 __sysmmu_set_prefbuf(drvdata->sfrbase + pbuf_offset[3],
315 prefbuf[i].base, prefbuf[i].size, 0);
316 __raw_writel(prefbuf[i].config | 1,
317 drvdata->sfrbase + REG_PB_CFG);
318 SYSMMU_EVENT_LOG_PBSET(SYSMMU_DRVDATA_TO_LOG(drvdata),
319 prefbuf[i].config | 1, prefbuf[i].base,
320 prefbuf[i].size - 1 + prefbuf[i].base);
321 } else {
322 if (prefbuf[i].size == 0) {
323 dev_err(drvdata->sysmmu,
324 "%s: Trying to init PB[%d/%d]with zero-size\n",
325 __func__, i, num_bufs);
326 }
327
328 SYSMMU_EVENT_LOG_PBSET(SYSMMU_DRVDATA_TO_LOG(drvdata),
329 0, 0, 0);
330 }
331 }
332}
333
334static void (*func_set_pbuf[NUM_MINOR_OF_SYSMMU_V3])
335 (struct sysmmu_drvdata *, struct sysmmu_prefbuf *, int) = {
336 __sysmmu_set_pbuf_ver31,
337 __sysmmu_set_pbuf_ver31,
338 __sysmmu_set_pbuf_ver32,
339 __sysmmu_set_pbuf_ver33,
340};
341
342
343static void __sysmmu_disable_pbuf_ver31(struct sysmmu_drvdata *drvdata)
344{
345 unsigned int cfg = __raw_readl(drvdata->sfrbase + REG_MMU_CFG);
346
347 cfg &= CFG_MASK;
348 __raw_writel(cfg, drvdata->sfrbase + REG_MMU_CFG);
349
350 SYSMMU_EVENT_LOG_PBLMM(SYSMMU_DRVDATA_TO_LOG(drvdata), cfg, 0);
351}
352
353#define __sysmmu_disable_pbuf_ver32 __sysmmu_disable_pbuf_ver31
354
355static void __sysmmu_disable_pbuf_ver33(struct sysmmu_drvdata *drvdata)
356{
357 unsigned int i, num_pb;
358
359 num_pb = PB_INFO_NUM(__raw_readl(drvdata->sfrbase + REG_PB_INFO));
360
361 __sysmmu_init_pb(drvdata->sfrbase,
362 find_num_pb(num_pb,
363 __raw_readl(drvdata->sfrbase + REG_PB_LMM)));
364
365 __raw_writel(0, drvdata->sfrbase + REG_PB_LMM);
366
367 SYSMMU_EVENT_LOG_PBLMM(SYSMMU_DRVDATA_TO_LOG(drvdata), 0, 0);
368
369 for (i = 0; i < num_pb; i++) {
370 __raw_writel(i, drvdata->sfrbase + REG_PB_INDICATE);
371 __raw_writel(0, drvdata->sfrbase + REG_PB_CFG);
372 SYSMMU_EVENT_LOG_PBSET(SYSMMU_DRVDATA_TO_LOG(drvdata), 0, 0, 0);
373 }
374}
375
376static void (*func_disable_pbuf[NUM_MINOR_OF_SYSMMU_V3])
377 (struct sysmmu_drvdata *) = {
378 __sysmmu_disable_pbuf_ver31,
379 __sysmmu_disable_pbuf_ver31,
380 __sysmmu_disable_pbuf_ver32,
381 __sysmmu_disable_pbuf_ver33,
382};
383
384static unsigned int __sysmmu_get_num_pb(struct sysmmu_drvdata *drvdata,
385 int *min)
386{
387 if (!has_sysmmu_capable_pbuf(drvdata->sfrbase, min))
388 return 0;
389
390 switch (*min) {
391 case 0:
392 case 1:
393 return 2;
394 case 2:
395 return 3;
396 case 3:
397 return PB_INFO_NUM(__raw_readl(drvdata->sfrbase + REG_PB_INFO));
398 default:
399 BUG();
400 }
401
402 return 0;
403}
404
405void __exynos_sysmmu_set_prefbuf_by_region(struct sysmmu_drvdata *drvdata,
406 struct sysmmu_prefbuf pb_reg[], unsigned int num_reg)
407{
408 unsigned int i;
409 int num_bufs = 0;
410 struct sysmmu_prefbuf prefbuf[6];
411 unsigned int version;
412
413 version = __raw_sysmmu_version(drvdata->sfrbase);
414 if (version < MAKE_MMU_VER(3, 0))
415 return;
416
417 if ((num_reg == 0) || (pb_reg == NULL)) {
418 /* Disabling prefetch buffers */
419 func_disable_pbuf[MMU_MIN_VER(version)](drvdata);
420 return;
421 }
422
423 for (i = 0; i < num_reg; i++) {
424 if (((pb_reg[i].config & SYSMMU_PBUFCFG_WRITE) &&
425 (drvdata->prop & SYSMMU_PROP_WRITE)) ||
426 (!(pb_reg[i].config & SYSMMU_PBUFCFG_WRITE) &&
427 (drvdata->prop & SYSMMU_PROP_READ)))
428 prefbuf[num_bufs++] = pb_reg[i];
429 }
430
431 func_set_pbuf[MMU_MIN_VER(version)](drvdata, prefbuf, num_bufs);
432}
433
434void __exynos_sysmmu_set_prefbuf_by_plane(struct sysmmu_drvdata *drvdata,
435 unsigned int inplanes, unsigned int onplanes,
436 unsigned int ipoption, unsigned int opoption)
437{
438 unsigned int num_pb;
439 int num_bufs, min;
440 struct sysmmu_prefbuf prefbuf[6];
441
442 num_pb = __sysmmu_get_num_pb(drvdata, &min);
443 if (num_pb == 0) /* No Prefetch buffers */
444 return;
445
446 num_bufs = __prepare_prefetch_buffers_by_plane(drvdata,
447 prefbuf, num_pb, inplanes, onplanes,
448 ipoption, opoption);
449
450 if (num_bufs == 0)
451 func_disable_pbuf[min](drvdata);
452 else
453 func_set_pbuf[min](drvdata, prefbuf, num_bufs);
454}
455
456static void dump_sysmmu_pb_v31(void __iomem *sfrbase)
457{
458 unsigned int cfg, i;
459 cfg = __raw_readl(sfrbase + REG_MMU_CFG);
460
461 switch ((cfg >> 28) & 0x3) {
462 case 2:
463 pr_crit("PB[1] [%#010x, %#010x] Cached VA: %08x\n",
464 __raw_readl(sfrbase + 0x54),
465 __raw_readl(sfrbase + 0x58),
466 __raw_readl(sfrbase + 0x60));
467 for (i = 0; i < 16; i++) {
468 __raw_writel((i << 4) | 1, sfrbase + 0x70);
469 pr_crit("PB[1][%2d] %08x\n", i,
470 __raw_readl(sfrbase + 0x74));
471 }
472 /* fall trhough */
473 case 1:
474 pr_crit("PB[0] [%#010x, %#010x] Cached VA: %08x\n",
475 __raw_readl(sfrbase + 0x4C),
476 __raw_readl(sfrbase + 0x50),
477 __raw_readl(sfrbase + 0x5C));
478 for (i = 0; i < 16; i++) {
479 __raw_writel((i << 4) | 1, sfrbase + 0x68);
480 pr_crit("PB[0][%2d] %08x\n", i,
481 __raw_readl(sfrbase + 0x6C));
482 }
483 break;
484 case 3:
485 pr_crit("PB[0] [%#010x, %#010x] Cached VA: %08x\n",
486 __raw_readl(sfrbase + 0x4C),
487 __raw_readl(sfrbase + 0x50),
488 __raw_readl(sfrbase + 0x5C));
489 for (i = 0; i < 32; i++) {
490 __raw_writel((i << 4) | 1, sfrbase + 0x68);
491 pr_crit("PB[0][%2d] %08x\n", i,
492 __raw_readl(sfrbase + 0x6C));
493 }
494 case 0:
495 break;
496 }
497}
498
499static void dump_sysmmu_pb_v32(void __iomem *sfrbase)
500{
501 unsigned int cfg, i;
502 cfg = __raw_readl(sfrbase + REG_MMU_CFG);
503
504 if (cfg & (1 << 19)) {
505 pr_crit("PB[0] [%#010x, %#010x] Cached VA: %08x\n",
506 __raw_readl(sfrbase + 0x70),
507 __raw_readl(sfrbase + 0x74),
508 __raw_readl(sfrbase + 0x88));
509 for (i = 0; i < 64; i++) {
510 __raw_writel((i << 4) | 1, sfrbase + 0x98);
511 pr_crit("PB[0][%2d] %08x\n", i,
512 __raw_readl(sfrbase + 0x9C));
513 }
514 return;
515 } else if (cfg & (1 << 21)) {
516 pr_crit("PB[0] [%#010x, %#010x] Cached VA: %08x\n",
517 __raw_readl(sfrbase + 0x70),
518 __raw_readl(sfrbase + 0x74),
519 __raw_readl(sfrbase + 0x88));
520 for (i = 0; i < 32; i++) {
521 __raw_writel((i << 4) | 1, sfrbase + 0x98);
522 pr_crit("PB[0][%2d] %08x\n", i,
523 __raw_readl(sfrbase + 0x9C));
524 }
525
526 if ((cfg & (1 << 18)) == 0)
527 return;
528
529 pr_crit("PB[2] [%#010x, %#010x] Cached VA: %08x\n",
530 __raw_readl(sfrbase + 0x80),
531 __raw_readl(sfrbase + 0x84),
532 __raw_readl(sfrbase + 0x90));
533 for (i = 0; i < 32; i++) {
534 __raw_writel((i << 4) | 1, sfrbase + 0xA8);
535 pr_crit("PB[2][%2d] %08x\n", i,
536 __raw_readl(sfrbase + 0xAC));
537 }
538
539 return;
540 }
541
542 if (cfg & (1 << 16)) {
543 pr_crit("PB[0] [%#010x, %#010x] Cached VA: %08x\n",
544 __raw_readl(sfrbase + 0x70),
545 __raw_readl(sfrbase + 0x74),
546 __raw_readl(sfrbase + 0x88));
547 for (i = 0; i < 16; i++) {
548 __raw_writel((i << 4) | 1, sfrbase + 0x98);
549 pr_crit("PB[0][%2d] %08x\n", i,
550 __raw_readl(sfrbase + 0x9C));
551 }
552 }
553
554 if (cfg & (1 << 17)) {
555 pr_crit("PB[1] [%#010x, %#010x] Cached VA: %08x\n",
556 __raw_readl(sfrbase + 0x78),
557 __raw_readl(sfrbase + 0x7C),
558 __raw_readl(sfrbase + 0x8C));
559 for (i = 0; i < 16; i++) {
560 __raw_writel((i << 4) | 1, sfrbase + 0xA0);
561 pr_crit("PB[1][%2d] %08x\n", i,
562 __raw_readl(sfrbase + 0xA4));
563 }
564 }
565
566 if (cfg & (1 << 18)) {
567 pr_crit("PB[2] [%#010x, %#010x] Cached VA: %08x\n",
568 __raw_readl(sfrbase + 0x80),
569 __raw_readl(sfrbase + 0x84),
570 __raw_readl(sfrbase + 0x90));
571 for (i = 0; i < 32; i++) {
572 __raw_writel((i << 4) | 1, sfrbase + 0xA8);
573 pr_crit("PB[2][%2d] %08x\n", i,
574 __raw_readl(sfrbase + 0xAC));
575 }
576 }
577}
578
579void dump_sysmmu_tlb_pb(void __iomem *sfrbase)
580{
581 unsigned int i, capa, lmm, tlb_ent_num, ver;
582
583 ver = MMU_RAW_VER(__raw_readl(sfrbase + REG_MMU_VERSION));
584
585 pr_crit("---------- System MMU Status -----------------------------\n");
586 pr_crit("VERSION %d.%d, MMU_CFG: %#010x, MMU_STATUS: %#010x\n",
587 MMU_MAJ_VER(ver), MMU_MIN_VER(ver),
588 __raw_readl(sfrbase + REG_MMU_CFG),
589 __raw_readl(sfrbase + REG_MMU_STATUS));
590
591 /* TODO: dump tlb with vpn for sysmmu v1 */
592 if (MMU_MAJ_VER(ver) < 2)
593 return;
594
595 pr_crit("---------- Level 1 TLB -----------------------------------\n");
596
597 tlb_ent_num = MMU_TLB_ENT_NUM(__raw_readl(sfrbase + REG_MMU_VERSION));
598 for (i = 0; i < tlb_ent_num; i++) {
599 __raw_writel((i << 4) | 1, sfrbase + REG_L1TLB_READ_ENTRY);
600 pr_crit("[%02d] VPN: %#010x, PPN: %#010x\n",
601 i, __raw_readl(sfrbase + REG_L1TLB_ENTRY_VPN),
602 __raw_readl(sfrbase + REG_L1TLB_ENTRY_PPN));
603 }
604
605 if (MMU_MAJ_VER(ver) < 3)
606 return;
607
608 pr_crit("---------- Prefetch Buffers ------------------------------\n");
609
610 if (MMU_MIN_VER(ver) < 2) {
611 dump_sysmmu_pb_v31(sfrbase);
612 return;
613 }
614
615 if (MMU_MIN_VER(ver) < 3) {
616 dump_sysmmu_pb_v32(sfrbase);
617 return;
618 }
619
620 capa = __raw_readl(sfrbase + REG_PB_INFO);
621 lmm = __raw_readl(sfrbase + REG_PB_LMM);
622
623 pr_crit("PB_INFO: %#010x, PB_LMM: %#010x\n", capa, lmm);
624
625 capa = find_num_pb(capa & 0xFF, lmm);
626
627 for (i = 0; i < capa; i++) {
628 __raw_writel(i, sfrbase + REG_PB_INDICATE);
629 pr_crit("PB[%d] = CFG: %#010x, START: %#010x, END: %#010x\n", i,
630 __raw_readl(sfrbase + REG_PB_CFG),
631 __raw_readl(sfrbase + REG_PB_START_ADDR),
632 __raw_readl(sfrbase + REG_PB_END_ADDR));
633
634 pr_crit("PB[%d]_SUB0 BASE_VPN = %#010x\n", i,
635 __raw_readl(sfrbase + REG_SPB_BASE_VPN));
636 __raw_writel(i | 0x100, sfrbase + REG_PB_INDICATE);
637 pr_crit("PB[%d]_SUB1 BASE_VPN = %#010x\n", i,
638 __raw_readl(sfrbase + REG_SPB_BASE_VPN));
639 }
640
641 /* Reading L2TLB is not provided by H/W */
642}
643
644static void show_fault_information(struct sysmmu_drvdata *drvdata,
645 enum exynos_sysmmu_inttype itype,
646 unsigned long fault_addr)
647{
648 unsigned int info;
649 phys_addr_t pgtable;
650 unsigned int version;
651
652 pgtable = __raw_readl(drvdata->sfrbase + REG_PT_BASE_ADDR);
653
654 pr_crit("----------------------------------------------------------\n");
655 pr_crit("%s %s at %#010lx by %s (page table @ %#010x)\n",
656 dev_name(drvdata->sysmmu),
657 sysmmu_fault_name[itype], fault_addr,
658 dev_name(drvdata->master), pgtable);
659
660 if (itype== SYSMMU_FAULT_UNDEF) {
661 pr_crit("The fault is not caused by this System MMU.\n");
662 pr_crit("Please check IRQ and SFR base address.\n");
663 goto finish;
664 }
665
666 version = __raw_sysmmu_version(drvdata->sfrbase);
667 if (version == MAKE_MMU_VER(3, 3)) {
668 info = __raw_readl(drvdata->sfrbase +
669 REG_FAULT_TRANS_INFO);
670 pr_crit("AxID: %#x, AxLEN: %#x RW: %s\n",
671 info & 0xFFFF, (info >> 16) & 0xF,
672 (info >> 20) ? "WRITE" : "READ");
673 }
674
675 if (pgtable != drvdata->pgtable)
676 pr_crit("Page table base of driver: %#010x\n",
677 drvdata->pgtable);
678
679 if (itype == SYSMMU_BUSERROR) {
680 pr_crit("System MMU has failed to access page table\n");
681 goto finish;
682 }
683
684 if (!pfn_valid(pgtable >> PAGE_SHIFT)) {
685 pr_crit("Page table base is not in a valid memory region\n");
686 } else {
687 sysmmu_pte_t *ent;
688 ent = section_entry(phys_to_virt(pgtable), fault_addr);
689 pr_crit("Lv1 entry: %#010x\n", *ent);
690
691 if (lv1ent_page(ent)) {
692 ent = page_entry(ent, fault_addr);
693 pr_crit("Lv2 entry: %#010x\n", *ent);
694 }
695 }
696
697 dump_sysmmu_tlb_pb(drvdata->sfrbase);
698
699finish:
700 pr_crit("----------------------------------------------------------\n");
701}
702
703irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
704{
705 /* SYSMMU is in blocked when interrupt occurred. */
706 struct sysmmu_drvdata *drvdata = dev_id;
707 unsigned int itype;
708 unsigned long addr = -1;
709 int ret = -ENOSYS;
710 int flags = 0;
711
712 WARN(!is_sysmmu_active(drvdata),
713 "Fault occurred while System MMU %s is not enabled!\n",
714 dev_name(drvdata->sysmmu));
715
716 itype = __ffs(__raw_readl(drvdata->sfrbase + REG_INT_STATUS));
717
718 if (WARN_ON(!((itype >= 0) && (itype < SYSMMU_FAULT_UNDEF))))
719 itype = SYSMMU_FAULT_UNDEF;
720 else
721 addr = __raw_readl(drvdata->sfrbase + fault_reg_offset[itype]);
722
723 show_fault_information(drvdata, itype, addr);
724
725 if (drvdata->domain) /* master is set if drvdata->domain exists */
726 ret = report_iommu_fault(drvdata->domain,
727 drvdata->master, addr, flags);
728
729 panic("Unrecoverable System MMU Fault!!");
730
731 return IRQ_HANDLED;
732}
733
734void __sysmmu_init_config(struct sysmmu_drvdata *drvdata)
735{
736 unsigned long cfg = CFG_LRU | CFG_QOS(drvdata->qos);
737 unsigned int version;
738
739 __raw_writel(0, drvdata->sfrbase + REG_MMU_CTRL);
740
741 version = __raw_sysmmu_version(drvdata->sfrbase);
742 if (version < MAKE_MMU_VER(3, 0))
743 goto set_cfg;
744
745 if (MMU_MAJ_VER(version) != 3)
746 panic("%s: Failed to read version (%d.%d), master: %s\n",
747 dev_name(drvdata->sysmmu), MMU_MAJ_VER(version),
748 MMU_MIN_VER(version), dev_name(drvdata->master));
749
750
751 if (MMU_MIN_VER(version) < 2)
752 goto set_pb;
753
754 BUG_ON(MMU_MIN_VER(version) > 3);
755
756 cfg |= CFG_FLPDCACHE;
757 cfg |= (MMU_MIN_VER(version) == 2) ? CFG_SYSSEL : CFG_ACGEN;
758 cfg |= CFG_QOS_OVRRIDE;
759
760set_pb:
761 __exynos_sysmmu_set_prefbuf_by_plane(drvdata, 0, 0,
762 SYSMMU_PBUFCFG_DEFAULT_INPUT,
763 SYSMMU_PBUFCFG_DEFAULT_OUTPUT);
764set_cfg:
765 cfg |= __raw_readl(drvdata->sfrbase + REG_MMU_CFG) & ~CFG_MASK;
766 __raw_writel(cfg, drvdata->sfrbase + REG_MMU_CFG);
767}
768
769void dump_sysmmu_ppc_cnt(struct sysmmu_drvdata *drvdata)
770{
771 static char ppc_event_name_preset[4][8] = { /* [type][event_name] */
772 {8, 8, -1, -1, 0, 0, -1, -1}, /*SYSMMU v1.2 */
773 {1, 1, 2, 2, 0, 0, -1, -1}, /* SYSMMU v2.1 */
774 {8, 8, 5, 5, 0, 0, 4, 4}, /* SYSMMU v3.1/2 */
775 {0, 8, 4, 5, 3, 9, 6, 7}, /* SYSMMU v3.3 */
776 };
777
778 unsigned int ver, type, offset;
779 int i, maj;
780 u32 cfg;
781
782 ver = __raw_sysmmu_version(drvdata->sfrbase);
783 maj = MMU_MAJ_VER(ver);
784
785 offset = (maj < 2) ? 0x40 : 0x58;
786
787 pr_crit("------------- System MMU PPC Status --------------\n");
788 for (i = 0; i < drvdata->event_cnt; i++) {
789 int event, write = 0;
790 unsigned char preset;
791 cfg = __raw_readl(drvdata->sfrbase +
792 REG_PPC_EVENT_SEL(offset, i));
793 event = cfg & 0xf;
794 if (ver == MAKE_MMU_VER(3, 3)) {
795 type = maj;
796 if (event > 0x7) {
797 write = 1;
798 event -= 0x8;
799 };
800 } else {
801 type = maj - 1;
802 write = !(event % 2);
803 }
804
805 if (ppc_event_name_preset[type][event] < 0) {
806 pr_err("PPC event value is unknown");
807 continue;
808 }
809
810 preset = ppc_event_name_preset[type][event];
811 pr_crit("%s %s %s CNT : %d", dev_name(drvdata->sysmmu),
812 write ? "WRITE" : "READ", ppc_event_name[preset],
813 __raw_readl(drvdata->sfrbase + REG_PPC_PMCNT(i)));
814 }
815 pr_crit("--------------------------------------------------\n");
816}
817
818int sysmmu_set_ppc_event(struct sysmmu_drvdata *drvdata, int event)
819{
820 static char ppc_event_preset[4][2][10] = { /* [type][write][event] */
821 { /* SYSMMU v1.2 */
822 {5, -1, -1, -1, -1, -1, -1, -1, 1, -1},
823 {4, -1, -1, -1, -1, -1, -1, -1, 0, -1},
824 },
825 { /* SYSMMU v2.x */
826 {5, 1, 3, -1, -1, -1, -1, -1, -1, -1},
827 {4, 0, 2, -1, -1, -1, -1, -1, -1, -1},
828 },
829 { /* SYSMMU v3.1/2 */
830 {5, -1, -1, -1, 7, 3, -1, -1, 1, -1},
831 {4, -1, -1, -1, 6, 2, -1, -1, 0, -1},
832 },
833 { /* SYSMMU v3.3 */
834 {0, -1, -1, 4, 2, 3, 6, 7, 1, 5},
835 {8, -1, -1, 12, 10, 11, 14, 15, 9, 13},
836 },
837 };
838 unsigned int ver, type, offset;
839 u32 cfg, write = 0;
840 char event_sel;
841
842 if (event < 0 || event > TOTAL_ID_NUM)
843 return -EINVAL;
844
845 if (event > READ_FLPD_MISS_PREFETCH) {
846 write = 1;
847 event -= 0x10;
848 }
849
850 ver = __raw_sysmmu_version(drvdata->sfrbase);
851
852 offset = (MMU_MAJ_VER(ver) < 2) ? 0x40 : 0x58;
853
854 if (!(ver == MAKE_MMU_VER(3, 3)))
855 type = MMU_MAJ_VER(ver) - 1;
856 else
857 type = MMU_MAJ_VER(ver);
858
859
860 event_sel = ppc_event_preset[type][write][event];
861 if (event_sel < 0)
862 return -EINVAL;
863
864 if (!drvdata->event_cnt)
865 __raw_writel(0x1, drvdata->sfrbase + REG_PPC_PMNC);
866
867 __raw_writel(event_sel, drvdata->sfrbase +
868 REG_PPC_EVENT_SEL(offset, drvdata->event_cnt));
869 cfg = __raw_readl(drvdata->sfrbase +
870 REG_PPC_CNTENS);
871 __raw_writel(cfg | 0x1 << drvdata->event_cnt,
872 drvdata->sfrbase + REG_PPC_CNTENS);
873
874 return 0;
875}