Commit | Line | Data |
---|---|---|
1da177e4 | 1 | /* |
4baa9922 | 2 | * arch/arm/include/asm/tlbflush.h |
1da177e4 LT |
3 | * |
4 | * Copyright (C) 1999-2003 Russell King | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 as | |
8 | * published by the Free Software Foundation. | |
9 | */ | |
10 | #ifndef _ASMARM_TLBFLUSH_H | |
11 | #define _ASMARM_TLBFLUSH_H | |
12 | ||
0157903e HC |
13 | |
14 | #ifndef CONFIG_MMU | |
15 | ||
16 | #define tlb_flush(tlb) ((void) tlb) | |
17 | ||
fb1c7762 | 18 | #else /* CONFIG_MMU */ |
0157903e | 19 | |
1da177e4 LT |
20 | #include <asm/glue.h> |
21 | ||
22 | #define TLB_V3_PAGE (1 << 0) | |
23 | #define TLB_V4_U_PAGE (1 << 1) | |
24 | #define TLB_V4_D_PAGE (1 << 2) | |
25 | #define TLB_V4_I_PAGE (1 << 3) | |
26 | #define TLB_V6_U_PAGE (1 << 4) | |
27 | #define TLB_V6_D_PAGE (1 << 5) | |
28 | #define TLB_V6_I_PAGE (1 << 6) | |
29 | ||
30 | #define TLB_V3_FULL (1 << 8) | |
31 | #define TLB_V4_U_FULL (1 << 9) | |
32 | #define TLB_V4_D_FULL (1 << 10) | |
33 | #define TLB_V4_I_FULL (1 << 11) | |
34 | #define TLB_V6_U_FULL (1 << 12) | |
35 | #define TLB_V6_D_FULL (1 << 13) | |
36 | #define TLB_V6_I_FULL (1 << 14) | |
37 | ||
38 | #define TLB_V6_U_ASID (1 << 16) | |
39 | #define TLB_V6_D_ASID (1 << 17) | |
40 | #define TLB_V6_I_ASID (1 << 18) | |
41 | ||
bba7d0b9 | 42 | #define TLB_BTB (1 << 28) |
faa7bc51 CM |
43 | |
44 | /* Unified Inner Shareable TLB operations (ARMv7 MP extensions) */ | |
45 | #define TLB_V7_UIS_PAGE (1 << 19) | |
46 | #define TLB_V7_UIS_FULL (1 << 20) | |
47 | #define TLB_V7_UIS_ASID (1 << 21) | |
48 | ||
b8349b56 CM |
49 | /* Inner Shareable BTB operation (ARMv7 MP extensions) */ |
50 | #define TLB_V7_IS_BTB (1 << 22) | |
51 | ||
99c6dc11 | 52 | #define TLB_L2CLEAN_FR (1 << 29) /* Feroceon */ |
1da177e4 LT |
53 | #define TLB_DCLEAN (1 << 30) |
54 | #define TLB_WB (1 << 31) | |
55 | ||
56 | /* | |
57 | * MMU TLB Model | |
58 | * ============= | |
59 | * | |
60 | * We have the following to choose from: | |
61 | * v3 - ARMv3 | |
62 | * v4 - ARMv4 without write buffer | |
63 | * v4wb - ARMv4 with write buffer without I TLB flush entry instruction | |
64 | * v4wbi - ARMv4 with write buffer with I TLB flush entry instruction | |
99c6dc11 | 65 | * fr - Feroceon (v4wbi with non-outer-cacheable page table walks) |
28853ac8 | 66 | * fa - Faraday (v4 with write buffer with UTLB and branch target buffer (BTB)) |
1da177e4 | 67 | * v6wbi - ARMv6 with write buffer with I TLB flush entry instruction |
61db7fb1 | 68 | * v7wbi - identical to v6wbi |
1da177e4 LT |
69 | */ |
70 | #undef _TLB | |
71 | #undef MULTI_TLB | |
72 | ||
f00ec48f RK |
73 | #ifdef CONFIG_SMP_ON_UP |
74 | #define MULTI_TLB 1 | |
75 | #endif | |
76 | ||
1da177e4 LT |
77 | #define v3_tlb_flags (TLB_V3_FULL | TLB_V3_PAGE) |
78 | ||
79 | #ifdef CONFIG_CPU_TLB_V3 | |
80 | # define v3_possible_flags v3_tlb_flags | |
81 | # define v3_always_flags v3_tlb_flags | |
82 | # ifdef _TLB | |
83 | # define MULTI_TLB 1 | |
84 | # else | |
85 | # define _TLB v3 | |
86 | # endif | |
87 | #else | |
88 | # define v3_possible_flags 0 | |
89 | # define v3_always_flags (-1UL) | |
90 | #endif | |
91 | ||
92 | #define v4_tlb_flags (TLB_V4_U_FULL | TLB_V4_U_PAGE) | |
93 | ||
94 | #ifdef CONFIG_CPU_TLB_V4WT | |
95 | # define v4_possible_flags v4_tlb_flags | |
96 | # define v4_always_flags v4_tlb_flags | |
97 | # ifdef _TLB | |
98 | # define MULTI_TLB 1 | |
99 | # else | |
100 | # define _TLB v4 | |
101 | # endif | |
102 | #else | |
103 | # define v4_possible_flags 0 | |
104 | # define v4_always_flags (-1UL) | |
105 | #endif | |
106 | ||
28853ac8 PZ |
107 | #define fa_tlb_flags (TLB_WB | TLB_BTB | TLB_DCLEAN | \ |
108 | TLB_V4_U_FULL | TLB_V4_U_PAGE) | |
109 | ||
110 | #ifdef CONFIG_CPU_TLB_FA | |
111 | # define fa_possible_flags fa_tlb_flags | |
112 | # define fa_always_flags fa_tlb_flags | |
113 | # ifdef _TLB | |
114 | # define MULTI_TLB 1 | |
115 | # else | |
116 | # define _TLB fa | |
117 | # endif | |
118 | #else | |
119 | # define fa_possible_flags 0 | |
120 | # define fa_always_flags (-1UL) | |
121 | #endif | |
122 | ||
1da177e4 LT |
123 | #define v4wbi_tlb_flags (TLB_WB | TLB_DCLEAN | \ |
124 | TLB_V4_I_FULL | TLB_V4_D_FULL | \ | |
125 | TLB_V4_I_PAGE | TLB_V4_D_PAGE) | |
126 | ||
127 | #ifdef CONFIG_CPU_TLB_V4WBI | |
128 | # define v4wbi_possible_flags v4wbi_tlb_flags | |
129 | # define v4wbi_always_flags v4wbi_tlb_flags | |
130 | # ifdef _TLB | |
131 | # define MULTI_TLB 1 | |
132 | # else | |
133 | # define _TLB v4wbi | |
134 | # endif | |
135 | #else | |
136 | # define v4wbi_possible_flags 0 | |
137 | # define v4wbi_always_flags (-1UL) | |
138 | #endif | |
139 | ||
99c6dc11 LB |
140 | #define fr_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_L2CLEAN_FR | \ |
141 | TLB_V4_I_FULL | TLB_V4_D_FULL | \ | |
142 | TLB_V4_I_PAGE | TLB_V4_D_PAGE) | |
143 | ||
144 | #ifdef CONFIG_CPU_TLB_FEROCEON | |
145 | # define fr_possible_flags fr_tlb_flags | |
146 | # define fr_always_flags fr_tlb_flags | |
147 | # ifdef _TLB | |
148 | # define MULTI_TLB 1 | |
149 | # else | |
150 | # define _TLB v4wbi | |
151 | # endif | |
152 | #else | |
153 | # define fr_possible_flags 0 | |
154 | # define fr_always_flags (-1UL) | |
155 | #endif | |
156 | ||
1da177e4 LT |
157 | #define v4wb_tlb_flags (TLB_WB | TLB_DCLEAN | \ |
158 | TLB_V4_I_FULL | TLB_V4_D_FULL | \ | |
159 | TLB_V4_D_PAGE) | |
160 | ||
161 | #ifdef CONFIG_CPU_TLB_V4WB | |
162 | # define v4wb_possible_flags v4wb_tlb_flags | |
163 | # define v4wb_always_flags v4wb_tlb_flags | |
164 | # ifdef _TLB | |
165 | # define MULTI_TLB 1 | |
166 | # else | |
167 | # define _TLB v4wb | |
168 | # endif | |
169 | #else | |
170 | # define v4wb_possible_flags 0 | |
171 | # define v4wb_always_flags (-1UL) | |
172 | #endif | |
173 | ||
bba7d0b9 | 174 | #define v6wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BTB | \ |
1da177e4 LT |
175 | TLB_V6_I_FULL | TLB_V6_D_FULL | \ |
176 | TLB_V6_I_PAGE | TLB_V6_D_PAGE | \ | |
177 | TLB_V6_I_ASID | TLB_V6_D_ASID) | |
178 | ||
179 | #ifdef CONFIG_CPU_TLB_V6 | |
180 | # define v6wbi_possible_flags v6wbi_tlb_flags | |
181 | # define v6wbi_always_flags v6wbi_tlb_flags | |
182 | # ifdef _TLB | |
183 | # define MULTI_TLB 1 | |
184 | # else | |
185 | # define _TLB v6wbi | |
186 | # endif | |
187 | #else | |
188 | # define v6wbi_possible_flags 0 | |
189 | # define v6wbi_always_flags (-1UL) | |
190 | #endif | |
191 | ||
f00ec48f | 192 | #define v7wbi_tlb_flags_smp (TLB_WB | TLB_DCLEAN | TLB_V7_IS_BTB | \ |
faa7bc51 | 193 | TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID) |
f00ec48f | 194 | #define v7wbi_tlb_flags_up (TLB_WB | TLB_DCLEAN | TLB_BTB | \ |
faa7bc51 | 195 | TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID) |
faa7bc51 | 196 | |
2ccdd1e7 | 197 | #ifdef CONFIG_CPU_TLB_V7 |
f00ec48f RK |
198 | |
199 | # ifdef CONFIG_SMP_ON_UP | |
200 | # define v7wbi_possible_flags (v7wbi_tlb_flags_smp | v7wbi_tlb_flags_up) | |
201 | # define v7wbi_always_flags (v7wbi_tlb_flags_smp & v7wbi_tlb_flags_up) | |
202 | # elif defined(CONFIG_SMP) | |
203 | # define v7wbi_possible_flags v7wbi_tlb_flags_smp | |
204 | # define v7wbi_always_flags v7wbi_tlb_flags_smp | |
205 | # else | |
206 | # define v7wbi_possible_flags v7wbi_tlb_flags_up | |
207 | # define v7wbi_always_flags v7wbi_tlb_flags_up | |
208 | # endif | |
2ccdd1e7 CM |
209 | # ifdef _TLB |
210 | # define MULTI_TLB 1 | |
211 | # else | |
212 | # define _TLB v7wbi | |
213 | # endif | |
214 | #else | |
215 | # define v7wbi_possible_flags 0 | |
216 | # define v7wbi_always_flags (-1UL) | |
217 | #endif | |
218 | ||
1da177e4 LT |
219 | #ifndef _TLB |
220 | #error Unknown TLB model | |
221 | #endif | |
222 | ||
223 | #ifndef __ASSEMBLY__ | |
224 | ||
e8edc6e0 AD |
225 | #include <linux/sched.h> |
226 | ||
1da177e4 LT |
227 | struct cpu_tlb_fns { |
228 | void (*flush_user_range)(unsigned long, unsigned long, struct vm_area_struct *); | |
229 | void (*flush_kern_range)(unsigned long, unsigned long); | |
230 | unsigned long tlb_flags; | |
231 | }; | |
232 | ||
233 | /* | |
234 | * Select the calling method | |
235 | */ | |
236 | #ifdef MULTI_TLB | |
237 | ||
238 | #define __cpu_flush_user_tlb_range cpu_tlb.flush_user_range | |
239 | #define __cpu_flush_kern_tlb_range cpu_tlb.flush_kern_range | |
240 | ||
241 | #else | |
242 | ||
243 | #define __cpu_flush_user_tlb_range __glue(_TLB,_flush_user_tlb_range) | |
244 | #define __cpu_flush_kern_tlb_range __glue(_TLB,_flush_kern_tlb_range) | |
245 | ||
246 | extern void __cpu_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *); | |
247 | extern void __cpu_flush_kern_tlb_range(unsigned long, unsigned long); | |
248 | ||
249 | #endif | |
250 | ||
251 | extern struct cpu_tlb_fns cpu_tlb; | |
252 | ||
253 | #define __cpu_tlb_flags cpu_tlb.tlb_flags | |
254 | ||
255 | /* | |
256 | * TLB Management | |
257 | * ============== | |
258 | * | |
259 | * The arch/arm/mm/tlb-*.S files implement these methods. | |
260 | * | |
261 | * The TLB specific code is expected to perform whatever tests it | |
262 | * needs to determine if it should invalidate the TLB for each | |
263 | * call. Start addresses are inclusive and end addresses are | |
264 | * exclusive; it is safe to round these addresses down. | |
265 | * | |
266 | * flush_tlb_all() | |
267 | * | |
268 | * Invalidate the entire TLB. | |
269 | * | |
270 | * flush_tlb_mm(mm) | |
271 | * | |
272 | * Invalidate all TLB entries in a particular address | |
273 | * space. | |
274 | * - mm - mm_struct describing address space | |
275 | * | |
276 | * flush_tlb_range(mm,start,end) | |
277 | * | |
278 | * Invalidate a range of TLB entries in the specified | |
279 | * address space. | |
280 | * - mm - mm_struct describing address space | |
281 | * - start - start address (may not be aligned) | |
282 | * - end - end address (exclusive, may not be aligned) | |
283 | * | |
284 | * flush_tlb_page(vaddr,vma) | |
285 | * | |
286 | * Invalidate the specified page in the specified address range. | |
287 | * - vaddr - virtual address (may not be aligned) | |
288 | * - vma - vma_struct describing address range | |
289 | * | |
290 | * flush_kern_tlb_page(kaddr) | |
291 | * | |
292 | * Invalidate the TLB entry for the specified page. The address | |
293 | * will be in the kernels virtual memory space. Current uses | |
294 | * only require the D-TLB to be invalidated. | |
295 | * - kaddr - Kernel virtual memory address | |
296 | */ | |
297 | ||
298 | /* | |
299 | * We optimise the code below by: | |
300 | * - building a set of TLB flags that might be set in __cpu_tlb_flags | |
301 | * - building a set of TLB flags that will always be set in __cpu_tlb_flags | |
302 | * - if we're going to need __cpu_tlb_flags, access it once and only once | |
303 | * | |
304 | * This allows us to build optimal assembly for the single-CPU type case, | |
305 | * and as close to optimal given the compiler constrants for multi-CPU | |
306 | * case. We could do better for the multi-CPU case if the compiler | |
307 | * implemented the "%?" method, but this has been discontinued due to too | |
308 | * many people getting it wrong. | |
309 | */ | |
310 | #define possible_tlb_flags (v3_possible_flags | \ | |
311 | v4_possible_flags | \ | |
312 | v4wbi_possible_flags | \ | |
99c6dc11 | 313 | fr_possible_flags | \ |
1da177e4 | 314 | v4wb_possible_flags | \ |
28853ac8 | 315 | fa_possible_flags | \ |
61db7fb1 PW |
316 | v6wbi_possible_flags | \ |
317 | v7wbi_possible_flags) | |
1da177e4 LT |
318 | |
319 | #define always_tlb_flags (v3_always_flags & \ | |
320 | v4_always_flags & \ | |
321 | v4wbi_always_flags & \ | |
99c6dc11 | 322 | fr_always_flags & \ |
1da177e4 | 323 | v4wb_always_flags & \ |
28853ac8 | 324 | fa_always_flags & \ |
61db7fb1 PW |
325 | v6wbi_always_flags & \ |
326 | v7wbi_always_flags) | |
1da177e4 LT |
327 | |
328 | #define tlb_flag(f) ((always_tlb_flags & (f)) || (__tlb_flag & possible_tlb_flags & (f))) | |
329 | ||
603fff54 | 330 | static inline void local_flush_tlb_all(void) |
1da177e4 LT |
331 | { |
332 | const int zero = 0; | |
333 | const unsigned int __tlb_flag = __cpu_tlb_flags; | |
334 | ||
335 | if (tlb_flag(TLB_WB)) | |
e6a5d66f | 336 | dsb(); |
1da177e4 LT |
337 | |
338 | if (tlb_flag(TLB_V3_FULL)) | |
6a39dd62 | 339 | asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (zero) : "cc"); |
1da177e4 | 340 | if (tlb_flag(TLB_V4_U_FULL | TLB_V6_U_FULL)) |
6a39dd62 | 341 | asm("mcr p15, 0, %0, c8, c7, 0" : : "r" (zero) : "cc"); |
1da177e4 | 342 | if (tlb_flag(TLB_V4_D_FULL | TLB_V6_D_FULL)) |
6a39dd62 | 343 | asm("mcr p15, 0, %0, c8, c6, 0" : : "r" (zero) : "cc"); |
1da177e4 | 344 | if (tlb_flag(TLB_V4_I_FULL | TLB_V6_I_FULL)) |
6a39dd62 | 345 | asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); |
faa7bc51 CM |
346 | if (tlb_flag(TLB_V7_UIS_FULL)) |
347 | asm("mcr p15, 0, %0, c8, c3, 0" : : "r" (zero) : "cc"); | |
e6a5d66f | 348 | |
bba7d0b9 | 349 | if (tlb_flag(TLB_BTB)) { |
e6a5d66f CM |
350 | /* flush the branch target cache */ |
351 | asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); | |
352 | dsb(); | |
353 | isb(); | |
354 | } | |
b8349b56 CM |
355 | if (tlb_flag(TLB_V7_IS_BTB)) { |
356 | /* flush the branch target cache */ | |
357 | asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc"); | |
358 | dsb(); | |
359 | isb(); | |
360 | } | |
1da177e4 LT |
361 | } |
362 | ||
603fff54 | 363 | static inline void local_flush_tlb_mm(struct mm_struct *mm) |
1da177e4 LT |
364 | { |
365 | const int zero = 0; | |
366 | const int asid = ASID(mm); | |
367 | const unsigned int __tlb_flag = __cpu_tlb_flags; | |
368 | ||
369 | if (tlb_flag(TLB_WB)) | |
e6a5d66f | 370 | dsb(); |
1da177e4 | 371 | |
daaeb6c9 | 372 | if (cpumask_test_cpu(get_cpu(), mm_cpumask(mm))) { |
1da177e4 | 373 | if (tlb_flag(TLB_V3_FULL)) |
6a39dd62 | 374 | asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (zero) : "cc"); |
1da177e4 | 375 | if (tlb_flag(TLB_V4_U_FULL)) |
6a39dd62 | 376 | asm("mcr p15, 0, %0, c8, c7, 0" : : "r" (zero) : "cc"); |
1da177e4 | 377 | if (tlb_flag(TLB_V4_D_FULL)) |
6a39dd62 | 378 | asm("mcr p15, 0, %0, c8, c6, 0" : : "r" (zero) : "cc"); |
1da177e4 | 379 | if (tlb_flag(TLB_V4_I_FULL)) |
6a39dd62 | 380 | asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); |
1da177e4 | 381 | } |
daaeb6c9 | 382 | put_cpu(); |
1da177e4 LT |
383 | |
384 | if (tlb_flag(TLB_V6_U_ASID)) | |
6a39dd62 | 385 | asm("mcr p15, 0, %0, c8, c7, 2" : : "r" (asid) : "cc"); |
1da177e4 | 386 | if (tlb_flag(TLB_V6_D_ASID)) |
6a39dd62 | 387 | asm("mcr p15, 0, %0, c8, c6, 2" : : "r" (asid) : "cc"); |
1da177e4 | 388 | if (tlb_flag(TLB_V6_I_ASID)) |
6a39dd62 | 389 | asm("mcr p15, 0, %0, c8, c5, 2" : : "r" (asid) : "cc"); |
faa7bc51 | 390 | if (tlb_flag(TLB_V7_UIS_ASID)) |
cdf357f1 WD |
391 | #ifdef CONFIG_ARM_ERRATA_720789 |
392 | asm("mcr p15, 0, %0, c8, c3, 0" : : "r" (zero) : "cc"); | |
393 | #else | |
faa7bc51 | 394 | asm("mcr p15, 0, %0, c8, c3, 2" : : "r" (asid) : "cc"); |
cdf357f1 | 395 | #endif |
e6a5d66f | 396 | |
bba7d0b9 | 397 | if (tlb_flag(TLB_BTB)) { |
e6a5d66f CM |
398 | /* flush the branch target cache */ |
399 | asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); | |
400 | dsb(); | |
401 | } | |
b8349b56 CM |
402 | if (tlb_flag(TLB_V7_IS_BTB)) { |
403 | /* flush the branch target cache */ | |
404 | asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc"); | |
405 | dsb(); | |
406 | isb(); | |
407 | } | |
1da177e4 LT |
408 | } |
409 | ||
410 | static inline void | |
603fff54 | 411 | local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) |
1da177e4 LT |
412 | { |
413 | const int zero = 0; | |
414 | const unsigned int __tlb_flag = __cpu_tlb_flags; | |
415 | ||
416 | uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm); | |
417 | ||
418 | if (tlb_flag(TLB_WB)) | |
e6a5d66f | 419 | dsb(); |
1da177e4 | 420 | |
56f8ba83 | 421 | if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) { |
1da177e4 | 422 | if (tlb_flag(TLB_V3_PAGE)) |
6a39dd62 | 423 | asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (uaddr) : "cc"); |
1da177e4 | 424 | if (tlb_flag(TLB_V4_U_PAGE)) |
6a39dd62 | 425 | asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (uaddr) : "cc"); |
1da177e4 | 426 | if (tlb_flag(TLB_V4_D_PAGE)) |
6a39dd62 | 427 | asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (uaddr) : "cc"); |
1da177e4 | 428 | if (tlb_flag(TLB_V4_I_PAGE)) |
6a39dd62 | 429 | asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (uaddr) : "cc"); |
1da177e4 | 430 | if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL)) |
6a39dd62 | 431 | asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); |
1da177e4 LT |
432 | } |
433 | ||
434 | if (tlb_flag(TLB_V6_U_PAGE)) | |
6a39dd62 | 435 | asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (uaddr) : "cc"); |
1da177e4 | 436 | if (tlb_flag(TLB_V6_D_PAGE)) |
6a39dd62 | 437 | asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (uaddr) : "cc"); |
1da177e4 | 438 | if (tlb_flag(TLB_V6_I_PAGE)) |
6a39dd62 | 439 | asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (uaddr) : "cc"); |
faa7bc51 | 440 | if (tlb_flag(TLB_V7_UIS_PAGE)) |
cdf357f1 WD |
441 | #ifdef CONFIG_ARM_ERRATA_720789 |
442 | asm("mcr p15, 0, %0, c8, c3, 3" : : "r" (uaddr & PAGE_MASK) : "cc"); | |
443 | #else | |
faa7bc51 | 444 | asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (uaddr) : "cc"); |
cdf357f1 | 445 | #endif |
e6a5d66f | 446 | |
bba7d0b9 | 447 | if (tlb_flag(TLB_BTB)) { |
e6a5d66f CM |
448 | /* flush the branch target cache */ |
449 | asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); | |
450 | dsb(); | |
451 | } | |
b8349b56 CM |
452 | if (tlb_flag(TLB_V7_IS_BTB)) { |
453 | /* flush the branch target cache */ | |
454 | asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc"); | |
455 | dsb(); | |
456 | isb(); | |
457 | } | |
1da177e4 LT |
458 | } |
459 | ||
603fff54 | 460 | static inline void local_flush_tlb_kernel_page(unsigned long kaddr) |
1da177e4 LT |
461 | { |
462 | const int zero = 0; | |
463 | const unsigned int __tlb_flag = __cpu_tlb_flags; | |
464 | ||
465 | kaddr &= PAGE_MASK; | |
466 | ||
467 | if (tlb_flag(TLB_WB)) | |
e6a5d66f | 468 | dsb(); |
1da177e4 LT |
469 | |
470 | if (tlb_flag(TLB_V3_PAGE)) | |
6a39dd62 | 471 | asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (kaddr) : "cc"); |
1da177e4 | 472 | if (tlb_flag(TLB_V4_U_PAGE)) |
6a39dd62 | 473 | asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (kaddr) : "cc"); |
1da177e4 | 474 | if (tlb_flag(TLB_V4_D_PAGE)) |
6a39dd62 | 475 | asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (kaddr) : "cc"); |
1da177e4 | 476 | if (tlb_flag(TLB_V4_I_PAGE)) |
6a39dd62 | 477 | asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (kaddr) : "cc"); |
1da177e4 | 478 | if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL)) |
6a39dd62 | 479 | asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); |
1da177e4 LT |
480 | |
481 | if (tlb_flag(TLB_V6_U_PAGE)) | |
6a39dd62 | 482 | asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (kaddr) : "cc"); |
1da177e4 | 483 | if (tlb_flag(TLB_V6_D_PAGE)) |
6a39dd62 | 484 | asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (kaddr) : "cc"); |
1da177e4 | 485 | if (tlb_flag(TLB_V6_I_PAGE)) |
6a39dd62 | 486 | asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (kaddr) : "cc"); |
faa7bc51 CM |
487 | if (tlb_flag(TLB_V7_UIS_PAGE)) |
488 | asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (kaddr) : "cc"); | |
6a0e2430 | 489 | |
bba7d0b9 | 490 | if (tlb_flag(TLB_BTB)) { |
e6a5d66f CM |
491 | /* flush the branch target cache */ |
492 | asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); | |
493 | dsb(); | |
494 | isb(); | |
495 | } | |
b8349b56 CM |
496 | if (tlb_flag(TLB_V7_IS_BTB)) { |
497 | /* flush the branch target cache */ | |
498 | asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc"); | |
499 | dsb(); | |
500 | isb(); | |
501 | } | |
1da177e4 LT |
502 | } |
503 | ||
504 | /* | |
505 | * flush_pmd_entry | |
506 | * | |
507 | * Flush a PMD entry (word aligned, or double-word aligned) to | |
508 | * RAM if the TLB for the CPU we are running on requires this. | |
509 | * This is typically used when we are creating PMD entries. | |
510 | * | |
511 | * clean_pmd_entry | |
512 | * | |
513 | * Clean (but don't drain the write buffer) if the CPU requires | |
514 | * these operations. This is typically used when we are removing | |
515 | * PMD entries. | |
516 | */ | |
517 | static inline void flush_pmd_entry(pmd_t *pmd) | |
518 | { | |
1da177e4 LT |
519 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
520 | ||
521 | if (tlb_flag(TLB_DCLEAN)) | |
6a39dd62 DJ |
522 | asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pmd" |
523 | : : "r" (pmd) : "cc"); | |
99c6dc11 LB |
524 | |
525 | if (tlb_flag(TLB_L2CLEAN_FR)) | |
526 | asm("mcr p15, 1, %0, c15, c9, 1 @ L2 flush_pmd" | |
527 | : : "r" (pmd) : "cc"); | |
528 | ||
1da177e4 | 529 | if (tlb_flag(TLB_WB)) |
e6a5d66f | 530 | dsb(); |
1da177e4 LT |
531 | } |
532 | ||
533 | static inline void clean_pmd_entry(pmd_t *pmd) | |
534 | { | |
535 | const unsigned int __tlb_flag = __cpu_tlb_flags; | |
536 | ||
537 | if (tlb_flag(TLB_DCLEAN)) | |
6a39dd62 DJ |
538 | asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pmd" |
539 | : : "r" (pmd) : "cc"); | |
99c6dc11 LB |
540 | |
541 | if (tlb_flag(TLB_L2CLEAN_FR)) | |
542 | asm("mcr p15, 1, %0, c15, c9, 1 @ L2 flush_pmd" | |
543 | : : "r" (pmd) : "cc"); | |
1da177e4 LT |
544 | } |
545 | ||
546 | #undef tlb_flag | |
547 | #undef always_tlb_flags | |
548 | #undef possible_tlb_flags | |
549 | ||
550 | /* | |
551 | * Convert calls to our calling convention. | |
552 | */ | |
603fff54 RK |
553 | #define local_flush_tlb_range(vma,start,end) __cpu_flush_user_tlb_range(start,end,vma) |
554 | #define local_flush_tlb_kernel_range(s,e) __cpu_flush_kern_tlb_range(s,e) | |
555 | ||
556 | #ifndef CONFIG_SMP | |
557 | #define flush_tlb_all local_flush_tlb_all | |
558 | #define flush_tlb_mm local_flush_tlb_mm | |
559 | #define flush_tlb_page local_flush_tlb_page | |
560 | #define flush_tlb_kernel_page local_flush_tlb_kernel_page | |
561 | #define flush_tlb_range local_flush_tlb_range | |
562 | #define flush_tlb_kernel_range local_flush_tlb_kernel_range | |
563 | #else | |
564 | extern void flush_tlb_all(void); | |
565 | extern void flush_tlb_mm(struct mm_struct *mm); | |
566 | extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr); | |
567 | extern void flush_tlb_kernel_page(unsigned long kaddr); | |
568 | extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); | |
569 | extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); | |
570 | #endif | |
1da177e4 LT |
571 | |
572 | /* | |
c0177800 | 573 | * If PG_dcache_clean is not set for the page, we need to ensure that any |
1da177e4 | 574 | * cache entries for the kernels virtual memory range are written |
6012191a CM |
575 | * back to the page. On ARMv6 and later, the cache coherency is handled via |
576 | * the set_pte_at() function. | |
1da177e4 | 577 | */ |
6012191a | 578 | #if __LINUX_ARM_ARCH__ < 6 |
4b3073e1 RK |
579 | extern void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, |
580 | pte_t *ptep); | |
6012191a CM |
581 | #else |
582 | static inline void update_mmu_cache(struct vm_area_struct *vma, | |
583 | unsigned long addr, pte_t *ptep) | |
584 | { | |
585 | } | |
586 | #endif | |
1da177e4 | 587 | |
1da177e4 LT |
588 | #endif |
589 | ||
0157903e HC |
590 | #endif /* CONFIG_MMU */ |
591 | ||
1da177e4 | 592 | #endif |