Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * TLB support routines. | |
3 | * | |
4 | * Copyright (C) 1998-2001, 2003 Hewlett-Packard Co | |
5 | * David Mosberger-Tang <davidm@hpl.hp.com> | |
6 | * | |
7 | * 08/02/00 A. Mallick <asit.k.mallick@intel.com> | |
8 | * Modified RID allocation for SMP | |
9 | * Goutham Rao <goutham.rao@intel.com> | |
10 | * IPI based ptc implementation and A-step IPI implementation. | |
dcc17d1b PK |
11 | * Rohit Seth <rohit.seth@intel.com> |
12 | * Ken Chen <kenneth.w.chen@intel.com> | |
1da177e4 | 13 | */ |
1da177e4 LT |
14 | #include <linux/module.h> |
15 | #include <linux/init.h> | |
16 | #include <linux/kernel.h> | |
17 | #include <linux/sched.h> | |
18 | #include <linux/smp.h> | |
19 | #include <linux/mm.h> | |
dcc17d1b | 20 | #include <linux/bootmem.h> |
1da177e4 LT |
21 | |
22 | #include <asm/delay.h> | |
23 | #include <asm/mmu_context.h> | |
24 | #include <asm/pgalloc.h> | |
25 | #include <asm/pal.h> | |
26 | #include <asm/tlbflush.h> | |
dcc17d1b | 27 | #include <asm/dma.h> |
1da177e4 LT |
28 | |
29 | static struct { | |
30 | unsigned long mask; /* mask of supported purge page-sizes */ | |
58cd9082 | 31 | unsigned long max_bits; /* log2 of largest supported purge page-size */ |
1da177e4 LT |
32 | } purge; |
33 | ||
34 | struct ia64_ctx ia64_ctx = { | |
8737d595 MAC |
35 | .lock = __SPIN_LOCK_UNLOCKED(ia64_ctx.lock), |
36 | .next = 1, | |
37 | .max_ctx = ~0U | |
1da177e4 LT |
38 | }; |
39 | ||
40 | DEFINE_PER_CPU(u8, ia64_need_tlb_flush); | |
41 | ||
dcc17d1b PK |
42 | /* |
43 | * Initializes the ia64_ctx.bitmap array based on max_ctx+1. | |
44 | * Called after cpu_init() has setup ia64_ctx.max_ctx based on | |
45 | * maximum RID that is supported by boot CPU. | |
46 | */ | |
47 | void __init | |
48 | mmu_context_init (void) | |
49 | { | |
50 | ia64_ctx.bitmap = alloc_bootmem((ia64_ctx.max_ctx+1)>>3); | |
51 | ia64_ctx.flushmap = alloc_bootmem((ia64_ctx.max_ctx+1)>>3); | |
52 | } | |
53 | ||
1da177e4 LT |
54 | /* |
55 | * Acquire the ia64_ctx.lock before calling this function! | |
56 | */ | |
57 | void | |
58 | wrap_mmu_context (struct mm_struct *mm) | |
59 | { | |
58cd9082 | 60 | int i, cpu; |
dcc17d1b | 61 | unsigned long flush_bit; |
1da177e4 | 62 | |
dcc17d1b PK |
63 | for (i=0; i <= ia64_ctx.max_ctx / BITS_PER_LONG; i++) { |
64 | flush_bit = xchg(&ia64_ctx.flushmap[i], 0); | |
65 | ia64_ctx.bitmap[i] ^= flush_bit; | |
1da177e4 | 66 | } |
dcc17d1b PK |
67 | |
68 | /* use offset at 300 to skip daemons */ | |
69 | ia64_ctx.next = find_next_zero_bit(ia64_ctx.bitmap, | |
70 | ia64_ctx.max_ctx, 300); | |
71 | ia64_ctx.limit = find_next_bit(ia64_ctx.bitmap, | |
72 | ia64_ctx.max_ctx, ia64_ctx.next); | |
73 | ||
58cd9082 KC |
74 | /* |
75 | * can't call flush_tlb_all() here because of race condition | |
76 | * with O(1) scheduler [EF] | |
77 | */ | |
78 | cpu = get_cpu(); /* prevent preemption/migration */ | |
79 | for_each_online_cpu(i) | |
80 | if (i != cpu) | |
81 | per_cpu(ia64_need_tlb_flush, i) = 1; | |
82 | put_cpu(); | |
1da177e4 LT |
83 | local_flush_tlb_all(); |
84 | } | |
85 | ||
86 | void | |
58cd9082 KC |
87 | ia64_global_tlb_purge (struct mm_struct *mm, unsigned long start, |
88 | unsigned long end, unsigned long nbits) | |
1da177e4 LT |
89 | { |
90 | static DEFINE_SPINLOCK(ptcg_lock); | |
91 | ||
cfbb1426 | 92 | if (mm != current->active_mm || !current->mm) { |
c1902aae DR |
93 | flush_tlb_all(); |
94 | return; | |
95 | } | |
96 | ||
1da177e4 LT |
97 | /* HW requires global serialization of ptc.ga. */ |
98 | spin_lock(&ptcg_lock); | |
99 | { | |
100 | do { | |
101 | /* | |
102 | * Flush ALAT entries also. | |
103 | */ | |
104 | ia64_ptcga(start, (nbits<<2)); | |
105 | ia64_srlz_i(); | |
106 | start += (1UL << nbits); | |
107 | } while (start < end); | |
108 | } | |
109 | spin_unlock(&ptcg_lock); | |
110 | } | |
111 | ||
112 | void | |
113 | local_flush_tlb_all (void) | |
114 | { | |
115 | unsigned long i, j, flags, count0, count1, stride0, stride1, addr; | |
116 | ||
117 | addr = local_cpu_data->ptce_base; | |
118 | count0 = local_cpu_data->ptce_count[0]; | |
119 | count1 = local_cpu_data->ptce_count[1]; | |
120 | stride0 = local_cpu_data->ptce_stride[0]; | |
121 | stride1 = local_cpu_data->ptce_stride[1]; | |
122 | ||
123 | local_irq_save(flags); | |
124 | for (i = 0; i < count0; ++i) { | |
125 | for (j = 0; j < count1; ++j) { | |
126 | ia64_ptce(addr); | |
127 | addr += stride1; | |
128 | } | |
129 | addr += stride0; | |
130 | } | |
131 | local_irq_restore(flags); | |
132 | ia64_srlz_i(); /* srlz.i implies srlz.d */ | |
133 | } | |
134 | ||
135 | void | |
58cd9082 KC |
136 | flush_tlb_range (struct vm_area_struct *vma, unsigned long start, |
137 | unsigned long end) | |
1da177e4 LT |
138 | { |
139 | struct mm_struct *mm = vma->vm_mm; | |
140 | unsigned long size = end - start; | |
141 | unsigned long nbits; | |
142 | ||
c1902aae | 143 | #ifndef CONFIG_SMP |
1da177e4 | 144 | if (mm != current->active_mm) { |
1da177e4 | 145 | mm->context = 0; |
1da177e4 LT |
146 | return; |
147 | } | |
c1902aae | 148 | #endif |
1da177e4 LT |
149 | |
150 | nbits = ia64_fls(size + 0xfff); | |
58cd9082 KC |
151 | while (unlikely (((1UL << nbits) & purge.mask) == 0) && |
152 | (nbits < purge.max_bits)) | |
1da177e4 LT |
153 | ++nbits; |
154 | if (nbits > purge.max_bits) | |
155 | nbits = purge.max_bits; | |
156 | start &= ~((1UL << nbits) - 1); | |
157 | ||
663b97f7 | 158 | preempt_disable(); |
ce9eed5a KC |
159 | #ifdef CONFIG_SMP |
160 | if (mm != current->active_mm || cpus_weight(mm->cpu_vm_mask) != 1) { | |
161 | platform_global_tlb_purge(mm, start, end, nbits); | |
162 | preempt_enable(); | |
163 | return; | |
164 | } | |
165 | #endif | |
1da177e4 LT |
166 | do { |
167 | ia64_ptcl(start, (nbits<<2)); | |
168 | start += (1UL << nbits); | |
169 | } while (start < end); | |
663b97f7 | 170 | preempt_enable(); |
1da177e4 LT |
171 | ia64_srlz_i(); /* srlz.i implies srlz.d */ |
172 | } | |
173 | EXPORT_SYMBOL(flush_tlb_range); | |
174 | ||
175 | void __devinit | |
176 | ia64_tlb_init (void) | |
177 | { | |
256a7e09 | 178 | ia64_ptce_info_t uninitialized_var(ptce_info); /* GCC be quiet */ |
1da177e4 LT |
179 | unsigned long tr_pgbits; |
180 | long status; | |
181 | ||
182 | if ((status = ia64_pal_vm_page_size(&tr_pgbits, &purge.mask)) != 0) { | |
c2eeb321 | 183 | printk(KERN_ERR "PAL_VM_PAGE_SIZE failed with status=%ld; " |
1da177e4 LT |
184 | "defaulting to architected purge page-sizes.\n", status); |
185 | purge.mask = 0x115557000UL; | |
186 | } | |
187 | purge.max_bits = ia64_fls(purge.mask); | |
188 | ||
189 | ia64_get_ptce(&ptce_info); | |
190 | local_cpu_data->ptce_base = ptce_info.base; | |
191 | local_cpu_data->ptce_count[0] = ptce_info.count[0]; | |
192 | local_cpu_data->ptce_count[1] = ptce_info.count[1]; | |
193 | local_cpu_data->ptce_stride[0] = ptce_info.stride[0]; | |
194 | local_cpu_data->ptce_stride[1] = ptce_info.stride[1]; | |
195 | ||
58cd9082 | 196 | local_flush_tlb_all(); /* nuke left overs from bootstrapping... */ |
1da177e4 | 197 | } |