2 * TLB support routines.
4 * Copyright (C) 1998-2001, 2003 Hewlett-Packard Co
5 * David Mosberger-Tang <davidm@hpl.hp.com>
7 * 08/02/00 A. Mallick <asit.k.mallick@intel.com>
8 * Modified RID allocation for SMP
9 * Goutham Rao <goutham.rao@intel.com>
10 * IPI based ptc implementation and A-step IPI implementation.
11 * Rohit Seth <rohit.seth@intel.com>
12 * Ken Chen <kenneth.w.chen@intel.com>
14 #include <linux/module.h>
15 #include <linux/init.h>
16 #include <linux/kernel.h>
17 #include <linux/sched.h>
18 #include <linux/smp.h>
20 #include <linux/bootmem.h>
22 #include <asm/delay.h>
23 #include <asm/mmu_context.h>
24 #include <asm/pgalloc.h>
26 #include <asm/tlbflush.h>
30 unsigned long mask
; /* mask of supported purge page-sizes */
31 unsigned long max_bits
; /* log2 of largest supported purge page-size */
34 struct ia64_ctx ia64_ctx
= {
35 .lock
= __SPIN_LOCK_UNLOCKED(ia64_ctx
.lock
),
40 DEFINE_PER_CPU(u8
, ia64_need_tlb_flush
);
43 * Initializes the ia64_ctx.bitmap array based on max_ctx+1.
44 * Called after cpu_init() has setup ia64_ctx.max_ctx based on
45 * maximum RID that is supported by boot CPU.
48 mmu_context_init (void)
50 ia64_ctx
.bitmap
= alloc_bootmem((ia64_ctx
.max_ctx
+1)>>3);
51 ia64_ctx
.flushmap
= alloc_bootmem((ia64_ctx
.max_ctx
+1)>>3);
55 * Acquire the ia64_ctx.lock before calling this function!
58 wrap_mmu_context (struct mm_struct
*mm
)
61 unsigned long flush_bit
;
63 for (i
=0; i
<= ia64_ctx
.max_ctx
/ BITS_PER_LONG
; i
++) {
64 flush_bit
= xchg(&ia64_ctx
.flushmap
[i
], 0);
65 ia64_ctx
.bitmap
[i
] ^= flush_bit
;
68 /* use offset at 300 to skip daemons */
69 ia64_ctx
.next
= find_next_zero_bit(ia64_ctx
.bitmap
,
70 ia64_ctx
.max_ctx
, 300);
71 ia64_ctx
.limit
= find_next_bit(ia64_ctx
.bitmap
,
72 ia64_ctx
.max_ctx
, ia64_ctx
.next
);
75 * can't call flush_tlb_all() here because of race condition
76 * with O(1) scheduler [EF]
78 cpu
= get_cpu(); /* prevent preemption/migration */
79 for_each_online_cpu(i
)
81 per_cpu(ia64_need_tlb_flush
, i
) = 1;
83 local_flush_tlb_all();
87 ia64_global_tlb_purge (struct mm_struct
*mm
, unsigned long start
,
88 unsigned long end
, unsigned long nbits
)
90 static DEFINE_SPINLOCK(ptcg_lock
);
92 if (mm
!= current
->active_mm
|| !current
->mm
) {
97 /* HW requires global serialization of ptc.ga. */
98 spin_lock(&ptcg_lock
);
102 * Flush ALAT entries also.
104 ia64_ptcga(start
, (nbits
<<2));
106 start
+= (1UL << nbits
);
107 } while (start
< end
);
109 spin_unlock(&ptcg_lock
);
113 local_flush_tlb_all (void)
115 unsigned long i
, j
, flags
, count0
, count1
, stride0
, stride1
, addr
;
117 addr
= local_cpu_data
->ptce_base
;
118 count0
= local_cpu_data
->ptce_count
[0];
119 count1
= local_cpu_data
->ptce_count
[1];
120 stride0
= local_cpu_data
->ptce_stride
[0];
121 stride1
= local_cpu_data
->ptce_stride
[1];
123 local_irq_save(flags
);
124 for (i
= 0; i
< count0
; ++i
) {
125 for (j
= 0; j
< count1
; ++j
) {
131 local_irq_restore(flags
);
132 ia64_srlz_i(); /* srlz.i implies srlz.d */
136 flush_tlb_range (struct vm_area_struct
*vma
, unsigned long start
,
139 struct mm_struct
*mm
= vma
->vm_mm
;
140 unsigned long size
= end
- start
;
144 if (mm
!= current
->active_mm
) {
150 nbits
= ia64_fls(size
+ 0xfff);
151 while (unlikely (((1UL << nbits
) & purge
.mask
) == 0) &&
152 (nbits
< purge
.max_bits
))
154 if (nbits
> purge
.max_bits
)
155 nbits
= purge
.max_bits
;
156 start
&= ~((1UL << nbits
) - 1);
160 if (mm
!= current
->active_mm
|| cpus_weight(mm
->cpu_vm_mask
) != 1) {
161 platform_global_tlb_purge(mm
, start
, end
, nbits
);
167 ia64_ptcl(start
, (nbits
<<2));
168 start
+= (1UL << nbits
);
169 } while (start
< end
);
171 ia64_srlz_i(); /* srlz.i implies srlz.d */
173 EXPORT_SYMBOL(flush_tlb_range
);
178 ia64_ptce_info_t
uninitialized_var(ptce_info
); /* GCC be quiet */
179 unsigned long tr_pgbits
;
182 if ((status
= ia64_pal_vm_page_size(&tr_pgbits
, &purge
.mask
)) != 0) {
183 printk(KERN_ERR
"PAL_VM_PAGE_SIZE failed with status=%ld; "
184 "defaulting to architected purge page-sizes.\n", status
);
185 purge
.mask
= 0x115557000UL
;
187 purge
.max_bits
= ia64_fls(purge
.mask
);
189 ia64_get_ptce(&ptce_info
);
190 local_cpu_data
->ptce_base
= ptce_info
.base
;
191 local_cpu_data
->ptce_count
[0] = ptce_info
.count
[0];
192 local_cpu_data
->ptce_count
[1] = ptce_info
.count
[1];
193 local_cpu_data
->ptce_stride
[0] = ptce_info
.stride
[0];
194 local_cpu_data
->ptce_stride
[1] = ptce_info
.stride
[1];
196 local_flush_tlb_all(); /* nuke left overs from bootstrapping... */