2 * arch/sparc64/mm/init.c
4 * Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu)
5 * Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
8 #include <linux/module.h>
9 #include <linux/kernel.h>
10 #include <linux/sched.h>
11 #include <linux/string.h>
12 #include <linux/init.h>
13 #include <linux/bootmem.h>
15 #include <linux/hugetlb.h>
16 #include <linux/initrd.h>
17 #include <linux/swap.h>
18 #include <linux/pagemap.h>
19 #include <linux/poison.h>
21 #include <linux/seq_file.h>
22 #include <linux/kprobes.h>
23 #include <linux/cache.h>
24 #include <linux/sort.h>
25 #include <linux/percpu.h>
26 #include <linux/memblock.h>
27 #include <linux/mmzone.h>
28 #include <linux/gfp.h>
32 #include <asm/pgalloc.h>
33 #include <asm/pgtable.h>
34 #include <asm/oplib.h>
35 #include <asm/iommu.h>
37 #include <asm/uaccess.h>
38 #include <asm/mmu_context.h>
39 #include <asm/tlbflush.h>
41 #include <asm/starfire.h>
43 #include <asm/spitfire.h>
44 #include <asm/sections.h>
46 #include <asm/hypervisor.h>
48 #include <asm/mdesc.h>
49 #include <asm/cpudata.h>
54 unsigned long kern_linear_pte_xor
[4] __read_mostly
;
56 /* A bitmap, two bits for every 256MB of physical memory. These two
57 * bits determine what page size we use for kernel linear
58 * translations. They form an index into kern_linear_pte_xor[]. The
59 * value in the indexed slot is XOR'd with the TLB miss virtual
60 * address to form the resulting TTE. The mapping is:
67 * All sun4v chips support 256MB pages. Only SPARC-T4 and later
68 * support 2GB pages, and hopefully future cpus will support the 16GB
69 * pages as well. For slots 2 and 3, we encode a 256MB TTE xor there
70 * if these larger page sizes are not supported by the cpu.
72 * It would be nice to determine this from the machine description
73 * 'cpu' properties, but we need to have this table setup before the
74 * MDESC is initialized.
76 unsigned long kpte_linear_bitmap
[KPTE_BITMAP_BYTES
/ sizeof(unsigned long)];
78 #ifndef CONFIG_DEBUG_PAGEALLOC
79 /* A special kernel TSB for 4MB, 256MB, 2GB and 16GB linear mappings.
80 * Space is allocated for this right after the trap table in
81 * arch/sparc64/kernel/head.S
83 extern struct tsb swapper_4m_tsb
[KERNEL_TSB4M_NENTRIES
];
86 static unsigned long cpu_pgsz_mask
;
90 static struct linux_prom64_registers pavail
[MAX_BANKS
];
91 static int pavail_ents
;
93 static int cmp_p64(const void *a
, const void *b
)
95 const struct linux_prom64_registers
*x
= a
, *y
= b
;
97 if (x
->phys_addr
> y
->phys_addr
)
99 if (x
->phys_addr
< y
->phys_addr
)
104 static void __init
read_obp_memory(const char *property
,
105 struct linux_prom64_registers
*regs
,
108 phandle node
= prom_finddevice("/memory");
109 int prop_size
= prom_getproplen(node
, property
);
112 ents
= prop_size
/ sizeof(struct linux_prom64_registers
);
113 if (ents
> MAX_BANKS
) {
114 prom_printf("The machine has more %s property entries than "
115 "this kernel can support (%d).\n",
116 property
, MAX_BANKS
);
120 ret
= prom_getproperty(node
, property
, (char *) regs
, prop_size
);
122 prom_printf("Couldn't get %s property from /memory.\n",
127 /* Sanitize what we got from the firmware, by page aligning
130 for (i
= 0; i
< ents
; i
++) {
131 unsigned long base
, size
;
133 base
= regs
[i
].phys_addr
;
134 size
= regs
[i
].reg_size
;
137 if (base
& ~PAGE_MASK
) {
138 unsigned long new_base
= PAGE_ALIGN(base
);
140 size
-= new_base
- base
;
141 if ((long) size
< 0L)
146 /* If it is empty, simply get rid of it.
147 * This simplifies the logic of the other
148 * functions that process these arrays.
150 memmove(®s
[i
], ®s
[i
+ 1],
151 (ents
- i
- 1) * sizeof(regs
[0]));
156 regs
[i
].phys_addr
= base
;
157 regs
[i
].reg_size
= size
;
162 sort(regs
, ents
, sizeof(struct linux_prom64_registers
),
166 unsigned long sparc64_valid_addr_bitmap
[VALID_ADDR_BITMAP_BYTES
/
167 sizeof(unsigned long)];
168 EXPORT_SYMBOL(sparc64_valid_addr_bitmap
);
170 /* Kernel physical address base and size in bytes. */
171 unsigned long kern_base __read_mostly
;
172 unsigned long kern_size __read_mostly
;
174 /* Initial ramdisk setup */
175 extern unsigned long sparc_ramdisk_image64
;
176 extern unsigned int sparc_ramdisk_image
;
177 extern unsigned int sparc_ramdisk_size
;
179 struct page
*mem_map_zero __read_mostly
;
180 EXPORT_SYMBOL(mem_map_zero
);
182 unsigned int sparc64_highest_unlocked_tlb_ent __read_mostly
;
184 unsigned long sparc64_kern_pri_context __read_mostly
;
185 unsigned long sparc64_kern_pri_nuc_bits __read_mostly
;
186 unsigned long sparc64_kern_sec_context __read_mostly
;
188 int num_kernel_image_mappings
;
190 #ifdef CONFIG_DEBUG_DCFLUSH
191 atomic_t dcpage_flushes
= ATOMIC_INIT(0);
193 atomic_t dcpage_flushes_xcall
= ATOMIC_INIT(0);
197 inline void flush_dcache_page_impl(struct page
*page
)
199 BUG_ON(tlb_type
== hypervisor
);
200 #ifdef CONFIG_DEBUG_DCFLUSH
201 atomic_inc(&dcpage_flushes
);
204 #ifdef DCACHE_ALIASING_POSSIBLE
205 __flush_dcache_page(page_address(page
),
206 ((tlb_type
== spitfire
) &&
207 page_mapping(page
) != NULL
));
209 if (page_mapping(page
) != NULL
&&
210 tlb_type
== spitfire
)
211 __flush_icache_page(__pa(page_address(page
)));
215 #define PG_dcache_dirty PG_arch_1
216 #define PG_dcache_cpu_shift 32UL
217 #define PG_dcache_cpu_mask \
218 ((1UL<<ilog2(roundup_pow_of_two(NR_CPUS)))-1UL)
220 #define dcache_dirty_cpu(page) \
221 (((page)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask)
223 static inline void set_dcache_dirty(struct page
*page
, int this_cpu
)
225 unsigned long mask
= this_cpu
;
226 unsigned long non_cpu_bits
;
228 non_cpu_bits
= ~(PG_dcache_cpu_mask
<< PG_dcache_cpu_shift
);
229 mask
= (mask
<< PG_dcache_cpu_shift
) | (1UL << PG_dcache_dirty
);
231 __asm__
__volatile__("1:\n\t"
233 "and %%g7, %1, %%g1\n\t"
234 "or %%g1, %0, %%g1\n\t"
235 "casx [%2], %%g7, %%g1\n\t"
237 "bne,pn %%xcc, 1b\n\t"
240 : "r" (mask
), "r" (non_cpu_bits
), "r" (&page
->flags
)
244 static inline void clear_dcache_dirty_cpu(struct page
*page
, unsigned long cpu
)
246 unsigned long mask
= (1UL << PG_dcache_dirty
);
248 __asm__
__volatile__("! test_and_clear_dcache_dirty\n"
251 "srlx %%g7, %4, %%g1\n\t"
252 "and %%g1, %3, %%g1\n\t"
254 "bne,pn %%icc, 2f\n\t"
255 " andn %%g7, %1, %%g1\n\t"
256 "casx [%2], %%g7, %%g1\n\t"
258 "bne,pn %%xcc, 1b\n\t"
262 : "r" (cpu
), "r" (mask
), "r" (&page
->flags
),
263 "i" (PG_dcache_cpu_mask
),
264 "i" (PG_dcache_cpu_shift
)
268 static inline void tsb_insert(struct tsb
*ent
, unsigned long tag
, unsigned long pte
)
270 unsigned long tsb_addr
= (unsigned long) ent
;
272 if (tlb_type
== cheetah_plus
|| tlb_type
== hypervisor
)
273 tsb_addr
= __pa(tsb_addr
);
275 __tsb_insert(tsb_addr
, tag
, pte
);
278 unsigned long _PAGE_ALL_SZ_BITS __read_mostly
;
280 static void flush_dcache(unsigned long pfn
)
284 page
= pfn_to_page(pfn
);
286 unsigned long pg_flags
;
288 pg_flags
= page
->flags
;
289 if (pg_flags
& (1UL << PG_dcache_dirty
)) {
290 int cpu
= ((pg_flags
>> PG_dcache_cpu_shift
) &
292 int this_cpu
= get_cpu();
294 /* This is just to optimize away some function calls
298 flush_dcache_page_impl(page
);
300 smp_flush_dcache_page_impl(page
, cpu
);
302 clear_dcache_dirty_cpu(page
, cpu
);
309 /* mm->context.lock must be held */
310 static void __update_mmu_tsb_insert(struct mm_struct
*mm
, unsigned long tsb_index
,
311 unsigned long tsb_hash_shift
, unsigned long address
,
314 struct tsb
*tsb
= mm
->context
.tsb_block
[tsb_index
].tsb
;
320 tsb
+= ((address
>> tsb_hash_shift
) &
321 (mm
->context
.tsb_block
[tsb_index
].tsb_nentries
- 1UL));
322 tag
= (address
>> 22UL);
323 tsb_insert(tsb
, tag
, tte
);
326 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
327 static inline bool is_hugetlb_pte(pte_t pte
)
329 if ((tlb_type
== hypervisor
&&
330 (pte_val(pte
) & _PAGE_SZALL_4V
) == _PAGE_SZHUGE_4V
) ||
331 (tlb_type
!= hypervisor
&&
332 (pte_val(pte
) & _PAGE_SZALL_4U
) == _PAGE_SZHUGE_4U
))
338 void update_mmu_cache(struct vm_area_struct
*vma
, unsigned long address
, pte_t
*ptep
)
340 struct mm_struct
*mm
;
344 if (tlb_type
!= hypervisor
) {
345 unsigned long pfn
= pte_pfn(pte
);
353 spin_lock_irqsave(&mm
->context
.lock
, flags
);
355 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
356 if (mm
->context
.huge_pte_count
&& is_hugetlb_pte(pte
))
357 __update_mmu_tsb_insert(mm
, MM_TSB_HUGE
, HPAGE_SHIFT
,
358 address
, pte_val(pte
));
361 __update_mmu_tsb_insert(mm
, MM_TSB_BASE
, PAGE_SHIFT
,
362 address
, pte_val(pte
));
364 spin_unlock_irqrestore(&mm
->context
.lock
, flags
);
367 void flush_dcache_page(struct page
*page
)
369 struct address_space
*mapping
;
372 if (tlb_type
== hypervisor
)
375 /* Do not bother with the expensive D-cache flush if it
376 * is merely the zero page. The 'bigcore' testcase in GDB
377 * causes this case to run millions of times.
379 if (page
== ZERO_PAGE(0))
382 this_cpu
= get_cpu();
384 mapping
= page_mapping(page
);
385 if (mapping
&& !mapping_mapped(mapping
)) {
386 int dirty
= test_bit(PG_dcache_dirty
, &page
->flags
);
388 int dirty_cpu
= dcache_dirty_cpu(page
);
390 if (dirty_cpu
== this_cpu
)
392 smp_flush_dcache_page_impl(page
, dirty_cpu
);
394 set_dcache_dirty(page
, this_cpu
);
396 /* We could delay the flush for the !page_mapping
397 * case too. But that case is for exec env/arg
398 * pages and those are %99 certainly going to get
399 * faulted into the tlb (and thus flushed) anyways.
401 flush_dcache_page_impl(page
);
407 EXPORT_SYMBOL(flush_dcache_page
);
409 void __kprobes
flush_icache_range(unsigned long start
, unsigned long end
)
411 /* Cheetah and Hypervisor platform cpus have coherent I-cache. */
412 if (tlb_type
== spitfire
) {
415 /* This code only runs on Spitfire cpus so this is
416 * why we can assume _PAGE_PADDR_4U.
418 for (kaddr
= start
; kaddr
< end
; kaddr
+= PAGE_SIZE
) {
419 unsigned long paddr
, mask
= _PAGE_PADDR_4U
;
421 if (kaddr
>= PAGE_OFFSET
)
422 paddr
= kaddr
& mask
;
424 pgd_t
*pgdp
= pgd_offset_k(kaddr
);
425 pud_t
*pudp
= pud_offset(pgdp
, kaddr
);
426 pmd_t
*pmdp
= pmd_offset(pudp
, kaddr
);
427 pte_t
*ptep
= pte_offset_kernel(pmdp
, kaddr
);
429 paddr
= pte_val(*ptep
) & mask
;
431 __flush_icache_page(paddr
);
435 EXPORT_SYMBOL(flush_icache_range
);
437 void mmu_info(struct seq_file
*m
)
439 static const char *pgsz_strings
[] = {
440 "8K", "64K", "512K", "4MB", "32MB",
441 "256MB", "2GB", "16GB",
445 if (tlb_type
== cheetah
)
446 seq_printf(m
, "MMU Type\t: Cheetah\n");
447 else if (tlb_type
== cheetah_plus
)
448 seq_printf(m
, "MMU Type\t: Cheetah+\n");
449 else if (tlb_type
== spitfire
)
450 seq_printf(m
, "MMU Type\t: Spitfire\n");
451 else if (tlb_type
== hypervisor
)
452 seq_printf(m
, "MMU Type\t: Hypervisor (sun4v)\n");
454 seq_printf(m
, "MMU Type\t: ???\n");
456 seq_printf(m
, "MMU PGSZs\t: ");
458 for (i
= 0; i
< ARRAY_SIZE(pgsz_strings
); i
++) {
459 if (cpu_pgsz_mask
& (1UL << i
)) {
460 seq_printf(m
, "%s%s",
461 printed
? "," : "", pgsz_strings
[i
]);
467 #ifdef CONFIG_DEBUG_DCFLUSH
468 seq_printf(m
, "DCPageFlushes\t: %d\n",
469 atomic_read(&dcpage_flushes
));
471 seq_printf(m
, "DCPageFlushesXC\t: %d\n",
472 atomic_read(&dcpage_flushes_xcall
));
473 #endif /* CONFIG_SMP */
474 #endif /* CONFIG_DEBUG_DCFLUSH */
477 struct linux_prom_translation prom_trans
[512] __read_mostly
;
478 unsigned int prom_trans_ents __read_mostly
;
480 unsigned long kern_locked_tte_data
;
482 /* The obp translations are saved based on 8k pagesize, since obp can
483 * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS ->
484 * HI_OBP_ADDRESS range are handled in ktlb.S.
486 static inline int in_obp_range(unsigned long vaddr
)
488 return (vaddr
>= LOW_OBP_ADDRESS
&&
489 vaddr
< HI_OBP_ADDRESS
);
492 static int cmp_ptrans(const void *a
, const void *b
)
494 const struct linux_prom_translation
*x
= a
, *y
= b
;
496 if (x
->virt
> y
->virt
)
498 if (x
->virt
< y
->virt
)
503 /* Read OBP translations property into 'prom_trans[]'. */
504 static void __init
read_obp_translations(void)
506 int n
, node
, ents
, first
, last
, i
;
508 node
= prom_finddevice("/virtual-memory");
509 n
= prom_getproplen(node
, "translations");
510 if (unlikely(n
== 0 || n
== -1)) {
511 prom_printf("prom_mappings: Couldn't get size.\n");
514 if (unlikely(n
> sizeof(prom_trans
))) {
515 prom_printf("prom_mappings: Size %d is too big.\n", n
);
519 if ((n
= prom_getproperty(node
, "translations",
520 (char *)&prom_trans
[0],
521 sizeof(prom_trans
))) == -1) {
522 prom_printf("prom_mappings: Couldn't get property.\n");
526 n
= n
/ sizeof(struct linux_prom_translation
);
530 sort(prom_trans
, ents
, sizeof(struct linux_prom_translation
),
533 /* Now kick out all the non-OBP entries. */
534 for (i
= 0; i
< ents
; i
++) {
535 if (in_obp_range(prom_trans
[i
].virt
))
539 for (; i
< ents
; i
++) {
540 if (!in_obp_range(prom_trans
[i
].virt
))
545 for (i
= 0; i
< (last
- first
); i
++) {
546 struct linux_prom_translation
*src
= &prom_trans
[i
+ first
];
547 struct linux_prom_translation
*dest
= &prom_trans
[i
];
551 for (; i
< ents
; i
++) {
552 struct linux_prom_translation
*dest
= &prom_trans
[i
];
553 dest
->virt
= dest
->size
= dest
->data
= 0x0UL
;
556 prom_trans_ents
= last
- first
;
558 if (tlb_type
== spitfire
) {
559 /* Clear diag TTE bits. */
560 for (i
= 0; i
< prom_trans_ents
; i
++)
561 prom_trans
[i
].data
&= ~0x0003fe0000000000UL
;
564 /* Force execute bit on. */
565 for (i
= 0; i
< prom_trans_ents
; i
++)
566 prom_trans
[i
].data
|= (tlb_type
== hypervisor
?
567 _PAGE_EXEC_4V
: _PAGE_EXEC_4U
);
570 static void __init
hypervisor_tlb_lock(unsigned long vaddr
,
574 unsigned long ret
= sun4v_mmu_map_perm_addr(vaddr
, 0, pte
, mmu
);
577 prom_printf("hypervisor_tlb_lock[%lx:%x:%lx:%lx]: "
578 "errors with %lx\n", vaddr
, 0, pte
, mmu
, ret
);
583 static unsigned long kern_large_tte(unsigned long paddr
);
585 static void __init
remap_kernel(void)
587 unsigned long phys_page
, tte_vaddr
, tte_data
;
588 int i
, tlb_ent
= sparc64_highest_locked_tlbent();
590 tte_vaddr
= (unsigned long) KERNBASE
;
591 phys_page
= (prom_boot_mapping_phys_low
>> 22UL) << 22UL;
592 tte_data
= kern_large_tte(phys_page
);
594 kern_locked_tte_data
= tte_data
;
596 /* Now lock us into the TLBs via Hypervisor or OBP. */
597 if (tlb_type
== hypervisor
) {
598 for (i
= 0; i
< num_kernel_image_mappings
; i
++) {
599 hypervisor_tlb_lock(tte_vaddr
, tte_data
, HV_MMU_DMMU
);
600 hypervisor_tlb_lock(tte_vaddr
, tte_data
, HV_MMU_IMMU
);
601 tte_vaddr
+= 0x400000;
602 tte_data
+= 0x400000;
605 for (i
= 0; i
< num_kernel_image_mappings
; i
++) {
606 prom_dtlb_load(tlb_ent
- i
, tte_data
, tte_vaddr
);
607 prom_itlb_load(tlb_ent
- i
, tte_data
, tte_vaddr
);
608 tte_vaddr
+= 0x400000;
609 tte_data
+= 0x400000;
611 sparc64_highest_unlocked_tlb_ent
= tlb_ent
- i
;
613 if (tlb_type
== cheetah_plus
) {
614 sparc64_kern_pri_context
= (CTX_CHEETAH_PLUS_CTX0
|
615 CTX_CHEETAH_PLUS_NUC
);
616 sparc64_kern_pri_nuc_bits
= CTX_CHEETAH_PLUS_NUC
;
617 sparc64_kern_sec_context
= CTX_CHEETAH_PLUS_CTX0
;
622 static void __init
inherit_prom_mappings(void)
624 /* Now fixup OBP's idea about where we really are mapped. */
625 printk("Remapping the kernel... ");
630 void prom_world(int enter
)
635 __asm__
__volatile__("flushw");
638 void __flush_dcache_range(unsigned long start
, unsigned long end
)
642 if (tlb_type
== spitfire
) {
645 for (va
= start
; va
< end
; va
+= 32) {
646 spitfire_put_dcache_tag(va
& 0x3fe0, 0x0);
650 } else if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
) {
653 for (va
= start
; va
< end
; va
+= 32)
654 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
658 "i" (ASI_DCACHE_INVALIDATE
));
661 EXPORT_SYMBOL(__flush_dcache_range
);
663 /* get_new_mmu_context() uses "cache + 1". */
664 DEFINE_SPINLOCK(ctx_alloc_lock
);
665 unsigned long tlb_context_cache
= CTX_FIRST_VERSION
- 1;
666 #define MAX_CTX_NR (1UL << CTX_NR_BITS)
667 #define CTX_BMAP_SLOTS BITS_TO_LONGS(MAX_CTX_NR)
668 DECLARE_BITMAP(mmu_context_bmap
, MAX_CTX_NR
);
670 /* Caller does TLB context flushing on local CPU if necessary.
671 * The caller also ensures that CTX_VALID(mm->context) is false.
673 * We must be careful about boundary cases so that we never
674 * let the user have CTX 0 (nucleus) or we ever use a CTX
675 * version of zero (and thus NO_CONTEXT would not be caught
676 * by version mis-match tests in mmu_context.h).
678 * Always invoked with interrupts disabled.
680 void get_new_mmu_context(struct mm_struct
*mm
)
682 unsigned long ctx
, new_ctx
;
683 unsigned long orig_pgsz_bits
;
687 spin_lock_irqsave(&ctx_alloc_lock
, flags
);
688 orig_pgsz_bits
= (mm
->context
.sparc64_ctx_val
& CTX_PGSZ_MASK
);
689 ctx
= (tlb_context_cache
+ 1) & CTX_NR_MASK
;
690 new_ctx
= find_next_zero_bit(mmu_context_bmap
, 1 << CTX_NR_BITS
, ctx
);
692 if (new_ctx
>= (1 << CTX_NR_BITS
)) {
693 new_ctx
= find_next_zero_bit(mmu_context_bmap
, ctx
, 1);
694 if (new_ctx
>= ctx
) {
696 new_ctx
= (tlb_context_cache
& CTX_VERSION_MASK
) +
699 new_ctx
= CTX_FIRST_VERSION
;
701 /* Don't call memset, for 16 entries that's just
704 mmu_context_bmap
[0] = 3;
705 mmu_context_bmap
[1] = 0;
706 mmu_context_bmap
[2] = 0;
707 mmu_context_bmap
[3] = 0;
708 for (i
= 4; i
< CTX_BMAP_SLOTS
; i
+= 4) {
709 mmu_context_bmap
[i
+ 0] = 0;
710 mmu_context_bmap
[i
+ 1] = 0;
711 mmu_context_bmap
[i
+ 2] = 0;
712 mmu_context_bmap
[i
+ 3] = 0;
718 mmu_context_bmap
[new_ctx
>>6] |= (1UL << (new_ctx
& 63));
719 new_ctx
|= (tlb_context_cache
& CTX_VERSION_MASK
);
721 tlb_context_cache
= new_ctx
;
722 mm
->context
.sparc64_ctx_val
= new_ctx
| orig_pgsz_bits
;
723 spin_unlock_irqrestore(&ctx_alloc_lock
, flags
);
725 if (unlikely(new_version
))
726 smp_new_mmu_context_version();
729 static int numa_enabled
= 1;
730 static int numa_debug
;
732 static int __init
early_numa(char *p
)
737 if (strstr(p
, "off"))
740 if (strstr(p
, "debug"))
745 early_param("numa", early_numa
);
747 #define numadbg(f, a...) \
748 do { if (numa_debug) \
749 printk(KERN_INFO f, ## a); \
752 static void __init
find_ramdisk(unsigned long phys_base
)
754 #ifdef CONFIG_BLK_DEV_INITRD
755 if (sparc_ramdisk_image
|| sparc_ramdisk_image64
) {
756 unsigned long ramdisk_image
;
758 /* Older versions of the bootloader only supported a
759 * 32-bit physical address for the ramdisk image
760 * location, stored at sparc_ramdisk_image. Newer
761 * SILO versions set sparc_ramdisk_image to zero and
762 * provide a full 64-bit physical address at
763 * sparc_ramdisk_image64.
765 ramdisk_image
= sparc_ramdisk_image
;
767 ramdisk_image
= sparc_ramdisk_image64
;
769 /* Another bootloader quirk. The bootloader normalizes
770 * the physical address to KERNBASE, so we have to
771 * factor that back out and add in the lowest valid
772 * physical page address to get the true physical address.
774 ramdisk_image
-= KERNBASE
;
775 ramdisk_image
+= phys_base
;
777 numadbg("Found ramdisk at physical address 0x%lx, size %u\n",
778 ramdisk_image
, sparc_ramdisk_size
);
780 initrd_start
= ramdisk_image
;
781 initrd_end
= ramdisk_image
+ sparc_ramdisk_size
;
783 memblock_reserve(initrd_start
, sparc_ramdisk_size
);
785 initrd_start
+= PAGE_OFFSET
;
786 initrd_end
+= PAGE_OFFSET
;
791 struct node_mem_mask
{
795 static struct node_mem_mask node_masks
[MAX_NUMNODES
];
796 static int num_node_masks
;
798 int numa_cpu_lookup_table
[NR_CPUS
];
799 cpumask_t numa_cpumask_lookup_table
[MAX_NUMNODES
];
801 #ifdef CONFIG_NEED_MULTIPLE_NODES
803 struct mdesc_mblock
{
806 u64 offset
; /* RA-to-PA */
808 static struct mdesc_mblock
*mblocks
;
809 static int num_mblocks
;
811 static unsigned long ra_to_pa(unsigned long addr
)
815 for (i
= 0; i
< num_mblocks
; i
++) {
816 struct mdesc_mblock
*m
= &mblocks
[i
];
818 if (addr
>= m
->base
&&
819 addr
< (m
->base
+ m
->size
)) {
827 static int find_node(unsigned long addr
)
831 addr
= ra_to_pa(addr
);
832 for (i
= 0; i
< num_node_masks
; i
++) {
833 struct node_mem_mask
*p
= &node_masks
[i
];
835 if ((addr
& p
->mask
) == p
->val
)
841 static u64
memblock_nid_range(u64 start
, u64 end
, int *nid
)
843 *nid
= find_node(start
);
845 while (start
< end
) {
846 int n
= find_node(start
);
860 /* This must be invoked after performing all of the necessary
861 * memblock_set_node() calls for 'nid'. We need to be able to get
862 * correct data from get_pfn_range_for_nid().
864 static void __init
allocate_node_data(int nid
)
866 struct pglist_data
*p
;
867 unsigned long start_pfn
, end_pfn
;
868 #ifdef CONFIG_NEED_MULTIPLE_NODES
871 paddr
= memblock_alloc_try_nid(sizeof(struct pglist_data
), SMP_CACHE_BYTES
, nid
);
873 prom_printf("Cannot allocate pglist_data for nid[%d]\n", nid
);
876 NODE_DATA(nid
) = __va(paddr
);
877 memset(NODE_DATA(nid
), 0, sizeof(struct pglist_data
));
879 NODE_DATA(nid
)->node_id
= nid
;
884 get_pfn_range_for_nid(nid
, &start_pfn
, &end_pfn
);
885 p
->node_start_pfn
= start_pfn
;
886 p
->node_spanned_pages
= end_pfn
- start_pfn
;
889 static void init_node_masks_nonnuma(void)
893 numadbg("Initializing tables for non-numa.\n");
895 node_masks
[0].mask
= node_masks
[0].val
= 0;
898 for (i
= 0; i
< NR_CPUS
; i
++)
899 numa_cpu_lookup_table
[i
] = 0;
901 cpumask_setall(&numa_cpumask_lookup_table
[0]);
904 #ifdef CONFIG_NEED_MULTIPLE_NODES
905 struct pglist_data
*node_data
[MAX_NUMNODES
];
907 EXPORT_SYMBOL(numa_cpu_lookup_table
);
908 EXPORT_SYMBOL(numa_cpumask_lookup_table
);
909 EXPORT_SYMBOL(node_data
);
911 struct mdesc_mlgroup
{
917 static struct mdesc_mlgroup
*mlgroups
;
918 static int num_mlgroups
;
920 static int scan_pio_for_cfg_handle(struct mdesc_handle
*md
, u64 pio
,
925 mdesc_for_each_arc(arc
, md
, pio
, MDESC_ARC_TYPE_FWD
) {
926 u64 target
= mdesc_arc_target(md
, arc
);
929 val
= mdesc_get_property(md
, target
,
931 if (val
&& *val
== cfg_handle
)
937 static int scan_arcs_for_cfg_handle(struct mdesc_handle
*md
, u64 grp
,
940 u64 arc
, candidate
, best_latency
= ~(u64
)0;
942 candidate
= MDESC_NODE_NULL
;
943 mdesc_for_each_arc(arc
, md
, grp
, MDESC_ARC_TYPE_FWD
) {
944 u64 target
= mdesc_arc_target(md
, arc
);
945 const char *name
= mdesc_node_name(md
, target
);
948 if (strcmp(name
, "pio-latency-group"))
951 val
= mdesc_get_property(md
, target
, "latency", NULL
);
955 if (*val
< best_latency
) {
961 if (candidate
== MDESC_NODE_NULL
)
964 return scan_pio_for_cfg_handle(md
, candidate
, cfg_handle
);
967 int of_node_to_nid(struct device_node
*dp
)
969 const struct linux_prom64_registers
*regs
;
970 struct mdesc_handle
*md
;
975 /* This is the right thing to do on currently supported
976 * SUN4U NUMA platforms as well, as the PCI controller does
977 * not sit behind any particular memory controller.
982 regs
= of_get_property(dp
, "reg", NULL
);
986 cfg_handle
= (regs
->phys_addr
>> 32UL) & 0x0fffffff;
992 mdesc_for_each_node_by_name(md
, grp
, "group") {
993 if (!scan_arcs_for_cfg_handle(md
, grp
, cfg_handle
)) {
1005 static void __init
add_node_ranges(void)
1007 struct memblock_region
*reg
;
1009 for_each_memblock(memory
, reg
) {
1010 unsigned long size
= reg
->size
;
1011 unsigned long start
, end
;
1015 while (start
< end
) {
1016 unsigned long this_end
;
1019 this_end
= memblock_nid_range(start
, end
, &nid
);
1021 numadbg("Setting memblock NUMA node nid[%d] "
1022 "start[%lx] end[%lx]\n",
1023 nid
, start
, this_end
);
1025 memblock_set_node(start
, this_end
- start
, nid
);
1031 static int __init
grab_mlgroups(struct mdesc_handle
*md
)
1033 unsigned long paddr
;
1037 mdesc_for_each_node_by_name(md
, node
, "memory-latency-group")
1042 paddr
= memblock_alloc(count
* sizeof(struct mdesc_mlgroup
),
1047 mlgroups
= __va(paddr
);
1048 num_mlgroups
= count
;
1051 mdesc_for_each_node_by_name(md
, node
, "memory-latency-group") {
1052 struct mdesc_mlgroup
*m
= &mlgroups
[count
++];
1057 val
= mdesc_get_property(md
, node
, "latency", NULL
);
1059 val
= mdesc_get_property(md
, node
, "address-match", NULL
);
1061 val
= mdesc_get_property(md
, node
, "address-mask", NULL
);
1064 numadbg("MLGROUP[%d]: node[%llx] latency[%llx] "
1065 "match[%llx] mask[%llx]\n",
1066 count
- 1, m
->node
, m
->latency
, m
->match
, m
->mask
);
1072 static int __init
grab_mblocks(struct mdesc_handle
*md
)
1074 unsigned long paddr
;
1078 mdesc_for_each_node_by_name(md
, node
, "mblock")
1083 paddr
= memblock_alloc(count
* sizeof(struct mdesc_mblock
),
1088 mblocks
= __va(paddr
);
1089 num_mblocks
= count
;
1092 mdesc_for_each_node_by_name(md
, node
, "mblock") {
1093 struct mdesc_mblock
*m
= &mblocks
[count
++];
1096 val
= mdesc_get_property(md
, node
, "base", NULL
);
1098 val
= mdesc_get_property(md
, node
, "size", NULL
);
1100 val
= mdesc_get_property(md
, node
,
1101 "address-congruence-offset", NULL
);
1104 numadbg("MBLOCK[%d]: base[%llx] size[%llx] offset[%llx]\n",
1105 count
- 1, m
->base
, m
->size
, m
->offset
);
1111 static void __init
numa_parse_mdesc_group_cpus(struct mdesc_handle
*md
,
1112 u64 grp
, cpumask_t
*mask
)
1116 cpumask_clear(mask
);
1118 mdesc_for_each_arc(arc
, md
, grp
, MDESC_ARC_TYPE_BACK
) {
1119 u64 target
= mdesc_arc_target(md
, arc
);
1120 const char *name
= mdesc_node_name(md
, target
);
1123 if (strcmp(name
, "cpu"))
1125 id
= mdesc_get_property(md
, target
, "id", NULL
);
1126 if (*id
< nr_cpu_ids
)
1127 cpumask_set_cpu(*id
, mask
);
1131 static struct mdesc_mlgroup
* __init
find_mlgroup(u64 node
)
1135 for (i
= 0; i
< num_mlgroups
; i
++) {
1136 struct mdesc_mlgroup
*m
= &mlgroups
[i
];
1137 if (m
->node
== node
)
1143 static int __init
numa_attach_mlgroup(struct mdesc_handle
*md
, u64 grp
,
1146 struct mdesc_mlgroup
*candidate
= NULL
;
1147 u64 arc
, best_latency
= ~(u64
)0;
1148 struct node_mem_mask
*n
;
1150 mdesc_for_each_arc(arc
, md
, grp
, MDESC_ARC_TYPE_FWD
) {
1151 u64 target
= mdesc_arc_target(md
, arc
);
1152 struct mdesc_mlgroup
*m
= find_mlgroup(target
);
1155 if (m
->latency
< best_latency
) {
1157 best_latency
= m
->latency
;
1163 if (num_node_masks
!= index
) {
1164 printk(KERN_ERR
"Inconsistent NUMA state, "
1165 "index[%d] != num_node_masks[%d]\n",
1166 index
, num_node_masks
);
1170 n
= &node_masks
[num_node_masks
++];
1172 n
->mask
= candidate
->mask
;
1173 n
->val
= candidate
->match
;
1175 numadbg("NUMA NODE[%d]: mask[%lx] val[%lx] (latency[%llx])\n",
1176 index
, n
->mask
, n
->val
, candidate
->latency
);
1181 static int __init
numa_parse_mdesc_group(struct mdesc_handle
*md
, u64 grp
,
1187 numa_parse_mdesc_group_cpus(md
, grp
, &mask
);
1189 for_each_cpu(cpu
, &mask
)
1190 numa_cpu_lookup_table
[cpu
] = index
;
1191 cpumask_copy(&numa_cpumask_lookup_table
[index
], &mask
);
1194 printk(KERN_INFO
"NUMA GROUP[%d]: cpus [ ", index
);
1195 for_each_cpu(cpu
, &mask
)
1200 return numa_attach_mlgroup(md
, grp
, index
);
1203 static int __init
numa_parse_mdesc(void)
1205 struct mdesc_handle
*md
= mdesc_grab();
1209 node
= mdesc_node_by_name(md
, MDESC_NODE_NULL
, "latency-groups");
1210 if (node
== MDESC_NODE_NULL
) {
1215 err
= grab_mblocks(md
);
1219 err
= grab_mlgroups(md
);
1224 mdesc_for_each_node_by_name(md
, node
, "group") {
1225 err
= numa_parse_mdesc_group(md
, node
, count
);
1233 for (i
= 0; i
< num_node_masks
; i
++) {
1234 allocate_node_data(i
);
1244 static int __init
numa_parse_jbus(void)
1246 unsigned long cpu
, index
;
1248 /* NUMA node id is encoded in bits 36 and higher, and there is
1249 * a 1-to-1 mapping from CPU ID to NUMA node ID.
1252 for_each_present_cpu(cpu
) {
1253 numa_cpu_lookup_table
[cpu
] = index
;
1254 cpumask_copy(&numa_cpumask_lookup_table
[index
], cpumask_of(cpu
));
1255 node_masks
[index
].mask
= ~((1UL << 36UL) - 1UL);
1256 node_masks
[index
].val
= cpu
<< 36UL;
1260 num_node_masks
= index
;
1264 for (index
= 0; index
< num_node_masks
; index
++) {
1265 allocate_node_data(index
);
1266 node_set_online(index
);
1272 static int __init
numa_parse_sun4u(void)
1274 if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
) {
1277 __asm__ ("rdpr %%ver, %0" : "=r" (ver
));
1278 if ((ver
>> 32UL) == __JALAPENO_ID
||
1279 (ver
>> 32UL) == __SERRANO_ID
)
1280 return numa_parse_jbus();
1285 static int __init
bootmem_init_numa(void)
1289 numadbg("bootmem_init_numa()\n");
1292 if (tlb_type
== hypervisor
)
1293 err
= numa_parse_mdesc();
1295 err
= numa_parse_sun4u();
1302 static int bootmem_init_numa(void)
1309 static void __init
bootmem_init_nonnuma(void)
1311 unsigned long top_of_ram
= memblock_end_of_DRAM();
1312 unsigned long total_ram
= memblock_phys_mem_size();
1314 numadbg("bootmem_init_nonnuma()\n");
1316 printk(KERN_INFO
"Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
1317 top_of_ram
, total_ram
);
1318 printk(KERN_INFO
"Memory hole size: %ldMB\n",
1319 (top_of_ram
- total_ram
) >> 20);
1321 init_node_masks_nonnuma();
1322 memblock_set_node(0, (phys_addr_t
)ULLONG_MAX
, 0);
1323 allocate_node_data(0);
1327 static unsigned long __init
bootmem_init(unsigned long phys_base
)
1329 unsigned long end_pfn
;
1331 end_pfn
= memblock_end_of_DRAM() >> PAGE_SHIFT
;
1332 max_pfn
= max_low_pfn
= end_pfn
;
1333 min_low_pfn
= (phys_base
>> PAGE_SHIFT
);
1335 if (bootmem_init_numa() < 0)
1336 bootmem_init_nonnuma();
1338 /* Dump memblock with node info. */
1339 memblock_dump_all();
1341 /* XXX cpu notifier XXX */
1343 sparse_memory_present_with_active_regions(MAX_NUMNODES
);
1349 static struct linux_prom64_registers pall
[MAX_BANKS
] __initdata
;
1350 static int pall_ents __initdata
;
1352 #ifdef CONFIG_DEBUG_PAGEALLOC
1353 static unsigned long __ref
kernel_map_range(unsigned long pstart
,
1354 unsigned long pend
, pgprot_t prot
)
1356 unsigned long vstart
= PAGE_OFFSET
+ pstart
;
1357 unsigned long vend
= PAGE_OFFSET
+ pend
;
1358 unsigned long alloc_bytes
= 0UL;
1360 if ((vstart
& ~PAGE_MASK
) || (vend
& ~PAGE_MASK
)) {
1361 prom_printf("kernel_map: Unaligned physmem[%lx:%lx]\n",
1366 while (vstart
< vend
) {
1367 unsigned long this_end
, paddr
= __pa(vstart
);
1368 pgd_t
*pgd
= pgd_offset_k(vstart
);
1373 pud
= pud_offset(pgd
, vstart
);
1374 if (pud_none(*pud
)) {
1377 new = __alloc_bootmem(PAGE_SIZE
, PAGE_SIZE
, PAGE_SIZE
);
1378 alloc_bytes
+= PAGE_SIZE
;
1379 pud_populate(&init_mm
, pud
, new);
1382 pmd
= pmd_offset(pud
, vstart
);
1383 if (!pmd_present(*pmd
)) {
1386 new = __alloc_bootmem(PAGE_SIZE
, PAGE_SIZE
, PAGE_SIZE
);
1387 alloc_bytes
+= PAGE_SIZE
;
1388 pmd_populate_kernel(&init_mm
, pmd
, new);
1391 pte
= pte_offset_kernel(pmd
, vstart
);
1392 this_end
= (vstart
+ PMD_SIZE
) & PMD_MASK
;
1393 if (this_end
> vend
)
1396 while (vstart
< this_end
) {
1397 pte_val(*pte
) = (paddr
| pgprot_val(prot
));
1399 vstart
+= PAGE_SIZE
;
1408 extern unsigned int kvmap_linear_patch
[1];
1409 #endif /* CONFIG_DEBUG_PAGEALLOC */
1411 static void __init
kpte_set_val(unsigned long index
, unsigned long val
)
1413 unsigned long *ptr
= kpte_linear_bitmap
;
1415 val
<<= ((index
% (BITS_PER_LONG
/ 2)) * 2);
1416 ptr
+= (index
/ (BITS_PER_LONG
/ 2));
1421 static const unsigned long kpte_shift_min
= 28; /* 256MB */
1422 static const unsigned long kpte_shift_max
= 34; /* 16GB */
1423 static const unsigned long kpte_shift_incr
= 3;
1425 static unsigned long kpte_mark_using_shift(unsigned long start
, unsigned long end
,
1426 unsigned long shift
)
1428 unsigned long size
= (1UL << shift
);
1429 unsigned long mask
= (size
- 1UL);
1430 unsigned long remains
= end
- start
;
1433 if (remains
< size
|| (start
& mask
))
1438 * shift 28 --> kern_linear_pte_xor index 1
1439 * shift 31 --> kern_linear_pte_xor index 2
1440 * shift 34 --> kern_linear_pte_xor index 3
1442 val
= ((shift
- kpte_shift_min
) / kpte_shift_incr
) + 1;
1445 if (shift
!= kpte_shift_max
)
1449 unsigned long index
= start
>> kpte_shift_min
;
1451 kpte_set_val(index
, val
);
1453 start
+= 1UL << kpte_shift_min
;
1454 remains
-= 1UL << kpte_shift_min
;
1460 static void __init
mark_kpte_bitmap(unsigned long start
, unsigned long end
)
1462 unsigned long smallest_size
, smallest_mask
;
1465 smallest_size
= (1UL << kpte_shift_min
);
1466 smallest_mask
= (smallest_size
- 1UL);
1468 while (start
< end
) {
1469 unsigned long orig_start
= start
;
1471 for (s
= kpte_shift_max
; s
>= kpte_shift_min
; s
-= kpte_shift_incr
) {
1472 start
= kpte_mark_using_shift(start
, end
, s
);
1474 if (start
!= orig_start
)
1478 if (start
== orig_start
)
1479 start
= (start
+ smallest_size
) & ~smallest_mask
;
1483 static void __init
init_kpte_bitmap(void)
1487 for (i
= 0; i
< pall_ents
; i
++) {
1488 unsigned long phys_start
, phys_end
;
1490 phys_start
= pall
[i
].phys_addr
;
1491 phys_end
= phys_start
+ pall
[i
].reg_size
;
1493 mark_kpte_bitmap(phys_start
, phys_end
);
1497 static void __init
kernel_physical_mapping_init(void)
1499 #ifdef CONFIG_DEBUG_PAGEALLOC
1500 unsigned long i
, mem_alloced
= 0UL;
1502 for (i
= 0; i
< pall_ents
; i
++) {
1503 unsigned long phys_start
, phys_end
;
1505 phys_start
= pall
[i
].phys_addr
;
1506 phys_end
= phys_start
+ pall
[i
].reg_size
;
1508 mem_alloced
+= kernel_map_range(phys_start
, phys_end
,
1512 printk("Allocated %ld bytes for kernel page tables.\n",
1515 kvmap_linear_patch
[0] = 0x01000000; /* nop */
1516 flushi(&kvmap_linear_patch
[0]);
1522 #ifdef CONFIG_DEBUG_PAGEALLOC
1523 void kernel_map_pages(struct page
*page
, int numpages
, int enable
)
1525 unsigned long phys_start
= page_to_pfn(page
) << PAGE_SHIFT
;
1526 unsigned long phys_end
= phys_start
+ (numpages
* PAGE_SIZE
);
1528 kernel_map_range(phys_start
, phys_end
,
1529 (enable
? PAGE_KERNEL
: __pgprot(0)));
1531 flush_tsb_kernel_range(PAGE_OFFSET
+ phys_start
,
1532 PAGE_OFFSET
+ phys_end
);
1534 /* we should perform an IPI and flush all tlbs,
1535 * but that can deadlock->flush only current cpu.
1537 __flush_tlb_kernel_range(PAGE_OFFSET
+ phys_start
,
1538 PAGE_OFFSET
+ phys_end
);
1542 unsigned long __init
find_ecache_flush_span(unsigned long size
)
1546 for (i
= 0; i
< pavail_ents
; i
++) {
1547 if (pavail
[i
].reg_size
>= size
)
1548 return pavail
[i
].phys_addr
;
1554 static void __init
tsb_phys_patch(void)
1556 struct tsb_ldquad_phys_patch_entry
*pquad
;
1557 struct tsb_phys_patch_entry
*p
;
1559 pquad
= &__tsb_ldquad_phys_patch
;
1560 while (pquad
< &__tsb_ldquad_phys_patch_end
) {
1561 unsigned long addr
= pquad
->addr
;
1563 if (tlb_type
== hypervisor
)
1564 *(unsigned int *) addr
= pquad
->sun4v_insn
;
1566 *(unsigned int *) addr
= pquad
->sun4u_insn
;
1568 __asm__
__volatile__("flush %0"
1575 p
= &__tsb_phys_patch
;
1576 while (p
< &__tsb_phys_patch_end
) {
1577 unsigned long addr
= p
->addr
;
1579 *(unsigned int *) addr
= p
->insn
;
1581 __asm__
__volatile__("flush %0"
1589 /* Don't mark as init, we give this to the Hypervisor. */
1590 #ifndef CONFIG_DEBUG_PAGEALLOC
1591 #define NUM_KTSB_DESCR 2
1593 #define NUM_KTSB_DESCR 1
1595 static struct hv_tsb_descr ktsb_descr
[NUM_KTSB_DESCR
];
1596 extern struct tsb swapper_tsb
[KERNEL_TSB_NENTRIES
];
1598 static void patch_one_ktsb_phys(unsigned int *start
, unsigned int *end
, unsigned long pa
)
1600 pa
>>= KTSB_PHYS_SHIFT
;
1602 while (start
< end
) {
1603 unsigned int *ia
= (unsigned int *)(unsigned long)*start
;
1605 ia
[0] = (ia
[0] & ~0x3fffff) | (pa
>> 10);
1606 __asm__
__volatile__("flush %0" : : "r" (ia
));
1608 ia
[1] = (ia
[1] & ~0x3ff) | (pa
& 0x3ff);
1609 __asm__
__volatile__("flush %0" : : "r" (ia
+ 1));
1615 static void ktsb_phys_patch(void)
1617 extern unsigned int __swapper_tsb_phys_patch
;
1618 extern unsigned int __swapper_tsb_phys_patch_end
;
1619 unsigned long ktsb_pa
;
1621 ktsb_pa
= kern_base
+ ((unsigned long)&swapper_tsb
[0] - KERNBASE
);
1622 patch_one_ktsb_phys(&__swapper_tsb_phys_patch
,
1623 &__swapper_tsb_phys_patch_end
, ktsb_pa
);
1624 #ifndef CONFIG_DEBUG_PAGEALLOC
1626 extern unsigned int __swapper_4m_tsb_phys_patch
;
1627 extern unsigned int __swapper_4m_tsb_phys_patch_end
;
1628 ktsb_pa
= (kern_base
+
1629 ((unsigned long)&swapper_4m_tsb
[0] - KERNBASE
));
1630 patch_one_ktsb_phys(&__swapper_4m_tsb_phys_patch
,
1631 &__swapper_4m_tsb_phys_patch_end
, ktsb_pa
);
1636 static void __init
sun4v_ktsb_init(void)
1638 unsigned long ktsb_pa
;
1640 /* First KTSB for PAGE_SIZE mappings. */
1641 ktsb_pa
= kern_base
+ ((unsigned long)&swapper_tsb
[0] - KERNBASE
);
1643 switch (PAGE_SIZE
) {
1646 ktsb_descr
[0].pgsz_idx
= HV_PGSZ_IDX_8K
;
1647 ktsb_descr
[0].pgsz_mask
= HV_PGSZ_MASK_8K
;
1651 ktsb_descr
[0].pgsz_idx
= HV_PGSZ_IDX_64K
;
1652 ktsb_descr
[0].pgsz_mask
= HV_PGSZ_MASK_64K
;
1656 ktsb_descr
[0].pgsz_idx
= HV_PGSZ_IDX_512K
;
1657 ktsb_descr
[0].pgsz_mask
= HV_PGSZ_MASK_512K
;
1660 case 4 * 1024 * 1024:
1661 ktsb_descr
[0].pgsz_idx
= HV_PGSZ_IDX_4MB
;
1662 ktsb_descr
[0].pgsz_mask
= HV_PGSZ_MASK_4MB
;
1666 ktsb_descr
[0].assoc
= 1;
1667 ktsb_descr
[0].num_ttes
= KERNEL_TSB_NENTRIES
;
1668 ktsb_descr
[0].ctx_idx
= 0;
1669 ktsb_descr
[0].tsb_base
= ktsb_pa
;
1670 ktsb_descr
[0].resv
= 0;
1672 #ifndef CONFIG_DEBUG_PAGEALLOC
1673 /* Second KTSB for 4MB/256MB/2GB/16GB mappings. */
1674 ktsb_pa
= (kern_base
+
1675 ((unsigned long)&swapper_4m_tsb
[0] - KERNBASE
));
1677 ktsb_descr
[1].pgsz_idx
= HV_PGSZ_IDX_4MB
;
1678 ktsb_descr
[1].pgsz_mask
= ((HV_PGSZ_MASK_4MB
|
1679 HV_PGSZ_MASK_256MB
|
1681 HV_PGSZ_MASK_16GB
) &
1683 ktsb_descr
[1].assoc
= 1;
1684 ktsb_descr
[1].num_ttes
= KERNEL_TSB4M_NENTRIES
;
1685 ktsb_descr
[1].ctx_idx
= 0;
1686 ktsb_descr
[1].tsb_base
= ktsb_pa
;
1687 ktsb_descr
[1].resv
= 0;
1691 void __cpuinit
sun4v_ktsb_register(void)
1693 unsigned long pa
, ret
;
1695 pa
= kern_base
+ ((unsigned long)&ktsb_descr
[0] - KERNBASE
);
1697 ret
= sun4v_mmu_tsb_ctx0(NUM_KTSB_DESCR
, pa
);
1699 prom_printf("hypervisor_mmu_tsb_ctx0[%lx]: "
1700 "errors with %lx\n", pa
, ret
);
1705 static void __init
sun4u_linear_pte_xor_finalize(void)
1707 #ifndef CONFIG_DEBUG_PAGEALLOC
1708 /* This is where we would add Panther support for
1709 * 32MB and 256MB pages.
1714 static void __init
sun4v_linear_pte_xor_finalize(void)
1716 #ifndef CONFIG_DEBUG_PAGEALLOC
1717 if (cpu_pgsz_mask
& HV_PGSZ_MASK_256MB
) {
1718 kern_linear_pte_xor
[1] = (_PAGE_VALID
| _PAGE_SZ256MB_4V
) ^
1719 0xfffff80000000000UL
;
1720 kern_linear_pte_xor
[1] |= (_PAGE_CP_4V
| _PAGE_CV_4V
|
1721 _PAGE_P_4V
| _PAGE_W_4V
);
1723 kern_linear_pte_xor
[1] = kern_linear_pte_xor
[0];
1726 if (cpu_pgsz_mask
& HV_PGSZ_MASK_2GB
) {
1727 kern_linear_pte_xor
[2] = (_PAGE_VALID
| _PAGE_SZ2GB_4V
) ^
1728 0xfffff80000000000UL
;
1729 kern_linear_pte_xor
[2] |= (_PAGE_CP_4V
| _PAGE_CV_4V
|
1730 _PAGE_P_4V
| _PAGE_W_4V
);
1732 kern_linear_pte_xor
[2] = kern_linear_pte_xor
[1];
1735 if (cpu_pgsz_mask
& HV_PGSZ_MASK_16GB
) {
1736 kern_linear_pte_xor
[3] = (_PAGE_VALID
| _PAGE_SZ16GB_4V
) ^
1737 0xfffff80000000000UL
;
1738 kern_linear_pte_xor
[3] |= (_PAGE_CP_4V
| _PAGE_CV_4V
|
1739 _PAGE_P_4V
| _PAGE_W_4V
);
1741 kern_linear_pte_xor
[3] = kern_linear_pte_xor
[2];
1746 /* paging_init() sets up the page tables */
1748 static unsigned long last_valid_pfn
;
1749 pgd_t swapper_pg_dir
[2048];
1751 static void sun4u_pgprot_init(void);
1752 static void sun4v_pgprot_init(void);
1754 void __init
paging_init(void)
1756 unsigned long end_pfn
, shift
, phys_base
;
1757 unsigned long real_end
, i
;
1760 /* These build time checkes make sure that the dcache_dirty_cpu()
1761 * page->flags usage will work.
1763 * When a page gets marked as dcache-dirty, we store the
1764 * cpu number starting at bit 32 in the page->flags. Also,
1765 * functions like clear_dcache_dirty_cpu use the cpu mask
1766 * in 13-bit signed-immediate instruction fields.
1770 * Page flags must not reach into upper 32 bits that are used
1771 * for the cpu number
1773 BUILD_BUG_ON(NR_PAGEFLAGS
> 32);
1776 * The bit fields placed in the high range must not reach below
1777 * the 32 bit boundary. Otherwise we cannot place the cpu field
1778 * at the 32 bit boundary.
1780 BUILD_BUG_ON(SECTIONS_WIDTH
+ NODES_WIDTH
+ ZONES_WIDTH
+
1781 ilog2(roundup_pow_of_two(NR_CPUS
)) > 32);
1783 BUILD_BUG_ON(NR_CPUS
> 4096);
1785 kern_base
= (prom_boot_mapping_phys_low
>> 22UL) << 22UL;
1786 kern_size
= (unsigned long)&_end
- (unsigned long)KERNBASE
;
1788 /* Invalidate both kernel TSBs. */
1789 memset(swapper_tsb
, 0x40, sizeof(swapper_tsb
));
1790 #ifndef CONFIG_DEBUG_PAGEALLOC
1791 memset(swapper_4m_tsb
, 0x40, sizeof(swapper_4m_tsb
));
1794 if (tlb_type
== hypervisor
)
1795 sun4v_pgprot_init();
1797 sun4u_pgprot_init();
1799 if (tlb_type
== cheetah_plus
||
1800 tlb_type
== hypervisor
) {
1805 if (tlb_type
== hypervisor
)
1806 sun4v_patch_tlb_handlers();
1808 /* Find available physical memory...
1810 * Read it twice in order to work around a bug in openfirmware.
1811 * The call to grab this table itself can cause openfirmware to
1812 * allocate memory, which in turn can take away some space from
1813 * the list of available memory. Reading it twice makes sure
1814 * we really do get the final value.
1816 read_obp_translations();
1817 read_obp_memory("reg", &pall
[0], &pall_ents
);
1818 read_obp_memory("available", &pavail
[0], &pavail_ents
);
1819 read_obp_memory("available", &pavail
[0], &pavail_ents
);
1821 phys_base
= 0xffffffffffffffffUL
;
1822 for (i
= 0; i
< pavail_ents
; i
++) {
1823 phys_base
= min(phys_base
, pavail
[i
].phys_addr
);
1824 memblock_add(pavail
[i
].phys_addr
, pavail
[i
].reg_size
);
1827 memblock_reserve(kern_base
, kern_size
);
1829 find_ramdisk(phys_base
);
1831 memblock_enforce_memory_limit(cmdline_memory_size
);
1833 memblock_allow_resize();
1834 memblock_dump_all();
1836 set_bit(0, mmu_context_bmap
);
1838 shift
= kern_base
+ PAGE_OFFSET
- ((unsigned long)KERNBASE
);
1840 real_end
= (unsigned long)_end
;
1841 num_kernel_image_mappings
= DIV_ROUND_UP(real_end
- KERNBASE
, 1 << 22);
1842 printk("Kernel: Using %d locked TLB entries for main kernel image.\n",
1843 num_kernel_image_mappings
);
1845 /* Set kernel pgd to upper alias so physical page computations
1848 init_mm
.pgd
+= ((shift
) / (sizeof(pgd_t
)));
1850 memset(swapper_low_pmd_dir
, 0, sizeof(swapper_low_pmd_dir
));
1852 /* Now can init the kernel/bad page tables. */
1853 pud_set(pud_offset(&swapper_pg_dir
[0], 0),
1854 swapper_low_pmd_dir
+ (shift
/ sizeof(pgd_t
)));
1856 inherit_prom_mappings();
1860 /* Ok, we can use our TLB miss and window trap handlers safely. */
1865 prom_build_devicetree();
1866 of_populate_present_mask();
1868 of_fill_in_cpu_data();
1871 if (tlb_type
== hypervisor
) {
1873 mdesc_populate_present_mask(cpu_all_mask
);
1875 mdesc_fill_in_cpu_data(cpu_all_mask
);
1877 mdesc_get_page_sizes(cpu_all_mask
, &cpu_pgsz_mask
);
1879 sun4v_linear_pte_xor_finalize();
1882 sun4v_ktsb_register();
1884 unsigned long impl
, ver
;
1886 cpu_pgsz_mask
= (HV_PGSZ_MASK_8K
| HV_PGSZ_MASK_64K
|
1887 HV_PGSZ_MASK_512K
| HV_PGSZ_MASK_4MB
);
1889 __asm__
__volatile__("rdpr %%ver, %0" : "=r" (ver
));
1890 impl
= ((ver
>> 32) & 0xffff);
1891 if (impl
== PANTHER_IMPL
)
1892 cpu_pgsz_mask
|= (HV_PGSZ_MASK_32MB
|
1893 HV_PGSZ_MASK_256MB
);
1895 sun4u_linear_pte_xor_finalize();
1898 /* Flush the TLBs and the 4M TSB so that the updated linear
1899 * pte XOR settings are realized for all mappings.
1902 #ifndef CONFIG_DEBUG_PAGEALLOC
1903 memset(swapper_4m_tsb
, 0x40, sizeof(swapper_4m_tsb
));
1907 /* Setup bootmem... */
1908 last_valid_pfn
= end_pfn
= bootmem_init(phys_base
);
1910 /* Once the OF device tree and MDESC have been setup, we know
1911 * the list of possible cpus. Therefore we can allocate the
1914 for_each_possible_cpu(i
) {
1915 node
= cpu_to_node(i
);
1917 softirq_stack
[i
] = __alloc_bootmem_node(NODE_DATA(node
),
1920 hardirq_stack
[i
] = __alloc_bootmem_node(NODE_DATA(node
),
1925 kernel_physical_mapping_init();
1928 unsigned long max_zone_pfns
[MAX_NR_ZONES
];
1930 memset(max_zone_pfns
, 0, sizeof(max_zone_pfns
));
1932 max_zone_pfns
[ZONE_NORMAL
] = end_pfn
;
1934 free_area_init_nodes(max_zone_pfns
);
1937 printk("Booting Linux...\n");
1940 int page_in_phys_avail(unsigned long paddr
)
1946 for (i
= 0; i
< pavail_ents
; i
++) {
1947 unsigned long start
, end
;
1949 start
= pavail
[i
].phys_addr
;
1950 end
= start
+ pavail
[i
].reg_size
;
1952 if (paddr
>= start
&& paddr
< end
)
1955 if (paddr
>= kern_base
&& paddr
< (kern_base
+ kern_size
))
1957 #ifdef CONFIG_BLK_DEV_INITRD
1958 if (paddr
>= __pa(initrd_start
) &&
1959 paddr
< __pa(PAGE_ALIGN(initrd_end
)))
1966 static struct linux_prom64_registers pavail_rescan
[MAX_BANKS
] __initdata
;
1967 static int pavail_rescan_ents __initdata
;
1969 /* Certain OBP calls, such as fetching "available" properties, can
1970 * claim physical memory. So, along with initializing the valid
1971 * address bitmap, what we do here is refetch the physical available
1972 * memory list again, and make sure it provides at least as much
1973 * memory as 'pavail' does.
1975 static void __init
setup_valid_addr_bitmap_from_pavail(unsigned long *bitmap
)
1979 read_obp_memory("available", &pavail_rescan
[0], &pavail_rescan_ents
);
1981 for (i
= 0; i
< pavail_ents
; i
++) {
1982 unsigned long old_start
, old_end
;
1984 old_start
= pavail
[i
].phys_addr
;
1985 old_end
= old_start
+ pavail
[i
].reg_size
;
1986 while (old_start
< old_end
) {
1989 for (n
= 0; n
< pavail_rescan_ents
; n
++) {
1990 unsigned long new_start
, new_end
;
1992 new_start
= pavail_rescan
[n
].phys_addr
;
1993 new_end
= new_start
+
1994 pavail_rescan
[n
].reg_size
;
1996 if (new_start
<= old_start
&&
1997 new_end
>= (old_start
+ PAGE_SIZE
)) {
1998 set_bit(old_start
>> 22, bitmap
);
2003 prom_printf("mem_init: Lost memory in pavail\n");
2004 prom_printf("mem_init: OLD start[%lx] size[%lx]\n",
2005 pavail
[i
].phys_addr
,
2006 pavail
[i
].reg_size
);
2007 prom_printf("mem_init: NEW start[%lx] size[%lx]\n",
2008 pavail_rescan
[i
].phys_addr
,
2009 pavail_rescan
[i
].reg_size
);
2010 prom_printf("mem_init: Cannot continue, aborting.\n");
2014 old_start
+= PAGE_SIZE
;
2019 static void __init
patch_tlb_miss_handler_bitmap(void)
2021 extern unsigned int valid_addr_bitmap_insn
[];
2022 extern unsigned int valid_addr_bitmap_patch
[];
2024 valid_addr_bitmap_insn
[1] = valid_addr_bitmap_patch
[1];
2026 valid_addr_bitmap_insn
[0] = valid_addr_bitmap_patch
[0];
2027 flushi(&valid_addr_bitmap_insn
[0]);
2030 static void __init
register_page_bootmem_info(void)
2032 #ifdef CONFIG_NEED_MULTIPLE_NODES
2035 for_each_online_node(i
)
2036 if (NODE_DATA(i
)->node_spanned_pages
)
2037 register_page_bootmem_info_node(NODE_DATA(i
));
2040 void __init
mem_init(void)
2042 unsigned long codepages
, datapages
, initpages
;
2043 unsigned long addr
, last
;
2045 addr
= PAGE_OFFSET
+ kern_base
;
2046 last
= PAGE_ALIGN(kern_size
) + addr
;
2047 while (addr
< last
) {
2048 set_bit(__pa(addr
) >> 22, sparc64_valid_addr_bitmap
);
2052 setup_valid_addr_bitmap_from_pavail(sparc64_valid_addr_bitmap
);
2053 patch_tlb_miss_handler_bitmap();
2055 high_memory
= __va(last_valid_pfn
<< PAGE_SHIFT
);
2057 register_page_bootmem_info();
2058 totalram_pages
= free_all_bootmem();
2060 /* We subtract one to account for the mem_map_zero page
2063 totalram_pages
-= 1;
2064 num_physpages
= totalram_pages
;
2067 * Set up the zero page, mark it reserved, so that page count
2068 * is not manipulated when freeing the page from user ptes.
2070 mem_map_zero
= alloc_pages(GFP_KERNEL
|__GFP_ZERO
, 0);
2071 if (mem_map_zero
== NULL
) {
2072 prom_printf("paging_init: Cannot alloc zero page.\n");
2075 SetPageReserved(mem_map_zero
);
2077 codepages
= (((unsigned long) _etext
) - ((unsigned long) _start
));
2078 codepages
= PAGE_ALIGN(codepages
) >> PAGE_SHIFT
;
2079 datapages
= (((unsigned long) _edata
) - ((unsigned long) _etext
));
2080 datapages
= PAGE_ALIGN(datapages
) >> PAGE_SHIFT
;
2081 initpages
= (((unsigned long) __init_end
) - ((unsigned long) __init_begin
));
2082 initpages
= PAGE_ALIGN(initpages
) >> PAGE_SHIFT
;
2084 printk("Memory: %luk available (%ldk kernel code, %ldk data, %ldk init) [%016lx,%016lx]\n",
2085 nr_free_pages() << (PAGE_SHIFT
-10),
2086 codepages
<< (PAGE_SHIFT
-10),
2087 datapages
<< (PAGE_SHIFT
-10),
2088 initpages
<< (PAGE_SHIFT
-10),
2089 PAGE_OFFSET
, (last_valid_pfn
<< PAGE_SHIFT
));
2091 if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
)
2092 cheetah_ecache_flush_init();
2095 void free_initmem(void)
2097 unsigned long addr
, initend
;
2100 /* If the physical memory maps were trimmed by kernel command
2101 * line options, don't even try freeing this initmem stuff up.
2102 * The kernel image could have been in the trimmed out region
2103 * and if so the freeing below will free invalid page structs.
2105 if (cmdline_memory_size
)
2109 * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.
2111 addr
= PAGE_ALIGN((unsigned long)(__init_begin
));
2112 initend
= (unsigned long)(__init_end
) & PAGE_MASK
;
2113 for (; addr
< initend
; addr
+= PAGE_SIZE
) {
2118 ((unsigned long) __va(kern_base
)) -
2119 ((unsigned long) KERNBASE
));
2120 memset((void *)addr
, POISON_FREE_INITMEM
, PAGE_SIZE
);
2123 p
= virt_to_page(page
);
2125 ClearPageReserved(p
);
2134 #ifdef CONFIG_BLK_DEV_INITRD
2135 void free_initrd_mem(unsigned long start
, unsigned long end
)
2138 printk ("Freeing initrd memory: %ldk freed\n", (end
- start
) >> 10);
2139 for (; start
< end
; start
+= PAGE_SIZE
) {
2140 struct page
*p
= virt_to_page(start
);
2142 ClearPageReserved(p
);
2151 #define _PAGE_CACHE_4U (_PAGE_CP_4U | _PAGE_CV_4U)
2152 #define _PAGE_CACHE_4V (_PAGE_CP_4V | _PAGE_CV_4V)
2153 #define __DIRTY_BITS_4U (_PAGE_MODIFIED_4U | _PAGE_WRITE_4U | _PAGE_W_4U)
2154 #define __DIRTY_BITS_4V (_PAGE_MODIFIED_4V | _PAGE_WRITE_4V | _PAGE_W_4V)
2155 #define __ACCESS_BITS_4U (_PAGE_ACCESSED_4U | _PAGE_READ_4U | _PAGE_R)
2156 #define __ACCESS_BITS_4V (_PAGE_ACCESSED_4V | _PAGE_READ_4V | _PAGE_R)
2158 pgprot_t PAGE_KERNEL __read_mostly
;
2159 EXPORT_SYMBOL(PAGE_KERNEL
);
2161 pgprot_t PAGE_KERNEL_LOCKED __read_mostly
;
2162 pgprot_t PAGE_COPY __read_mostly
;
2164 pgprot_t PAGE_SHARED __read_mostly
;
2165 EXPORT_SYMBOL(PAGE_SHARED
);
2167 unsigned long pg_iobits __read_mostly
;
2169 unsigned long _PAGE_IE __read_mostly
;
2170 EXPORT_SYMBOL(_PAGE_IE
);
2172 unsigned long _PAGE_E __read_mostly
;
2173 EXPORT_SYMBOL(_PAGE_E
);
2175 unsigned long _PAGE_CACHE __read_mostly
;
2176 EXPORT_SYMBOL(_PAGE_CACHE
);
2178 #ifdef CONFIG_SPARSEMEM_VMEMMAP
2179 unsigned long vmemmap_table
[VMEMMAP_SIZE
];
2181 static long __meminitdata addr_start
, addr_end
;
2182 static int __meminitdata node_start
;
2184 int __meminit
vmemmap_populate(struct page
*start
, unsigned long nr
, int node
)
2186 unsigned long vstart
= (unsigned long) start
;
2187 unsigned long vend
= (unsigned long) (start
+ nr
);
2188 unsigned long phys_start
= (vstart
- VMEMMAP_BASE
);
2189 unsigned long phys_end
= (vend
- VMEMMAP_BASE
);
2190 unsigned long addr
= phys_start
& VMEMMAP_CHUNK_MASK
;
2191 unsigned long end
= VMEMMAP_ALIGN(phys_end
);
2192 unsigned long pte_base
;
2194 pte_base
= (_PAGE_VALID
| _PAGE_SZ4MB_4U
|
2195 _PAGE_CP_4U
| _PAGE_CV_4U
|
2196 _PAGE_P_4U
| _PAGE_W_4U
);
2197 if (tlb_type
== hypervisor
)
2198 pte_base
= (_PAGE_VALID
| _PAGE_SZ4MB_4V
|
2199 _PAGE_CP_4V
| _PAGE_CV_4V
|
2200 _PAGE_P_4V
| _PAGE_W_4V
);
2202 for (; addr
< end
; addr
+= VMEMMAP_CHUNK
) {
2203 unsigned long *vmem_pp
=
2204 vmemmap_table
+ (addr
>> VMEMMAP_CHUNK_SHIFT
);
2207 if (!(*vmem_pp
& _PAGE_VALID
)) {
2208 block
= vmemmap_alloc_block(1UL << 22, node
);
2212 *vmem_pp
= pte_base
| __pa(block
);
2214 /* check to see if we have contiguous blocks */
2215 if (addr_end
!= addr
|| node_start
!= node
) {
2217 printk(KERN_DEBUG
" [%lx-%lx] on node %d\n",
2218 addr_start
, addr_end
-1, node_start
);
2222 addr_end
= addr
+ VMEMMAP_CHUNK
;
2228 void __meminit
vmemmap_populate_print_last(void)
2231 printk(KERN_DEBUG
" [%lx-%lx] on node %d\n",
2232 addr_start
, addr_end
-1, node_start
);
2239 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
2241 static void prot_init_common(unsigned long page_none
,
2242 unsigned long page_shared
,
2243 unsigned long page_copy
,
2244 unsigned long page_readonly
,
2245 unsigned long page_exec_bit
)
2247 PAGE_COPY
= __pgprot(page_copy
);
2248 PAGE_SHARED
= __pgprot(page_shared
);
2250 protection_map
[0x0] = __pgprot(page_none
);
2251 protection_map
[0x1] = __pgprot(page_readonly
& ~page_exec_bit
);
2252 protection_map
[0x2] = __pgprot(page_copy
& ~page_exec_bit
);
2253 protection_map
[0x3] = __pgprot(page_copy
& ~page_exec_bit
);
2254 protection_map
[0x4] = __pgprot(page_readonly
);
2255 protection_map
[0x5] = __pgprot(page_readonly
);
2256 protection_map
[0x6] = __pgprot(page_copy
);
2257 protection_map
[0x7] = __pgprot(page_copy
);
2258 protection_map
[0x8] = __pgprot(page_none
);
2259 protection_map
[0x9] = __pgprot(page_readonly
& ~page_exec_bit
);
2260 protection_map
[0xa] = __pgprot(page_shared
& ~page_exec_bit
);
2261 protection_map
[0xb] = __pgprot(page_shared
& ~page_exec_bit
);
2262 protection_map
[0xc] = __pgprot(page_readonly
);
2263 protection_map
[0xd] = __pgprot(page_readonly
);
2264 protection_map
[0xe] = __pgprot(page_shared
);
2265 protection_map
[0xf] = __pgprot(page_shared
);
2268 static void __init
sun4u_pgprot_init(void)
2270 unsigned long page_none
, page_shared
, page_copy
, page_readonly
;
2271 unsigned long page_exec_bit
;
2274 PAGE_KERNEL
= __pgprot (_PAGE_PRESENT_4U
| _PAGE_VALID
|
2275 _PAGE_CACHE_4U
| _PAGE_P_4U
|
2276 __ACCESS_BITS_4U
| __DIRTY_BITS_4U
|
2278 PAGE_KERNEL_LOCKED
= __pgprot (_PAGE_PRESENT_4U
| _PAGE_VALID
|
2279 _PAGE_CACHE_4U
| _PAGE_P_4U
|
2280 __ACCESS_BITS_4U
| __DIRTY_BITS_4U
|
2281 _PAGE_EXEC_4U
| _PAGE_L_4U
);
2283 _PAGE_IE
= _PAGE_IE_4U
;
2284 _PAGE_E
= _PAGE_E_4U
;
2285 _PAGE_CACHE
= _PAGE_CACHE_4U
;
2287 pg_iobits
= (_PAGE_VALID
| _PAGE_PRESENT_4U
| __DIRTY_BITS_4U
|
2288 __ACCESS_BITS_4U
| _PAGE_E_4U
);
2290 #ifdef CONFIG_DEBUG_PAGEALLOC
2291 kern_linear_pte_xor
[0] = _PAGE_VALID
^ 0xfffff80000000000UL
;
2293 kern_linear_pte_xor
[0] = (_PAGE_VALID
| _PAGE_SZ4MB_4U
) ^
2294 0xfffff80000000000UL
;
2296 kern_linear_pte_xor
[0] |= (_PAGE_CP_4U
| _PAGE_CV_4U
|
2297 _PAGE_P_4U
| _PAGE_W_4U
);
2299 for (i
= 1; i
< 4; i
++)
2300 kern_linear_pte_xor
[i
] = kern_linear_pte_xor
[0];
2302 _PAGE_ALL_SZ_BITS
= (_PAGE_SZ4MB_4U
| _PAGE_SZ512K_4U
|
2303 _PAGE_SZ64K_4U
| _PAGE_SZ8K_4U
|
2304 _PAGE_SZ32MB_4U
| _PAGE_SZ256MB_4U
);
2307 page_none
= _PAGE_PRESENT_4U
| _PAGE_ACCESSED_4U
| _PAGE_CACHE_4U
;
2308 page_shared
= (_PAGE_VALID
| _PAGE_PRESENT_4U
| _PAGE_CACHE_4U
|
2309 __ACCESS_BITS_4U
| _PAGE_WRITE_4U
| _PAGE_EXEC_4U
);
2310 page_copy
= (_PAGE_VALID
| _PAGE_PRESENT_4U
| _PAGE_CACHE_4U
|
2311 __ACCESS_BITS_4U
| _PAGE_EXEC_4U
);
2312 page_readonly
= (_PAGE_VALID
| _PAGE_PRESENT_4U
| _PAGE_CACHE_4U
|
2313 __ACCESS_BITS_4U
| _PAGE_EXEC_4U
);
2315 page_exec_bit
= _PAGE_EXEC_4U
;
2317 prot_init_common(page_none
, page_shared
, page_copy
, page_readonly
,
2321 static void __init
sun4v_pgprot_init(void)
2323 unsigned long page_none
, page_shared
, page_copy
, page_readonly
;
2324 unsigned long page_exec_bit
;
2327 PAGE_KERNEL
= __pgprot (_PAGE_PRESENT_4V
| _PAGE_VALID
|
2328 _PAGE_CACHE_4V
| _PAGE_P_4V
|
2329 __ACCESS_BITS_4V
| __DIRTY_BITS_4V
|
2331 PAGE_KERNEL_LOCKED
= PAGE_KERNEL
;
2333 _PAGE_IE
= _PAGE_IE_4V
;
2334 _PAGE_E
= _PAGE_E_4V
;
2335 _PAGE_CACHE
= _PAGE_CACHE_4V
;
2337 #ifdef CONFIG_DEBUG_PAGEALLOC
2338 kern_linear_pte_xor
[0] = _PAGE_VALID
^ 0xfffff80000000000UL
;
2340 kern_linear_pte_xor
[0] = (_PAGE_VALID
| _PAGE_SZ4MB_4V
) ^
2341 0xfffff80000000000UL
;
2343 kern_linear_pte_xor
[0] |= (_PAGE_CP_4V
| _PAGE_CV_4V
|
2344 _PAGE_P_4V
| _PAGE_W_4V
);
2346 for (i
= 1; i
< 4; i
++)
2347 kern_linear_pte_xor
[i
] = kern_linear_pte_xor
[0];
2349 pg_iobits
= (_PAGE_VALID
| _PAGE_PRESENT_4V
| __DIRTY_BITS_4V
|
2350 __ACCESS_BITS_4V
| _PAGE_E_4V
);
2352 _PAGE_ALL_SZ_BITS
= (_PAGE_SZ16GB_4V
| _PAGE_SZ2GB_4V
|
2353 _PAGE_SZ256MB_4V
| _PAGE_SZ32MB_4V
|
2354 _PAGE_SZ4MB_4V
| _PAGE_SZ512K_4V
|
2355 _PAGE_SZ64K_4V
| _PAGE_SZ8K_4V
);
2357 page_none
= _PAGE_PRESENT_4V
| _PAGE_ACCESSED_4V
| _PAGE_CACHE_4V
;
2358 page_shared
= (_PAGE_VALID
| _PAGE_PRESENT_4V
| _PAGE_CACHE_4V
|
2359 __ACCESS_BITS_4V
| _PAGE_WRITE_4V
| _PAGE_EXEC_4V
);
2360 page_copy
= (_PAGE_VALID
| _PAGE_PRESENT_4V
| _PAGE_CACHE_4V
|
2361 __ACCESS_BITS_4V
| _PAGE_EXEC_4V
);
2362 page_readonly
= (_PAGE_VALID
| _PAGE_PRESENT_4V
| _PAGE_CACHE_4V
|
2363 __ACCESS_BITS_4V
| _PAGE_EXEC_4V
);
2365 page_exec_bit
= _PAGE_EXEC_4V
;
2367 prot_init_common(page_none
, page_shared
, page_copy
, page_readonly
,
2371 unsigned long pte_sz_bits(unsigned long sz
)
2373 if (tlb_type
== hypervisor
) {
2377 return _PAGE_SZ8K_4V
;
2379 return _PAGE_SZ64K_4V
;
2381 return _PAGE_SZ512K_4V
;
2382 case 4 * 1024 * 1024:
2383 return _PAGE_SZ4MB_4V
;
2389 return _PAGE_SZ8K_4U
;
2391 return _PAGE_SZ64K_4U
;
2393 return _PAGE_SZ512K_4U
;
2394 case 4 * 1024 * 1024:
2395 return _PAGE_SZ4MB_4U
;
2400 pte_t
mk_pte_io(unsigned long page
, pgprot_t prot
, int space
, unsigned long page_size
)
2404 pte_val(pte
) = page
| pgprot_val(pgprot_noncached(prot
));
2405 pte_val(pte
) |= (((unsigned long)space
) << 32);
2406 pte_val(pte
) |= pte_sz_bits(page_size
);
2411 static unsigned long kern_large_tte(unsigned long paddr
)
2415 val
= (_PAGE_VALID
| _PAGE_SZ4MB_4U
|
2416 _PAGE_CP_4U
| _PAGE_CV_4U
| _PAGE_P_4U
|
2417 _PAGE_EXEC_4U
| _PAGE_L_4U
| _PAGE_W_4U
);
2418 if (tlb_type
== hypervisor
)
2419 val
= (_PAGE_VALID
| _PAGE_SZ4MB_4V
|
2420 _PAGE_CP_4V
| _PAGE_CV_4V
| _PAGE_P_4V
|
2421 _PAGE_EXEC_4V
| _PAGE_W_4V
);
2426 /* If not locked, zap it. */
2427 void __flush_tlb_all(void)
2429 unsigned long pstate
;
2432 __asm__
__volatile__("flushw\n\t"
2433 "rdpr %%pstate, %0\n\t"
2434 "wrpr %0, %1, %%pstate"
2437 if (tlb_type
== hypervisor
) {
2438 sun4v_mmu_demap_all();
2439 } else if (tlb_type
== spitfire
) {
2440 for (i
= 0; i
< 64; i
++) {
2441 /* Spitfire Errata #32 workaround */
2442 /* NOTE: Always runs on spitfire, so no
2443 * cheetah+ page size encodings.
2445 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
2449 "r" (PRIMARY_CONTEXT
), "i" (ASI_DMMU
));
2451 if (!(spitfire_get_dtlb_data(i
) & _PAGE_L_4U
)) {
2452 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
2455 : "r" (TLB_TAG_ACCESS
), "i" (ASI_DMMU
));
2456 spitfire_put_dtlb_data(i
, 0x0UL
);
2459 /* Spitfire Errata #32 workaround */
2460 /* NOTE: Always runs on spitfire, so no
2461 * cheetah+ page size encodings.
2463 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
2467 "r" (PRIMARY_CONTEXT
), "i" (ASI_DMMU
));
2469 if (!(spitfire_get_itlb_data(i
) & _PAGE_L_4U
)) {
2470 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
2473 : "r" (TLB_TAG_ACCESS
), "i" (ASI_IMMU
));
2474 spitfire_put_itlb_data(i
, 0x0UL
);
2477 } else if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
) {
2478 cheetah_flush_dtlb_all();
2479 cheetah_flush_itlb_all();
2481 __asm__
__volatile__("wrpr %0, 0, %%pstate"
2485 static pte_t
*get_from_cache(struct mm_struct
*mm
)
2490 spin_lock(&mm
->page_table_lock
);
2491 page
= mm
->context
.pgtable_page
;
2494 void *p
= page_address(page
);
2496 mm
->context
.pgtable_page
= NULL
;
2498 ret
= (pte_t
*) (p
+ (PAGE_SIZE
/ 2));
2500 spin_unlock(&mm
->page_table_lock
);
2505 static struct page
*__alloc_for_cache(struct mm_struct
*mm
)
2507 struct page
*page
= alloc_page(GFP_KERNEL
| __GFP_NOTRACK
|
2508 __GFP_REPEAT
| __GFP_ZERO
);
2511 spin_lock(&mm
->page_table_lock
);
2512 if (!mm
->context
.pgtable_page
) {
2513 atomic_set(&page
->_count
, 2);
2514 mm
->context
.pgtable_page
= page
;
2516 spin_unlock(&mm
->page_table_lock
);
2521 pte_t
*pte_alloc_one_kernel(struct mm_struct
*mm
,
2522 unsigned long address
)
2527 pte
= get_from_cache(mm
);
2531 page
= __alloc_for_cache(mm
);
2533 pte
= (pte_t
*) page_address(page
);
2538 pgtable_t
pte_alloc_one(struct mm_struct
*mm
,
2539 unsigned long address
)
2544 pte
= get_from_cache(mm
);
2548 page
= __alloc_for_cache(mm
);
2550 pgtable_page_ctor(page
);
2551 pte
= (pte_t
*) page_address(page
);
2557 void pte_free_kernel(struct mm_struct
*mm
, pte_t
*pte
)
2559 struct page
*page
= virt_to_page(pte
);
2560 if (put_page_testzero(page
))
2561 free_hot_cold_page(page
, 0);
2564 static void __pte_free(pgtable_t pte
)
2566 struct page
*page
= virt_to_page(pte
);
2567 if (put_page_testzero(page
)) {
2568 pgtable_page_dtor(page
);
2569 free_hot_cold_page(page
, 0);
2573 void pte_free(struct mm_struct
*mm
, pgtable_t pte
)
2578 void pgtable_free(void *table
, bool is_page
)
2583 kmem_cache_free(pgtable_cache
, table
);
2586 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2587 static pmd_t
pmd_set_protbits(pmd_t pmd
, pgprot_t pgprot
, bool for_modify
)
2589 if (pgprot_val(pgprot
) & _PAGE_VALID
)
2590 pmd_val(pmd
) |= PMD_HUGE_PRESENT
;
2591 if (tlb_type
== hypervisor
) {
2592 if (pgprot_val(pgprot
) & _PAGE_WRITE_4V
)
2593 pmd_val(pmd
) |= PMD_HUGE_WRITE
;
2594 if (pgprot_val(pgprot
) & _PAGE_EXEC_4V
)
2595 pmd_val(pmd
) |= PMD_HUGE_EXEC
;
2598 if (pgprot_val(pgprot
) & _PAGE_ACCESSED_4V
)
2599 pmd_val(pmd
) |= PMD_HUGE_ACCESSED
;
2600 if (pgprot_val(pgprot
) & _PAGE_MODIFIED_4V
)
2601 pmd_val(pmd
) |= PMD_HUGE_DIRTY
;
2604 if (pgprot_val(pgprot
) & _PAGE_WRITE_4U
)
2605 pmd_val(pmd
) |= PMD_HUGE_WRITE
;
2606 if (pgprot_val(pgprot
) & _PAGE_EXEC_4U
)
2607 pmd_val(pmd
) |= PMD_HUGE_EXEC
;
2610 if (pgprot_val(pgprot
) & _PAGE_ACCESSED_4U
)
2611 pmd_val(pmd
) |= PMD_HUGE_ACCESSED
;
2612 if (pgprot_val(pgprot
) & _PAGE_MODIFIED_4U
)
2613 pmd_val(pmd
) |= PMD_HUGE_DIRTY
;
2620 pmd_t
pfn_pmd(unsigned long page_nr
, pgprot_t pgprot
)
2624 pmd_val(pmd
) = (page_nr
<< ((PAGE_SHIFT
- PMD_PADDR_SHIFT
)));
2625 pmd_val(pmd
) |= PMD_ISHUGE
;
2626 pmd
= pmd_set_protbits(pmd
, pgprot
, false);
2630 pmd_t
pmd_modify(pmd_t pmd
, pgprot_t newprot
)
2632 pmd_val(pmd
) &= ~(PMD_HUGE_PRESENT
|
2635 pmd
= pmd_set_protbits(pmd
, newprot
, true);
2639 pgprot_t
pmd_pgprot(pmd_t entry
)
2641 unsigned long pte
= 0;
2643 if (pmd_val(entry
) & PMD_HUGE_PRESENT
)
2646 if (tlb_type
== hypervisor
) {
2647 if (pmd_val(entry
) & PMD_HUGE_PRESENT
)
2648 pte
|= _PAGE_PRESENT_4V
;
2649 if (pmd_val(entry
) & PMD_HUGE_EXEC
)
2650 pte
|= _PAGE_EXEC_4V
;
2651 if (pmd_val(entry
) & PMD_HUGE_WRITE
)
2653 if (pmd_val(entry
) & PMD_HUGE_ACCESSED
)
2654 pte
|= _PAGE_ACCESSED_4V
;
2655 if (pmd_val(entry
) & PMD_HUGE_DIRTY
)
2656 pte
|= _PAGE_MODIFIED_4V
;
2657 pte
|= _PAGE_CP_4V
|_PAGE_CV_4V
;
2659 if (pmd_val(entry
) & PMD_HUGE_PRESENT
)
2660 pte
|= _PAGE_PRESENT_4U
;
2661 if (pmd_val(entry
) & PMD_HUGE_EXEC
)
2662 pte
|= _PAGE_EXEC_4U
;
2663 if (pmd_val(entry
) & PMD_HUGE_WRITE
)
2665 if (pmd_val(entry
) & PMD_HUGE_ACCESSED
)
2666 pte
|= _PAGE_ACCESSED_4U
;
2667 if (pmd_val(entry
) & PMD_HUGE_DIRTY
)
2668 pte
|= _PAGE_MODIFIED_4U
;
2669 pte
|= _PAGE_CP_4U
|_PAGE_CV_4U
;
2672 return __pgprot(pte
);
2675 void update_mmu_cache_pmd(struct vm_area_struct
*vma
, unsigned long addr
,
2678 unsigned long pte
, flags
;
2679 struct mm_struct
*mm
;
2683 if (!pmd_large(entry
) || !pmd_young(entry
))
2686 pte
= (pmd_val(entry
) & ~PMD_HUGE_PROTBITS
);
2687 pte
<<= PMD_PADDR_SHIFT
;
2690 prot
= pmd_pgprot(entry
);
2692 if (tlb_type
== hypervisor
)
2693 pgprot_val(prot
) |= _PAGE_SZHUGE_4V
;
2695 pgprot_val(prot
) |= _PAGE_SZHUGE_4U
;
2697 pte
|= pgprot_val(prot
);
2701 spin_lock_irqsave(&mm
->context
.lock
, flags
);
2703 if (mm
->context
.tsb_block
[MM_TSB_HUGE
].tsb
!= NULL
)
2704 __update_mmu_tsb_insert(mm
, MM_TSB_HUGE
, HPAGE_SHIFT
,
2707 spin_unlock_irqrestore(&mm
->context
.lock
, flags
);
2709 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2711 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
2712 static void context_reload(void *__data
)
2714 struct mm_struct
*mm
= __data
;
2716 if (mm
== current
->mm
)
2717 load_secondary_context(mm
);
2720 void hugetlb_setup(struct pt_regs
*regs
)
2722 struct mm_struct
*mm
= current
->mm
;
2723 struct tsb_config
*tp
;
2725 if (in_atomic() || !mm
) {
2726 const struct exception_table_entry
*entry
;
2728 entry
= search_exception_tables(regs
->tpc
);
2730 regs
->tpc
= entry
->fixup
;
2731 regs
->tnpc
= regs
->tpc
+ 4;
2734 pr_alert("Unexpected HugeTLB setup in atomic context.\n");
2735 die_if_kernel("HugeTSB in atomic", regs
);
2738 tp
= &mm
->context
.tsb_block
[MM_TSB_HUGE
];
2739 if (likely(tp
->tsb
== NULL
))
2740 tsb_grow(mm
, MM_TSB_HUGE
, 0);
2742 tsb_context_switch(mm
);
2745 /* On UltraSPARC-III+ and later, configure the second half of
2746 * the Data-TLB for huge pages.
2748 if (tlb_type
== cheetah_plus
) {
2751 spin_lock(&ctx_alloc_lock
);
2752 ctx
= mm
->context
.sparc64_ctx_val
;
2753 ctx
&= ~CTX_PGSZ_MASK
;
2754 ctx
|= CTX_PGSZ_BASE
<< CTX_PGSZ0_SHIFT
;
2755 ctx
|= CTX_PGSZ_HUGE
<< CTX_PGSZ1_SHIFT
;
2757 if (ctx
!= mm
->context
.sparc64_ctx_val
) {
2758 /* When changing the page size fields, we
2759 * must perform a context flush so that no
2760 * stale entries match. This flush must
2761 * occur with the original context register
2764 do_flush_tlb_mm(mm
);
2766 /* Reload the context register of all processors
2767 * also executing in this address space.
2769 mm
->context
.sparc64_ctx_val
= ctx
;
2770 on_each_cpu(context_reload
, mm
, 0);
2772 spin_unlock(&ctx_alloc_lock
);