obj-y += $(cacheops-y)
mmu-y := nommu.o extable_32.o
-mmu-$(CONFIG_MMU) := extable_$(BITS).o fault_$(BITS).o gup.o \
- ioremap.o kmap.o pgtable.o tlbflush_$(BITS).o
+mmu-$(CONFIG_MMU) := extable_$(BITS).o fault.o gup.o ioremap.o kmap.o \
+ pgtable.o tlbex_$(BITS).o tlbflush_$(BITS).o
obj-y += $(mmu-y)
--- /dev/null
+/*
+ * Page fault handler for SH with an MMU.
+ *
+ * Copyright (C) 1999 Niibe Yutaka
+ * Copyright (C) 2003 - 2012 Paul Mundt
+ *
+ * Based on linux/arch/i386/mm/fault.c:
+ * Copyright (C) 1995 Linus Torvalds
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/hardirq.h>
+#include <linux/kprobes.h>
+#include <linux/perf_event.h>
+#include <linux/kdebug.h>
+#include <asm/io_trapped.h>
+#include <asm/mmu_context.h>
+#include <asm/tlbflush.h>
+#include <asm/traps.h>
+
+static inline int notify_page_fault(struct pt_regs *regs, int trap)
+{
+ int ret = 0;
+
+ if (kprobes_built_in() && !user_mode(regs)) {
+ preempt_disable();
+ if (kprobe_running() && kprobe_fault_handler(regs, trap))
+ ret = 1;
+ preempt_enable();
+ }
+
+ return ret;
+}
+
+static void
+force_sig_info_fault(int si_signo, int si_code, unsigned long address,
+ struct task_struct *tsk)
+{
+ siginfo_t info;
+
+ info.si_signo = si_signo;
+ info.si_errno = 0;
+ info.si_code = si_code;
+ info.si_addr = (void __user *)address;
+
+ force_sig_info(si_signo, &info, tsk);
+}
+
+/*
+ * This is useful to dump out the page tables associated with
+ * 'addr' in mm 'mm'.
+ */
+static void show_pte(struct mm_struct *mm, unsigned long addr)
+{
+ pgd_t *pgd;
+
+ if (mm)
+ pgd = mm->pgd;
+ else
+ pgd = get_TTB();
+
+ printk(KERN_ALERT "pgd = %p\n", pgd);
+ pgd += pgd_index(addr);
+ printk(KERN_ALERT "[%08lx] *pgd=%0*Lx", addr,
+ (u32)(sizeof(*pgd) * 2), (u64)pgd_val(*pgd));
+
+ do {
+ pud_t *pud;
+ pmd_t *pmd;
+ pte_t *pte;
+
+ if (pgd_none(*pgd))
+ break;
+
+ if (pgd_bad(*pgd)) {
+ printk("(bad)");
+ break;
+ }
+
+ pud = pud_offset(pgd, addr);
+ if (PTRS_PER_PUD != 1)
+ printk(", *pud=%0*Lx", (u32)(sizeof(*pud) * 2),
+ (u64)pud_val(*pud));
+
+ if (pud_none(*pud))
+ break;
+
+ if (pud_bad(*pud)) {
+ printk("(bad)");
+ break;
+ }
+
+ pmd = pmd_offset(pud, addr);
+ if (PTRS_PER_PMD != 1)
+ printk(", *pmd=%0*Lx", (u32)(sizeof(*pmd) * 2),
+ (u64)pmd_val(*pmd));
+
+ if (pmd_none(*pmd))
+ break;
+
+ if (pmd_bad(*pmd)) {
+ printk("(bad)");
+ break;
+ }
+
+ /* We must not map this if we have highmem enabled */
+ if (PageHighMem(pfn_to_page(pmd_val(*pmd) >> PAGE_SHIFT)))
+ break;
+
+ pte = pte_offset_kernel(pmd, addr);
+ printk(", *pte=%0*Lx", (u32)(sizeof(*pte) * 2),
+ (u64)pte_val(*pte));
+ } while (0);
+
+ printk("\n");
+}
+
+static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
+{
+ unsigned index = pgd_index(address);
+ pgd_t *pgd_k;
+ pud_t *pud, *pud_k;
+ pmd_t *pmd, *pmd_k;
+
+ pgd += index;
+ pgd_k = init_mm.pgd + index;
+
+ if (!pgd_present(*pgd_k))
+ return NULL;
+
+ pud = pud_offset(pgd, address);
+ pud_k = pud_offset(pgd_k, address);
+ if (!pud_present(*pud_k))
+ return NULL;
+
+ if (!pud_present(*pud))
+ set_pud(pud, *pud_k);
+
+ pmd = pmd_offset(pud, address);
+ pmd_k = pmd_offset(pud_k, address);
+ if (!pmd_present(*pmd_k))
+ return NULL;
+
+ if (!pmd_present(*pmd))
+ set_pmd(pmd, *pmd_k);
+ else {
+ /*
+ * The page tables are fully synchronised so there must
+ * be another reason for the fault. Return NULL here to
+ * signal that we have not taken care of the fault.
+ */
+ BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
+ return NULL;
+ }
+
+ return pmd_k;
+}
+
+/*
+ * Handle a fault on the vmalloc or module mapping area
+ */
+static noinline int vmalloc_fault(unsigned long address)
+{
+ pgd_t *pgd_k;
+ pmd_t *pmd_k;
+ pte_t *pte_k;
+
+ /* Make sure we are in vmalloc/module area: */
+ if (!is_vmalloc_addr((void *)address))
+ return -1;
+
+ /*
+ * Synchronize this task's top level page-table
+ * with the 'reference' page table.
+ *
+ * Do _not_ use "current" here. We might be inside
+ * an interrupt in the middle of a task switch..
+ */
+ pgd_k = get_TTB();
+ pmd_k = vmalloc_sync_one(pgd_k, address);
+ if (!pmd_k)
+ return -1;
+
+ pte_k = pte_offset_kernel(pmd_k, address);
+ if (!pte_present(*pte_k))
+ return -1;
+
+ return 0;
+}
+
+static void
+show_fault_oops(struct pt_regs *regs, unsigned long address)
+{
+ if (!oops_may_print())
+ return;
+
+ printk(KERN_ALERT "BUG: unable to handle kernel ");
+ if (address < PAGE_SIZE)
+ printk(KERN_CONT "NULL pointer dereference");
+ else
+ printk(KERN_CONT "paging request");
+
+ printk(KERN_CONT " at %08lx\n", address);
+ printk(KERN_ALERT "PC:");
+ printk_address(regs->pc, 1);
+
+ show_pte(NULL, address);
+}
+
+static noinline void
+no_context(struct pt_regs *regs, unsigned long error_code,
+ unsigned long address)
+{
+ /* Are we prepared to handle this kernel fault? */
+ if (fixup_exception(regs))
+ return;
+
+ if (handle_trapped_io(regs, address))
+ return;
+
+ /*
+ * Oops. The kernel tried to access some bad page. We'll have to
+ * terminate things with extreme prejudice.
+ */
+ bust_spinlocks(1);
+
+ show_fault_oops(regs, address);
+
+ die("Oops", regs, error_code);
+ bust_spinlocks(0);
+ do_exit(SIGKILL);
+}
+
+static void
+__bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
+ unsigned long address, int si_code)
+{
+ struct task_struct *tsk = current;
+
+ /* User mode accesses just cause a SIGSEGV */
+ if (user_mode(regs)) {
+ /*
+ * It's possible to have interrupts off here:
+ */
+ local_irq_enable();
+
+ force_sig_info_fault(SIGSEGV, si_code, address, tsk);
+
+ return;
+ }
+
+ no_context(regs, error_code, address);
+}
+
+static noinline void
+bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
+ unsigned long address)
+{
+ __bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR);
+}
+
+static void
+__bad_area(struct pt_regs *regs, unsigned long error_code,
+ unsigned long address, int si_code)
+{
+ struct mm_struct *mm = current->mm;
+
+ /*
+ * Something tried to access memory that isn't in our memory map..
+ * Fix it, but check if it's kernel or user first..
+ */
+ up_read(&mm->mmap_sem);
+
+ __bad_area_nosemaphore(regs, error_code, address, si_code);
+}
+
+static noinline void
+bad_area(struct pt_regs *regs, unsigned long error_code, unsigned long address)
+{
+ __bad_area(regs, error_code, address, SEGV_MAPERR);
+}
+
+static noinline void
+bad_area_access_error(struct pt_regs *regs, unsigned long error_code,
+ unsigned long address)
+{
+ __bad_area(regs, error_code, address, SEGV_ACCERR);
+}
+
+static void out_of_memory(void)
+{
+ /*
+ * We ran out of memory, call the OOM killer, and return the userspace
+ * (which will retry the fault, or kill us if we got oom-killed):
+ */
+ up_read(¤t->mm->mmap_sem);
+
+ pagefault_out_of_memory();
+}
+
+static void
+do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address)
+{
+ struct task_struct *tsk = current;
+ struct mm_struct *mm = tsk->mm;
+
+ up_read(&mm->mmap_sem);
+
+ /* Kernel mode? Handle exceptions or die: */
+ if (!user_mode(regs))
+ no_context(regs, error_code, address);
+
+ force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
+}
+
+static noinline int
+mm_fault_error(struct pt_regs *regs, unsigned long error_code,
+ unsigned long address, unsigned int fault)
+{
+ /*
+ * Pagefault was interrupted by SIGKILL. We have no reason to
+ * continue pagefault.
+ */
+ if (fatal_signal_pending(current)) {
+ if (!(fault & VM_FAULT_RETRY))
+ up_read(¤t->mm->mmap_sem);
+ if (!user_mode(regs))
+ no_context(regs, error_code, address);
+ return 1;
+ }
+
+ if (!(fault & VM_FAULT_ERROR))
+ return 0;
+
+ if (fault & VM_FAULT_OOM) {
+ /* Kernel mode? Handle exceptions or die: */
+ if (!user_mode(regs)) {
+ up_read(¤t->mm->mmap_sem);
+ no_context(regs, error_code, address);
+ return 1;
+ }
+
+ out_of_memory();
+ } else {
+ if (fault & VM_FAULT_SIGBUS)
+ do_sigbus(regs, error_code, address);
+ else
+ BUG();
+ }
+
+ return 1;
+}
+
+static inline int access_error(int error_code, struct vm_area_struct *vma)
+{
+ if (error_code & FAULT_CODE_WRITE) {
+ /* write, present and write, not present: */
+ if (unlikely(!(vma->vm_flags & VM_WRITE)))
+ return 1;
+ return 0;
+ }
+
+ /* ITLB miss on NX page */
+ if (unlikely((error_code & FAULT_CODE_ITLB) &&
+ !(vma->vm_flags & VM_EXEC)))
+ return 1;
+
+ /* read, not present: */
+ if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))))
+ return 1;
+
+ return 0;
+}
+
+static int fault_in_kernel_space(unsigned long address)
+{
+ return address >= TASK_SIZE;
+}
+
+/*
+ * This routine handles page faults. It determines the address,
+ * and the problem, and then passes it off to one of the appropriate
+ * routines.
+ */
+asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
+ unsigned long error_code,
+ unsigned long address)
+{
+ unsigned long vec;
+ struct task_struct *tsk;
+ struct mm_struct *mm;
+ struct vm_area_struct * vma;
+ int fault;
+ int write = error_code & FAULT_CODE_WRITE;
+ unsigned int flags = (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE |
+ (write ? FAULT_FLAG_WRITE : 0));
+
+ tsk = current;
+ mm = tsk->mm;
+ vec = lookup_exception_vector();
+
+ /*
+ * We fault-in kernel-space virtual memory on-demand. The
+ * 'reference' page table is init_mm.pgd.
+ *
+ * NOTE! We MUST NOT take any locks for this case. We may
+ * be in an interrupt or a critical region, and should
+ * only copy the information from the master page table,
+ * nothing more.
+ */
+ if (unlikely(fault_in_kernel_space(address))) {
+ if (vmalloc_fault(address) >= 0)
+ return;
+ if (notify_page_fault(regs, vec))
+ return;
+
+ bad_area_nosemaphore(regs, error_code, address);
+ return;
+ }
+
+ if (unlikely(notify_page_fault(regs, vec)))
+ return;
+
+ /* Only enable interrupts if they were on before the fault */
+ if ((regs->sr & SR_IMASK) != SR_IMASK)
+ local_irq_enable();
+
+ perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
+
+ /*
+ * If we're in an interrupt, have no user context or are running
+ * in an atomic region then we must not take the fault:
+ */
+ if (unlikely(in_atomic() || !mm)) {
+ bad_area_nosemaphore(regs, error_code, address);
+ return;
+ }
+
+retry:
+ down_read(&mm->mmap_sem);
+
+ vma = find_vma(mm, address);
+ if (unlikely(!vma)) {
+ bad_area(regs, error_code, address);
+ return;
+ }
+ if (likely(vma->vm_start <= address))
+ goto good_area;
+ if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
+ bad_area(regs, error_code, address);
+ return;
+ }
+ if (unlikely(expand_stack(vma, address))) {
+ bad_area(regs, error_code, address);
+ return;
+ }
+
+ /*
+ * Ok, we have a good vm_area for this memory access, so
+ * we can handle it..
+ */
+good_area:
+ if (unlikely(access_error(error_code, vma))) {
+ bad_area_access_error(regs, error_code, address);
+ return;
+ }
+
+ set_thread_fault_code(error_code);
+
+ /*
+ * If for any reason at all we couldn't handle the fault,
+ * make sure we exit gracefully rather than endlessly redo
+ * the fault.
+ */
+ fault = handle_mm_fault(mm, vma, address, flags);
+
+ if (unlikely(fault & (VM_FAULT_RETRY | VM_FAULT_ERROR)))
+ if (mm_fault_error(regs, error_code, address, fault))
+ return;
+
+ if (flags & FAULT_FLAG_ALLOW_RETRY) {
+ if (fault & VM_FAULT_MAJOR) {
+ tsk->maj_flt++;
+ perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
+ regs, address);
+ } else {
+ tsk->min_flt++;
+ perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
+ regs, address);
+ }
+ if (fault & VM_FAULT_RETRY) {
+ flags &= ~FAULT_FLAG_ALLOW_RETRY;
+
+ /*
+ * No need to up_read(&mm->mmap_sem) as we would
+ * have already released it in __lock_page_or_retry
+ * in mm/filemap.c.
+ */
+ goto retry;
+ }
+ }
+
+ up_read(&mm->mmap_sem);
+}
+++ /dev/null
-/*
- * Page fault handler for SH with an MMU.
- *
- * Copyright (C) 1999 Niibe Yutaka
- * Copyright (C) 2003 - 2012 Paul Mundt
- *
- * Based on linux/arch/i386/mm/fault.c:
- * Copyright (C) 1995 Linus Torvalds
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file "COPYING" in the main directory of this archive
- * for more details.
- */
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/hardirq.h>
-#include <linux/kprobes.h>
-#include <linux/perf_event.h>
-#include <linux/kdebug.h>
-#include <asm/io_trapped.h>
-#include <asm/mmu_context.h>
-#include <asm/tlbflush.h>
-#include <asm/traps.h>
-
-static inline int notify_page_fault(struct pt_regs *regs, int trap)
-{
- int ret = 0;
-
- if (kprobes_built_in() && !user_mode(regs)) {
- preempt_disable();
- if (kprobe_running() && kprobe_fault_handler(regs, trap))
- ret = 1;
- preempt_enable();
- }
-
- return ret;
-}
-
-static void
-force_sig_info_fault(int si_signo, int si_code, unsigned long address,
- struct task_struct *tsk)
-{
- siginfo_t info;
-
- info.si_signo = si_signo;
- info.si_errno = 0;
- info.si_code = si_code;
- info.si_addr = (void __user *)address;
-
- force_sig_info(si_signo, &info, tsk);
-}
-
-/*
- * This is useful to dump out the page tables associated with
- * 'addr' in mm 'mm'.
- */
-static void show_pte(struct mm_struct *mm, unsigned long addr)
-{
- pgd_t *pgd;
-
- if (mm)
- pgd = mm->pgd;
- else
- pgd = get_TTB();
-
- printk(KERN_ALERT "pgd = %p\n", pgd);
- pgd += pgd_index(addr);
- printk(KERN_ALERT "[%08lx] *pgd=%0*Lx", addr,
- sizeof(*pgd) * 2, (u64)pgd_val(*pgd));
-
- do {
- pud_t *pud;
- pmd_t *pmd;
- pte_t *pte;
-
- if (pgd_none(*pgd))
- break;
-
- if (pgd_bad(*pgd)) {
- printk("(bad)");
- break;
- }
-
- pud = pud_offset(pgd, addr);
- if (PTRS_PER_PUD != 1)
- printk(", *pud=%0*Lx", sizeof(*pud) * 2,
- (u64)pud_val(*pud));
-
- if (pud_none(*pud))
- break;
-
- if (pud_bad(*pud)) {
- printk("(bad)");
- break;
- }
-
- pmd = pmd_offset(pud, addr);
- if (PTRS_PER_PMD != 1)
- printk(", *pmd=%0*Lx", sizeof(*pmd) * 2,
- (u64)pmd_val(*pmd));
-
- if (pmd_none(*pmd))
- break;
-
- if (pmd_bad(*pmd)) {
- printk("(bad)");
- break;
- }
-
- /* We must not map this if we have highmem enabled */
- if (PageHighMem(pfn_to_page(pmd_val(*pmd) >> PAGE_SHIFT)))
- break;
-
- pte = pte_offset_kernel(pmd, addr);
- printk(", *pte=%0*Lx", sizeof(*pte) * 2, (u64)pte_val(*pte));
- } while (0);
-
- printk("\n");
-}
-
-static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
-{
- unsigned index = pgd_index(address);
- pgd_t *pgd_k;
- pud_t *pud, *pud_k;
- pmd_t *pmd, *pmd_k;
-
- pgd += index;
- pgd_k = init_mm.pgd + index;
-
- if (!pgd_present(*pgd_k))
- return NULL;
-
- pud = pud_offset(pgd, address);
- pud_k = pud_offset(pgd_k, address);
- if (!pud_present(*pud_k))
- return NULL;
-
- if (!pud_present(*pud))
- set_pud(pud, *pud_k);
-
- pmd = pmd_offset(pud, address);
- pmd_k = pmd_offset(pud_k, address);
- if (!pmd_present(*pmd_k))
- return NULL;
-
- if (!pmd_present(*pmd))
- set_pmd(pmd, *pmd_k);
- else {
- /*
- * The page tables are fully synchronised so there must
- * be another reason for the fault. Return NULL here to
- * signal that we have not taken care of the fault.
- */
- BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
- return NULL;
- }
-
- return pmd_k;
-}
-
-/*
- * Handle a fault on the vmalloc or module mapping area
- */
-static noinline int vmalloc_fault(unsigned long address)
-{
- pgd_t *pgd_k;
- pmd_t *pmd_k;
- pte_t *pte_k;
-
- /* Make sure we are in vmalloc/module area: */
- if (!is_vmalloc_addr((void *)address))
- return -1;
-
- /*
- * Synchronize this task's top level page-table
- * with the 'reference' page table.
- *
- * Do _not_ use "current" here. We might be inside
- * an interrupt in the middle of a task switch..
- */
- pgd_k = get_TTB();
- pmd_k = vmalloc_sync_one(pgd_k, address);
- if (!pmd_k)
- return -1;
-
- pte_k = pte_offset_kernel(pmd_k, address);
- if (!pte_present(*pte_k))
- return -1;
-
- return 0;
-}
-
-static void
-show_fault_oops(struct pt_regs *regs, unsigned long address)
-{
- if (!oops_may_print())
- return;
-
- printk(KERN_ALERT "BUG: unable to handle kernel ");
- if (address < PAGE_SIZE)
- printk(KERN_CONT "NULL pointer dereference");
- else
- printk(KERN_CONT "paging request");
-
- printk(KERN_CONT " at %08lx\n", address);
- printk(KERN_ALERT "PC:");
- printk_address(regs->pc, 1);
-
- show_pte(NULL, address);
-}
-
-static noinline void
-no_context(struct pt_regs *regs, unsigned long error_code,
- unsigned long address)
-{
- /* Are we prepared to handle this kernel fault? */
- if (fixup_exception(regs))
- return;
-
- if (handle_trapped_io(regs, address))
- return;
-
- /*
- * Oops. The kernel tried to access some bad page. We'll have to
- * terminate things with extreme prejudice.
- */
- bust_spinlocks(1);
-
- show_fault_oops(regs, address);
-
- die("Oops", regs, error_code);
- bust_spinlocks(0);
- do_exit(SIGKILL);
-}
-
-static void
-__bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
- unsigned long address, int si_code)
-{
- struct task_struct *tsk = current;
-
- /* User mode accesses just cause a SIGSEGV */
- if (user_mode(regs)) {
- /*
- * It's possible to have interrupts off here:
- */
- local_irq_enable();
-
- force_sig_info_fault(SIGSEGV, si_code, address, tsk);
-
- return;
- }
-
- no_context(regs, error_code, address);
-}
-
-static noinline void
-bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
- unsigned long address)
-{
- __bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR);
-}
-
-static void
-__bad_area(struct pt_regs *regs, unsigned long error_code,
- unsigned long address, int si_code)
-{
- struct mm_struct *mm = current->mm;
-
- /*
- * Something tried to access memory that isn't in our memory map..
- * Fix it, but check if it's kernel or user first..
- */
- up_read(&mm->mmap_sem);
-
- __bad_area_nosemaphore(regs, error_code, address, si_code);
-}
-
-static noinline void
-bad_area(struct pt_regs *regs, unsigned long error_code, unsigned long address)
-{
- __bad_area(regs, error_code, address, SEGV_MAPERR);
-}
-
-static noinline void
-bad_area_access_error(struct pt_regs *regs, unsigned long error_code,
- unsigned long address)
-{
- __bad_area(regs, error_code, address, SEGV_ACCERR);
-}
-
-static void out_of_memory(void)
-{
- /*
- * We ran out of memory, call the OOM killer, and return the userspace
- * (which will retry the fault, or kill us if we got oom-killed):
- */
- up_read(¤t->mm->mmap_sem);
-
- pagefault_out_of_memory();
-}
-
-static void
-do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address)
-{
- struct task_struct *tsk = current;
- struct mm_struct *mm = tsk->mm;
-
- up_read(&mm->mmap_sem);
-
- /* Kernel mode? Handle exceptions or die: */
- if (!user_mode(regs))
- no_context(regs, error_code, address);
-
- force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
-}
-
-static noinline int
-mm_fault_error(struct pt_regs *regs, unsigned long error_code,
- unsigned long address, unsigned int fault)
-{
- /*
- * Pagefault was interrupted by SIGKILL. We have no reason to
- * continue pagefault.
- */
- if (fatal_signal_pending(current)) {
- if (!(fault & VM_FAULT_RETRY))
- up_read(¤t->mm->mmap_sem);
- if (!user_mode(regs))
- no_context(regs, error_code, address);
- return 1;
- }
-
- if (!(fault & VM_FAULT_ERROR))
- return 0;
-
- if (fault & VM_FAULT_OOM) {
- /* Kernel mode? Handle exceptions or die: */
- if (!user_mode(regs)) {
- up_read(¤t->mm->mmap_sem);
- no_context(regs, error_code, address);
- return 1;
- }
-
- out_of_memory();
- } else {
- if (fault & VM_FAULT_SIGBUS)
- do_sigbus(regs, error_code, address);
- else
- BUG();
- }
-
- return 1;
-}
-
-static inline int access_error(int write, struct vm_area_struct *vma)
-{
- if (write) {
- /* write, present and write, not present: */
- if (unlikely(!(vma->vm_flags & VM_WRITE)))
- return 1;
- return 0;
- }
-
- /* read, not present: */
- if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))))
- return 1;
-
- return 0;
-}
-
-static int fault_in_kernel_space(unsigned long address)
-{
- return address >= TASK_SIZE;
-}
-
-/*
- * This routine handles page faults. It determines the address,
- * and the problem, and then passes it off to one of the appropriate
- * routines.
- */
-asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
- unsigned long error_code,
- unsigned long address)
-{
- unsigned long vec;
- struct task_struct *tsk;
- struct mm_struct *mm;
- struct vm_area_struct * vma;
- int fault;
- int write = error_code & FAULT_CODE_WRITE;
- unsigned int flags = (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE |
- (write ? FAULT_FLAG_WRITE : 0));
-
- tsk = current;
- mm = tsk->mm;
- vec = lookup_exception_vector();
-
- /*
- * We fault-in kernel-space virtual memory on-demand. The
- * 'reference' page table is init_mm.pgd.
- *
- * NOTE! We MUST NOT take any locks for this case. We may
- * be in an interrupt or a critical region, and should
- * only copy the information from the master page table,
- * nothing more.
- */
- if (unlikely(fault_in_kernel_space(address))) {
- if (vmalloc_fault(address) >= 0)
- return;
- if (notify_page_fault(regs, vec))
- return;
-
- bad_area_nosemaphore(regs, error_code, address);
- return;
- }
-
- if (unlikely(notify_page_fault(regs, vec)))
- return;
-
- /* Only enable interrupts if they were on before the fault */
- if ((regs->sr & SR_IMASK) != SR_IMASK)
- local_irq_enable();
-
- perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
-
- /*
- * If we're in an interrupt, have no user context or are running
- * in an atomic region then we must not take the fault:
- */
- if (unlikely(in_atomic() || !mm)) {
- bad_area_nosemaphore(regs, error_code, address);
- return;
- }
-
-retry:
- down_read(&mm->mmap_sem);
-
- vma = find_vma(mm, address);
- if (unlikely(!vma)) {
- bad_area(regs, error_code, address);
- return;
- }
- if (likely(vma->vm_start <= address))
- goto good_area;
- if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
- bad_area(regs, error_code, address);
- return;
- }
- if (unlikely(expand_stack(vma, address))) {
- bad_area(regs, error_code, address);
- return;
- }
-
- /*
- * Ok, we have a good vm_area for this memory access, so
- * we can handle it..
- */
-good_area:
- if (unlikely(access_error(error_code, vma))) {
- bad_area_access_error(regs, error_code, address);
- return;
- }
-
- set_thread_fault_code(error_code);
-
- /*
- * If for any reason at all we couldn't handle the fault,
- * make sure we exit gracefully rather than endlessly redo
- * the fault.
- */
- fault = handle_mm_fault(mm, vma, address, flags);
-
- if (unlikely(fault & (VM_FAULT_RETRY | VM_FAULT_ERROR)))
- if (mm_fault_error(regs, error_code, address, fault))
- return;
-
- if (flags & FAULT_FLAG_ALLOW_RETRY) {
- if (fault & VM_FAULT_MAJOR) {
- tsk->maj_flt++;
- perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
- regs, address);
- } else {
- tsk->min_flt++;
- perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
- regs, address);
- }
- if (fault & VM_FAULT_RETRY) {
- flags &= ~FAULT_FLAG_ALLOW_RETRY;
-
- /*
- * No need to up_read(&mm->mmap_sem) as we would
- * have already released it in __lock_page_or_retry
- * in mm/filemap.c.
- */
- goto retry;
- }
- }
-
- up_read(&mm->mmap_sem);
-}
-
-/*
- * Called with interrupts disabled.
- */
-asmlinkage int __kprobes
-handle_tlbmiss(struct pt_regs *regs, unsigned long error_code,
- unsigned long address)
-{
- pgd_t *pgd;
- pud_t *pud;
- pmd_t *pmd;
- pte_t *pte;
- pte_t entry;
-
- /*
- * We don't take page faults for P1, P2, and parts of P4, these
- * are always mapped, whether it be due to legacy behaviour in
- * 29-bit mode, or due to PMB configuration in 32-bit mode.
- */
- if (address >= P3SEG && address < P3_ADDR_MAX) {
- pgd = pgd_offset_k(address);
- } else {
- if (unlikely(address >= TASK_SIZE || !current->mm))
- return 1;
-
- pgd = pgd_offset(current->mm, address);
- }
-
- pud = pud_offset(pgd, address);
- if (pud_none_or_clear_bad(pud))
- return 1;
- pmd = pmd_offset(pud, address);
- if (pmd_none_or_clear_bad(pmd))
- return 1;
- pte = pte_offset_kernel(pmd, address);
- entry = *pte;
- if (unlikely(pte_none(entry) || pte_not_present(entry)))
- return 1;
- if (unlikely(error_code && !pte_write(entry)))
- return 1;
-
- if (error_code)
- entry = pte_mkdirty(entry);
- entry = pte_mkyoung(entry);
-
- set_pte(pte, entry);
-
-#if defined(CONFIG_CPU_SH4) && !defined(CONFIG_SMP)
- /*
- * SH-4 does not set MMUCR.RC to the corresponding TLB entry in
- * the case of an initial page write exception, so we need to
- * flush it in order to avoid potential TLB entry duplication.
- */
- if (error_code == FAULT_CODE_INITIAL)
- local_flush_tlb_one(get_asid(), address & PAGE_MASK);
-#endif
-
- set_thread_fault_code(error_code);
- update_mmu_cache(NULL, address, pte);
-
- return 0;
-}
+++ /dev/null
-/*
- * The SH64 TLB miss.
- *
- * Original code from fault.c
- * Copyright (C) 2000, 2001 Paolo Alberelli
- *
- * Fast PTE->TLB refill path
- * Copyright (C) 2003 Richard.Curnow@superh.com
- *
- * IMPORTANT NOTES :
- * The do_fast_page_fault function is called from a context in entry.S
- * where very few registers have been saved. In particular, the code in
- * this file must be compiled not to use ANY caller-save registers that
- * are not part of the restricted save set. Also, it means that code in
- * this file must not make calls to functions elsewhere in the kernel, or
- * else the excepting context will see corruption in its caller-save
- * registers. Plus, the entry.S save area is non-reentrant, so this code
- * has to run with SR.BL==1, i.e. no interrupts taken inside it and panic
- * on any exception.
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file "COPYING" in the main directory of this archive
- * for more details.
- */
-#include <linux/signal.h>
-#include <linux/sched.h>
-#include <linux/kernel.h>
-#include <linux/errno.h>
-#include <linux/string.h>
-#include <linux/types.h>
-#include <linux/ptrace.h>
-#include <linux/mman.h>
-#include <linux/mm.h>
-#include <linux/smp.h>
-#include <linux/interrupt.h>
-#include <asm/tlb.h>
-#include <asm/io.h>
-#include <asm/uaccess.h>
-#include <asm/pgalloc.h>
-#include <asm/mmu_context.h>
-#include <cpu/registers.h>
-
-/* Callable from fault.c, so not static */
-inline void __do_tlb_refill(unsigned long address,
- unsigned long long is_text_not_data, pte_t *pte)
-{
- unsigned long long ptel;
- unsigned long long pteh=0;
- struct tlb_info *tlbp;
- unsigned long long next;
-
- /* Get PTEL first */
- ptel = pte_val(*pte);
-
- /*
- * Set PTEH register
- */
- pteh = neff_sign_extend(address & MMU_VPN_MASK);
-
- /* Set the ASID. */
- pteh |= get_asid() << PTEH_ASID_SHIFT;
- pteh |= PTEH_VALID;
-
- /* Set PTEL register, set_pte has performed the sign extension */
- ptel &= _PAGE_FLAGS_HARDWARE_MASK; /* drop software flags */
-
- tlbp = is_text_not_data ? &(cpu_data->itlb) : &(cpu_data->dtlb);
- next = tlbp->next;
- __flush_tlb_slot(next);
- asm volatile ("putcfg %0,1,%2\n\n\t"
- "putcfg %0,0,%1\n"
- : : "r" (next), "r" (pteh), "r" (ptel) );
-
- next += TLB_STEP;
- if (next > tlbp->last) next = tlbp->first;
- tlbp->next = next;
-
-}
-
-static int handle_vmalloc_fault(struct mm_struct *mm,
- unsigned long protection_flags,
- unsigned long long textaccess,
- unsigned long address)
-{
- pgd_t *dir;
- pud_t *pud;
- pmd_t *pmd;
- static pte_t *pte;
- pte_t entry;
-
- dir = pgd_offset_k(address);
-
- pud = pud_offset(dir, address);
- if (pud_none_or_clear_bad(pud))
- return 0;
-
- pmd = pmd_offset(pud, address);
- if (pmd_none_or_clear_bad(pmd))
- return 0;
-
- pte = pte_offset_kernel(pmd, address);
- entry = *pte;
-
- if (pte_none(entry) || !pte_present(entry))
- return 0;
- if ((pte_val(entry) & protection_flags) != protection_flags)
- return 0;
-
- __do_tlb_refill(address, textaccess, pte);
-
- return 1;
-}
-
-static int handle_tlbmiss(struct mm_struct *mm,
- unsigned long long protection_flags,
- unsigned long long textaccess,
- unsigned long address)
-{
- pgd_t *dir;
- pud_t *pud;
- pmd_t *pmd;
- pte_t *pte;
- pte_t entry;
-
- /* NB. The PGD currently only contains a single entry - there is no
- page table tree stored for the top half of the address space since
- virtual pages in that region should never be mapped in user mode.
- (In kernel mode, the only things in that region are the 512Mb super
- page (locked in), and vmalloc (modules) + I/O device pages (handled
- by handle_vmalloc_fault), so no PGD for the upper half is required
- by kernel mode either).
-
- See how mm->pgd is allocated and initialised in pgd_alloc to see why
- the next test is necessary. - RPC */
- if (address >= (unsigned long) TASK_SIZE)
- /* upper half - never has page table entries. */
- return 0;
-
- dir = pgd_offset(mm, address);
- if (pgd_none(*dir) || !pgd_present(*dir))
- return 0;
- if (!pgd_present(*dir))
- return 0;
-
- pud = pud_offset(dir, address);
- if (pud_none(*pud) || !pud_present(*pud))
- return 0;
-
- pmd = pmd_offset(pud, address);
- if (pmd_none(*pmd) || !pmd_present(*pmd))
- return 0;
-
- pte = pte_offset_kernel(pmd, address);
- entry = *pte;
-
- if (pte_none(entry) || !pte_present(entry))
- return 0;
-
- /*
- * If the page doesn't have sufficient protection bits set to
- * service the kind of fault being handled, there's not much
- * point doing the TLB refill. Punt the fault to the general
- * handler.
- */
- if ((pte_val(entry) & protection_flags) != protection_flags)
- return 0;
-
- __do_tlb_refill(address, textaccess, pte);
-
- return 1;
-}
-
-/*
- * Put all this information into one structure so that everything is just
- * arithmetic relative to a single base address. This reduces the number
- * of movi/shori pairs needed just to load addresses of static data.
- */
-struct expevt_lookup {
- unsigned short protection_flags[8];
- unsigned char is_text_access[8];
- unsigned char is_write_access[8];
-};
-
-#define PRU (1<<9)
-#define PRW (1<<8)
-#define PRX (1<<7)
-#define PRR (1<<6)
-
-#define DIRTY (_PAGE_DIRTY | _PAGE_ACCESSED)
-#define YOUNG (_PAGE_ACCESSED)
-
-/* Sized as 8 rather than 4 to allow checking the PTE's PRU bit against whether
- the fault happened in user mode or privileged mode. */
-static struct expevt_lookup expevt_lookup_table = {
- .protection_flags = {PRX, PRX, 0, 0, PRR, PRR, PRW, PRW},
- .is_text_access = {1, 1, 0, 0, 0, 0, 0, 0}
-};
-
-/*
- This routine handles page faults that can be serviced just by refilling a
- TLB entry from an existing page table entry. (This case represents a very
- large majority of page faults.) Return 1 if the fault was successfully
- handled. Return 0 if the fault could not be handled. (This leads into the
- general fault handling in fault.c which deals with mapping file-backed
- pages, stack growth, segmentation faults, swapping etc etc)
- */
-asmlinkage int do_fast_page_fault(unsigned long long ssr_md,
- unsigned long long expevt,
- unsigned long address)
-{
- struct task_struct *tsk;
- struct mm_struct *mm;
- unsigned long long textaccess;
- unsigned long long protection_flags;
- unsigned long long index;
- unsigned long long expevt4;
-
- /* The next few lines implement a way of hashing EXPEVT into a
- * small array index which can be used to lookup parameters
- * specific to the type of TLBMISS being handled.
- *
- * Note:
- * ITLBMISS has EXPEVT==0xa40
- * RTLBMISS has EXPEVT==0x040
- * WTLBMISS has EXPEVT==0x060
- */
- expevt4 = (expevt >> 4);
- /* TODO : xor ssr_md into this expression too. Then we can check
- * that PRU is set when it needs to be. */
- index = expevt4 ^ (expevt4 >> 5);
- index &= 7;
- protection_flags = expevt_lookup_table.protection_flags[index];
- textaccess = expevt_lookup_table.is_text_access[index];
-
- /* SIM
- * Note this is now called with interrupts still disabled
- * This is to cope with being called for a missing IO port
- * address with interrupts disabled. This should be fixed as
- * soon as we have a better 'fast path' miss handler.
- *
- * Plus take care how you try and debug this stuff.
- * For example, writing debug data to a port which you
- * have just faulted on is not going to work.
- */
-
- tsk = current;
- mm = tsk->mm;
-
- if ((address >= VMALLOC_START && address < VMALLOC_END)) {
- if (ssr_md)
- /*
- * Process-contexts can never have this address
- * range mapped
- */
- if (handle_vmalloc_fault(mm, protection_flags,
- textaccess, address))
- return 1;
- } else if (!in_interrupt() && mm) {
- if (handle_tlbmiss(mm, protection_flags, textaccess, address))
- return 1;
- }
-
- return 0;
-}
--- /dev/null
+/*
+ * TLB miss handler for SH with an MMU.
+ *
+ * Copyright (C) 1999 Niibe Yutaka
+ * Copyright (C) 2003 - 2012 Paul Mundt
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/kprobes.h>
+#include <linux/kdebug.h>
+#include <asm/mmu_context.h>
+#include <asm/thread_info.h>
+
+/*
+ * Called with interrupts disabled.
+ */
+asmlinkage int __kprobes
+handle_tlbmiss(struct pt_regs *regs, unsigned long error_code,
+ unsigned long address)
+{
+ pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd;
+ pte_t *pte;
+ pte_t entry;
+
+ /*
+ * We don't take page faults for P1, P2, and parts of P4, these
+ * are always mapped, whether it be due to legacy behaviour in
+ * 29-bit mode, or due to PMB configuration in 32-bit mode.
+ */
+ if (address >= P3SEG && address < P3_ADDR_MAX) {
+ pgd = pgd_offset_k(address);
+ } else {
+ if (unlikely(address >= TASK_SIZE || !current->mm))
+ return 1;
+
+ pgd = pgd_offset(current->mm, address);
+ }
+
+ pud = pud_offset(pgd, address);
+ if (pud_none_or_clear_bad(pud))
+ return 1;
+ pmd = pmd_offset(pud, address);
+ if (pmd_none_or_clear_bad(pmd))
+ return 1;
+ pte = pte_offset_kernel(pmd, address);
+ entry = *pte;
+ if (unlikely(pte_none(entry) || pte_not_present(entry)))
+ return 1;
+ if (unlikely(error_code && !pte_write(entry)))
+ return 1;
+
+ if (error_code)
+ entry = pte_mkdirty(entry);
+ entry = pte_mkyoung(entry);
+
+ set_pte(pte, entry);
+
+#if defined(CONFIG_CPU_SH4) && !defined(CONFIG_SMP)
+ /*
+ * SH-4 does not set MMUCR.RC to the corresponding TLB entry in
+ * the case of an initial page write exception, so we need to
+ * flush it in order to avoid potential TLB entry duplication.
+ */
+ if (error_code == FAULT_CODE_INITIAL)
+ local_flush_tlb_one(get_asid(), address & PAGE_MASK);
+#endif
+
+ set_thread_fault_code(error_code);
+ update_mmu_cache(NULL, address, pte);
+
+ return 0;
+}
--- /dev/null
+/*
+ * The SH64 TLB miss.
+ *
+ * Original code from fault.c
+ * Copyright (C) 2000, 2001 Paolo Alberelli
+ *
+ * Fast PTE->TLB refill path
+ * Copyright (C) 2003 Richard.Curnow@superh.com
+ *
+ * IMPORTANT NOTES :
+ * The do_fast_page_fault function is called from a context in entry.S
+ * where very few registers have been saved. In particular, the code in
+ * this file must be compiled not to use ANY caller-save registers that
+ * are not part of the restricted save set. Also, it means that code in
+ * this file must not make calls to functions elsewhere in the kernel, or
+ * else the excepting context will see corruption in its caller-save
+ * registers. Plus, the entry.S save area is non-reentrant, so this code
+ * has to run with SR.BL==1, i.e. no interrupts taken inside it and panic
+ * on any exception.
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#include <linux/signal.h>
+#include <linux/sched.h>
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/ptrace.h>
+#include <linux/mman.h>
+#include <linux/mm.h>
+#include <linux/smp.h>
+#include <linux/interrupt.h>
+#include <asm/tlb.h>
+#include <asm/io.h>
+#include <asm/uaccess.h>
+#include <asm/pgalloc.h>
+#include <asm/mmu_context.h>
+#include <cpu/registers.h>
+
+/* Callable from fault.c, so not static */
+inline void __do_tlb_refill(unsigned long address,
+ unsigned long long is_text_not_data, pte_t *pte)
+{
+ unsigned long long ptel;
+ unsigned long long pteh=0;
+ struct tlb_info *tlbp;
+ unsigned long long next;
+
+ /* Get PTEL first */
+ ptel = pte_val(*pte);
+
+ /*
+ * Set PTEH register
+ */
+ pteh = neff_sign_extend(address & MMU_VPN_MASK);
+
+ /* Set the ASID. */
+ pteh |= get_asid() << PTEH_ASID_SHIFT;
+ pteh |= PTEH_VALID;
+
+ /* Set PTEL register, set_pte has performed the sign extension */
+ ptel &= _PAGE_FLAGS_HARDWARE_MASK; /* drop software flags */
+
+ tlbp = is_text_not_data ? &(cpu_data->itlb) : &(cpu_data->dtlb);
+ next = tlbp->next;
+ __flush_tlb_slot(next);
+ asm volatile ("putcfg %0,1,%2\n\n\t"
+ "putcfg %0,0,%1\n"
+ : : "r" (next), "r" (pteh), "r" (ptel) );
+
+ next += TLB_STEP;
+ if (next > tlbp->last) next = tlbp->first;
+ tlbp->next = next;
+
+}
+
+static int handle_vmalloc_fault(struct mm_struct *mm,
+ unsigned long protection_flags,
+ unsigned long long textaccess,
+ unsigned long address)
+{
+ pgd_t *dir;
+ pud_t *pud;
+ pmd_t *pmd;
+ static pte_t *pte;
+ pte_t entry;
+
+ dir = pgd_offset_k(address);
+
+ pud = pud_offset(dir, address);
+ if (pud_none_or_clear_bad(pud))
+ return 0;
+
+ pmd = pmd_offset(pud, address);
+ if (pmd_none_or_clear_bad(pmd))
+ return 0;
+
+ pte = pte_offset_kernel(pmd, address);
+ entry = *pte;
+
+ if (pte_none(entry) || !pte_present(entry))
+ return 0;
+ if ((pte_val(entry) & protection_flags) != protection_flags)
+ return 0;
+
+ __do_tlb_refill(address, textaccess, pte);
+
+ return 1;
+}
+
+static int handle_tlbmiss(struct mm_struct *mm,
+ unsigned long long protection_flags,
+ unsigned long long textaccess,
+ unsigned long address)
+{
+ pgd_t *dir;
+ pud_t *pud;
+ pmd_t *pmd;
+ pte_t *pte;
+ pte_t entry;
+
+ /* NB. The PGD currently only contains a single entry - there is no
+ page table tree stored for the top half of the address space since
+ virtual pages in that region should never be mapped in user mode.
+ (In kernel mode, the only things in that region are the 512Mb super
+ page (locked in), and vmalloc (modules) + I/O device pages (handled
+ by handle_vmalloc_fault), so no PGD for the upper half is required
+ by kernel mode either).
+
+ See how mm->pgd is allocated and initialised in pgd_alloc to see why
+ the next test is necessary. - RPC */
+ if (address >= (unsigned long) TASK_SIZE)
+ /* upper half - never has page table entries. */
+ return 0;
+
+ dir = pgd_offset(mm, address);
+ if (pgd_none(*dir) || !pgd_present(*dir))
+ return 0;
+ if (!pgd_present(*dir))
+ return 0;
+
+ pud = pud_offset(dir, address);
+ if (pud_none(*pud) || !pud_present(*pud))
+ return 0;
+
+ pmd = pmd_offset(pud, address);
+ if (pmd_none(*pmd) || !pmd_present(*pmd))
+ return 0;
+
+ pte = pte_offset_kernel(pmd, address);
+ entry = *pte;
+
+ if (pte_none(entry) || !pte_present(entry))
+ return 0;
+
+ /*
+ * If the page doesn't have sufficient protection bits set to
+ * service the kind of fault being handled, there's not much
+ * point doing the TLB refill. Punt the fault to the general
+ * handler.
+ */
+ if ((pte_val(entry) & protection_flags) != protection_flags)
+ return 0;
+
+ __do_tlb_refill(address, textaccess, pte);
+
+ return 1;
+}
+
+/*
+ * Put all this information into one structure so that everything is just
+ * arithmetic relative to a single base address. This reduces the number
+ * of movi/shori pairs needed just to load addresses of static data.
+ */
+struct expevt_lookup {
+ unsigned short protection_flags[8];
+ unsigned char is_text_access[8];
+ unsigned char is_write_access[8];
+};
+
+#define PRU (1<<9)
+#define PRW (1<<8)
+#define PRX (1<<7)
+#define PRR (1<<6)
+
+#define DIRTY (_PAGE_DIRTY | _PAGE_ACCESSED)
+#define YOUNG (_PAGE_ACCESSED)
+
+/* Sized as 8 rather than 4 to allow checking the PTE's PRU bit against whether
+ the fault happened in user mode or privileged mode. */
+static struct expevt_lookup expevt_lookup_table = {
+ .protection_flags = {PRX, PRX, 0, 0, PRR, PRR, PRW, PRW},
+ .is_text_access = {1, 1, 0, 0, 0, 0, 0, 0}
+};
+
+/*
+ This routine handles page faults that can be serviced just by refilling a
+ TLB entry from an existing page table entry. (This case represents a very
+ large majority of page faults.) Return 1 if the fault was successfully
+ handled. Return 0 if the fault could not be handled. (This leads into the
+ general fault handling in fault.c which deals with mapping file-backed
+ pages, stack growth, segmentation faults, swapping etc etc)
+ */
+asmlinkage int do_fast_page_fault(unsigned long long ssr_md,
+ unsigned long long expevt,
+ unsigned long address)
+{
+ struct task_struct *tsk;
+ struct mm_struct *mm;
+ unsigned long long textaccess;
+ unsigned long long protection_flags;
+ unsigned long long index;
+ unsigned long long expevt4;
+
+ /* The next few lines implement a way of hashing EXPEVT into a
+ * small array index which can be used to lookup parameters
+ * specific to the type of TLBMISS being handled.
+ *
+ * Note:
+ * ITLBMISS has EXPEVT==0xa40
+ * RTLBMISS has EXPEVT==0x040
+ * WTLBMISS has EXPEVT==0x060
+ */
+ expevt4 = (expevt >> 4);
+ /* TODO : xor ssr_md into this expression too. Then we can check
+ * that PRU is set when it needs to be. */
+ index = expevt4 ^ (expevt4 >> 5);
+ index &= 7;
+ protection_flags = expevt_lookup_table.protection_flags[index];
+ textaccess = expevt_lookup_table.is_text_access[index];
+
+ /* SIM
+ * Note this is now called with interrupts still disabled
+ * This is to cope with being called for a missing IO port
+ * address with interrupts disabled. This should be fixed as
+ * soon as we have a better 'fast path' miss handler.
+ *
+ * Plus take care how you try and debug this stuff.
+ * For example, writing debug data to a port which you
+ * have just faulted on is not going to work.
+ */
+
+ tsk = current;
+ mm = tsk->mm;
+
+ if (is_vmalloc_addr((void *)address)) {
+ if (ssr_md)
+ /*
+ * Process-contexts can never have this address
+ * range mapped
+ */
+ if (handle_vmalloc_fault(mm, protection_flags,
+ textaccess, address))
+ return 1;
+ } else if (!in_interrupt() && mm) {
+ if (handle_tlbmiss(mm, protection_flags, textaccess, address))
+ return 1;
+ }
+
+ return 0;
+}
#include <asm/pgalloc.h>
#include <asm/mmu_context.h>
-static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address)
-{
- pgd_t *dir;
- pud_t *pud;
- pmd_t *pmd;
- pte_t *pte;
- pte_t entry;
-
- dir = pgd_offset(mm, address);
- if (pgd_none(*dir))
- return NULL;
-
- pud = pud_offset(dir, address);
- if (pud_none(*pud))
- return NULL;
-
- pmd = pmd_offset(pud, address);
- if (pmd_none(*pmd))
- return NULL;
-
- pte = pte_offset_kernel(pmd, address);
- entry = *pte;
- if (pte_none(entry) || !pte_present(entry))
- return NULL;
-
- return pte;
-}
-
-/*
- * This routine handles page faults. It determines the address,
- * and the problem, and then passes it off to one of the appropriate
- * routines.
- */
-asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code,
- unsigned long address)
-{
- struct task_struct *tsk;
- struct mm_struct *mm;
- struct vm_area_struct * vma;
- const struct exception_table_entry *fixup;
- int write = error_code & FAULT_CODE_WRITE;
- int textaccess = error_code & FAULT_CODE_ITLB;
- unsigned int flags = (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE |
- (write ? FAULT_FLAG_WRITE : 0));
- pte_t *pte;
- int fault;
-
- /* SIM
- * Note this is now called with interrupts still disabled
- * This is to cope with being called for a missing IO port
- * address with interrupts disabled. This should be fixed as
- * soon as we have a better 'fast path' miss handler.
- *
- * Plus take care how you try and debug this stuff.
- * For example, writing debug data to a port which you
- * have just faulted on is not going to work.
- */
-
- tsk = current;
- mm = tsk->mm;
-
- /* Not an IO address, so reenable interrupts */
- local_irq_enable();
-
- perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
-
- /*
- * If we're in an interrupt or have no user
- * context, we must not take the fault..
- */
- if (in_atomic() || !mm)
- goto no_context;
-
-retry:
- /* TLB misses upon some cache flushes get done under cli() */
- down_read(&mm->mmap_sem);
-
- vma = find_vma(mm, address);
- if (!vma)
- goto bad_area;
- if (vma->vm_start <= address)
- goto good_area;
- if (!(vma->vm_flags & VM_GROWSDOWN))
- goto bad_area;
- if (expand_stack(vma, address))
- goto bad_area;
-
-/*
- * Ok, we have a good vm_area for this memory access, so
- * we can handle it..
- */
-good_area:
- if (textaccess) {
- if (!(vma->vm_flags & VM_EXEC))
- goto bad_area;
- } else {
- if (write) {
- if (!(vma->vm_flags & VM_WRITE))
- goto bad_area;
- } else {
- if (!(vma->vm_flags & VM_READ))
- goto bad_area;
- }
- }
-
- /*
- * If for any reason at all we couldn't handle the fault,
- * make sure we exit gracefully rather than endlessly redo
- * the fault.
- */
- fault = handle_mm_fault(mm, vma, address, flags);
-
- if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
- return;
-
- if (unlikely(fault & VM_FAULT_ERROR)) {
- if (fault & VM_FAULT_OOM)
- goto out_of_memory;
- else if (fault & VM_FAULT_SIGBUS)
- goto do_sigbus;
- BUG();
- }
-
- if (flags & FAULT_FLAG_ALLOW_RETRY) {
- if (fault & VM_FAULT_MAJOR) {
- tsk->maj_flt++;
- perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
- regs, address);
- } else {
- tsk->min_flt++;
- perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
- regs, address);
- }
-
- if (fault & VM_FAULT_RETRY) {
- flags &= ~FAULT_FLAG_ALLOW_RETRY;
-
- /*
- * No need to up_read(&mm->mmap_sem) as we would
- * have already released it in __lock_page_or_retry
- * in mm/filemap.c.
- */
- goto retry;
- }
- }
-
- /* If we get here, the page fault has been handled. Do the TLB refill
- now from the newly-setup PTE, to avoid having to fault again right
- away on the same instruction. */
- pte = lookup_pte (mm, address);
- if (!pte) {
- /* From empirical evidence, we can get here, due to
- !pte_present(pte). (e.g. if a swap-in occurs, and the page
- is swapped back out again before the process that wanted it
- gets rescheduled?) */
- goto no_pte;
- }
-
- __do_tlb_refill(address, textaccess, pte);
-
-no_pte:
-
- up_read(&mm->mmap_sem);
- return;
-
-/*
- * Something tried to access memory that isn't in our memory map..
- * Fix it, but check if it's kernel or user first..
- */
-bad_area:
- up_read(&mm->mmap_sem);
-
- if (user_mode(regs)) {
- static int count=0;
- siginfo_t info;
- if (count < 4) {
- /* This is really to help debug faults when starting
- * usermode, so only need a few */
- count++;
- printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n",
- address, task_pid_nr(current), current->comm,
- (unsigned long) regs->pc);
- }
- if (is_global_init(tsk)) {
- panic("INIT had user mode bad_area\n");
- }
- tsk->thread.address = address;
- info.si_signo = SIGSEGV;
- info.si_errno = 0;
- info.si_addr = (void *) address;
- force_sig_info(SIGSEGV, &info, tsk);
- return;
- }
-
-no_context:
- /* Are we prepared to handle this kernel fault? */
- fixup = search_exception_tables(regs->pc);
- if (fixup) {
- regs->pc = fixup->fixup;
- return;
- }
-
-/*
- * Oops. The kernel tried to access some bad page. We'll have to
- * terminate things with extreme prejudice.
- *
- */
- if (address < PAGE_SIZE)
- printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
- else
- printk(KERN_ALERT "Unable to handle kernel paging request");
- printk(" at virtual address %08lx\n", address);
- printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff);
- die("Oops", regs, error_code);
- do_exit(SIGKILL);
-
-/*
- * We ran out of memory, or some other thing happened to us that made
- * us unable to handle the page fault gracefully.
- */
-out_of_memory:
- up_read(&mm->mmap_sem);
- if (!user_mode(regs))
- goto no_context;
- pagefault_out_of_memory();
- return;
-
-do_sigbus:
- printk("fault:Do sigbus\n");
- up_read(&mm->mmap_sem);
-
- /*
- * Send a sigbus, regardless of whether we were in kernel
- * or user mode.
- */
- tsk->thread.address = address;
- force_sig(SIGBUS, tsk);
-
- /* Kernel mode? Handle exceptions or die */
- if (!user_mode(regs))
- goto no_context;
-}
-
void local_flush_tlb_one(unsigned long asid, unsigned long page)
{
unsigned long long match, pteh=0, lpage;
projects / GitHub / LineageOS / G12 / android_kernel_amlogic_linux-4.9.git / commitdiff