return (((tmp >> TICKET_SHIFT) - tmp) & ((1 << TICKET_SHIFT) - 1)) > 1;
}
-- --------- -#ifdef CONFIG_PARAVIRT
-- --------- -/*
-- --------- - * Define virtualization-friendly old-style lock byte lock, for use in
-- --------- - * pv_lock_ops if desired.
-- --------- - *
-- --------- - * This differs from the pre-2.6.24 spinlock by always using xchgb
-- --------- - * rather than decb to take the lock; this allows it to use a
-- --------- - * zero-initialized lock structure. It also maintains a 1-byte
-- --------- - * contention counter, so that we can implement
-- --------- - * __byte_spin_is_contended.
-- --------- - */
-- --------- -struct __byte_spinlock {
-- --------- - s8 lock;
-- --------- - s8 spinners;
-- --------- -};
-- --------- -
-- --------- -static inline int __byte_spin_is_locked(raw_spinlock_t *lock)
-- --------- -{
-- --------- - struct __byte_spinlock *bl = (struct __byte_spinlock *)lock;
-- --------- - return bl->lock != 0;
-- --------- -}
-- --------- -
-- --------- -static inline int __byte_spin_is_contended(raw_spinlock_t *lock)
-- --------- -{
-- --------- - struct __byte_spinlock *bl = (struct __byte_spinlock *)lock;
-- --------- - return bl->spinners != 0;
-- --------- -}
-- --------- -
-- --------- -static inline void __byte_spin_lock(raw_spinlock_t *lock)
-- --------- -{
-- --------- - struct __byte_spinlock *bl = (struct __byte_spinlock *)lock;
-- --------- - s8 val = 1;
- --------- -
- --------- - asm("1: xchgb %1, %0\n"
- --------- - " test %1,%1\n"
- --------- - " jz 3f\n"
- --------- - " " LOCK_PREFIX "incb %2\n"
- --------- - "2: rep;nop\n"
- --------- - " cmpb $1, %0\n"
- --------- - " je 2b\n"
- --------- - " " LOCK_PREFIX "decb %2\n"
- --------- - " jmp 1b\n"
- --------- - "3:"
- --------- - : "+m" (bl->lock), "+q" (val), "+m" (bl->spinners): : "memory");
- --------- -}
- --------- -
- --------- -static inline int __byte_spin_trylock(raw_spinlock_t *lock)
- --------- -{
- --------- - struct __byte_spinlock *bl = (struct __byte_spinlock *)lock;
- --------- - u8 old = 1;
- --------- -
- --------- - asm("xchgb %1,%0"
- --------- - : "+m" (bl->lock), "+q" (old) : : "memory");
++ +++++++++ +#ifndef CONFIG_PARAVIRT
- asm("1: xchgb %1, %0\n"
- " test %1,%1\n"
- " jz 3f\n"
- " " LOCK_PREFIX "incb %2\n"
- "2: rep;nop\n"
- " cmpb $1, %0\n"
- " je 2b\n"
- " " LOCK_PREFIX "decb %2\n"
- " jmp 1b\n"
- "3:"
- : "+m" (bl->lock), "+q" (val), "+m" (bl->spinners): : "memory");
- }
-
- static inline int __byte_spin_trylock(raw_spinlock_t *lock)
- {
- struct __byte_spinlock *bl = (struct __byte_spinlock *)lock;
- u8 old = 1;
-
- asm("xchgb %1,%0"
- : "+m" (bl->lock), "+q" (old) : : "memory");
-
-- --------- - return old == 0;
-- --------- -}
-- --------- -
-- --------- -static inline void __byte_spin_unlock(raw_spinlock_t *lock)
-- --------- -{
-- --------- - struct __byte_spinlock *bl = (struct __byte_spinlock *)lock;
-- --------- - smp_wmb();
-- --------- - bl->lock = 0;
-- --------- -}
-- --------- -#else /* !CONFIG_PARAVIRT */
static inline int __raw_spin_is_locked(raw_spinlock_t *lock)
{
return __ticket_spin_is_locked(lock);
{
unsigned long flags = oops_begin();
int sig = SIGKILL;
------ ------ struct task_struct *tsk;
++++++ ++++++ struct task_struct *tsk = current;
printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
------ ------ current->comm, address);
++++++ ++++++ tsk->comm, address);
dump_pagetable(address);
------- ------ tsk = current;
tsk->thread.cr2 = address;
tsk->thread.trap_no = 14;
tsk->thread.error_code = error_code;
}
#endif
++++++ ++++++static noinline void no_context(struct pt_regs *regs,
++++++ ++++++ unsigned long error_code, unsigned long address)
++++++ ++++++{
++++++ ++++++ struct task_struct *tsk = current;
+++++++++++++ unsigned long *stackend;
+++++++++++++
++++++ ++++++#ifdef CONFIG_X86_64
++++++ ++++++ unsigned long flags;
++++++ ++++++ int sig;
++++++ ++++++#endif
++++++ ++++++
++++++ ++++++ /* Are we prepared to handle this kernel fault? */
++++++ ++++++ if (fixup_exception(regs))
++++++ ++++++ return;
++++++ ++++++
++++++ ++++++ /*
++++++ ++++++ * X86_32
++++++ ++++++ * Valid to do another page fault here, because if this fault
++++++ ++++++ * had been triggered by is_prefetch fixup_exception would have
++++++ ++++++ * handled it.
++++++ ++++++ *
++++++ ++++++ * X86_64
++++++ ++++++ * Hall of shame of CPU/BIOS bugs.
++++++ ++++++ */
++++++ ++++++ if (is_prefetch(regs, error_code, address))
++++++ ++++++ return;
++++++ ++++++
++++++ ++++++ if (is_errata93(regs, address))
++++++ ++++++ return;
++++++ ++++++
++++++ ++++++ /*
++++++ ++++++ * Oops. The kernel tried to access some bad page. We'll have to
++++++ ++++++ * terminate things with extreme prejudice.
++++++ ++++++ */
++++++ ++++++#ifdef CONFIG_X86_32
++++++ ++++++ bust_spinlocks(1);
++++++ ++++++#else
++++++ ++++++ flags = oops_begin();
++++++ ++++++#endif
++++++ ++++++
++++++ ++++++ show_fault_oops(regs, error_code, address);
++++++ ++++++
+++++++++++++ stackend = end_of_stack(tsk);
+++++++++++++ if (*stackend != STACK_END_MAGIC)
+++++++++++++ printk(KERN_ALERT "Thread overran stack, or stack corrupted\n");
+++++++++++++
++++++ ++++++ tsk->thread.cr2 = address;
++++++ ++++++ tsk->thread.trap_no = 14;
++++++ ++++++ tsk->thread.error_code = error_code;
++++++ ++++++
++++++ ++++++#ifdef CONFIG_X86_32
++++++ ++++++ die("Oops", regs, error_code);
++++++ ++++++ bust_spinlocks(0);
++++++ ++++++ do_exit(SIGKILL);
++++++ ++++++#else
++++++ ++++++ sig = SIGKILL;
++++++ ++++++ if (__die("Oops", regs, error_code))
++++++ ++++++ sig = 0;
++++++ ++++++ /* Executive summary in case the body of the oops scrolled away */
++++++ ++++++ printk(KERN_EMERG "CR2: %016lx\n", address);
++++++ ++++++ oops_end(flags, regs, sig);
++++++ ++++++#endif
++++++ ++++++}
++++++ ++++++
++++++ ++++++static void __bad_area_nosemaphore(struct pt_regs *regs,
++++++ ++++++ unsigned long error_code, unsigned long address,
++++++ ++++++ int si_code)
++++++ ++++++{
++++++ ++++++ struct task_struct *tsk = current;
++++++ ++++++
++++++ ++++++ /* User mode accesses just cause a SIGSEGV */
++++++ ++++++ if (error_code & PF_USER) {
++++++ ++++++ /*
++++++ ++++++ * It's possible to have interrupts off here.
++++++ ++++++ */
++++++ ++++++ local_irq_enable();
++++++ ++++++
++++++ ++++++ /*
++++++ ++++++ * Valid to do another page fault here because this one came
++++++ ++++++ * from user space.
++++++ ++++++ */
++++++ ++++++ if (is_prefetch(regs, error_code, address))
++++++ ++++++ return;
++++++ ++++++
++++++ ++++++ if (is_errata100(regs, address))
++++++ ++++++ return;
++++++ ++++++
++++++ ++++++ if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
++++++ ++++++ printk_ratelimit()) {
++++++ ++++++ printk(
++++++ ++++++ "%s%s[%d]: segfault at %lx ip %p sp %p error %lx",
++++++ ++++++ task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
++++++ ++++++ tsk->comm, task_pid_nr(tsk), address,
++++++ ++++++ (void *) regs->ip, (void *) regs->sp, error_code);
++++++ ++++++ print_vma_addr(" in ", regs->ip);
++++++ ++++++ printk("\n");
++++++ ++++++ }
++++++ ++++++
++++++ ++++++ tsk->thread.cr2 = address;
++++++ ++++++ /* Kernel addresses are always protection faults */
++++++ ++++++ tsk->thread.error_code = error_code | (address >= TASK_SIZE);
++++++ ++++++ tsk->thread.trap_no = 14;
++++++ ++++++ force_sig_info_fault(SIGSEGV, si_code, address, tsk);
++++++ ++++++ return;
++++++ ++++++ }
++++++ ++++++
++++++ ++++++ if (is_f00f_bug(regs, address))
++++++ ++++++ return;
++++++ ++++++
++++++ ++++++ no_context(regs, error_code, address);
++++++ ++++++}
++++++ ++++++
++++++ ++++++static noinline void bad_area_nosemaphore(struct pt_regs *regs,
++++++ ++++++ unsigned long error_code, unsigned long address)
++++++ ++++++{
++++++ ++++++ __bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR);
++++++ ++++++}
++++++ ++++++
++++++ ++++++static void __bad_area(struct pt_regs *regs,
++++++ ++++++ unsigned long error_code, unsigned long address,
++++++ ++++++ int si_code)
++++++ ++++++{
++++++ ++++++ struct mm_struct *mm = current->mm;
++++++ ++++++
++++++ ++++++ /*
++++++ ++++++ * Something tried to access memory that isn't in our memory map..
++++++ ++++++ * Fix it, but check if it's kernel or user first..
++++++ ++++++ */
++++++ ++++++ up_read(&mm->mmap_sem);
++++++ ++++++
++++++ ++++++ __bad_area_nosemaphore(regs, error_code, address, si_code);
++++++ ++++++}
++++++ ++++++
++++++ ++++++static noinline void bad_area(struct pt_regs *regs,
++++++ ++++++ unsigned long error_code, unsigned long address)
++++++ ++++++{
++++++ ++++++ __bad_area(regs, error_code, address, SEGV_MAPERR);
++++++ ++++++}
++++++ ++++++
++++++ ++++++static noinline void bad_area_access_error(struct pt_regs *regs,
++++++ ++++++ unsigned long error_code, unsigned long address)
++++++ ++++++{
++++++ ++++++ __bad_area(regs, error_code, address, SEGV_ACCERR);
++++++ ++++++}
++++++ ++++++
++++++ ++++++/* TODO: fixup for "mm-invoke-oom-killer-from-page-fault.patch" */
++++++ ++++++static void out_of_memory(struct pt_regs *regs,
++++++ ++++++ unsigned long error_code, unsigned long address)
++++++ ++++++{
++++++ ++++++ /*
++++++ ++++++ * We ran out of memory, call the OOM killer, and return the userspace
++++++ ++++++ * (which will retry the fault, or kill us if we got oom-killed).
++++++ ++++++ */
++++++ ++++++ up_read(¤t->mm->mmap_sem);
++++++ ++++++ pagefault_out_of_memory();
++++++ ++++++}
++++++ ++++++
++++++ ++++++static void do_sigbus(struct pt_regs *regs,
++++++ ++++++ unsigned long error_code, unsigned long address)
++++++ ++++++{
++++++ ++++++ struct task_struct *tsk = current;
++++++ ++++++ struct mm_struct *mm = tsk->mm;
++++++ ++++++
++++++ ++++++ up_read(&mm->mmap_sem);
++++++ ++++++
++++++ ++++++ /* Kernel mode? Handle exceptions or die */
++++++ ++++++ if (!(error_code & PF_USER))
++++++ ++++++ no_context(regs, error_code, address);
++++++ ++++++#ifdef CONFIG_X86_32
++++++ ++++++ /* User space => ok to do another page fault */
++++++ ++++++ if (is_prefetch(regs, error_code, address))
++++++ ++++++ return;
++++++ ++++++#endif
++++++ ++++++ tsk->thread.cr2 = address;
++++++ ++++++ tsk->thread.error_code = error_code;
++++++ ++++++ tsk->thread.trap_no = 14;
++++++ ++++++ force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
++++++ ++++++}
++++++ ++++++
++++++ ++++++static noinline void mm_fault_error(struct pt_regs *regs,
++++++ ++++++ unsigned long error_code, unsigned long address, unsigned int fault)
++++++ ++++++{
++++++ ++++++ if (fault & VM_FAULT_OOM)
++++++ ++++++ out_of_memory(regs, error_code, address);
++++++ ++++++ else if (fault & VM_FAULT_SIGBUS)
++++++ ++++++ do_sigbus(regs, error_code, address);
++++++ ++++++ else
++++++ ++++++ BUG();
++++++ ++++++}
++++++ ++++++
static int spurious_fault_check(unsigned long error_code, pte_t *pte)
{
if ((error_code & PF_WRITE) && !pte_write(*pte))
* Reserved non RAM regions only and after successful reserve_memtype,
* this func also keeps identity mapping (if any) in sync with this new prot.
*/
-- - --static int reserve_pfn_range(u64 paddr, unsigned long size, pgprot_t vma_prot)
++ + ++static int reserve_pfn_range(u64 paddr, unsigned long size, pgprot_t *vma_prot,
++ + ++ int strict_prot)
{
int is_ram = 0;
- int ret;
+ int id_sz, ret;
unsigned long flags;
-- - -- unsigned long want_flags = (pgprot_val(vma_prot) & _PAGE_CACHE_MASK);
++ + ++ unsigned long want_flags = (pgprot_val(*vma_prot) & _PAGE_CACHE_MASK);
is_ram = pagerange_is_ram(paddr, paddr + size);
return ret;
if (flags != want_flags) {
-- - -- free_memtype(paddr, paddr + size);
-- - -- printk(KERN_ERR
-- - -- "%s:%d map pfn expected mapping type %s for %Lx-%Lx, got %s\n",
-- - -- current->comm, current->pid,
-- - -- cattr_name(want_flags),
-- - -- (unsigned long long)paddr,
-- - -- (unsigned long long)(paddr + size),
-- - -- cattr_name(flags));
-- - -- return -EINVAL;
++ + ++ if (strict_prot || !is_new_memtype_allowed(want_flags, flags)) {
++ + ++ free_memtype(paddr, paddr + size);
++ + ++ printk(KERN_ERR "%s:%d map pfn expected mapping type %s"
++ + ++ " for %Lx-%Lx, got %s\n",
++ + ++ current->comm, current->pid,
++ + ++ cattr_name(want_flags),
++ + ++ (unsigned long long)paddr,
++ + ++ (unsigned long long)(paddr + size),
++ + ++ cattr_name(flags));
++ + ++ return -EINVAL;
++ + ++ }
++ + ++ /*
++ + ++ * We allow returning different type than the one requested in
++ + ++ * non strict case.
++ + ++ */
++ + ++ *vma_prot = __pgprot((pgprot_val(*vma_prot) &
++ + ++ (~_PAGE_CACHE_MASK)) |
++ + ++ flags);
}
- if (kernel_map_sync_memtype(paddr, size, flags)) {
+ /* Need to keep identity mapping in sync */
+ if (paddr >= __pa(high_memory))
+ return 0;
+
+ id_sz = (__pa(high_memory) < paddr + size) ?
+ __pa(high_memory) - paddr :
+ size;
+
+ if (ioremap_change_attr((unsigned long)__va(paddr), id_sz, flags) < 0) {
free_memtype(paddr, paddr + size);
printk(KERN_ERR
"%s:%d reserve_pfn_range ioremap_change_attr failed %s "