arch/ppc/kernel/entry.S

   1 /*
   2  *  PowerPC version
   3  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
   4  *  Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP
   5  *    Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com>
   6  *  Adapted for Power Macintosh by Paul Mackerras.
   7  *  Low-level exception handlers and MMU support
   8  *  rewritten by Paul Mackerras.
   9  *    Copyright (C) 1996 Paul Mackerras.
  10  *  MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
  11  *
  12  *  This file contains the system call entry code, context switch
  13  *  code, and exception/interrupt return code for PowerPC.
  14  *
  15  *  This program is free software; you can redistribute it and/or
  16  *  modify it under the terms of the GNU General Public License
  17  *  as published by the Free Software Foundation; either version
  18  *  2 of the License, or (at your option) any later version.
  19  *
  20  */
  21
  22 #include <linux/config.h>
  23 #include <linux/errno.h>
  24 #include <linux/sys.h>
  25 #include <linux/threads.h>
  26 #include <asm/processor.h>
  27 #include <asm/page.h>
  28 #include <asm/mmu.h>
  29 #include <asm/cputable.h>
  30 #include <asm/thread_info.h>
  31 #include <asm/ppc_asm.h>
  32 #include <asm/offsets.h>
  33 #include <asm/unistd.h>
  34
  35 #undef SHOW_SYSCALLS
  36 #undef SHOW_SYSCALLS_TASK
  37
  38 /*
  39  * MSR_KERNEL is > 0x10000 on 4xx/Book-E since it include MSR_CE.
  40  */
  41 #if MSR_KERNEL >= 0x10000
  42 #define LOAD_MSR_KERNEL(r, x)   lis r,(x)@h; ori r,r,(x)@l
  43 #else
  44 #define LOAD_MSR_KERNEL(r, x)   li r,(x)
  45 #endif
  46
  47 #ifdef CONFIG_BOOKE
  48 #include "head_booke.h"
  49         .globl  mcheck_transfer_to_handler
  50 mcheck_transfer_to_handler:
  51         mtspr   MCHECK_SPRG,r8
  52         BOOKE_LOAD_MCHECK_STACK
  53         lwz     r0,GPR10-INT_FRAME_SIZE(r8)
  54         stw     r0,GPR10(r11)
  55         lwz     r0,GPR11-INT_FRAME_SIZE(r8)
  56         stw     r0,GPR11(r11)
  57         mfspr   r8,MCHECK_SPRG
  58         b       transfer_to_handler_full
  59
  60         .globl  crit_transfer_to_handler
  61 crit_transfer_to_handler:
  62         mtspr   CRIT_SPRG,r8
  63         BOOKE_LOAD_CRIT_STACK
  64         lwz     r0,GPR10-INT_FRAME_SIZE(r8)
  65         stw     r0,GPR10(r11)
  66         lwz     r0,GPR11-INT_FRAME_SIZE(r8)
  67         stw     r0,GPR11(r11)
  68         mfspr   r8,CRIT_SPRG
  69         /* fall through */
  70 #endif
  71
  72 #ifdef CONFIG_40x
  73         .globl  crit_transfer_to_handler
  74 crit_transfer_to_handler:
  75         lwz     r0,crit_r10@l(0)
  76         stw     r0,GPR10(r11)
  77         lwz     r0,crit_r11@l(0)
  78         stw     r0,GPR11(r11)
  79         /* fall through */
  80 #endif
  81
  82 /*
  83  * This code finishes saving the registers to the exception frame
  84  * and jumps to the appropriate handler for the exception, turning
  85  * on address translation.
  86  * Note that we rely on the caller having set cr0.eq iff the exception
  87  * occurred in kernel mode (i.e. MSR:PR = 0).
  88  */
  89         .globl  transfer_to_handler_full
  90 transfer_to_handler_full:
  91         SAVE_NVGPRS(r11)
  92         /* fall through */
  93
  94         .globl  transfer_to_handler
  95 transfer_to_handler:
  96         stw     r2,GPR2(r11)
  97         stw     r12,_NIP(r11)
  98         stw     r9,_MSR(r11)
  99         andi.   r2,r9,MSR_PR
 100         mfctr   r12
 101         mfspr   r2,SPRN_XER
 102         stw     r12,_CTR(r11)
 103         stw     r2,_XER(r11)
 104         mfspr   r12,SPRN_SPRG3
 105         addi    r2,r12,-THREAD
 106         tovirt(r2,r2)                   /* set r2 to current */
 107         beq     2f                      /* if from user, fix up THREAD.regs */
 108         addi    r11,r1,STACK_FRAME_OVERHEAD
 109         stw     r11,PT_REGS(r12)
 110 #if defined(CONFIG_40x) || defined(CONFIG_BOOKE)
 111         /* Check to see if the dbcr0 register is set up to debug.  Use the
 112            single-step bit to do this. */
 113         lwz     r12,THREAD_DBCR0(r12)
 114         andis.  r12,r12,DBCR0_IC@h
 115         beq+    3f
 116         /* From user and task is ptraced - load up global dbcr0 */
 117         li      r12,-1                  /* clear all pending debug events */
 118         mtspr   SPRN_DBSR,r12
 119         lis     r11,global_dbcr0@ha
 120         tophys(r11,r11)
 121         addi    r11,r11,global_dbcr0@l
 122         lwz     r12,0(r11)
 123         mtspr   SPRN_DBCR0,r12
 124         lwz     r12,4(r11)
 125         addi    r12,r12,-1
 126         stw     r12,4(r11)
 127 #endif
 128         b       3f
 129 2:      /* if from kernel, check interrupted DOZE/NAP mode and
 130          * check for stack overflow
 131          */
 132 #ifdef CONFIG_6xx
 133         mfspr   r11,SPRN_HID0
 134         mtcr    r11
 135 BEGIN_FTR_SECTION
 136         bt-     8,power_save_6xx_restore        /* Check DOZE */
 137 END_FTR_SECTION_IFSET(CPU_FTR_CAN_DOZE)
 138 BEGIN_FTR_SECTION
 139         bt-     9,power_save_6xx_restore        /* Check NAP */
 140 END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP)
 141 #endif /* CONFIG_6xx */
 142         .globl transfer_to_handler_cont
 143 transfer_to_handler_cont:
 144         lwz     r11,THREAD_INFO-THREAD(r12)
 145         cmplw   r1,r11                  /* if r1 <= current->thread_info */
 146         ble-    stack_ovf               /* then the kernel stack overflowed */
 147 3:
 148         mflr    r9
 149         lwz     r11,0(r9)               /* virtual address of handler */
 150         lwz     r9,4(r9)                /* where to go when done */
 151         FIX_SRR1(r10,r12)
 152         mtspr   SPRN_SRR0,r11
 153         mtspr   SPRN_SRR1,r10
 154         mtlr    r9
 155         SYNC
 156         RFI                             /* jump to handler, enable MMU */
 157
 158 /*
 159  * On kernel stack overflow, load up an initial stack pointer
 160  * and call StackOverflow(regs), which should not return.
 161  */
 162 stack_ovf:
 163         /* sometimes we use a statically-allocated stack, which is OK. */
 164         lis     r11,_end@h
 165         ori     r11,r11,_end@l
 166         cmplw   r1,r11
 167         ble     3b                      /* r1 <= &_end is OK */
 168         SAVE_NVGPRS(r11)
 169         addi    r3,r1,STACK_FRAME_OVERHEAD
 170         lis     r1,init_thread_union@ha
 171         addi    r1,r1,init_thread_union@l
 172         addi    r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
 173         lis     r9,StackOverflow@ha
 174         addi    r9,r9,StackOverflow@l
 175         LOAD_MSR_KERNEL(r10,MSR_KERNEL)
 176         FIX_SRR1(r10,r12)
 177         mtspr   SPRN_SRR0,r9
 178         mtspr   SPRN_SRR1,r10
 179         SYNC
 180         RFI
 181
 182 /*
 183  * Handle a system call.
 184  */
 185         .stabs  "arch/ppc/kernel/",N_SO,0,0,0f
 186         .stabs  "entry.S",N_SO,0,0,0f
 187 0:
 188
 189 _GLOBAL(DoSyscall)
 190         stw     r0,THREAD+LAST_SYSCALL(r2)
 191         stw     r3,ORIG_GPR3(r1)
 192         li      r12,0
 193         stw     r12,RESULT(r1)
 194         lwz     r11,_CCR(r1)    /* Clear SO bit in CR */
 195         rlwinm  r11,r11,0,4,2
 196         stw     r11,_CCR(r1)
 197 #ifdef SHOW_SYSCALLS
 198         bl      do_show_syscall
 199 #endif /* SHOW_SYSCALLS */
 200         rlwinm  r10,r1,0,0,18   /* current_thread_info() */
 201         lwz     r11,TI_LOCAL_FLAGS(r10)
 202         rlwinm  r11,r11,0,~_TIFL_FORCE_NOERROR
 203         stw     r11,TI_LOCAL_FLAGS(r10)
 204         lwz     r11,TI_FLAGS(r10)
 205         andi.   r11,r11,_TIF_SYSCALL_TRACE
 206         bne-    syscall_dotrace
 207 syscall_dotrace_cont:
 208         cmplwi  0,r0,NR_syscalls
 209         lis     r10,sys_call_table@h
 210         ori     r10,r10,sys_call_table@l
 211         slwi    r0,r0,2
 212         bge-    66f
 213         lwzx    r10,r10,r0      /* Fetch system call handler [ptr] */
 214         mtlr    r10
 215         addi    r9,r1,STACK_FRAME_OVERHEAD
 216         blrl                    /* Call handler */
 217         .globl  ret_from_syscall
 218 ret_from_syscall:
 219 #ifdef SHOW_SYSCALLS
 220         bl      do_show_syscall_exit
 221 #endif
 222         mr      r6,r3
 223         li      r11,-_LAST_ERRNO
 224         cmplw   0,r3,r11
 225         rlwinm  r12,r1,0,0,18   /* current_thread_info() */
 226         blt+    30f
 227         lwz     r11,TI_LOCAL_FLAGS(r12)
 228         andi.   r11,r11,_TIFL_FORCE_NOERROR
 229         bne     30f
 230         neg     r3,r3
 231         lwz     r10,_CCR(r1)    /* Set SO bit in CR */
 232         oris    r10,r10,0x1000
 233         stw     r10,_CCR(r1)
 234
 235         /* disable interrupts so current_thread_info()->flags can't change */
 236 30:     LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */
 237         SYNC
 238         MTMSRD(r10)
 239         lwz     r9,TI_FLAGS(r12)
 240         andi.   r0,r9,(_TIF_SYSCALL_TRACE|_TIF_SIGPENDING|_TIF_NEED_RESCHED)
 241         bne-    syscall_exit_work
 242 syscall_exit_cont:
 243 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
 244         /* If the process has its own DBCR0 value, load it up.  The single
 245            step bit tells us that dbcr0 should be loaded. */
 246         lwz     r0,THREAD+THREAD_DBCR0(r2)
 247         andis.  r10,r0,DBCR0_IC@h
 248         bnel-   load_dbcr0
 249 #endif
 250         stwcx.  r0,0,r1                 /* to clear the reservation */
 251         lwz     r4,_LINK(r1)
 252         lwz     r5,_CCR(r1)
 253         mtlr    r4
 254         mtcr    r5
 255         lwz     r7,_NIP(r1)
 256         lwz     r8,_MSR(r1)
 257         FIX_SRR1(r8, r0)
 258         lwz     r2,GPR2(r1)
 259         lwz     r1,GPR1(r1)
 260         mtspr   SPRN_SRR0,r7
 261         mtspr   SPRN_SRR1,r8
 262         SYNC
 263         RFI
 264
 265 66:     li      r3,-ENOSYS
 266         b       ret_from_syscall
 267
 268         .globl  ret_from_fork
 269 ret_from_fork:
 270         REST_NVGPRS(r1)
 271         bl      schedule_tail
 272         li      r3,0
 273         b       ret_from_syscall
 274
 275 /* Traced system call support */
 276 syscall_dotrace:
 277         SAVE_NVGPRS(r1)
 278         li      r0,0xc00
 279         stw     r0,TRAP(r1)
 280         bl      do_syscall_trace
 281         lwz     r0,GPR0(r1)     /* Restore original registers */
 282         lwz     r3,GPR3(r1)
 283         lwz     r4,GPR4(r1)
 284         lwz     r5,GPR5(r1)
 285         lwz     r6,GPR6(r1)
 286         lwz     r7,GPR7(r1)
 287         lwz     r8,GPR8(r1)
 288         REST_NVGPRS(r1)
 289         b       syscall_dotrace_cont
 290
 291 syscall_exit_work:
 292         stw     r6,RESULT(r1)   /* Save result */
 293         stw     r3,GPR3(r1)     /* Update return value */
 294         andi.   r0,r9,_TIF_SYSCALL_TRACE
 295         beq     5f
 296         ori     r10,r10,MSR_EE
 297         SYNC
 298         MTMSRD(r10)             /* re-enable interrupts */
 299         lwz     r4,TRAP(r1)
 300         andi.   r4,r4,1
 301         beq     4f
 302         SAVE_NVGPRS(r1)
 303         li      r4,0xc00
 304         stw     r4,TRAP(r1)
 305 4:
 306         bl      do_syscall_trace
 307         REST_NVGPRS(r1)
 308 2:
 309         lwz     r3,GPR3(r1)
 310         LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */
 311         SYNC
 312         MTMSRD(r10)             /* disable interrupts again */
 313         rlwinm  r12,r1,0,0,18   /* current_thread_info() */
 314         lwz     r9,TI_FLAGS(r12)
 315 5:
 316         andi.   r0,r9,_TIF_NEED_RESCHED
 317         bne     1f
 318         lwz     r5,_MSR(r1)
 319         andi.   r5,r5,MSR_PR
 320         beq     syscall_exit_cont
 321         andi.   r0,r9,_TIF_SIGPENDING
 322         beq     syscall_exit_cont
 323         b       do_user_signal
 324 1:
 325         ori     r10,r10,MSR_EE
 326         SYNC
 327         MTMSRD(r10)             /* re-enable interrupts */
 328         bl      schedule
 329         b       2b
 330
 331 #ifdef SHOW_SYSCALLS
 332 do_show_syscall:
 333 #ifdef SHOW_SYSCALLS_TASK
 334         lis     r11,show_syscalls_task@ha
 335         lwz     r11,show_syscalls_task@l(r11)
 336         cmp     0,r2,r11
 337         bnelr
 338 #endif
 339         stw     r31,GPR31(r1)
 340         mflr    r31
 341         lis     r3,7f@ha
 342         addi    r3,r3,7f@l
 343         lwz     r4,GPR0(r1)
 344         lwz     r5,GPR3(r1)
 345         lwz     r6,GPR4(r1)
 346         lwz     r7,GPR5(r1)
 347         lwz     r8,GPR6(r1)
 348         lwz     r9,GPR7(r1)
 349         bl      printk
 350         lis     r3,77f@ha
 351         addi    r3,r3,77f@l
 352         lwz     r4,GPR8(r1)
 353         mr      r5,r2
 354         bl      printk
 355         lwz     r0,GPR0(r1)
 356         lwz     r3,GPR3(r1)
 357         lwz     r4,GPR4(r1)
 358         lwz     r5,GPR5(r1)
 359         lwz     r6,GPR6(r1)
 360         lwz     r7,GPR7(r1)
 361         lwz     r8,GPR8(r1)
 362         mtlr    r31
 363         lwz     r31,GPR31(r1)
 364         blr
 365
 366 do_show_syscall_exit:
 367 #ifdef SHOW_SYSCALLS_TASK
 368         lis     r11,show_syscalls_task@ha
 369         lwz     r11,show_syscalls_task@l(r11)
 370         cmp     0,r2,r11
 371         bnelr
 372 #endif
 373         stw     r31,GPR31(r1)
 374         mflr    r31
 375         stw     r3,RESULT(r1)   /* Save result */
 376         mr      r4,r3
 377         lis     r3,79f@ha
 378         addi    r3,r3,79f@l
 379         bl      printk
 380         lwz     r3,RESULT(r1)
 381         mtlr    r31
 382         lwz     r31,GPR31(r1)
 383         blr
 384
 385 7:      .string "syscall %d(%x, %x, %x, %x, %x, "
 386 77:     .string "%x), current=%p\n"
 387 79:     .string " -> %x\n"
 388         .align  2,0
 389
 390 #ifdef SHOW_SYSCALLS_TASK
 391         .data
 392         .globl  show_syscalls_task
 393 show_syscalls_task:
 394         .long   -1
 395         .text
 396 #endif
 397 #endif /* SHOW_SYSCALLS */
 398
 399 /*
 400  * The sigsuspend and rt_sigsuspend system calls can call do_signal
 401  * and thus put the process into the stopped state where we might
 402  * want to examine its user state with ptrace.  Therefore we need
 403  * to save all the nonvolatile registers (r13 - r31) before calling
 404  * the C code.
 405  */
 406         .globl  ppc_sigsuspend
 407 ppc_sigsuspend:
 408         SAVE_NVGPRS(r1)
 409         lwz     r0,TRAP(r1)
 410         rlwinm  r0,r0,0,0,30            /* clear LSB to indicate full */
 411         stw     r0,TRAP(r1)             /* register set saved */
 412         b       sys_sigsuspend
 413
 414         .globl  ppc_rt_sigsuspend
 415 ppc_rt_sigsuspend:
 416         SAVE_NVGPRS(r1)
 417         lwz     r0,TRAP(r1)
 418         rlwinm  r0,r0,0,0,30
 419         stw     r0,TRAP(r1)
 420         b       sys_rt_sigsuspend
 421
 422         .globl  ppc_fork
 423 ppc_fork:
 424         SAVE_NVGPRS(r1)
 425         lwz     r0,TRAP(r1)
 426         rlwinm  r0,r0,0,0,30            /* clear LSB to indicate full */
 427         stw     r0,TRAP(r1)             /* register set saved */
 428         b       sys_fork
 429
 430         .globl  ppc_vfork
 431 ppc_vfork:
 432         SAVE_NVGPRS(r1)
 433         lwz     r0,TRAP(r1)
 434         rlwinm  r0,r0,0,0,30            /* clear LSB to indicate full */
 435         stw     r0,TRAP(r1)             /* register set saved */
 436         b       sys_vfork
 437
 438         .globl  ppc_clone
 439 ppc_clone:
 440         SAVE_NVGPRS(r1)
 441         lwz     r0,TRAP(r1)
 442         rlwinm  r0,r0,0,0,30            /* clear LSB to indicate full */
 443         stw     r0,TRAP(r1)             /* register set saved */
 444         b       sys_clone
 445
 446         .globl  ppc_swapcontext
 447 ppc_swapcontext:
 448         SAVE_NVGPRS(r1)
 449         lwz     r0,TRAP(r1)
 450         rlwinm  r0,r0,0,0,30            /* clear LSB to indicate full */
 451         stw     r0,TRAP(r1)             /* register set saved */
 452         b       sys_swapcontext
 453
 454 /*
 455  * Top-level page fault handling.
 456  * This is in assembler because if do_page_fault tells us that
 457  * it is a bad kernel page fault, we want to save the non-volatile
 458  * registers before calling bad_page_fault.
 459  */
 460         .globl  handle_page_fault
 461 handle_page_fault:
 462         stw     r4,_DAR(r1)
 463         addi    r3,r1,STACK_FRAME_OVERHEAD
 464         bl      do_page_fault
 465         cmpwi   r3,0
 466         beq+    ret_from_except
 467         SAVE_NVGPRS(r1)
 468         lwz     r0,TRAP(r1)
 469         clrrwi  r0,r0,1
 470         stw     r0,TRAP(r1)
 471         mr      r5,r3
 472         addi    r3,r1,STACK_FRAME_OVERHEAD
 473         lwz     r4,_DAR(r1)
 474         bl      bad_page_fault
 475         b       ret_from_except_full
 476
 477 /*
 478  * This routine switches between two different tasks.  The process
 479  * state of one is saved on its kernel stack.  Then the state
 480  * of the other is restored from its kernel stack.  The memory
 481  * management hardware is updated to the second process's state.
 482  * Finally, we can return to the second process.
 483  * On entry, r3 points to the THREAD for the current task, r4
 484  * points to the THREAD for the new task.
 485  *
 486  * This routine is always called with interrupts disabled.
 487  *
 488  * Note: there are two ways to get to the "going out" portion
 489  * of this code; either by coming in via the entry (_switch)
 490  * or via "fork" which must set up an environment equivalent
 491  * to the "_switch" path.  If you change this , you'll have to
 492  * change the fork code also.
 493  *
 494  * The code which creates the new task context is in 'copy_thread'
 495  * in arch/ppc/kernel/process.c
 496  */
 497 _GLOBAL(_switch)
 498         stwu    r1,-INT_FRAME_SIZE(r1)
 499         mflr    r0
 500         stw     r0,INT_FRAME_SIZE+4(r1)
 501         /* r3-r12 are caller saved -- Cort */
 502         SAVE_NVGPRS(r1)
 503         stw     r0,_NIP(r1)     /* Return to switch caller */
 504         mfmsr   r11
 505         li      r0,MSR_FP       /* Disable floating-point */
 506 #ifdef CONFIG_ALTIVEC
 507 BEGIN_FTR_SECTION
 508         oris    r0,r0,MSR_VEC@h /* Disable altivec */
 509         mfspr   r12,SPRN_VRSAVE /* save vrsave register value */
 510         stw     r12,THREAD+THREAD_VRSAVE(r2)
 511 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
 512 #endif /* CONFIG_ALTIVEC */
 513 #ifdef CONFIG_SPE
 514         oris    r0,r0,MSR_SPE@h  /* Disable SPE */
 515         mfspr   r12,SPRN_SPEFSCR /* save spefscr register value */
 516         stw     r12,THREAD+THREAD_SPEFSCR(r2)
 517 #endif /* CONFIG_SPE */
 518         and.    r0,r0,r11       /* FP or altivec or SPE enabled? */
 519         beq+    1f
 520         andc    r11,r11,r0
 521         MTMSRD(r11)
 522         isync
 523 1:      stw     r11,_MSR(r1)
 524         mfcr    r10
 525         stw     r10,_CCR(r1)
 526         stw     r1,KSP(r3)      /* Set old stack pointer */
 527
 528 #ifdef CONFIG_SMP
 529         /* We need a sync somewhere here to make sure that if the
 530          * previous task gets rescheduled on another CPU, it sees all
 531          * stores it has performed on this one.
 532          */
 533         sync
 534 #endif /* CONFIG_SMP */
 535
 536         tophys(r0,r4)
 537         CLR_TOP32(r0)
 538         mtspr   SPRN_SPRG3,r0   /* Update current THREAD phys addr */
 539         lwz     r1,KSP(r4)      /* Load new stack pointer */
 540
 541         /* save the old current 'last' for return value */
 542         mr      r3,r2
 543         addi    r2,r4,-THREAD   /* Update current */
 544
 545 #ifdef CONFIG_ALTIVEC
 546 BEGIN_FTR_SECTION
 547         lwz     r0,THREAD+THREAD_VRSAVE(r2)
 548         mtspr   SPRN_VRSAVE,r0          /* if G4, restore VRSAVE reg */
 549 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
 550 #endif /* CONFIG_ALTIVEC */
 551 #ifdef CONFIG_SPE
 552         lwz     r0,THREAD+THREAD_SPEFSCR(r2)
 553         mtspr   SPRN_SPEFSCR,r0         /* restore SPEFSCR reg */
 554 #endif /* CONFIG_SPE */
 555
 556         lwz     r0,_CCR(r1)
 557         mtcrf   0xFF,r0
 558         /* r3-r12 are destroyed -- Cort */
 559         REST_NVGPRS(r1)
 560
 561         lwz     r4,_NIP(r1)     /* Return to _switch caller in new task */
 562         mtlr    r4
 563         addi    r1,r1,INT_FRAME_SIZE
 564         blr
 565
 566         .globl  sigreturn_exit
 567 sigreturn_exit:
 568         subi    r1,r3,STACK_FRAME_OVERHEAD
 569         rlwinm  r12,r1,0,0,18   /* current_thread_info() */
 570         lwz     r9,TI_FLAGS(r12)
 571         andi.   r0,r9,_TIF_SYSCALL_TRACE
 572         bnel-   do_syscall_trace
 573         /* fall through */
 574
 575         .globl  ret_from_except_full
 576 ret_from_except_full:
 577         REST_NVGPRS(r1)
 578         /* fall through */
 579
 580         .globl  ret_from_except
 581 ret_from_except:
 582         /* Hard-disable interrupts so that current_thread_info()->flags
 583          * can't change between when we test it and when we return
 584          * from the interrupt. */
 585         LOAD_MSR_KERNEL(r10,MSR_KERNEL)
 586         SYNC                    /* Some chip revs have problems here... */
 587         MTMSRD(r10)             /* disable interrupts */
 588
 589         lwz     r3,_MSR(r1)     /* Returning to user mode? */
 590         andi.   r0,r3,MSR_PR
 591         beq     resume_kernel
 592
 593 user_exc_return:                /* r10 contains MSR_KERNEL here */
 594         /* Check current_thread_info()->flags */
 595         rlwinm  r9,r1,0,0,18
 596         lwz     r9,TI_FLAGS(r9)
 597         andi.   r0,r9,(_TIF_SIGPENDING|_TIF_NEED_RESCHED)
 598         bne     do_work
 599
 600 restore_user:
 601 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
 602         /* Check whether this process has its own DBCR0 value.  The single
 603            step bit tells us that dbcr0 should be loaded. */
 604         lwz     r0,THREAD+THREAD_DBCR0(r2)
 605         andis.  r10,r0,DBCR0_IC@h
 606         bnel-   load_dbcr0
 607 #endif
 608
 609 #ifdef CONFIG_PREEMPT
 610         b       restore
 611
 612 /* N.B. the only way to get here is from the beq following ret_from_except. */
 613 resume_kernel:
 614         /* check current_thread_info->preempt_count */
 615         rlwinm  r9,r1,0,0,18
 616         lwz     r0,TI_PREEMPT(r9)
 617         cmpwi   0,r0,0          /* if non-zero, just restore regs and return */
 618         bne     restore
 619         lwz     r0,TI_FLAGS(r9)
 620         andi.   r0,r0,_TIF_NEED_RESCHED
 621         beq+    restore
 622         andi.   r0,r3,MSR_EE    /* interrupts off? */
 623         beq     restore         /* don't schedule if so */
 624 1:      bl      preempt_schedule_irq
 625         rlwinm  r9,r1,0,0,18
 626         lwz     r3,TI_FLAGS(r9)
 627         andi.   r0,r3,_TIF_NEED_RESCHED
 628         bne-    1b
 629 #else
 630 resume_kernel:
 631 #endif /* CONFIG_PREEMPT */
 632
 633         /* interrupts are hard-disabled at this point */
 634 restore:
 635         lwz     r0,GPR0(r1)
 636         lwz     r2,GPR2(r1)
 637         REST_4GPRS(3, r1)
 638         REST_2GPRS(7, r1)
 639
 640         lwz     r10,_XER(r1)
 641         lwz     r11,_CTR(r1)
 642         mtspr   SPRN_XER,r10
 643         mtctr   r11
 644
 645         PPC405_ERR77(0,r1)
 646         stwcx.  r0,0,r1                 /* to clear the reservation */
 647
 648 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
 649         lwz     r9,_MSR(r1)
 650         andi.   r10,r9,MSR_RI           /* check if this exception occurred */
 651         beql    nonrecoverable          /* at a bad place (MSR:RI = 0) */
 652
 653         lwz     r10,_CCR(r1)
 654         lwz     r11,_LINK(r1)
 655         mtcrf   0xFF,r10
 656         mtlr    r11
 657
 658         /*
 659          * Once we put values in SRR0 and SRR1, we are in a state
 660          * where exceptions are not recoverable, since taking an
 661          * exception will trash SRR0 and SRR1.  Therefore we clear the
 662          * MSR:RI bit to indicate this.  If we do take an exception,
 663          * we can't return to the point of the exception but we
 664          * can restart the exception exit path at the label
 665          * exc_exit_restart below.  -- paulus
 666          */
 667         LOAD_MSR_KERNEL(r10,MSR_KERNEL & ~MSR_RI)
 668         SYNC
 669         MTMSRD(r10)             /* clear the RI bit */
 670         .globl exc_exit_restart
 671 exc_exit_restart:
 672         lwz     r9,_MSR(r1)
 673         lwz     r12,_NIP(r1)
 674         FIX_SRR1(r9,r10)
 675         mtspr   SPRN_SRR0,r12
 676         mtspr   SPRN_SRR1,r9
 677         REST_4GPRS(9, r1)
 678         lwz     r1,GPR1(r1)
 679         .globl exc_exit_restart_end
 680 exc_exit_restart_end:
 681         SYNC
 682         RFI
 683
 684 #else /* !(CONFIG_4xx || CONFIG_BOOKE) */
 685         /*
 686          * This is a bit different on 4xx/Book-E because it doesn't have
 687          * the RI bit in the MSR.
 688          * The TLB miss handler checks if we have interrupted
 689          * the exception exit path and restarts it if so
 690          * (well maybe one day it will... :).
 691          */
 692         lwz     r11,_LINK(r1)
 693         mtlr    r11
 694         lwz     r10,_CCR(r1)
 695         mtcrf   0xff,r10
 696         REST_2GPRS(9, r1)
 697         .globl exc_exit_restart
 698 exc_exit_restart:
 699         lwz     r11,_NIP(r1)
 700         lwz     r12,_MSR(r1)
 701 exc_exit_start:
 702         mtspr   SPRN_SRR0,r11
 703         mtspr   SPRN_SRR1,r12
 704         REST_2GPRS(11, r1)
 705         lwz     r1,GPR1(r1)
 706         .globl exc_exit_restart_end
 707 exc_exit_restart_end:
 708         PPC405_ERR77_SYNC
 709         rfi
 710         b       .                       /* prevent prefetch past rfi */
 711
 712 /*
 713  * Returning from a critical interrupt in user mode doesn't need
 714  * to be any different from a normal exception.  For a critical
 715  * interrupt in the kernel, we just return (without checking for
 716  * preemption) since the interrupt may have happened at some crucial
 717  * place (e.g. inside the TLB miss handler), and because we will be
 718  * running with r1 pointing into critical_stack, not the current
 719  * process's kernel stack (and therefore current_thread_info() will
 720  * give the wrong answer).
 721  * We have to restore various SPRs that may have been in use at the
 722  * time of the critical interrupt.
 723  *
 724  */
 725         .globl  ret_from_crit_exc
 726 ret_from_crit_exc:
 727         REST_NVGPRS(r1)
 728         lwz     r3,_MSR(r1)
 729         andi.   r3,r3,MSR_PR
 730         LOAD_MSR_KERNEL(r10,MSR_KERNEL)
 731         bne     user_exc_return
 732
 733         lwz     r0,GPR0(r1)
 734         lwz     r2,GPR2(r1)
 735         REST_4GPRS(3, r1)
 736         REST_2GPRS(7, r1)
 737
 738         lwz     r10,_XER(r1)
 739         lwz     r11,_CTR(r1)
 740         mtspr   SPRN_XER,r10
 741         mtctr   r11
 742
 743         PPC405_ERR77(0,r1)
 744         stwcx.  r0,0,r1                 /* to clear the reservation */
 745
 746         lwz     r11,_LINK(r1)
 747         mtlr    r11
 748         lwz     r10,_CCR(r1)
 749         mtcrf   0xff,r10
 750 #ifdef CONFIG_40x
 751         /* avoid any possible TLB misses here by turning off MSR.DR, we
 752          * assume the instructions here are mapped by a pinned TLB entry */
 753         li      r10,MSR_IR
 754         mtmsr   r10
 755         isync
 756         tophys(r1, r1)
 757 #endif
 758         lwz     r9,_DEAR(r1)
 759         lwz     r10,_ESR(r1)
 760         mtspr   SPRN_DEAR,r9
 761         mtspr   SPRN_ESR,r10
 762         lwz     r11,_NIP(r1)
 763         lwz     r12,_MSR(r1)
 764         mtspr   SPRN_CSRR0,r11
 765         mtspr   SPRN_CSRR1,r12
 766         lwz     r9,GPR9(r1)
 767         lwz     r12,GPR12(r1)
 768         lwz     r10,GPR10(r1)
 769         lwz     r11,GPR11(r1)
 770         lwz     r1,GPR1(r1)
 771         PPC405_ERR77_SYNC
 772         rfci
 773         b       .               /* prevent prefetch past rfci */
 774
 775 #ifdef CONFIG_BOOKE
 776 /*
 777  * Return from a machine check interrupt, similar to a critical
 778  * interrupt.
 779  */
 780         .globl  ret_from_mcheck_exc
 781 ret_from_mcheck_exc:
 782         REST_NVGPRS(r1)
 783         lwz     r3,_MSR(r1)
 784         andi.   r3,r3,MSR_PR
 785         LOAD_MSR_KERNEL(r10,MSR_KERNEL)
 786         bne     user_exc_return
 787
 788         lwz     r0,GPR0(r1)
 789         lwz     r2,GPR2(r1)
 790         REST_4GPRS(3, r1)
 791         REST_2GPRS(7, r1)
 792
 793         lwz     r10,_XER(r1)
 794         lwz     r11,_CTR(r1)
 795         mtspr   SPRN_XER,r10
 796         mtctr   r11
 797
 798         stwcx.  r0,0,r1                 /* to clear the reservation */
 799
 800         lwz     r11,_LINK(r1)
 801         mtlr    r11
 802         lwz     r10,_CCR(r1)
 803         mtcrf   0xff,r10
 804         lwz     r9,_DEAR(r1)
 805         lwz     r10,_ESR(r1)
 806         mtspr   SPRN_DEAR,r9
 807         mtspr   SPRN_ESR,r10
 808         lwz     r11,_NIP(r1)
 809         lwz     r12,_MSR(r1)
 810         mtspr   SPRN_MCSRR0,r11
 811         mtspr   SPRN_MCSRR1,r12
 812         lwz     r9,GPR9(r1)
 813         lwz     r12,GPR12(r1)
 814         lwz     r10,GPR10(r1)
 815         lwz     r11,GPR11(r1)
 816         lwz     r1,GPR1(r1)
 817         RFMCI
 818 #endif /* CONFIG_BOOKE */
 819
 820 /*
 821  * Load the DBCR0 value for a task that is being ptraced,
 822  * having first saved away the global DBCR0.  Note that r0
 823  * has the dbcr0 value to set upon entry to this.
 824  */
 825 load_dbcr0:
 826         mfmsr   r10             /* first disable debug exceptions */
 827         rlwinm  r10,r10,0,~MSR_DE
 828         mtmsr   r10
 829         isync
 830         mfspr   r10,SPRN_DBCR0
 831         lis     r11,global_dbcr0@ha
 832         addi    r11,r11,global_dbcr0@l
 833         stw     r10,0(r11)
 834         mtspr   SPRN_DBCR0,r0
 835         lwz     r10,4(r11)
 836         addi    r10,r10,1
 837         stw     r10,4(r11)
 838         li      r11,-1
 839         mtspr   SPRN_DBSR,r11   /* clear all pending debug events */
 840         blr
 841
 842         .comm   global_dbcr0,8
 843 #endif /* !(CONFIG_4xx || CONFIG_BOOKE) */
 844
 845 do_work:                        /* r10 contains MSR_KERNEL here */
 846         andi.   r0,r9,_TIF_NEED_RESCHED
 847         beq     do_user_signal
 848
 849 do_resched:                     /* r10 contains MSR_KERNEL here */
 850         ori     r10,r10,MSR_EE
 851         SYNC
 852         MTMSRD(r10)             /* hard-enable interrupts */
 853         bl      schedule
 854 recheck:
 855         LOAD_MSR_KERNEL(r10,MSR_KERNEL)
 856         SYNC
 857         MTMSRD(r10)             /* disable interrupts */
 858         rlwinm  r9,r1,0,0,18
 859         lwz     r9,TI_FLAGS(r9)
 860         andi.   r0,r9,_TIF_NEED_RESCHED
 861         bne-    do_resched
 862         andi.   r0,r9,_TIF_SIGPENDING
 863         beq     restore_user
 864 do_user_signal:                 /* r10 contains MSR_KERNEL here */
 865         ori     r10,r10,MSR_EE
 866         SYNC
 867         MTMSRD(r10)             /* hard-enable interrupts */
 868         /* save r13-r31 in the exception frame, if not already done */
 869         lwz     r3,TRAP(r1)
 870         andi.   r0,r3,1
 871         beq     2f
 872         SAVE_NVGPRS(r1)
 873         rlwinm  r3,r3,0,0,30
 874         stw     r3,TRAP(r1)
 875 2:      li      r3,0
 876         addi    r4,r1,STACK_FRAME_OVERHEAD
 877         bl      do_signal
 878         REST_NVGPRS(r1)
 879         b       recheck
 880
 881 /*
 882  * We come here when we are at the end of handling an exception
 883  * that occurred at a place where taking an exception will lose
 884  * state information, such as the contents of SRR0 and SRR1.
 885  */
 886 nonrecoverable:
 887         lis     r10,exc_exit_restart_end@ha
 888         addi    r10,r10,exc_exit_restart_end@l
 889         cmplw   r12,r10
 890         bge     3f
 891         lis     r11,exc_exit_restart@ha
 892         addi    r11,r11,exc_exit_restart@l
 893         cmplw   r12,r11
 894         blt     3f
 895         lis     r10,ee_restarts@ha
 896         lwz     r12,ee_restarts@l(r10)
 897         addi    r12,r12,1
 898         stw     r12,ee_restarts@l(r10)
 899         mr      r12,r11         /* restart at exc_exit_restart */
 900         blr
 901 3:      /* OK, we can't recover, kill this process */
 902         /* but the 601 doesn't implement the RI bit, so assume it's OK */
 903 BEGIN_FTR_SECTION
 904         blr
 905 END_FTR_SECTION_IFSET(CPU_FTR_601)
 906         lwz     r3,TRAP(r1)
 907         andi.   r0,r3,1
 908         beq     4f
 909         SAVE_NVGPRS(r1)
 910         rlwinm  r3,r3,0,0,30
 911         stw     r3,TRAP(r1)
 912 4:      addi    r3,r1,STACK_FRAME_OVERHEAD
 913         bl      nonrecoverable_exception
 914         /* shouldn't return */
 915         b       4b
 916
 917         .comm   ee_restarts,4
 918
 919 /*
 920  * PROM code for specific machines follows.  Put it
 921  * here so it's easy to add arch-specific sections later.
 922  * -- Cort
 923  */
 924 #ifdef CONFIG_PPC_OF
 925 /*
 926  * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
 927  * called with the MMU off.
 928  */
 929 _GLOBAL(enter_rtas)
 930         stwu    r1,-INT_FRAME_SIZE(r1)
 931         mflr    r0
 932         stw     r0,INT_FRAME_SIZE+4(r1)
 933         lis     r4,rtas_data@ha
 934         lwz     r4,rtas_data@l(r4)
 935         lis     r6,1f@ha        /* physical return address for rtas */
 936         addi    r6,r6,1f@l
 937         tophys(r6,r6)
 938         tophys(r7,r1)
 939         lis     r8,rtas_entry@ha
 940         lwz     r8,rtas_entry@l(r8)
 941         mfmsr   r9
 942         stw     r9,8(r1)
 943         LOAD_MSR_KERNEL(r0,MSR_KERNEL)
 944         SYNC                    /* disable interrupts so SRR0/1 */
 945         MTMSRD(r0)              /* don't get trashed */
 946         li      r9,MSR_KERNEL & ~(MSR_IR|MSR_DR)
 947         mtlr    r6
 948         CLR_TOP32(r7)
 949         mtspr   SPRN_SPRG2,r7
 950         mtspr   SPRN_SRR0,r8
 951         mtspr   SPRN_SRR1,r9
 952         RFI
 953 1:      tophys(r9,r1)
 954         lwz     r8,INT_FRAME_SIZE+4(r9) /* get return address */
 955         lwz     r9,8(r9)        /* original msr value */
 956         FIX_SRR1(r9,r0)
 957         addi    r1,r1,INT_FRAME_SIZE
 958         li      r0,0
 959         mtspr   SPRN_SPRG2,r0
 960         mtspr   SPRN_SRR0,r8
 961         mtspr   SPRN_SRR1,r9
 962         RFI                     /* return to caller */
 963
 964         .globl  machine_check_in_rtas
 965 machine_check_in_rtas:
 966         twi     31,0,0
 967         /* XXX load up BATs and panic */
 968
 969 #endif /* CONFIG_PPC_OF */