[IA64] Workaround for RSE issue
authorTony Luck <tony.luck@intel.com>
Tue, 27 May 2008 20:23:16 +0000 (13:23 -0700)
committerTony Luck <tony.luck@intel.com>
Tue, 27 May 2008 20:24:39 +0000 (13:24 -0700)
Problem: An application violating the architectural rules regarding
operation dependencies and having specific Register Stack Engine (RSE)
state at the time of the violation, may result in an illegal operation
fault and invalid RSE state.  Such faults may initiate a cascade of
repeated illegal operation faults within OS interruption handlers.
The specific behavior is OS dependent.

Implication: An application causing an illegal operation fault with
specific RSE state may result in a series of illegal operation faults
and an eventual OS stack overflow condition.

Workaround: OS interruption handlers that switch to kernel backing
store implement a check for invalid RSE state to avoid the series
of illegal operation faults.

The core of the workaround is the RSE_WORKAROUND code sequence
inserted into each invocation of the SAVE_MIN_WITH_COVER and
SAVE_MIN_WITH_COVER_R19 macros.  This sequence includes hard-coded
constants that depend on the number of stacked physical registers
being 96.  The rest of this patch consists of code to disable this
workaround should this not be the case (with the presumption that
if a future Itanium processor increases the number of registers, it
would also remove the need for this patch).

Move the start of the RBS up to a mod32 boundary to avoid some
corner cases.

The dispatch_illegal_op_fault code outgrew the spot it was
squatting in when built with this patch and CONFIG_VIRT_CPU_ACCOUNTING=y
Move it out to the end of the ivt.

Signed-off-by: Tony Luck <tony.luck@intel.com>
arch/ia64/kernel/ivt.S
arch/ia64/kernel/minstate.h
arch/ia64/kernel/patch.c
arch/ia64/kernel/setup.c
arch/ia64/kernel/vmlinux.lds.S
include/asm-ia64/patch.h
include/asm-ia64/ptrace.h
include/asm-ia64/sections.h

index 6678c49daba3c40fcf9972c572ad6087e4690ce5..80b44ea052d74420abca826c119d8ee192743144 100644 (file)
@@ -1076,48 +1076,6 @@ END(ia64_syscall_setup)
        DBG_FAULT(15)
        FAULT(15)
 
-       /*
-        * Squatting in this space ...
-        *
-        * This special case dispatcher for illegal operation faults allows preserved
-        * registers to be modified through a callback function (asm only) that is handed
-        * back from the fault handler in r8. Up to three arguments can be passed to the
-        * callback function by returning an aggregate with the callback as its first
-        * element, followed by the arguments.
-        */
-ENTRY(dispatch_illegal_op_fault)
-       .prologue
-       .body
-       SAVE_MIN_WITH_COVER
-       ssm psr.ic | PSR_DEFAULT_BITS
-       ;;
-       srlz.i          // guarantee that interruption collection is on
-       ;;
-(p15)  ssm psr.i       // restore psr.i
-       adds r3=8,r2    // set up second base pointer for SAVE_REST
-       ;;
-       alloc r14=ar.pfs,0,0,1,0        // must be first in insn group
-       mov out0=ar.ec
-       ;;
-       SAVE_REST
-       PT_REGS_UNWIND_INFO(0)
-       ;;
-       br.call.sptk.many rp=ia64_illegal_op_fault
-.ret0: ;;
-       alloc r14=ar.pfs,0,0,3,0        // must be first in insn group
-       mov out0=r9
-       mov out1=r10
-       mov out2=r11
-       movl r15=ia64_leave_kernel
-       ;;
-       mov rp=r15
-       mov b6=r8
-       ;;
-       cmp.ne p6,p0=0,r8
-(p6)   br.call.dpnt.many b6=b6         // call returns to ia64_leave_kernel
-       br.sptk.many ia64_leave_kernel
-END(dispatch_illegal_op_fault)
-
        .org ia64_ivt+0x4000
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x4000 Entry 16 (size 64 bundles) Reserved
@@ -1715,6 +1673,48 @@ END(ia32_interrupt)
        DBG_FAULT(67)
        FAULT(67)
 
+       /*
+        * Squatting in this space ...
+        *
+        * This special case dispatcher for illegal operation faults allows preserved
+        * registers to be modified through a callback function (asm only) that is handed
+        * back from the fault handler in r8. Up to three arguments can be passed to the
+        * callback function by returning an aggregate with the callback as its first
+        * element, followed by the arguments.
+        */
+ENTRY(dispatch_illegal_op_fault)
+       .prologue
+       .body
+       SAVE_MIN_WITH_COVER
+       ssm psr.ic | PSR_DEFAULT_BITS
+       ;;
+       srlz.i          // guarantee that interruption collection is on
+       ;;
+(p15)  ssm psr.i       // restore psr.i
+       adds r3=8,r2    // set up second base pointer for SAVE_REST
+       ;;
+       alloc r14=ar.pfs,0,0,1,0        // must be first in insn group
+       mov out0=ar.ec
+       ;;
+       SAVE_REST
+       PT_REGS_UNWIND_INFO(0)
+       ;;
+       br.call.sptk.many rp=ia64_illegal_op_fault
+.ret0: ;;
+       alloc r14=ar.pfs,0,0,3,0        // must be first in insn group
+       mov out0=r9
+       mov out1=r10
+       mov out2=r11
+       movl r15=ia64_leave_kernel
+       ;;
+       mov rp=r15
+       mov b6=r8
+       ;;
+       cmp.ne p6,p0=0,r8
+(p6)   br.call.dpnt.many b6=b6         // call returns to ia64_leave_kernel
+       br.sptk.many ia64_leave_kernel
+END(dispatch_illegal_op_fault)
+
 #ifdef CONFIG_IA32_SUPPORT
 
        /*
index 7c548ac52bbc0956b4683a197f713c29436d0c71..74b6d670aaef2d42a4c7574a3543b3fb26e6dd98 100644 (file)
@@ -15,6 +15,9 @@
 #define ACCOUNT_SYS_ENTER
 #endif
 
+.section ".data.patch.rse", "a"
+.previous
+
 /*
  * DO_SAVE_MIN switches to the kernel stacks (if necessary) and saves
  * the minimum state necessary that allows us to turn psr.ic back
@@ -40,7 +43,7 @@
  * Note that psr.ic is NOT turned on by this macro.  This is so that
  * we can pass interruption state as arguments to a handler.
  */
-#define DO_SAVE_MIN(COVER,SAVE_IFS,EXTRA)                                                      \
+#define DO_SAVE_MIN(COVER,SAVE_IFS,EXTRA,WORKAROUND)                                           \
        mov r16=IA64_KR(CURRENT);       /* M */                                                 \
        mov r27=ar.rsc;                 /* M */                                                 \
        mov r20=r1;                     /* A */                                                 \
@@ -87,6 +90,7 @@
        tbit.nz p15,p0=r29,IA64_PSR_I_BIT;                                                      \
        mov r29=b0                                                                              \
        ;;                                                                                      \
+       WORKAROUND;                                                                             \
        adds r16=PT(R8),r1;     /* initialize first base pointer */                             \
        adds r17=PT(R9),r1;     /* initialize second base pointer */                            \
 (pKStk)        mov r18=r0;             /* make sure r18 isn't NaT */                                   \
        st8 [r25]=r10;          /* ar.ssd */    \
        ;;
 
-#define SAVE_MIN_WITH_COVER    DO_SAVE_MIN(cover, mov r30=cr.ifs,)
-#define SAVE_MIN_WITH_COVER_R19        DO_SAVE_MIN(cover, mov r30=cr.ifs, mov r15=r19)
-#define SAVE_MIN               DO_SAVE_MIN(     , mov r30=r0, )
+#define RSE_WORKAROUND                         \
+(pUStk) extr.u r17=r18,3,6;                    \
+(pUStk)        sub r16=r18,r22;                        \
+[1:](pKStk)    br.cond.sptk.many 1f;           \
+       .xdata4 ".data.patch.rse",1b-.          \
+       ;;                                      \
+       cmp.ge p6,p7 = 33,r17;                  \
+       ;;                                      \
+(p6)   mov r17=0x310;                          \
+(p7)   mov r17=0x308;                          \
+       ;;                                      \
+       cmp.leu p1,p0=r16,r17;                  \
+(p1)   br.cond.sptk.many 1f;                   \
+       dep.z r17=r26,0,62;                     \
+       movl r16=2f;                            \
+       ;;                                      \
+       mov ar.pfs=r17;                         \
+       dep r27=r0,r27,16,14;                   \
+       mov b0=r16;                             \
+       ;;                                      \
+       br.ret.sptk b0;                         \
+       ;;                                      \
+2:                                             \
+       mov ar.rsc=r0                           \
+       ;;                                      \
+       flushrs;                                \
+       ;;                                      \
+       mov ar.bspstore=r22                     \
+       ;;                                      \
+       mov r18=ar.bsp;                         \
+       ;;                                      \
+1:                                             \
+       .pred.rel "mutex", pKStk, pUStk
+
+#define SAVE_MIN_WITH_COVER    DO_SAVE_MIN(cover, mov r30=cr.ifs, , RSE_WORKAROUND)
+#define SAVE_MIN_WITH_COVER_R19        DO_SAVE_MIN(cover, mov r30=cr.ifs, mov r15=r19, RSE_WORKAROUND)
+#define SAVE_MIN                       DO_SAVE_MIN(     , mov r30=r0, , )
index e0dca8743dbb11b87031b7ec9e8e845638a7bb4a..b83b2c516008d4cdac5db340a53c74dfa8f4444a 100644 (file)
@@ -115,6 +115,29 @@ ia64_patch_vtop (unsigned long start, unsigned long end)
        ia64_srlz_i();
 }
 
+/*
+ * Disable the RSE workaround by turning the conditional branch
+ * that we tagged in each place the workaround was used into an
+ * unconditional branch.
+ */
+void __init
+ia64_patch_rse (unsigned long start, unsigned long end)
+{
+       s32 *offp = (s32 *) start;
+       u64 ip, *b;
+
+       while (offp < (s32 *) end) {
+               ip = (u64) offp + *offp;
+
+               b = (u64 *)(ip & -16);
+               b[1] &= ~0xf800000L;
+               ia64_fc((void *) ip);
+               ++offp;
+       }
+       ia64_sync_i();
+       ia64_srlz_i();
+}
+
 void __init
 ia64_patch_mckinley_e9 (unsigned long start, unsigned long end)
 {
index e9596cd0cdab21595f7c5f85d7da7d2952beb5d6..f48a809c686d899be2d431a96487e292329dc9c8 100644 (file)
@@ -560,6 +560,17 @@ setup_arch (char **cmdline_p)
        /* process SAL system table: */
        ia64_sal_init(__va(efi.sal_systab));
 
+#ifdef CONFIG_ITANIUM
+       ia64_patch_rse((u64) __start___rse_patchlist, (u64) __end___rse_patchlist);
+#else
+       {
+               u64 num_phys_stacked;
+
+               if (ia64_pal_rse_info(&num_phys_stacked, 0) == 0 && num_phys_stacked > 96)
+                       ia64_patch_rse((u64) __start___rse_patchlist, (u64) __end___rse_patchlist);
+       }
+#endif
+
 #ifdef CONFIG_SMP
        cpu_physical_id(0) = hard_smp_processor_id();
 #endif
index 80622acc95deea98c83f5e2ce97cc11cc4bc883a..5929ab10a289d8adeec09ee13ba0a4b03ed4d2ca 100644 (file)
@@ -156,6 +156,13 @@ SECTIONS
          __end___vtop_patchlist = .;
        }
 
+  .data.patch.rse : AT(ADDR(.data.patch.rse) - LOAD_OFFSET)
+       {
+         __start___rse_patchlist = .;
+         *(.data.patch.rse)
+         __end___rse_patchlist = .;
+       }
+
   .data.patch.mckinley_e9 : AT(ADDR(.data.patch.mckinley_e9) - LOAD_OFFSET)
        {
          __start___mckinley_e9_bundles = .;
index a71543084fb41dc51cd90b168824a6a5a4b5eb26..295fe6ab4584c4db9585c6a84ff9ac450d590c77 100644 (file)
@@ -21,6 +21,7 @@ extern void ia64_patch_imm60 (u64 insn_addr, u64 val);                /* patch "brl" w/ip-rel
 extern void ia64_patch_mckinley_e9 (unsigned long start, unsigned long end);
 extern void ia64_patch_vtop (unsigned long start, unsigned long end);
 extern void ia64_patch_phys_stack_reg(unsigned long val);
+extern void ia64_patch_rse (unsigned long start, unsigned long end);
 extern void ia64_patch_gate (void);
 
 #endif /* _ASM_IA64_PATCH_H */
index 4b2a8d40ebc5ed398538a5aa63650605b561d803..15f8dcfe6eeef8d675d07cb13d7aa71ceea0fc00 100644 (file)
@@ -76,7 +76,7 @@
 # define KERNEL_STACK_SIZE_ORDER               0
 #endif
 
-#define IA64_RBS_OFFSET                        ((IA64_TASK_SIZE + IA64_THREAD_INFO_SIZE + 15) & ~15)
+#define IA64_RBS_OFFSET                        ((IA64_TASK_SIZE + IA64_THREAD_INFO_SIZE + 31) & ~31)
 #define IA64_STK_OFFSET                        ((1 << KERNEL_STACK_SIZE_ORDER)*PAGE_SIZE)
 
 #define KERNEL_STACK_SIZE              IA64_STK_OFFSET
index dc42a359894fe759617fdf47b5e6a048eda8190b..7286e4a9fe8449e1b02e5e762eb9dc7e1aeb53ce 100644 (file)
@@ -10,6 +10,7 @@
 
 extern char __per_cpu_start[], __per_cpu_end[], __phys_per_cpu_start[];
 extern char __start___vtop_patchlist[], __end___vtop_patchlist[];
+extern char __start___rse_patchlist[], __end___rse_patchlist[];
 extern char __start___mckinley_e9_bundles[], __end___mckinley_e9_bundles[];
 extern char __start___phys_stack_reg_patchlist[], __end___phys_stack_reg_patchlist[];
 extern char __start_gate_section[];