arch/powerpc/mm/hash_low_64.S

   1 /*
   2  * ppc64 MMU hashtable management routines
   3  *
   4  * (c) Copyright IBM Corp. 2003, 2005
   5  *
   6  * Maintained by: Benjamin Herrenschmidt
   7  *                <benh@kernel.crashing.org>
   8  *
   9  * This file is covered by the GNU Public Licence v2 as
  10  * described in the kernel's COPYING file.
  11  */
  12
  13 #include <asm/reg.h>
  14 #include <asm/pgtable.h>
  15 #include <asm/mmu.h>
  16 #include <asm/page.h>
  17 #include <asm/types.h>
  18 #include <asm/ppc_asm.h>
  19 #include <asm/asm-offsets.h>
  20 #include <asm/cputable.h>
  21
  22         .text
  23
  24 /*
  25  * Stackframe:
  26  *
  27  *         +-> Back chain                       (SP + 256)
  28  *         |   General register save area       (SP + 112)
  29  *         |   Parameter save area              (SP + 48)
  30  *         |   TOC save area                    (SP + 40)
  31  *         |   link editor doubleword           (SP + 32)
  32  *         |   compiler doubleword              (SP + 24)
  33  *         |   LR save area                     (SP + 16)
  34  *         |   CR save area                     (SP + 8)
  35  * SP ---> +-- Back chain                       (SP + 0)
  36  */
  37 #define STACKFRAMESIZE  256
  38
  39 /* Save parameters offsets */
  40 #define STK_PARM(i)     (STACKFRAMESIZE + 48 + ((i)-3)*8)
  41
  42 /* Save non-volatile offsets */
  43 #define STK_REG(i)      (112 + ((i)-14)*8)
  44
  45
  46 #ifndef CONFIG_PPC_64K_PAGES
  47
  48 /*****************************************************************************
  49  *                                                                           *
  50  *           4K SW & 4K HW pages implementation                              *
  51  *                                                                           *
  52  *****************************************************************************/
  53
  54
  55 /*
  56  * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
  57  *               pte_t *ptep, unsigned long trap, int local, int ssize)
  58  *
  59  * Adds a 4K page to the hash table in a segment of 4K pages only
  60  */
  61
  62 _GLOBAL(__hash_page_4K)
  63         mflr    r0
  64         std     r0,16(r1)
  65         stdu    r1,-STACKFRAMESIZE(r1)
  66         /* Save all params that we need after a function call */
  67         std     r6,STK_PARM(r6)(r1)
  68         std     r8,STK_PARM(r8)(r1)
  69         std     r9,STK_PARM(r9)(r1)
  70
  71         /* Save non-volatile registers.
  72          * r31 will hold "old PTE"
  73          * r30 is "new PTE"
  74          * r29 is "va"
  75          * r28 is a hash value
  76          * r27 is hashtab mask (maybe dynamic patched instead ?)
  77          */
  78         std     r27,STK_REG(r27)(r1)
  79         std     r28,STK_REG(r28)(r1)
  80         std     r29,STK_REG(r29)(r1)
  81         std     r30,STK_REG(r30)(r1)
  82         std     r31,STK_REG(r31)(r1)
  83
  84         /* Step 1:
  85          *
  86          * Check permissions, atomically mark the linux PTE busy
  87          * and hashed.
  88          */
  89 1:
  90         ldarx   r31,0,r6
  91         /* Check access rights (access & ~(pte_val(*ptep))) */
  92         andc.   r0,r4,r31
  93         bne-    htab_wrong_access
  94         /* Check if PTE is busy */
  95         andi.   r0,r31,_PAGE_BUSY
  96         /* If so, just bail out and refault if needed. Someone else
  97          * is changing this PTE anyway and might hash it.
  98          */
  99         bne-    htab_bail_ok
 100
 101         /* Prepare new PTE value (turn access RW into DIRTY, then
 102          * add BUSY,HASHPTE and ACCESSED)
 103          */
 104         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 105         or      r30,r30,r31
 106         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
 107         /* Write the linux PTE atomically (setting busy) */
 108         stdcx.  r30,0,r6
 109         bne-    1b
 110         isync
 111
 112         /* Step 2:
 113          *
 114          * Insert/Update the HPTE in the hash table. At this point,
 115          * r4 (access) is re-useable, we use it for the new HPTE flags
 116          */
 117
 118 BEGIN_FTR_SECTION
 119         cmpdi   r9,0                    /* check segment size */
 120         bne     3f
 121 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
 122         /* Calc va and put it in r29 */
 123         rldicr  r29,r5,28,63-28
 124         rldicl  r3,r3,0,36
 125         or      r29,r3,r29
 126
 127         /* Calculate hash value for primary slot and store it in r28 */
 128         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 129         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
 130         xor     r28,r5,r0
 131         b       4f
 132
 133 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
 134         sldi    r29,r5,40               /* vsid << 40 */
 135         clrldi  r3,r3,24                /* ea & 0xffffffffff */
 136         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 137         clrldi  r5,r5,40                /* vsid & 0xffffff */
 138         rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
 139         xor     r28,r28,r5
 140         or      r29,r3,r29              /* VA */
 141         xor     r28,r28,r0              /* hash */
 142
 143         /* Convert linux PTE bits into HW equivalents */
 144 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 145         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 146         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 147         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 148         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 149         andc    r0,r30,r0               /* r0 = pte & ~r0 */
 150         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 151         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 152
 153         /* We eventually do the icache sync here (maybe inline that
 154          * code rather than call a C function...)
 155          */
 156 BEGIN_FTR_SECTION
 157         mr      r4,r30
 158         mr      r5,r7
 159         bl      .hash_page_do_lazy_icache
 160 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 161
 162         /* At this point, r3 contains new PP bits, save them in
 163          * place of "access" in the param area (sic)
 164          */
 165         std     r3,STK_PARM(r4)(r1)
 166
 167         /* Get htab_hash_mask */
 168         ld      r4,htab_hash_mask@got(2)
 169         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 170
 171         /* Check if we may already be in the hashtable, in this case, we
 172          * go to out-of-line code to try to modify the HPTE
 173          */
 174         andi.   r0,r31,_PAGE_HASHPTE
 175         bne     htab_modify_pte
 176
 177 htab_insert_pte:
 178         /* Clear hpte bits in new pte (we also clear BUSY btw) and
 179          * add _PAGE_HASHPTE
 180          */
 181         lis     r0,_PAGE_HPTEFLAGS@h
 182         ori     r0,r0,_PAGE_HPTEFLAGS@l
 183         andc    r30,r30,r0
 184         ori     r30,r30,_PAGE_HASHPTE
 185
 186         /* physical address r5 */
 187         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 188         sldi    r5,r5,PAGE_SHIFT
 189
 190         /* Calculate primary group hash */
 191         and     r0,r28,r27
 192         rldicr  r3,r0,3,63-3            /* r3 = (hash & mask) << 3 */
 193
 194         /* Call ppc_md.hpte_insert */
 195         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 196         mr      r4,r29                  /* Retreive va */
 197         li      r7,0                    /* !bolted, !secondary */
 198         li      r8,MMU_PAGE_4K          /* page size */
 199         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 200 _GLOBAL(htab_call_hpte_insert1)
 201         bl      .                       /* Patched by htab_finish_init() */
 202         cmpdi   0,r3,0
 203         bge     htab_pte_insert_ok      /* Insertion successful */
 204         cmpdi   0,r3,-2                 /* Critical failure */
 205         beq-    htab_pte_insert_failure
 206
 207         /* Now try secondary slot */
 208
 209         /* physical address r5 */
 210         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 211         sldi    r5,r5,PAGE_SHIFT
 212
 213         /* Calculate secondary group hash */
 214         andc    r0,r27,r28
 215         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 216
 217         /* Call ppc_md.hpte_insert */
 218         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 219         mr      r4,r29                  /* Retreive va */
 220         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 221         li      r8,MMU_PAGE_4K          /* page size */
 222         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 223 _GLOBAL(htab_call_hpte_insert2)
 224         bl      .                       /* Patched by htab_finish_init() */
 225         cmpdi   0,r3,0
 226         bge+    htab_pte_insert_ok      /* Insertion successful */
 227         cmpdi   0,r3,-2                 /* Critical failure */
 228         beq-    htab_pte_insert_failure
 229
 230         /* Both are full, we need to evict something */
 231         mftb    r0
 232         /* Pick a random group based on TB */
 233         andi.   r0,r0,1
 234         mr      r5,r28
 235         bne     2f
 236         not     r5,r5
 237 2:      and     r0,r5,r27
 238         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 239         /* Call ppc_md.hpte_remove */
 240 _GLOBAL(htab_call_hpte_remove)
 241         bl      .                       /* Patched by htab_finish_init() */
 242
 243         /* Try all again */
 244         b       htab_insert_pte
 245
 246 htab_bail_ok:
 247         li      r3,0
 248         b       htab_bail
 249
 250 htab_pte_insert_ok:
 251         /* Insert slot number & secondary bit in PTE */
 252         rldimi  r30,r3,12,63-15
 253
 254         /* Write out the PTE with a normal write
 255          * (maybe add eieio may be good still ?)
 256          */
 257 htab_write_out_pte:
 258         ld      r6,STK_PARM(r6)(r1)
 259         std     r30,0(r6)
 260         li      r3, 0
 261 htab_bail:
 262         ld      r27,STK_REG(r27)(r1)
 263         ld      r28,STK_REG(r28)(r1)
 264         ld      r29,STK_REG(r29)(r1)
 265         ld      r30,STK_REG(r30)(r1)
 266         ld      r31,STK_REG(r31)(r1)
 267         addi    r1,r1,STACKFRAMESIZE
 268         ld      r0,16(r1)
 269         mtlr    r0
 270         blr
 271
 272 htab_modify_pte:
 273         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 274         mr      r4,r3
 275         rlwinm  r3,r31,32-12,29,31
 276
 277         /* Secondary group ? if yes, get a inverted hash value */
 278         mr      r5,r28
 279         andi.   r0,r31,_PAGE_SECONDARY
 280         beq     1f
 281         not     r5,r5
 282 1:
 283         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 284         and     r0,r5,r27
 285         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 286         add     r3,r0,r3        /* add slot idx */
 287
 288         /* Call ppc_md.hpte_updatepp */
 289         mr      r5,r29                  /* va */
 290         li      r6,MMU_PAGE_4K          /* page size */
 291         ld      r7,STK_PARM(r9)(r1)     /* segment size */
 292         ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
 293 _GLOBAL(htab_call_hpte_updatepp)
 294         bl      .                       /* Patched by htab_finish_init() */
 295
 296         /* if we failed because typically the HPTE wasn't really here
 297          * we try an insertion.
 298          */
 299         cmpdi   0,r3,-1
 300         beq-    htab_insert_pte
 301
 302         /* Clear the BUSY bit and Write out the PTE */
 303         li      r0,_PAGE_BUSY
 304         andc    r30,r30,r0
 305         b       htab_write_out_pte
 306
 307 htab_wrong_access:
 308         /* Bail out clearing reservation */
 309         stdcx.  r31,0,r6
 310         li      r3,1
 311         b       htab_bail
 312
 313 htab_pte_insert_failure:
 314         /* Bail out restoring old PTE */
 315         ld      r6,STK_PARM(r6)(r1)
 316         std     r31,0(r6)
 317         li      r3,-1
 318         b       htab_bail
 319
 320
 321 #else /* CONFIG_PPC_64K_PAGES */
 322
 323
 324 /*****************************************************************************
 325  *                                                                           *
 326  *           64K SW & 4K or 64K HW in a 4K segment pages implementation      *
 327  *                                                                           *
 328  *****************************************************************************/
 329
 330 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 331  *               pte_t *ptep, unsigned long trap, int local, int ssize,
 332  *               int subpg_prot)
 333  */
 334
 335 /*
 336  * For now, we do NOT implement Admixed pages
 337  */
 338 _GLOBAL(__hash_page_4K)
 339         mflr    r0
 340         std     r0,16(r1)
 341         stdu    r1,-STACKFRAMESIZE(r1)
 342         /* Save all params that we need after a function call */
 343         std     r6,STK_PARM(r6)(r1)
 344         std     r8,STK_PARM(r8)(r1)
 345         std     r9,STK_PARM(r9)(r1)
 346
 347         /* Save non-volatile registers.
 348          * r31 will hold "old PTE"
 349          * r30 is "new PTE"
 350          * r29 is "va"
 351          * r28 is a hash value
 352          * r27 is hashtab mask (maybe dynamic patched instead ?)
 353          * r26 is the hidx mask
 354          * r25 is the index in combo page
 355          */
 356         std     r25,STK_REG(r25)(r1)
 357         std     r26,STK_REG(r26)(r1)
 358         std     r27,STK_REG(r27)(r1)
 359         std     r28,STK_REG(r28)(r1)
 360         std     r29,STK_REG(r29)(r1)
 361         std     r30,STK_REG(r30)(r1)
 362         std     r31,STK_REG(r31)(r1)
 363
 364         /* Step 1:
 365          *
 366          * Check permissions, atomically mark the linux PTE busy
 367          * and hashed.
 368          */
 369 1:
 370         ldarx   r31,0,r6
 371         /* Check access rights (access & ~(pte_val(*ptep))) */
 372         andc.   r0,r4,r31
 373         bne-    htab_wrong_access
 374         /* Check if PTE is busy */
 375         andi.   r0,r31,_PAGE_BUSY
 376         /* If so, just bail out and refault if needed. Someone else
 377          * is changing this PTE anyway and might hash it.
 378          */
 379         bne-    htab_bail_ok
 380         /* Prepare new PTE value (turn access RW into DIRTY, then
 381          * add BUSY and ACCESSED)
 382          */
 383         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 384         or      r30,r30,r31
 385         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
 386         oris    r30,r30,_PAGE_COMBO@h
 387         /* Write the linux PTE atomically (setting busy) */
 388         stdcx.  r30,0,r6
 389         bne-    1b
 390         isync
 391
 392         /* Step 2:
 393          *
 394          * Insert/Update the HPTE in the hash table. At this point,
 395          * r4 (access) is re-useable, we use it for the new HPTE flags
 396          */
 397
 398         /* Load the hidx index */
 399         rldicl  r25,r3,64-12,60
 400
 401 BEGIN_FTR_SECTION
 402         cmpdi   r9,0                    /* check segment size */
 403         bne     3f
 404 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
 405         /* Calc va and put it in r29 */
 406         rldicr  r29,r5,28,63-28         /* r29 = (vsid << 28) */
 407         rldicl  r3,r3,0,36              /* r3 = (ea & 0x0fffffff) */
 408         or      r29,r3,r29              /* r29 = va */
 409
 410         /* Calculate hash value for primary slot and store it in r28 */
 411         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 412         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
 413         xor     r28,r5,r0
 414         b       4f
 415
 416 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
 417         sldi    r29,r5,40               /* vsid << 40 */
 418         clrldi  r3,r3,24                /* ea & 0xffffffffff */
 419         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 420         clrldi  r5,r5,40                /* vsid & 0xffffff */
 421         rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
 422         xor     r28,r28,r5
 423         or      r29,r3,r29              /* VA */
 424         xor     r28,r28,r0              /* hash */
 425
 426         /* Convert linux PTE bits into HW equivalents */
 427 4:
 428 #ifdef CONFIG_PPC_SUBPAGE_PROT
 429         andc    r10,r30,r10
 430         andi.   r3,r10,0x1fe            /* Get basic set of flags */
 431         rlwinm  r0,r10,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 432 #else
 433         andi.   r3,r30,0x1fe            /* Get basic set of flags */
 434         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 435 #endif
 436         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 437         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 438         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 439         andc    r0,r3,r0                /* r0 = pte & ~r0 */
 440         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 441         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 442
 443         /* We eventually do the icache sync here (maybe inline that
 444          * code rather than call a C function...)
 445          */
 446 BEGIN_FTR_SECTION
 447         mr      r4,r30
 448         mr      r5,r7
 449         bl      .hash_page_do_lazy_icache
 450 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 451
 452         /* At this point, r3 contains new PP bits, save them in
 453          * place of "access" in the param area (sic)
 454          */
 455         std     r3,STK_PARM(r4)(r1)
 456
 457         /* Get htab_hash_mask */
 458         ld      r4,htab_hash_mask@got(2)
 459         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 460
 461         /* Check if we may already be in the hashtable, in this case, we
 462          * go to out-of-line code to try to modify the HPTE. We look for
 463          * the bit at (1 >> (index + 32))
 464          */
 465         rldicl. r0,r31,64-12,48
 466         li      r26,0                   /* Default hidx */
 467         beq     htab_insert_pte
 468
 469         /*
 470          * Check if the pte was already inserted into the hash table
 471          * as a 64k HW page, and invalidate the 64k HPTE if so.
 472          */
 473         andis.  r0,r31,_PAGE_COMBO@h
 474         beq     htab_inval_old_hpte
 475
 476         ld      r6,STK_PARM(r6)(r1)
 477         ori     r26,r6,0x8000           /* Load the hidx mask */
 478         ld      r26,0(r26)
 479         addi    r5,r25,36               /* Check actual HPTE_SUB bit, this */
 480         rldcr.  r0,r31,r5,0             /* must match pgtable.h definition */
 481         bne     htab_modify_pte
 482
 483 htab_insert_pte:
 484         /* real page number in r5, PTE RPN value + index */
 485         andis.  r0,r31,_PAGE_4K_PFN@h
 486         srdi    r5,r31,PTE_RPN_SHIFT
 487         bne-    htab_special_pfn
 488         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
 489         add     r5,r5,r25
 490 htab_special_pfn:
 491         sldi    r5,r5,HW_PAGE_SHIFT
 492
 493         /* Calculate primary group hash */
 494         and     r0,r28,r27
 495         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 496
 497         /* Call ppc_md.hpte_insert */
 498         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 499         mr      r4,r29                  /* Retreive va */
 500         li      r7,0                    /* !bolted, !secondary */
 501         li      r8,MMU_PAGE_4K          /* page size */
 502         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 503 _GLOBAL(htab_call_hpte_insert1)
 504         bl      .                       /* patched by htab_finish_init() */
 505         cmpdi   0,r3,0
 506         bge     htab_pte_insert_ok      /* Insertion successful */
 507         cmpdi   0,r3,-2                 /* Critical failure */
 508         beq-    htab_pte_insert_failure
 509
 510         /* Now try secondary slot */
 511
 512         /* real page number in r5, PTE RPN value + index */
 513         andis.  r0,r31,_PAGE_4K_PFN@h
 514         srdi    r5,r31,PTE_RPN_SHIFT
 515         bne-    3f
 516         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
 517         add     r5,r5,r25
 518 3:      sldi    r5,r5,HW_PAGE_SHIFT
 519
 520         /* Calculate secondary group hash */
 521         andc    r0,r27,r28
 522         rldicr  r3,r0,3,63-3            /* r0 = (~hash & mask) << 3 */
 523
 524         /* Call ppc_md.hpte_insert */
 525         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 526         mr      r4,r29                  /* Retreive va */
 527         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 528         li      r8,MMU_PAGE_4K          /* page size */
 529         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 530 _GLOBAL(htab_call_hpte_insert2)
 531         bl      .                       /* patched by htab_finish_init() */
 532         cmpdi   0,r3,0
 533         bge+    htab_pte_insert_ok      /* Insertion successful */
 534         cmpdi   0,r3,-2                 /* Critical failure */
 535         beq-    htab_pte_insert_failure
 536
 537         /* Both are full, we need to evict something */
 538         mftb    r0
 539         /* Pick a random group based on TB */
 540         andi.   r0,r0,1
 541         mr      r5,r28
 542         bne     2f
 543         not     r5,r5
 544 2:      and     r0,r5,r27
 545         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 546         /* Call ppc_md.hpte_remove */
 547 _GLOBAL(htab_call_hpte_remove)
 548         bl      .                       /* patched by htab_finish_init() */
 549
 550         /* Try all again */
 551         b       htab_insert_pte
 552
 553         /*
 554          * Call out to C code to invalidate an 64k HW HPTE that is
 555          * useless now that the segment has been switched to 4k pages.
 556          */
 557 htab_inval_old_hpte:
 558         mr      r3,r29                  /* virtual addr */
 559         mr      r4,r31                  /* PTE.pte */
 560         li      r5,0                    /* PTE.hidx */
 561         li      r6,MMU_PAGE_64K         /* psize */
 562         ld      r7,STK_PARM(r9)(r1)     /* ssize */
 563         ld      r8,STK_PARM(r8)(r1)     /* local */
 564         bl      .flush_hash_page
 565         /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */
 566         lis     r0,_PAGE_HPTE_SUB@h
 567         ori     r0,r0,_PAGE_HPTE_SUB@l
 568         andc    r30,r30,r0
 569         b       htab_insert_pte
 570
 571 htab_bail_ok:
 572         li      r3,0
 573         b       htab_bail
 574
 575 htab_pte_insert_ok:
 576         /* Insert slot number & secondary bit in PTE second half,
 577          * clear _PAGE_BUSY and set approriate HPTE slot bit
 578          */
 579         ld      r6,STK_PARM(r6)(r1)
 580         li      r0,_PAGE_BUSY
 581         andc    r30,r30,r0
 582         /* HPTE SUB bit */
 583         li      r0,1
 584         subfic  r5,r25,27               /* Must match bit position in */
 585         sld     r0,r0,r5                /* pgtable.h */
 586         or      r30,r30,r0
 587         /* hindx */
 588         sldi    r5,r25,2
 589         sld     r3,r3,r5
 590         li      r4,0xf
 591         sld     r4,r4,r5
 592         andc    r26,r26,r4
 593         or      r26,r26,r3
 594         ori     r5,r6,0x8000
 595         std     r26,0(r5)
 596         lwsync
 597         std     r30,0(r6)
 598         li      r3, 0
 599 htab_bail:
 600         ld      r25,STK_REG(r25)(r1)
 601         ld      r26,STK_REG(r26)(r1)
 602         ld      r27,STK_REG(r27)(r1)
 603         ld      r28,STK_REG(r28)(r1)
 604         ld      r29,STK_REG(r29)(r1)
 605         ld      r30,STK_REG(r30)(r1)
 606         ld      r31,STK_REG(r31)(r1)
 607         addi    r1,r1,STACKFRAMESIZE
 608         ld      r0,16(r1)
 609         mtlr    r0
 610         blr
 611
 612 htab_modify_pte:
 613         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 614         mr      r4,r3
 615         sldi    r5,r25,2
 616         srd     r3,r26,r5
 617
 618         /* Secondary group ? if yes, get a inverted hash value */
 619         mr      r5,r28
 620         andi.   r0,r3,0x8 /* page secondary ? */
 621         beq     1f
 622         not     r5,r5
 623 1:      andi.   r3,r3,0x7 /* extract idx alone */
 624
 625         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 626         and     r0,r5,r27
 627         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 628         add     r3,r0,r3        /* add slot idx */
 629
 630         /* Call ppc_md.hpte_updatepp */
 631         mr      r5,r29                  /* va */
 632         li      r6,MMU_PAGE_4K          /* page size */
 633         ld      r7,STK_PARM(r9)(r1)     /* segment size */
 634         ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
 635 _GLOBAL(htab_call_hpte_updatepp)
 636         bl      .                       /* patched by htab_finish_init() */
 637
 638         /* if we failed because typically the HPTE wasn't really here
 639          * we try an insertion.
 640          */
 641         cmpdi   0,r3,-1
 642         beq-    htab_insert_pte
 643
 644         /* Clear the BUSY bit and Write out the PTE */
 645         li      r0,_PAGE_BUSY
 646         andc    r30,r30,r0
 647         ld      r6,STK_PARM(r6)(r1)
 648         std     r30,0(r6)
 649         li      r3,0
 650         b       htab_bail
 651
 652 htab_wrong_access:
 653         /* Bail out clearing reservation */
 654         stdcx.  r31,0,r6
 655         li      r3,1
 656         b       htab_bail
 657
 658 htab_pte_insert_failure:
 659         /* Bail out restoring old PTE */
 660         ld      r6,STK_PARM(r6)(r1)
 661         std     r31,0(r6)
 662         li      r3,-1
 663         b       htab_bail
 664
 665 #endif /* CONFIG_PPC_64K_PAGES */
 666
 667 #ifdef CONFIG_PPC_HAS_HASH_64K
 668
 669 /*****************************************************************************
 670  *                                                                           *
 671  *           64K SW & 64K HW in a 64K segment pages implementation           *
 672  *                                                                           *
 673  *****************************************************************************/
 674
 675 _GLOBAL(__hash_page_64K)
 676         mflr    r0
 677         std     r0,16(r1)
 678         stdu    r1,-STACKFRAMESIZE(r1)
 679         /* Save all params that we need after a function call */
 680         std     r6,STK_PARM(r6)(r1)
 681         std     r8,STK_PARM(r8)(r1)
 682         std     r9,STK_PARM(r9)(r1)
 683
 684         /* Save non-volatile registers.
 685          * r31 will hold "old PTE"
 686          * r30 is "new PTE"
 687          * r29 is "va"
 688          * r28 is a hash value
 689          * r27 is hashtab mask (maybe dynamic patched instead ?)
 690          */
 691         std     r27,STK_REG(r27)(r1)
 692         std     r28,STK_REG(r28)(r1)
 693         std     r29,STK_REG(r29)(r1)
 694         std     r30,STK_REG(r30)(r1)
 695         std     r31,STK_REG(r31)(r1)
 696
 697         /* Step 1:
 698          *
 699          * Check permissions, atomically mark the linux PTE busy
 700          * and hashed.
 701          */
 702 1:
 703         ldarx   r31,0,r6
 704         /* Check access rights (access & ~(pte_val(*ptep))) */
 705         andc.   r0,r4,r31
 706         bne-    ht64_wrong_access
 707         /* Check if PTE is busy */
 708         andi.   r0,r31,_PAGE_BUSY
 709         /* If so, just bail out and refault if needed. Someone else
 710          * is changing this PTE anyway and might hash it.
 711          */
 712         bne-    ht64_bail_ok
 713 BEGIN_FTR_SECTION
 714         /* Check if PTE has the cache-inhibit bit set */
 715         andi.   r0,r31,_PAGE_NO_CACHE
 716         /* If so, bail out and refault as a 4k page */
 717         bne-    ht64_bail_ok
 718 END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
 719         /* Prepare new PTE value (turn access RW into DIRTY, then
 720          * add BUSY and ACCESSED)
 721          */
 722         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 723         or      r30,r30,r31
 724         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
 725         /* Write the linux PTE atomically (setting busy) */
 726         stdcx.  r30,0,r6
 727         bne-    1b
 728         isync
 729
 730         /* Step 2:
 731          *
 732          * Insert/Update the HPTE in the hash table. At this point,
 733          * r4 (access) is re-useable, we use it for the new HPTE flags
 734          */
 735
 736 BEGIN_FTR_SECTION
 737         cmpdi   r9,0                    /* check segment size */
 738         bne     3f
 739 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
 740         /* Calc va and put it in r29 */
 741         rldicr  r29,r5,28,63-28
 742         rldicl  r3,r3,0,36
 743         or      r29,r3,r29
 744
 745         /* Calculate hash value for primary slot and store it in r28 */
 746         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 747         rldicl  r0,r3,64-16,52          /* (ea >> 16) & 0xfff */
 748         xor     r28,r5,r0
 749         b       4f
 750
 751 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
 752         sldi    r29,r5,40               /* vsid << 40 */
 753         clrldi  r3,r3,24                /* ea & 0xffffffffff */
 754         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 755         clrldi  r5,r5,40                /* vsid & 0xffffff */
 756         rldicl  r0,r3,64-16,40          /* (ea >> 16) & 0xffffff */
 757         xor     r28,r28,r5
 758         or      r29,r3,r29              /* VA */
 759         xor     r28,r28,r0              /* hash */
 760
 761         /* Convert linux PTE bits into HW equivalents */
 762 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 763         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 764         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 765         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 766         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 767         andc    r0,r30,r0               /* r0 = pte & ~r0 */
 768         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 769         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 770
 771         /* We eventually do the icache sync here (maybe inline that
 772          * code rather than call a C function...)
 773          */
 774 BEGIN_FTR_SECTION
 775         mr      r4,r30
 776         mr      r5,r7
 777         bl      .hash_page_do_lazy_icache
 778 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 779
 780         /* At this point, r3 contains new PP bits, save them in
 781          * place of "access" in the param area (sic)
 782          */
 783         std     r3,STK_PARM(r4)(r1)
 784
 785         /* Get htab_hash_mask */
 786         ld      r4,htab_hash_mask@got(2)
 787         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 788
 789         /* Check if we may already be in the hashtable, in this case, we
 790          * go to out-of-line code to try to modify the HPTE
 791          */
 792         rldicl. r0,r31,64-12,48
 793         bne     ht64_modify_pte
 794
 795 ht64_insert_pte:
 796         /* Clear hpte bits in new pte (we also clear BUSY btw) and
 797          * add _PAGE_HPTE_SUB0
 798          */
 799         lis     r0,_PAGE_HPTEFLAGS@h
 800         ori     r0,r0,_PAGE_HPTEFLAGS@l
 801         andc    r30,r30,r0
 802 #ifdef CONFIG_PPC_64K_PAGES
 803         oris    r30,r30,_PAGE_HPTE_SUB0@h
 804 #else
 805         ori     r30,r30,_PAGE_HASHPTE
 806 #endif
 807         /* Phyical address in r5 */
 808         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 809         sldi    r5,r5,PAGE_SHIFT
 810
 811         /* Calculate primary group hash */
 812         and     r0,r28,r27
 813         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 814
 815         /* Call ppc_md.hpte_insert */
 816         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 817         mr      r4,r29                  /* Retreive va */
 818         li      r7,0                    /* !bolted, !secondary */
 819         li      r8,MMU_PAGE_64K
 820         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 821 _GLOBAL(ht64_call_hpte_insert1)
 822         bl      .                       /* patched by htab_finish_init() */
 823         cmpdi   0,r3,0
 824         bge     ht64_pte_insert_ok      /* Insertion successful */
 825         cmpdi   0,r3,-2                 /* Critical failure */
 826         beq-    ht64_pte_insert_failure
 827
 828         /* Now try secondary slot */
 829
 830         /* Phyical address in r5 */
 831         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 832         sldi    r5,r5,PAGE_SHIFT
 833
 834         /* Calculate secondary group hash */
 835         andc    r0,r27,r28
 836         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 837
 838         /* Call ppc_md.hpte_insert */
 839         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 840         mr      r4,r29                  /* Retreive va */
 841         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 842         li      r8,MMU_PAGE_64K
 843         ld      r9,STK_PARM(r9)(r1)     /* segment size */
 844 _GLOBAL(ht64_call_hpte_insert2)
 845         bl      .                       /* patched by htab_finish_init() */
 846         cmpdi   0,r3,0
 847         bge+    ht64_pte_insert_ok      /* Insertion successful */
 848         cmpdi   0,r3,-2                 /* Critical failure */
 849         beq-    ht64_pte_insert_failure
 850
 851         /* Both are full, we need to evict something */
 852         mftb    r0
 853         /* Pick a random group based on TB */
 854         andi.   r0,r0,1
 855         mr      r5,r28
 856         bne     2f
 857         not     r5,r5
 858 2:      and     r0,r5,r27
 859         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 860         /* Call ppc_md.hpte_remove */
 861 _GLOBAL(ht64_call_hpte_remove)
 862         bl      .                       /* patched by htab_finish_init() */
 863
 864         /* Try all again */
 865         b       ht64_insert_pte
 866
 867 ht64_bail_ok:
 868         li      r3,0
 869         b       ht64_bail
 870
 871 ht64_pte_insert_ok:
 872         /* Insert slot number & secondary bit in PTE */
 873         rldimi  r30,r3,12,63-15
 874
 875         /* Write out the PTE with a normal write
 876          * (maybe add eieio may be good still ?)
 877          */
 878 ht64_write_out_pte:
 879         ld      r6,STK_PARM(r6)(r1)
 880         std     r30,0(r6)
 881         li      r3, 0
 882 ht64_bail:
 883         ld      r27,STK_REG(r27)(r1)
 884         ld      r28,STK_REG(r28)(r1)
 885         ld      r29,STK_REG(r29)(r1)
 886         ld      r30,STK_REG(r30)(r1)
 887         ld      r31,STK_REG(r31)(r1)
 888         addi    r1,r1,STACKFRAMESIZE
 889         ld      r0,16(r1)
 890         mtlr    r0
 891         blr
 892
 893 ht64_modify_pte:
 894         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 895         mr      r4,r3
 896         rlwinm  r3,r31,32-12,29,31
 897
 898         /* Secondary group ? if yes, get a inverted hash value */
 899         mr      r5,r28
 900         andi.   r0,r31,_PAGE_F_SECOND
 901         beq     1f
 902         not     r5,r5
 903 1:
 904         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 905         and     r0,r5,r27
 906         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 907         add     r3,r0,r3        /* add slot idx */
 908
 909         /* Call ppc_md.hpte_updatepp */
 910         mr      r5,r29                  /* va */
 911         li      r6,MMU_PAGE_64K
 912         ld      r7,STK_PARM(r9)(r1)     /* segment size */
 913         ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
 914 _GLOBAL(ht64_call_hpte_updatepp)
 915         bl      .                       /* patched by htab_finish_init() */
 916
 917         /* if we failed because typically the HPTE wasn't really here
 918          * we try an insertion.
 919          */
 920         cmpdi   0,r3,-1
 921         beq-    ht64_insert_pte
 922
 923         /* Clear the BUSY bit and Write out the PTE */
 924         li      r0,_PAGE_BUSY
 925         andc    r30,r30,r0
 926         b       ht64_write_out_pte
 927
 928 ht64_wrong_access:
 929         /* Bail out clearing reservation */
 930         stdcx.  r31,0,r6
 931         li      r3,1
 932         b       ht64_bail
 933
 934 ht64_pte_insert_failure:
 935         /* Bail out restoring old PTE */
 936         ld      r6,STK_PARM(r6)(r1)
 937         std     r31,0(r6)
 938         li      r3,-1
 939         b       ht64_bail
 940
 941
 942 #endif /* CONFIG_PPC_HAS_HASH_64K */
 943
 944
 945 /*****************************************************************************
 946  *                                                                           *
 947  *           Huge pages implementation is in hugetlbpage.c                   *
 948  *                                                                           *
 949  *****************************************************************************/