mm/huge_memory.c

   1 /*
   2  *  Copyright (C) 2009  Red Hat, Inc.
   3  *
   4  *  This work is licensed under the terms of the GNU GPL, version 2. See
   5  *  the COPYING file in the top-level directory.
   6  */
   7
   8 #include <linux/mm.h>
   9 #include <linux/sched.h>
  10 #include <linux/highmem.h>
  11 #include <linux/hugetlb.h>
  12 #include <linux/mmu_notifier.h>
  13 #include <linux/rmap.h>
  14 #include <linux/swap.h>
  15 #include <asm/tlb.h>
  16 #include <asm/pgalloc.h>
  17 #include "internal.h"
  18
  19 unsigned long transparent_hugepage_flags __read_mostly =
  20         (1<<TRANSPARENT_HUGEPAGE_FLAG);
  21
  22 #ifdef CONFIG_SYSFS
  23 static ssize_t double_flag_show(struct kobject *kobj,
  24                                 struct kobj_attribute *attr, char *buf,
  25                                 enum transparent_hugepage_flag enabled,
  26                                 enum transparent_hugepage_flag req_madv)
  27 {
  28         if (test_bit(enabled, &transparent_hugepage_flags)) {
  29                 VM_BUG_ON(test_bit(req_madv, &transparent_hugepage_flags));
  30                 return sprintf(buf, "[always] madvise never\n");
  31         } else if (test_bit(req_madv, &transparent_hugepage_flags))
  32                 return sprintf(buf, "always [madvise] never\n");
  33         else
  34                 return sprintf(buf, "always madvise [never]\n");
  35 }
  36 static ssize_t double_flag_store(struct kobject *kobj,
  37                                  struct kobj_attribute *attr,
  38                                  const char *buf, size_t count,
  39                                  enum transparent_hugepage_flag enabled,
  40                                  enum transparent_hugepage_flag req_madv)
  41 {
  42         if (!memcmp("always", buf,
  43                     min(sizeof("always")-1, count))) {
  44                 set_bit(enabled, &transparent_hugepage_flags);
  45                 clear_bit(req_madv, &transparent_hugepage_flags);
  46         } else if (!memcmp("madvise", buf,
  47                            min(sizeof("madvise")-1, count))) {
  48                 clear_bit(enabled, &transparent_hugepage_flags);
  49                 set_bit(req_madv, &transparent_hugepage_flags);
  50         } else if (!memcmp("never", buf,
  51                            min(sizeof("never")-1, count))) {
  52                 clear_bit(enabled, &transparent_hugepage_flags);
  53                 clear_bit(req_madv, &transparent_hugepage_flags);
  54         } else
  55                 return -EINVAL;
  56
  57         return count;
  58 }
  59
  60 static ssize_t enabled_show(struct kobject *kobj,
  61                             struct kobj_attribute *attr, char *buf)
  62 {
  63         return double_flag_show(kobj, attr, buf,
  64                                 TRANSPARENT_HUGEPAGE_FLAG,
  65                                 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
  66 }
  67 static ssize_t enabled_store(struct kobject *kobj,
  68                              struct kobj_attribute *attr,
  69                              const char *buf, size_t count)
  70 {
  71         return double_flag_store(kobj, attr, buf, count,
  72                                  TRANSPARENT_HUGEPAGE_FLAG,
  73                                  TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
  74 }
  75 static struct kobj_attribute enabled_attr =
  76         __ATTR(enabled, 0644, enabled_show, enabled_store);
  77
  78 static ssize_t single_flag_show(struct kobject *kobj,
  79                                 struct kobj_attribute *attr, char *buf,
  80                                 enum transparent_hugepage_flag flag)
  81 {
  82         if (test_bit(flag, &transparent_hugepage_flags))
  83                 return sprintf(buf, "[yes] no\n");
  84         else
  85                 return sprintf(buf, "yes [no]\n");
  86 }
  87 static ssize_t single_flag_store(struct kobject *kobj,
  88                                  struct kobj_attribute *attr,
  89                                  const char *buf, size_t count,
  90                                  enum transparent_hugepage_flag flag)
  91 {
  92         if (!memcmp("yes", buf,
  93                     min(sizeof("yes")-1, count))) {
  94                 set_bit(flag, &transparent_hugepage_flags);
  95         } else if (!memcmp("no", buf,
  96                            min(sizeof("no")-1, count))) {
  97                 clear_bit(flag, &transparent_hugepage_flags);
  98         } else
  99                 return -EINVAL;
 100
 101         return count;
 102 }
 103
 104 /*
 105  * Currently defrag only disables __GFP_NOWAIT for allocation. A blind
 106  * __GFP_REPEAT is too aggressive, it's never worth swapping tons of
 107  * memory just to allocate one more hugepage.
 108  */
 109 static ssize_t defrag_show(struct kobject *kobj,
 110                            struct kobj_attribute *attr, char *buf)
 111 {
 112         return double_flag_show(kobj, attr, buf,
 113                                 TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
 114                                 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
 115 }
 116 static ssize_t defrag_store(struct kobject *kobj,
 117                             struct kobj_attribute *attr,
 118                             const char *buf, size_t count)
 119 {
 120         return double_flag_store(kobj, attr, buf, count,
 121                                  TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
 122                                  TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
 123 }
 124 static struct kobj_attribute defrag_attr =
 125         __ATTR(defrag, 0644, defrag_show, defrag_store);
 126
 127 #ifdef CONFIG_DEBUG_VM
 128 static ssize_t debug_cow_show(struct kobject *kobj,
 129                                 struct kobj_attribute *attr, char *buf)
 130 {
 131         return single_flag_show(kobj, attr, buf,
 132                                 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
 133 }
 134 static ssize_t debug_cow_store(struct kobject *kobj,
 135                                struct kobj_attribute *attr,
 136                                const char *buf, size_t count)
 137 {
 138         return single_flag_store(kobj, attr, buf, count,
 139                                  TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
 140 }
 141 static struct kobj_attribute debug_cow_attr =
 142         __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store);
 143 #endif /* CONFIG_DEBUG_VM */
 144
 145 static struct attribute *hugepage_attr[] = {
 146         &enabled_attr.attr,
 147         &defrag_attr.attr,
 148 #ifdef CONFIG_DEBUG_VM
 149         &debug_cow_attr.attr,
 150 #endif
 151         NULL,
 152 };
 153
 154 static struct attribute_group hugepage_attr_group = {
 155         .attrs = hugepage_attr,
 156         .name = "transparent_hugepage",
 157 };
 158 #endif /* CONFIG_SYSFS */
 159
 160 static int __init hugepage_init(void)
 161 {
 162 #ifdef CONFIG_SYSFS
 163         int err;
 164
 165         err = sysfs_create_group(mm_kobj, &hugepage_attr_group);
 166         if (err)
 167                 printk(KERN_ERR "hugepage: register sysfs failed\n");
 168 #endif
 169         return 0;
 170 }
 171 module_init(hugepage_init)
 172
 173 static int __init setup_transparent_hugepage(char *str)
 174 {
 175         int ret = 0;
 176         if (!str)
 177                 goto out;
 178         if (!strcmp(str, "always")) {
 179                 set_bit(TRANSPARENT_HUGEPAGE_FLAG,
 180                         &transparent_hugepage_flags);
 181                 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
 182                           &transparent_hugepage_flags);
 183                 ret = 1;
 184         } else if (!strcmp(str, "madvise")) {
 185                 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
 186                           &transparent_hugepage_flags);
 187                 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
 188                         &transparent_hugepage_flags);
 189                 ret = 1;
 190         } else if (!strcmp(str, "never")) {
 191                 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
 192                           &transparent_hugepage_flags);
 193                 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
 194                           &transparent_hugepage_flags);
 195                 ret = 1;
 196         }
 197 out:
 198         if (!ret)
 199                 printk(KERN_WARNING
 200                        "transparent_hugepage= cannot parse, ignored\n");
 201         return ret;
 202 }
 203 __setup("transparent_hugepage=", setup_transparent_hugepage);
 204
 205 static void prepare_pmd_huge_pte(pgtable_t pgtable,
 206                                  struct mm_struct *mm)
 207 {
 208         assert_spin_locked(&mm->page_table_lock);
 209
 210         /* FIFO */
 211         if (!mm->pmd_huge_pte)
 212                 INIT_LIST_HEAD(&pgtable->lru);
 213         else
 214                 list_add(&pgtable->lru, &mm->pmd_huge_pte->lru);
 215         mm->pmd_huge_pte = pgtable;
 216 }
 217
 218 static inline pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
 219 {
 220         if (likely(vma->vm_flags & VM_WRITE))
 221                 pmd = pmd_mkwrite(pmd);
 222         return pmd;
 223 }
 224
 225 static int __do_huge_pmd_anonymous_page(struct mm_struct *mm,
 226                                         struct vm_area_struct *vma,
 227                                         unsigned long haddr, pmd_t *pmd,
 228                                         struct page *page)
 229 {
 230         int ret = 0;
 231         pgtable_t pgtable;
 232
 233         VM_BUG_ON(!PageCompound(page));
 234         pgtable = pte_alloc_one(mm, haddr);
 235         if (unlikely(!pgtable)) {
 236                 put_page(page);
 237                 return VM_FAULT_OOM;
 238         }
 239
 240         clear_huge_page(page, haddr, HPAGE_PMD_NR);
 241         __SetPageUptodate(page);
 242
 243         spin_lock(&mm->page_table_lock);
 244         if (unlikely(!pmd_none(*pmd))) {
 245                 spin_unlock(&mm->page_table_lock);
 246                 put_page(page);
 247                 pte_free(mm, pgtable);
 248         } else {
 249                 pmd_t entry;
 250                 entry = mk_pmd(page, vma->vm_page_prot);
 251                 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
 252                 entry = pmd_mkhuge(entry);
 253                 /*
 254                  * The spinlocking to take the lru_lock inside
 255                  * page_add_new_anon_rmap() acts as a full memory
 256                  * barrier to be sure clear_huge_page writes become
 257                  * visible after the set_pmd_at() write.
 258                  */
 259                 page_add_new_anon_rmap(page, vma, haddr);
 260                 set_pmd_at(mm, haddr, pmd, entry);
 261                 prepare_pmd_huge_pte(pgtable, mm);
 262                 add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR);
 263                 spin_unlock(&mm->page_table_lock);
 264         }
 265
 266         return ret;
 267 }
 268
 269 static inline struct page *alloc_hugepage(int defrag)
 270 {
 271         return alloc_pages(GFP_TRANSHUGE & ~(defrag ? 0 : __GFP_WAIT),
 272                            HPAGE_PMD_ORDER);
 273 }
 274
 275 int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
 276                                unsigned long address, pmd_t *pmd,
 277                                unsigned int flags)
 278 {
 279         struct page *page;
 280         unsigned long haddr = address & HPAGE_PMD_MASK;
 281         pte_t *pte;
 282
 283         if (haddr >= vma->vm_start && haddr + HPAGE_PMD_SIZE <= vma->vm_end) {
 284                 if (unlikely(anon_vma_prepare(vma)))
 285                         return VM_FAULT_OOM;
 286                 page = alloc_hugepage(transparent_hugepage_defrag(vma));
 287                 if (unlikely(!page))
 288                         goto out;
 289
 290                 return __do_huge_pmd_anonymous_page(mm, vma, haddr, pmd, page);
 291         }
 292 out:
 293         /*
 294          * Use __pte_alloc instead of pte_alloc_map, because we can't
 295          * run pte_offset_map on the pmd, if an huge pmd could
 296          * materialize from under us from a different thread.
 297          */
 298         if (unlikely(__pte_alloc(mm, vma, pmd, address)))
 299                 return VM_FAULT_OOM;
 300         /* if an huge pmd materialized from under us just retry later */
 301         if (unlikely(pmd_trans_huge(*pmd)))
 302                 return 0;
 303         /*
 304          * A regular pmd is established and it can't morph into a huge pmd
 305          * from under us anymore at this point because we hold the mmap_sem
 306          * read mode and khugepaged takes it in write mode. So now it's
 307          * safe to run pte_offset_map().
 308          */
 309         pte = pte_offset_map(pmd, address);
 310         return handle_pte_fault(mm, vma, address, pte, pmd, flags);
 311 }
 312
 313 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 314                   pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
 315                   struct vm_area_struct *vma)
 316 {
 317         struct page *src_page;
 318         pmd_t pmd;
 319         pgtable_t pgtable;
 320         int ret;
 321
 322         ret = -ENOMEM;
 323         pgtable = pte_alloc_one(dst_mm, addr);
 324         if (unlikely(!pgtable))
 325                 goto out;
 326
 327         spin_lock(&dst_mm->page_table_lock);
 328         spin_lock_nested(&src_mm->page_table_lock, SINGLE_DEPTH_NESTING);
 329
 330         ret = -EAGAIN;
 331         pmd = *src_pmd;
 332         if (unlikely(!pmd_trans_huge(pmd))) {
 333                 pte_free(dst_mm, pgtable);
 334                 goto out_unlock;
 335         }
 336         if (unlikely(pmd_trans_splitting(pmd))) {
 337                 /* split huge page running from under us */
 338                 spin_unlock(&src_mm->page_table_lock);
 339                 spin_unlock(&dst_mm->page_table_lock);
 340                 pte_free(dst_mm, pgtable);
 341
 342                 wait_split_huge_page(vma->anon_vma, src_pmd); /* src_vma */
 343                 goto out;
 344         }
 345         src_page = pmd_page(pmd);
 346         VM_BUG_ON(!PageHead(src_page));
 347         get_page(src_page);
 348         page_dup_rmap(src_page);
 349         add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
 350
 351         pmdp_set_wrprotect(src_mm, addr, src_pmd);
 352         pmd = pmd_mkold(pmd_wrprotect(pmd));
 353         set_pmd_at(dst_mm, addr, dst_pmd, pmd);
 354         prepare_pmd_huge_pte(pgtable, dst_mm);
 355
 356         ret = 0;
 357 out_unlock:
 358         spin_unlock(&src_mm->page_table_lock);
 359         spin_unlock(&dst_mm->page_table_lock);
 360 out:
 361         return ret;
 362 }
 363
 364 /* no "address" argument so destroys page coloring of some arch */
 365 pgtable_t get_pmd_huge_pte(struct mm_struct *mm)
 366 {
 367         pgtable_t pgtable;
 368
 369         assert_spin_locked(&mm->page_table_lock);
 370
 371         /* FIFO */
 372         pgtable = mm->pmd_huge_pte;
 373         if (list_empty(&pgtable->lru))
 374                 mm->pmd_huge_pte = NULL;
 375         else {
 376                 mm->pmd_huge_pte = list_entry(pgtable->lru.next,
 377                                               struct page, lru);
 378                 list_del(&pgtable->lru);
 379         }
 380         return pgtable;
 381 }
 382
 383 static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
 384                                         struct vm_area_struct *vma,
 385                                         unsigned long address,
 386                                         pmd_t *pmd, pmd_t orig_pmd,
 387                                         struct page *page,
 388                                         unsigned long haddr)
 389 {
 390         pgtable_t pgtable;
 391         pmd_t _pmd;
 392         int ret = 0, i;
 393         struct page **pages;
 394
 395         pages = kmalloc(sizeof(struct page *) * HPAGE_PMD_NR,
 396                         GFP_KERNEL);
 397         if (unlikely(!pages)) {
 398                 ret |= VM_FAULT_OOM;
 399                 goto out;
 400         }
 401
 402         for (i = 0; i < HPAGE_PMD_NR; i++) {
 403                 pages[i] = alloc_page_vma(GFP_HIGHUSER_MOVABLE,
 404                                           vma, address);
 405                 if (unlikely(!pages[i])) {
 406                         while (--i >= 0)
 407                                 put_page(pages[i]);
 408                         kfree(pages);
 409                         ret |= VM_FAULT_OOM;
 410                         goto out;
 411                 }
 412         }
 413
 414         for (i = 0; i < HPAGE_PMD_NR; i++) {
 415                 copy_user_highpage(pages[i], page + i,
 416                                    haddr + PAGE_SHIFT*i, vma);
 417                 __SetPageUptodate(pages[i]);
 418                 cond_resched();
 419         }
 420
 421         spin_lock(&mm->page_table_lock);
 422         if (unlikely(!pmd_same(*pmd, orig_pmd)))
 423                 goto out_free_pages;
 424         VM_BUG_ON(!PageHead(page));
 425
 426         pmdp_clear_flush_notify(vma, haddr, pmd);
 427         /* leave pmd empty until pte is filled */
 428
 429         pgtable = get_pmd_huge_pte(mm);
 430         pmd_populate(mm, &_pmd, pgtable);
 431
 432         for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
 433                 pte_t *pte, entry;
 434                 entry = mk_pte(pages[i], vma->vm_page_prot);
 435                 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
 436                 page_add_new_anon_rmap(pages[i], vma, haddr);
 437                 pte = pte_offset_map(&_pmd, haddr);
 438                 VM_BUG_ON(!pte_none(*pte));
 439                 set_pte_at(mm, haddr, pte, entry);
 440                 pte_unmap(pte);
 441         }
 442         kfree(pages);
 443
 444         mm->nr_ptes++;
 445         smp_wmb(); /* make pte visible before pmd */
 446         pmd_populate(mm, pmd, pgtable);
 447         page_remove_rmap(page);
 448         spin_unlock(&mm->page_table_lock);
 449
 450         ret |= VM_FAULT_WRITE;
 451         put_page(page);
 452
 453 out:
 454         return ret;
 455
 456 out_free_pages:
 457         spin_unlock(&mm->page_table_lock);
 458         for (i = 0; i < HPAGE_PMD_NR; i++)
 459                 put_page(pages[i]);
 460         kfree(pages);
 461         goto out;
 462 }
 463
 464 int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
 465                         unsigned long address, pmd_t *pmd, pmd_t orig_pmd)
 466 {
 467         int ret = 0;
 468         struct page *page, *new_page;
 469         unsigned long haddr;
 470
 471         VM_BUG_ON(!vma->anon_vma);
 472         spin_lock(&mm->page_table_lock);
 473         if (unlikely(!pmd_same(*pmd, orig_pmd)))
 474                 goto out_unlock;
 475
 476         page = pmd_page(orig_pmd);
 477         VM_BUG_ON(!PageCompound(page) || !PageHead(page));
 478         haddr = address & HPAGE_PMD_MASK;
 479         if (page_mapcount(page) == 1) {
 480                 pmd_t entry;
 481                 entry = pmd_mkyoung(orig_pmd);
 482                 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
 483                 if (pmdp_set_access_flags(vma, haddr, pmd, entry,  1))
 484                         update_mmu_cache(vma, address, entry);
 485                 ret |= VM_FAULT_WRITE;
 486                 goto out_unlock;
 487         }
 488         get_page(page);
 489         spin_unlock(&mm->page_table_lock);
 490
 491         if (transparent_hugepage_enabled(vma) &&
 492             !transparent_hugepage_debug_cow())
 493                 new_page = alloc_hugepage(transparent_hugepage_defrag(vma));
 494         else
 495                 new_page = NULL;
 496
 497         if (unlikely(!new_page)) {
 498                 ret = do_huge_pmd_wp_page_fallback(mm, vma, address,
 499                                                    pmd, orig_pmd, page, haddr);
 500                 put_page(page);
 501                 goto out;
 502         }
 503
 504         copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR);
 505         __SetPageUptodate(new_page);
 506
 507         spin_lock(&mm->page_table_lock);
 508         put_page(page);
 509         if (unlikely(!pmd_same(*pmd, orig_pmd)))
 510                 put_page(new_page);
 511         else {
 512                 pmd_t entry;
 513                 VM_BUG_ON(!PageHead(page));
 514                 entry = mk_pmd(new_page, vma->vm_page_prot);
 515                 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
 516                 entry = pmd_mkhuge(entry);
 517                 pmdp_clear_flush_notify(vma, haddr, pmd);
 518                 page_add_new_anon_rmap(new_page, vma, haddr);
 519                 set_pmd_at(mm, haddr, pmd, entry);
 520                 update_mmu_cache(vma, address, entry);
 521                 page_remove_rmap(page);
 522                 put_page(page);
 523                 ret |= VM_FAULT_WRITE;
 524         }
 525 out_unlock:
 526         spin_unlock(&mm->page_table_lock);
 527 out:
 528         return ret;
 529 }
 530
 531 struct page *follow_trans_huge_pmd(struct mm_struct *mm,
 532                                    unsigned long addr,
 533                                    pmd_t *pmd,
 534                                    unsigned int flags)
 535 {
 536         struct page *page = NULL;
 537
 538         assert_spin_locked(&mm->page_table_lock);
 539
 540         if (flags & FOLL_WRITE && !pmd_write(*pmd))
 541                 goto out;
 542
 543         page = pmd_page(*pmd);
 544         VM_BUG_ON(!PageHead(page));
 545         if (flags & FOLL_TOUCH) {
 546                 pmd_t _pmd;
 547                 /*
 548                  * We should set the dirty bit only for FOLL_WRITE but
 549                  * for now the dirty bit in the pmd is meaningless.
 550                  * And if the dirty bit will become meaningful and
 551                  * we'll only set it with FOLL_WRITE, an atomic
 552                  * set_bit will be required on the pmd to set the
 553                  * young bit, instead of the current set_pmd_at.
 554                  */
 555                 _pmd = pmd_mkyoung(pmd_mkdirty(*pmd));
 556                 set_pmd_at(mm, addr & HPAGE_PMD_MASK, pmd, _pmd);
 557         }
 558         page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
 559         VM_BUG_ON(!PageCompound(page));
 560         if (flags & FOLL_GET)
 561                 get_page(page);
 562
 563 out:
 564         return page;
 565 }
 566
 567 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 568                  pmd_t *pmd)
 569 {
 570         int ret = 0;
 571
 572         spin_lock(&tlb->mm->page_table_lock);
 573         if (likely(pmd_trans_huge(*pmd))) {
 574                 if (unlikely(pmd_trans_splitting(*pmd))) {
 575                         spin_unlock(&tlb->mm->page_table_lock);
 576                         wait_split_huge_page(vma->anon_vma,
 577                                              pmd);
 578                 } else {
 579                         struct page *page;
 580                         pgtable_t pgtable;
 581                         pgtable = get_pmd_huge_pte(tlb->mm);
 582                         page = pmd_page(*pmd);
 583                         pmd_clear(pmd);
 584                         page_remove_rmap(page);
 585                         VM_BUG_ON(page_mapcount(page) < 0);
 586                         add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
 587                         VM_BUG_ON(!PageHead(page));
 588                         spin_unlock(&tlb->mm->page_table_lock);
 589                         tlb_remove_page(tlb, page);
 590                         pte_free(tlb->mm, pgtable);
 591                         ret = 1;
 592                 }
 593         } else
 594                 spin_unlock(&tlb->mm->page_table_lock);
 595
 596         return ret;
 597 }
 598
 599 pmd_t *page_check_address_pmd(struct page *page,
 600                               struct mm_struct *mm,
 601                               unsigned long address,
 602                               enum page_check_address_pmd_flag flag)
 603 {
 604         pgd_t *pgd;
 605         pud_t *pud;
 606         pmd_t *pmd, *ret = NULL;
 607
 608         if (address & ~HPAGE_PMD_MASK)
 609                 goto out;
 610
 611         pgd = pgd_offset(mm, address);
 612         if (!pgd_present(*pgd))
 613                 goto out;
 614
 615         pud = pud_offset(pgd, address);
 616         if (!pud_present(*pud))
 617                 goto out;
 618
 619         pmd = pmd_offset(pud, address);
 620         if (pmd_none(*pmd))
 621                 goto out;
 622         if (pmd_page(*pmd) != page)
 623                 goto out;
 624         VM_BUG_ON(flag == PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG &&
 625                   pmd_trans_splitting(*pmd));
 626         if (pmd_trans_huge(*pmd)) {
 627                 VM_BUG_ON(flag == PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG &&
 628                           !pmd_trans_splitting(*pmd));
 629                 ret = pmd;
 630         }
 631 out:
 632         return ret;
 633 }
 634
 635 static int __split_huge_page_splitting(struct page *page,
 636                                        struct vm_area_struct *vma,
 637                                        unsigned long address)
 638 {
 639         struct mm_struct *mm = vma->vm_mm;
 640         pmd_t *pmd;
 641         int ret = 0;
 642
 643         spin_lock(&mm->page_table_lock);
 644         pmd = page_check_address_pmd(page, mm, address,
 645                                      PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG);
 646         if (pmd) {
 647                 /*
 648                  * We can't temporarily set the pmd to null in order
 649                  * to split it, the pmd must remain marked huge at all
 650                  * times or the VM won't take the pmd_trans_huge paths
 651                  * and it won't wait on the anon_vma->root->lock to
 652                  * serialize against split_huge_page*.
 653                  */
 654                 pmdp_splitting_flush_notify(vma, address, pmd);
 655                 ret = 1;
 656         }
 657         spin_unlock(&mm->page_table_lock);
 658
 659         return ret;
 660 }
 661
 662 static void __split_huge_page_refcount(struct page *page)
 663 {
 664         int i;
 665         unsigned long head_index = page->index;
 666         struct zone *zone = page_zone(page);
 667
 668         /* prevent PageLRU to go away from under us, and freeze lru stats */
 669         spin_lock_irq(&zone->lru_lock);
 670         compound_lock(page);
 671
 672         for (i = 1; i < HPAGE_PMD_NR; i++) {
 673                 struct page *page_tail = page + i;
 674
 675                 /* tail_page->_count cannot change */
 676                 atomic_sub(atomic_read(&page_tail->_count), &page->_count);
 677                 BUG_ON(page_count(page) <= 0);
 678                 atomic_add(page_mapcount(page) + 1, &page_tail->_count);
 679                 BUG_ON(atomic_read(&page_tail->_count) <= 0);
 680
 681                 /* after clearing PageTail the gup refcount can be released */
 682                 smp_mb();
 683
 684                 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
 685                 page_tail->flags |= (page->flags &
 686                                      ((1L << PG_referenced) |
 687                                       (1L << PG_swapbacked) |
 688                                       (1L << PG_mlocked) |
 689                                       (1L << PG_uptodate)));
 690                 page_tail->flags |= (1L << PG_dirty);
 691
 692                 /*
 693                  * 1) clear PageTail before overwriting first_page
 694                  * 2) clear PageTail before clearing PageHead for VM_BUG_ON
 695                  */
 696                 smp_wmb();
 697
 698                 /*
 699                  * __split_huge_page_splitting() already set the
 700                  * splitting bit in all pmd that could map this
 701                  * hugepage, that will ensure no CPU can alter the
 702                  * mapcount on the head page. The mapcount is only
 703                  * accounted in the head page and it has to be
 704                  * transferred to all tail pages in the below code. So
 705                  * for this code to be safe, the split the mapcount
 706                  * can't change. But that doesn't mean userland can't
 707                  * keep changing and reading the page contents while
 708                  * we transfer the mapcount, so the pmd splitting
 709                  * status is achieved setting a reserved bit in the
 710                  * pmd, not by clearing the present bit.
 711                 */
 712                 BUG_ON(page_mapcount(page_tail));
 713                 page_tail->_mapcount = page->_mapcount;
 714
 715                 BUG_ON(page_tail->mapping);
 716                 page_tail->mapping = page->mapping;
 717
 718                 page_tail->index = ++head_index;
 719
 720                 BUG_ON(!PageAnon(page_tail));
 721                 BUG_ON(!PageUptodate(page_tail));
 722                 BUG_ON(!PageDirty(page_tail));
 723                 BUG_ON(!PageSwapBacked(page_tail));
 724
 725                 lru_add_page_tail(zone, page, page_tail);
 726         }
 727
 728         ClearPageCompound(page);
 729         compound_unlock(page);
 730         spin_unlock_irq(&zone->lru_lock);
 731
 732         for (i = 1; i < HPAGE_PMD_NR; i++) {
 733                 struct page *page_tail = page + i;
 734                 BUG_ON(page_count(page_tail) <= 0);
 735                 /*
 736                  * Tail pages may be freed if there wasn't any mapping
 737                  * like if add_to_swap() is running on a lru page that
 738                  * had its mapping zapped. And freeing these pages
 739                  * requires taking the lru_lock so we do the put_page
 740                  * of the tail pages after the split is complete.
 741                  */
 742                 put_page(page_tail);
 743         }
 744
 745         /*
 746          * Only the head page (now become a regular page) is required
 747          * to be pinned by the caller.
 748          */
 749         BUG_ON(page_count(page) <= 0);
 750 }
 751
 752 static int __split_huge_page_map(struct page *page,
 753                                  struct vm_area_struct *vma,
 754                                  unsigned long address)
 755 {
 756         struct mm_struct *mm = vma->vm_mm;
 757         pmd_t *pmd, _pmd;
 758         int ret = 0, i;
 759         pgtable_t pgtable;
 760         unsigned long haddr;
 761
 762         spin_lock(&mm->page_table_lock);
 763         pmd = page_check_address_pmd(page, mm, address,
 764                                      PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG);
 765         if (pmd) {
 766                 pgtable = get_pmd_huge_pte(mm);
 767                 pmd_populate(mm, &_pmd, pgtable);
 768
 769                 for (i = 0, haddr = address; i < HPAGE_PMD_NR;
 770                      i++, haddr += PAGE_SIZE) {
 771                         pte_t *pte, entry;
 772                         BUG_ON(PageCompound(page+i));
 773                         entry = mk_pte(page + i, vma->vm_page_prot);
 774                         entry = maybe_mkwrite(pte_mkdirty(entry), vma);
 775                         if (!pmd_write(*pmd))
 776                                 entry = pte_wrprotect(entry);
 777                         else
 778                                 BUG_ON(page_mapcount(page) != 1);
 779                         if (!pmd_young(*pmd))
 780                                 entry = pte_mkold(entry);
 781                         pte = pte_offset_map(&_pmd, haddr);
 782                         BUG_ON(!pte_none(*pte));
 783                         set_pte_at(mm, haddr, pte, entry);
 784                         pte_unmap(pte);
 785                 }
 786
 787                 mm->nr_ptes++;
 788                 smp_wmb(); /* make pte visible before pmd */
 789                 /*
 790                  * Up to this point the pmd is present and huge and
 791                  * userland has the whole access to the hugepage
 792                  * during the split (which happens in place). If we
 793                  * overwrite the pmd with the not-huge version
 794                  * pointing to the pte here (which of course we could
 795                  * if all CPUs were bug free), userland could trigger
 796                  * a small page size TLB miss on the small sized TLB
 797                  * while the hugepage TLB entry is still established
 798                  * in the huge TLB. Some CPU doesn't like that. See
 799                  * http://support.amd.com/us/Processor_TechDocs/41322.pdf,
 800                  * Erratum 383 on page 93. Intel should be safe but is
 801                  * also warns that it's only safe if the permission
 802                  * and cache attributes of the two entries loaded in
 803                  * the two TLB is identical (which should be the case
 804                  * here). But it is generally safer to never allow
 805                  * small and huge TLB entries for the same virtual
 806                  * address to be loaded simultaneously. So instead of
 807                  * doing "pmd_populate(); flush_tlb_range();" we first
 808                  * mark the current pmd notpresent (atomically because
 809                  * here the pmd_trans_huge and pmd_trans_splitting
 810                  * must remain set at all times on the pmd until the
 811                  * split is complete for this pmd), then we flush the
 812                  * SMP TLB and finally we write the non-huge version
 813                  * of the pmd entry with pmd_populate.
 814                  */
 815                 set_pmd_at(mm, address, pmd, pmd_mknotpresent(*pmd));
 816                 flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
 817                 pmd_populate(mm, pmd, pgtable);
 818                 ret = 1;
 819         }
 820         spin_unlock(&mm->page_table_lock);
 821
 822         return ret;
 823 }
 824
 825 /* must be called with anon_vma->root->lock hold */
 826 static void __split_huge_page(struct page *page,
 827                               struct anon_vma *anon_vma)
 828 {
 829         int mapcount, mapcount2;
 830         struct anon_vma_chain *avc;
 831
 832         BUG_ON(!PageHead(page));
 833         BUG_ON(PageTail(page));
 834
 835         mapcount = 0;
 836         list_for_each_entry(avc, &anon_vma->head, same_anon_vma) {
 837                 struct vm_area_struct *vma = avc->vma;
 838                 unsigned long addr = vma_address(page, vma);
 839                 BUG_ON(is_vma_temporary_stack(vma));
 840                 if (addr == -EFAULT)
 841                         continue;
 842                 mapcount += __split_huge_page_splitting(page, vma, addr);
 843         }
 844         BUG_ON(mapcount != page_mapcount(page));
 845
 846         __split_huge_page_refcount(page);
 847
 848         mapcount2 = 0;
 849         list_for_each_entry(avc, &anon_vma->head, same_anon_vma) {
 850                 struct vm_area_struct *vma = avc->vma;
 851                 unsigned long addr = vma_address(page, vma);
 852                 BUG_ON(is_vma_temporary_stack(vma));
 853                 if (addr == -EFAULT)
 854                         continue;
 855                 mapcount2 += __split_huge_page_map(page, vma, addr);
 856         }
 857         BUG_ON(mapcount != mapcount2);
 858 }
 859
 860 int split_huge_page(struct page *page)
 861 {
 862         struct anon_vma *anon_vma;
 863         int ret = 1;
 864
 865         BUG_ON(!PageAnon(page));
 866         anon_vma = page_lock_anon_vma(page);
 867         if (!anon_vma)
 868                 goto out;
 869         ret = 0;
 870         if (!PageCompound(page))
 871                 goto out_unlock;
 872
 873         BUG_ON(!PageSwapBacked(page));
 874         __split_huge_page(page, anon_vma);
 875
 876         BUG_ON(PageCompound(page));
 877 out_unlock:
 878         page_unlock_anon_vma(anon_vma);
 879 out:
 880         return ret;
 881 }
 882
 883 void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd)
 884 {
 885         struct page *page;
 886
 887         spin_lock(&mm->page_table_lock);
 888         if (unlikely(!pmd_trans_huge(*pmd))) {
 889                 spin_unlock(&mm->page_table_lock);
 890                 return;
 891         }
 892         page = pmd_page(*pmd);
 893         VM_BUG_ON(!page_count(page));
 894         get_page(page);
 895         spin_unlock(&mm->page_table_lock);
 896
 897         split_huge_page(page);
 898
 899         put_page(page);
 900         BUG_ON(pmd_trans_huge(*pmd));
 901 }