Make page->private usable in compound pages

If we add a new flag so that we can distinguish between the first page and the
tail pages then we can avoid to use page->private in the first page.
page->private == page for the first page, so there is no real information in
there.

Freeing up page->private makes the use of compound pages more transparent.
They become more usable like real pages.  Right now we have to be careful f.e.
 if we are going beyond PAGE_SIZE allocations in the slab on i386 because we
can then no longer use the private field.  This is one of the issues that
cause us not to support debugging for page size slabs in SLAB.

Having page->private available for SLUB would allow more meta information in
the page struct.  I can probably avoid the 16 bit ints that I have in there
right now.

Also if page->private is available then a compound page may be equipped with
buffer heads.  This may free up the way for filesystems to support larger
blocks than page size.

We add PageTail as an alias of PageReclaim.  Compound pages cannot currently
be reclaimed.  Because of the alias one needs to check PageCompound first.

The RFC for the this approach was discussed at
http://marc.info/?t=117574302800001&r=1&w=2

[nacc@us.ibm.com: fix hugetlbfs]
Signed-off-by: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Nishanth Aravamudan <nacc@us.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c
index 4f36987eea7241bfea99187f0ec5765202426bd1..2da841110727b223c4e9c4720a6351a94c92a7c1 100644 (file)
--- a/arch/ia64/mm/init.c
+++ b/arch/ia64/mm/init.c
@@ -121,7 +121,7 @@ lazy_mmu_prot_update (pte_t pte)
                return;                         /* i-cache is already coherent with d-cache */
 
        if (PageCompound(page)) {
-               order = (unsigned long) (page[1].lru.prev);
+               order = compound_order(page);
                flush_icache_range(addr, addr + (1UL << order << PAGE_SHIFT));
        }
        else

diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
index 9ba71b252f3ece82706cf5545c669076998e52e2..8e1b7825e2f3fd61dddc50d5c54a7d694708c387 100644 (file)
--- a/fs/hugetlbfs/inode.c
+++ b/fs/hugetlbfs/inode.c
@@ -450,7 +450,7 @@ static int hugetlbfs_symlink(struct inode *dir,
  */
 static int hugetlbfs_set_page_dirty(struct page *page)
 {
-       struct page *head = (struct page *)page_private(page);
+       struct page *head = compound_head(page);
 
        SetPageDirty(head);
        return 0;

diff --git a/include/linux/mm.h b/include/linux/mm.h
index c95d96ebd5ade0fd094f05273a011ed93d8df5a4..8c149fa4491d751bd70997b024d220a5a139f8d4 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -267,17 +267,28 @@ static inline int get_page_unless_zero(struct page *page)
        return atomic_inc_not_zero(&page->_count);
 }
 
+static inline struct page *compound_head(struct page *page)
+{
+       /*
+        * We could avoid the PageCompound(page) check if
+        * we would not overload PageTail().
+        *
+        * This check has to be done in several performance critical
+        * paths of the slab etc. IMHO PageTail deserves its own flag.
+        */
+       if (unlikely(PageCompound(page) && PageTail(page)))
+               return page->first_page;
+       return page;
+}
+
 static inline int page_count(struct page *page)
 {
-       if (unlikely(PageCompound(page)))
-               page = (struct page *)page_private(page);
-       return atomic_read(&page->_count);
+       return atomic_read(&compound_head(page)->_count);
 }
 
 static inline void get_page(struct page *page)
 {
-       if (unlikely(PageCompound(page)))
-               page = (struct page *)page_private(page);
+       page = compound_head(page);
        VM_BUG_ON(atomic_read(&page->_count) == 0);
        atomic_inc(&page->_count);
 }
@@ -314,6 +325,18 @@ static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
        return (compound_page_dtor *)page[1].lru.next;
 }
 
+static inline int compound_order(struct page *page)
+{
+       if (!PageCompound(page) || PageTail(page))
+               return 0;
+       return (unsigned long)page[1].lru.prev;
+}
+
+static inline void set_compound_order(struct page *page, unsigned long order)
+{
+       page[1].lru.prev = (void *)order;
+}
+
 /*
  * Multiple processes may "see" the same page. E.g. for untouched
  * mappings of /dev/null, all processes see the same page full of

diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 96326594e55d9fec9ae803ea61f9659c1e7f6b64..a1e143634946f08128737148f53358550b356ec8 100644 (file)
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -94,6 +94,12 @@
 /* PG_owner_priv_1 users should have descriptive aliases */
 #define PG_checked             PG_owner_priv_1 /* Used by some filesystems */
 
+/*
+ * Marks tail portion of a compound page. We currently do not reclaim
+ * compound pages so we can reuse a flag only used for reclaim here.
+ */
+#define PG_tail                        PG_reclaim
+
 #if (BITS_PER_LONG > 32)
 /*
  * 64-bit-only flags build down from bit 31
@@ -241,6 +247,14 @@ static inline void SetPageUptodate(struct page *page)
 #define __SetPageCompound(page)        __set_bit(PG_compound, &(page)->flags)
 #define __ClearPageCompound(page) __clear_bit(PG_compound, &(page)->flags)
 
+/*
+ * Note: PG_tail is an alias of another page flag. The result of PageTail()
+ * is only valid if PageCompound(page) is true.
+ */
+#define PageTail(page) test_bit(PG_tail, &(page)->flags)
+#define __SetPageTail(page)    __set_bit(PG_tail, &(page)->flags)
+#define __ClearPageTail(page)  __clear_bit(PG_tail, &(page)->flags)
+
 #ifdef CONFIG_SWAP
 #define PageSwapCache(page)    test_bit(PG_swapcache, &(page)->flags)
 #define SetPageSwapCache(page) set_bit(PG_swapcache, &(page)->flags)

diff --git a/mm/internal.h b/mm/internal.h
index d527b80b292fc7d96308ab91f459dbe7bae874c8..a3110c02aea7d4b6f8edbd06e5a06477d3c2fc8b 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -24,7 +24,7 @@ static inline void set_page_count(struct page *page, int v)
  */
 static inline void set_page_refcounted(struct page *page)
 {
-       VM_BUG_ON(PageCompound(page) && page_private(page) != (unsigned long)page);
+       VM_BUG_ON(PageCompound(page) && PageTail(page));
        VM_BUG_ON(atomic_read(&page->_count));
        set_page_count(page, 1);
 }

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 542fc088ff508b15bf1416dcd68c80e8c3fb2fac..fc241fe295abf21a9c015b66a21d0e1eb18ea7d8 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -225,7 +225,7 @@ static void bad_page(struct page *page)
 
 static void free_compound_page(struct page *page)
 {
-       __free_pages_ok(page, (unsigned long)page[1].lru.prev);
+       __free_pages_ok(page, compound_order(page));
 }
 
 static void prep_compound_page(struct page *page, unsigned long order)
@@ -234,12 +234,14 @@ static void prep_compound_page(struct page *page, unsigned long order)
        int nr_pages = 1 << order;
 
        set_compound_page_dtor(page, free_compound_page);
-       page[1].lru.prev = (void *)order;
-       for (i = 0; i < nr_pages; i++) {
+       set_compound_order(page, order);
+       __SetPageCompound(page);
+       for (i = 1; i < nr_pages; i++) {
                struct page *p = page + i;
 
+               __SetPageTail(p);
                __SetPageCompound(p);
-               set_page_private(p, (unsigned long)page);
+               p->first_page = page;
        }
 }
 
@@ -248,15 +250,19 @@ static void destroy_compound_page(struct page *page, unsigned long order)
        int i;
        int nr_pages = 1 << order;
 
-       if (unlikely((unsigned long)page[1].lru.prev != order))
+       if (unlikely(compound_order(page) != order))
                bad_page(page);
 
-       for (i = 0; i < nr_pages; i++) {
+       if (unlikely(!PageCompound(page)))
+                       bad_page(page);
+       __ClearPageCompound(page);
+       for (i = 1; i < nr_pages; i++) {
                struct page *p = page + i;
 
-               if (unlikely(!PageCompound(p) |
-                               (page_private(p) != (unsigned long)page)))
+               if (unlikely(!PageCompound(p) | !PageTail(p) |
+                               (p->first_page != page)))
                        bad_page(page);
+               __ClearPageTail(p);
                __ClearPageCompound(p);
        }
 }
@@ -429,13 +435,18 @@ static inline int free_pages_check(struct page *page)
                        1 << PG_private |
                        1 << PG_locked  |
                        1 << PG_active  |
-                       1 << PG_reclaim |
                        1 << PG_slab    |
                        1 << PG_swapcache |
                        1 << PG_writeback |
                        1 << PG_reserved |
                        1 << PG_buddy ))))
                bad_page(page);
+       /*
+        * PageReclaim == PageTail. It is only an error
+        * for PageReclaim to be set if PageCompound is clear.
+        */
+       if (unlikely(!PageCompound(page) && PageReclaim(page)))
+               bad_page(page);
        if (PageDirty(page))
                __ClearPageDirty(page);
        /*

diff --git a/mm/slab.c b/mm/slab.c
index 9cd01fa600043e0a0f3520b523db4694d8d988e4..f4b2e22b5c616bdb7a29e33d207fdf8fbe66a1f2 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -602,8 +602,7 @@ static inline void page_set_cache(struct page *page, struct kmem_cache *cache)
 
 static inline struct kmem_cache *page_get_cache(struct page *page)
 {
-       if (unlikely(PageCompound(page)))
-               page = (struct page *)page_private(page);
+       page = compound_head(page);
        BUG_ON(!PageSlab(page));
        return (struct kmem_cache *)page->lru.next;
 }
@@ -615,8 +614,7 @@ static inline void page_set_slab(struct page *page, struct slab *slab)
 
 static inline struct slab *page_get_slab(struct page *page)
 {
-       if (unlikely(PageCompound(page)))
-               page = (struct page *)page_private(page);
+       page = compound_head(page);
        BUG_ON(!PageSlab(page));
        return (struct slab *)page->lru.prev;
 }

diff --git a/mm/slub.c b/mm/slub.c
index 9d52cce7c9992f6f89cf33902461d9c67c52e08c..8fa1c6e937f5ccc475ee270115f2d7e1dd106e8b 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1325,9 +1325,7 @@ void kmem_cache_free(struct kmem_cache *s, void *x)
 
        page = virt_to_page(x);
 
-       if (unlikely(PageCompound(page)))
-               page = page->first_page;
-
+       page = compound_head(page);
 
        if (unlikely(PageError(page) && (s->flags & SLAB_STORE_USER)))
                set_tracking(s, x, TRACK_FREE);
@@ -1338,10 +1336,7 @@ EXPORT_SYMBOL(kmem_cache_free);
 /* Figure out on which slab object the object resides */
 static struct page *get_object_page(const void *x)
 {
-       struct page *page = virt_to_page(x);
-
-       if (unlikely(PageCompound(page)))
-               page = page->first_page;
+       struct page *page = compound_head(virt_to_page(x));
 
        if (!PageSlab(page))
                return NULL;
@@ -2081,10 +2076,7 @@ void kfree(const void *x)
        if (!x)
                return;
 
-       page = virt_to_page(x);
-
-       if (unlikely(PageCompound(page)))
-               page = page->first_page;
+       page = compound_head(virt_to_page(x));
 
        s = page->slab;
 
@@ -2120,10 +2112,7 @@ void *krealloc(const void *p, size_t new_size, gfp_t flags)
                return NULL;
        }
 
-       page = virt_to_page(p);
-
-       if (unlikely(PageCompound(page)))
-               page = page->first_page;
+       page = compound_head(virt_to_page(p));
 
        new_cache = get_slab(new_size, flags);
 

diff --git a/mm/swap.c b/mm/swap.c
index 2ed7be39795e3034986988502b75982698a1febb..218c52a24a216831a347736cde51045563854770 100644 (file)
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -55,7 +55,7 @@ static void fastcall __page_cache_release(struct page *page)
 
 static void put_compound_page(struct page *page)
 {
-       page = (struct page *)page_private(page);
+       page = compound_head(page);
        if (put_page_testzero(page)) {
                compound_page_dtor *dtor;