[PATCH] mm: VM_BUG_ON

Introduce a VM_BUG_ON, which is turned on with CONFIG_DEBUG_VM.  Use this
in the lightweight, inline refcounting functions; PageLRU and PageActive
checks in vmscan, because they're pretty well confined to vmscan.  And in
page allocate/free fastpaths which can be the hottest parts of the kernel
for kbuilds.

Unlike BUG_ON, VM_BUG_ON must not be used to execute statements with
side-effects, and should not be used outside core mm code.

Signed-off-by: Nick Piggin <npiggin@suse.de>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 224178a000d2e1930c13b5db60642de3d0bd4ff6..7d20b25c58fcb055f1644759af6552dd394f3636 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -278,6 +278,12 @@ struct page {
  */
 #include <linux/page-flags.h>
 
+#ifdef CONFIG_DEBUG_VM
+#define VM_BUG_ON(cond) BUG_ON(cond)
+#else
+#define VM_BUG_ON(condition) do { } while(0)
+#endif
+
 /*
  * Methods to modify the page usage count.
  *
@@ -297,7 +303,7 @@ struct page {
  */
 static inline int put_page_testzero(struct page *page)
 {
-       BUG_ON(atomic_read(&page->_count) == 0);
+       VM_BUG_ON(atomic_read(&page->_count) == 0);
        return atomic_dec_and_test(&page->_count);
 }
 
@@ -307,6 +313,7 @@ static inline int put_page_testzero(struct page *page)
  */
 static inline int get_page_unless_zero(struct page *page)
 {
+       VM_BUG_ON(PageCompound(page));
        return atomic_inc_not_zero(&page->_count);
 }
 
@@ -323,6 +330,7 @@ static inline void get_page(struct page *page)
 {
        if (unlikely(PageCompound(page)))
                page = (struct page *)page_private(page);
+       VM_BUG_ON(atomic_read(&page->_count) == 0);
        atomic_inc(&page->_count);
 }
 

diff --git a/mm/internal.h b/mm/internal.h
index d20e3cc4aef02d6d299c14cc52d5f2e875bbdb79..d527b80b292fc7d96308ab91f459dbe7bae874c8 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -24,8 +24,8 @@ static inline void set_page_count(struct page *page, int v)
  */
 static inline void set_page_refcounted(struct page *page)
 {
-       BUG_ON(PageCompound(page) && page_private(page) != (unsigned long)page);
-       BUG_ON(atomic_read(&page->_count));
+       VM_BUG_ON(PageCompound(page) && page_private(page) != (unsigned long)page);
+       VM_BUG_ON(atomic_read(&page->_count));
        set_page_count(page, 1);
 }
 

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 8a52ba9fe693bcf749486c3ec5ae6f1e26849700..4b33878e94883d865836e249cbd886584af7ce29 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -127,7 +127,6 @@ static int bad_range(struct zone *zone, struct page *page)
 
        return 0;
 }
-
 #else
 static inline int bad_range(struct zone *zone, struct page *page)
 {
@@ -218,12 +217,12 @@ static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags)
 {
        int i;
 
-       BUG_ON((gfp_flags & (__GFP_WAIT | __GFP_HIGHMEM)) == __GFP_HIGHMEM);
+       VM_BUG_ON((gfp_flags & (__GFP_WAIT | __GFP_HIGHMEM)) == __GFP_HIGHMEM);
        /*
         * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO
         * and __GFP_HIGHMEM from hard or soft interrupt context.
         */
-       BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt());
+       VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt());
        for (i = 0; i < (1 << order); i++)
                clear_highpage(page + i);
 }
@@ -347,8 +346,8 @@ static inline void __free_one_page(struct page *page,
 
        page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);
 
-       BUG_ON(page_idx & (order_size - 1));
-       BUG_ON(bad_range(zone, page));
+       VM_BUG_ON(page_idx & (order_size - 1));
+       VM_BUG_ON(bad_range(zone, page));
 
        zone->free_pages += order_size;
        while (order < MAX_ORDER-1) {
@@ -421,7 +420,7 @@ static void free_pages_bulk(struct zone *zone, int count,
        while (count--) {
                struct page *page;
 
-               BUG_ON(list_empty(list));
+               VM_BUG_ON(list_empty(list));
                page = list_entry(list->prev, struct page, lru);
                /* have to delete it as __free_one_page list manipulates */
                list_del(&page->lru);
@@ -512,7 +511,7 @@ static inline void expand(struct zone *zone, struct page *page,
                area--;
                high--;
                size >>= 1;
-               BUG_ON(bad_range(zone, &page[size]));
+               VM_BUG_ON(bad_range(zone, &page[size]));
                list_add(&page[size].lru, &area->free_list);
                area->nr_free++;
                set_page_order(&page[size], high);
@@ -761,8 +760,8 @@ void split_page(struct page *page, unsigned int order)
 {
        int i;
 
-       BUG_ON(PageCompound(page));
-       BUG_ON(!page_count(page));
+       VM_BUG_ON(PageCompound(page));
+       VM_BUG_ON(!page_count(page));
        for (i = 1; i < (1 << order); i++)
                set_page_refcounted(page + i);
 }
@@ -809,7 +808,7 @@ again:
        local_irq_restore(flags);
        put_cpu();
 
-       BUG_ON(bad_range(zone, page));
+       VM_BUG_ON(bad_range(zone, page));
        if (prep_new_page(page, order, gfp_flags))
                goto again;
        return page;
@@ -1083,7 +1082,7 @@ fastcall unsigned long get_zeroed_page(gfp_t gfp_mask)
         * get_zeroed_page() returns a 32-bit address, which cannot represent
         * a highmem page
         */
-       BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
+       VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
 
        page = alloc_pages(gfp_mask | __GFP_ZERO, 0);
        if (page)
@@ -1116,7 +1115,7 @@ EXPORT_SYMBOL(__free_pages);
 fastcall void free_pages(unsigned long addr, unsigned int order)
 {
        if (addr != 0) {
-               BUG_ON(!virt_addr_valid((void *)addr));
+               VM_BUG_ON(!virt_addr_valid((void *)addr));
                __free_pages(virt_to_page((void *)addr), order);
        }
 }

diff --git a/mm/swap.c b/mm/swap.c
index 687686a61f7cb81cc75cd65a636ade65eb00d08d..600235e43704f0b4e2628a908b54f13a8de5c695 100644 (file)
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -233,7 +233,7 @@ void fastcall __page_cache_release(struct page *page)
                struct zone *zone = page_zone(page);
 
                spin_lock_irqsave(&zone->lru_lock, flags);
-               BUG_ON(!PageLRU(page));
+               VM_BUG_ON(!PageLRU(page));
                __ClearPageLRU(page);
                del_page_from_lru(zone, page);
                spin_unlock_irqrestore(&zone->lru_lock, flags);
@@ -284,7 +284,7 @@ void release_pages(struct page **pages, int nr, int cold)
                                zone = pagezone;
                                spin_lock_irq(&zone->lru_lock);
                        }
-                       BUG_ON(!PageLRU(page));
+                       VM_BUG_ON(!PageLRU(page));
                        __ClearPageLRU(page);
                        del_page_from_lru(zone, page);
                }
@@ -337,7 +337,7 @@ void __pagevec_release_nonlru(struct pagevec *pvec)
        for (i = 0; i < pagevec_count(pvec); i++) {
                struct page *page = pvec->pages[i];
 
-               BUG_ON(PageLRU(page));
+               VM_BUG_ON(PageLRU(page));
                if (put_page_testzero(page))
                        pagevec_add(&pages_to_free, page);
        }
@@ -364,7 +364,7 @@ void __pagevec_lru_add(struct pagevec *pvec)
                        zone = pagezone;
                        spin_lock_irq(&zone->lru_lock);
                }
-               BUG_ON(PageLRU(page));
+               VM_BUG_ON(PageLRU(page));
                SetPageLRU(page);
                add_page_to_inactive_list(zone, page);
        }
@@ -391,9 +391,9 @@ void __pagevec_lru_add_active(struct pagevec *pvec)
                        zone = pagezone;
                        spin_lock_irq(&zone->lru_lock);
                }
-               BUG_ON(PageLRU(page));
+               VM_BUG_ON(PageLRU(page));
                SetPageLRU(page);
-               BUG_ON(PageActive(page));
+               VM_BUG_ON(PageActive(page));
                SetPageActive(page);
                add_page_to_active_list(zone, page);
        }

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 5d4c4d02254dc977d84951845dada98a80bd0627..41a3da3d6ccce37437131c8f717ec15aebbb0dbe 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -440,7 +440,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                if (TestSetPageLocked(page))
                        goto keep;
 
-               BUG_ON(PageActive(page));
+               VM_BUG_ON(PageActive(page));
 
                sc->nr_scanned++;
 
@@ -564,7 +564,7 @@ keep_locked:
                unlock_page(page);
 keep:
                list_add(&page->lru, &ret_pages);
-               BUG_ON(PageLRU(page));
+               VM_BUG_ON(PageLRU(page));
        }
        list_splice(&ret_pages, page_list);
        if (pagevec_count(&freed_pvec))
@@ -603,7 +603,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
                page = lru_to_page(src);
                prefetchw_prev_lru_page(page, src, flags);
 
-               BUG_ON(!PageLRU(page));
+               VM_BUG_ON(!PageLRU(page));
 
                list_del(&page->lru);
                target = src;
@@ -674,7 +674,7 @@ static unsigned long shrink_inactive_list(unsigned long max_scan,
                 */
                while (!list_empty(&page_list)) {
                        page = lru_to_page(&page_list);
-                       BUG_ON(PageLRU(page));
+                       VM_BUG_ON(PageLRU(page));
                        SetPageLRU(page);
                        list_del(&page->lru);
                        if (PageActive(page))
@@ -797,9 +797,9 @@ static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
        while (!list_empty(&l_inactive)) {
                page = lru_to_page(&l_inactive);
                prefetchw_prev_lru_page(page, &l_inactive, flags);
-               BUG_ON(PageLRU(page));
+               VM_BUG_ON(PageLRU(page));
                SetPageLRU(page);
-               BUG_ON(!PageActive(page));
+               VM_BUG_ON(!PageActive(page));
                ClearPageActive(page);
 
                list_move(&page->lru, &zone->inactive_list);
@@ -827,9 +827,9 @@ static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
        while (!list_empty(&l_active)) {
                page = lru_to_page(&l_active);
                prefetchw_prev_lru_page(page, &l_active, flags);
-               BUG_ON(PageLRU(page));
+               VM_BUG_ON(PageLRU(page));
                SetPageLRU(page);
-               BUG_ON(!PageActive(page));
+               VM_BUG_ON(!PageActive(page));
                list_move(&page->lru, &zone->active_list);
                pgmoved++;
                if (!pagevec_add(&pvec, page)) {