[PATCH] mm: PageLRU no testset

PG_lru is protected by zone->lru_lock. It does not need TestSet/TestClear
operations.

Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index d52999c433368c1f7801ae787f13ee8beff714ba..58856c823f8b6885d54de0cebd26970d3fc55204 100644 (file)
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -239,10 +239,9 @@ extern void __mod_page_state_offset(unsigned long offset, unsigned long delta);
 #define __ClearPageDirty(page) __clear_bit(PG_dirty, &(page)->flags)
 #define TestClearPageDirty(page) test_and_clear_bit(PG_dirty, &(page)->flags)
 
-#define SetPageLRU(page)       set_bit(PG_lru, &(page)->flags)
 #define PageLRU(page)          test_bit(PG_lru, &(page)->flags)
-#define TestSetPageLRU(page)   test_and_set_bit(PG_lru, &(page)->flags)
-#define TestClearPageLRU(page) test_and_clear_bit(PG_lru, &(page)->flags)
+#define SetPageLRU(page)       set_bit(PG_lru, &(page)->flags)
+#define ClearPageLRU(page)     clear_bit(PG_lru, &(page)->flags)
 
 #define PageActive(page)       test_bit(PG_active, &(page)->flags)
 #define SetPageActive(page)    set_bit(PG_active, &(page)->flags)

diff --git a/mm/swap.c b/mm/swap.c
index 3045a0f4c4519ac5052cbc052450dd0723056e2e..985324ee9368810a765b384a6ac77346ef53b649 100644 (file)
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -214,8 +214,8 @@ void fastcall __page_cache_release(struct page *page)
                struct zone *zone = page_zone(page);
 
                spin_lock_irqsave(&zone->lru_lock, flags);
-               if (!TestClearPageLRU(page))
-                       BUG();
+               BUG_ON(!PageLRU(page));
+               ClearPageLRU(page);
                del_page_from_lru(zone, page);
                spin_unlock_irqrestore(&zone->lru_lock, flags);
        }
@@ -265,8 +265,8 @@ void release_pages(struct page **pages, int nr, int cold)
                                zone = pagezone;
                                spin_lock_irq(&zone->lru_lock);
                        }
-                       if (!TestClearPageLRU(page))
-                               BUG();
+                       BUG_ON(!PageLRU(page));
+                       ClearPageLRU(page);
                        del_page_from_lru(zone, page);
                }
 
@@ -345,8 +345,8 @@ void __pagevec_lru_add(struct pagevec *pvec)
                        zone = pagezone;
                        spin_lock_irq(&zone->lru_lock);
                }
-               if (TestSetPageLRU(page))
-                       BUG();
+               BUG_ON(PageLRU(page));
+               SetPageLRU(page);
                add_page_to_inactive_list(zone, page);
        }
        if (zone)
@@ -372,8 +372,8 @@ void __pagevec_lru_add_active(struct pagevec *pvec)
                        zone = pagezone;
                        spin_lock_irq(&zone->lru_lock);
                }
-               if (TestSetPageLRU(page))
-                       BUG();
+               BUG_ON(PageLRU(page));
+               SetPageLRU(page);
                if (TestSetPageActive(page))
                        BUG();
                add_page_to_active_list(zone, page);

diff --git a/mm/vmscan.c b/mm/vmscan.c
index acb7611cd5257e5efba27e4a70526130bc436620..40fb37828e8c4993bbc848b816d84fc279299762 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1042,9 +1042,10 @@ int isolate_lru_page(struct page *page)
        if (PageLRU(page)) {
                struct zone *zone = page_zone(page);
                spin_lock_irq(&zone->lru_lock);
-               if (TestClearPageLRU(page)) {
+               if (PageLRU(page)) {
                        ret = 1;
                        get_page(page);
+                       ClearPageLRU(page);
                        if (PageActive(page))
                                del_page_from_active_list(zone, page);
                        else
@@ -1085,6 +1086,8 @@ static int isolate_lru_pages(int nr_to_scan, struct list_head *src,
                page = lru_to_page(src);
                prefetchw_prev_lru_page(page, src, flags);
 
+               BUG_ON(!PageLRU(page));
+
                list_del(&page->lru);
                if (unlikely(get_page_testone(page))) {
                        /*
@@ -1100,8 +1103,7 @@ static int isolate_lru_pages(int nr_to_scan, struct list_head *src,
                 * the page is not being freed elsewhere -- the page release
                 * code relies on it.
                 */
-               if (!TestClearPageLRU(page))
-                       BUG();
+               ClearPageLRU(page);
                list_add(&page->lru, dst);
                nr_taken++;
        }
@@ -1156,8 +1158,8 @@ static void shrink_cache(struct zone *zone, struct scan_control *sc)
                 */
                while (!list_empty(&page_list)) {
                        page = lru_to_page(&page_list);
-                       if (TestSetPageLRU(page))
-                               BUG();
+                       BUG_ON(PageLRU(page));
+                       SetPageLRU(page);
                        list_del(&page->lru);
                        if (PageActive(page))
                                add_page_to_active_list(zone, page);
@@ -1276,8 +1278,8 @@ refill_inactive_zone(struct zone *zone, struct scan_control *sc)
        while (!list_empty(&l_inactive)) {
                page = lru_to_page(&l_inactive);
                prefetchw_prev_lru_page(page, &l_inactive, flags);
-               if (TestSetPageLRU(page))
-                       BUG();
+               BUG_ON(PageLRU(page));
+               SetPageLRU(page);
                if (!TestClearPageActive(page))
                        BUG();
                list_move(&page->lru, &zone->inactive_list);
@@ -1305,8 +1307,8 @@ refill_inactive_zone(struct zone *zone, struct scan_control *sc)
        while (!list_empty(&l_active)) {
                page = lru_to_page(&l_active);
                prefetchw_prev_lru_page(page, &l_active, flags);
-               if (TestSetPageLRU(page))
-                       BUG();
+               BUG_ON(PageLRU(page));
+               SetPageLRU(page);
                BUG_ON(!PageActive(page));
                list_move(&page->lru, &zone->active_list);
                pgmoved++;