mm: update_lru_size do the __mod_zone_page_state

Konstantin Khlebnikov pointed out (nearly four years ago, when lumpy
reclaim was removed) that lru_size can be updated by -nr_taken once per
call to isolate_lru_pages(), instead of page by page.

Update it inside isolate_lru_pages(), or at its two callsites? I chose
to update it at the callsites, rearranging and grouping the updates by
nr_taken and nr_scanned together in both.

With one exception, mem_cgroup_update_lru_size(,lru,) is then used where
__mod_zone_page_state(,NR_LRU_BASE+lru,) is used; and we shall be adding
some more calls in a future commit.  Make the code a little smaller and
simpler by incorporating stat update in lru_size update.

The exception was move_active_pages_to_lru(), which aggregated the
pgmoved stat update separately from the individual lru_size updates; but
I still think this a simplification worth making.

However, the __mod_zone_page_state is not peculiar to mem_cgroups: so
better use the name update_lru_size, calls mem_cgroup_update_lru_size
when CONFIG_MEMCG.

Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andres Lagar-Cavilla <andreslc@google.com>
Cc: Yang Shi <yang.shi@linaro.org>
Cc: Ning Qu <quning@gmail.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Konstantin Khlebnikov <koct9i@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 1191d79aa495e698fdd233e91c2b2b2e317cba9d..94da96738df3fcc2b770e913f0743d4121ecb065 100644 (file)
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -658,12 +658,6 @@ mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
        return 0;
 }
 
-static inline void
-mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
-                             int increment)
-{
-}
-
 static inline unsigned long
 mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
                             int nid, unsigned int lru_mask)

diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index d8cea81ab1aca8947124b2a9d4f121b5fd7a1708..5bd29ba4f174f531d1921f8ea2616b7727b069aa 100644 (file)
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -22,22 +22,34 @@ static inline int page_is_file_cache(struct page *page)
        return !PageSwapBacked(page);
 }
 
+static __always_inline void __update_lru_size(struct lruvec *lruvec,
+                               enum lru_list lru, int nr_pages)
+{
+       __mod_zone_page_state(lruvec_zone(lruvec), NR_LRU_BASE + lru, nr_pages);
+}
+
+static __always_inline void update_lru_size(struct lruvec *lruvec,
+                               enum lru_list lru, int nr_pages)
+{
+#ifdef CONFIG_MEMCG
+       mem_cgroup_update_lru_size(lruvec, lru, nr_pages);
+#else
+       __update_lru_size(lruvec, lru, nr_pages);
+#endif
+}
+
 static __always_inline void add_page_to_lru_list(struct page *page,
                                struct lruvec *lruvec, enum lru_list lru)
 {
-       int nr_pages = hpage_nr_pages(page);
-       mem_cgroup_update_lru_size(lruvec, lru, nr_pages);
+       update_lru_size(lruvec, lru, hpage_nr_pages(page));
        list_add(&page->lru, &lruvec->lists[lru]);
-       __mod_zone_page_state(lruvec_zone(lruvec), NR_LRU_BASE + lru, nr_pages);
 }
 
 static __always_inline void del_page_from_lru_list(struct page *page,
                                struct lruvec *lruvec, enum lru_list lru)
 {
-       int nr_pages = hpage_nr_pages(page);
        list_del(&page->lru);
-       mem_cgroup_update_lru_size(lruvec, lru, -nr_pages);
-       __mod_zone_page_state(lruvec_zone(lruvec), NR_LRU_BASE + lru, -nr_pages);
+       update_lru_size(lruvec, lru, -hpage_nr_pages(page));
 }
 
 /**

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 6a0199706f00bc428968f017ea32df0889e4cd2b..1b40dcad2b90f9395a99dfe027cb0331b0565ec2 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -1035,6 +1035,8 @@ void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
        long size;
        bool empty;
 
+       __update_lru_size(lruvec, lru, nr_pages);
+
        if (mem_cgroup_disabled())
                return;
 

diff --git a/mm/vmscan.c b/mm/vmscan.c
index d3a02ac3eed7df63e6bfb0325e1414ed5e2353fd..dcfdfc1a09420714e9f4dcff8fd3b66de807455e 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1374,7 +1374,6 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
        for (scan = 0; scan < nr_to_scan && nr_taken < nr_to_scan &&
                                        !list_empty(src); scan++) {
                struct page *page;
-               int nr_pages;
 
                page = lru_to_page(src);
                prefetchw_prev_lru_page(page, src, flags);
@@ -1383,10 +1382,8 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 
                switch (__isolate_lru_page(page, mode)) {
                case 0:
-                       nr_pages = hpage_nr_pages(page);
-                       mem_cgroup_update_lru_size(lruvec, lru, -nr_pages);
+                       nr_taken += hpage_nr_pages(page);
                        list_move(&page->lru, dst);
-                       nr_taken += nr_pages;
                        break;
 
                case -EBUSY:
@@ -1602,8 +1599,9 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
        nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
                                     &nr_scanned, sc, isolate_mode, lru);
 
-       __mod_zone_page_state(zone, NR_LRU_BASE + lru, -nr_taken);
+       update_lru_size(lruvec, lru, -nr_taken);
        __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, nr_taken);
+       reclaim_stat->recent_scanned[file] += nr_taken;
 
        if (global_reclaim(sc)) {
                __mod_zone_page_state(zone, NR_PAGES_SCANNED, nr_scanned);
@@ -1624,8 +1622,6 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
 
        spin_lock_irq(&zone->lru_lock);
 
-       reclaim_stat->recent_scanned[file] += nr_taken;
-
        if (global_reclaim(sc)) {
                if (current_is_kswapd())
                        __count_zone_vm_events(PGSTEAL_KSWAPD, zone,
@@ -1742,7 +1738,7 @@ static void move_active_pages_to_lru(struct lruvec *lruvec,
                SetPageLRU(page);
 
                nr_pages = hpage_nr_pages(page);
-               mem_cgroup_update_lru_size(lruvec, lru, nr_pages);
+               update_lru_size(lruvec, lru, nr_pages);
                list_move(&page->lru, &lruvec->lists[lru]);
                pgmoved += nr_pages;
 
@@ -1760,7 +1756,7 @@ static void move_active_pages_to_lru(struct lruvec *lruvec,
                                list_add(&page->lru, pages_to_free);
                }
        }
-       __mod_zone_page_state(zone, NR_LRU_BASE + lru, pgmoved);
+
        if (!is_active_lru(lru))
                __count_vm_events(PGDEACTIVATE, pgmoved);
 }
@@ -1794,14 +1790,15 @@ static void shrink_active_list(unsigned long nr_to_scan,
 
        nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,
                                     &nr_scanned, sc, isolate_mode, lru);
-       if (global_reclaim(sc))
-               __mod_zone_page_state(zone, NR_PAGES_SCANNED, nr_scanned);
 
+       update_lru_size(lruvec, lru, -nr_taken);
+       __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, nr_taken);
        reclaim_stat->recent_scanned[file] += nr_taken;
 
+       if (global_reclaim(sc))
+               __mod_zone_page_state(zone, NR_PAGES_SCANNED, nr_scanned);
        __count_zone_vm_events(PGREFILL, zone, nr_scanned);
-       __mod_zone_page_state(zone, NR_LRU_BASE + lru, -nr_taken);
-       __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, nr_taken);
+
        spin_unlock_irq(&zone->lru_lock);
 
        while (!list_empty(&l_hold)) {