mm/memcg: move reclaim_stat into lruvec

With mem_cgroup_disabled() now explicit, it becomes clear that the
zone_reclaim_stat structure actually belongs in lruvec, per-zone when
memcg is disabled but per-memcg per-zone when it's enabled.

We can delete mem_cgroup_get_reclaim_stat(), and change
update_page_reclaim_stat() to update just the one set of stats, the one
which get_scan_count() will actually use.

Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Konstantin Khlebnikov <khlebnikov@openvz.org>
Acked-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Acked-by: Michal Hocko <mhocko@suse.cz>
Reviewed-by: Minchan Kim <minchan@kernel.org>
Reviewed-by: Michal Hocko <mhocko@suse.cz>
Cc: Glauber Costa <glommer@parallels.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 18ea0b7baf32e0183a68adb8c53bbb61630caac3..cfe9050ad8da61eed59d5e8d0ff499876ed275ab 100644 (file)
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -126,8 +126,6 @@ int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg,
 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
 unsigned long mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg,
                                        int nid, int zid, unsigned int lrumask);
-struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
-                                                     struct zone *zone);
 struct zone_reclaim_stat*
 mem_cgroup_get_reclaim_stat_from_page(struct page *page);
 extern void mem_cgroup_print_oom_info(struct mem_cgroup *memcg,
@@ -356,13 +354,6 @@ mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
        return 0;
 }
 
-
-static inline struct zone_reclaim_stat*
-mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg, struct zone *zone)
-{
-       return NULL;
-}
-
 static inline struct zone_reclaim_stat*
 mem_cgroup_get_reclaim_stat_from_page(struct page *page)
 {

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 4871e31ae27780ffcc5efd7e7a5d24a912f43afa..1b89861eedc0bd296fedcf65e049114568ebcf95 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -185,8 +185,22 @@ static inline int is_unevictable_lru(enum lru_list lru)
        return (lru == LRU_UNEVICTABLE);
 }
 
+struct zone_reclaim_stat {
+       /*
+        * The pageout code in vmscan.c keeps track of how many of the
+        * mem/swap backed and file backed pages are refeferenced.
+        * The higher the rotated/scanned ratio, the more valuable
+        * that cache is.
+        *
+        * The anon LRU stats live in [0], file LRU stats in [1]
+        */
+       unsigned long           recent_rotated[2];
+       unsigned long           recent_scanned[2];
+};
+
 struct lruvec {
        struct list_head lists[NR_LRU_LISTS];
+       struct zone_reclaim_stat reclaim_stat;
 };
 
 /* Mask used at gathering information at once (see memcontrol.c) */
@@ -313,19 +327,6 @@ enum zone_type {
 #error ZONES_SHIFT -- too many zones configured adjust calculation
 #endif
 
-struct zone_reclaim_stat {
-       /*
-        * The pageout code in vmscan.c keeps track of how many of the
-        * mem/swap backed and file backed pages are refeferenced.
-        * The higher the rotated/scanned ratio, the more valuable
-        * that cache is.
-        *
-        * The anon LRU stats live in [0], file LRU stats in [1]
-        */
-       unsigned long           recent_rotated[2];
-       unsigned long           recent_scanned[2];
-};
-
 struct zone {
        /* Fields commonly accessed by the page allocator */
 
@@ -407,8 +408,6 @@ struct zone {
        spinlock_t              lru_lock;
        struct lruvec           lruvec;
 
-       struct zone_reclaim_stat reclaim_stat;
-
        unsigned long           pages_scanned;     /* since last reclaim */
        unsigned long           flags;             /* zone flags, see below */
 

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 30f938c8645394b306eca3c89d5c0e984cfb0fcd..00c8898dbb81e0eb3d8e1a60e2c5966e74e08696 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -138,7 +138,6 @@ struct mem_cgroup_per_zone {
 
        struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1];
 
-       struct zone_reclaim_stat reclaim_stat;
        struct rb_node          tree_node;      /* RB tree node */
        unsigned long long      usage_in_excess;/* Set to the value by which */
                                                /* the soft limit is exceeded*/
@@ -1243,16 +1242,6 @@ int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg, struct zone *zone)
        return (active > inactive);
 }
 
-struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
-                                                     struct zone *zone)
-{
-       int nid = zone_to_nid(zone);
-       int zid = zone_idx(zone);
-       struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
-
-       return &mz->reclaim_stat;
-}
-
 struct zone_reclaim_stat *
 mem_cgroup_get_reclaim_stat_from_page(struct page *page)
 {
@@ -1268,7 +1257,7 @@ mem_cgroup_get_reclaim_stat_from_page(struct page *page)
        /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
        smp_rmb();
        mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
-       return &mz->reclaim_stat;
+       return &mz->lruvec.reclaim_stat;
 }
 
 #define mem_cgroup_from_res_counter(counter, member)   \
@@ -4216,21 +4205,19 @@ static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
        {
                int nid, zid;
                struct mem_cgroup_per_zone *mz;
+               struct zone_reclaim_stat *rstat;
                unsigned long recent_rotated[2] = {0, 0};
                unsigned long recent_scanned[2] = {0, 0};
 
                for_each_online_node(nid)
                        for (zid = 0; zid < MAX_NR_ZONES; zid++) {
                                mz = mem_cgroup_zoneinfo(memcg, nid, zid);
+                               rstat = &mz->lruvec.reclaim_stat;
 
-                               recent_rotated[0] +=
-                                       mz->reclaim_stat.recent_rotated[0];
-                               recent_rotated[1] +=
-                                       mz->reclaim_stat.recent_rotated[1];
-                               recent_scanned[0] +=
-                                       mz->reclaim_stat.recent_scanned[0];
-                               recent_scanned[1] +=
-                                       mz->reclaim_stat.recent_scanned[1];
+                               recent_rotated[0] += rstat->recent_rotated[0];
+                               recent_rotated[1] += rstat->recent_rotated[1];
+                               recent_scanned[0] += rstat->recent_scanned[0];
+                               recent_scanned[1] += rstat->recent_scanned[1];
                        }
                cb->fill(cb, "recent_rotated_anon", recent_rotated[0]);
                cb->fill(cb, "recent_rotated_file", recent_rotated[1]);

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 4fc462b5fcf104f2edf0dee45026f91aea4ed910..8cbfc38e68ac914b6b661c0081bba4318018e522 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -4410,10 +4410,10 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
                zone_pcp_init(zone);
                for_each_lru(lru)
                        INIT_LIST_HEAD(&zone->lruvec.lists[lru]);
-               zone->reclaim_stat.recent_rotated[0] = 0;
-               zone->reclaim_stat.recent_rotated[1] = 0;
-               zone->reclaim_stat.recent_scanned[0] = 0;
-               zone->reclaim_stat.recent_scanned[1] = 0;
+               zone->lruvec.reclaim_stat.recent_rotated[0] = 0;
+               zone->lruvec.reclaim_stat.recent_rotated[1] = 0;
+               zone->lruvec.reclaim_stat.recent_scanned[0] = 0;
+               zone->lruvec.reclaim_stat.recent_scanned[1] = 0;
                zap_zone_vm_stats(zone);
                zone->flags = 0;
                if (!size)

diff --git a/mm/swap.c b/mm/swap.c
index 6fdd72ec15b01b4552480e02ad639de0a73d79de..0503ad705e7c0c65804ff2adaa8c9eb4432a45f7 100644 (file)
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -312,21 +312,15 @@ void rotate_reclaimable_page(struct page *page)
 static void update_page_reclaim_stat(struct zone *zone, struct page *page,
                                     int file, int rotated)
 {
-       struct zone_reclaim_stat *reclaim_stat = &zone->reclaim_stat;
-       struct zone_reclaim_stat *memcg_reclaim_stat;
+       struct zone_reclaim_stat *reclaim_stat;
 
-       memcg_reclaim_stat = mem_cgroup_get_reclaim_stat_from_page(page);
+       reclaim_stat = mem_cgroup_get_reclaim_stat_from_page(page);
+       if (!reclaim_stat)
+               reclaim_stat = &zone->lruvec.reclaim_stat;
 
        reclaim_stat->recent_scanned[file]++;
        if (rotated)
                reclaim_stat->recent_rotated[file]++;
-
-       if (!memcg_reclaim_stat)
-               return;
-
-       memcg_reclaim_stat->recent_scanned[file]++;
-       if (rotated)
-               memcg_reclaim_stat->recent_rotated[file]++;
 }
 
 static void __activate_page(struct page *page, void *arg)

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 52fac58b4461c2e3acb0591cdfebd0841bbdb2f3..e234ada187473dd66d8cc1923c9d951412839fb9 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -149,10 +149,7 @@ static bool global_reclaim(struct scan_control *sc)
 
 static struct zone_reclaim_stat *get_reclaim_stat(struct mem_cgroup_zone *mz)
 {
-       if (!mem_cgroup_disabled())
-               return mem_cgroup_get_reclaim_stat(mz->mem_cgroup, mz->zone);
-
-       return &mz->zone->reclaim_stat;
+       return &mem_cgroup_zone_lruvec(mz->zone, mz->mem_cgroup)->reclaim_stat;
 }
 
 static unsigned long zone_nr_lru_pages(struct mem_cgroup_zone *mz,