mm, vmscan: move lru_lock to the node

Node-based reclaim requires node-based LRUs and locking.  This is a
preparation patch that just moves the lru_lock to the node so later
patches are easier to review.  It is a mechanical change but note this
patch makes contention worse because the LRU lock is hotter and direct
reclaim and kswapd can contend on the same lock even when reclaiming
from different zones.

Link: http://lkml.kernel.org/r/1467970510-21195-3-git-send-email-mgorman@techsingularity.net
Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Reviewed-by: Minchan Kim <minchan@kernel.org>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Rik van Riel <riel@surriel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/Documentation/cgroup-v1/memcg_test.txt b/Documentation/cgroup-v1/memcg_test.txt
index 8870b02121502430a420e129b61e7d887ee0c762..78a8c2963b38816b152cb07c0f6b82a953b32eaa 100644 (file)
--- a/Documentation/cgroup-v1/memcg_test.txt
+++ b/Documentation/cgroup-v1/memcg_test.txt
@@ -107,9 +107,9 @@ Under below explanation, we assume CONFIG_MEM_RES_CTRL_SWAP=y.
 
 8. LRU
         Each memcg has its own private LRU. Now, its handling is under global
-       VM's control (means that it's handled under global zone->lru_lock).
+       VM's control (means that it's handled under global zone_lru_lock).
        Almost all routines around memcg's LRU is called by global LRU's
-       list management functions under zone->lru_lock().
+       list management functions under zone_lru_lock().
 
        A special function is mem_cgroup_isolate_pages(). This scans
        memcg's private LRU and call __isolate_lru_page() to extract a page

diff --git a/Documentation/cgroup-v1/memory.txt b/Documentation/cgroup-v1/memory.txt
index b14abf217239e892e19cd9f8c6366e3646ee640f..946e69103cdd78134d2071483aa12b22c699e670 100644 (file)
--- a/Documentation/cgroup-v1/memory.txt
+++ b/Documentation/cgroup-v1/memory.txt
@@ -267,11 +267,11 @@ When oom event notifier is registered, event will be delivered.
    Other lock order is following:
    PG_locked.
    mm->page_table_lock
-       zone->lru_lock
+       zone_lru_lock
          lock_page_cgroup.
   In many cases, just lock_page_cgroup() is called.
   per-zone-per-cgroup LRU (cgroup's private LRU) is just guarded by
-  zone->lru_lock, it has no lock of its own.
+  zone_lru_lock, it has no lock of its own.
 
 2.7 Kernel Memory Extension (CONFIG_MEMCG_KMEM)
 

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 79472b22d23f0866cd251e3e5e47c902522994e0..903200f4ec41ce03c50c15bfa079d5a01f903afa 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -118,7 +118,7 @@ struct page {
         */
        union {
                struct list_head lru;   /* Pageout list, eg. active_list
-                                        * protected by zone->lru_lock !
+                                        * protected by zone_lru_lock !
                                         * Can be used as a generic list
                                         * by the page owner.
                                         */

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 078ecb81e209821f814a59e48f0c4ad41cabdb4b..cfa870107abee2cd9f53512967b5dde56a7c5c8f 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -93,7 +93,7 @@ struct free_area {
 struct pglist_data;
 
 /*
- * zone->lock and zone->lru_lock are two of the hottest locks in the kernel.
+ * zone->lock and the zone lru_lock are two of the hottest locks in the kernel.
  * So add a wild amount of padding here to ensure that they fall into separate
  * cachelines.  There are very few zone structures in the machine, so space
  * consumption is not a concern here.
@@ -496,7 +496,6 @@ struct zone {
        /* Write-intensive fields used by page reclaim */
 
        /* Fields commonly accessed by the page reclaim scanner */
-       spinlock_t              lru_lock;
        struct lruvec           lruvec;
 
        /*
@@ -690,6 +689,9 @@ typedef struct pglist_data {
        /* Number of pages migrated during the rate limiting time interval */
        unsigned long numabalancing_migrate_nr_pages;
 #endif
+       /* Write-intensive fields used by page reclaim */
+       ZONE_PADDING(_pad1_)
+       spinlock_t              lru_lock;
 
 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
        /*
@@ -721,6 +723,10 @@ typedef struct pglist_data {
 
 #define node_start_pfn(nid)    (NODE_DATA(nid)->node_start_pfn)
 #define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid))
+static inline spinlock_t *zone_lru_lock(struct zone *zone)
+{
+       return &zone->zone_pgdat->lru_lock;
+}
 
 static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat)
 {

diff --git a/mm/compaction.c b/mm/compaction.c
index 45eaa2a56517e0c2d72c972d5b2c701fb2f370c0..5c65fad3f330f941bfc0e243de9cdc2696b92671 100644 (file)
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -752,7 +752,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                 * if contended.
                 */
                if (!(low_pfn % SWAP_CLUSTER_MAX)
-                   && compact_unlock_should_abort(&zone->lru_lock, flags,
+                   && compact_unlock_should_abort(zone_lru_lock(zone), flags,
                                                                &locked, cc))
                        break;
 
@@ -813,7 +813,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                        if (unlikely(__PageMovable(page)) &&
                                        !PageIsolated(page)) {
                                if (locked) {
-                                       spin_unlock_irqrestore(&zone->lru_lock,
+                                       spin_unlock_irqrestore(zone_lru_lock(zone),
                                                                        flags);
                                        locked = false;
                                }
@@ -836,7 +836,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
 
                /* If we already hold the lock, we can skip some rechecking */
                if (!locked) {
-                       locked = compact_trylock_irqsave(&zone->lru_lock,
+                       locked = compact_trylock_irqsave(zone_lru_lock(zone),
                                                                &flags, cc);
                        if (!locked)
                                break;
@@ -899,7 +899,7 @@ isolate_fail:
                 */
                if (nr_isolated) {
                        if (locked) {
-                               spin_unlock_irqrestore(&zone->lru_lock, flags);
+                               spin_unlock_irqrestore(zone_lru_lock(zone), flags);
                                locked = false;
                        }
                        acct_isolated(zone, cc);
@@ -927,7 +927,7 @@ isolate_fail:
                low_pfn = end_pfn;
 
        if (locked)
-               spin_unlock_irqrestore(&zone->lru_lock, flags);
+               spin_unlock_irqrestore(zone_lru_lock(zone), flags);
 
        /*
         * Update the pageblock-skip information and cached scanner pfn,

diff --git a/mm/filemap.c b/mm/filemap.c
index e90c1543ec2dc0fa577bf07c5a6eea824c0eecf6..7ec50bd6f88c5885631cad8dee3fc6a2008f99ff 100644 (file)
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -95,8 +95,8 @@
  *    ->swap_lock              (try_to_unmap_one)
  *    ->private_lock           (try_to_unmap_one)
  *    ->tree_lock              (try_to_unmap_one)
- *    ->zone.lru_lock          (follow_page->mark_page_accessed)
- *    ->zone.lru_lock          (check_pte_range->isolate_lru_page)
+ *    ->zone_lru_lock(zone)    (follow_page->mark_page_accessed)
+ *    ->zone_lru_lock(zone)    (check_pte_range->isolate_lru_page)
  *    ->private_lock           (page_remove_rmap->set_page_dirty)
  *    ->tree_lock              (page_remove_rmap->set_page_dirty)
  *    bdi.wb->list_lock                (page_remove_rmap->set_page_dirty)

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 3647334c2ef916ce9824153a359cb13cff3d9aba..99578b63814bfc1cae31455eda1bc77a9851b638 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1848,7 +1848,7 @@ static void __split_huge_page(struct page *page, struct list_head *list,
                spin_unlock(&head->mapping->tree_lock);
        }
 
-       spin_unlock_irqrestore(&page_zone(head)->lru_lock, flags);
+       spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags);
 
        unfreeze_page(head);
 
@@ -2034,7 +2034,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
                lru_add_drain();
 
        /* prevent PageLRU to go away from under us, and freeze lru stats */
-       spin_lock_irqsave(&page_zone(head)->lru_lock, flags);
+       spin_lock_irqsave(zone_lru_lock(page_zone(head)), flags);
 
        if (mapping) {
                void **pslot;
@@ -2077,7 +2077,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
                spin_unlock(&pgdata->split_queue_lock);
 fail:          if (mapping)
                        spin_unlock(&mapping->tree_lock);
-               spin_unlock_irqrestore(&page_zone(head)->lru_lock, flags);
+               spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags);
                unfreeze_page(head);
                ret = -EBUSY;
        }

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 40dfca3ef4bb13276bf5309df1fa161c530b00da..9b70f9ca8ddfabe8508dc3e31273d872494ca566 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2065,7 +2065,7 @@ static void lock_page_lru(struct page *page, int *isolated)
 {
        struct zone *zone = page_zone(page);
 
-       spin_lock_irq(&zone->lru_lock);
+       spin_lock_irq(zone_lru_lock(zone));
        if (PageLRU(page)) {
                struct lruvec *lruvec;
 
@@ -2089,7 +2089,7 @@ static void unlock_page_lru(struct page *page, int isolated)
                SetPageLRU(page);
                add_page_to_lru_list(page, lruvec, page_lru(page));
        }
-       spin_unlock_irq(&zone->lru_lock);
+       spin_unlock_irq(zone_lru_lock(zone));
 }
 
 static void commit_charge(struct page *page, struct mem_cgroup *memcg,
@@ -2389,7 +2389,7 @@ void memcg_kmem_uncharge(struct page *page, int order)
 
 /*
  * Because tail pages are not marked as "used", set it. We're under
- * zone->lru_lock and migration entries setup in all page mappings.
+ * zone_lru_lock and migration entries setup in all page mappings.
  */
 void mem_cgroup_split_huge_fixup(struct page *head)
 {

diff --git a/mm/mlock.c b/mm/mlock.c
index ef8dc9f395c4cb52e5e80c73f31654dc6936223b..997f63082ff583fed2153bf81472ad55b262b3f7 100644 (file)
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -188,7 +188,7 @@ unsigned int munlock_vma_page(struct page *page)
         * might otherwise copy PageMlocked to part of the tail pages before
         * we clear it in the head page. It also stabilizes hpage_nr_pages().
         */
-       spin_lock_irq(&zone->lru_lock);
+       spin_lock_irq(zone_lru_lock(zone));
 
        nr_pages = hpage_nr_pages(page);
        if (!TestClearPageMlocked(page))
@@ -197,14 +197,14 @@ unsigned int munlock_vma_page(struct page *page)
        __mod_zone_page_state(zone, NR_MLOCK, -nr_pages);
 
        if (__munlock_isolate_lru_page(page, true)) {
-               spin_unlock_irq(&zone->lru_lock);
+               spin_unlock_irq(zone_lru_lock(zone));
                __munlock_isolated_page(page);
                goto out;
        }
        __munlock_isolation_failed(page);
 
 unlock_out:
-       spin_unlock_irq(&zone->lru_lock);
+       spin_unlock_irq(zone_lru_lock(zone));
 
 out:
        return nr_pages - 1;
@@ -289,7 +289,7 @@ static void __munlock_pagevec(struct pagevec *pvec, struct zone *zone)
        pagevec_init(&pvec_putback, 0);
 
        /* Phase 1: page isolation */
-       spin_lock_irq(&zone->lru_lock);
+       spin_lock_irq(zone_lru_lock(zone));
        for (i = 0; i < nr; i++) {
                struct page *page = pvec->pages[i];
 
@@ -315,7 +315,7 @@ static void __munlock_pagevec(struct pagevec *pvec, struct zone *zone)
        }
        delta_munlocked = -nr + pagevec_count(&pvec_putback);
        __mod_zone_page_state(zone, NR_MLOCK, delta_munlocked);
-       spin_unlock_irq(&zone->lru_lock);
+       spin_unlock_irq(zone_lru_lock(zone));
 
        /* Now we can release pins of pages that we are not munlocking */
        pagevec_release(&pvec_putback);

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 7d4ff81b973f0fdc0527b2ceba09587c23cb7658..5760c626c3092e9e7966582aef1c68c45f4da2d2 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -5904,6 +5904,7 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat)
        init_waitqueue_head(&pgdat->kcompactd_wait);
 #endif
        pgdat_page_ext_init(pgdat);
+       spin_lock_init(&pgdat->lru_lock);
 
        for (j = 0; j < MAX_NR_ZONES; j++) {
                struct zone *zone = pgdat->node_zones + j;
@@ -5958,10 +5959,9 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat)
                zone->min_slab_pages = (freesize * sysctl_min_slab_ratio) / 100;
 #endif
                zone->name = zone_names[j];
+               zone->zone_pgdat = pgdat;
                spin_lock_init(&zone->lock);
-               spin_lock_init(&zone->lru_lock);
                zone_seqlock_init(zone);
-               zone->zone_pgdat = pgdat;
                zone_pcp_init(zone);
 
                /* For bootup, initialized properly in watermark setup */

diff --git a/mm/page_idle.c b/mm/page_idle.c
index 4ea9c4ef5146b8b784848a6710b70e5fc0dfcd41..ae11aa914e5569afbd5ee67f5349b81912121d02 100644 (file)
--- a/mm/page_idle.c
+++ b/mm/page_idle.c
@@ -41,12 +41,12 @@ static struct page *page_idle_get_page(unsigned long pfn)
                return NULL;
 
        zone = page_zone(page);
-       spin_lock_irq(&zone->lru_lock);
+       spin_lock_irq(zone_lru_lock(zone));
        if (unlikely(!PageLRU(page))) {
                put_page(page);
                page = NULL;
        }
-       spin_unlock_irq(&zone->lru_lock);
+       spin_unlock_irq(zone_lru_lock(zone));
        return page;
 }
 

diff --git a/mm/rmap.c b/mm/rmap.c
index 8a13d9f7b5666b153d64788576f46029bdc1d99f..dc28bfecbf80c1438779353db4a831beb9a72c44 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -27,7 +27,7 @@
  *         mapping->i_mmap_rwsem
  *           anon_vma->rwsem
  *             mm->page_table_lock or pte_lock
- *               zone->lru_lock (in mark_page_accessed, isolate_lru_page)
+ *               zone_lru_lock (in mark_page_accessed, isolate_lru_page)
  *               swap_lock (in swap_duplicate, swap_info_get)
  *                 mmlist_lock (in mmput, drain_mmlist and others)
  *                 mapping->private_lock (in __set_page_dirty_buffers)

diff --git a/mm/swap.c b/mm/swap.c
index 616df4ddd8708ee7dc4a16fcd6f4814256a86d82..bf37e5cfae812f6325af2cb5c0cb6933ebab2f7c 100644 (file)
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -62,12 +62,12 @@ static void __page_cache_release(struct page *page)
                struct lruvec *lruvec;
                unsigned long flags;
 
-               spin_lock_irqsave(&zone->lru_lock, flags);
+               spin_lock_irqsave(zone_lru_lock(zone), flags);
                lruvec = mem_cgroup_page_lruvec(page, zone);
                VM_BUG_ON_PAGE(!PageLRU(page), page);
                __ClearPageLRU(page);
                del_page_from_lru_list(page, lruvec, page_off_lru(page));
-               spin_unlock_irqrestore(&zone->lru_lock, flags);
+               spin_unlock_irqrestore(zone_lru_lock(zone), flags);
        }
        mem_cgroup_uncharge(page);
 }
@@ -189,16 +189,16 @@ static void pagevec_lru_move_fn(struct pagevec *pvec,
 
                if (pagezone != zone) {
                        if (zone)
-                               spin_unlock_irqrestore(&zone->lru_lock, flags);
+                               spin_unlock_irqrestore(zone_lru_lock(zone), flags);
                        zone = pagezone;
-                       spin_lock_irqsave(&zone->lru_lock, flags);
+                       spin_lock_irqsave(zone_lru_lock(zone), flags);
                }
 
                lruvec = mem_cgroup_page_lruvec(page, zone);
                (*move_fn)(page, lruvec, arg);
        }
        if (zone)
-               spin_unlock_irqrestore(&zone->lru_lock, flags);
+               spin_unlock_irqrestore(zone_lru_lock(zone), flags);
        release_pages(pvec->pages, pvec->nr, pvec->cold);
        pagevec_reinit(pvec);
 }
@@ -318,9 +318,9 @@ void activate_page(struct page *page)
        struct zone *zone = page_zone(page);
 
        page = compound_head(page);
-       spin_lock_irq(&zone->lru_lock);
+       spin_lock_irq(zone_lru_lock(zone));
        __activate_page(page, mem_cgroup_page_lruvec(page, zone), NULL);
-       spin_unlock_irq(&zone->lru_lock);
+       spin_unlock_irq(zone_lru_lock(zone));
 }
 #endif
 
@@ -448,13 +448,13 @@ void add_page_to_unevictable_list(struct page *page)
        struct zone *zone = page_zone(page);
        struct lruvec *lruvec;
 
-       spin_lock_irq(&zone->lru_lock);
+       spin_lock_irq(zone_lru_lock(zone));
        lruvec = mem_cgroup_page_lruvec(page, zone);
        ClearPageActive(page);
        SetPageUnevictable(page);
        SetPageLRU(page);
        add_page_to_lru_list(page, lruvec, LRU_UNEVICTABLE);
-       spin_unlock_irq(&zone->lru_lock);
+       spin_unlock_irq(zone_lru_lock(zone));
 }
 
 /**
@@ -744,7 +744,7 @@ void release_pages(struct page **pages, int nr, bool cold)
                 * same zone. The lock is held only if zone != NULL.
                 */
                if (zone && ++lock_batch == SWAP_CLUSTER_MAX) {
-                       spin_unlock_irqrestore(&zone->lru_lock, flags);
+                       spin_unlock_irqrestore(zone_lru_lock(zone), flags);
                        zone = NULL;
                }
 
@@ -759,7 +759,7 @@ void release_pages(struct page **pages, int nr, bool cold)
 
                if (PageCompound(page)) {
                        if (zone) {
-                               spin_unlock_irqrestore(&zone->lru_lock, flags);
+                               spin_unlock_irqrestore(zone_lru_lock(zone), flags);
                                zone = NULL;
                        }
                        __put_compound_page(page);
@@ -771,11 +771,11 @@ void release_pages(struct page **pages, int nr, bool cold)
 
                        if (pagezone != zone) {
                                if (zone)
-                                       spin_unlock_irqrestore(&zone->lru_lock,
+                                       spin_unlock_irqrestore(zone_lru_lock(zone),
                                                                        flags);
                                lock_batch = 0;
                                zone = pagezone;
-                               spin_lock_irqsave(&zone->lru_lock, flags);
+                               spin_lock_irqsave(zone_lru_lock(zone), flags);
                        }
 
                        lruvec = mem_cgroup_page_lruvec(page, zone);
@@ -790,7 +790,7 @@ void release_pages(struct page **pages, int nr, bool cold)
                list_add(&page->lru, &pages_to_free);
        }
        if (zone)
-               spin_unlock_irqrestore(&zone->lru_lock, flags);
+               spin_unlock_irqrestore(zone_lru_lock(zone), flags);
 
        mem_cgroup_uncharge_list(&pages_to_free);
        free_hot_cold_page_list(&pages_to_free, cold);
@@ -826,7 +826,7 @@ void lru_add_page_tail(struct page *page, struct page *page_tail,
        VM_BUG_ON_PAGE(PageCompound(page_tail), page);
        VM_BUG_ON_PAGE(PageLRU(page_tail), page);
        VM_BUG_ON(NR_CPUS != 1 &&
-                 !spin_is_locked(&lruvec_zone(lruvec)->lru_lock));
+                 !spin_is_locked(zone_lru_lock(lruvec_zone(lruvec))));
 
        if (!list)
                SetPageLRU(page_tail);

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 21d417ccff69d4efb8154d78c976b97e353a3609..e7ffcd259cc42bacacaaa7fccafd8b2af3a7e51f 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1349,7 +1349,7 @@ int __isolate_lru_page(struct page *page, isolate_mode_t mode)
 }
 
 /*
- * zone->lru_lock is heavily contended.  Some of the functions that
+ * zone_lru_lock is heavily contended.  Some of the functions that
  * shrink the lists perform better by taking out a batch of pages
  * and working on them outside the LRU lock.
  *
@@ -1444,7 +1444,7 @@ int isolate_lru_page(struct page *page)
                struct zone *zone = page_zone(page);
                struct lruvec *lruvec;
 
-               spin_lock_irq(&zone->lru_lock);
+               spin_lock_irq(zone_lru_lock(zone));
                lruvec = mem_cgroup_page_lruvec(page, zone);
                if (PageLRU(page)) {
                        int lru = page_lru(page);
@@ -1453,7 +1453,7 @@ int isolate_lru_page(struct page *page)
                        del_page_from_lru_list(page, lruvec, lru);
                        ret = 0;
                }
-               spin_unlock_irq(&zone->lru_lock);
+               spin_unlock_irq(zone_lru_lock(zone));
        }
        return ret;
 }
@@ -1512,9 +1512,9 @@ putback_inactive_pages(struct lruvec *lruvec, struct list_head *page_list)
                VM_BUG_ON_PAGE(PageLRU(page), page);
                list_del(&page->lru);
                if (unlikely(!page_evictable(page))) {
-                       spin_unlock_irq(&zone->lru_lock);
+                       spin_unlock_irq(zone_lru_lock(zone));
                        putback_lru_page(page);
-                       spin_lock_irq(&zone->lru_lock);
+                       spin_lock_irq(zone_lru_lock(zone));
                        continue;
                }
 
@@ -1535,10 +1535,10 @@ putback_inactive_pages(struct lruvec *lruvec, struct list_head *page_list)
                        del_page_from_lru_list(page, lruvec, lru);
 
                        if (unlikely(PageCompound(page))) {
-                               spin_unlock_irq(&zone->lru_lock);
+                               spin_unlock_irq(zone_lru_lock(zone));
                                mem_cgroup_uncharge(page);
                                (*get_compound_page_dtor(page))(page);
-                               spin_lock_irq(&zone->lru_lock);
+                               spin_lock_irq(zone_lru_lock(zone));
                        } else
                                list_add(&page->lru, &pages_to_free);
                }
@@ -1600,7 +1600,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
        if (!sc->may_writepage)
                isolate_mode |= ISOLATE_CLEAN;
 
-       spin_lock_irq(&zone->lru_lock);
+       spin_lock_irq(zone_lru_lock(zone));
 
        nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
                                     &nr_scanned, sc, isolate_mode, lru);
@@ -1616,7 +1616,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
                else
                        __count_zone_vm_events(PGSCAN_DIRECT, zone, nr_scanned);
        }
-       spin_unlock_irq(&zone->lru_lock);
+       spin_unlock_irq(zone_lru_lock(zone));
 
        if (nr_taken == 0)
                return 0;
@@ -1626,7 +1626,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
                                &nr_writeback, &nr_immediate,
                                false);
 
-       spin_lock_irq(&zone->lru_lock);
+       spin_lock_irq(zone_lru_lock(zone));
 
        if (global_reclaim(sc)) {
                if (current_is_kswapd())
@@ -1641,7 +1641,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
 
        __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, -nr_taken);
 
-       spin_unlock_irq(&zone->lru_lock);
+       spin_unlock_irq(zone_lru_lock(zone));
 
        mem_cgroup_uncharge_list(&page_list);
        free_hot_cold_page_list(&page_list, true);
@@ -1715,9 +1715,9 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
  * processes, from rmap.
  *
  * If the pages are mostly unmapped, the processing is fast and it is
- * appropriate to hold zone->lru_lock across the whole operation.  But if
+ * appropriate to hold zone_lru_lock across the whole operation.  But if
  * the pages are mapped, the processing is slow (page_referenced()) so we
- * should drop zone->lru_lock around each page.  It's impossible to balance
+ * should drop zone_lru_lock around each page.  It's impossible to balance
  * this, so instead we remove the pages from the LRU while processing them.
  * It is safe to rely on PG_active against the non-LRU pages in here because
  * nobody will play with that bit on a non-LRU page.
@@ -1754,10 +1754,10 @@ static void move_active_pages_to_lru(struct lruvec *lruvec,
                        del_page_from_lru_list(page, lruvec, lru);
 
                        if (unlikely(PageCompound(page))) {
-                               spin_unlock_irq(&zone->lru_lock);
+                               spin_unlock_irq(zone_lru_lock(zone));
                                mem_cgroup_uncharge(page);
                                (*get_compound_page_dtor(page))(page);
-                               spin_lock_irq(&zone->lru_lock);
+                               spin_lock_irq(zone_lru_lock(zone));
                        } else
                                list_add(&page->lru, pages_to_free);
                }
@@ -1792,7 +1792,7 @@ static void shrink_active_list(unsigned long nr_to_scan,
        if (!sc->may_writepage)
                isolate_mode |= ISOLATE_CLEAN;
 
-       spin_lock_irq(&zone->lru_lock);
+       spin_lock_irq(zone_lru_lock(zone));
 
        nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,
                                     &nr_scanned, sc, isolate_mode, lru);
@@ -1805,7 +1805,7 @@ static void shrink_active_list(unsigned long nr_to_scan,
                __mod_zone_page_state(zone, NR_PAGES_SCANNED, nr_scanned);
        __count_zone_vm_events(PGREFILL, zone, nr_scanned);
 
-       spin_unlock_irq(&zone->lru_lock);
+       spin_unlock_irq(zone_lru_lock(zone));
 
        while (!list_empty(&l_hold)) {
                cond_resched();
@@ -1850,7 +1850,7 @@ static void shrink_active_list(unsigned long nr_to_scan,
        /*
         * Move pages back to the lru list.
         */
-       spin_lock_irq(&zone->lru_lock);
+       spin_lock_irq(zone_lru_lock(zone));
        /*
         * Count referenced pages from currently used mappings as rotated,
         * even though only some of them are actually re-activated.  This
@@ -1862,7 +1862,7 @@ static void shrink_active_list(unsigned long nr_to_scan,
        move_active_pages_to_lru(lruvec, &l_active, &l_hold, lru);
        move_active_pages_to_lru(lruvec, &l_inactive, &l_hold, lru - LRU_ACTIVE);
        __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, -nr_taken);
-       spin_unlock_irq(&zone->lru_lock);
+       spin_unlock_irq(zone_lru_lock(zone));
 
        mem_cgroup_uncharge_list(&l_hold);
        free_hot_cold_page_list(&l_hold, true);
@@ -2077,7 +2077,7 @@ static void get_scan_count(struct lruvec *lruvec, struct mem_cgroup *memcg,
        file  = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE) +
                lruvec_lru_size(lruvec, LRU_INACTIVE_FILE);
 
-       spin_lock_irq(&zone->lru_lock);
+       spin_lock_irq(zone_lru_lock(zone));
        if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) {
                reclaim_stat->recent_scanned[0] /= 2;
                reclaim_stat->recent_rotated[0] /= 2;
@@ -2098,7 +2098,7 @@ static void get_scan_count(struct lruvec *lruvec, struct mem_cgroup *memcg,
 
        fp = file_prio * (reclaim_stat->recent_scanned[1] + 1);
        fp /= reclaim_stat->recent_rotated[1] + 1;
-       spin_unlock_irq(&zone->lru_lock);
+       spin_unlock_irq(zone_lru_lock(zone));
 
        fraction[0] = ap;
        fraction[1] = fp;
@@ -3791,9 +3791,9 @@ void check_move_unevictable_pages(struct page **pages, int nr_pages)
                pagezone = page_zone(page);
                if (pagezone != zone) {
                        if (zone)
-                               spin_unlock_irq(&zone->lru_lock);
+                               spin_unlock_irq(zone_lru_lock(zone));
                        zone = pagezone;
-                       spin_lock_irq(&zone->lru_lock);
+                       spin_lock_irq(zone_lru_lock(zone));
                }
                lruvec = mem_cgroup_page_lruvec(page, zone);
 
@@ -3814,7 +3814,7 @@ void check_move_unevictable_pages(struct page **pages, int nr_pages)
        if (zone) {
                __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
                __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
-               spin_unlock_irq(&zone->lru_lock);
+               spin_unlock_irq(zone_lru_lock(zone));
        }
 }
 #endif /* CONFIG_SHMEM */