f2fs: split free nid list

commit b8559dc242d1d47dcf99660a4d6afded727e0cc0 upstream.

During free nid allocation, in order to do preallocation, we will tag free
nid entry as allocated one and still leave it in free nid list, for other
allocators who want to grab free nids, it needs to traverse the free nid
list for lookup. It becomes overhead in scenario of allocating free nid
intensively by multithreads.

This patch splits free nid list to two list: {free,alloc}_nid_list, to
keep free nids and preallocated free nids separately, after that, traverse
latency will be gone, besides split nid_cnt for separate statistic.

Additionally, introduce __insert_nid_to_list and __remove_nid_from_list for
cleanup.

Signed-off-by: Chao Yu <yuchao0@huawei.com>
[Jaegeuk Kim: modify f2fs_bug_on to avoid needless branches]
Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>

diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c
index 1c35e80732e02b7228d43de72934d4595fb95152..5cab80dfa4dc22c599ab387c3b5af9eb9a73ce74 100644 (file)
--- a/fs/f2fs/debug.c
+++ b/fs/f2fs/debug.c
@@ -74,7 +74,8 @@ static void update_general_status(struct f2fs_sb_info *sbi)
        si->dirty_nats = NM_I(sbi)->dirty_nat_cnt;
        si->sits = MAIN_SEGS(sbi);
        si->dirty_sits = SIT_I(sbi)->dirty_sentries;
-       si->fnids = NM_I(sbi)->fcnt;
+       si->free_nids = NM_I(sbi)->nid_cnt[FREE_NID_LIST];
+       si->alloc_nids = NM_I(sbi)->nid_cnt[ALLOC_NID_LIST];
        si->bg_gc = sbi->bg_gc;
        si->util_free = (int)(free_user_blocks(sbi) >> sbi->log_blocks_per_seg)
                * 100 / (int)(sbi->user_block_count >> sbi->log_blocks_per_seg)
@@ -194,7 +195,9 @@ get_cache:
                si->cache_mem += sizeof(struct flush_cmd_control);
 
        /* free nids */
-       si->cache_mem += NM_I(sbi)->fcnt * sizeof(struct free_nid);
+       si->cache_mem += (NM_I(sbi)->nid_cnt[FREE_NID_LIST] +
+                               NM_I(sbi)->nid_cnt[ALLOC_NID_LIST]) *
+                               sizeof(struct free_nid);
        si->cache_mem += NM_I(sbi)->nat_cnt * sizeof(struct nat_entry);
        si->cache_mem += NM_I(sbi)->dirty_nat_cnt *
                                        sizeof(struct nat_entry_set);
@@ -324,8 +327,8 @@ static int stat_show(struct seq_file *s, void *v)
                           si->ndirty_imeta);
                seq_printf(s, "  - NATs: %9d/%9d\n  - SITs: %9d/%9d\n",
                           si->dirty_nats, si->nats, si->dirty_sits, si->sits);
-               seq_printf(s, "  - free_nids: %9d\n",
-                          si->fnids);
+               seq_printf(s, "  - free_nids: %9d, alloc_nids: %9d\n",
+                          si->free_nids, si->alloc_nids);
                seq_puts(s, "\nDistribution of User Blocks:");
                seq_puts(s, " [ valid | invalid | free ]\n");
                seq_puts(s, "  [");

diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
index 45d1e4522760937cf7ce896a58a1e61b841646db..cec025852c221143a10daafeb0a3e19a0f6ffe58 100644 (file)
--- a/fs/f2fs/f2fs.h
+++ b/fs/f2fs/f2fs.h
@@ -529,6 +529,12 @@ static inline void __try_update_largest_extent(struct inode *inode,
        }
 }
 
+enum nid_list {
+       FREE_NID_LIST,
+       ALLOC_NID_LIST,
+       MAX_NID_LIST,
+};
+
 struct f2fs_nm_info {
        block_t nat_blkaddr;            /* base disk address of NAT */
        nid_t max_nid;                  /* maximum possible node ids */
@@ -548,9 +554,9 @@ struct f2fs_nm_info {
 
        /* free node ids management */
        struct radix_tree_root free_nid_root;/* root of the free_nid cache */
-       struct list_head free_nid_list; /* a list for free nids */
-       spinlock_t free_nid_list_lock;  /* protect free nid list */
-       unsigned int fcnt;              /* the number of free node id */
+       struct list_head nid_list[MAX_NID_LIST];/* lists for free nids */
+       unsigned int nid_cnt[MAX_NID_LIST];     /* the number of free node id */
+       spinlock_t nid_list_lock;       /* protect nid lists ops */
        struct mutex build_lock;        /* lock for build free nids */
 
        /* for checkpoint */
@@ -2214,7 +2220,7 @@ struct f2fs_stat_info {
        s64 ndirty_node, ndirty_dent, ndirty_meta, ndirty_data, ndirty_imeta;
        s64 inmem_pages;
        unsigned int ndirty_dirs, ndirty_files, ndirty_all;
-       int nats, dirty_nats, sits, dirty_sits, fnids;
+       int nats, dirty_nats, sits, dirty_sits, free_nids, alloc_nids;
        int total_count, utilization;
        int bg_gc, wb_bios;
        int inline_xattr, inline_inode, inline_dir, orphans;

diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c
index d2ba37a84f8e826f57eea4791ed5d83280ebee52..5bb2fa324e685fcf8ac6dabc078b6eae26bd9eae 100644 (file)
--- a/fs/f2fs/node.c
+++ b/fs/f2fs/node.c
@@ -45,8 +45,8 @@ bool available_free_memory(struct f2fs_sb_info *sbi, int type)
         * give 25%, 25%, 50%, 50%, 50% memory for each components respectively
         */
        if (type == FREE_NIDS) {
-               mem_size = (nm_i->fcnt * sizeof(struct free_nid)) >>
-                                                       PAGE_SHIFT;
+               mem_size = (nm_i->nid_cnt[FREE_NID_LIST] *
+                               sizeof(struct free_nid)) >> PAGE_SHIFT;
                res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);
        } else if (type == NAT_ENTRIES) {
                mem_size = (nm_i->nat_cnt * sizeof(struct nat_entry)) >>
@@ -1699,10 +1699,31 @@ static struct free_nid *__lookup_free_nid_list(struct f2fs_nm_info *nm_i,
 static void __del_from_free_nid_list(struct f2fs_nm_info *nm_i,
                                                struct free_nid *i)
 {
-       list_del(&i->list);
        radix_tree_delete(&nm_i->free_nid_root, i->nid);
 }
 
+static void __insert_nid_to_list(struct f2fs_sb_info *sbi,
+                                       struct free_nid *i, enum nid_list list)
+{
+       struct f2fs_nm_info *nm_i = NM_I(sbi);
+
+       f2fs_bug_on(sbi, list == FREE_NID_LIST ? i->state != NID_NEW :
+                                               i->state != NID_ALLOC);
+       nm_i->nid_cnt[list]++;
+       list_add_tail(&i->list, &nm_i->nid_list[list]);
+}
+
+static void __remove_nid_from_list(struct f2fs_sb_info *sbi,
+                                       struct free_nid *i, enum nid_list list)
+{
+       struct f2fs_nm_info *nm_i = NM_I(sbi);
+
+       f2fs_bug_on(sbi, list == FREE_NID_LIST ? i->state != NID_NEW :
+                                               i->state != NID_ALLOC);
+       nm_i->nid_cnt[list]--;
+       list_del(&i->list);
+}
+
 static int add_free_nid(struct f2fs_sb_info *sbi, nid_t nid, bool build)
 {
        struct f2fs_nm_info *nm_i = NM_I(sbi);
@@ -1733,33 +1754,33 @@ static int add_free_nid(struct f2fs_sb_info *sbi, nid_t nid, bool build)
                return 0;
        }
 
-       spin_lock(&nm_i->free_nid_list_lock);
+       spin_lock(&nm_i->nid_list_lock);
        if (radix_tree_insert(&nm_i->free_nid_root, i->nid, i)) {
-               spin_unlock(&nm_i->free_nid_list_lock);
+               spin_unlock(&nm_i->nid_list_lock);
                radix_tree_preload_end();
                kmem_cache_free(free_nid_slab, i);
                return 0;
        }
-       list_add_tail(&i->list, &nm_i->free_nid_list);
-       nm_i->fcnt++;
-       spin_unlock(&nm_i->free_nid_list_lock);
+       __insert_nid_to_list(sbi, i, FREE_NID_LIST);
+       spin_unlock(&nm_i->nid_list_lock);
        radix_tree_preload_end();
        return 1;
 }
 
-static void remove_free_nid(struct f2fs_nm_info *nm_i, nid_t nid)
+static void remove_free_nid(struct f2fs_sb_info *sbi, nid_t nid)
 {
+       struct f2fs_nm_info *nm_i = NM_I(sbi);
        struct free_nid *i;
        bool need_free = false;
 
-       spin_lock(&nm_i->free_nid_list_lock);
+       spin_lock(&nm_i->nid_list_lock);
        i = __lookup_free_nid_list(nm_i, nid);
        if (i && i->state == NID_NEW) {
+               __remove_nid_from_list(sbi, i, FREE_NID_LIST);
                __del_from_free_nid_list(nm_i, i);
-               nm_i->fcnt--;
                need_free = true;
        }
-       spin_unlock(&nm_i->free_nid_list_lock);
+       spin_unlock(&nm_i->nid_list_lock);
 
        if (need_free)
                kmem_cache_free(free_nid_slab, i);
@@ -1798,7 +1819,7 @@ void __build_free_nids(struct f2fs_sb_info *sbi)
        nid_t nid = nm_i->next_scan_nid;
 
        /* Enough entries */
-       if (nm_i->fcnt >= NAT_ENTRY_PER_BLOCK)
+       if (nm_i->nid_cnt[FREE_NID_LIST] >= NAT_ENTRY_PER_BLOCK)
                return;
 
        /* readahead nat pages to be scanned */
@@ -1834,7 +1855,7 @@ void __build_free_nids(struct f2fs_sb_info *sbi)
                if (addr == NULL_ADDR)
                        add_free_nid(sbi, nid, true);
                else
-                       remove_free_nid(nm_i, nid);
+                       remove_free_nid(sbi, nid);
        }
        up_read(&curseg->journal_rwsem);
        up_read(&nm_i->nat_tree_lock);
@@ -1867,23 +1888,22 @@ retry:
        if (unlikely(sbi->total_valid_node_count + 1 > nm_i->available_nids))
                return false;
 
-       spin_lock(&nm_i->free_nid_list_lock);
+       spin_lock(&nm_i->nid_list_lock);
 
        /* We should not use stale free nids created by build_free_nids */
-       if (nm_i->fcnt && !on_build_free_nids(nm_i)) {
-               f2fs_bug_on(sbi, list_empty(&nm_i->free_nid_list));
-               list_for_each_entry(i, &nm_i->free_nid_list, list)
-                       if (i->state == NID_NEW)
-                               break;
-
-               f2fs_bug_on(sbi, i->state != NID_NEW);
+       if (nm_i->nid_cnt[FREE_NID_LIST] && !on_build_free_nids(nm_i)) {
+               f2fs_bug_on(sbi, list_empty(&nm_i->nid_list[FREE_NID_LIST]));
+               i = list_first_entry(&nm_i->nid_list[FREE_NID_LIST],
+                                       struct free_nid, list);
                *nid = i->nid;
+
+               __remove_nid_from_list(sbi, i, FREE_NID_LIST);
                i->state = NID_ALLOC;
-               nm_i->fcnt--;
-               spin_unlock(&nm_i->free_nid_list_lock);
+               __insert_nid_to_list(sbi, i, ALLOC_NID_LIST);
+               spin_unlock(&nm_i->nid_list_lock);
                return true;
        }
-       spin_unlock(&nm_i->free_nid_list_lock);
+       spin_unlock(&nm_i->nid_list_lock);
 
        /* Let's scan nat pages and its caches to get free nids */
        build_free_nids(sbi);
@@ -1898,11 +1918,12 @@ void alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid)
        struct f2fs_nm_info *nm_i = NM_I(sbi);
        struct free_nid *i;
 
-       spin_lock(&nm_i->free_nid_list_lock);
+       spin_lock(&nm_i->nid_list_lock);
        i = __lookup_free_nid_list(nm_i, nid);
-       f2fs_bug_on(sbi, !i || i->state != NID_ALLOC);
+       f2fs_bug_on(sbi, !i);
+       __remove_nid_from_list(sbi, i, ALLOC_NID_LIST);
        __del_from_free_nid_list(nm_i, i);
-       spin_unlock(&nm_i->free_nid_list_lock);
+       spin_unlock(&nm_i->nid_list_lock);
 
        kmem_cache_free(free_nid_slab, i);
 }
@@ -1919,17 +1940,20 @@ void alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid)
        if (!nid)
                return;
 
-       spin_lock(&nm_i->free_nid_list_lock);
+       spin_lock(&nm_i->nid_list_lock);
        i = __lookup_free_nid_list(nm_i, nid);
-       f2fs_bug_on(sbi, !i || i->state != NID_ALLOC);
+       f2fs_bug_on(sbi, !i);
+
+       __remove_nid_from_list(sbi, i, ALLOC_NID_LIST);
+
        if (!available_free_memory(sbi, FREE_NIDS)) {
                __del_from_free_nid_list(nm_i, i);
                need_free = true;
        } else {
                i->state = NID_NEW;
-               nm_i->fcnt++;
+               __insert_nid_to_list(sbi, i, FREE_NID_LIST);
        }
-       spin_unlock(&nm_i->free_nid_list_lock);
+       spin_unlock(&nm_i->nid_list_lock);
 
        if (need_free)
                kmem_cache_free(free_nid_slab, i);
@@ -1941,24 +1965,26 @@ int try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink)
        struct free_nid *i, *next;
        int nr = nr_shrink;
 
-       if (nm_i->fcnt <= MAX_FREE_NIDS)
+       if (nm_i->nid_cnt[FREE_NID_LIST] <= MAX_FREE_NIDS)
                return 0;
 
        if (!mutex_trylock(&nm_i->build_lock))
                return 0;
 
-       spin_lock(&nm_i->free_nid_list_lock);
-       list_for_each_entry_safe(i, next, &nm_i->free_nid_list, list) {
-               if (nr_shrink <= 0 || nm_i->fcnt <= MAX_FREE_NIDS)
+       spin_lock(&nm_i->nid_list_lock);
+       list_for_each_entry_safe(i, next, &nm_i->nid_list[FREE_NID_LIST],
+                                                                       list) {
+               if (nr_shrink <= 0 ||
+                               nm_i->nid_cnt[FREE_NID_LIST] <= MAX_FREE_NIDS)
                        break;
-               if (i->state == NID_ALLOC)
-                       continue;
+
+               __remove_nid_from_list(sbi, i, FREE_NID_LIST);
                __del_from_free_nid_list(nm_i, i);
+
                kmem_cache_free(free_nid_slab, i);
-               nm_i->fcnt--;
                nr_shrink--;
        }
-       spin_unlock(&nm_i->free_nid_list_lock);
+       spin_unlock(&nm_i->nid_list_lock);
        mutex_unlock(&nm_i->build_lock);
 
        return nr - nr_shrink;
@@ -2014,7 +2040,7 @@ recover_xnid:
        if (unlikely(!inc_valid_node_count(sbi, inode)))
                f2fs_bug_on(sbi, 1);
 
-       remove_free_nid(NM_I(sbi), new_xnid);
+       remove_free_nid(sbi, new_xnid);
        get_node_info(sbi, new_xnid, &ni);
        ni.ino = inode->i_ino;
        set_node_addr(sbi, &ni, NEW_ADDR, false);
@@ -2044,7 +2070,7 @@ retry:
        }
 
        /* Should not use this inode from free nid list */
-       remove_free_nid(NM_I(sbi), ino);
+       remove_free_nid(sbi, ino);
 
        if (!PageUptodate(ipage))
                SetPageUptodate(ipage);
@@ -2278,20 +2304,22 @@ static int init_node_manager(struct f2fs_sb_info *sbi)
 
        /* not used nids: 0, node, meta, (and root counted as valid node) */
        nm_i->available_nids = nm_i->max_nid - F2FS_RESERVED_NODE_NUM;
-       nm_i->fcnt = 0;
+       nm_i->nid_cnt[FREE_NID_LIST] = 0;
+       nm_i->nid_cnt[ALLOC_NID_LIST] = 0;
        nm_i->nat_cnt = 0;
        nm_i->ram_thresh = DEF_RAM_THRESHOLD;
        nm_i->ra_nid_pages = DEF_RA_NID_PAGES;
        nm_i->dirty_nats_ratio = DEF_DIRTY_NAT_RATIO_THRESHOLD;
 
        INIT_RADIX_TREE(&nm_i->free_nid_root, GFP_ATOMIC);
-       INIT_LIST_HEAD(&nm_i->free_nid_list);
+       INIT_LIST_HEAD(&nm_i->nid_list[FREE_NID_LIST]);
+       INIT_LIST_HEAD(&nm_i->nid_list[ALLOC_NID_LIST]);
        INIT_RADIX_TREE(&nm_i->nat_root, GFP_NOIO);
        INIT_RADIX_TREE(&nm_i->nat_set_root, GFP_NOIO);
        INIT_LIST_HEAD(&nm_i->nat_entries);
 
        mutex_init(&nm_i->build_lock);
-       spin_lock_init(&nm_i->free_nid_list_lock);
+       spin_lock_init(&nm_i->nid_list_lock);
        init_rwsem(&nm_i->nat_tree_lock);
 
        nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid);
@@ -2336,17 +2364,19 @@ void destroy_node_manager(struct f2fs_sb_info *sbi)
                return;
 
        /* destroy free nid list */
-       spin_lock(&nm_i->free_nid_list_lock);
-       list_for_each_entry_safe(i, next_i, &nm_i->free_nid_list, list) {
-               f2fs_bug_on(sbi, i->state == NID_ALLOC);
+       spin_lock(&nm_i->nid_list_lock);
+       list_for_each_entry_safe(i, next_i, &nm_i->nid_list[FREE_NID_LIST],
+                                                                       list) {
+               __remove_nid_from_list(sbi, i, FREE_NID_LIST);
                __del_from_free_nid_list(nm_i, i);
-               nm_i->fcnt--;
-               spin_unlock(&nm_i->free_nid_list_lock);
+               spin_unlock(&nm_i->nid_list_lock);
                kmem_cache_free(free_nid_slab, i);
-               spin_lock(&nm_i->free_nid_list_lock);
+               spin_lock(&nm_i->nid_list_lock);
        }
-       f2fs_bug_on(sbi, nm_i->fcnt);
-       spin_unlock(&nm_i->free_nid_list_lock);
+       f2fs_bug_on(sbi, nm_i->nid_cnt[FREE_NID_LIST]);
+       f2fs_bug_on(sbi, nm_i->nid_cnt[ALLOC_NID_LIST]);
+       f2fs_bug_on(sbi, !list_empty(&nm_i->nid_list[ALLOC_NID_LIST]));
+       spin_unlock(&nm_i->nid_list_lock);
 
        /* destroy nat cache */
        down_write(&nm_i->nat_tree_lock);

diff --git a/fs/f2fs/node.h b/fs/f2fs/node.h
index cfdcf98516a1f911ebd60f158c073b40e16f6bc3..e7997e2403660799de6ed30605f5b090e938fd5f 100644 (file)
--- a/fs/f2fs/node.h
+++ b/fs/f2fs/node.h
@@ -169,14 +169,15 @@ static inline void next_free_nid(struct f2fs_sb_info *sbi, nid_t *nid)
        struct f2fs_nm_info *nm_i = NM_I(sbi);
        struct free_nid *fnid;
 
-       spin_lock(&nm_i->free_nid_list_lock);
-       if (nm_i->fcnt <= 0) {
-               spin_unlock(&nm_i->free_nid_list_lock);
+       spin_lock(&nm_i->nid_list_lock);
+       if (nm_i->nid_cnt[FREE_NID_LIST] <= 0) {
+               spin_unlock(&nm_i->nid_list_lock);
                return;
        }
-       fnid = list_entry(nm_i->free_nid_list.next, struct free_nid, list);
+       fnid = list_entry(nm_i->nid_list[FREE_NID_LIST].next,
+                                               struct free_nid, list);
        *nid = fnid->nid;
-       spin_unlock(&nm_i->free_nid_list_lock);
+       spin_unlock(&nm_i->nid_list_lock);
 }
 
 /*

diff --git a/fs/f2fs/shrinker.c b/fs/f2fs/shrinker.c
index 46c9154259239f284cb3a8a73c20ec7ded2356c4..ec539f407cc49ab7225a748be8e27ea536e3bc43 100644 (file)
--- a/fs/f2fs/shrinker.c
+++ b/fs/f2fs/shrinker.c
@@ -26,8 +26,8 @@ static unsigned long __count_nat_entries(struct f2fs_sb_info *sbi)
 
 static unsigned long __count_free_nids(struct f2fs_sb_info *sbi)
 {
-       if (NM_I(sbi)->fcnt > MAX_FREE_NIDS)
-               return NM_I(sbi)->fcnt - MAX_FREE_NIDS;
+       if (NM_I(sbi)->nid_cnt[FREE_NID_LIST] > MAX_FREE_NIDS)
+               return NM_I(sbi)->nid_cnt[FREE_NID_LIST] - MAX_FREE_NIDS;
        return 0;
 }