f2fs: introduce io_list for serialize data/node IOs

Serialize data/node IOs by using fifo list instead of mutex lock,
it will help to enhance concurrency of f2fs, meanwhile keeping LFS
IO semantics.

Signed-off-by: Chao Yu <yuchao0@huawei.com>
Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>

diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c
index 13828f63a8710753c117e20ef570e5c53ef8e639..12559a4b6c244470364f739dcda97585d2a99cd1 100644 (file)
--- a/fs/f2fs/checkpoint.c
+++ b/fs/f2fs/checkpoint.c
@@ -162,6 +162,7 @@ int ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
                .op = REQ_OP_READ,
                .op_flags = sync ? (REQ_META | REQ_PRIO) : REQ_RAHEAD,
                .encrypted_page = NULL,
+               .in_list = false,
        };
        struct blk_plug plug;
 

diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index 779a306858a22a414139c7247a17d76b02190ba0..2ed90f5db83201d7a71a26146f0b69ca1e888c27 100644 (file)
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -397,6 +397,20 @@ int f2fs_submit_page_write(struct f2fs_io_info *fio)
 
        f2fs_bug_on(sbi, is_read_io(fio->op));
 
+       down_write(&io->io_rwsem);
+next:
+       if (fio->in_list) {
+               spin_lock(&io->io_lock);
+               if (list_empty(&io->io_list)) {
+                       spin_unlock(&io->io_lock);
+                       goto out_fail;
+               }
+               fio = list_first_entry(&io->io_list,
+                                               struct f2fs_io_info, list);
+               list_del(&fio->list);
+               spin_unlock(&io->io_lock);
+       }
+
        if (fio->old_blkaddr != NEW_ADDR)
                verify_block_addr(sbi, fio->old_blkaddr);
        verify_block_addr(sbi, fio->new_blkaddr);
@@ -408,8 +422,6 @@ int f2fs_submit_page_write(struct f2fs_io_info *fio)
 
        inc_page_count(sbi, WB_DATA_TYPE(bio_page));
 
-       down_write(&io->io_rwsem);
-
        if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 ||
            (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) ||
                        !__same_bdev(sbi, fio->new_blkaddr, io->bio)))
@@ -434,9 +446,13 @@ alloc_new:
 
        io->last_block_in_bio = fio->new_blkaddr;
        f2fs_trace_ios(fio, 0);
+
+       trace_f2fs_submit_page_write(fio->page, fio);
+
+       if (fio->in_list)
+               goto next;
 out_fail:
        up_write(&io->io_rwsem);
-       trace_f2fs_submit_page_write(fio->page, fio);
        return err;
 }
 
@@ -749,7 +765,7 @@ alloc:
        set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
 
        allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr,
-                                               &sum, CURSEG_WARM_DATA);
+                                       &sum, CURSEG_WARM_DATA, NULL, false);
        set_data_blkaddr(dn);
 
        /* update i_size */

diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
index 093d68a7ae4783f8542d9003f6c3c9067c77bc3e..cd777cf30be2a88fbd9ed6fb885a82fc2fe610c2 100644 (file)
--- a/fs/f2fs/f2fs.h
+++ b/fs/f2fs/f2fs.h
@@ -815,8 +815,10 @@ struct f2fs_io_info {
        block_t old_blkaddr;    /* old block address before Cow */
        struct page *page;      /* page to be written */
        struct page *encrypted_page;    /* encrypted page */
+       struct list_head list;          /* serialize IOs */
        bool submitted;         /* indicate IO submission */
        int need_lock;          /* indicate we need to lock cp_rwsem */
+       bool in_list;           /* indicate fio is in io_list */
 };
 
 #define is_read_io(rw) ((rw) == READ)
@@ -826,6 +828,8 @@ struct f2fs_bio_info {
        sector_t last_block_in_bio;     /* last block number */
        struct f2fs_io_info fio;        /* store buffered io info. */
        struct rw_semaphore io_rwsem;   /* blocking op for bio */
+       spinlock_t io_lock;             /* serialize DATA/NODE IOs */
+       struct list_head io_list;       /* track fios */
 };
 
 #define FDEV(i)                                (sbi->devs[i])
@@ -2294,7 +2298,8 @@ void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
                        bool recover_newaddr);
 void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
                        block_t old_blkaddr, block_t *new_blkaddr,
-                       struct f2fs_summary *sum, int type);
+                       struct f2fs_summary *sum, int type,
+                       struct f2fs_io_info *fio, bool add_list);
 void f2fs_wait_on_page_writeback(struct page *page,
                        enum page_type type, bool ordered);
 void f2fs_wait_on_encrypted_page_writeback(struct f2fs_sb_info *sbi,

diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c
index 570480571d72238ddd39ef6189e9df6a64f83c5c..fa3d2e2df8e70e883011afc5e46c9989e0e834ff 100644 (file)
--- a/fs/f2fs/gc.c
+++ b/fs/f2fs/gc.c
@@ -600,6 +600,7 @@ static void move_encrypted_block(struct inode *inode, block_t bidx,
                .op = REQ_OP_READ,
                .op_flags = 0,
                .encrypted_page = NULL,
+               .in_list = false,
        };
        struct dnode_of_data dn;
        struct f2fs_summary sum;
@@ -643,7 +644,7 @@ static void move_encrypted_block(struct inode *inode, block_t bidx,
        fio.new_blkaddr = fio.old_blkaddr = dn.data_blkaddr;
 
        allocate_data_block(fio.sbi, NULL, fio.old_blkaddr, &newaddr,
-                                                       &sum, CURSEG_COLD_DATA);
+                                       &sum, CURSEG_COLD_DATA, NULL, false);
 
        fio.encrypted_page = pagecache_get_page(META_MAPPING(fio.sbi), newaddr,
                                        FGP_LOCK | FGP_CREAT, GFP_NOFS);

diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c
index 63850e023c106e4f53df8862f057483738db4778..a3766bc1ba4d9bed5d6e84da546a011ab8562297 100644 (file)
--- a/fs/f2fs/segment.c
+++ b/fs/f2fs/segment.c
@@ -2109,7 +2109,8 @@ static int __get_segment_type(struct f2fs_io_info *fio)
 
 void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
                block_t old_blkaddr, block_t *new_blkaddr,
-               struct f2fs_summary *sum, int type)
+               struct f2fs_summary *sum, int type,
+               struct f2fs_io_info *fio, bool add_list)
 {
        struct sit_info *sit_i = SIT_I(sbi);
        struct curseg_info *curseg = CURSEG_I(sbi, type);
@@ -2145,6 +2146,17 @@ void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
        if (page && IS_NODESEG(type))
                fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
 
+       if (add_list) {
+               struct f2fs_bio_info *io;
+
+               INIT_LIST_HEAD(&fio->list);
+               fio->in_list = true;
+               io = sbi->write_io[fio->type] + fio->temp;
+               spin_lock(&io->io_lock);
+               list_add_tail(&fio->list, &io->io_list);
+               spin_unlock(&io->io_lock);
+       }
+
        mutex_unlock(&curseg->curseg_mutex);
 }
 
@@ -2153,11 +2165,9 @@ static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
        int type = __get_segment_type(fio);
        int err;
 
-       if (fio->type == NODE || fio->type == DATA)
-               mutex_lock(&fio->sbi->wio_mutex[fio->type][fio->temp]);
 reallocate:
        allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
-                                       &fio->new_blkaddr, sum, type);
+                       &fio->new_blkaddr, sum, type, fio, true);
 
        /* writeout dirty page into bdev */
        err = f2fs_submit_page_write(fio);
@@ -2165,9 +2175,6 @@ reallocate:
                fio->old_blkaddr = fio->new_blkaddr;
                goto reallocate;
        }
-
-       if (fio->type == NODE || fio->type == DATA)
-               mutex_unlock(&fio->sbi->wio_mutex[fio->type][fio->temp]);
 }
 
 void write_meta_page(struct f2fs_sb_info *sbi, struct page *page)
@@ -2181,6 +2188,7 @@ void write_meta_page(struct f2fs_sb_info *sbi, struct page *page)
                .new_blkaddr = page->index,
                .page = page,
                .encrypted_page = NULL,
+               .in_list = false,
        };
 
        if (unlikely(page->index >= MAIN_BLKADDR(sbi)))

diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
index b700766d0cbfc466051a841a6f8b5f2a28781733..d6af34d1e6a839fc56f0982e809a3519a44cbc2f 100644 (file)
--- a/fs/f2fs/super.c
+++ b/fs/f2fs/super.c
@@ -1983,6 +1983,8 @@ try_onemore:
                        init_rwsem(&sbi->write_io[i][j].io_rwsem);
                        sbi->write_io[i][j].sbi = sbi;
                        sbi->write_io[i][j].bio = NULL;
+                       spin_lock_init(&sbi->write_io[i][j].io_lock);
+                       INIT_LIST_HEAD(&sbi->write_io[i][j].io_list);
                }
        }