fs/f2fs/data.c

   1 /*
   2  * fs/f2fs/data.c
   3  *
   4  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
   5  *             http://www.samsung.com/
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/fs.h>
  12 #include <linux/f2fs_fs.h>
  13 #include <linux/buffer_head.h>
  14 #include <linux/mpage.h>
  15 #include <linux/aio.h>
  16 #include <linux/writeback.h>
  17 #include <linux/backing-dev.h>
  18 #include <linux/blkdev.h>
  19 #include <linux/bio.h>
  20 #include <linux/prefetch.h>
  21
  22 #include "f2fs.h"
  23 #include "node.h"
  24 #include "segment.h"
  25 #include <trace/events/f2fs.h>
  26
  27 /*
  28  * Lock ordering for the change of data block address:
  29  * ->data_page
  30  *  ->node_page
  31  *    update block addresses in the node page
  32  */
  33 static void __set_data_blkaddr(struct dnode_of_data *dn, block_t new_addr)
  34 {
  35         struct f2fs_node *rn;
  36         __le32 *addr_array;
  37         struct page *node_page = dn->node_page;
  38         unsigned int ofs_in_node = dn->ofs_in_node;
  39
  40         f2fs_wait_on_page_writeback(node_page, NODE, false);
  41
  42         rn = F2FS_NODE(node_page);
  43
  44         /* Get physical address of data block */
  45         addr_array = blkaddr_in_node(rn);
  46         addr_array[ofs_in_node] = cpu_to_le32(new_addr);
  47         set_page_dirty(node_page);
  48 }
  49
  50 int reserve_new_block(struct dnode_of_data *dn)
  51 {
  52         struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
  53
  54         if (is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC))
  55                 return -EPERM;
  56         if (!inc_valid_block_count(sbi, dn->inode, 1))
  57                 return -ENOSPC;
  58
  59         trace_f2fs_reserve_new_block(dn->inode, dn->nid, dn->ofs_in_node);
  60
  61         __set_data_blkaddr(dn, NEW_ADDR);
  62         dn->data_blkaddr = NEW_ADDR;
  63         sync_inode_page(dn);
  64         return 0;
  65 }
  66
  67 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
  68 {
  69         bool need_put = dn->inode_page ? false : true;
  70         int err;
  71
  72         err = get_dnode_of_data(dn, index, ALLOC_NODE);
  73         if (err)
  74                 return err;
  75         if (dn->data_blkaddr == NULL_ADDR)
  76                 err = reserve_new_block(dn);
  77
  78         if (need_put)
  79                 f2fs_put_dnode(dn);
  80         return err;
  81 }
  82
  83 static int check_extent_cache(struct inode *inode, pgoff_t pgofs,
  84                                         struct buffer_head *bh_result)
  85 {
  86         struct f2fs_inode_info *fi = F2FS_I(inode);
  87         pgoff_t start_fofs, end_fofs;
  88         block_t start_blkaddr;
  89
  90         if (is_inode_flag_set(fi, FI_NO_EXTENT))
  91                 return 0;
  92
  93         read_lock(&fi->ext.ext_lock);
  94         if (fi->ext.len == 0) {
  95                 read_unlock(&fi->ext.ext_lock);
  96                 return 0;
  97         }
  98
  99         stat_inc_total_hit(inode->i_sb);
 100
 101         start_fofs = fi->ext.fofs;
 102         end_fofs = fi->ext.fofs + fi->ext.len - 1;
 103         start_blkaddr = fi->ext.blk_addr;
 104
 105         if (pgofs >= start_fofs && pgofs <= end_fofs) {
 106                 unsigned int blkbits = inode->i_sb->s_blocksize_bits;
 107                 size_t count;
 108
 109                 clear_buffer_new(bh_result);
 110                 map_bh(bh_result, inode->i_sb,
 111                                 start_blkaddr + pgofs - start_fofs);
 112                 count = end_fofs - pgofs + 1;
 113                 if (count < (UINT_MAX >> blkbits))
 114                         bh_result->b_size = (count << blkbits);
 115                 else
 116                         bh_result->b_size = UINT_MAX;
 117
 118                 stat_inc_read_hit(inode->i_sb);
 119                 read_unlock(&fi->ext.ext_lock);
 120                 return 1;
 121         }
 122         read_unlock(&fi->ext.ext_lock);
 123         return 0;
 124 }
 125
 126 void update_extent_cache(block_t blk_addr, struct dnode_of_data *dn)
 127 {
 128         struct f2fs_inode_info *fi = F2FS_I(dn->inode);
 129         pgoff_t fofs, start_fofs, end_fofs;
 130         block_t start_blkaddr, end_blkaddr;
 131         int need_update = true;
 132
 133         f2fs_bug_on(blk_addr == NEW_ADDR);
 134         fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) +
 135                                                         dn->ofs_in_node;
 136
 137         /* Update the page address in the parent node */
 138         __set_data_blkaddr(dn, blk_addr);
 139
 140         if (is_inode_flag_set(fi, FI_NO_EXTENT))
 141                 return;
 142
 143         write_lock(&fi->ext.ext_lock);
 144
 145         start_fofs = fi->ext.fofs;
 146         end_fofs = fi->ext.fofs + fi->ext.len - 1;
 147         start_blkaddr = fi->ext.blk_addr;
 148         end_blkaddr = fi->ext.blk_addr + fi->ext.len - 1;
 149
 150         /* Drop and initialize the matched extent */
 151         if (fi->ext.len == 1 && fofs == start_fofs)
 152                 fi->ext.len = 0;
 153
 154         /* Initial extent */
 155         if (fi->ext.len == 0) {
 156                 if (blk_addr != NULL_ADDR) {
 157                         fi->ext.fofs = fofs;
 158                         fi->ext.blk_addr = blk_addr;
 159                         fi->ext.len = 1;
 160                 }
 161                 goto end_update;
 162         }
 163
 164         /* Front merge */
 165         if (fofs == start_fofs - 1 && blk_addr == start_blkaddr - 1) {
 166                 fi->ext.fofs--;
 167                 fi->ext.blk_addr--;
 168                 fi->ext.len++;
 169                 goto end_update;
 170         }
 171
 172         /* Back merge */
 173         if (fofs == end_fofs + 1 && blk_addr == end_blkaddr + 1) {
 174                 fi->ext.len++;
 175                 goto end_update;
 176         }
 177
 178         /* Split the existing extent */
 179         if (fi->ext.len > 1 &&
 180                 fofs >= start_fofs && fofs <= end_fofs) {
 181                 if ((end_fofs - fofs) < (fi->ext.len >> 1)) {
 182                         fi->ext.len = fofs - start_fofs;
 183                 } else {
 184                         fi->ext.fofs = fofs + 1;
 185                         fi->ext.blk_addr = start_blkaddr +
 186                                         fofs - start_fofs + 1;
 187                         fi->ext.len -= fofs - start_fofs + 1;
 188                 }
 189         } else {
 190                 need_update = false;
 191         }
 192
 193         /* Finally, if the extent is very fragmented, let's drop the cache. */
 194         if (fi->ext.len < F2FS_MIN_EXTENT_LEN) {
 195                 fi->ext.len = 0;
 196                 set_inode_flag(fi, FI_NO_EXTENT);
 197                 need_update = true;
 198         }
 199 end_update:
 200         write_unlock(&fi->ext.ext_lock);
 201         if (need_update)
 202                 sync_inode_page(dn);
 203         return;
 204 }
 205
 206 struct page *find_data_page(struct inode *inode, pgoff_t index, bool sync)
 207 {
 208         struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
 209         struct address_space *mapping = inode->i_mapping;
 210         struct dnode_of_data dn;
 211         struct page *page;
 212         int err;
 213
 214         page = find_get_page(mapping, index);
 215         if (page && PageUptodate(page))
 216                 return page;
 217         f2fs_put_page(page, 0);
 218
 219         set_new_dnode(&dn, inode, NULL, NULL, 0);
 220         err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
 221         if (err)
 222                 return ERR_PTR(err);
 223         f2fs_put_dnode(&dn);
 224
 225         if (dn.data_blkaddr == NULL_ADDR)
 226                 return ERR_PTR(-ENOENT);
 227
 228         /* By fallocate(), there is no cached page, but with NEW_ADDR */
 229         if (dn.data_blkaddr == NEW_ADDR)
 230                 return ERR_PTR(-EINVAL);
 231
 232         page = grab_cache_page_write_begin(mapping, index, AOP_FLAG_NOFS);
 233         if (!page)
 234                 return ERR_PTR(-ENOMEM);
 235
 236         if (PageUptodate(page)) {
 237                 unlock_page(page);
 238                 return page;
 239         }
 240
 241         err = f2fs_readpage(sbi, page, dn.data_blkaddr,
 242                                         sync ? READ_SYNC : READA);
 243         if (err)
 244                 return ERR_PTR(err);
 245
 246         if (sync) {
 247                 wait_on_page_locked(page);
 248                 if (!PageUptodate(page)) {
 249                         f2fs_put_page(page, 0);
 250                         return ERR_PTR(-EIO);
 251                 }
 252         }
 253         return page;
 254 }
 255
 256 /*
 257  * If it tries to access a hole, return an error.
 258  * Because, the callers, functions in dir.c and GC, should be able to know
 259  * whether this page exists or not.
 260  */
 261 struct page *get_lock_data_page(struct inode *inode, pgoff_t index)
 262 {
 263         struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
 264         struct address_space *mapping = inode->i_mapping;
 265         struct dnode_of_data dn;
 266         struct page *page;
 267         int err;
 268
 269 repeat:
 270         page = grab_cache_page_write_begin(mapping, index, AOP_FLAG_NOFS);
 271         if (!page)
 272                 return ERR_PTR(-ENOMEM);
 273
 274         set_new_dnode(&dn, inode, NULL, NULL, 0);
 275         err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
 276         if (err) {
 277                 f2fs_put_page(page, 1);
 278                 return ERR_PTR(err);
 279         }
 280         f2fs_put_dnode(&dn);
 281
 282         if (dn.data_blkaddr == NULL_ADDR) {
 283                 f2fs_put_page(page, 1);
 284                 return ERR_PTR(-ENOENT);
 285         }
 286
 287         if (PageUptodate(page))
 288                 return page;
 289
 290         /*
 291          * A new dentry page is allocated but not able to be written, since its
 292          * new inode page couldn't be allocated due to -ENOSPC.
 293          * In such the case, its blkaddr can be remained as NEW_ADDR.
 294          * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata.
 295          */
 296         if (dn.data_blkaddr == NEW_ADDR) {
 297                 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
 298                 SetPageUptodate(page);
 299                 return page;
 300         }
 301
 302         err = f2fs_readpage(sbi, page, dn.data_blkaddr, READ_SYNC);
 303         if (err)
 304                 return ERR_PTR(err);
 305
 306         lock_page(page);
 307         if (!PageUptodate(page)) {
 308                 f2fs_put_page(page, 1);
 309                 return ERR_PTR(-EIO);
 310         }
 311         if (page->mapping != mapping) {
 312                 f2fs_put_page(page, 1);
 313                 goto repeat;
 314         }
 315         return page;
 316 }
 317
 318 /*
 319  * Caller ensures that this data page is never allocated.
 320  * A new zero-filled data page is allocated in the page cache.
 321  *
 322  * Also, caller should grab and release a mutex by calling mutex_lock_op() and
 323  * mutex_unlock_op().
 324  * Note that, npage is set only by make_empty_dir.
 325  */
 326 struct page *get_new_data_page(struct inode *inode,
 327                 struct page *npage, pgoff_t index, bool new_i_size)
 328 {
 329         struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
 330         struct address_space *mapping = inode->i_mapping;
 331         struct page *page;
 332         struct dnode_of_data dn;
 333         int err;
 334
 335         set_new_dnode(&dn, inode, npage, npage, 0);
 336         err = f2fs_reserve_block(&dn, index);
 337         if (err)
 338                 return ERR_PTR(err);
 339
 340 repeat:
 341         page = grab_cache_page(mapping, index);
 342         if (!page)
 343                 return ERR_PTR(-ENOMEM);
 344
 345         if (PageUptodate(page))
 346                 return page;
 347
 348         if (dn.data_blkaddr == NEW_ADDR) {
 349                 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
 350                 SetPageUptodate(page);
 351         } else {
 352                 err = f2fs_readpage(sbi, page, dn.data_blkaddr, READ_SYNC);
 353                 if (err)
 354                         return ERR_PTR(err);
 355                 lock_page(page);
 356                 if (!PageUptodate(page)) {
 357                         f2fs_put_page(page, 1);
 358                         return ERR_PTR(-EIO);
 359                 }
 360                 if (page->mapping != mapping) {
 361                         f2fs_put_page(page, 1);
 362                         goto repeat;
 363                 }
 364         }
 365
 366         if (new_i_size &&
 367                 i_size_read(inode) < ((index + 1) << PAGE_CACHE_SHIFT)) {
 368                 i_size_write(inode, ((index + 1) << PAGE_CACHE_SHIFT));
 369                 /* Only the directory inode sets new_i_size */
 370                 set_inode_flag(F2FS_I(inode), FI_UPDATE_DIR);
 371                 mark_inode_dirty_sync(inode);
 372         }
 373         return page;
 374 }
 375
 376 static void read_end_io(struct bio *bio, int err)
 377 {
 378         const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
 379         struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
 380
 381         do {
 382                 struct page *page = bvec->bv_page;
 383
 384                 if (--bvec >= bio->bi_io_vec)
 385                         prefetchw(&bvec->bv_page->flags);
 386
 387                 if (uptodate) {
 388                         SetPageUptodate(page);
 389                 } else {
 390                         ClearPageUptodate(page);
 391                         SetPageError(page);
 392                 }
 393                 unlock_page(page);
 394         } while (bvec >= bio->bi_io_vec);
 395         bio_put(bio);
 396 }
 397
 398 /*
 399  * Fill the locked page with data located in the block address.
 400  * Return unlocked page.
 401  */
 402 int f2fs_readpage(struct f2fs_sb_info *sbi, struct page *page,
 403                                         block_t blk_addr, int type)
 404 {
 405         struct block_device *bdev = sbi->sb->s_bdev;
 406         struct bio *bio;
 407
 408         trace_f2fs_readpage(page, blk_addr, type);
 409
 410         /* Allocate a new bio */
 411         bio = f2fs_bio_alloc(bdev, 1);
 412
 413         /* Initialize the bio */
 414         bio->bi_sector = SECTOR_FROM_BLOCK(sbi, blk_addr);
 415         bio->bi_end_io = read_end_io;
 416
 417         if (bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) < PAGE_CACHE_SIZE) {
 418                 bio_put(bio);
 419                 f2fs_put_page(page, 1);
 420                 return -EFAULT;
 421         }
 422
 423         submit_bio(type, bio);
 424         return 0;
 425 }
 426
 427 void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, int rw)
 428 {
 429         struct f2fs_bio_info *io = &sbi->read_io;
 430
 431         if (!io->bio)
 432                 return;
 433
 434         trace_f2fs_submit_read_bio(sbi->sb, rw, META, io->bio);
 435
 436         mutex_lock(&io->io_mutex);
 437         if (io->bio) {
 438                 submit_bio(rw, io->bio);
 439                 io->bio = NULL;
 440         }
 441         mutex_unlock(&io->io_mutex);
 442 }
 443
 444 void submit_read_page(struct f2fs_sb_info *sbi, struct page *page,
 445                                         block_t blk_addr, int rw)
 446 {
 447         struct block_device *bdev = sbi->sb->s_bdev;
 448         struct f2fs_bio_info *io = &sbi->read_io;
 449         int bio_blocks;
 450
 451         verify_block_addr(sbi, blk_addr);
 452
 453         mutex_lock(&io->io_mutex);
 454
 455         if (io->bio && io->last_block_in_bio != blk_addr - 1) {
 456                 submit_bio(rw, io->bio);
 457                 io->bio = NULL;
 458         }
 459 alloc_new:
 460         if (io->bio == NULL) {
 461                 bio_blocks = MAX_BIO_BLOCKS(max_hw_blocks(sbi));
 462                 io->bio = f2fs_bio_alloc(bdev, bio_blocks);
 463                 io->bio->bi_sector = SECTOR_FROM_BLOCK(sbi, blk_addr);
 464                 io->bio->bi_end_io = read_end_io;
 465         }
 466
 467         if (bio_add_page(io->bio, page, PAGE_CACHE_SIZE, 0) <
 468                                                         PAGE_CACHE_SIZE) {
 469                 submit_bio(rw, io->bio);
 470                 io->bio = NULL;
 471                 goto alloc_new;
 472         }
 473
 474         io->last_block_in_bio = blk_addr;
 475
 476         mutex_unlock(&io->io_mutex);
 477         trace_f2fs_submit_read_page(page, rw, META, blk_addr);
 478 }
 479
 480 /*
 481  * This function should be used by the data read flow only where it
 482  * does not check the "create" flag that indicates block allocation.
 483  * The reason for this special functionality is to exploit VFS readahead
 484  * mechanism.
 485  */
 486 static int get_data_block_ro(struct inode *inode, sector_t iblock,
 487                         struct buffer_head *bh_result, int create)
 488 {
 489         unsigned int blkbits = inode->i_sb->s_blocksize_bits;
 490         unsigned maxblocks = bh_result->b_size >> blkbits;
 491         struct dnode_of_data dn;
 492         pgoff_t pgofs;
 493         int err;
 494
 495         /* Get the page offset from the block offset(iblock) */
 496         pgofs = (pgoff_t)(iblock >> (PAGE_CACHE_SHIFT - blkbits));
 497
 498         if (check_extent_cache(inode, pgofs, bh_result)) {
 499                 trace_f2fs_get_data_block(inode, iblock, bh_result, 0);
 500                 return 0;
 501         }
 502
 503         /* When reading holes, we need its node page */
 504         set_new_dnode(&dn, inode, NULL, NULL, 0);
 505         err = get_dnode_of_data(&dn, pgofs, LOOKUP_NODE_RA);
 506         if (err) {
 507                 trace_f2fs_get_data_block(inode, iblock, bh_result, err);
 508                 return (err == -ENOENT) ? 0 : err;
 509         }
 510
 511         /* It does not support data allocation */
 512         f2fs_bug_on(create);
 513
 514         if (dn.data_blkaddr != NEW_ADDR && dn.data_blkaddr != NULL_ADDR) {
 515                 int i;
 516                 unsigned int end_offset;
 517
 518                 end_offset = IS_INODE(dn.node_page) ?
 519                                 ADDRS_PER_INODE(F2FS_I(inode)) :
 520                                 ADDRS_PER_BLOCK;
 521
 522                 clear_buffer_new(bh_result);
 523
 524                 /* Give more consecutive addresses for the read ahead */
 525                 for (i = 0; i < end_offset - dn.ofs_in_node; i++)
 526                         if (((datablock_addr(dn.node_page,
 527                                                         dn.ofs_in_node + i))
 528                                 != (dn.data_blkaddr + i)) || maxblocks == i)
 529                                 break;
 530                 map_bh(bh_result, inode->i_sb, dn.data_blkaddr);
 531                 bh_result->b_size = (((size_t)i) << blkbits);
 532         }
 533         f2fs_put_dnode(&dn);
 534         trace_f2fs_get_data_block(inode, iblock, bh_result, 0);
 535         return 0;
 536 }
 537
 538 static int f2fs_read_data_page(struct file *file, struct page *page)
 539 {
 540         return mpage_readpage(page, get_data_block_ro);
 541 }
 542
 543 static int f2fs_read_data_pages(struct file *file,
 544                         struct address_space *mapping,
 545                         struct list_head *pages, unsigned nr_pages)
 546 {
 547         return mpage_readpages(mapping, pages, nr_pages, get_data_block_ro);
 548 }
 549
 550 int do_write_data_page(struct page *page)
 551 {
 552         struct inode *inode = page->mapping->host;
 553         block_t old_blk_addr, new_blk_addr;
 554         struct dnode_of_data dn;
 555         int err = 0;
 556
 557         set_new_dnode(&dn, inode, NULL, NULL, 0);
 558         err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
 559         if (err)
 560                 return err;
 561
 562         old_blk_addr = dn.data_blkaddr;
 563
 564         /* This page is already truncated */
 565         if (old_blk_addr == NULL_ADDR)
 566                 goto out_writepage;
 567
 568         set_page_writeback(page);
 569
 570         /*
 571          * If current allocation needs SSR,
 572          * it had better in-place writes for updated data.
 573          */
 574         if (unlikely(old_blk_addr != NEW_ADDR &&
 575                         !is_cold_data(page) &&
 576                         need_inplace_update(inode))) {
 577                 rewrite_data_page(F2FS_SB(inode->i_sb), page,
 578                                                 old_blk_addr);
 579         } else {
 580                 write_data_page(inode, page, &dn,
 581                                 old_blk_addr, &new_blk_addr);
 582                 update_extent_cache(new_blk_addr, &dn);
 583         }
 584 out_writepage:
 585         f2fs_put_dnode(&dn);
 586         return err;
 587 }
 588
 589 static int f2fs_write_data_page(struct page *page,
 590                                         struct writeback_control *wbc)
 591 {
 592         struct inode *inode = page->mapping->host;
 593         struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
 594         loff_t i_size = i_size_read(inode);
 595         const pgoff_t end_index = ((unsigned long long) i_size)
 596                                                         >> PAGE_CACHE_SHIFT;
 597         unsigned offset;
 598         bool need_balance_fs = false;
 599         int err = 0;
 600
 601         if (page->index < end_index)
 602                 goto write;
 603
 604         /*
 605          * If the offset is out-of-range of file size,
 606          * this page does not have to be written to disk.
 607          */
 608         offset = i_size & (PAGE_CACHE_SIZE - 1);
 609         if ((page->index >= end_index + 1) || !offset) {
 610                 if (S_ISDIR(inode->i_mode)) {
 611                         dec_page_count(sbi, F2FS_DIRTY_DENTS);
 612                         inode_dec_dirty_dents(inode);
 613                 }
 614                 goto out;
 615         }
 616
 617         zero_user_segment(page, offset, PAGE_CACHE_SIZE);
 618 write:
 619         if (sbi->por_doing) {
 620                 err = AOP_WRITEPAGE_ACTIVATE;
 621                 goto redirty_out;
 622         }
 623
 624         /* Dentry blocks are controlled by checkpoint */
 625         if (S_ISDIR(inode->i_mode)) {
 626                 dec_page_count(sbi, F2FS_DIRTY_DENTS);
 627                 inode_dec_dirty_dents(inode);
 628                 err = do_write_data_page(page);
 629         } else {
 630                 f2fs_lock_op(sbi);
 631                 err = do_write_data_page(page);
 632                 f2fs_unlock_op(sbi);
 633                 need_balance_fs = true;
 634         }
 635         if (err == -ENOENT)
 636                 goto out;
 637         else if (err)
 638                 goto redirty_out;
 639
 640         if (wbc->for_reclaim)
 641                 f2fs_submit_bio(sbi, DATA, true);
 642
 643         clear_cold_data(page);
 644 out:
 645         unlock_page(page);
 646         if (need_balance_fs)
 647                 f2fs_balance_fs(sbi);
 648         return 0;
 649
 650 redirty_out:
 651         wbc->pages_skipped++;
 652         set_page_dirty(page);
 653         return err;
 654 }
 655
 656 #define MAX_DESIRED_PAGES_WP    4096
 657
 658 static int __f2fs_writepage(struct page *page, struct writeback_control *wbc,
 659                         void *data)
 660 {
 661         struct address_space *mapping = data;
 662         int ret = mapping->a_ops->writepage(page, wbc);
 663         mapping_set_error(mapping, ret);
 664         return ret;
 665 }
 666
 667 static int f2fs_write_data_pages(struct address_space *mapping,
 668                             struct writeback_control *wbc)
 669 {
 670         struct inode *inode = mapping->host;
 671         struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
 672         bool locked = false;
 673         int ret;
 674         long excess_nrtw = 0, desired_nrtw;
 675
 676         /* deal with chardevs and other special file */
 677         if (!mapping->a_ops->writepage)
 678                 return 0;
 679
 680         if (wbc->nr_to_write < MAX_DESIRED_PAGES_WP) {
 681                 desired_nrtw = MAX_DESIRED_PAGES_WP;
 682                 excess_nrtw = desired_nrtw - wbc->nr_to_write;
 683                 wbc->nr_to_write = desired_nrtw;
 684         }
 685
 686         if (!S_ISDIR(inode->i_mode)) {
 687                 mutex_lock(&sbi->writepages);
 688                 locked = true;
 689         }
 690         ret = write_cache_pages(mapping, wbc, __f2fs_writepage, mapping);
 691         if (locked)
 692                 mutex_unlock(&sbi->writepages);
 693         f2fs_submit_bio(sbi, DATA, (wbc->sync_mode == WB_SYNC_ALL));
 694
 695         remove_dirty_dir_inode(inode);
 696
 697         wbc->nr_to_write -= excess_nrtw;
 698         return ret;
 699 }
 700
 701 static int f2fs_write_begin(struct file *file, struct address_space *mapping,
 702                 loff_t pos, unsigned len, unsigned flags,
 703                 struct page **pagep, void **fsdata)
 704 {
 705         struct inode *inode = mapping->host;
 706         struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
 707         struct page *page;
 708         pgoff_t index = ((unsigned long long) pos) >> PAGE_CACHE_SHIFT;
 709         struct dnode_of_data dn;
 710         int err = 0;
 711
 712         f2fs_balance_fs(sbi);
 713 repeat:
 714         page = grab_cache_page_write_begin(mapping, index, flags);
 715         if (!page)
 716                 return -ENOMEM;
 717         *pagep = page;
 718
 719         f2fs_lock_op(sbi);
 720         set_new_dnode(&dn, inode, NULL, NULL, 0);
 721         err = f2fs_reserve_block(&dn, index);
 722         f2fs_unlock_op(sbi);
 723
 724         if (err) {
 725                 f2fs_put_page(page, 1);
 726                 return err;
 727         }
 728
 729         if ((len == PAGE_CACHE_SIZE) || PageUptodate(page))
 730                 return 0;
 731
 732         if ((pos & PAGE_CACHE_MASK) >= i_size_read(inode)) {
 733                 unsigned start = pos & (PAGE_CACHE_SIZE - 1);
 734                 unsigned end = start + len;
 735
 736                 /* Reading beyond i_size is simple: memset to zero */
 737                 zero_user_segments(page, 0, start, end, PAGE_CACHE_SIZE);
 738                 goto out;
 739         }
 740
 741         if (dn.data_blkaddr == NEW_ADDR) {
 742                 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
 743         } else {
 744                 err = f2fs_readpage(sbi, page, dn.data_blkaddr, READ_SYNC);
 745                 if (err)
 746                         return err;
 747                 lock_page(page);
 748                 if (!PageUptodate(page)) {
 749                         f2fs_put_page(page, 1);
 750                         return -EIO;
 751                 }
 752                 if (page->mapping != mapping) {
 753                         f2fs_put_page(page, 1);
 754                         goto repeat;
 755                 }
 756         }
 757 out:
 758         SetPageUptodate(page);
 759         clear_cold_data(page);
 760         return 0;
 761 }
 762
 763 static int f2fs_write_end(struct file *file,
 764                         struct address_space *mapping,
 765                         loff_t pos, unsigned len, unsigned copied,
 766                         struct page *page, void *fsdata)
 767 {
 768         struct inode *inode = page->mapping->host;
 769
 770         SetPageUptodate(page);
 771         set_page_dirty(page);
 772
 773         if (pos + copied > i_size_read(inode)) {
 774                 i_size_write(inode, pos + copied);
 775                 mark_inode_dirty(inode);
 776                 update_inode_page(inode);
 777         }
 778
 779         f2fs_put_page(page, 1);
 780         return copied;
 781 }
 782
 783 static ssize_t f2fs_direct_IO(int rw, struct kiocb *iocb,
 784                 const struct iovec *iov, loff_t offset, unsigned long nr_segs)
 785 {
 786         struct file *file = iocb->ki_filp;
 787         struct inode *inode = file->f_mapping->host;
 788
 789         if (rw == WRITE)
 790                 return 0;
 791
 792         /* Needs synchronization with the cleaner */
 793         return blockdev_direct_IO(rw, iocb, inode, iov, offset, nr_segs,
 794                                                   get_data_block_ro);
 795 }
 796
 797 static void f2fs_invalidate_data_page(struct page *page, unsigned int offset,
 798                                       unsigned int length)
 799 {
 800         struct inode *inode = page->mapping->host;
 801         struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
 802         if (S_ISDIR(inode->i_mode) && PageDirty(page)) {
 803                 dec_page_count(sbi, F2FS_DIRTY_DENTS);
 804                 inode_dec_dirty_dents(inode);
 805         }
 806         ClearPagePrivate(page);
 807 }
 808
 809 static int f2fs_release_data_page(struct page *page, gfp_t wait)
 810 {
 811         ClearPagePrivate(page);
 812         return 1;
 813 }
 814
 815 static int f2fs_set_data_page_dirty(struct page *page)
 816 {
 817         struct address_space *mapping = page->mapping;
 818         struct inode *inode = mapping->host;
 819
 820         trace_f2fs_set_page_dirty(page, DATA);
 821
 822         SetPageUptodate(page);
 823         if (!PageDirty(page)) {
 824                 __set_page_dirty_nobuffers(page);
 825                 set_dirty_dir_page(inode, page);
 826                 return 1;
 827         }
 828         return 0;
 829 }
 830
 831 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
 832 {
 833         return generic_block_bmap(mapping, block, get_data_block_ro);
 834 }
 835
 836 const struct address_space_operations f2fs_dblock_aops = {
 837         .readpage       = f2fs_read_data_page,
 838         .readpages      = f2fs_read_data_pages,
 839         .writepage      = f2fs_write_data_page,
 840         .writepages     = f2fs_write_data_pages,
 841         .write_begin    = f2fs_write_begin,
 842         .write_end      = f2fs_write_end,
 843         .set_page_dirty = f2fs_set_data_page_dirty,
 844         .invalidatepage = f2fs_invalidate_data_page,
 845         .releasepage    = f2fs_release_data_page,
 846         .direct_IO      = f2fs_direct_IO,
 847         .bmap           = f2fs_bmap,
 848 };