projects / GitHub / exynos8895 / android_kernel_samsung_universal8895.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
f2fs: introduce a new direct_IO write path
[GitHub/exynos8895/android_kernel_samsung_universal8895.git] / fs / f2fs / data.c
1 /*
2 * fs/f2fs/data.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11 #include <linux/fs.h>
12 #include <linux/f2fs_fs.h>
13 #include <linux/buffer_head.h>
14 #include <linux/mpage.h>
15 #include <linux/aio.h>
16 #include <linux/writeback.h>
17 #include <linux/backing-dev.h>
18 #include <linux/blkdev.h>
19 #include <linux/bio.h>
20 #include <linux/prefetch.h>
21
22 #include "f2fs.h"
23 #include "node.h"
24 #include "segment.h"
25 #include <trace/events/f2fs.h>
26
27 /*
28 * Low-level block read/write IO operations.
29 */
30 static struct bio *__bio_alloc(struct block_device *bdev, int npages)
31 {
32 struct bio *bio;
33
34 /* No failure on bio allocation */
35 bio = bio_alloc(GFP_NOIO, npages);
36 bio->bi_bdev = bdev;
37 bio->bi_private = NULL;
38 return bio;
39 }
40
41 static void f2fs_read_end_io(struct bio *bio, int err)
42 {
43 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
44 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
45
46 do {
47 struct page *page = bvec->bv_page;
48
49 if (--bvec >= bio->bi_io_vec)
50 prefetchw(&bvec->bv_page->flags);
51
52 if (unlikely(!uptodate)) {
53 ClearPageUptodate(page);
54 SetPageError(page);
55 } else {
56 SetPageUptodate(page);
57 }
58 unlock_page(page);
59 } while (bvec >= bio->bi_io_vec);
60
61 bio_put(bio);
62 }
63
64 static void f2fs_write_end_io(struct bio *bio, int err)
65 {
66 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
67 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
68 struct f2fs_sb_info *sbi = F2FS_SB(bvec->bv_page->mapping->host->i_sb);
69
70 do {
71 struct page *page = bvec->bv_page;
72
73 if (--bvec >= bio->bi_io_vec)
74 prefetchw(&bvec->bv_page->flags);
75
76 if (unlikely(!uptodate)) {
77 SetPageError(page);
78 set_bit(AS_EIO, &page->mapping->flags);
79 set_ckpt_flags(sbi->ckpt, CP_ERROR_FLAG);
80 sbi->sb->s_flags |= MS_RDONLY;
81 }
82 end_page_writeback(page);
83 dec_page_count(sbi, F2FS_WRITEBACK);
84 } while (bvec >= bio->bi_io_vec);
85
86 if (bio->bi_private)
87 complete(bio->bi_private);
88
89 if (!get_pages(sbi, F2FS_WRITEBACK) &&
90 !list_empty(&sbi->cp_wait.task_list))
91 wake_up(&sbi->cp_wait);
92
93 bio_put(bio);
94 }
95
96 static void __submit_merged_bio(struct f2fs_bio_info *io)
97 {
98 struct f2fs_io_info *fio = &io->fio;
99 int rw;
100
101 if (!io->bio)
102 return;
103
104 rw = fio->rw | fio->rw_flag;
105
106 if (is_read_io(rw)) {
107 trace_f2fs_submit_read_bio(io->sbi->sb, rw, fio->type, io->bio);
108 submit_bio(rw, io->bio);
109 io->bio = NULL;
110 return;
111 }
112 trace_f2fs_submit_write_bio(io->sbi->sb, rw, fio->type, io->bio);
113
114 /*
115 * META_FLUSH is only from the checkpoint procedure, and we should wait
116 * this metadata bio for FS consistency.
117 */
118 if (fio->type == META_FLUSH) {
119 DECLARE_COMPLETION_ONSTACK(wait);
120 io->bio->bi_private = &wait;
121 submit_bio(rw, io->bio);
122 wait_for_completion(&wait);
123 } else {
124 submit_bio(rw, io->bio);
125 }
126 io->bio = NULL;
127 }
128
129 void f2fs_submit_merged_bio(struct f2fs_sb_info *sbi,
130 enum page_type type, int rw)
131 {
132 enum page_type btype = PAGE_TYPE_OF_BIO(type);
133 struct f2fs_bio_info *io;
134
135 io = is_read_io(rw) ? &sbi->read_io : &sbi->write_io[btype];
136
137 mutex_lock(&io->io_mutex);
138
139 /* change META to META_FLUSH in the checkpoint procedure */
140 if (type >= META_FLUSH) {
141 io->fio.type = META_FLUSH;
142 io->fio.rw = WRITE_FLUSH_FUA;
143 }
144 __submit_merged_bio(io);
145 mutex_unlock(&io->io_mutex);
146 }
147
148 /*
149 * Fill the locked page with data located in the block address.
150 * Return unlocked page.
151 */
152 int f2fs_submit_page_bio(struct f2fs_sb_info *sbi, struct page *page,
153 block_t blk_addr, int rw)
154 {
155 struct block_device *bdev = sbi->sb->s_bdev;
156 struct bio *bio;
157
158 trace_f2fs_submit_page_bio(page, blk_addr, rw);
159
160 /* Allocate a new bio */
161 bio = __bio_alloc(bdev, 1);
162
163 /* Initialize the bio */
164 bio->bi_sector = SECTOR_FROM_BLOCK(sbi, blk_addr);
165 bio->bi_end_io = is_read_io(rw) ? f2fs_read_end_io : f2fs_write_end_io;
166
167 if (bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) < PAGE_CACHE_SIZE) {
168 bio_put(bio);
169 f2fs_put_page(page, 1);
170 return -EFAULT;
171 }
172
173 submit_bio(rw, bio);
174 return 0;
175 }
176
177 void f2fs_submit_page_mbio(struct f2fs_sb_info *sbi, struct page *page,
178 block_t blk_addr, struct f2fs_io_info *fio)
179 {
180 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
181 struct block_device *bdev = sbi->sb->s_bdev;
182 struct f2fs_bio_info *io;
183 int bio_blocks;
184
185 io = is_read_io(fio->rw) ? &sbi->read_io : &sbi->write_io[btype];
186
187 verify_block_addr(sbi, blk_addr);
188
189 mutex_lock(&io->io_mutex);
190
191 if (!is_read_io(fio->rw))
192 inc_page_count(sbi, F2FS_WRITEBACK);
193
194 if (io->bio && (io->last_block_in_bio != blk_addr - 1 ||
195 io->fio.rw != fio->rw))
196 __submit_merged_bio(io);
197 alloc_new:
198 if (io->bio == NULL) {
199 bio_blocks = MAX_BIO_BLOCKS(max_hw_blocks(sbi));
200 io->bio = __bio_alloc(bdev, bio_blocks);
201 io->bio->bi_sector = SECTOR_FROM_BLOCK(sbi, blk_addr);
202 io->bio->bi_end_io = is_read_io(fio->rw) ? f2fs_read_end_io :
203 f2fs_write_end_io;
204 io->fio = *fio;
205 /*
206 * The end_io will be assigned at the sumbission phase.
207 * Until then, let bio_add_page() merge consecutive IOs as much
208 * as possible.
209 */
210 }
211
212 if (bio_add_page(io->bio, page, PAGE_CACHE_SIZE, 0) <
213 PAGE_CACHE_SIZE) {
214 __submit_merged_bio(io);
215 goto alloc_new;
216 }
217
218 io->last_block_in_bio = blk_addr;
219
220 mutex_unlock(&io->io_mutex);
221 trace_f2fs_submit_page_mbio(page, fio->rw, fio->type, blk_addr);
222 }
223
224 /*
225 * Lock ordering for the change of data block address:
226 * ->data_page
227 * ->node_page
228 * update block addresses in the node page
229 */
230 static void __set_data_blkaddr(struct dnode_of_data *dn, block_t new_addr)
231 {
232 struct f2fs_node *rn;
233 __le32 *addr_array;
234 struct page *node_page = dn->node_page;
235 unsigned int ofs_in_node = dn->ofs_in_node;
236
237 f2fs_wait_on_page_writeback(node_page, NODE, false);
238
239 rn = F2FS_NODE(node_page);
240
241 /* Get physical address of data block */
242 addr_array = blkaddr_in_node(rn);
243 addr_array[ofs_in_node] = cpu_to_le32(new_addr);
244 set_page_dirty(node_page);
245 }
246
247 int reserve_new_block(struct dnode_of_data *dn)
248 {
249 struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
250
251 if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
252 return -EPERM;
253 if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))
254 return -ENOSPC;
255
256 trace_f2fs_reserve_new_block(dn->inode, dn->nid, dn->ofs_in_node);
257
258 __set_data_blkaddr(dn, NEW_ADDR);
259 dn->data_blkaddr = NEW_ADDR;
260 sync_inode_page(dn);
261 return 0;
262 }
263
264 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
265 {
266 bool need_put = dn->inode_page ? false : true;
267 int err;
268
269 err = get_dnode_of_data(dn, index, ALLOC_NODE);
270 if (err)
271 return err;
272 if (dn->data_blkaddr == NULL_ADDR)
273 err = reserve_new_block(dn);
274
275 if (need_put)
276 f2fs_put_dnode(dn);
277 return err;
278 }
279
280 static int check_extent_cache(struct inode *inode, pgoff_t pgofs,
281 struct buffer_head *bh_result)
282 {
283 struct f2fs_inode_info *fi = F2FS_I(inode);
284 pgoff_t start_fofs, end_fofs;
285 block_t start_blkaddr;
286
287 if (is_inode_flag_set(fi, FI_NO_EXTENT))
288 return 0;
289
290 read_lock(&fi->ext.ext_lock);
291 if (fi->ext.len == 0) {
292 read_unlock(&fi->ext.ext_lock);
293 return 0;
294 }
295
296 stat_inc_total_hit(inode->i_sb);
297
298 start_fofs = fi->ext.fofs;
299 end_fofs = fi->ext.fofs + fi->ext.len - 1;
300 start_blkaddr = fi->ext.blk_addr;
301
302 if (pgofs >= start_fofs && pgofs <= end_fofs) {
303 unsigned int blkbits = inode->i_sb->s_blocksize_bits;
304 size_t count;
305
306 clear_buffer_new(bh_result);
307 map_bh(bh_result, inode->i_sb,
308 start_blkaddr + pgofs - start_fofs);
309 count = end_fofs - pgofs + 1;
310 if (count < (UINT_MAX >> blkbits))
311 bh_result->b_size = (count << blkbits);
312 else
313 bh_result->b_size = UINT_MAX;
314
315 stat_inc_read_hit(inode->i_sb);
316 read_unlock(&fi->ext.ext_lock);
317 return 1;
318 }
319 read_unlock(&fi->ext.ext_lock);
320 return 0;
321 }
322
323 void update_extent_cache(block_t blk_addr, struct dnode_of_data *dn)
324 {
325 struct f2fs_inode_info *fi = F2FS_I(dn->inode);
326 pgoff_t fofs, start_fofs, end_fofs;
327 block_t start_blkaddr, end_blkaddr;
328 int need_update = true;
329
330 f2fs_bug_on(blk_addr == NEW_ADDR);
331 fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) +
332 dn->ofs_in_node;
333
334 /* Update the page address in the parent node */
335 __set_data_blkaddr(dn, blk_addr);
336
337 if (is_inode_flag_set(fi, FI_NO_EXTENT))
338 return;
339
340 write_lock(&fi->ext.ext_lock);
341
342 start_fofs = fi->ext.fofs;
343 end_fofs = fi->ext.fofs + fi->ext.len - 1;
344 start_blkaddr = fi->ext.blk_addr;
345 end_blkaddr = fi->ext.blk_addr + fi->ext.len - 1;
346
347 /* Drop and initialize the matched extent */
348 if (fi->ext.len == 1 && fofs == start_fofs)
349 fi->ext.len = 0;
350
351 /* Initial extent */
352 if (fi->ext.len == 0) {
353 if (blk_addr != NULL_ADDR) {
354 fi->ext.fofs = fofs;
355 fi->ext.blk_addr = blk_addr;
356 fi->ext.len = 1;
357 }
358 goto end_update;
359 }
360
361 /* Front merge */
362 if (fofs == start_fofs - 1 && blk_addr == start_blkaddr - 1) {
363 fi->ext.fofs--;
364 fi->ext.blk_addr--;
365 fi->ext.len++;
366 goto end_update;
367 }
368
369 /* Back merge */
370 if (fofs == end_fofs + 1 && blk_addr == end_blkaddr + 1) {
371 fi->ext.len++;
372 goto end_update;
373 }
374
375 /* Split the existing extent */
376 if (fi->ext.len > 1 &&
377 fofs >= start_fofs && fofs <= end_fofs) {
378 if ((end_fofs - fofs) < (fi->ext.len >> 1)) {
379 fi->ext.len = fofs - start_fofs;
380 } else {
381 fi->ext.fofs = fofs + 1;
382 fi->ext.blk_addr = start_blkaddr +
383 fofs - start_fofs + 1;
384 fi->ext.len -= fofs - start_fofs + 1;
385 }
386 } else {
387 need_update = false;
388 }
389
390 /* Finally, if the extent is very fragmented, let's drop the cache. */
391 if (fi->ext.len < F2FS_MIN_EXTENT_LEN) {
392 fi->ext.len = 0;
393 set_inode_flag(fi, FI_NO_EXTENT);
394 need_update = true;
395 }
396 end_update:
397 write_unlock(&fi->ext.ext_lock);
398 if (need_update)
399 sync_inode_page(dn);
400 return;
401 }
402
403 struct page *find_data_page(struct inode *inode, pgoff_t index, bool sync)
404 {
405 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
406 struct address_space *mapping = inode->i_mapping;
407 struct dnode_of_data dn;
408 struct page *page;
409 int err;
410
411 page = find_get_page(mapping, index);
412 if (page && PageUptodate(page))
413 return page;
414 f2fs_put_page(page, 0);
415
416 set_new_dnode(&dn, inode, NULL, NULL, 0);
417 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
418 if (err)
419 return ERR_PTR(err);
420 f2fs_put_dnode(&dn);
421
422 if (dn.data_blkaddr == NULL_ADDR)
423 return ERR_PTR(-ENOENT);
424
425 /* By fallocate(), there is no cached page, but with NEW_ADDR */
426 if (unlikely(dn.data_blkaddr == NEW_ADDR))
427 return ERR_PTR(-EINVAL);
428
429 page = grab_cache_page_write_begin(mapping, index, AOP_FLAG_NOFS);
430 if (!page)
431 return ERR_PTR(-ENOMEM);
432
433 if (PageUptodate(page)) {
434 unlock_page(page);
435 return page;
436 }
437
438 err = f2fs_submit_page_bio(sbi, page, dn.data_blkaddr,
439 sync ? READ_SYNC : READA);
440 if (err)
441 return ERR_PTR(err);
442
443 if (sync) {
444 wait_on_page_locked(page);
445 if (unlikely(!PageUptodate(page))) {
446 f2fs_put_page(page, 0);
447 return ERR_PTR(-EIO);
448 }
449 }
450 return page;
451 }
452
453 /*
454 * If it tries to access a hole, return an error.
455 * Because, the callers, functions in dir.c and GC, should be able to know
456 * whether this page exists or not.
457 */
458 struct page *get_lock_data_page(struct inode *inode, pgoff_t index)
459 {
460 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
461 struct address_space *mapping = inode->i_mapping;
462 struct dnode_of_data dn;
463 struct page *page;
464 int err;
465
466 repeat:
467 page = grab_cache_page_write_begin(mapping, index, AOP_FLAG_NOFS);
468 if (!page)
469 return ERR_PTR(-ENOMEM);
470
471 set_new_dnode(&dn, inode, NULL, NULL, 0);
472 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
473 if (err) {
474 f2fs_put_page(page, 1);
475 return ERR_PTR(err);
476 }
477 f2fs_put_dnode(&dn);
478
479 if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
480 f2fs_put_page(page, 1);
481 return ERR_PTR(-ENOENT);
482 }
483
484 if (PageUptodate(page))
485 return page;
486
487 /*
488 * A new dentry page is allocated but not able to be written, since its
489 * new inode page couldn't be allocated due to -ENOSPC.
490 * In such the case, its blkaddr can be remained as NEW_ADDR.
491 * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata.
492 */
493 if (dn.data_blkaddr == NEW_ADDR) {
494 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
495 SetPageUptodate(page);
496 return page;
497 }
498
499 err = f2fs_submit_page_bio(sbi, page, dn.data_blkaddr, READ_SYNC);
500 if (err)
501 return ERR_PTR(err);
502
503 lock_page(page);
504 if (unlikely(!PageUptodate(page))) {
505 f2fs_put_page(page, 1);
506 return ERR_PTR(-EIO);
507 }
508 if (unlikely(page->mapping != mapping)) {
509 f2fs_put_page(page, 1);
510 goto repeat;
511 }
512 return page;
513 }
514
515 /*
516 * Caller ensures that this data page is never allocated.
517 * A new zero-filled data page is allocated in the page cache.
518 *
519 * Also, caller should grab and release a mutex by calling mutex_lock_op() and
520 * mutex_unlock_op().
521 * Note that, npage is set only by make_empty_dir.
522 */
523 struct page *get_new_data_page(struct inode *inode,
524 struct page *npage, pgoff_t index, bool new_i_size)
525 {
526 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
527 struct address_space *mapping = inode->i_mapping;
528 struct page *page;
529 struct dnode_of_data dn;
530 int err;
531
532 set_new_dnode(&dn, inode, npage, npage, 0);
533 err = f2fs_reserve_block(&dn, index);
534 if (err)
535 return ERR_PTR(err);
536 repeat:
537 page = grab_cache_page(mapping, index);
538 if (!page)
539 return ERR_PTR(-ENOMEM);
540
541 if (PageUptodate(page))
542 return page;
543
544 if (dn.data_blkaddr == NEW_ADDR) {
545 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
546 SetPageUptodate(page);
547 } else {
548 err = f2fs_submit_page_bio(sbi, page, dn.data_blkaddr,
549 READ_SYNC);
550 if (err)
551 return ERR_PTR(err);
552 lock_page(page);
553 if (unlikely(!PageUptodate(page))) {
554 f2fs_put_page(page, 1);
555 return ERR_PTR(-EIO);
556 }
557 if (unlikely(page->mapping != mapping)) {
558 f2fs_put_page(page, 1);
559 goto repeat;
560 }
561 }
562
563 if (new_i_size &&
564 i_size_read(inode) < ((index + 1) << PAGE_CACHE_SHIFT)) {
565 i_size_write(inode, ((index + 1) << PAGE_CACHE_SHIFT));
566 /* Only the directory inode sets new_i_size */
567 set_inode_flag(F2FS_I(inode), FI_UPDATE_DIR);
568 mark_inode_dirty_sync(inode);
569 }
570 return page;
571 }
572
573 static int __allocate_data_block(struct dnode_of_data *dn)
574 {
575 struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
576 struct f2fs_summary sum;
577 block_t new_blkaddr;
578 struct node_info ni;
579 int type;
580
581 if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
582 return -EPERM;
583 if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))
584 return -ENOSPC;
585
586 __set_data_blkaddr(dn, NEW_ADDR);
587 dn->data_blkaddr = NEW_ADDR;
588
589 get_node_info(sbi, dn->nid, &ni);
590 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
591
592 type = CURSEG_WARM_DATA;
593
594 allocate_data_block(sbi, NULL, NULL_ADDR, &new_blkaddr, &sum, type);
595
596 /* direct IO doesn't use extent cache to maximize the performance */
597 set_inode_flag(F2FS_I(dn->inode), FI_NO_EXTENT);
598 update_extent_cache(new_blkaddr, dn);
599 clear_inode_flag(F2FS_I(dn->inode), FI_NO_EXTENT);
600
601 dn->data_blkaddr = new_blkaddr;
602 return 0;
603 }
604
605 /*
606 * This function should be used by the data read flow only where it
607 * does not check the "create" flag that indicates block allocation.
608 * The reason for this special functionality is to exploit VFS readahead
609 * mechanism.
610 */
611 static int get_data_block(struct inode *inode, sector_t iblock,
612 struct buffer_head *bh_result, int create)
613 {
614 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
615 unsigned int blkbits = inode->i_sb->s_blocksize_bits;
616 unsigned maxblocks = bh_result->b_size >> blkbits;
617 struct dnode_of_data dn;
618 int mode = create ? ALLOC_NODE : LOOKUP_NODE_RA;
619 pgoff_t pgofs, end_offset;
620 int err = 0, ofs = 1;
621 bool allocated = false;
622
623 /* Get the page offset from the block offset(iblock) */
624 pgofs = (pgoff_t)(iblock >> (PAGE_CACHE_SHIFT - blkbits));
625
626 if (check_extent_cache(inode, pgofs, bh_result))
627 goto out;
628
629 if (create)
630 f2fs_lock_op(sbi);
631
632 /* When reading holes, we need its node page */
633 set_new_dnode(&dn, inode, NULL, NULL, 0);
634 err = get_dnode_of_data(&dn, pgofs, mode);
635 if (err || dn.data_blkaddr == NEW_ADDR) {
636 if (err == -ENOENT)
637 err = 0;
638 goto unlock_out;
639 }
640
641 if (dn.data_blkaddr != NULL_ADDR) {
642 map_bh(bh_result, inode->i_sb, dn.data_blkaddr);
643 } else if (create) {
644 err = __allocate_data_block(&dn);
645 if (err)
646 goto put_out;
647 allocated = true;
648 map_bh(bh_result, inode->i_sb, dn.data_blkaddr);
649 } else {
650 goto put_out;
651 }
652
653 end_offset = IS_INODE(dn.node_page) ?
654 ADDRS_PER_INODE(F2FS_I(inode)) : ADDRS_PER_BLOCK;
655 bh_result->b_size = (((size_t)1) << blkbits);
656 dn.ofs_in_node++;
657 pgofs++;
658
659 get_next:
660 if (dn.ofs_in_node >= end_offset) {
661 if (allocated)
662 sync_inode_page(&dn);
663 allocated = false;
664 f2fs_put_dnode(&dn);
665
666 set_new_dnode(&dn, inode, NULL, NULL, 0);
667 err = get_dnode_of_data(&dn, pgofs, mode);
668 if (err || dn.data_blkaddr == NEW_ADDR) {
669 if (err == -ENOENT)
670 err = 0;
671 goto unlock_out;
672 }
673 end_offset = IS_INODE(dn.node_page) ?
674 ADDRS_PER_INODE(F2FS_I(inode)) : ADDRS_PER_BLOCK;
675 }
676
677 if (maxblocks > (bh_result->b_size >> blkbits)) {
678 block_t blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node);
679 if (blkaddr == NULL_ADDR && create) {
680 err = __allocate_data_block(&dn);
681 if (err)
682 goto sync_out;
683 allocated = true;
684 blkaddr = dn.data_blkaddr;
685 }
686 /* Give more consecutive addresses for the read ahead */
687 if (blkaddr == (bh_result->b_blocknr + ofs)) {
688 ofs++;
689 dn.ofs_in_node++;
690 pgofs++;
691 bh_result->b_size += (((size_t)1) << blkbits);
692 goto get_next;
693 }
694 }
695 sync_out:
696 if (allocated)
697 sync_inode_page(&dn);
698 put_out:
699 f2fs_put_dnode(&dn);
700 unlock_out:
701 if (create)
702 f2fs_unlock_op(sbi);
703 out:
704 trace_f2fs_get_data_block(inode, iblock, bh_result, err);
705 return err;
706 }
707
708 static int f2fs_read_data_page(struct file *file, struct page *page)
709 {
710 return mpage_readpage(page, get_data_block);
711 }
712
713 static int f2fs_read_data_pages(struct file *file,
714 struct address_space *mapping,
715 struct list_head *pages, unsigned nr_pages)
716 {
717 return mpage_readpages(mapping, pages, nr_pages, get_data_block);
718 }
719
720 int do_write_data_page(struct page *page, struct f2fs_io_info *fio)
721 {
722 struct inode *inode = page->mapping->host;
723 block_t old_blkaddr, new_blkaddr;
724 struct dnode_of_data dn;
725 int err = 0;
726
727 set_new_dnode(&dn, inode, NULL, NULL, 0);
728 err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
729 if (err)
730 return err;
731
732 old_blkaddr = dn.data_blkaddr;
733
734 /* This page is already truncated */
735 if (old_blkaddr == NULL_ADDR)
736 goto out_writepage;
737
738 set_page_writeback(page);
739
740 /*
741 * If current allocation needs SSR,
742 * it had better in-place writes for updated data.
743 */
744 if (unlikely(old_blkaddr != NEW_ADDR &&
745 !is_cold_data(page) &&
746 need_inplace_update(inode))) {
747 rewrite_data_page(page, old_blkaddr, fio);
748 } else {
749 write_data_page(page, &dn, &new_blkaddr, fio);
750 update_extent_cache(new_blkaddr, &dn);
751 }
752 out_writepage:
753 f2fs_put_dnode(&dn);
754 return err;
755 }
756
757 static int f2fs_write_data_page(struct page *page,
758 struct writeback_control *wbc)
759 {
760 struct inode *inode = page->mapping->host;
761 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
762 loff_t i_size = i_size_read(inode);
763 const pgoff_t end_index = ((unsigned long long) i_size)
764 >> PAGE_CACHE_SHIFT;
765 unsigned offset;
766 bool need_balance_fs = false;
767 int err = 0;
768 struct f2fs_io_info fio = {
769 .type = DATA,
770 .rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC: WRITE,
771 .rw_flag = 0,
772 };
773
774 if (page->index < end_index)
775 goto write;
776
777 /*
778 * If the offset is out-of-range of file size,
779 * this page does not have to be written to disk.
780 */
781 offset = i_size & (PAGE_CACHE_SIZE - 1);
782 if ((page->index >= end_index + 1) || !offset) {
783 if (S_ISDIR(inode->i_mode)) {
784 dec_page_count(sbi, F2FS_DIRTY_DENTS);
785 inode_dec_dirty_dents(inode);
786 }
787 goto out;
788 }
789
790 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
791 write:
792 if (unlikely(sbi->por_doing)) {
793 err = AOP_WRITEPAGE_ACTIVATE;
794 goto redirty_out;
795 }
796
797 /* Dentry blocks are controlled by checkpoint */
798 if (S_ISDIR(inode->i_mode)) {
799 dec_page_count(sbi, F2FS_DIRTY_DENTS);
800 inode_dec_dirty_dents(inode);
801 err = do_write_data_page(page, &fio);
802 } else {
803 f2fs_lock_op(sbi);
804 err = do_write_data_page(page, &fio);
805 f2fs_unlock_op(sbi);
806 need_balance_fs = true;
807 }
808 if (err == -ENOENT)
809 goto out;
810 else if (err)
811 goto redirty_out;
812
813 if (wbc->for_reclaim)
814 f2fs_submit_merged_bio(sbi, DATA, WRITE);
815
816 clear_cold_data(page);
817 out:
818 unlock_page(page);
819 if (need_balance_fs)
820 f2fs_balance_fs(sbi);
821 return 0;
822
823 redirty_out:
824 wbc->pages_skipped++;
825 set_page_dirty(page);
826 return err;
827 }
828
829 #define MAX_DESIRED_PAGES_WP 4096
830
831 static int __f2fs_writepage(struct page *page, struct writeback_control *wbc,
832 void *data)
833 {
834 struct address_space *mapping = data;
835 int ret = mapping->a_ops->writepage(page, wbc);
836 mapping_set_error(mapping, ret);
837 return ret;
838 }
839
840 static int f2fs_write_data_pages(struct address_space *mapping,
841 struct writeback_control *wbc)
842 {
843 struct inode *inode = mapping->host;
844 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
845 bool locked = false;
846 int ret;
847 long excess_nrtw = 0, desired_nrtw;
848
849 /* deal with chardevs and other special file */
850 if (!mapping->a_ops->writepage)
851 return 0;
852
853 if (wbc->nr_to_write < MAX_DESIRED_PAGES_WP) {
854 desired_nrtw = MAX_DESIRED_PAGES_WP;
855 excess_nrtw = desired_nrtw - wbc->nr_to_write;
856 wbc->nr_to_write = desired_nrtw;
857 }
858
859 if (!S_ISDIR(inode->i_mode)) {
860 mutex_lock(&sbi->writepages);
861 locked = true;
862 }
863 ret = write_cache_pages(mapping, wbc, __f2fs_writepage, mapping);
864 if (locked)
865 mutex_unlock(&sbi->writepages);
866
867 f2fs_submit_merged_bio(sbi, DATA, WRITE);
868
869 remove_dirty_dir_inode(inode);
870
871 wbc->nr_to_write -= excess_nrtw;
872 return ret;
873 }
874
875 static int f2fs_write_begin(struct file *file, struct address_space *mapping,
876 loff_t pos, unsigned len, unsigned flags,
877 struct page **pagep, void **fsdata)
878 {
879 struct inode *inode = mapping->host;
880 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
881 struct page *page;
882 pgoff_t index = ((unsigned long long) pos) >> PAGE_CACHE_SHIFT;
883 struct dnode_of_data dn;
884 int err = 0;
885
886 f2fs_balance_fs(sbi);
887 repeat:
888 page = grab_cache_page_write_begin(mapping, index, flags);
889 if (!page)
890 return -ENOMEM;
891 *pagep = page;
892
893 f2fs_lock_op(sbi);
894 set_new_dnode(&dn, inode, NULL, NULL, 0);
895 err = f2fs_reserve_block(&dn, index);
896 f2fs_unlock_op(sbi);
897
898 if (err) {
899 f2fs_put_page(page, 1);
900 return err;
901 }
902
903 if ((len == PAGE_CACHE_SIZE) || PageUptodate(page))
904 return 0;
905
906 if ((pos & PAGE_CACHE_MASK) >= i_size_read(inode)) {
907 unsigned start = pos & (PAGE_CACHE_SIZE - 1);
908 unsigned end = start + len;
909
910 /* Reading beyond i_size is simple: memset to zero */
911 zero_user_segments(page, 0, start, end, PAGE_CACHE_SIZE);
912 goto out;
913 }
914
915 if (dn.data_blkaddr == NEW_ADDR) {
916 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
917 } else {
918 err = f2fs_submit_page_bio(sbi, page, dn.data_blkaddr,
919 READ_SYNC);
920 if (err)
921 return err;
922 lock_page(page);
923 if (unlikely(!PageUptodate(page))) {
924 f2fs_put_page(page, 1);
925 return -EIO;
926 }
927 if (unlikely(page->mapping != mapping)) {
928 f2fs_put_page(page, 1);
929 goto repeat;
930 }
931 }
932 out:
933 SetPageUptodate(page);
934 clear_cold_data(page);
935 return 0;
936 }
937
938 static int f2fs_write_end(struct file *file,
939 struct address_space *mapping,
940 loff_t pos, unsigned len, unsigned copied,
941 struct page *page, void *fsdata)
942 {
943 struct inode *inode = page->mapping->host;
944
945 SetPageUptodate(page);
946 set_page_dirty(page);
947
948 if (pos + copied > i_size_read(inode)) {
949 i_size_write(inode, pos + copied);
950 mark_inode_dirty(inode);
951 update_inode_page(inode);
952 }
953
954 f2fs_put_page(page, 1);
955 return copied;
956 }
957
958 static ssize_t f2fs_direct_IO(int rw, struct kiocb *iocb,
959 const struct iovec *iov, loff_t offset, unsigned long nr_segs)
960 {
961 struct file *file = iocb->ki_filp;
962 struct inode *inode = file->f_mapping->host;
963 return blockdev_direct_IO(rw, iocb, inode, iov, offset, nr_segs,
964 get_data_block);
965 }
966
967 static void f2fs_invalidate_data_page(struct page *page, unsigned int offset,
968 unsigned int length)
969 {
970 struct inode *inode = page->mapping->host;
971 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
972 if (S_ISDIR(inode->i_mode) && PageDirty(page)) {
973 dec_page_count(sbi, F2FS_DIRTY_DENTS);
974 inode_dec_dirty_dents(inode);
975 }
976 ClearPagePrivate(page);
977 }
978
979 static int f2fs_release_data_page(struct page *page, gfp_t wait)
980 {
981 ClearPagePrivate(page);
982 return 1;
983 }
984
985 static int f2fs_set_data_page_dirty(struct page *page)
986 {
987 struct address_space *mapping = page->mapping;
988 struct inode *inode = mapping->host;
989
990 trace_f2fs_set_page_dirty(page, DATA);
991
992 SetPageUptodate(page);
993 if (!PageDirty(page)) {
994 __set_page_dirty_nobuffers(page);
995 set_dirty_dir_page(inode, page);
996 return 1;
997 }
998 return 0;
999 }
1000
1001 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
1002 {
1003 return generic_block_bmap(mapping, block, get_data_block);
1004 }
1005
1006 const struct address_space_operations f2fs_dblock_aops = {
1007 .readpage = f2fs_read_data_page,
1008 .readpages = f2fs_read_data_pages,
1009 .writepage = f2fs_write_data_page,
1010 .writepages = f2fs_write_data_pages,
1011 .write_begin = f2fs_write_begin,
1012 .write_end = f2fs_write_end,
1013 .set_page_dirty = f2fs_set_data_page_dirty,
1014 .invalidatepage = f2fs_invalidate_data_page,
1015 .releasepage = f2fs_release_data_page,
1016 .direct_IO = f2fs_direct_IO,
1017 .bmap = f2fs_bmap,
1018 };