projects / GitHub / exynos8895 / android_kernel_samsung_universal8895.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
f2fs: cleanup parameters for trace_f2fs_submit_{read_,write_,page_,page_m}bio with fio
[GitHub/exynos8895/android_kernel_samsung_universal8895.git] / fs / f2fs / data.c
1 /*
2 * fs/f2fs/data.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11 #include <linux/fs.h>
12 #include <linux/f2fs_fs.h>
13 #include <linux/buffer_head.h>
14 #include <linux/mpage.h>
15 #include <linux/aio.h>
16 #include <linux/writeback.h>
17 #include <linux/backing-dev.h>
18 #include <linux/blkdev.h>
19 #include <linux/bio.h>
20 #include <linux/prefetch.h>
21
22 #include "f2fs.h"
23 #include "node.h"
24 #include "segment.h"
25 #include "trace.h"
26 #include <trace/events/f2fs.h>
27
28 static void f2fs_read_end_io(struct bio *bio, int err)
29 {
30 struct bio_vec *bvec;
31 int i;
32
33 bio_for_each_segment_all(bvec, bio, i) {
34 struct page *page = bvec->bv_page;
35
36 if (!err) {
37 SetPageUptodate(page);
38 } else {
39 ClearPageUptodate(page);
40 SetPageError(page);
41 }
42 unlock_page(page);
43 }
44 bio_put(bio);
45 }
46
47 static void f2fs_write_end_io(struct bio *bio, int err)
48 {
49 struct f2fs_sb_info *sbi = bio->bi_private;
50 struct bio_vec *bvec;
51 int i;
52
53 bio_for_each_segment_all(bvec, bio, i) {
54 struct page *page = bvec->bv_page;
55
56 if (unlikely(err)) {
57 set_page_dirty(page);
58 set_bit(AS_EIO, &page->mapping->flags);
59 f2fs_stop_checkpoint(sbi);
60 }
61 end_page_writeback(page);
62 dec_page_count(sbi, F2FS_WRITEBACK);
63 }
64
65 if (!get_pages(sbi, F2FS_WRITEBACK) &&
66 !list_empty(&sbi->cp_wait.task_list))
67 wake_up(&sbi->cp_wait);
68
69 bio_put(bio);
70 }
71
72 /*
73 * Low-level block read/write IO operations.
74 */
75 static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
76 int npages, bool is_read)
77 {
78 struct bio *bio;
79
80 /* No failure on bio allocation */
81 bio = bio_alloc(GFP_NOIO, npages);
82
83 bio->bi_bdev = sbi->sb->s_bdev;
84 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
85 bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io;
86 bio->bi_private = sbi;
87
88 return bio;
89 }
90
91 static void __submit_merged_bio(struct f2fs_bio_info *io)
92 {
93 struct f2fs_io_info *fio = &io->fio;
94
95 if (!io->bio)
96 return;
97
98 if (is_read_io(fio->rw))
99 trace_f2fs_submit_read_bio(io->sbi->sb, fio, io->bio);
100 else
101 trace_f2fs_submit_write_bio(io->sbi->sb, fio, io->bio);
102
103 submit_bio(fio->rw, io->bio);
104 io->bio = NULL;
105 }
106
107 void f2fs_submit_merged_bio(struct f2fs_sb_info *sbi,
108 enum page_type type, int rw)
109 {
110 enum page_type btype = PAGE_TYPE_OF_BIO(type);
111 struct f2fs_bio_info *io;
112
113 io = is_read_io(rw) ? &sbi->read_io : &sbi->write_io[btype];
114
115 down_write(&io->io_rwsem);
116
117 /* change META to META_FLUSH in the checkpoint procedure */
118 if (type >= META_FLUSH) {
119 io->fio.type = META_FLUSH;
120 if (test_opt(sbi, NOBARRIER))
121 io->fio.rw = WRITE_FLUSH | REQ_META | REQ_PRIO;
122 else
123 io->fio.rw = WRITE_FLUSH_FUA | REQ_META | REQ_PRIO;
124 }
125 __submit_merged_bio(io);
126 up_write(&io->io_rwsem);
127 }
128
129 /*
130 * Fill the locked page with data located in the block address.
131 * Return unlocked page.
132 */
133 int f2fs_submit_page_bio(struct f2fs_sb_info *sbi, struct page *page,
134 struct f2fs_io_info *fio)
135 {
136 struct bio *bio;
137
138 trace_f2fs_submit_page_bio(page, fio);
139 f2fs_trace_ios(page, fio, 0);
140
141 /* Allocate a new bio */
142 bio = __bio_alloc(sbi, fio->blk_addr, 1, is_read_io(fio->rw));
143
144 if (bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) < PAGE_CACHE_SIZE) {
145 bio_put(bio);
146 f2fs_put_page(page, 1);
147 return -EFAULT;
148 }
149
150 submit_bio(fio->rw, bio);
151 return 0;
152 }
153
154 void f2fs_submit_page_mbio(struct f2fs_sb_info *sbi, struct page *page,
155 struct f2fs_io_info *fio)
156 {
157 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
158 struct f2fs_bio_info *io;
159 bool is_read = is_read_io(fio->rw);
160
161 io = is_read ? &sbi->read_io : &sbi->write_io[btype];
162
163 verify_block_addr(sbi, fio->blk_addr);
164
165 down_write(&io->io_rwsem);
166
167 if (!is_read)
168 inc_page_count(sbi, F2FS_WRITEBACK);
169
170 if (io->bio && (io->last_block_in_bio != fio->blk_addr - 1 ||
171 io->fio.rw != fio->rw))
172 __submit_merged_bio(io);
173 alloc_new:
174 if (io->bio == NULL) {
175 int bio_blocks = MAX_BIO_BLOCKS(sbi);
176
177 io->bio = __bio_alloc(sbi, fio->blk_addr, bio_blocks, is_read);
178 io->fio = *fio;
179 }
180
181 if (bio_add_page(io->bio, page, PAGE_CACHE_SIZE, 0) <
182 PAGE_CACHE_SIZE) {
183 __submit_merged_bio(io);
184 goto alloc_new;
185 }
186
187 io->last_block_in_bio = fio->blk_addr;
188 f2fs_trace_ios(page, fio, 0);
189
190 up_write(&io->io_rwsem);
191 trace_f2fs_submit_page_mbio(page, fio);
192 }
193
194 /*
195 * Lock ordering for the change of data block address:
196 * ->data_page
197 * ->node_page
198 * update block addresses in the node page
199 */
200 static void __set_data_blkaddr(struct dnode_of_data *dn, block_t new_addr)
201 {
202 struct f2fs_node *rn;
203 __le32 *addr_array;
204 struct page *node_page = dn->node_page;
205 unsigned int ofs_in_node = dn->ofs_in_node;
206
207 f2fs_wait_on_page_writeback(node_page, NODE);
208
209 rn = F2FS_NODE(node_page);
210
211 /* Get physical address of data block */
212 addr_array = blkaddr_in_node(rn);
213 addr_array[ofs_in_node] = cpu_to_le32(new_addr);
214 set_page_dirty(node_page);
215 }
216
217 int reserve_new_block(struct dnode_of_data *dn)
218 {
219 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
220
221 if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
222 return -EPERM;
223 if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))
224 return -ENOSPC;
225
226 trace_f2fs_reserve_new_block(dn->inode, dn->nid, dn->ofs_in_node);
227
228 __set_data_blkaddr(dn, NEW_ADDR);
229 dn->data_blkaddr = NEW_ADDR;
230 mark_inode_dirty(dn->inode);
231 sync_inode_page(dn);
232 return 0;
233 }
234
235 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
236 {
237 bool need_put = dn->inode_page ? false : true;
238 int err;
239
240 err = get_dnode_of_data(dn, index, ALLOC_NODE);
241 if (err)
242 return err;
243
244 if (dn->data_blkaddr == NULL_ADDR)
245 err = reserve_new_block(dn);
246 if (err || need_put)
247 f2fs_put_dnode(dn);
248 return err;
249 }
250
251 static int check_extent_cache(struct inode *inode, pgoff_t pgofs,
252 struct buffer_head *bh_result)
253 {
254 struct f2fs_inode_info *fi = F2FS_I(inode);
255 pgoff_t start_fofs, end_fofs;
256 block_t start_blkaddr;
257
258 if (is_inode_flag_set(fi, FI_NO_EXTENT))
259 return 0;
260
261 read_lock(&fi->ext.ext_lock);
262 if (fi->ext.len == 0) {
263 read_unlock(&fi->ext.ext_lock);
264 return 0;
265 }
266
267 stat_inc_total_hit(inode->i_sb);
268
269 start_fofs = fi->ext.fofs;
270 end_fofs = fi->ext.fofs + fi->ext.len - 1;
271 start_blkaddr = fi->ext.blk_addr;
272
273 if (pgofs >= start_fofs && pgofs <= end_fofs) {
274 unsigned int blkbits = inode->i_sb->s_blocksize_bits;
275 size_t count;
276
277 clear_buffer_new(bh_result);
278 map_bh(bh_result, inode->i_sb,
279 start_blkaddr + pgofs - start_fofs);
280 count = end_fofs - pgofs + 1;
281 if (count < (UINT_MAX >> blkbits))
282 bh_result->b_size = (count << blkbits);
283 else
284 bh_result->b_size = UINT_MAX;
285
286 stat_inc_read_hit(inode->i_sb);
287 read_unlock(&fi->ext.ext_lock);
288 return 1;
289 }
290 read_unlock(&fi->ext.ext_lock);
291 return 0;
292 }
293
294 void update_extent_cache(block_t blk_addr, struct dnode_of_data *dn)
295 {
296 struct f2fs_inode_info *fi = F2FS_I(dn->inode);
297 pgoff_t fofs, start_fofs, end_fofs;
298 block_t start_blkaddr, end_blkaddr;
299 int need_update = true;
300
301 f2fs_bug_on(F2FS_I_SB(dn->inode), blk_addr == NEW_ADDR);
302 fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) +
303 dn->ofs_in_node;
304
305 /* Update the page address in the parent node */
306 __set_data_blkaddr(dn, blk_addr);
307
308 if (is_inode_flag_set(fi, FI_NO_EXTENT))
309 return;
310
311 write_lock(&fi->ext.ext_lock);
312
313 start_fofs = fi->ext.fofs;
314 end_fofs = fi->ext.fofs + fi->ext.len - 1;
315 start_blkaddr = fi->ext.blk_addr;
316 end_blkaddr = fi->ext.blk_addr + fi->ext.len - 1;
317
318 /* Drop and initialize the matched extent */
319 if (fi->ext.len == 1 && fofs == start_fofs)
320 fi->ext.len = 0;
321
322 /* Initial extent */
323 if (fi->ext.len == 0) {
324 if (blk_addr != NULL_ADDR) {
325 fi->ext.fofs = fofs;
326 fi->ext.blk_addr = blk_addr;
327 fi->ext.len = 1;
328 }
329 goto end_update;
330 }
331
332 /* Front merge */
333 if (fofs == start_fofs - 1 && blk_addr == start_blkaddr - 1) {
334 fi->ext.fofs--;
335 fi->ext.blk_addr--;
336 fi->ext.len++;
337 goto end_update;
338 }
339
340 /* Back merge */
341 if (fofs == end_fofs + 1 && blk_addr == end_blkaddr + 1) {
342 fi->ext.len++;
343 goto end_update;
344 }
345
346 /* Split the existing extent */
347 if (fi->ext.len > 1 &&
348 fofs >= start_fofs && fofs <= end_fofs) {
349 if ((end_fofs - fofs) < (fi->ext.len >> 1)) {
350 fi->ext.len = fofs - start_fofs;
351 } else {
352 fi->ext.fofs = fofs + 1;
353 fi->ext.blk_addr = start_blkaddr +
354 fofs - start_fofs + 1;
355 fi->ext.len -= fofs - start_fofs + 1;
356 }
357 } else {
358 need_update = false;
359 }
360
361 /* Finally, if the extent is very fragmented, let's drop the cache. */
362 if (fi->ext.len < F2FS_MIN_EXTENT_LEN) {
363 fi->ext.len = 0;
364 set_inode_flag(fi, FI_NO_EXTENT);
365 need_update = true;
366 }
367 end_update:
368 write_unlock(&fi->ext.ext_lock);
369 if (need_update)
370 sync_inode_page(dn);
371 return;
372 }
373
374 struct page *find_data_page(struct inode *inode, pgoff_t index, bool sync)
375 {
376 struct address_space *mapping = inode->i_mapping;
377 struct dnode_of_data dn;
378 struct page *page;
379 int err;
380 struct f2fs_io_info fio = {
381 .type = DATA,
382 .rw = sync ? READ_SYNC : READA,
383 };
384
385 page = find_get_page(mapping, index);
386 if (page && PageUptodate(page))
387 return page;
388 f2fs_put_page(page, 0);
389
390 set_new_dnode(&dn, inode, NULL, NULL, 0);
391 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
392 if (err)
393 return ERR_PTR(err);
394 f2fs_put_dnode(&dn);
395
396 if (dn.data_blkaddr == NULL_ADDR)
397 return ERR_PTR(-ENOENT);
398
399 /* By fallocate(), there is no cached page, but with NEW_ADDR */
400 if (unlikely(dn.data_blkaddr == NEW_ADDR))
401 return ERR_PTR(-EINVAL);
402
403 page = grab_cache_page(mapping, index);
404 if (!page)
405 return ERR_PTR(-ENOMEM);
406
407 if (PageUptodate(page)) {
408 unlock_page(page);
409 return page;
410 }
411
412 fio.blk_addr = dn.data_blkaddr;
413 err = f2fs_submit_page_bio(F2FS_I_SB(inode), page, &fio);
414 if (err)
415 return ERR_PTR(err);
416
417 if (sync) {
418 wait_on_page_locked(page);
419 if (unlikely(!PageUptodate(page))) {
420 f2fs_put_page(page, 0);
421 return ERR_PTR(-EIO);
422 }
423 }
424 return page;
425 }
426
427 /*
428 * If it tries to access a hole, return an error.
429 * Because, the callers, functions in dir.c and GC, should be able to know
430 * whether this page exists or not.
431 */
432 struct page *get_lock_data_page(struct inode *inode, pgoff_t index)
433 {
434 struct address_space *mapping = inode->i_mapping;
435 struct dnode_of_data dn;
436 struct page *page;
437 int err;
438 struct f2fs_io_info fio = {
439 .type = DATA,
440 .rw = READ_SYNC,
441 };
442 repeat:
443 page = grab_cache_page(mapping, index);
444 if (!page)
445 return ERR_PTR(-ENOMEM);
446
447 set_new_dnode(&dn, inode, NULL, NULL, 0);
448 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
449 if (err) {
450 f2fs_put_page(page, 1);
451 return ERR_PTR(err);
452 }
453 f2fs_put_dnode(&dn);
454
455 if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
456 f2fs_put_page(page, 1);
457 return ERR_PTR(-ENOENT);
458 }
459
460 if (PageUptodate(page))
461 return page;
462
463 /*
464 * A new dentry page is allocated but not able to be written, since its
465 * new inode page couldn't be allocated due to -ENOSPC.
466 * In such the case, its blkaddr can be remained as NEW_ADDR.
467 * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata.
468 */
469 if (dn.data_blkaddr == NEW_ADDR) {
470 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
471 SetPageUptodate(page);
472 return page;
473 }
474
475 fio.blk_addr = dn.data_blkaddr;
476 err = f2fs_submit_page_bio(F2FS_I_SB(inode), page, &fio);
477 if (err)
478 return ERR_PTR(err);
479
480 lock_page(page);
481 if (unlikely(!PageUptodate(page))) {
482 f2fs_put_page(page, 1);
483 return ERR_PTR(-EIO);
484 }
485 if (unlikely(page->mapping != mapping)) {
486 f2fs_put_page(page, 1);
487 goto repeat;
488 }
489 return page;
490 }
491
492 /*
493 * Caller ensures that this data page is never allocated.
494 * A new zero-filled data page is allocated in the page cache.
495 *
496 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
497 * f2fs_unlock_op().
498 * Note that, ipage is set only by make_empty_dir.
499 */
500 struct page *get_new_data_page(struct inode *inode,
501 struct page *ipage, pgoff_t index, bool new_i_size)
502 {
503 struct address_space *mapping = inode->i_mapping;
504 struct page *page;
505 struct dnode_of_data dn;
506 int err;
507
508 set_new_dnode(&dn, inode, ipage, NULL, 0);
509 err = f2fs_reserve_block(&dn, index);
510 if (err)
511 return ERR_PTR(err);
512 repeat:
513 page = grab_cache_page(mapping, index);
514 if (!page) {
515 err = -ENOMEM;
516 goto put_err;
517 }
518
519 if (PageUptodate(page))
520 return page;
521
522 if (dn.data_blkaddr == NEW_ADDR) {
523 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
524 SetPageUptodate(page);
525 } else {
526 struct f2fs_io_info fio = {
527 .type = DATA,
528 .rw = READ_SYNC,
529 .blk_addr = dn.data_blkaddr,
530 };
531 err = f2fs_submit_page_bio(F2FS_I_SB(inode), page, &fio);
532 if (err)
533 goto put_err;
534
535 lock_page(page);
536 if (unlikely(!PageUptodate(page))) {
537 f2fs_put_page(page, 1);
538 err = -EIO;
539 goto put_err;
540 }
541 if (unlikely(page->mapping != mapping)) {
542 f2fs_put_page(page, 1);
543 goto repeat;
544 }
545 }
546
547 if (new_i_size &&
548 i_size_read(inode) < ((index + 1) << PAGE_CACHE_SHIFT)) {
549 i_size_write(inode, ((index + 1) << PAGE_CACHE_SHIFT));
550 /* Only the directory inode sets new_i_size */
551 set_inode_flag(F2FS_I(inode), FI_UPDATE_DIR);
552 }
553 return page;
554
555 put_err:
556 f2fs_put_dnode(&dn);
557 return ERR_PTR(err);
558 }
559
560 static int __allocate_data_block(struct dnode_of_data *dn)
561 {
562 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
563 struct f2fs_inode_info *fi = F2FS_I(dn->inode);
564 struct f2fs_summary sum;
565 block_t new_blkaddr;
566 struct node_info ni;
567 pgoff_t fofs;
568 int type;
569
570 if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
571 return -EPERM;
572 if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))
573 return -ENOSPC;
574
575 __set_data_blkaddr(dn, NEW_ADDR);
576 dn->data_blkaddr = NEW_ADDR;
577
578 get_node_info(sbi, dn->nid, &ni);
579 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
580
581 type = CURSEG_WARM_DATA;
582
583 allocate_data_block(sbi, NULL, NULL_ADDR, &new_blkaddr, &sum, type);
584
585 /* direct IO doesn't use extent cache to maximize the performance */
586 set_inode_flag(F2FS_I(dn->inode), FI_NO_EXTENT);
587 update_extent_cache(new_blkaddr, dn);
588 clear_inode_flag(F2FS_I(dn->inode), FI_NO_EXTENT);
589
590 /* update i_size */
591 fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) +
592 dn->ofs_in_node;
593 if (i_size_read(dn->inode) < ((fofs + 1) << PAGE_CACHE_SHIFT))
594 i_size_write(dn->inode, ((fofs + 1) << PAGE_CACHE_SHIFT));
595
596 dn->data_blkaddr = new_blkaddr;
597 return 0;
598 }
599
600 /*
601 * get_data_block() now supported readahead/bmap/rw direct_IO with mapped bh.
602 * If original data blocks are allocated, then give them to blockdev.
603 * Otherwise,
604 * a. preallocate requested block addresses
605 * b. do not use extent cache for better performance
606 * c. give the block addresses to blockdev
607 */
608 static int __get_data_block(struct inode *inode, sector_t iblock,
609 struct buffer_head *bh_result, int create, bool fiemap)
610 {
611 unsigned int blkbits = inode->i_sb->s_blocksize_bits;
612 unsigned maxblocks = bh_result->b_size >> blkbits;
613 struct dnode_of_data dn;
614 int mode = create ? ALLOC_NODE : LOOKUP_NODE_RA;
615 pgoff_t pgofs, end_offset;
616 int err = 0, ofs = 1;
617 bool allocated = false;
618
619 /* Get the page offset from the block offset(iblock) */
620 pgofs = (pgoff_t)(iblock >> (PAGE_CACHE_SHIFT - blkbits));
621
622 if (check_extent_cache(inode, pgofs, bh_result))
623 goto out;
624
625 if (create) {
626 f2fs_balance_fs(F2FS_I_SB(inode));
627 f2fs_lock_op(F2FS_I_SB(inode));
628 }
629
630 /* When reading holes, we need its node page */
631 set_new_dnode(&dn, inode, NULL, NULL, 0);
632 err = get_dnode_of_data(&dn, pgofs, mode);
633 if (err) {
634 if (err == -ENOENT)
635 err = 0;
636 goto unlock_out;
637 }
638 if (dn.data_blkaddr == NEW_ADDR && !fiemap)
639 goto put_out;
640
641 if (dn.data_blkaddr != NULL_ADDR) {
642 map_bh(bh_result, inode->i_sb, dn.data_blkaddr);
643 } else if (create) {
644 err = __allocate_data_block(&dn);
645 if (err)
646 goto put_out;
647 allocated = true;
648 map_bh(bh_result, inode->i_sb, dn.data_blkaddr);
649 } else {
650 goto put_out;
651 }
652
653 end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode));
654 bh_result->b_size = (((size_t)1) << blkbits);
655 dn.ofs_in_node++;
656 pgofs++;
657
658 get_next:
659 if (dn.ofs_in_node >= end_offset) {
660 if (allocated)
661 sync_inode_page(&dn);
662 allocated = false;
663 f2fs_put_dnode(&dn);
664
665 set_new_dnode(&dn, inode, NULL, NULL, 0);
666 err = get_dnode_of_data(&dn, pgofs, mode);
667 if (err) {
668 if (err == -ENOENT)
669 err = 0;
670 goto unlock_out;
671 }
672 if (dn.data_blkaddr == NEW_ADDR && !fiemap)
673 goto put_out;
674
675 end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode));
676 }
677
678 if (maxblocks > (bh_result->b_size >> blkbits)) {
679 block_t blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node);
680 if (blkaddr == NULL_ADDR && create) {
681 err = __allocate_data_block(&dn);
682 if (err)
683 goto sync_out;
684 allocated = true;
685 blkaddr = dn.data_blkaddr;
686 }
687 /* Give more consecutive addresses for the readahead */
688 if (blkaddr == (bh_result->b_blocknr + ofs)) {
689 ofs++;
690 dn.ofs_in_node++;
691 pgofs++;
692 bh_result->b_size += (((size_t)1) << blkbits);
693 goto get_next;
694 }
695 }
696 sync_out:
697 if (allocated)
698 sync_inode_page(&dn);
699 put_out:
700 f2fs_put_dnode(&dn);
701 unlock_out:
702 if (create)
703 f2fs_unlock_op(F2FS_I_SB(inode));
704 out:
705 trace_f2fs_get_data_block(inode, iblock, bh_result, err);
706 return err;
707 }
708
709 static int get_data_block(struct inode *inode, sector_t iblock,
710 struct buffer_head *bh_result, int create)
711 {
712 return __get_data_block(inode, iblock, bh_result, create, false);
713 }
714
715 static int get_data_block_fiemap(struct inode *inode, sector_t iblock,
716 struct buffer_head *bh_result, int create)
717 {
718 return __get_data_block(inode, iblock, bh_result, create, true);
719 }
720
721 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
722 u64 start, u64 len)
723 {
724 return generic_block_fiemap(inode, fieinfo,
725 start, len, get_data_block_fiemap);
726 }
727
728 static int f2fs_read_data_page(struct file *file, struct page *page)
729 {
730 struct inode *inode = page->mapping->host;
731 int ret = -EAGAIN;
732
733 trace_f2fs_readpage(page, DATA);
734
735 /* If the file has inline data, try to read it directly */
736 if (f2fs_has_inline_data(inode))
737 ret = f2fs_read_inline_data(inode, page);
738 if (ret == -EAGAIN)
739 ret = mpage_readpage(page, get_data_block);
740
741 return ret;
742 }
743
744 static int f2fs_read_data_pages(struct file *file,
745 struct address_space *mapping,
746 struct list_head *pages, unsigned nr_pages)
747 {
748 struct inode *inode = file->f_mapping->host;
749
750 /* If the file has inline data, skip readpages */
751 if (f2fs_has_inline_data(inode))
752 return 0;
753
754 return mpage_readpages(mapping, pages, nr_pages, get_data_block);
755 }
756
757 int do_write_data_page(struct page *page, struct f2fs_io_info *fio)
758 {
759 struct inode *inode = page->mapping->host;
760 struct dnode_of_data dn;
761 int err = 0;
762
763 set_new_dnode(&dn, inode, NULL, NULL, 0);
764 err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
765 if (err)
766 return err;
767
768 fio->blk_addr = dn.data_blkaddr;
769
770 /* This page is already truncated */
771 if (fio->blk_addr == NULL_ADDR)
772 goto out_writepage;
773
774 set_page_writeback(page);
775
776 /*
777 * If current allocation needs SSR,
778 * it had better in-place writes for updated data.
779 */
780 if (unlikely(fio->blk_addr != NEW_ADDR &&
781 !is_cold_data(page) &&
782 need_inplace_update(inode))) {
783 rewrite_data_page(page, fio);
784 set_inode_flag(F2FS_I(inode), FI_UPDATE_WRITE);
785 } else {
786 write_data_page(page, &dn, fio);
787 update_extent_cache(fio->blk_addr, &dn);
788 set_inode_flag(F2FS_I(inode), FI_APPEND_WRITE);
789 }
790 out_writepage:
791 f2fs_put_dnode(&dn);
792 return err;
793 }
794
795 static int f2fs_write_data_page(struct page *page,
796 struct writeback_control *wbc)
797 {
798 struct inode *inode = page->mapping->host;
799 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
800 loff_t i_size = i_size_read(inode);
801 const pgoff_t end_index = ((unsigned long long) i_size)
802 >> PAGE_CACHE_SHIFT;
803 unsigned offset = 0;
804 bool need_balance_fs = false;
805 int err = 0;
806 struct f2fs_io_info fio = {
807 .type = DATA,
808 .rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE,
809 };
810
811 trace_f2fs_writepage(page, DATA);
812
813 if (page->index < end_index)
814 goto write;
815
816 /*
817 * If the offset is out-of-range of file size,
818 * this page does not have to be written to disk.
819 */
820 offset = i_size & (PAGE_CACHE_SIZE - 1);
821 if ((page->index >= end_index + 1) || !offset)
822 goto out;
823
824 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
825 write:
826 if (unlikely(sbi->por_doing))
827 goto redirty_out;
828 if (f2fs_is_drop_cache(inode))
829 goto out;
830 if (f2fs_is_volatile_file(inode) && !wbc->for_reclaim &&
831 available_free_memory(sbi, BASE_CHECK))
832 goto redirty_out;
833
834 /* Dentry blocks are controlled by checkpoint */
835 if (S_ISDIR(inode->i_mode)) {
836 if (unlikely(f2fs_cp_error(sbi)))
837 goto redirty_out;
838 err = do_write_data_page(page, &fio);
839 goto done;
840 }
841
842 /* we should bypass data pages to proceed the kworkder jobs */
843 if (unlikely(f2fs_cp_error(sbi))) {
844 SetPageError(page);
845 unlock_page(page);
846 goto out;
847 }
848
849 if (!wbc->for_reclaim)
850 need_balance_fs = true;
851 else if (has_not_enough_free_secs(sbi, 0))
852 goto redirty_out;
853
854 err = -EAGAIN;
855 f2fs_lock_op(sbi);
856 if (f2fs_has_inline_data(inode))
857 err = f2fs_write_inline_data(inode, page);
858 if (err == -EAGAIN)
859 err = do_write_data_page(page, &fio);
860 f2fs_unlock_op(sbi);
861 done:
862 if (err && err != -ENOENT)
863 goto redirty_out;
864
865 clear_cold_data(page);
866 out:
867 inode_dec_dirty_pages(inode);
868 unlock_page(page);
869 if (need_balance_fs)
870 f2fs_balance_fs(sbi);
871 if (wbc->for_reclaim)
872 f2fs_submit_merged_bio(sbi, DATA, WRITE);
873 return 0;
874
875 redirty_out:
876 redirty_page_for_writepage(wbc, page);
877 return AOP_WRITEPAGE_ACTIVATE;
878 }
879
880 static int __f2fs_writepage(struct page *page, struct writeback_control *wbc,
881 void *data)
882 {
883 struct address_space *mapping = data;
884 int ret = mapping->a_ops->writepage(page, wbc);
885 mapping_set_error(mapping, ret);
886 return ret;
887 }
888
889 static int f2fs_write_data_pages(struct address_space *mapping,
890 struct writeback_control *wbc)
891 {
892 struct inode *inode = mapping->host;
893 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
894 bool locked = false;
895 int ret;
896 long diff;
897
898 trace_f2fs_writepages(mapping->host, wbc, DATA);
899
900 /* deal with chardevs and other special file */
901 if (!mapping->a_ops->writepage)
902 return 0;
903
904 if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE &&
905 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
906 available_free_memory(sbi, DIRTY_DENTS))
907 goto skip_write;
908
909 diff = nr_pages_to_write(sbi, DATA, wbc);
910
911 if (!S_ISDIR(inode->i_mode)) {
912 mutex_lock(&sbi->writepages);
913 locked = true;
914 }
915 ret = write_cache_pages(mapping, wbc, __f2fs_writepage, mapping);
916 if (locked)
917 mutex_unlock(&sbi->writepages);
918
919 f2fs_submit_merged_bio(sbi, DATA, WRITE);
920
921 remove_dirty_dir_inode(inode);
922
923 wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff);
924 return ret;
925
926 skip_write:
927 wbc->pages_skipped += get_dirty_pages(inode);
928 return 0;
929 }
930
931 static void f2fs_write_failed(struct address_space *mapping, loff_t to)
932 {
933 struct inode *inode = mapping->host;
934
935 if (to > inode->i_size) {
936 truncate_pagecache(inode, inode->i_size);
937 truncate_blocks(inode, inode->i_size, true);
938 }
939 }
940
941 static int f2fs_write_begin(struct file *file, struct address_space *mapping,
942 loff_t pos, unsigned len, unsigned flags,
943 struct page **pagep, void **fsdata)
944 {
945 struct inode *inode = mapping->host;
946 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
947 struct page *page, *ipage;
948 pgoff_t index = ((unsigned long long) pos) >> PAGE_CACHE_SHIFT;
949 struct dnode_of_data dn;
950 int err = 0;
951
952 trace_f2fs_write_begin(inode, pos, len, flags);
953
954 f2fs_balance_fs(sbi);
955
956 /*
957 * We should check this at this moment to avoid deadlock on inode page
958 * and #0 page. The locking rule for inline_data conversion should be:
959 * lock_page(page #0) -> lock_page(inode_page)
960 */
961 if (index != 0) {
962 err = f2fs_convert_inline_inode(inode);
963 if (err)
964 goto fail;
965 }
966 repeat:
967 page = grab_cache_page_write_begin(mapping, index, flags);
968 if (!page) {
969 err = -ENOMEM;
970 goto fail;
971 }
972
973 *pagep = page;
974
975 f2fs_lock_op(sbi);
976
977 /* check inline_data */
978 ipage = get_node_page(sbi, inode->i_ino);
979 if (IS_ERR(ipage)) {
980 err = PTR_ERR(ipage);
981 goto unlock_fail;
982 }
983
984 set_new_dnode(&dn, inode, ipage, ipage, 0);
985
986 if (f2fs_has_inline_data(inode)) {
987 if (pos + len <= MAX_INLINE_DATA) {
988 read_inline_data(page, ipage);
989 set_inode_flag(F2FS_I(inode), FI_DATA_EXIST);
990 sync_inode_page(&dn);
991 goto put_next;
992 }
993 err = f2fs_convert_inline_page(&dn, page);
994 if (err)
995 goto put_fail;
996 }
997 err = f2fs_reserve_block(&dn, index);
998 if (err)
999 goto put_fail;
1000 put_next:
1001 f2fs_put_dnode(&dn);
1002 f2fs_unlock_op(sbi);
1003
1004 if ((len == PAGE_CACHE_SIZE) || PageUptodate(page))
1005 return 0;
1006
1007 f2fs_wait_on_page_writeback(page, DATA);
1008
1009 if ((pos & PAGE_CACHE_MASK) >= i_size_read(inode)) {
1010 unsigned start = pos & (PAGE_CACHE_SIZE - 1);
1011 unsigned end = start + len;
1012
1013 /* Reading beyond i_size is simple: memset to zero */
1014 zero_user_segments(page, 0, start, end, PAGE_CACHE_SIZE);
1015 goto out;
1016 }
1017
1018 if (dn.data_blkaddr == NEW_ADDR) {
1019 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
1020 } else {
1021 struct f2fs_io_info fio = {
1022 .type = DATA,
1023 .rw = READ_SYNC,
1024 .blk_addr = dn.data_blkaddr,
1025 };
1026 err = f2fs_submit_page_bio(sbi, page, &fio);
1027 if (err)
1028 goto fail;
1029
1030 lock_page(page);
1031 if (unlikely(!PageUptodate(page))) {
1032 f2fs_put_page(page, 1);
1033 err = -EIO;
1034 goto fail;
1035 }
1036 if (unlikely(page->mapping != mapping)) {
1037 f2fs_put_page(page, 1);
1038 goto repeat;
1039 }
1040 }
1041 out:
1042 SetPageUptodate(page);
1043 clear_cold_data(page);
1044 return 0;
1045
1046 put_fail:
1047 f2fs_put_dnode(&dn);
1048 unlock_fail:
1049 f2fs_unlock_op(sbi);
1050 f2fs_put_page(page, 1);
1051 fail:
1052 f2fs_write_failed(mapping, pos + len);
1053 return err;
1054 }
1055
1056 static int f2fs_write_end(struct file *file,
1057 struct address_space *mapping,
1058 loff_t pos, unsigned len, unsigned copied,
1059 struct page *page, void *fsdata)
1060 {
1061 struct inode *inode = page->mapping->host;
1062
1063 trace_f2fs_write_end(inode, pos, len, copied);
1064
1065 set_page_dirty(page);
1066
1067 if (pos + copied > i_size_read(inode)) {
1068 i_size_write(inode, pos + copied);
1069 mark_inode_dirty(inode);
1070 update_inode_page(inode);
1071 }
1072
1073 f2fs_put_page(page, 1);
1074 return copied;
1075 }
1076
1077 static int check_direct_IO(struct inode *inode, int rw,
1078 struct iov_iter *iter, loff_t offset)
1079 {
1080 unsigned blocksize_mask = inode->i_sb->s_blocksize - 1;
1081
1082 if (rw == READ)
1083 return 0;
1084
1085 if (offset & blocksize_mask)
1086 return -EINVAL;
1087
1088 if (iov_iter_alignment(iter) & blocksize_mask)
1089 return -EINVAL;
1090
1091 return 0;
1092 }
1093
1094 static ssize_t f2fs_direct_IO(int rw, struct kiocb *iocb,
1095 struct iov_iter *iter, loff_t offset)
1096 {
1097 struct file *file = iocb->ki_filp;
1098 struct address_space *mapping = file->f_mapping;
1099 struct inode *inode = mapping->host;
1100 size_t count = iov_iter_count(iter);
1101 int err;
1102
1103 /* we don't need to use inline_data strictly */
1104 if (f2fs_has_inline_data(inode)) {
1105 err = f2fs_convert_inline_inode(inode);
1106 if (err)
1107 return err;
1108 }
1109
1110 if (check_direct_IO(inode, rw, iter, offset))
1111 return 0;
1112
1113 trace_f2fs_direct_IO_enter(inode, offset, count, rw);
1114
1115 err = blockdev_direct_IO(rw, iocb, inode, iter, offset, get_data_block);
1116 if (err < 0 && (rw & WRITE))
1117 f2fs_write_failed(mapping, offset + count);
1118
1119 trace_f2fs_direct_IO_exit(inode, offset, count, rw, err);
1120
1121 return err;
1122 }
1123
1124 static void f2fs_invalidate_data_page(struct page *page, unsigned int offset,
1125 unsigned int length)
1126 {
1127 struct inode *inode = page->mapping->host;
1128
1129 if (offset % PAGE_CACHE_SIZE || length != PAGE_CACHE_SIZE)
1130 return;
1131
1132 if (PageDirty(page))
1133 inode_dec_dirty_pages(inode);
1134 ClearPagePrivate(page);
1135 }
1136
1137 static int f2fs_release_data_page(struct page *page, gfp_t wait)
1138 {
1139 ClearPagePrivate(page);
1140 return 1;
1141 }
1142
1143 static int f2fs_set_data_page_dirty(struct page *page)
1144 {
1145 struct address_space *mapping = page->mapping;
1146 struct inode *inode = mapping->host;
1147
1148 trace_f2fs_set_page_dirty(page, DATA);
1149
1150 SetPageUptodate(page);
1151
1152 if (f2fs_is_atomic_file(inode)) {
1153 register_inmem_page(inode, page);
1154 return 1;
1155 }
1156
1157 mark_inode_dirty(inode);
1158
1159 if (!PageDirty(page)) {
1160 __set_page_dirty_nobuffers(page);
1161 update_dirty_page(inode, page);
1162 return 1;
1163 }
1164 return 0;
1165 }
1166
1167 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
1168 {
1169 struct inode *inode = mapping->host;
1170
1171 /* we don't need to use inline_data strictly */
1172 if (f2fs_has_inline_data(inode)) {
1173 int err = f2fs_convert_inline_inode(inode);
1174 if (err)
1175 return err;
1176 }
1177 return generic_block_bmap(mapping, block, get_data_block);
1178 }
1179
1180 const struct address_space_operations f2fs_dblock_aops = {
1181 .readpage = f2fs_read_data_page,
1182 .readpages = f2fs_read_data_pages,
1183 .writepage = f2fs_write_data_page,
1184 .writepages = f2fs_write_data_pages,
1185 .write_begin = f2fs_write_begin,
1186 .write_end = f2fs_write_end,
1187 .set_page_dirty = f2fs_set_data_page_dirty,
1188 .invalidatepage = f2fs_invalidate_data_page,
1189 .releasepage = f2fs_release_data_page,
1190 .direct_IO = f2fs_direct_IO,
1191 .bmap = f2fs_bmap,
1192 };