projects / GitHub / MotorolaMobilityLLC / kernel-slsi.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
[RAMEN9610-21536]f2fs: avoid wrong decrypted data from disk
[GitHub/MotorolaMobilityLLC/kernel-slsi.git] / fs / f2fs / data.c
1 /*
2 * fs/f2fs/data.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11 #include <linux/fs.h>
12 #include <linux/f2fs_fs.h>
13 #include <linux/buffer_head.h>
14 #include <linux/mpage.h>
15 #include <linux/writeback.h>
16 #include <linux/backing-dev.h>
17 #include <linux/pagevec.h>
18 #include <linux/blkdev.h>
19 #include <linux/bio.h>
20 #include <linux/prefetch.h>
21 #include <linux/uio.h>
22 #include <linux/cleancache.h>
23 #include <linux/sched/signal.h>
24
25 #include "f2fs.h"
26 #include "node.h"
27 #include "segment.h"
28 #include "trace.h"
29 #include <trace/events/f2fs.h>
30 #include <trace/events/android_fs.h>
31
32 #define NUM_PREALLOC_POST_READ_CTXS 128
33
34 static struct kmem_cache *bio_post_read_ctx_cache;
35 static mempool_t *bio_post_read_ctx_pool;
36
37 static bool __is_cp_guaranteed(struct page *page)
38 {
39 struct address_space *mapping = page->mapping;
40 struct inode *inode;
41 struct f2fs_sb_info *sbi;
42
43 if (!mapping)
44 return false;
45
46 inode = mapping->host;
47 sbi = F2FS_I_SB(inode);
48
49 if (inode->i_ino == F2FS_META_INO(sbi) ||
50 inode->i_ino == F2FS_NODE_INO(sbi) ||
51 S_ISDIR(inode->i_mode) ||
52 is_cold_data(page))
53 return true;
54 return false;
55 }
56
57 /* postprocessing steps for read bios */
58 enum bio_post_read_step {
59 STEP_INITIAL = 0,
60 STEP_DECRYPT,
61 };
62
63 struct bio_post_read_ctx {
64 struct bio *bio;
65 struct work_struct work;
66 unsigned int cur_step;
67 unsigned int enabled_steps;
68 };
69
70 /* device unit number for iv sector */
71 #define PG_DUN(i,p) \
72 ((((i)->i_ino & 0xffffffff) << 32) | ((p)->index & 0xffffffff))
73
74 static inline bool f2fs_may_encrypt_bio(struct inode *inode,
75 struct f2fs_io_info *fio)
76 {
77 #ifdef CONFIG_CRYPTO_DISKCIPHER
78 if (fio && (fio->type != DATA || fio->encrypted_page))
79 return false;
80
81 return (f2fs_encrypted_file(inode) &&
82 fscrypt_disk_encrypted(inode));
83 #else
84 return false;
85 #endif
86 }
87
88 static inline bool f2fs_bio_disk_encrypted(unsigned int bi_opf)
89 {
90 if (bi_opf & REQ_CRYPT)
91 return true;
92 else
93 return false;
94 }
95
96 static bool f2fs_mergeable_bio(struct bio *bio, u64 dun, void *ci, bool bio_encrypted)
97 {
98 #ifdef CONFIG_CRYPTO_DISKCIPHER
99 if (!bio)
100 return true;
101
102 /* if both of them are not encrypted, no further check is needed */
103 if (!f2fs_bio_disk_encrypted(bio->bi_opf) && !bio_encrypted)
104 return true;
105
106 #ifdef CONFIG_CRYPTO_DISKCIPHER_DUN
107 if (bio->bi_aux_private == ci)
108 return bio_end_dun(bio) == dun;
109 else
110 return false;
111 #else
112 return bio->bi_aux_private == ci;
113 #endif
114 #else
115 return true;
116 #endif
117 }
118
119 static void __read_end_io(struct bio *bio)
120 {
121 struct page *page;
122 struct bio_vec *bv;
123 int i;
124
125 bio_for_each_segment_all(bv, bio, i) {
126 page = bv->bv_page;
127
128 /* PG_error was set if any post_read step failed */
129 if (bio->bi_status || PageError(page)) {
130 ClearPageUptodate(page);
131 SetPageError(page);
132 } else {
133 SetPageUptodate(page);
134 }
135 unlock_page(page);
136 }
137 if (bio->bi_private)
138 mempool_free(bio->bi_private, bio_post_read_ctx_pool);
139 bio_put(bio);
140 }
141
142 static void bio_post_read_processing(struct bio_post_read_ctx *ctx);
143
144 static void decrypt_work(struct work_struct *work)
145 {
146 struct bio_post_read_ctx *ctx =
147 container_of(work, struct bio_post_read_ctx, work);
148
149 fscrypt_decrypt_bio(ctx->bio);
150
151 bio_post_read_processing(ctx);
152 }
153
154 static void bio_post_read_processing(struct bio_post_read_ctx *ctx)
155 {
156 switch (++ctx->cur_step) {
157 case STEP_DECRYPT:
158 if (ctx->enabled_steps & (1 << STEP_DECRYPT)) {
159 INIT_WORK(&ctx->work, decrypt_work);
160 fscrypt_enqueue_decrypt_work(&ctx->work);
161 return;
162 }
163 ctx->cur_step++;
164 /* fall-through */
165 default:
166 __read_end_io(ctx->bio);
167 }
168 }
169
170 static bool f2fs_bio_post_read_required(struct bio *bio)
171 {
172 return bio->bi_private && !bio->bi_status;
173 }
174
175 static void f2fs_read_end_io(struct bio *bio)
176 {
177 #ifdef CONFIG_F2FS_FAULT_INJECTION
178 if (time_to_inject(F2FS_P_SB(bio->bi_io_vec->bv_page), FAULT_IO)) {
179 f2fs_show_injection_info(FAULT_IO);
180 bio->bi_status = BLK_STS_IOERR;
181 }
182 #endif
183
184 if(f2fs_bio_disk_encrypted(bio->bi_opf))
185 goto end_io;
186
187 if (f2fs_bio_post_read_required(bio)) {
188 struct bio_post_read_ctx *ctx = bio->bi_private;
189
190 ctx->cur_step = STEP_INITIAL;
191 bio_post_read_processing(ctx);
192 return;
193 }
194
195 end_io:
196 __read_end_io(bio);
197 }
198
199 static void f2fs_write_end_io(struct bio *bio)
200 {
201 struct f2fs_sb_info *sbi = bio->bi_private;
202 struct bio_vec *bvec;
203 int i;
204
205 bio_for_each_segment_all(bvec, bio, i) {
206 struct page *page = bvec->bv_page;
207 enum count_type type = WB_DATA_TYPE(page);
208
209 if (IS_DUMMY_WRITTEN_PAGE(page)) {
210 set_page_private(page, (unsigned long)NULL);
211 ClearPagePrivate(page);
212 unlock_page(page);
213 mempool_free(page, sbi->write_io_dummy);
214
215 if (unlikely(bio->bi_status))
216 f2fs_stop_checkpoint(sbi, true);
217 continue;
218 }
219
220 fscrypt_pullback_bio_page(&page, true);
221
222 if (unlikely(bio->bi_status)) {
223 mapping_set_error(page->mapping, -EIO);
224 if (type == F2FS_WB_CP_DATA)
225 f2fs_stop_checkpoint(sbi, true);
226 }
227
228 f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) &&
229 page->index != nid_of_node(page));
230
231 dec_page_count(sbi, type);
232 clear_cold_data(page);
233 end_page_writeback(page);
234 }
235 if (!get_pages(sbi, F2FS_WB_CP_DATA) &&
236 wq_has_sleeper(&sbi->cp_wait))
237 wake_up(&sbi->cp_wait);
238
239 bio_put(bio);
240 }
241
242 /*
243 * Return true, if pre_bio's bdev is same as its target device.
244 */
245 struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
246 block_t blk_addr, struct bio *bio)
247 {
248 struct block_device *bdev = sbi->sb->s_bdev;
249 int i;
250
251 for (i = 0; i < sbi->s_ndevs; i++) {
252 if (FDEV(i).start_blk <= blk_addr &&
253 FDEV(i).end_blk >= blk_addr) {
254 blk_addr -= FDEV(i).start_blk;
255 bdev = FDEV(i).bdev;
256 break;
257 }
258 }
259 if (bio) {
260 bio_set_dev(bio, bdev);
261 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
262 }
263 return bdev;
264 }
265
266 int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr)
267 {
268 int i;
269
270 for (i = 0; i < sbi->s_ndevs; i++)
271 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
272 return i;
273 return 0;
274 }
275
276 static bool __same_bdev(struct f2fs_sb_info *sbi,
277 block_t blk_addr, struct bio *bio)
278 {
279 struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL);
280 return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno;
281 }
282
283 /*
284 * Low-level block read/write IO operations.
285 */
286 static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
287 struct writeback_control *wbc,
288 int npages, bool is_read,
289 enum page_type type, enum temp_type temp)
290 {
291 struct bio *bio;
292
293 bio = f2fs_bio_alloc(sbi, npages, true);
294
295 f2fs_target_device(sbi, blk_addr, bio);
296 if (is_read) {
297 bio->bi_end_io = f2fs_read_end_io;
298 bio->bi_private = NULL;
299 } else {
300 bio->bi_end_io = f2fs_write_end_io;
301 bio->bi_private = sbi;
302 bio->bi_write_hint = io_type_to_rw_hint(sbi, type, temp);
303 }
304 if (wbc)
305 wbc_init_bio(wbc, bio);
306
307 return bio;
308 }
309
310 static inline void __submit_bio(struct f2fs_sb_info *sbi,
311 struct bio *bio, enum page_type type)
312 {
313 if (!is_read_io(bio_op(bio))) {
314 unsigned int start;
315
316 if (type != DATA && type != NODE)
317 goto submit_io;
318
319 if (f2fs_sb_has_blkzoned(sbi->sb) && current->plug)
320 blk_finish_plug(current->plug);
321
322 start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS;
323 start %= F2FS_IO_SIZE(sbi);
324
325 if (start == 0)
326 goto submit_io;
327
328 /* fill dummy pages */
329 for (; start < F2FS_IO_SIZE(sbi); start++) {
330 struct page *page =
331 mempool_alloc(sbi->write_io_dummy,
332 GFP_NOIO | __GFP_ZERO | __GFP_NOFAIL);
333 f2fs_bug_on(sbi, !page);
334
335 SetPagePrivate(page);
336 set_page_private(page, (unsigned long)DUMMY_WRITTEN_PAGE);
337 lock_page(page);
338 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
339 f2fs_bug_on(sbi, 1);
340 }
341 /*
342 * In the NODE case, we lose next block address chain. So, we
343 * need to do checkpoint in f2fs_sync_file.
344 */
345 if (type == NODE)
346 set_sbi_flag(sbi, SBI_NEED_CP);
347 }
348 submit_io:
349 if (is_read_io(bio_op(bio)))
350 trace_f2fs_submit_read_bio(sbi->sb, type, bio);
351 else
352 trace_f2fs_submit_write_bio(sbi->sb, type, bio);
353 submit_bio(bio);
354 }
355
356 static void __submit_merged_bio(struct f2fs_bio_info *io)
357 {
358 struct f2fs_io_info *fio = &io->fio;
359
360 if (!io->bio)
361 return;
362
363 if (f2fs_bio_disk_encrypted(io->bio->bi_opf))
364 bio_set_op_attrs(io->bio, fio->op, fio->op_flags | REQ_CRYPT);
365 else
366 bio_set_op_attrs(io->bio, fio->op, fio->op_flags);
367
368 if (is_read_io(fio->op))
369 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);
370 else
371 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);
372
373 __submit_bio(io->sbi, io->bio, fio->type);
374 io->bio = NULL;
375 }
376
377 static bool __has_merged_page(struct f2fs_bio_info *io,
378 struct inode *inode, nid_t ino, pgoff_t idx)
379 {
380 struct bio_vec *bvec;
381 struct page *target;
382 int i;
383
384 if (!io->bio)
385 return false;
386
387 if (!inode && !ino)
388 return true;
389
390 bio_for_each_segment_all(bvec, io->bio, i) {
391
392 if (bvec->bv_page->mapping)
393 target = bvec->bv_page;
394 else
395 target = fscrypt_control_page(bvec->bv_page);
396
397 if (idx != target->index)
398 continue;
399
400 if (inode && inode == target->mapping->host)
401 return true;
402 if (ino && ino == ino_of_node(target))
403 return true;
404 }
405
406 return false;
407 }
408
409 static bool has_merged_page(struct f2fs_sb_info *sbi, struct inode *inode,
410 nid_t ino, pgoff_t idx, enum page_type type)
411 {
412 enum page_type btype = PAGE_TYPE_OF_BIO(type);
413 enum temp_type temp;
414 struct f2fs_bio_info *io;
415 bool ret = false;
416
417 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
418 io = sbi->write_io[btype] + temp;
419
420 down_read(&io->io_rwsem);
421 ret = __has_merged_page(io, inode, ino, idx);
422 up_read(&io->io_rwsem);
423
424 /* TODO: use HOT temp only for meta pages now. */
425 if (ret || btype == META)
426 break;
427 }
428 return ret;
429 }
430
431 static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
432 enum page_type type, enum temp_type temp)
433 {
434 enum page_type btype = PAGE_TYPE_OF_BIO(type);
435 struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
436
437 down_write(&io->io_rwsem);
438
439 /* change META to META_FLUSH in the checkpoint procedure */
440 if (type >= META_FLUSH) {
441 io->fio.type = META_FLUSH;
442 io->fio.op = REQ_OP_WRITE;
443 io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC;
444 if (!test_opt(sbi, NOBARRIER))
445 io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
446 }
447 __submit_merged_bio(io);
448 up_write(&io->io_rwsem);
449 }
450
451 static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
452 struct inode *inode, nid_t ino, pgoff_t idx,
453 enum page_type type, bool force)
454 {
455 enum temp_type temp;
456
457 if (!force && !has_merged_page(sbi, inode, ino, idx, type))
458 return;
459
460 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
461
462 __f2fs_submit_merged_write(sbi, type, temp);
463
464 /* TODO: use HOT temp only for meta pages now. */
465 if (type >= META)
466 break;
467 }
468 }
469
470 void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type)
471 {
472 __submit_merged_write_cond(sbi, NULL, 0, 0, type, true);
473 }
474
475 void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
476 struct inode *inode, nid_t ino, pgoff_t idx,
477 enum page_type type)
478 {
479 __submit_merged_write_cond(sbi, inode, ino, idx, type, false);
480 }
481
482 void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi)
483 {
484 f2fs_submit_merged_write(sbi, DATA);
485 f2fs_submit_merged_write(sbi, NODE);
486 f2fs_submit_merged_write(sbi, META);
487 }
488
489 /*
490 * Fill the locked page with data located in the block address.
491 * A caller needs to unlock the page on failure.
492 */
493 int f2fs_submit_page_bio(struct f2fs_io_info *fio)
494 {
495 struct bio *bio;
496 struct page *page = fio->encrypted_page ?
497 fio->encrypted_page : fio->page;
498 struct inode *inode = fio->page->mapping->host;
499
500 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,
501 __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC))
502 return -EFAULT;
503
504 trace_f2fs_submit_page_bio(page, fio);
505 f2fs_trace_ios(fio, 0);
506
507 /* Allocate a new bio */
508 bio = __bio_alloc(fio->sbi, fio->new_blkaddr, fio->io_wbc,
509 1, is_read_io(fio->op), fio->type, fio->temp);
510
511 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
512 bio_put(bio);
513 return -EFAULT;
514 }
515 fio->op_flags |= fio->encrypted_page ? REQ_NOENCRYPT : 0;
516 bio_set_op_attrs(bio, fio->op, fio->op_flags);
517
518 if (f2fs_may_encrypt_bio(inode, fio))
519 fscrypt_set_bio(inode, bio, PG_DUN(inode, fio->page));
520
521 if (!is_read_io(fio->op))
522 inc_page_count(fio->sbi, WB_DATA_TYPE(fio->page));
523
524 __submit_bio(fio->sbi, bio, fio->type);
525 return 0;
526 }
527
528 int f2fs_submit_page_write(struct f2fs_io_info *fio)
529 {
530 struct f2fs_sb_info *sbi = fio->sbi;
531 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
532 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
533 struct page *bio_page;
534 int err = 0;
535 struct inode *inode;
536 bool bio_encrypted;
537 u64 dun;
538
539 f2fs_bug_on(sbi, is_read_io(fio->op));
540
541 down_write(&io->io_rwsem);
542 next:
543 if (fio->in_list) {
544 spin_lock(&io->io_lock);
545 if (list_empty(&io->io_list)) {
546 spin_unlock(&io->io_lock);
547 goto out_fail;
548 }
549 fio = list_first_entry(&io->io_list,
550 struct f2fs_io_info, list);
551 list_del(&fio->list);
552 spin_unlock(&io->io_lock);
553 }
554
555 if (__is_valid_data_blkaddr(fio->old_blkaddr))
556 verify_block_addr(fio, fio->old_blkaddr);
557 verify_block_addr(fio, fio->new_blkaddr);
558
559 bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page;
560 inode = fio->page->mapping->host;
561 dun = PG_DUN(inode, fio->page);
562 bio_encrypted = f2fs_may_encrypt_bio(inode, fio);
563 fio->op_flags |= fio->encrypted_page ? REQ_NOENCRYPT : 0;
564
565 /* set submitted = true as a return value */
566 fio->submitted = true;
567
568 inc_page_count(sbi, WB_DATA_TYPE(bio_page));
569
570 if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 ||
571 (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) ||
572 !__same_bdev(sbi, fio->new_blkaddr, io->bio)))
573 __submit_merged_bio(io);
574
575 if (!f2fs_mergeable_bio(io->bio, dun, fscrypt_get_diskcipher(inode), bio_encrypted))
576 __submit_merged_bio(io);
577
578 alloc_new:
579 if (io->bio == NULL) {
580 if ((fio->type == DATA || fio->type == NODE) &&
581 fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) {
582 err = -EAGAIN;
583 dec_page_count(sbi, WB_DATA_TYPE(bio_page));
584 goto out_fail;
585 }
586 io->bio = __bio_alloc(sbi, fio->new_blkaddr, fio->io_wbc,
587 BIO_MAX_PAGES, false,
588 fio->type, fio->temp);
589 if (bio_encrypted)
590 fscrypt_set_bio(inode, io->bio, dun);
591
592 io->fio = *fio;
593 }
594
595 if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) {
596 __submit_merged_bio(io);
597 goto alloc_new;
598 }
599
600 if (fio->io_wbc)
601 wbc_account_io(fio->io_wbc, bio_page, PAGE_SIZE);
602
603 io->last_block_in_bio = fio->new_blkaddr;
604 f2fs_trace_ios(fio, 0);
605
606 trace_f2fs_submit_page_write(fio->page, fio);
607
608 if (fio->in_list)
609 goto next;
610 out_fail:
611 up_write(&io->io_rwsem);
612 return err;
613 }
614
615 static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
616 unsigned nr_pages)
617 {
618 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
619 struct bio *bio;
620 struct bio_post_read_ctx *ctx;
621 unsigned int post_read_steps = 0;
622
623 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC))
624 return ERR_PTR(-EFAULT);
625
626 bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES), false);
627 if (!bio)
628 return ERR_PTR(-ENOMEM);
629 f2fs_target_device(sbi, blkaddr, bio);
630 bio->bi_end_io = f2fs_read_end_io;
631 bio_set_op_attrs(bio, REQ_OP_READ, 0);
632
633 if (f2fs_encrypted_file(inode) && !fscrypt_disk_encrypted(inode))
634 post_read_steps |= 1 << STEP_DECRYPT;
635 if (post_read_steps) {
636 ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
637 if (!ctx) {
638 bio_put(bio);
639 return ERR_PTR(-ENOMEM);
640 }
641 ctx->bio = bio;
642 ctx->enabled_steps = post_read_steps;
643 bio->bi_private = ctx;
644 bio_set_op_attrs(bio, REQ_OP_READ,
645 (f2fs_encrypted_inode(inode) ?
646 REQ_NOENCRYPT :
647 0));
648 }
649
650 return bio;
651 }
652
653 /* This can handle encryption stuffs */
654 static int f2fs_submit_page_read(struct inode *inode, struct page *page,
655 block_t blkaddr)
656 {
657 struct bio *bio = f2fs_grab_read_bio(inode, blkaddr, 1);
658
659 if (IS_ERR(bio))
660 return PTR_ERR(bio);
661
662 /* wait for GCed page writeback via META_MAPPING */
663 f2fs_wait_on_block_writeback(inode, blkaddr);
664
665 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
666 bio_put(bio);
667 return -EFAULT;
668 }
669
670 if (f2fs_may_encrypt_bio(inode, NULL))
671 fscrypt_set_bio(inode, bio, PG_DUN(inode, page));
672
673 __submit_bio(F2FS_I_SB(inode), bio, DATA);
674 return 0;
675 }
676
677 static void __set_data_blkaddr(struct dnode_of_data *dn)
678 {
679 struct f2fs_node *rn = F2FS_NODE(dn->node_page);
680 __le32 *addr_array;
681 int base = 0;
682
683 if (IS_INODE(dn->node_page) && f2fs_has_extra_attr(dn->inode))
684 base = get_extra_isize(dn->inode);
685
686 /* Get physical address of data block */
687 addr_array = blkaddr_in_node(rn);
688 addr_array[base + dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
689 }
690
691 /*
692 * Lock ordering for the change of data block address:
693 * ->data_page
694 * ->node_page
695 * update block addresses in the node page
696 */
697 void set_data_blkaddr(struct dnode_of_data *dn)
698 {
699 f2fs_wait_on_page_writeback(dn->node_page, NODE, true);
700 __set_data_blkaddr(dn);
701 if (set_page_dirty(dn->node_page))
702 dn->node_changed = true;
703 }
704
705 void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
706 {
707 dn->data_blkaddr = blkaddr;
708 set_data_blkaddr(dn);
709 f2fs_update_extent_cache(dn);
710 }
711
712 /* dn->ofs_in_node will be returned with up-to-date last block pointer */
713 int reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count)
714 {
715 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
716 int err;
717
718 if (!count)
719 return 0;
720
721 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
722 return -EPERM;
723 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
724 return err;
725
726 trace_f2fs_reserve_new_blocks(dn->inode, dn->nid,
727 dn->ofs_in_node, count);
728
729 f2fs_wait_on_page_writeback(dn->node_page, NODE, true);
730
731 for (; count > 0; dn->ofs_in_node++) {
732 block_t blkaddr = datablock_addr(dn->inode,
733 dn->node_page, dn->ofs_in_node);
734 if (blkaddr == NULL_ADDR) {
735 dn->data_blkaddr = NEW_ADDR;
736 __set_data_blkaddr(dn);
737 count--;
738 }
739 }
740
741 if (set_page_dirty(dn->node_page))
742 dn->node_changed = true;
743 return 0;
744 }
745
746 /* Should keep dn->ofs_in_node unchanged */
747 int reserve_new_block(struct dnode_of_data *dn)
748 {
749 unsigned int ofs_in_node = dn->ofs_in_node;
750 int ret;
751
752 ret = reserve_new_blocks(dn, 1);
753 dn->ofs_in_node = ofs_in_node;
754 return ret;
755 }
756
757 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
758 {
759 bool need_put = dn->inode_page ? false : true;
760 int err;
761
762 err = get_dnode_of_data(dn, index, ALLOC_NODE);
763 if (err)
764 return err;
765
766 if (dn->data_blkaddr == NULL_ADDR)
767 err = reserve_new_block(dn);
768 if (err || need_put)
769 f2fs_put_dnode(dn);
770 return err;
771 }
772
773 int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
774 {
775 struct extent_info ei = {0,0,0};
776 struct inode *inode = dn->inode;
777
778 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
779 dn->data_blkaddr = ei.blk + index - ei.fofs;
780 return 0;
781 }
782
783 return f2fs_reserve_block(dn, index);
784 }
785
786 struct page *get_read_data_page(struct inode *inode, pgoff_t index,
787 int op_flags, bool for_write)
788 {
789 struct address_space *mapping = inode->i_mapping;
790 struct dnode_of_data dn;
791 struct page *page;
792 struct extent_info ei = {0,0,0};
793 int err;
794
795 page = f2fs_grab_cache_page(mapping, index, for_write);
796 if (!page)
797 return ERR_PTR(-ENOMEM);
798
799 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
800 dn.data_blkaddr = ei.blk + index - ei.fofs;
801 goto got_it;
802 }
803
804 set_new_dnode(&dn, inode, NULL, NULL, 0);
805 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
806 if (err)
807 goto put_err;
808 f2fs_put_dnode(&dn);
809
810 if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
811 err = -ENOENT;
812 goto put_err;
813 }
814 got_it:
815 if (PageUptodate(page)) {
816 unlock_page(page);
817 return page;
818 }
819
820 /*
821 * A new dentry page is allocated but not able to be written, since its
822 * new inode page couldn't be allocated due to -ENOSPC.
823 * In such the case, its blkaddr can be remained as NEW_ADDR.
824 * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata.
825 */
826 if (dn.data_blkaddr == NEW_ADDR) {
827 zero_user_segment(page, 0, PAGE_SIZE);
828 if (!PageUptodate(page))
829 SetPageUptodate(page);
830 unlock_page(page);
831 return page;
832 }
833
834 err = f2fs_submit_page_read(inode, page, dn.data_blkaddr);
835 if (err)
836 goto put_err;
837 return page;
838
839 put_err:
840 f2fs_put_page(page, 1);
841 return ERR_PTR(err);
842 }
843
844 struct page *find_data_page(struct inode *inode, pgoff_t index)
845 {
846 struct address_space *mapping = inode->i_mapping;
847 struct page *page;
848
849 page = find_get_page(mapping, index);
850 if (page && PageUptodate(page))
851 return page;
852 f2fs_put_page(page, 0);
853
854 page = get_read_data_page(inode, index, 0, false);
855 if (IS_ERR(page))
856 return page;
857
858 if (PageUptodate(page))
859 return page;
860
861 wait_on_page_locked(page);
862 if (unlikely(!PageUptodate(page))) {
863 f2fs_put_page(page, 0);
864 return ERR_PTR(-EIO);
865 }
866 return page;
867 }
868
869 /*
870 * If it tries to access a hole, return an error.
871 * Because, the callers, functions in dir.c and GC, should be able to know
872 * whether this page exists or not.
873 */
874 struct page *get_lock_data_page(struct inode *inode, pgoff_t index,
875 bool for_write)
876 {
877 struct address_space *mapping = inode->i_mapping;
878 struct page *page;
879 repeat:
880 page = get_read_data_page(inode, index, 0, for_write);
881 if (IS_ERR(page))
882 return page;
883
884 /* wait for read completion */
885 lock_page(page);
886 if (unlikely(page->mapping != mapping)) {
887 f2fs_put_page(page, 1);
888 goto repeat;
889 }
890 if (unlikely(!PageUptodate(page))) {
891 f2fs_put_page(page, 1);
892 return ERR_PTR(-EIO);
893 }
894 return page;
895 }
896
897 /*
898 * Caller ensures that this data page is never allocated.
899 * A new zero-filled data page is allocated in the page cache.
900 *
901 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
902 * f2fs_unlock_op().
903 * Note that, ipage is set only by make_empty_dir, and if any error occur,
904 * ipage should be released by this function.
905 */
906 struct page *get_new_data_page(struct inode *inode,
907 struct page *ipage, pgoff_t index, bool new_i_size)
908 {
909 struct address_space *mapping = inode->i_mapping;
910 struct page *page;
911 struct dnode_of_data dn;
912 int err;
913
914 page = f2fs_grab_cache_page(mapping, index, true);
915 if (!page) {
916 /*
917 * before exiting, we should make sure ipage will be released
918 * if any error occur.
919 */
920 f2fs_put_page(ipage, 1);
921 return ERR_PTR(-ENOMEM);
922 }
923
924 set_new_dnode(&dn, inode, ipage, NULL, 0);
925 err = f2fs_reserve_block(&dn, index);
926 if (err) {
927 f2fs_put_page(page, 1);
928 return ERR_PTR(err);
929 }
930 if (!ipage)
931 f2fs_put_dnode(&dn);
932
933 if (PageUptodate(page))
934 goto got_it;
935
936 if (dn.data_blkaddr == NEW_ADDR) {
937 zero_user_segment(page, 0, PAGE_SIZE);
938 if (!PageUptodate(page))
939 SetPageUptodate(page);
940 } else {
941 f2fs_put_page(page, 1);
942
943 /* if ipage exists, blkaddr should be NEW_ADDR */
944 f2fs_bug_on(F2FS_I_SB(inode), ipage);
945 page = get_lock_data_page(inode, index, true);
946 if (IS_ERR(page))
947 return page;
948 }
949 got_it:
950 if (new_i_size && i_size_read(inode) <
951 ((loff_t)(index + 1) << PAGE_SHIFT))
952 f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT));
953 return page;
954 }
955
956 static int __allocate_data_block(struct dnode_of_data *dn, int seg_type)
957 {
958 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
959 struct f2fs_summary sum;
960 struct node_info ni;
961 pgoff_t fofs;
962 blkcnt_t count = 1;
963 int err;
964
965 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
966 return -EPERM;
967
968 dn->data_blkaddr = datablock_addr(dn->inode,
969 dn->node_page, dn->ofs_in_node);
970 if (dn->data_blkaddr == NEW_ADDR)
971 goto alloc;
972
973 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
974 return err;
975
976 alloc:
977 get_node_info(sbi, dn->nid, &ni);
978 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
979
980 allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr,
981 &sum, seg_type, NULL, false);
982 set_data_blkaddr(dn);
983
984 /* update i_size */
985 fofs = start_bidx_of_node(ofs_of_node(dn->node_page), dn->inode) +
986 dn->ofs_in_node;
987 if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_SHIFT))
988 f2fs_i_size_write(dn->inode,
989 ((loff_t)(fofs + 1) << PAGE_SHIFT));
990 return 0;
991 }
992
993 int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
994 {
995 struct inode *inode = file_inode(iocb->ki_filp);
996 struct f2fs_map_blocks map;
997 int flag;
998 int err = 0;
999 bool direct_io = iocb->ki_flags & IOCB_DIRECT;
1000
1001 /* convert inline data for Direct I/O*/
1002 if (direct_io) {
1003 err = f2fs_convert_inline_inode(inode);
1004 if (err)
1005 return err;
1006 }
1007
1008 if (is_inode_flag_set(inode, FI_NO_PREALLOC))
1009 return 0;
1010
1011 map.m_lblk = F2FS_BLK_ALIGN(iocb->ki_pos);
1012 map.m_len = F2FS_BYTES_TO_BLK(iocb->ki_pos + iov_iter_count(from));
1013 if (map.m_len > map.m_lblk)
1014 map.m_len -= map.m_lblk;
1015 else
1016 map.m_len = 0;
1017
1018 map.m_next_pgofs = NULL;
1019 map.m_next_extent = NULL;
1020 map.m_seg_type = NO_CHECK_TYPE;
1021
1022 if (direct_io) {
1023 map.m_seg_type = rw_hint_to_seg_type(iocb->ki_hint);
1024 flag = f2fs_force_buffered_io(inode, WRITE) ?
1025 F2FS_GET_BLOCK_PRE_AIO :
1026 F2FS_GET_BLOCK_PRE_DIO;
1027 goto map_blocks;
1028 }
1029 if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA(inode)) {
1030 err = f2fs_convert_inline_inode(inode);
1031 if (err)
1032 return err;
1033 }
1034 if (f2fs_has_inline_data(inode))
1035 return err;
1036
1037 flag = F2FS_GET_BLOCK_PRE_AIO;
1038
1039 map_blocks:
1040 err = f2fs_map_blocks(inode, &map, 1, flag);
1041 if (map.m_len > 0 && err == -ENOSPC) {
1042 if (!direct_io)
1043 set_inode_flag(inode, FI_NO_PREALLOC);
1044 err = 0;
1045 }
1046 return err;
1047 }
1048
1049 static inline void __do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock)
1050 {
1051 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
1052 if (lock)
1053 down_read(&sbi->node_change);
1054 else
1055 up_read(&sbi->node_change);
1056 } else {
1057 if (lock)
1058 f2fs_lock_op(sbi);
1059 else
1060 f2fs_unlock_op(sbi);
1061 }
1062 }
1063
1064 /*
1065 * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with
1066 * f2fs_map_blocks structure.
1067 * If original data blocks are allocated, then give them to blockdev.
1068 * Otherwise,
1069 * a. preallocate requested block addresses
1070 * b. do not use extent cache for better performance
1071 * c. give the block addresses to blockdev
1072 */
1073 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
1074 int create, int flag)
1075 {
1076 unsigned int maxblocks = map->m_len;
1077 struct dnode_of_data dn;
1078 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1079 int mode = create ? ALLOC_NODE : LOOKUP_NODE;
1080 pgoff_t pgofs, end_offset, end;
1081 int err = 0, ofs = 1;
1082 unsigned int ofs_in_node, last_ofs_in_node;
1083 blkcnt_t prealloc;
1084 struct extent_info ei = {0,0,0};
1085 block_t blkaddr;
1086 unsigned int start_pgofs;
1087
1088 if (!maxblocks)
1089 return 0;
1090
1091 map->m_len = 0;
1092 map->m_flags = 0;
1093
1094 /* it only supports block size == page size */
1095 pgofs = (pgoff_t)map->m_lblk;
1096 end = pgofs + maxblocks;
1097
1098 if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
1099 map->m_pblk = ei.blk + pgofs - ei.fofs;
1100 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs);
1101 map->m_flags = F2FS_MAP_MAPPED;
1102 if (map->m_next_extent)
1103 *map->m_next_extent = pgofs + map->m_len;
1104 goto out;
1105 }
1106
1107 next_dnode:
1108 if (create)
1109 __do_map_lock(sbi, flag, true);
1110
1111 /* When reading holes, we need its node page */
1112 set_new_dnode(&dn, inode, NULL, NULL, 0);
1113 err = get_dnode_of_data(&dn, pgofs, mode);
1114 if (err) {
1115 if (flag == F2FS_GET_BLOCK_BMAP)
1116 map->m_pblk = 0;
1117 if (err == -ENOENT) {
1118 err = 0;
1119 if (map->m_next_pgofs)
1120 *map->m_next_pgofs =
1121 get_next_page_offset(&dn, pgofs);
1122 if (map->m_next_extent)
1123 *map->m_next_extent =
1124 get_next_page_offset(&dn, pgofs);
1125 }
1126 goto unlock_out;
1127 }
1128
1129 start_pgofs = pgofs;
1130 prealloc = 0;
1131 last_ofs_in_node = ofs_in_node = dn.ofs_in_node;
1132 end_offset = ADDRS_PER_PAGE(dn.node_page, inode);
1133
1134 next_block:
1135 blkaddr = datablock_addr(dn.inode, dn.node_page, dn.ofs_in_node);
1136
1137 if (__is_valid_data_blkaddr(blkaddr) &&
1138 !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC)) {
1139 err = -EFAULT;
1140 goto sync_out;
1141 }
1142
1143 if (!is_valid_data_blkaddr(sbi, blkaddr)) {
1144 if (create) {
1145 if (unlikely(f2fs_cp_error(sbi))) {
1146 err = -EIO;
1147 goto sync_out;
1148 }
1149 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
1150 if (blkaddr == NULL_ADDR) {
1151 prealloc++;
1152 last_ofs_in_node = dn.ofs_in_node;
1153 }
1154 } else {
1155 err = __allocate_data_block(&dn,
1156 map->m_seg_type);
1157 if (!err)
1158 set_inode_flag(inode, FI_APPEND_WRITE);
1159 }
1160 if (err)
1161 goto sync_out;
1162 map->m_flags |= F2FS_MAP_NEW;
1163 blkaddr = dn.data_blkaddr;
1164 } else {
1165 if (flag == F2FS_GET_BLOCK_BMAP) {
1166 map->m_pblk = 0;
1167 goto sync_out;
1168 }
1169 if (flag == F2FS_GET_BLOCK_PRECACHE)
1170 goto sync_out;
1171 if (flag == F2FS_GET_BLOCK_FIEMAP &&
1172 blkaddr == NULL_ADDR) {
1173 if (map->m_next_pgofs)
1174 *map->m_next_pgofs = pgofs + 1;
1175 goto sync_out;
1176 }
1177 if (flag != F2FS_GET_BLOCK_FIEMAP) {
1178 /* for defragment case */
1179 if (map->m_next_pgofs)
1180 *map->m_next_pgofs = pgofs + 1;
1181 goto sync_out;
1182 }
1183 }
1184 }
1185
1186 if (flag == F2FS_GET_BLOCK_PRE_AIO)
1187 goto skip;
1188
1189 if (map->m_len == 0) {
1190 /* preallocated unwritten block should be mapped for fiemap. */
1191 if (blkaddr == NEW_ADDR)
1192 map->m_flags |= F2FS_MAP_UNWRITTEN;
1193 map->m_flags |= F2FS_MAP_MAPPED;
1194
1195 map->m_pblk = blkaddr;
1196 map->m_len = 1;
1197 } else if ((map->m_pblk != NEW_ADDR &&
1198 blkaddr == (map->m_pblk + ofs)) ||
1199 (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) ||
1200 flag == F2FS_GET_BLOCK_PRE_DIO) {
1201 ofs++;
1202 map->m_len++;
1203 } else {
1204 goto sync_out;
1205 }
1206
1207 skip:
1208 dn.ofs_in_node++;
1209 pgofs++;
1210
1211 /* preallocate blocks in batch for one dnode page */
1212 if (flag == F2FS_GET_BLOCK_PRE_AIO &&
1213 (pgofs == end || dn.ofs_in_node == end_offset)) {
1214
1215 dn.ofs_in_node = ofs_in_node;
1216 err = reserve_new_blocks(&dn, prealloc);
1217 if (err)
1218 goto sync_out;
1219
1220 map->m_len += dn.ofs_in_node - ofs_in_node;
1221 if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) {
1222 err = -ENOSPC;
1223 goto sync_out;
1224 }
1225 dn.ofs_in_node = end_offset;
1226 }
1227
1228 if (pgofs >= end)
1229 goto sync_out;
1230 else if (dn.ofs_in_node < end_offset)
1231 goto next_block;
1232
1233 if (flag == F2FS_GET_BLOCK_PRECACHE) {
1234 if (map->m_flags & F2FS_MAP_MAPPED) {
1235 unsigned int ofs = start_pgofs - map->m_lblk;
1236
1237 f2fs_update_extent_cache_range(&dn,
1238 start_pgofs, map->m_pblk + ofs,
1239 map->m_len - ofs);
1240 }
1241 }
1242
1243 f2fs_put_dnode(&dn);
1244
1245 if (create) {
1246 __do_map_lock(sbi, flag, false);
1247 f2fs_balance_fs(sbi, dn.node_changed);
1248 }
1249 goto next_dnode;
1250
1251 sync_out:
1252 if (flag == F2FS_GET_BLOCK_PRECACHE) {
1253 if (map->m_flags & F2FS_MAP_MAPPED) {
1254 unsigned int ofs = start_pgofs - map->m_lblk;
1255
1256 f2fs_update_extent_cache_range(&dn,
1257 start_pgofs, map->m_pblk + ofs,
1258 map->m_len - ofs);
1259 }
1260 if (map->m_next_extent)
1261 *map->m_next_extent = pgofs + 1;
1262 }
1263 f2fs_put_dnode(&dn);
1264 unlock_out:
1265 if (create) {
1266 __do_map_lock(sbi, flag, false);
1267 f2fs_balance_fs(sbi, dn.node_changed);
1268 }
1269 out:
1270 trace_f2fs_map_blocks(inode, map, err);
1271 return err;
1272 }
1273
1274 bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len)
1275 {
1276 struct f2fs_map_blocks map;
1277 block_t last_lblk;
1278 int err;
1279
1280 if (pos + len > i_size_read(inode))
1281 return false;
1282
1283 map.m_lblk = F2FS_BYTES_TO_BLK(pos);
1284 map.m_next_pgofs = NULL;
1285 map.m_next_extent = NULL;
1286 map.m_seg_type = NO_CHECK_TYPE;
1287 last_lblk = F2FS_BLK_ALIGN(pos + len);
1288
1289 while (map.m_lblk < last_lblk) {
1290 map.m_len = last_lblk - map.m_lblk;
1291 err = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_DEFAULT);
1292 if (err || map.m_len == 0)
1293 return false;
1294 map.m_lblk += map.m_len;
1295 }
1296 return true;
1297 }
1298
1299 static int __get_data_block(struct inode *inode, sector_t iblock,
1300 struct buffer_head *bh, int create, int flag,
1301 pgoff_t *next_pgofs, int seg_type)
1302 {
1303 struct f2fs_map_blocks map;
1304 int err;
1305
1306 map.m_lblk = iblock;
1307 map.m_len = bh->b_size >> inode->i_blkbits;
1308 map.m_next_pgofs = next_pgofs;
1309 map.m_next_extent = NULL;
1310 map.m_seg_type = seg_type;
1311
1312 err = f2fs_map_blocks(inode, &map, create, flag);
1313 if (!err) {
1314 map_bh(bh, inode->i_sb, map.m_pblk);
1315 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags;
1316 bh->b_size = (u64)map.m_len << inode->i_blkbits;
1317 }
1318 return err;
1319 }
1320
1321 static int get_data_block(struct inode *inode, sector_t iblock,
1322 struct buffer_head *bh_result, int create, int flag,
1323 pgoff_t *next_pgofs)
1324 {
1325 return __get_data_block(inode, iblock, bh_result, create,
1326 flag, next_pgofs,
1327 NO_CHECK_TYPE);
1328 }
1329
1330 static int get_data_block_dio(struct inode *inode, sector_t iblock,
1331 struct buffer_head *bh_result, int create)
1332 {
1333 return __get_data_block(inode, iblock, bh_result, create,
1334 F2FS_GET_BLOCK_DEFAULT, NULL,
1335 rw_hint_to_seg_type(
1336 inode->i_write_hint));
1337 }
1338
1339 static int get_data_block_bmap(struct inode *inode, sector_t iblock,
1340 struct buffer_head *bh_result, int create)
1341 {
1342 /* Block number less than F2FS MAX BLOCKS */
1343 if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks))
1344 return -EFBIG;
1345
1346 return __get_data_block(inode, iblock, bh_result, create,
1347 F2FS_GET_BLOCK_BMAP, NULL,
1348 NO_CHECK_TYPE);
1349 }
1350
1351 static inline sector_t logical_to_blk(struct inode *inode, loff_t offset)
1352 {
1353 return (offset >> inode->i_blkbits);
1354 }
1355
1356 static inline loff_t blk_to_logical(struct inode *inode, sector_t blk)
1357 {
1358 return (blk << inode->i_blkbits);
1359 }
1360
1361 static int f2fs_xattr_fiemap(struct inode *inode,
1362 struct fiemap_extent_info *fieinfo)
1363 {
1364 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1365 struct page *page;
1366 struct node_info ni;
1367 __u64 phys = 0, len;
1368 __u32 flags;
1369 nid_t xnid = F2FS_I(inode)->i_xattr_nid;
1370 int err = 0;
1371
1372 if (f2fs_has_inline_xattr(inode)) {
1373 int offset;
1374
1375 page = f2fs_grab_cache_page(NODE_MAPPING(sbi),
1376 inode->i_ino, false);
1377 if (!page)
1378 return -ENOMEM;
1379
1380 get_node_info(sbi, inode->i_ino, &ni);
1381
1382 phys = (__u64)blk_to_logical(inode, ni.blk_addr);
1383 offset = offsetof(struct f2fs_inode, i_addr) +
1384 sizeof(__le32) * (DEF_ADDRS_PER_INODE -
1385 get_inline_xattr_addrs(inode));
1386
1387 phys += offset;
1388 len = inline_xattr_size(inode);
1389
1390 f2fs_put_page(page, 1);
1391
1392 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED;
1393
1394 if (!xnid)
1395 flags |= FIEMAP_EXTENT_LAST;
1396
1397 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
1398 if (err || err == 1)
1399 return err;
1400 }
1401
1402 if (xnid) {
1403 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), xnid, false);
1404 if (!page)
1405 return -ENOMEM;
1406
1407 get_node_info(sbi, xnid, &ni);
1408
1409 phys = (__u64)blk_to_logical(inode, ni.blk_addr);
1410 len = inode->i_sb->s_blocksize;
1411
1412 f2fs_put_page(page, 1);
1413
1414 flags = FIEMAP_EXTENT_LAST;
1415 }
1416
1417 if (phys)
1418 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
1419
1420 return (err < 0 ? err : 0);
1421 }
1422
1423 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
1424 u64 start, u64 len)
1425 {
1426 struct buffer_head map_bh;
1427 sector_t start_blk, last_blk;
1428 pgoff_t next_pgofs;
1429 u64 logical = 0, phys = 0, size = 0;
1430 u32 flags = 0;
1431 int ret = 0;
1432
1433 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
1434 ret = f2fs_precache_extents(inode);
1435 if (ret)
1436 return ret;
1437 }
1438
1439 ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC | FIEMAP_FLAG_XATTR);
1440 if (ret)
1441 return ret;
1442
1443 inode_lock(inode);
1444
1445 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
1446 ret = f2fs_xattr_fiemap(inode, fieinfo);
1447 goto out;
1448 }
1449
1450 if (f2fs_has_inline_data(inode)) {
1451 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
1452 if (ret != -EAGAIN)
1453 goto out;
1454 }
1455
1456 if (logical_to_blk(inode, len) == 0)
1457 len = blk_to_logical(inode, 1);
1458
1459 start_blk = logical_to_blk(inode, start);
1460 last_blk = logical_to_blk(inode, start + len - 1);
1461
1462 next:
1463 memset(&map_bh, 0, sizeof(struct buffer_head));
1464 map_bh.b_size = len;
1465
1466 ret = get_data_block(inode, start_blk, &map_bh, 0,
1467 F2FS_GET_BLOCK_FIEMAP, &next_pgofs);
1468 if (ret)
1469 goto out;
1470
1471 /* HOLE */
1472 if (!buffer_mapped(&map_bh)) {
1473 start_blk = next_pgofs;
1474
1475 if (blk_to_logical(inode, start_blk) < blk_to_logical(inode,
1476 F2FS_I_SB(inode)->max_file_blocks))
1477 goto prep_next;
1478
1479 flags |= FIEMAP_EXTENT_LAST;
1480 }
1481
1482 if (size) {
1483 if (f2fs_encrypted_inode(inode))
1484 flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;
1485
1486 ret = fiemap_fill_next_extent(fieinfo, logical,
1487 phys, size, flags);
1488 }
1489
1490 if (start_blk > last_blk || ret)
1491 goto out;
1492
1493 logical = blk_to_logical(inode, start_blk);
1494 phys = blk_to_logical(inode, map_bh.b_blocknr);
1495 size = map_bh.b_size;
1496 flags = 0;
1497 if (buffer_unwritten(&map_bh))
1498 flags = FIEMAP_EXTENT_UNWRITTEN;
1499
1500 start_blk += logical_to_blk(inode, size);
1501
1502 prep_next:
1503 cond_resched();
1504 if (fatal_signal_pending(current))
1505 ret = -EINTR;
1506 else
1507 goto next;
1508 out:
1509 if (ret == 1)
1510 ret = 0;
1511
1512 inode_unlock(inode);
1513 return ret;
1514 }
1515
1516 /*
1517 * This function was originally taken from fs/mpage.c, and customized for f2fs.
1518 * Major change was from block_size == page_size in f2fs by default.
1519 */
1520 static int f2fs_mpage_readpages(struct address_space *mapping,
1521 struct list_head *pages, struct page *page,
1522 unsigned nr_pages)
1523 {
1524 struct bio *bio = NULL;
1525 sector_t last_block_in_bio = 0;
1526 struct inode *inode = mapping->host;
1527 const unsigned blkbits = inode->i_blkbits;
1528 const unsigned blocksize = 1 << blkbits;
1529 sector_t block_in_file;
1530 sector_t last_block;
1531 sector_t last_block_in_file;
1532 sector_t block_nr;
1533 struct f2fs_map_blocks map;
1534 bool bio_encrypted;
1535 u64 dun;
1536
1537 map.m_pblk = 0;
1538 map.m_lblk = 0;
1539 map.m_len = 0;
1540 map.m_flags = 0;
1541 map.m_next_pgofs = NULL;
1542 map.m_next_extent = NULL;
1543 map.m_seg_type = NO_CHECK_TYPE;
1544
1545 for (; nr_pages; nr_pages--) {
1546 if (pages) {
1547 page = list_last_entry(pages, struct page, lru);
1548
1549 prefetchw(&page->flags);
1550 list_del(&page->lru);
1551 if (add_to_page_cache_lru(page, mapping,
1552 page->index,
1553 readahead_gfp_mask(mapping)))
1554 goto next_page;
1555 }
1556
1557 block_in_file = (sector_t)page->index;
1558 last_block = block_in_file + nr_pages;
1559 last_block_in_file = (i_size_read(inode) + blocksize - 1) >>
1560 blkbits;
1561 if (last_block > last_block_in_file)
1562 last_block = last_block_in_file;
1563
1564 /*
1565 * Map blocks using the previous result first.
1566 */
1567 if ((map.m_flags & F2FS_MAP_MAPPED) &&
1568 block_in_file > map.m_lblk &&
1569 block_in_file < (map.m_lblk + map.m_len))
1570 goto got_it;
1571
1572 /*
1573 * Then do more f2fs_map_blocks() calls until we are
1574 * done with this page.
1575 */
1576 map.m_flags = 0;
1577
1578 if (block_in_file < last_block) {
1579 map.m_lblk = block_in_file;
1580 map.m_len = last_block - block_in_file;
1581
1582 if (f2fs_map_blocks(inode, &map, 0,
1583 F2FS_GET_BLOCK_DEFAULT))
1584 goto set_error_page;
1585 }
1586 got_it:
1587 if ((map.m_flags & F2FS_MAP_MAPPED)) {
1588 block_nr = map.m_pblk + block_in_file - map.m_lblk;
1589 SetPageMappedToDisk(page);
1590
1591 if (!PageUptodate(page) && !cleancache_get_page(page)) {
1592 SetPageUptodate(page);
1593 goto confused;
1594 }
1595
1596 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr,
1597 DATA_GENERIC))
1598 goto set_error_page;
1599 } else {
1600 zero_user_segment(page, 0, PAGE_SIZE);
1601 if (!PageUptodate(page))
1602 SetPageUptodate(page);
1603 unlock_page(page);
1604 goto next_page;
1605 }
1606
1607 /*
1608 * This page will go to BIO. Do we need to send this
1609 * BIO off first?
1610 */
1611 if (bio && (last_block_in_bio != block_nr - 1 ||
1612 !__same_bdev(F2FS_I_SB(inode), block_nr, bio))) {
1613 submit_and_realloc:
1614 __submit_bio(F2FS_I_SB(inode), bio, DATA);
1615 bio = NULL;
1616 }
1617
1618 dun = PG_DUN(inode, page);
1619 bio_encrypted = f2fs_may_encrypt_bio(inode, NULL);
1620 if (!f2fs_mergeable_bio(bio, dun, fscrypt_get_diskcipher(inode), bio_encrypted)) {
1621 __submit_bio(F2FS_I_SB(inode), bio, DATA);
1622 bio = NULL;
1623 }
1624
1625 if (bio == NULL) {
1626 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages);
1627 if (IS_ERR(bio)) {
1628 bio = NULL;
1629 goto set_error_page;
1630 }
1631 if (f2fs_may_encrypt_bio(inode, NULL))
1632 fscrypt_set_bio(inode, bio, dun);
1633 }
1634
1635 /*
1636 * If the page is under writeback, we need to wait for
1637 * its completion to see the correct decrypted data.
1638 */
1639 f2fs_wait_on_block_writeback(inode, block_nr);
1640
1641 if (bio_add_page(bio, page, blocksize, 0) < blocksize)
1642 goto submit_and_realloc;
1643
1644 last_block_in_bio = block_nr;
1645 goto next_page;
1646 set_error_page:
1647 SetPageError(page);
1648 zero_user_segment(page, 0, PAGE_SIZE);
1649 unlock_page(page);
1650 goto next_page;
1651 confused:
1652 if (bio) {
1653 __submit_bio(F2FS_I_SB(inode), bio, DATA);
1654 bio = NULL;
1655 }
1656 unlock_page(page);
1657 next_page:
1658 if (pages)
1659 put_page(page);
1660 }
1661 BUG_ON(pages && !list_empty(pages));
1662 if (bio)
1663 __submit_bio(F2FS_I_SB(inode), bio, DATA);
1664 return 0;
1665 }
1666
1667 static int f2fs_read_data_page(struct file *file, struct page *page)
1668 {
1669 struct inode *inode = page->mapping->host;
1670 int ret = -EAGAIN;
1671
1672 trace_f2fs_readpage(page, DATA);
1673
1674 /* If the file has inline data, try to read it directly */
1675 if (f2fs_has_inline_data(inode))
1676 ret = f2fs_read_inline_data(inode, page);
1677 if (ret == -EAGAIN)
1678 ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1);
1679 return ret;
1680 }
1681
1682 static int f2fs_read_data_pages(struct file *file,
1683 struct address_space *mapping,
1684 struct list_head *pages, unsigned nr_pages)
1685 {
1686 struct inode *inode = mapping->host;
1687 struct page *page = list_last_entry(pages, struct page, lru);
1688
1689 trace_f2fs_readpages(inode, page, nr_pages);
1690
1691 /* If the file has inline data, skip readpages */
1692 if (f2fs_has_inline_data(inode))
1693 return 0;
1694
1695 return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages);
1696 }
1697
1698 static int encrypt_one_page(struct f2fs_io_info *fio)
1699 {
1700 struct inode *inode = fio->page->mapping->host;
1701 gfp_t gfp_flags = GFP_NOFS;
1702
1703 if (!f2fs_encrypted_file(inode))
1704 return 0;
1705
1706 /* wait for GCed page writeback via META_MAPPING */
1707 f2fs_wait_on_block_writeback(inode, fio->old_blkaddr);
1708
1709 retry_encrypt:
1710 if (fscrypt_disk_encrypted(inode))
1711 return 0;
1712
1713 fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page,
1714 PAGE_SIZE, 0, fio->page->index, gfp_flags);
1715 if (!IS_ERR(fio->encrypted_page))
1716 return 0;
1717
1718 /* flush pending IOs and wait for a while in the ENOMEM case */
1719 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
1720 f2fs_flush_merged_writes(fio->sbi);
1721 congestion_wait(BLK_RW_ASYNC, HZ/50);
1722 gfp_flags |= __GFP_NOFAIL;
1723 goto retry_encrypt;
1724 }
1725 return PTR_ERR(fio->encrypted_page);
1726 }
1727
1728 static inline bool check_inplace_update_policy(struct inode *inode,
1729 struct f2fs_io_info *fio)
1730 {
1731 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1732 unsigned int policy = SM_I(sbi)->ipu_policy;
1733
1734 if (policy & (0x1 << F2FS_IPU_FORCE))
1735 return true;
1736 if (policy & (0x1 << F2FS_IPU_SSR) && need_SSR(sbi))
1737 return true;
1738 if (policy & (0x1 << F2FS_IPU_UTIL) &&
1739 utilization(sbi) > SM_I(sbi)->min_ipu_util)
1740 return true;
1741 if (policy & (0x1 << F2FS_IPU_SSR_UTIL) && need_SSR(sbi) &&
1742 utilization(sbi) > SM_I(sbi)->min_ipu_util)
1743 return true;
1744
1745 /*
1746 * IPU for rewrite async pages
1747 */
1748 if (policy & (0x1 << F2FS_IPU_ASYNC) &&
1749 fio && fio->op == REQ_OP_WRITE &&
1750 !(fio->op_flags & REQ_SYNC) &&
1751 !f2fs_encrypted_inode(inode))
1752 return true;
1753
1754 /* this is only set during fdatasync */
1755 if (policy & (0x1 << F2FS_IPU_FSYNC) &&
1756 is_inode_flag_set(inode, FI_NEED_IPU))
1757 return true;
1758
1759 return false;
1760 }
1761
1762 bool should_update_inplace(struct inode *inode, struct f2fs_io_info *fio)
1763 {
1764 if (f2fs_is_pinned_file(inode))
1765 return true;
1766
1767 /* if this is cold file, we should overwrite to avoid fragmentation */
1768 if (file_is_cold(inode))
1769 return true;
1770
1771 return check_inplace_update_policy(inode, fio);
1772 }
1773
1774 bool should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)
1775 {
1776 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1777
1778 if (test_opt(sbi, LFS))
1779 return true;
1780 if (S_ISDIR(inode->i_mode))
1781 return true;
1782 if (f2fs_is_atomic_file(inode))
1783 return true;
1784 if (fio) {
1785 if (is_cold_data(fio->page))
1786 return true;
1787 if (IS_ATOMIC_WRITTEN_PAGE(fio->page))
1788 return true;
1789 }
1790 return false;
1791 }
1792
1793 static inline bool need_inplace_update(struct f2fs_io_info *fio)
1794 {
1795 struct inode *inode = fio->page->mapping->host;
1796
1797 if (should_update_outplace(inode, fio))
1798 return false;
1799
1800 return should_update_inplace(inode, fio);
1801 }
1802
1803 int do_write_data_page(struct f2fs_io_info *fio)
1804 {
1805 struct page *page = fio->page;
1806 struct inode *inode = page->mapping->host;
1807 struct dnode_of_data dn;
1808 struct extent_info ei = {0,0,0};
1809 bool ipu_force = false;
1810 int err = 0;
1811
1812 set_new_dnode(&dn, inode, NULL, NULL, 0);
1813 if (need_inplace_update(fio) &&
1814 f2fs_lookup_extent_cache(inode, page->index, &ei)) {
1815 fio->old_blkaddr = ei.blk + page->index - ei.fofs;
1816
1817 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
1818 DATA_GENERIC))
1819 return -EFAULT;
1820
1821 ipu_force = true;
1822 fio->need_lock = LOCK_DONE;
1823 goto got_it;
1824 }
1825
1826 /* Deadlock due to between page->lock and f2fs_lock_op */
1827 if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi))
1828 return -EAGAIN;
1829
1830 err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
1831 if (err)
1832 goto out;
1833
1834 fio->old_blkaddr = dn.data_blkaddr;
1835
1836 /* This page is already truncated */
1837 if (fio->old_blkaddr == NULL_ADDR) {
1838 ClearPageUptodate(page);
1839 goto out_writepage;
1840 }
1841 got_it:
1842 if (__is_valid_data_blkaddr(fio->old_blkaddr) &&
1843 !f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
1844 DATA_GENERIC)) {
1845 err = -EFAULT;
1846 goto out_writepage;
1847 }
1848 /*
1849 * If current allocation needs SSR,
1850 * it had better in-place writes for updated data.
1851 */
1852 if (ipu_force || (is_valid_data_blkaddr(fio->sbi, fio->old_blkaddr) &&
1853 need_inplace_update(fio))) {
1854 err = encrypt_one_page(fio);
1855 if (err)
1856 goto out_writepage;
1857
1858 set_page_writeback(page);
1859 ClearPageError(page);
1860 f2fs_put_dnode(&dn);
1861 if (fio->need_lock == LOCK_REQ)
1862 f2fs_unlock_op(fio->sbi);
1863 err = rewrite_data_page(fio);
1864 trace_f2fs_do_write_data_page(fio->page, IPU);
1865 set_inode_flag(inode, FI_UPDATE_WRITE);
1866 return err;
1867 }
1868
1869 if (fio->need_lock == LOCK_RETRY) {
1870 if (!f2fs_trylock_op(fio->sbi)) {
1871 err = -EAGAIN;
1872 goto out_writepage;
1873 }
1874 fio->need_lock = LOCK_REQ;
1875 }
1876
1877 err = encrypt_one_page(fio);
1878 if (err)
1879 goto out_writepage;
1880
1881 set_page_writeback(page);
1882 ClearPageError(page);
1883
1884 /* LFS mode write path */
1885 write_data_page(&dn, fio);
1886 trace_f2fs_do_write_data_page(page, OPU);
1887 set_inode_flag(inode, FI_APPEND_WRITE);
1888 if (page->index == 0)
1889 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
1890 out_writepage:
1891 f2fs_put_dnode(&dn);
1892 out:
1893 if (fio->need_lock == LOCK_REQ)
1894 f2fs_unlock_op(fio->sbi);
1895 return err;
1896 }
1897
1898 static int __write_data_page(struct page *page, bool *submitted,
1899 struct writeback_control *wbc,
1900 enum iostat_type io_type)
1901 {
1902 struct inode *inode = page->mapping->host;
1903 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1904 loff_t i_size = i_size_read(inode);
1905 const pgoff_t end_index = ((unsigned long long) i_size)
1906 >> PAGE_SHIFT;
1907 loff_t psize = (page->index + 1) << PAGE_SHIFT;
1908 unsigned offset = 0;
1909 bool need_balance_fs = false;
1910 int err = 0;
1911 struct f2fs_io_info fio = {
1912 .sbi = sbi,
1913 .ino = inode->i_ino,
1914 .type = DATA,
1915 .op = REQ_OP_WRITE,
1916 .op_flags = wbc_to_write_flags(wbc),
1917 .old_blkaddr = NULL_ADDR,
1918 .page = page,
1919 .encrypted_page = NULL,
1920 .submitted = false,
1921 .need_lock = LOCK_RETRY,
1922 .io_type = io_type,
1923 .io_wbc = wbc,
1924 };
1925
1926 trace_f2fs_writepage(page, DATA);
1927
1928 /* we should bypass data pages to proceed the kworkder jobs */
1929 if (unlikely(f2fs_cp_error(sbi))) {
1930 mapping_set_error(page->mapping, -EIO);
1931 goto out;
1932 }
1933
1934 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1935 goto redirty_out;
1936
1937 if (page->index < end_index)
1938 goto write;
1939
1940 /*
1941 * If the offset is out-of-range of file size,
1942 * this page does not have to be written to disk.
1943 */
1944 offset = i_size & (PAGE_SIZE - 1);
1945 if ((page->index >= end_index + 1) || !offset)
1946 goto out;
1947
1948 zero_user_segment(page, offset, PAGE_SIZE);
1949 write:
1950 if (f2fs_is_drop_cache(inode))
1951 goto out;
1952 /* we should not write 0'th page having journal header */
1953 if (f2fs_is_volatile_file(inode) && (!page->index ||
1954 (!wbc->for_reclaim &&
1955 available_free_memory(sbi, BASE_CHECK))))
1956 goto redirty_out;
1957
1958 /* Dentry blocks are controlled by checkpoint */
1959 if (S_ISDIR(inode->i_mode)) {
1960 fio.need_lock = LOCK_DONE;
1961 err = do_write_data_page(&fio);
1962 goto done;
1963 }
1964
1965 if (!wbc->for_reclaim)
1966 need_balance_fs = true;
1967 else if (has_not_enough_free_secs(sbi, 0, 0))
1968 goto redirty_out;
1969 else
1970 set_inode_flag(inode, FI_HOT_DATA);
1971
1972 err = -EAGAIN;
1973 if (f2fs_has_inline_data(inode)) {
1974 err = f2fs_write_inline_data(inode, page);
1975 if (!err)
1976 goto out;
1977 }
1978
1979 if (err == -EAGAIN) {
1980 err = do_write_data_page(&fio);
1981 if (err == -EAGAIN) {
1982 fio.need_lock = LOCK_REQ;
1983 err = do_write_data_page(&fio);
1984 }
1985 }
1986
1987 if (err) {
1988 file_set_keep_isize(inode);
1989 } else {
1990 down_write(&F2FS_I(inode)->i_sem);
1991 if (F2FS_I(inode)->last_disk_size < psize)
1992 F2FS_I(inode)->last_disk_size = psize;
1993 up_write(&F2FS_I(inode)->i_sem);
1994 }
1995
1996 done:
1997 if (err && err != -ENOENT)
1998 goto redirty_out;
1999
2000 out:
2001 inode_dec_dirty_pages(inode);
2002 if (err)
2003 ClearPageUptodate(page);
2004
2005 if (wbc->for_reclaim) {
2006 f2fs_submit_merged_write_cond(sbi, inode, 0, page->index, DATA);
2007 clear_inode_flag(inode, FI_HOT_DATA);
2008 remove_dirty_inode(inode);
2009 submitted = NULL;
2010 }
2011
2012 unlock_page(page);
2013 if (!S_ISDIR(inode->i_mode))
2014 f2fs_balance_fs(sbi, need_balance_fs);
2015
2016 if (unlikely(f2fs_cp_error(sbi))) {
2017 f2fs_submit_merged_write(sbi, DATA);
2018 submitted = NULL;
2019 }
2020
2021 if (submitted)
2022 *submitted = fio.submitted;
2023
2024 return 0;
2025
2026 redirty_out:
2027 redirty_page_for_writepage(wbc, page);
2028 /*
2029 * pageout() in MM traslates EAGAIN, so calls handle_write_error()
2030 * -> mapping_set_error() -> set_bit(AS_EIO, ...).
2031 * file_write_and_wait_range() will see EIO error, which is critical
2032 * to return value of fsync() followed by atomic_write failure to user.
2033 */
2034 if (!err || wbc->for_reclaim)
2035 return AOP_WRITEPAGE_ACTIVATE;
2036 unlock_page(page);
2037 return err;
2038 }
2039
2040 static int f2fs_write_data_page(struct page *page,
2041 struct writeback_control *wbc)
2042 {
2043 return __write_data_page(page, NULL, wbc, FS_DATA_IO);
2044 }
2045
2046 /*
2047 * This function was copied from write_cche_pages from mm/page-writeback.c.
2048 * The major change is making write step of cold data page separately from
2049 * warm/hot data page.
2050 */
2051 static int f2fs_write_cache_pages(struct address_space *mapping,
2052 struct writeback_control *wbc,
2053 enum iostat_type io_type)
2054 {
2055 int ret = 0;
2056 int done = 0;
2057 struct pagevec pvec;
2058 int nr_pages;
2059 pgoff_t uninitialized_var(writeback_index);
2060 pgoff_t index;
2061 pgoff_t end; /* Inclusive */
2062 pgoff_t done_index;
2063 pgoff_t last_idx = ULONG_MAX;
2064 int cycled;
2065 int range_whole = 0;
2066 int tag;
2067
2068 pagevec_init(&pvec, 0);
2069
2070 if (get_dirty_pages(mapping->host) <=
2071 SM_I(F2FS_M_SB(mapping))->min_hot_blocks)
2072 set_inode_flag(mapping->host, FI_HOT_DATA);
2073 else
2074 clear_inode_flag(mapping->host, FI_HOT_DATA);
2075
2076 if (wbc->range_cyclic) {
2077 writeback_index = mapping->writeback_index; /* prev offset */
2078 index = writeback_index;
2079 if (index == 0)
2080 cycled = 1;
2081 else
2082 cycled = 0;
2083 end = -1;
2084 } else {
2085 index = wbc->range_start >> PAGE_SHIFT;
2086 end = wbc->range_end >> PAGE_SHIFT;
2087 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2088 range_whole = 1;
2089 cycled = 1; /* ignore range_cyclic tests */
2090 }
2091 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2092 tag = PAGECACHE_TAG_TOWRITE;
2093 else
2094 tag = PAGECACHE_TAG_DIRTY;
2095 retry:
2096 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2097 tag_pages_for_writeback(mapping, index, end);
2098 done_index = index;
2099 while (!done && (index <= end)) {
2100 int i;
2101
2102 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
2103 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1);
2104 if (nr_pages == 0)
2105 break;
2106
2107 for (i = 0; i < nr_pages; i++) {
2108 struct page *page = pvec.pages[i];
2109 bool submitted = false;
2110
2111 if (page->index > end) {
2112 done = 1;
2113 break;
2114 }
2115
2116 done_index = page->index;
2117 retry_write:
2118 lock_page(page);
2119
2120 if (unlikely(page->mapping != mapping)) {
2121 continue_unlock:
2122 unlock_page(page);
2123 continue;
2124 }
2125
2126 if (!PageDirty(page)) {
2127 /* someone wrote it for us */
2128 goto continue_unlock;
2129 }
2130
2131 if (PageWriteback(page)) {
2132 if (wbc->sync_mode != WB_SYNC_NONE)
2133 f2fs_wait_on_page_writeback(page,
2134 DATA, true);
2135 else
2136 goto continue_unlock;
2137 }
2138
2139 BUG_ON(PageWriteback(page));
2140 if (!clear_page_dirty_for_io(page))
2141 goto continue_unlock;
2142
2143 ret = __write_data_page(page, &submitted, wbc, io_type);
2144 if (unlikely(ret)) {
2145 /*
2146 * keep nr_to_write, since vfs uses this to
2147 * get # of written pages.
2148 */
2149 if (ret == AOP_WRITEPAGE_ACTIVATE) {
2150 unlock_page(page);
2151 ret = 0;
2152 continue;
2153 } else if (ret == -EAGAIN) {
2154 ret = 0;
2155 if (wbc->sync_mode == WB_SYNC_ALL) {
2156 cond_resched();
2157 congestion_wait(BLK_RW_ASYNC,
2158 HZ/50);
2159 goto retry_write;
2160 }
2161 continue;
2162 }
2163 done_index = page->index + 1;
2164 done = 1;
2165 break;
2166 } else if (submitted) {
2167 last_idx = page->index;
2168 }
2169
2170 /* give a priority to WB_SYNC threads */
2171 if ((atomic_read(&F2FS_M_SB(mapping)->wb_sync_req) ||
2172 --wbc->nr_to_write <= 0) &&
2173 wbc->sync_mode == WB_SYNC_NONE) {
2174 done = 1;
2175 break;
2176 }
2177 }
2178 pagevec_release(&pvec);
2179 cond_resched();
2180 }
2181
2182 if (!cycled && !done) {
2183 cycled = 1;
2184 index = 0;
2185 end = writeback_index - 1;
2186 goto retry;
2187 }
2188 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
2189 mapping->writeback_index = done_index;
2190
2191 if (last_idx != ULONG_MAX)
2192 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host,
2193 0, last_idx, DATA);
2194
2195 return ret;
2196 }
2197
2198 int __f2fs_write_data_pages(struct address_space *mapping,
2199 struct writeback_control *wbc,
2200 enum iostat_type io_type)
2201 {
2202 struct inode *inode = mapping->host;
2203 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2204 struct blk_plug plug;
2205 int ret;
2206
2207 /* deal with chardevs and other special file */
2208 if (!mapping->a_ops->writepage)
2209 return 0;
2210
2211 /* skip writing if there is no dirty page in this inode */
2212 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
2213 return 0;
2214
2215 /* during POR, we don't need to trigger writepage at all. */
2216 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
2217 goto skip_write;
2218
2219 if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE &&
2220 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
2221 available_free_memory(sbi, DIRTY_DENTS))
2222 goto skip_write;
2223
2224 /* skip writing during file defragment */
2225 if (is_inode_flag_set(inode, FI_DO_DEFRAG))
2226 goto skip_write;
2227
2228 trace_f2fs_writepages(mapping->host, wbc, DATA);
2229
2230 /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */
2231 if (wbc->sync_mode == WB_SYNC_ALL)
2232 atomic_inc(&sbi->wb_sync_req);
2233 else if (atomic_read(&sbi->wb_sync_req))
2234 goto skip_write;
2235
2236 blk_start_plug(&plug);
2237 ret = f2fs_write_cache_pages(mapping, wbc, io_type);
2238 blk_finish_plug(&plug);
2239
2240 if (wbc->sync_mode == WB_SYNC_ALL)
2241 atomic_dec(&sbi->wb_sync_req);
2242 /*
2243 * if some pages were truncated, we cannot guarantee its mapping->host
2244 * to detect pending bios.
2245 */
2246
2247 remove_dirty_inode(inode);
2248 return ret;
2249
2250 skip_write:
2251 wbc->pages_skipped += get_dirty_pages(inode);
2252 trace_f2fs_writepages(mapping->host, wbc, DATA);
2253 return 0;
2254 }
2255
2256 static int f2fs_write_data_pages(struct address_space *mapping,
2257 struct writeback_control *wbc)
2258 {
2259 struct inode *inode = mapping->host;
2260
2261 return __f2fs_write_data_pages(mapping, wbc,
2262 F2FS_I(inode)->cp_task == current ?
2263 FS_CP_DATA_IO : FS_DATA_IO);
2264 }
2265
2266 static void f2fs_write_failed(struct address_space *mapping, loff_t to)
2267 {
2268 struct inode *inode = mapping->host;
2269 loff_t i_size = i_size_read(inode);
2270
2271 if (to > i_size) {
2272 down_write(&F2FS_I(inode)->i_mmap_sem);
2273 truncate_pagecache(inode, i_size);
2274 truncate_blocks(inode, i_size, true);
2275 up_write(&F2FS_I(inode)->i_mmap_sem);
2276 }
2277 }
2278
2279 static int prepare_write_begin(struct f2fs_sb_info *sbi,
2280 struct page *page, loff_t pos, unsigned len,
2281 block_t *blk_addr, bool *node_changed)
2282 {
2283 struct inode *inode = page->mapping->host;
2284 pgoff_t index = page->index;
2285 struct dnode_of_data dn;
2286 struct page *ipage;
2287 bool locked = false;
2288 struct extent_info ei = {0,0,0};
2289 int err = 0;
2290 int flag;
2291
2292 /*
2293 * we already allocated all the blocks, so we don't need to get
2294 * the block addresses when there is no need to fill the page.
2295 */
2296 if (!f2fs_has_inline_data(inode) && len == PAGE_SIZE &&
2297 !is_inode_flag_set(inode, FI_NO_PREALLOC))
2298 return 0;
2299
2300 /* f2fs_lock_op avoids race between write CP and convert_inline_page */
2301 if (f2fs_has_inline_data(inode) && pos + len > MAX_INLINE_DATA(inode))
2302 flag = F2FS_GET_BLOCK_DEFAULT;
2303 else
2304 flag = F2FS_GET_BLOCK_PRE_AIO;
2305
2306 if (f2fs_has_inline_data(inode) ||
2307 (pos & PAGE_MASK) >= i_size_read(inode)) {
2308 __do_map_lock(sbi, flag, true);
2309 locked = true;
2310 }
2311 restart:
2312 /* check inline_data */
2313 ipage = get_node_page(sbi, inode->i_ino);
2314 if (IS_ERR(ipage)) {
2315 err = PTR_ERR(ipage);
2316 goto unlock_out;
2317 }
2318
2319 set_new_dnode(&dn, inode, ipage, ipage, 0);
2320
2321 if (f2fs_has_inline_data(inode)) {
2322 if (pos + len <= MAX_INLINE_DATA(inode)) {
2323 read_inline_data(page, ipage);
2324 set_inode_flag(inode, FI_DATA_EXIST);
2325 if (inode->i_nlink)
2326 set_inline_node(ipage);
2327 } else {
2328 err = f2fs_convert_inline_page(&dn, page);
2329 if (err)
2330 goto out;
2331 if (dn.data_blkaddr == NULL_ADDR)
2332 err = f2fs_get_block(&dn, index);
2333 }
2334 } else if (locked) {
2335 err = f2fs_get_block(&dn, index);
2336 } else {
2337 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
2338 dn.data_blkaddr = ei.blk + index - ei.fofs;
2339 } else {
2340 /* hole case */
2341 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
2342 if (err || dn.data_blkaddr == NULL_ADDR) {
2343 f2fs_put_dnode(&dn);
2344 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO,
2345 true);
2346 WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO);
2347 locked = true;
2348 goto restart;
2349 }
2350 }
2351 }
2352
2353 /* convert_inline_page can make node_changed */
2354 *blk_addr = dn.data_blkaddr;
2355 *node_changed = dn.node_changed;
2356 out:
2357 f2fs_put_dnode(&dn);
2358 unlock_out:
2359 if (locked)
2360 __do_map_lock(sbi, flag, false);
2361 return err;
2362 }
2363
2364 static int f2fs_write_begin(struct file *file, struct address_space *mapping,
2365 loff_t pos, unsigned len, unsigned flags,
2366 struct page **pagep, void **fsdata)
2367 {
2368 struct inode *inode = mapping->host;
2369 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2370 struct page *page = NULL;
2371 pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT;
2372 bool need_balance = false, drop_atomic = false;
2373 block_t blkaddr = NULL_ADDR;
2374 int err = 0;
2375
2376 if (trace_android_fs_datawrite_start_enabled()) {
2377 char *path, pathbuf[MAX_TRACE_PATHBUF_LEN];
2378
2379 path = android_fstrace_get_pathname(pathbuf,
2380 MAX_TRACE_PATHBUF_LEN,
2381 inode);
2382 trace_android_fs_datawrite_start(inode, pos, len,
2383 current->pid, path,
2384 current->comm);
2385 }
2386 trace_f2fs_write_begin(inode, pos, len, flags);
2387
2388 if (f2fs_is_atomic_file(inode) &&
2389 !available_free_memory(sbi, INMEM_PAGES)) {
2390 err = -ENOMEM;
2391 drop_atomic = true;
2392 goto fail;
2393 }
2394
2395 /*
2396 * We should check this at this moment to avoid deadlock on inode page
2397 * and #0 page. The locking rule for inline_data conversion should be:
2398 * lock_page(page #0) -> lock_page(inode_page)
2399 */
2400 if (index != 0) {
2401 err = f2fs_convert_inline_inode(inode);
2402 if (err)
2403 goto fail;
2404 }
2405 repeat:
2406 /*
2407 * Do not use grab_cache_page_write_begin() to avoid deadlock due to
2408 * wait_for_stable_page. Will wait that below with our IO control.
2409 */
2410 page = f2fs_pagecache_get_page(mapping, index,
2411 FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS);
2412 if (!page) {
2413 err = -ENOMEM;
2414 goto fail;
2415 }
2416
2417 *pagep = page;
2418
2419 err = prepare_write_begin(sbi, page, pos, len,
2420 &blkaddr, &need_balance);
2421 if (err)
2422 goto fail;
2423
2424 if (need_balance && has_not_enough_free_secs(sbi, 0, 0)) {
2425 unlock_page(page);
2426 f2fs_balance_fs(sbi, true);
2427 lock_page(page);
2428 if (page->mapping != mapping) {
2429 /* The page got truncated from under us */
2430 f2fs_put_page(page, 1);
2431 goto repeat;
2432 }
2433 }
2434
2435 f2fs_wait_on_page_writeback(page, DATA, false);
2436
2437 if (len == PAGE_SIZE || PageUptodate(page))
2438 return 0;
2439
2440 if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode)) {
2441 zero_user_segment(page, len, PAGE_SIZE);
2442 return 0;
2443 }
2444
2445 if (blkaddr == NEW_ADDR) {
2446 zero_user_segment(page, 0, PAGE_SIZE);
2447 SetPageUptodate(page);
2448 } else {
2449 err = f2fs_submit_page_read(inode, page, blkaddr);
2450 if (err)
2451 goto fail;
2452
2453 lock_page(page);
2454 if (unlikely(page->mapping != mapping)) {
2455 f2fs_put_page(page, 1);
2456 goto repeat;
2457 }
2458 if (unlikely(!PageUptodate(page))) {
2459 err = -EIO;
2460 goto fail;
2461 }
2462 }
2463 return 0;
2464
2465 fail:
2466 f2fs_put_page(page, 1);
2467 f2fs_write_failed(mapping, pos + len);
2468 if (drop_atomic)
2469 drop_inmem_pages_all(sbi);
2470 return err;
2471 }
2472
2473 static int f2fs_write_end(struct file *file,
2474 struct address_space *mapping,
2475 loff_t pos, unsigned len, unsigned copied,
2476 struct page *page, void *fsdata)
2477 {
2478 struct inode *inode = page->mapping->host;
2479
2480 trace_android_fs_datawrite_end(inode, pos, len);
2481 trace_f2fs_write_end(inode, pos, len, copied);
2482
2483 /*
2484 * This should be come from len == PAGE_SIZE, and we expect copied
2485 * should be PAGE_SIZE. Otherwise, we treat it with zero copied and
2486 * let generic_perform_write() try to copy data again through copied=0.
2487 */
2488 if (!PageUptodate(page)) {
2489 if (unlikely(copied != len))
2490 copied = 0;
2491 else
2492 SetPageUptodate(page);
2493 }
2494 if (!copied)
2495 goto unlock_out;
2496
2497 set_page_dirty(page);
2498
2499 if (pos + copied > i_size_read(inode))
2500 f2fs_i_size_write(inode, pos + copied);
2501 unlock_out:
2502 f2fs_put_page(page, 1);
2503 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
2504 return copied;
2505 }
2506
2507 static int check_direct_IO(struct inode *inode, struct iov_iter *iter,
2508 loff_t offset)
2509 {
2510 unsigned blocksize_mask = inode->i_sb->s_blocksize - 1;
2511
2512 if (offset & blocksize_mask)
2513 return -EINVAL;
2514
2515 if (iov_iter_alignment(iter) & blocksize_mask)
2516 return -EINVAL;
2517
2518 return 0;
2519 }
2520
2521 static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
2522 {
2523 struct address_space *mapping = iocb->ki_filp->f_mapping;
2524 struct inode *inode = mapping->host;
2525 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2526 size_t count = iov_iter_count(iter);
2527 loff_t offset = iocb->ki_pos;
2528 int rw = iov_iter_rw(iter);
2529 int err;
2530 enum rw_hint hint = iocb->ki_hint;
2531 int whint_mode = F2FS_OPTION(sbi).whint_mode;
2532
2533 err = check_direct_IO(inode, iter, offset);
2534 if (err)
2535 return err;
2536
2537 if (f2fs_force_buffered_io(inode, rw))
2538 return 0;
2539
2540 trace_f2fs_direct_IO_enter(inode, offset, count, rw);
2541
2542 if (trace_android_fs_dataread_start_enabled() &&
2543 (rw == READ)) {
2544 char *path, pathbuf[MAX_TRACE_PATHBUF_LEN];
2545
2546 path = android_fstrace_get_pathname(pathbuf,
2547 MAX_TRACE_PATHBUF_LEN,
2548 inode);
2549 trace_android_fs_dataread_start(inode, offset,
2550 count, current->pid, path,
2551 current->comm);
2552 }
2553 if (trace_android_fs_datawrite_start_enabled() &&
2554 (rw == WRITE)) {
2555 char *path, pathbuf[MAX_TRACE_PATHBUF_LEN];
2556
2557 path = android_fstrace_get_pathname(pathbuf,
2558 MAX_TRACE_PATHBUF_LEN,
2559 inode);
2560 trace_android_fs_datawrite_start(inode, offset, count,
2561 current->pid, path,
2562 current->comm);
2563 }
2564 if (rw == WRITE && whint_mode == WHINT_MODE_OFF)
2565 iocb->ki_hint = WRITE_LIFE_NOT_SET;
2566
2567 if (!down_read_trylock(&F2FS_I(inode)->dio_rwsem[rw])) {
2568 if (iocb->ki_flags & IOCB_NOWAIT) {
2569 iocb->ki_hint = hint;
2570 err = -EAGAIN;
2571 goto out;
2572 }
2573 down_read(&F2FS_I(inode)->dio_rwsem[rw]);
2574 }
2575
2576 err = blockdev_direct_IO(iocb, inode, iter, get_data_block_dio);
2577 up_read(&F2FS_I(inode)->dio_rwsem[rw]);
2578
2579 if (rw == WRITE) {
2580 if (whint_mode == WHINT_MODE_OFF)
2581 iocb->ki_hint = hint;
2582 if (err > 0) {
2583 f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_IO,
2584 err);
2585 set_inode_flag(inode, FI_UPDATE_WRITE);
2586 } else if (err < 0) {
2587 f2fs_write_failed(mapping, offset + count);
2588 }
2589 }
2590 out:
2591 if (trace_android_fs_dataread_start_enabled() &&
2592 (rw == READ))
2593 trace_android_fs_dataread_end(inode, offset, count);
2594 if (trace_android_fs_datawrite_start_enabled() &&
2595 (rw == WRITE))
2596 trace_android_fs_datawrite_end(inode, offset, count);
2597
2598 trace_f2fs_direct_IO_exit(inode, offset, count, rw, err);
2599
2600 return err;
2601 }
2602
2603 void f2fs_invalidate_page(struct page *page, unsigned int offset,
2604 unsigned int length)
2605 {
2606 struct inode *inode = page->mapping->host;
2607 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2608
2609 if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
2610 (offset % PAGE_SIZE || length != PAGE_SIZE))
2611 return;
2612
2613 if (PageDirty(page)) {
2614 if (inode->i_ino == F2FS_META_INO(sbi)) {
2615 dec_page_count(sbi, F2FS_DIRTY_META);
2616 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
2617 dec_page_count(sbi, F2FS_DIRTY_NODES);
2618 } else {
2619 inode_dec_dirty_pages(inode);
2620 remove_dirty_inode(inode);
2621 }
2622 }
2623
2624 /* This is atomic written page, keep Private */
2625 if (IS_ATOMIC_WRITTEN_PAGE(page))
2626 return drop_inmem_page(inode, page);
2627
2628 set_page_private(page, 0);
2629 ClearPagePrivate(page);
2630 }
2631
2632 int f2fs_release_page(struct page *page, gfp_t wait)
2633 {
2634 /* If this is dirty page, keep PagePrivate */
2635 if (PageDirty(page))
2636 return 0;
2637
2638 /* This is atomic written page, keep Private */
2639 if (IS_ATOMIC_WRITTEN_PAGE(page))
2640 return 0;
2641
2642 set_page_private(page, 0);
2643 ClearPagePrivate(page);
2644 return 1;
2645 }
2646
2647 static int f2fs_set_data_page_dirty(struct page *page)
2648 {
2649 struct address_space *mapping = page->mapping;
2650 struct inode *inode = mapping->host;
2651
2652 trace_f2fs_set_page_dirty(page, DATA);
2653
2654 if (!PageUptodate(page))
2655 SetPageUptodate(page);
2656
2657 if (f2fs_is_atomic_file(inode) && !f2fs_is_commit_atomic_write(inode)) {
2658 if (!IS_ATOMIC_WRITTEN_PAGE(page)) {
2659 register_inmem_page(inode, page);
2660 return 1;
2661 }
2662 /*
2663 * Previously, this page has been registered, we just
2664 * return here.
2665 */
2666 return 0;
2667 }
2668
2669 if (!PageDirty(page)) {
2670 __set_page_dirty_nobuffers(page);
2671 update_dirty_page(inode, page);
2672 return 1;
2673 }
2674 return 0;
2675 }
2676
2677 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
2678 {
2679 struct inode *inode = mapping->host;
2680
2681 if (f2fs_has_inline_data(inode))
2682 return 0;
2683
2684 /* make sure allocating whole blocks */
2685 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2686 filemap_write_and_wait(mapping);
2687
2688 return generic_block_bmap(mapping, block, get_data_block_bmap);
2689 }
2690
2691 #ifdef CONFIG_MIGRATION
2692 #include <linux/migrate.h>
2693
2694 int f2fs_migrate_page(struct address_space *mapping,
2695 struct page *newpage, struct page *page, enum migrate_mode mode)
2696 {
2697 int rc, extra_count;
2698 struct f2fs_inode_info *fi = F2FS_I(mapping->host);
2699 bool atomic_written = IS_ATOMIC_WRITTEN_PAGE(page);
2700
2701 BUG_ON(PageWriteback(page));
2702
2703 /* migrating an atomic written page is safe with the inmem_lock hold */
2704 if (atomic_written) {
2705 if (mode != MIGRATE_SYNC)
2706 return -EBUSY;
2707 if (!mutex_trylock(&fi->inmem_lock))
2708 return -EAGAIN;
2709 }
2710
2711 /*
2712 * A reference is expected if PagePrivate set when move mapping,
2713 * however F2FS breaks this for maintaining dirty page counts when
2714 * truncating pages. So here adjusting the 'extra_count' make it work.
2715 */
2716 extra_count = (atomic_written ? 1 : 0) - page_has_private(page);
2717 rc = migrate_page_move_mapping(mapping, newpage,
2718 page, NULL, mode, extra_count);
2719 if (rc != MIGRATEPAGE_SUCCESS) {
2720 if (atomic_written)
2721 mutex_unlock(&fi->inmem_lock);
2722 return rc;
2723 }
2724
2725 if (atomic_written) {
2726 struct inmem_pages *cur;
2727 list_for_each_entry(cur, &fi->inmem_pages, list)
2728 if (cur->page == page) {
2729 cur->page = newpage;
2730 break;
2731 }
2732 mutex_unlock(&fi->inmem_lock);
2733 put_page(page);
2734 get_page(newpage);
2735 }
2736
2737 if (PagePrivate(page))
2738 SetPagePrivate(newpage);
2739 set_page_private(newpage, page_private(page));
2740
2741 if (mode != MIGRATE_SYNC_NO_COPY)
2742 migrate_page_copy(newpage, page);
2743 else
2744 migrate_page_states(newpage, page);
2745
2746 return MIGRATEPAGE_SUCCESS;
2747 }
2748 #endif
2749
2750 const struct address_space_operations f2fs_dblock_aops = {
2751 .readpage = f2fs_read_data_page,
2752 .readpages = f2fs_read_data_pages,
2753 .writepage = f2fs_write_data_page,
2754 .writepages = f2fs_write_data_pages,
2755 .write_begin = f2fs_write_begin,
2756 .write_end = f2fs_write_end,
2757 .set_page_dirty = f2fs_set_data_page_dirty,
2758 .invalidatepage = f2fs_invalidate_page,
2759 .releasepage = f2fs_release_page,
2760 .direct_IO = f2fs_direct_IO,
2761 .bmap = f2fs_bmap,
2762 #ifdef CONFIG_MIGRATION
2763 .migratepage = f2fs_migrate_page,
2764 #endif
2765 };
2766
2767 int __init f2fs_init_post_read_processing(void)
2768 {
2769 bio_post_read_ctx_cache = KMEM_CACHE(bio_post_read_ctx, 0);
2770 if (!bio_post_read_ctx_cache)
2771 goto fail;
2772 bio_post_read_ctx_pool =
2773 mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS,
2774 bio_post_read_ctx_cache);
2775 if (!bio_post_read_ctx_pool)
2776 goto fail_free_cache;
2777 return 0;
2778
2779 fail_free_cache:
2780 kmem_cache_destroy(bio_post_read_ctx_cache);
2781 fail:
2782 return -ENOMEM;
2783 }
2784
2785 void __exit f2fs_destroy_post_read_processing(void)
2786 {
2787 mempool_destroy(bio_post_read_ctx_pool);
2788 kmem_cache_destroy(bio_post_read_ctx_cache);
2789 }
Motorola kernel-slsi