projects / GitHub / MotorolaMobilityLLC / kernel-slsi.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
f2fs: avoid wrong decrypted data from disk
[GitHub/MotorolaMobilityLLC/kernel-slsi.git] / fs / f2fs / data.c
CommitLineData
0a8165d7 1/*
eb47b800
JK		 2 * fs/f2fs/data.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11#include <linux/fs.h>
12#include <linux/f2fs_fs.h>
13#include <linux/buffer_head.h>
14#include <linux/mpage.h>
15#include <linux/writeback.h>
16#include <linux/backing-dev.h>
8f46dcae 17#include <linux/pagevec.h>
eb47b800
JK		 18#include <linux/blkdev.h>
19#include <linux/bio.h>
690e4a3e 20#include <linux/prefetch.h>
e2e40f2c 21#include <linux/uio.h>
f1e88660 22#include <linux/cleancache.h>
174cd4b1 23#include <linux/sched/signal.h>
eb47b800
JK		 24
25#include "f2fs.h"
26#include "node.h"
27#include "segment.h"
db9f7c1a 28#include "trace.h"
848753aa 29#include <trace/events/f2fs.h>
45560fee 30#include <trace/events/android_fs.h>
eb47b800 31
c6e638ae
EB		 32#define NUM_PREALLOC_POST_READ_CTXS 128
33
34static struct kmem_cache *bio_post_read_ctx_cache;
35static mempool_t *bio_post_read_ctx_pool;
36
36951b38
CY		 37static bool __is_cp_guaranteed(struct page *page)
38{
39 struct address_space *mapping = page->mapping;
40 struct inode *inode;
41 struct f2fs_sb_info *sbi;
42
43 if (!mapping)
44 return false;
45
46 inode = mapping->host;
47 sbi = F2FS_I_SB(inode);
48
49 if (inode->i_ino == F2FS_META_INO(sbi) ||
50 inode->i_ino == F2FS_NODE_INO(sbi) ||
51 S_ISDIR(inode->i_mode) ||
52 is_cold_data(page))
53 return true;
54 return false;
55}
56
c6e638ae
EB		 57/* postprocessing steps for read bios */
58enum bio_post_read_step {
59 STEP_INITIAL = 0,
60 STEP_DECRYPT,
61};
62
63struct bio_post_read_ctx {
64 struct bio *bio;
65 struct work_struct work;
66 unsigned int cur_step;
67 unsigned int enabled_steps;
68};
69
49e3bbe3
BK		 70/* device unit number for iv sector */
71#define PG_DUN(i,p) \
72 ((((i)->i_ino & 0xffffffff) << 32) | ((p)->index & 0xffffffff))
73
51cc6215
BK		 74static inline bool f2fs_may_encrypt_bio(struct inode *inode,
75 struct f2fs_io_info *fio)
76{
49e3bbe3 77#ifdef CONFIG_CRYPTO_DISKCIPHER
51cc6215
BK		 78 if (fio && (fio->type != DATA || fio->encrypted_page))
79 return false;
80
81 return (f2fs_encrypted_file(inode) &&
82 fscrypt_disk_encrypted(inode));
49e3bbe3
BK		 83#else
84 return false;
85#endif
51cc6215
BK		 86}
87
88static inline bool f2fs_bio_disk_encrypted(unsigned int bi_opf)
89{
90 if (bi_opf & REQ_CRYPT)
91 return true;
92 else
93 return false;
94}
95
49e3bbe3 96static bool f2fs_mergeable_bio(struct bio *bio, u64 dun, void *ci, bool bio_encrypted)
51cc6215 97{
49e3bbe3 98#ifdef CONFIG_CRYPTO_DISKCIPHER
51cc6215
BK		 99 if (!bio)
100 return true;
101
102 /* if both of them are not encrypted, no further check is needed */
103 if (!f2fs_bio_disk_encrypted(bio->bi_opf) && !bio_encrypted)
104 return true;
105
49e3bbe3
BK		 106#ifdef CONFIG_CRYPTO_DISKCIPHER_DUN
107 if (bio->bi_aux_private == ci)
108 return bio_end_dun(bio) == dun;
109 else
110 return false;
111#else
51cc6215 112 return bio->bi_aux_private == ci;
49e3bbe3
BK		 113#endif
114#else
115 return true;
116#endif
51cc6215
BK		 117}
118
c6e638ae 119static void __read_end_io(struct bio *bio)
93dfe2ac 120{
c6e638ae
EB		 121 struct page *page;
122 struct bio_vec *bv;
f568849e 123 int i;
93dfe2ac 124
c6e638ae
EB		 125 bio_for_each_segment_all(bv, bio, i) {
126 page = bv->bv_page;
127
128 /* PG_error was set if any post_read step failed */
129 if (bio->bi_status || PageError(page)) {
130 ClearPageUptodate(page);
131 SetPageError(page);
132 } else {
133 SetPageUptodate(page);
134 }
135 unlock_page(page);
136 }
137 if (bio->bi_private)
138 mempool_free(bio->bi_private, bio_post_read_ctx_pool);
139 bio_put(bio);
140}
141
142static void bio_post_read_processing(struct bio_post_read_ctx *ctx);
143
144static void decrypt_work(struct work_struct *work)
145{
146 struct bio_post_read_ctx *ctx =
147 container_of(work, struct bio_post_read_ctx, work);
148
149 fscrypt_decrypt_bio(ctx->bio);
150
151 bio_post_read_processing(ctx);
152}
153
154static void bio_post_read_processing(struct bio_post_read_ctx *ctx)
155{
156 switch (++ctx->cur_step) {
157 case STEP_DECRYPT:
158 if (ctx->enabled_steps & (1 << STEP_DECRYPT)) {
159 INIT_WORK(&ctx->work, decrypt_work);
160 fscrypt_enqueue_decrypt_work(&ctx->work);
161 return;
162 }
163 ctx->cur_step++;
164 /* fall-through */
165 default:
166 __read_end_io(ctx->bio);
167 }
168}
169
170static bool f2fs_bio_post_read_required(struct bio *bio)
171{
172 return bio->bi_private && !bio->bi_status;
173}
174
175static void f2fs_read_end_io(struct bio *bio)
176{
8b038c70 177#ifdef CONFIG_F2FS_FAULT_INJECTION
55523519
CY		 178 if (time_to_inject(F2FS_P_SB(bio->bi_io_vec->bv_page), FAULT_IO)) {
179 f2fs_show_injection_info(FAULT_IO);
4e4cbee9 180 bio->bi_status = BLK_STS_IOERR;
55523519 181 }
8b038c70
CY		 182#endif
183
51cc6215
BK		 184 if(f2fs_bio_disk_encrypted(bio->bi_opf))
185 goto end_io;
186
c6e638ae
EB		 187 if (f2fs_bio_post_read_required(bio)) {
188 struct bio_post_read_ctx *ctx = bio->bi_private;
f1e88660 189
c6e638ae
EB		 190 ctx->cur_step = STEP_INITIAL;
191 bio_post_read_processing(ctx);
192 return;
f1e88660 193 }
c6e638ae 194
51cc6215 195end_io:
c6e638ae 196 __read_end_io(bio);
f1e88660
JK		 197}
198
4246a0b6 199static void f2fs_write_end_io(struct bio *bio)
93dfe2ac 200{
1b1f559f 201 struct f2fs_sb_info *sbi = bio->bi_private;
f568849e
LT		 202 struct bio_vec *bvec;
203 int i;
93dfe2ac 204
f568849e 205 bio_for_each_segment_all(bvec, bio, i) {
93dfe2ac 206 struct page *page = bvec->bv_page;
36951b38 207 enum count_type type = WB_DATA_TYPE(page);
93dfe2ac 208
0a595eba
JK		 209 if (IS_DUMMY_WRITTEN_PAGE(page)) {
210 set_page_private(page, (unsigned long)NULL);
211 ClearPagePrivate(page);
212 unlock_page(page);
213 mempool_free(page, sbi->write_io_dummy);
214
4e4cbee9 215 if (unlikely(bio->bi_status))
0a595eba
JK		 216 f2fs_stop_checkpoint(sbi, true);
217 continue;
218 }
219
0b81d077 220 fscrypt_pullback_bio_page(&page, true);
4375a336 221
4e4cbee9 222 if (unlikely(bio->bi_status)) {
5114a97a 223 mapping_set_error(page->mapping, -EIO);
ce767d9a
JK		 224 if (type == F2FS_WB_CP_DATA)
225 f2fs_stop_checkpoint(sbi, true);
93dfe2ac 226 }
ce767d9a
JK		 227
228 f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) &&
229 page->index != nid_of_node(page));
230
36951b38
CY		 231 dec_page_count(sbi, type);
232 clear_cold_data(page);
93dfe2ac 233 end_page_writeback(page);
f568849e 234 }
36951b38 235 if (!get_pages(sbi, F2FS_WB_CP_DATA) &&
f5730184 236 wq_has_sleeper(&sbi->cp_wait))
93dfe2ac
JK		 237 wake_up(&sbi->cp_wait);
238
239 bio_put(bio);
240}
241
3c62be17
JK		 242/*
243 * Return true, if pre_bio's bdev is same as its target device.
244 */
245struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
246 block_t blk_addr, struct bio *bio)
247{
248 struct block_device *bdev = sbi->sb->s_bdev;
249 int i;
250
251 for (i = 0; i < sbi->s_ndevs; i++) {
252 if (FDEV(i).start_blk <= blk_addr &&
253 FDEV(i).end_blk >= blk_addr) {
254 blk_addr -= FDEV(i).start_blk;
255 bdev = FDEV(i).bdev;
256 break;
257 }
258 }
259 if (bio) {
74d46992 260 bio_set_dev(bio, bdev);
3c62be17
JK		 261 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
262 }
263 return bdev;
264}
265
266int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr)
267{
268 int i;
269
270 for (i = 0; i < sbi->s_ndevs; i++)
271 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
272 return i;
273 return 0;
274}
275
276static bool __same_bdev(struct f2fs_sb_info *sbi,
277 block_t blk_addr, struct bio *bio)
278{
74d46992
CH		 279 struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL);
280 return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno;
3c62be17
JK		 281}
282
940a6d34
GZ		 283/*
284 * Low-level block read/write IO operations.
285 */
286static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
ce767d9a 287 struct writeback_control *wbc,
a2a9032a
JK		 288 int npages, bool is_read,
289 enum page_type type, enum temp_type temp)
940a6d34
GZ		 290{
291 struct bio *bio;
292
7fb3f3b2 293 bio = f2fs_bio_alloc(sbi, npages, true);
940a6d34 294
3c62be17 295 f2fs_target_device(sbi, blk_addr, bio);
a2a9032a
JK		 296 if (is_read) {
297 bio->bi_end_io = f2fs_read_end_io;
298 bio->bi_private = NULL;
299 } else {
300 bio->bi_end_io = f2fs_write_end_io;
301 bio->bi_private = sbi;
302 bio->bi_write_hint = io_type_to_rw_hint(sbi, type, temp);
303 }
ce767d9a
JK		 304 if (wbc)
305 wbc_init_bio(wbc, bio);
940a6d34
GZ		 306
307 return bio;
308}
309
4fc29c1a
LT		 310static inline void __submit_bio(struct f2fs_sb_info *sbi,
311 struct bio *bio, enum page_type type)
f5730184 312{
4fc29c1a 313 if (!is_read_io(bio_op(bio))) {
0a595eba
JK		 314 unsigned int start;
315
0a595eba
JK		 316 if (type != DATA && type != NODE)
317 goto submit_io;
318
a2a9032a
JK		 319 if (f2fs_sb_has_blkzoned(sbi->sb) && current->plug)
320 blk_finish_plug(current->plug);
321
0a595eba
JK		 322 start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS;
323 start %= F2FS_IO_SIZE(sbi);
324
325 if (start == 0)
326 goto submit_io;
327
328 /* fill dummy pages */
329 for (; start < F2FS_IO_SIZE(sbi); start++) {
330 struct page *page =
331 mempool_alloc(sbi->write_io_dummy,
332 GFP_NOIO | __GFP_ZERO | __GFP_NOFAIL);
333 f2fs_bug_on(sbi, !page);
334
335 SetPagePrivate(page);
336 set_page_private(page, (unsigned long)DUMMY_WRITTEN_PAGE);
337 lock_page(page);
338 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
339 f2fs_bug_on(sbi, 1);
340 }
341 /*
342 * In the NODE case, we lose next block address chain. So, we
343 * need to do checkpoint in f2fs_sync_file.
344 */
345 if (type == NODE)
346 set_sbi_flag(sbi, SBI_NEED_CP);
19a5f5e2 347 }
0a595eba 348submit_io:
554b5125
JK		 349 if (is_read_io(bio_op(bio)))
350 trace_f2fs_submit_read_bio(sbi->sb, type, bio);
351 else
352 trace_f2fs_submit_write_bio(sbi->sb, type, bio);
4e49ea4a 353 submit_bio(bio);
f5730184
JK		 354}
355
458e6197 356static void __submit_merged_bio(struct f2fs_bio_info *io)
93dfe2ac 357{
458e6197 358 struct f2fs_io_info *fio = &io->fio;
93dfe2ac
JK		 359
360 if (!io->bio)
361 return;
362
51cc6215
BK		 363 if (f2fs_bio_disk_encrypted(io->bio->bi_opf))
364 bio_set_op_attrs(io->bio, fio->op, fio->op_flags | REQ_CRYPT);
365 else
366 bio_set_op_attrs(io->bio, fio->op, fio->op_flags);
554b5125 367
04d328de 368 if (is_read_io(fio->op))
554b5125 369 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);
6a8f8ca5 370 else
554b5125 371 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);
04d328de 372
4fc29c1a 373 __submit_bio(io->sbi, io->bio, fio->type);
93dfe2ac
JK		 374 io->bio = NULL;
375}
376
942fd319
JK		 377static bool __has_merged_page(struct f2fs_bio_info *io,
378 struct inode *inode, nid_t ino, pgoff_t idx)
0fd785eb 379{
0fd785eb
CY		 380 struct bio_vec *bvec;
381 struct page *target;
382 int i;
383
0c3a5797 384 if (!io->bio)
0fd785eb 385 return false;
0c3a5797 386
942fd319 387 if (!inode && !ino)
0c3a5797 388 return true;
0fd785eb
CY		 389
390 bio_for_each_segment_all(bvec, io->bio, i) {
391
0b81d077 392 if (bvec->bv_page->mapping)
0fd785eb 393 target = bvec->bv_page;
0b81d077
JK		 394 else
395 target = fscrypt_control_page(bvec->bv_page);
0fd785eb 396
942fd319
JK		 397 if (idx != target->index)
398 continue;
399
0c3a5797
CY		 400 if (inode && inode == target->mapping->host)
401 return true;
0c3a5797 402 if (ino && ino == ino_of_node(target))
0fd785eb 403 return true;
0fd785eb
CY		 404 }
405
0fd785eb
CY		 406 return false;
407}
408
0c3a5797 409static bool has_merged_page(struct f2fs_sb_info *sbi, struct inode *inode,
942fd319 410 nid_t ino, pgoff_t idx, enum page_type type)
0c3a5797
CY		 411{
412 enum page_type btype = PAGE_TYPE_OF_BIO(type);
a912b54d
JK		 413 enum temp_type temp;
414 struct f2fs_bio_info *io;
415 bool ret = false;
0c3a5797 416
a912b54d
JK		 417 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
418 io = sbi->write_io[btype] + temp;
419
420 down_read(&io->io_rwsem);
421 ret = __has_merged_page(io, inode, ino, idx);
422 up_read(&io->io_rwsem);
0c3a5797 423
a912b54d
JK		 424 /* TODO: use HOT temp only for meta pages now. */
425 if (ret || btype == META)
426 break;
427 }
0c3a5797
CY		 428 return ret;
429}
430
b9109b0e 431static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
a912b54d 432 enum page_type type, enum temp_type temp)
93dfe2ac
JK		 433{
434 enum page_type btype = PAGE_TYPE_OF_BIO(type);
a912b54d 435 struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
93dfe2ac 436
df0f8dc0 437 down_write(&io->io_rwsem);
458e6197
JK		 438
439 /* change META to META_FLUSH in the checkpoint procedure */
440 if (type >= META_FLUSH) {
441 io->fio.type = META_FLUSH;
04d328de 442 io->fio.op = REQ_OP_WRITE;
3adc5fcb 443 io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC;
70fd7614 444 if (!test_opt(sbi, NOBARRIER))
7f54f51f 445 io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
458e6197
JK		 446 }
447 __submit_merged_bio(io);
df0f8dc0 448 up_write(&io->io_rwsem);
93dfe2ac
JK		 449}
450
a912b54d
JK		 451static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
452 struct inode *inode, nid_t ino, pgoff_t idx,
453 enum page_type type, bool force)
0c3a5797 454{
a912b54d
JK		 455 enum temp_type temp;
456
457 if (!force && !has_merged_page(sbi, inode, ino, idx, type))
458 return;
459
460 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
461
462 __f2fs_submit_merged_write(sbi, type, temp);
463
464 /* TODO: use HOT temp only for meta pages now. */
465 if (type >= META)
466 break;
467 }
0c3a5797
CY		 468}
469
b9109b0e 470void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type)
0c3a5797 471{
a912b54d 472 __submit_merged_write_cond(sbi, NULL, 0, 0, type, true);
0c3a5797
CY		 473}
474
b9109b0e 475void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
942fd319 476 struct inode *inode, nid_t ino, pgoff_t idx,
b9109b0e 477 enum page_type type)
0c3a5797 478{
a912b54d 479 __submit_merged_write_cond(sbi, inode, ino, idx, type, false);
0c3a5797
CY		 480}
481
b9109b0e 482void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi)
406657dd 483{
b9109b0e
JK		 484 f2fs_submit_merged_write(sbi, DATA);
485 f2fs_submit_merged_write(sbi, NODE);
486 f2fs_submit_merged_write(sbi, META);
406657dd
CY		 487}
488
93dfe2ac
JK		 489/*
490 * Fill the locked page with data located in the block address.
771a9a71 491 * A caller needs to unlock the page on failure.
93dfe2ac 492 */
05ca3632 493int f2fs_submit_page_bio(struct f2fs_io_info *fio)
93dfe2ac 494{
93dfe2ac 495 struct bio *bio;
0b81d077
JK		 496 struct page *page = fio->encrypted_page ?
497 fio->encrypted_page : fio->page;
51cc6215 498 struct inode *inode = fio->page->mapping->host;
93dfe2ac 499
ad19d1e7
CY		 500 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,
501 __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC))
502 return -EFAULT;
503
2ace38e0 504 trace_f2fs_submit_page_bio(page, fio);
05ca3632 505 f2fs_trace_ios(fio, 0);
93dfe2ac
JK		 506
507 /* Allocate a new bio */
ce767d9a 508 bio = __bio_alloc(fio->sbi, fio->new_blkaddr, fio->io_wbc,
a2a9032a 509 1, is_read_io(fio->op), fio->type, fio->temp);
93dfe2ac 510
09cbfeaf 511 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
93dfe2ac 512 bio_put(bio);
93dfe2ac
JK		 513 return -EFAULT;
514 }
c2f22fd2 515 fio->op_flags |= fio->encrypted_page ? REQ_NOENCRYPT : 0;
04d328de 516 bio_set_op_attrs(bio, fio->op, fio->op_flags);
93dfe2ac 517
51cc6215 518 if (f2fs_may_encrypt_bio(inode, fio))
49e3bbe3 519 fscrypt_set_bio(inode, bio, PG_DUN(inode, fio->page));
51cc6215 520
d1b3e72d
JK		 521 if (!is_read_io(fio->op))
522 inc_page_count(fio->sbi, WB_DATA_TYPE(fio->page));
2cc5dcf7
CY		 523
524 __submit_bio(fio->sbi, bio, fio->type);
93dfe2ac
JK		 525 return 0;
526}
527
b9109b0e 528int f2fs_submit_page_write(struct f2fs_io_info *fio)
93dfe2ac 529{
05ca3632 530 struct f2fs_sb_info *sbi = fio->sbi;
458e6197 531 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
a912b54d 532 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
4375a336 533 struct page *bio_page;
0a595eba 534 int err = 0;
51cc6215
BK		 535 struct inode *inode;
536 bool bio_encrypted;
49e3bbe3 537 u64 dun;
93dfe2ac 538
b9109b0e 539 f2fs_bug_on(sbi, is_read_io(fio->op));
93dfe2ac 540
fb830fc5
CY		 541 down_write(&io->io_rwsem);
542next:
543 if (fio->in_list) {
544 spin_lock(&io->io_lock);
545 if (list_empty(&io->io_list)) {
546 spin_unlock(&io->io_lock);
547 goto out_fail;
548 }
549 fio = list_first_entry(&io->io_list,
550 struct f2fs_io_info, list);
551 list_del(&fio->list);
552 spin_unlock(&io->io_lock);
553 }
93dfe2ac 554
eea71570 555 if (__is_valid_data_blkaddr(fio->old_blkaddr))
a2a9032a
JK		 556 verify_block_addr(fio, fio->old_blkaddr);
557 verify_block_addr(fio, fio->new_blkaddr);
93dfe2ac 558
36951b38 559 bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page;
51cc6215 560 inode = fio->page->mapping->host;
49e3bbe3 561 dun = PG_DUN(inode, fio->page);
51cc6215 562 bio_encrypted = f2fs_may_encrypt_bio(inode, fio);
c2f22fd2 563 fio->op_flags |= fio->encrypted_page ? REQ_NOENCRYPT : 0;
36951b38 564
7fb3f3b2
JK		 565 /* set submitted = true as a return value */
566 fio->submitted = true;
d68f735b 567
b9109b0e 568 inc_page_count(sbi, WB_DATA_TYPE(bio_page));
93dfe2ac 569
7a9d7548 570 if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 ||
3c62be17
JK		 571 (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) ||
572 !__same_bdev(sbi, fio->new_blkaddr, io->bio)))
458e6197 573 __submit_merged_bio(io);
51cc6215 574
49e3bbe3 575 if (!f2fs_mergeable_bio(io->bio, dun, fscrypt_get_diskcipher(inode), bio_encrypted))
51cc6215
BK		 576 __submit_merged_bio(io);
577
93dfe2ac
JK		 578alloc_new:
579 if (io->bio == NULL) {
0a595eba
JK		 580 if ((fio->type == DATA || fio->type == NODE) &&
581 fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) {
582 err = -EAGAIN;
b9109b0e 583 dec_page_count(sbi, WB_DATA_TYPE(bio_page));
0a595eba
JK		 584 goto out_fail;
585 }
ce767d9a 586 io->bio = __bio_alloc(sbi, fio->new_blkaddr, fio->io_wbc,
a2a9032a
JK		 587 BIO_MAX_PAGES, false,
588 fio->type, fio->temp);
51cc6215 589 if (bio_encrypted)
49e3bbe3 590 fscrypt_set_bio(inode, io->bio, dun);
51cc6215 591
458e6197 592 io->fio = *fio;
93dfe2ac
JK		 593 }
594
a912b54d 595 if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) {
458e6197 596 __submit_merged_bio(io);
93dfe2ac
JK		 597 goto alloc_new;
598 }
599
ce767d9a
JK		 600 if (fio->io_wbc)
601 wbc_account_io(fio->io_wbc, bio_page, PAGE_SIZE);
602
7a9d7548 603 io->last_block_in_bio = fio->new_blkaddr;
05ca3632 604 f2fs_trace_ios(fio, 0);
fb830fc5
CY		 605
606 trace_f2fs_submit_page_write(fio->page, fio);
607
608 if (fio->in_list)
609 goto next;
0a595eba 610out_fail:
df0f8dc0 611 up_write(&io->io_rwsem);
0a595eba 612 return err;
93dfe2ac
JK		 613}
614
13ba41e3
JK		 615static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
616 unsigned nr_pages)
617{
618 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
13ba41e3 619 struct bio *bio;
c6e638ae
EB		 620 struct bio_post_read_ctx *ctx;
621 unsigned int post_read_steps = 0;
13ba41e3 622
38fce19d
CY		 623 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC))
624 return ERR_PTR(-EFAULT);
625
7fb3f3b2 626 bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES), false);
c6e638ae 627 if (!bio)
13ba41e3 628 return ERR_PTR(-ENOMEM);
13ba41e3
JK		 629 f2fs_target_device(sbi, blkaddr, bio);
630 bio->bi_end_io = f2fs_read_end_io;
13ba41e3
JK		 631 bio_set_op_attrs(bio, REQ_OP_READ, 0);
632
51cc6215 633 if (f2fs_encrypted_file(inode) && !fscrypt_disk_encrypted(inode))
c6e638ae
EB		 634 post_read_steps |= 1 << STEP_DECRYPT;
635 if (post_read_steps) {
636 ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
637 if (!ctx) {
638 bio_put(bio);
639 return ERR_PTR(-ENOMEM);
640 }
641 ctx->bio = bio;
642 ctx->enabled_steps = post_read_steps;
643 bio->bi_private = ctx;
c2f22fd2
MH		 644 bio_set_op_attrs(bio, REQ_OP_READ,
645 (f2fs_encrypted_inode(inode) ?
646 REQ_NOENCRYPT :
647 0));
c6e638ae 648
c6e638ae
EB		 649 }
650
13ba41e3
JK		 651 return bio;
652}
653
654/* This can handle encryption stuffs */
655static int f2fs_submit_page_read(struct inode *inode, struct page *page,
656 block_t blkaddr)
657{
658 struct bio *bio = f2fs_grab_read_bio(inode, blkaddr, 1);
659
660 if (IS_ERR(bio))
661 return PTR_ERR(bio);
662
f9ef288b
JK		 663 /* wait for GCed page writeback via META_MAPPING */
664 f2fs_wait_on_block_writeback(inode, blkaddr);
665
13ba41e3
JK		 666 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
667 bio_put(bio);
668 return -EFAULT;
669 }
51cc6215
BK		 670
671 if (f2fs_may_encrypt_bio(inode, NULL))
49e3bbe3 672 fscrypt_set_bio(inode, bio, PG_DUN(inode, page));
51cc6215 673
13ba41e3
JK		 674 __submit_bio(F2FS_I_SB(inode), bio, DATA);
675 return 0;
676}
677
46008c6d
CY		 678static void __set_data_blkaddr(struct dnode_of_data *dn)
679{
680 struct f2fs_node *rn = F2FS_NODE(dn->node_page);
681 __le32 *addr_array;
7a2af766
CY		 682 int base = 0;
683
684 if (IS_INODE(dn->node_page) && f2fs_has_extra_attr(dn->inode))
685 base = get_extra_isize(dn->inode);
46008c6d
CY		 686
687 /* Get physical address of data block */
688 addr_array = blkaddr_in_node(rn);
7a2af766 689 addr_array[base + dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
46008c6d
CY		 690}
691
0a8165d7 692/*
eb47b800
JK		 693 * Lock ordering for the change of data block address:
694 * ->data_page
695 * ->node_page
696 * update block addresses in the node page
697 */
216a620a 698void set_data_blkaddr(struct dnode_of_data *dn)
eb47b800 699{
46008c6d
CY		 700 f2fs_wait_on_page_writeback(dn->node_page, NODE, true);
701 __set_data_blkaddr(dn);
702 if (set_page_dirty(dn->node_page))
12719ae1 703 dn->node_changed = true;
eb47b800
JK		 704}
705
f28b3434
CY		 706void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
707{
708 dn->data_blkaddr = blkaddr;
709 set_data_blkaddr(dn);
710 f2fs_update_extent_cache(dn);
711}
712
46008c6d
CY		 713/* dn->ofs_in_node will be returned with up-to-date last block pointer */
714int reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count)
eb47b800 715{
4081363f 716 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
0abd675e 717 int err;
eb47b800 718
46008c6d
CY		 719 if (!count)
720 return 0;
721
91942321 722 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
eb47b800 723 return -EPERM;
0abd675e
CY		 724 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
725 return err;
eb47b800 726
46008c6d
CY		 727 trace_f2fs_reserve_new_blocks(dn->inode, dn->nid,
728 dn->ofs_in_node, count);
729
730 f2fs_wait_on_page_writeback(dn->node_page, NODE, true);
731
732 for (; count > 0; dn->ofs_in_node++) {
7a2af766
CY		 733 block_t blkaddr = datablock_addr(dn->inode,
734 dn->node_page, dn->ofs_in_node);
46008c6d
CY		 735 if (blkaddr == NULL_ADDR) {
736 dn->data_blkaddr = NEW_ADDR;
737 __set_data_blkaddr(dn);
738 count--;
739 }
740 }
741
742 if (set_page_dirty(dn->node_page))
743 dn->node_changed = true;
eb47b800
JK		 744 return 0;
745}
746
46008c6d
CY		 747/* Should keep dn->ofs_in_node unchanged */
748int reserve_new_block(struct dnode_of_data *dn)
749{
750 unsigned int ofs_in_node = dn->ofs_in_node;
751 int ret;
752
753 ret = reserve_new_blocks(dn, 1);
754 dn->ofs_in_node = ofs_in_node;
755 return ret;
756}
757
b600965c
HL		 758int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
759{
760 bool need_put = dn->inode_page ? false : true;
761 int err;
762
763 err = get_dnode_of_data(dn, index, ALLOC_NODE);
764 if (err)
765 return err;
a8865372 766
b600965c
HL		 767 if (dn->data_blkaddr == NULL_ADDR)
768 err = reserve_new_block(dn);
a8865372 769 if (err || need_put)
b600965c
HL		 770 f2fs_put_dnode(dn);
771 return err;
772}
773
759af1c9 774int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
eb47b800 775{
e15882b6 776 struct extent_info ei = {0,0,0};
759af1c9 777 struct inode *inode = dn->inode;
028a41e8 778
759af1c9
FL		 779 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
780 dn->data_blkaddr = ei.blk + index - ei.fofs;
781 return 0;
429511cd 782 }
028a41e8 783
759af1c9 784 return f2fs_reserve_block(dn, index);
eb47b800
JK		 785}
786
a56c7c6f 787struct page *get_read_data_page(struct inode *inode, pgoff_t index,
04d328de 788 int op_flags, bool for_write)
eb47b800 789{
eb47b800
JK		 790 struct address_space *mapping = inode->i_mapping;
791 struct dnode_of_data dn;
792 struct page *page;
e15882b6 793 struct extent_info ei = {0,0,0};
eb47b800 794 int err;
4375a336 795
a56c7c6f 796 page = f2fs_grab_cache_page(mapping, index, for_write);
650495de
JK		 797 if (!page)
798 return ERR_PTR(-ENOMEM);
799
cb3bc9ee
CY		 800 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
801 dn.data_blkaddr = ei.blk + index - ei.fofs;
802 goto got_it;
803 }
804
eb47b800 805 set_new_dnode(&dn, inode, NULL, NULL, 0);
266e97a8 806 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
86531d6b
JK		 807 if (err)
808 goto put_err;
eb47b800
JK		 809 f2fs_put_dnode(&dn);
810
6bacf52f 811 if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
86531d6b
JK		 812 err = -ENOENT;
813 goto put_err;
650495de 814 }
cb3bc9ee 815got_it:
43f3eae1
JK		 816 if (PageUptodate(page)) {
817 unlock_page(page);
eb47b800 818 return page;
43f3eae1 819 }
eb47b800 820
d59ff4df
JK		 821 /*
822 * A new dentry page is allocated but not able to be written, since its
823 * new inode page couldn't be allocated due to -ENOSPC.
824 * In such the case, its blkaddr can be remained as NEW_ADDR.
825 * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata.
826 */
827 if (dn.data_blkaddr == NEW_ADDR) {
09cbfeaf 828 zero_user_segment(page, 0, PAGE_SIZE);
237c0790
JK		 829 if (!PageUptodate(page))
830 SetPageUptodate(page);
43f3eae1 831 unlock_page(page);
d59ff4df
JK		 832 return page;
833 }
eb47b800 834
13ba41e3 835 err = f2fs_submit_page_read(inode, page, dn.data_blkaddr);
393ff91f 836 if (err)
86531d6b 837 goto put_err;
43f3eae1 838 return page;
86531d6b
JK		 839
840put_err:
841 f2fs_put_page(page, 1);
842 return ERR_PTR(err);
43f3eae1
JK		 843}
844
845struct page *find_data_page(struct inode *inode, pgoff_t index)
846{
847 struct address_space *mapping = inode->i_mapping;
848 struct page *page;
849
850 page = find_get_page(mapping, index);
851 if (page && PageUptodate(page))
852 return page;
853 f2fs_put_page(page, 0);
854
70fd7614 855 page = get_read_data_page(inode, index, 0, false);
43f3eae1
JK		 856 if (IS_ERR(page))
857 return page;
858
859 if (PageUptodate(page))
860 return page;
861
862 wait_on_page_locked(page);
863 if (unlikely(!PageUptodate(page))) {
864 f2fs_put_page(page, 0);
865 return ERR_PTR(-EIO);
866 }
867 return page;
868}
869
870/*
871 * If it tries to access a hole, return an error.
872 * Because, the callers, functions in dir.c and GC, should be able to know
873 * whether this page exists or not.
874 */
a56c7c6f
JK		 875struct page *get_lock_data_page(struct inode *inode, pgoff_t index,
876 bool for_write)
43f3eae1
JK		 877{
878 struct address_space *mapping = inode->i_mapping;
879 struct page *page;
880repeat:
70fd7614 881 page = get_read_data_page(inode, index, 0, for_write);
43f3eae1
JK		 882 if (IS_ERR(page))
883 return page;
393ff91f 884
43f3eae1 885 /* wait for read completion */
393ff91f 886 lock_page(page);
6bacf52f 887 if (unlikely(page->mapping != mapping)) {
afcb7ca0
JK		 888 f2fs_put_page(page, 1);
889 goto repeat;
eb47b800 890 }
1563ac75
CY		 891 if (unlikely(!PageUptodate(page))) {
892 f2fs_put_page(page, 1);
893 return ERR_PTR(-EIO);
894 }
eb47b800
JK		 895 return page;
896}
897
0a8165d7 898/*
eb47b800
JK		 899 * Caller ensures that this data page is never allocated.
900 * A new zero-filled data page is allocated in the page cache.
39936837 901 *
4f4124d0
CY		 902 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
903 * f2fs_unlock_op().
470f00e9
CY		 904 * Note that, ipage is set only by make_empty_dir, and if any error occur,
905 * ipage should be released by this function.
eb47b800 906 */
64aa7ed9 907struct page *get_new_data_page(struct inode *inode,
a8865372 908 struct page *ipage, pgoff_t index, bool new_i_size)
eb47b800 909{
eb47b800
JK		 910 struct address_space *mapping = inode->i_mapping;
911 struct page *page;
912 struct dnode_of_data dn;
913 int err;
7612118a 914
a56c7c6f 915 page = f2fs_grab_cache_page(mapping, index, true);
470f00e9
CY		 916 if (!page) {
917 /*
918 * before exiting, we should make sure ipage will be released
919 * if any error occur.
920 */
921 f2fs_put_page(ipage, 1);
01f28610 922 return ERR_PTR(-ENOMEM);
470f00e9 923 }
eb47b800 924
a8865372 925 set_new_dnode(&dn, inode, ipage, NULL, 0);
b600965c 926 err = f2fs_reserve_block(&dn, index);
01f28610
JK		 927 if (err) {
928 f2fs_put_page(page, 1);
eb47b800 929 return ERR_PTR(err);
a8865372 930 }
01f28610
JK		 931 if (!ipage)
932 f2fs_put_dnode(&dn);
eb47b800
JK		 933
934 if (PageUptodate(page))
01f28610 935 goto got_it;
eb47b800
JK		 936
937 if (dn.data_blkaddr == NEW_ADDR) {
09cbfeaf 938 zero_user_segment(page, 0, PAGE_SIZE);
237c0790
JK		 939 if (!PageUptodate(page))
940 SetPageUptodate(page);
eb47b800 941 } else {
4375a336 942 f2fs_put_page(page, 1);
a8865372 943
7612118a
JK		 944 /* if ipage exists, blkaddr should be NEW_ADDR */
945 f2fs_bug_on(F2FS_I_SB(inode), ipage);
946 page = get_lock_data_page(inode, index, true);
4375a336 947 if (IS_ERR(page))
7612118a 948 return page;
eb47b800 949 }
01f28610 950got_it:
9edcdabf 951 if (new_i_size && i_size_read(inode) <
ee6d182f 952 ((loff_t)(index + 1) << PAGE_SHIFT))
fc9581c8 953 f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT));
eb47b800
JK		 954 return page;
955}
956
ce767d9a 957static int __allocate_data_block(struct dnode_of_data *dn, int seg_type)
bfad7c2d 958{
4081363f 959 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
bfad7c2d 960 struct f2fs_summary sum;
bfad7c2d 961 struct node_info ni;
976e4c50 962 pgoff_t fofs;
46008c6d 963 blkcnt_t count = 1;
0abd675e 964 int err;
bfad7c2d 965
91942321 966 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
bfad7c2d 967 return -EPERM;
df6136ef 968
7a2af766
CY		 969 dn->data_blkaddr = datablock_addr(dn->inode,
970 dn->node_page, dn->ofs_in_node);
df6136ef
CY		 971 if (dn->data_blkaddr == NEW_ADDR)
972 goto alloc;
973
0abd675e
CY		 974 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
975 return err;
bfad7c2d 976
df6136ef 977alloc:
bfad7c2d
JK		 978 get_node_info(sbi, dn->nid, &ni);
979 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
980
df6136ef 981 allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr,
ce767d9a 982 &sum, seg_type, NULL, false);
216a620a 983 set_data_blkaddr(dn);
bfad7c2d 984
976e4c50 985 /* update i_size */
81ca7350 986 fofs = start_bidx_of_node(ofs_of_node(dn->node_page), dn->inode) +
976e4c50 987 dn->ofs_in_node;
09cbfeaf 988 if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_SHIFT))
fc9581c8 989 f2fs_i_size_write(dn->inode,
09cbfeaf 990 ((loff_t)(fofs + 1) << PAGE_SHIFT));
bfad7c2d
JK		 991 return 0;
992}
993
a7de6086 994int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
59b802e5 995{
b439b103 996 struct inode *inode = file_inode(iocb->ki_filp);
5b8db7fa 997 struct f2fs_map_blocks map;
ce767d9a 998 int flag;
a7de6086 999 int err = 0;
ce767d9a 1000 bool direct_io = iocb->ki_flags & IOCB_DIRECT;
59b802e5 1001
7fb3f3b2 1002 /* convert inline data for Direct I/O*/
ce767d9a 1003 if (direct_io) {
7fb3f3b2
JK		 1004 err = f2fs_convert_inline_inode(inode);
1005 if (err)
1006 return err;
1007 }
1008
dc91de78
JK		 1009 if (is_inode_flag_set(inode, FI_NO_PREALLOC))
1010 return 0;
1011
0080c507 1012 map.m_lblk = F2FS_BLK_ALIGN(iocb->ki_pos);
dfd02e4d
CY		 1013 map.m_len = F2FS_BYTES_TO_BLK(iocb->ki_pos + iov_iter_count(from));
1014 if (map.m_len > map.m_lblk)
1015 map.m_len -= map.m_lblk;
1016 else
1017 map.m_len = 0;
1018
da85985c 1019 map.m_next_pgofs = NULL;
ce767d9a
JK		 1020 map.m_next_extent = NULL;
1021 map.m_seg_type = NO_CHECK_TYPE;
2a340760 1022
ce767d9a
JK		 1023 if (direct_io) {
1024 map.m_seg_type = rw_hint_to_seg_type(iocb->ki_hint);
a2a9032a 1025 flag = f2fs_force_buffered_io(inode, WRITE) ?
ce767d9a
JK		 1026 F2FS_GET_BLOCK_PRE_AIO :
1027 F2FS_GET_BLOCK_PRE_DIO;
1028 goto map_blocks;
1029 }
f2470371 1030 if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA(inode)) {
a7de6086
JK		 1031 err = f2fs_convert_inline_inode(inode);
1032 if (err)
1033 return err;
b439b103 1034 }
ce767d9a
JK		 1035 if (f2fs_has_inline_data(inode))
1036 return err;
1037
1038 flag = F2FS_GET_BLOCK_PRE_AIO;
1039
1040map_blocks:
1041 err = f2fs_map_blocks(inode, &map, 1, flag);
1042 if (map.m_len > 0 && err == -ENOSPC) {
1043 if (!direct_io)
1044 set_inode_flag(inode, FI_NO_PREALLOC);
1045 err = 0;
1046 }
a7de6086 1047 return err;
59b802e5
JK		 1048}
1049
59c9081b
YH		 1050static inline void __do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock)
1051{
1052 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
1053 if (lock)
1054 down_read(&sbi->node_change);
1055 else
1056 up_read(&sbi->node_change);
1057 } else {
1058 if (lock)
1059 f2fs_lock_op(sbi);
1060 else
1061 f2fs_unlock_op(sbi);
1062 }
1063}
1064
0a8165d7 1065/*
003a3e1d
JK		 1066 * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with
1067 * f2fs_map_blocks structure.
4f4124d0
CY		 1068 * If original data blocks are allocated, then give them to blockdev.
1069 * Otherwise,
1070 * a. preallocate requested block addresses
1071 * b. do not use extent cache for better performance
1072 * c. give the block addresses to blockdev
eb47b800 1073 */
d323d005 1074int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
e2b4e2bc 1075 int create, int flag)
eb47b800 1076{
003a3e1d 1077 unsigned int maxblocks = map->m_len;
eb47b800 1078 struct dnode_of_data dn;
f9811703 1079 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
ac6f1999 1080 int mode = create ? ALLOC_NODE : LOOKUP_NODE;
46008c6d 1081 pgoff_t pgofs, end_offset, end;
bfad7c2d 1082 int err = 0, ofs = 1;
46008c6d
CY		 1083 unsigned int ofs_in_node, last_ofs_in_node;
1084 blkcnt_t prealloc;
e15882b6 1085 struct extent_info ei = {0,0,0};
7df3a431 1086 block_t blkaddr;
ce767d9a 1087 unsigned int start_pgofs;
eb47b800 1088
dfd02e4d
CY		 1089 if (!maxblocks)
1090 return 0;
1091
003a3e1d
JK		 1092 map->m_len = 0;
1093 map->m_flags = 0;
1094
1095 /* it only supports block size == page size */
1096 pgofs = (pgoff_t)map->m_lblk;
46008c6d 1097 end = pgofs + maxblocks;
eb47b800 1098
24b84912 1099 if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
003a3e1d
JK		 1100 map->m_pblk = ei.blk + pgofs - ei.fofs;
1101 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs);
1102 map->m_flags = F2FS_MAP_MAPPED;
ce767d9a
JK		 1103 if (map->m_next_extent)
1104 *map->m_next_extent = pgofs + map->m_len;
bfad7c2d 1105 goto out;
a2e7d1bf 1106 }
bfad7c2d 1107
4fe71e88 1108next_dnode:
59b802e5 1109 if (create)
59c9081b 1110 __do_map_lock(sbi, flag, true);
eb47b800
JK		 1111
1112 /* When reading holes, we need its node page */
1113 set_new_dnode(&dn, inode, NULL, NULL, 0);
bfad7c2d 1114 err = get_dnode_of_data(&dn, pgofs, mode);
1ec79083 1115 if (err) {
43473f96
CY		 1116 if (flag == F2FS_GET_BLOCK_BMAP)
1117 map->m_pblk = 0;
da85985c 1118 if (err == -ENOENT) {
bfad7c2d 1119 err = 0;
da85985c
CY		 1120 if (map->m_next_pgofs)
1121 *map->m_next_pgofs =
1122 get_next_page_offset(&dn, pgofs);
ce767d9a
JK		 1123 if (map->m_next_extent)
1124 *map->m_next_extent =
1125 get_next_page_offset(&dn, pgofs);
da85985c 1126 }
bfad7c2d 1127 goto unlock_out;
848753aa 1128 }
973163fc 1129
ce767d9a 1130 start_pgofs = pgofs;
46008c6d 1131 prealloc = 0;
230436b3 1132 last_ofs_in_node = ofs_in_node = dn.ofs_in_node;
81ca7350 1133 end_offset = ADDRS_PER_PAGE(dn.node_page, inode);
4fe71e88
CY		 1134
1135next_block:
7a2af766 1136 blkaddr = datablock_addr(dn.inode, dn.node_page, dn.ofs_in_node);
4fe71e88 1137
ad19d1e7
CY		 1138 if (__is_valid_data_blkaddr(blkaddr) &&
1139 !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC)) {
1140 err = -EFAULT;
1141 goto sync_out;
1142 }
1143
eea71570 1144 if (!is_valid_data_blkaddr(sbi, blkaddr)) {
973163fc 1145 if (create) {
f9811703
CY		 1146 if (unlikely(f2fs_cp_error(sbi))) {
1147 err = -EIO;
4fe71e88 1148 goto sync_out;
f9811703 1149 }
24b84912 1150 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
46008c6d
CY		 1151 if (blkaddr == NULL_ADDR) {
1152 prealloc++;
1153 last_ofs_in_node = dn.ofs_in_node;
1154 }
24b84912 1155 } else {
ce767d9a
JK		 1156 err = __allocate_data_block(&dn,
1157 map->m_seg_type);
6f2d8ed6 1158 if (!err)
91942321 1159 set_inode_flag(inode, FI_APPEND_WRITE);
24b84912 1160 }
973163fc 1161 if (err)
4fe71e88 1162 goto sync_out;
3f2be043 1163 map->m_flags |= F2FS_MAP_NEW;
4fe71e88 1164 blkaddr = dn.data_blkaddr;
973163fc 1165 } else {
43473f96
CY		 1166 if (flag == F2FS_GET_BLOCK_BMAP) {
1167 map->m_pblk = 0;
1168 goto sync_out;
1169 }
ce767d9a
JK		 1170 if (flag == F2FS_GET_BLOCK_PRECACHE)
1171 goto sync_out;
da85985c
CY		 1172 if (flag == F2FS_GET_BLOCK_FIEMAP &&
1173 blkaddr == NULL_ADDR) {
1174 if (map->m_next_pgofs)
1175 *map->m_next_pgofs = pgofs + 1;
ce767d9a 1176 goto sync_out;
da85985c 1177 }
ce767d9a
JK		 1178 if (flag != F2FS_GET_BLOCK_FIEMAP) {
1179 /* for defragment case */
1180 if (map->m_next_pgofs)
1181 *map->m_next_pgofs = pgofs + 1;
4fe71e88 1182 goto sync_out;
ce767d9a 1183 }
e2b4e2bc 1184 }
e2b4e2bc 1185 }
eb47b800 1186
46008c6d
CY		 1187 if (flag == F2FS_GET_BLOCK_PRE_AIO)
1188 goto skip;
1189
4fe71e88
CY		 1190 if (map->m_len == 0) {
1191 /* preallocated unwritten block should be mapped for fiemap. */
1192 if (blkaddr == NEW_ADDR)
1193 map->m_flags |= F2FS_MAP_UNWRITTEN;
1194 map->m_flags |= F2FS_MAP_MAPPED;
1195
1196 map->m_pblk = blkaddr;
1197 map->m_len = 1;
1198 } else if ((map->m_pblk != NEW_ADDR &&
1199 blkaddr == (map->m_pblk + ofs)) ||
b439b103 1200 (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) ||
46008c6d 1201 flag == F2FS_GET_BLOCK_PRE_DIO) {
4fe71e88
CY		 1202 ofs++;
1203 map->m_len++;
1204 } else {
1205 goto sync_out;
1206 }
bfad7c2d 1207
46008c6d 1208skip:
bfad7c2d
JK		 1209 dn.ofs_in_node++;
1210 pgofs++;
1211
46008c6d
CY		 1212 /* preallocate blocks in batch for one dnode page */
1213 if (flag == F2FS_GET_BLOCK_PRE_AIO &&
1214 (pgofs == end || dn.ofs_in_node == end_offset)) {
7df3a431 1215
46008c6d
CY		 1216 dn.ofs_in_node = ofs_in_node;
1217 err = reserve_new_blocks(&dn, prealloc);
1218 if (err)
1219 goto sync_out;
bfad7c2d 1220
46008c6d
CY		 1221 map->m_len += dn.ofs_in_node - ofs_in_node;
1222 if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) {
1223 err = -ENOSPC;
1224 goto sync_out;
3104af35 1225 }
46008c6d
CY		 1226 dn.ofs_in_node = end_offset;
1227 }
1228
1229 if (pgofs >= end)
1230 goto sync_out;
1231 else if (dn.ofs_in_node < end_offset)
1232 goto next_block;
1233
ce767d9a
JK		 1234 if (flag == F2FS_GET_BLOCK_PRECACHE) {
1235 if (map->m_flags & F2FS_MAP_MAPPED) {
1236 unsigned int ofs = start_pgofs - map->m_lblk;
1237
1238 f2fs_update_extent_cache_range(&dn,
1239 start_pgofs, map->m_pblk + ofs,
1240 map->m_len - ofs);
1241 }
1242 }
1243
46008c6d
CY		 1244 f2fs_put_dnode(&dn);
1245
1246 if (create) {
59c9081b 1247 __do_map_lock(sbi, flag, false);
6f2d8ed6 1248 f2fs_balance_fs(sbi, dn.node_changed);
eb47b800 1249 }
46008c6d 1250 goto next_dnode;
7df3a431 1251
bfad7c2d 1252sync_out:
ce767d9a
JK		 1253 if (flag == F2FS_GET_BLOCK_PRECACHE) {
1254 if (map->m_flags & F2FS_MAP_MAPPED) {
1255 unsigned int ofs = start_pgofs - map->m_lblk;
1256
1257 f2fs_update_extent_cache_range(&dn,
1258 start_pgofs, map->m_pblk + ofs,
1259 map->m_len - ofs);
1260 }
1261 if (map->m_next_extent)
1262 *map->m_next_extent = pgofs + 1;
1263 }
eb47b800 1264 f2fs_put_dnode(&dn);
bfad7c2d 1265unlock_out:
2a340760 1266 if (create) {
59c9081b 1267 __do_map_lock(sbi, flag, false);
6f2d8ed6 1268 f2fs_balance_fs(sbi, dn.node_changed);
2a340760 1269 }
bfad7c2d 1270out:
003a3e1d 1271 trace_f2fs_map_blocks(inode, map, err);
bfad7c2d 1272 return err;
eb47b800
JK		 1273}
1274
a2a9032a
JK		 1275bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len)
1276{
1277 struct f2fs_map_blocks map;
1278 block_t last_lblk;
1279 int err;
1280
1281 if (pos + len > i_size_read(inode))
1282 return false;
1283
1284 map.m_lblk = F2FS_BYTES_TO_BLK(pos);
1285 map.m_next_pgofs = NULL;
1286 map.m_next_extent = NULL;
1287 map.m_seg_type = NO_CHECK_TYPE;
1288 last_lblk = F2FS_BLK_ALIGN(pos + len);
1289
1290 while (map.m_lblk < last_lblk) {
1291 map.m_len = last_lblk - map.m_lblk;
1292 err = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_DEFAULT);
1293 if (err || map.m_len == 0)
1294 return false;
1295 map.m_lblk += map.m_len;
1296 }
1297 return true;
1298}
1299
003a3e1d 1300static int __get_data_block(struct inode *inode, sector_t iblock,
da85985c 1301 struct buffer_head *bh, int create, int flag,
ce767d9a 1302 pgoff_t *next_pgofs, int seg_type)
003a3e1d
JK		 1303{
1304 struct f2fs_map_blocks map;
a7de6086 1305 int err;
003a3e1d
JK		 1306
1307 map.m_lblk = iblock;
1308 map.m_len = bh->b_size >> inode->i_blkbits;
da85985c 1309 map.m_next_pgofs = next_pgofs;
ce767d9a
JK		 1310 map.m_next_extent = NULL;
1311 map.m_seg_type = seg_type;
003a3e1d 1312
a7de6086
JK		 1313 err = f2fs_map_blocks(inode, &map, create, flag);
1314 if (!err) {
003a3e1d
JK		 1315 map_bh(bh, inode->i_sb, map.m_pblk);
1316 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags;
b86e3307 1317 bh->b_size = (u64)map.m_len << inode->i_blkbits;
003a3e1d 1318 }
a7de6086 1319 return err;
003a3e1d
JK		 1320}
1321
ccfb3000 1322static int get_data_block(struct inode *inode, sector_t iblock,
da85985c
CY		 1323 struct buffer_head *bh_result, int create, int flag,
1324 pgoff_t *next_pgofs)
e2b4e2bc 1325{
da85985c 1326 return __get_data_block(inode, iblock, bh_result, create,
ce767d9a
JK		 1327 flag, next_pgofs,
1328 NO_CHECK_TYPE);
e2b4e2bc
CY		 1329}
1330
1331static int get_data_block_dio(struct inode *inode, sector_t iblock,
ccfb3000
JK		 1332 struct buffer_head *bh_result, int create)
1333{
e2b4e2bc 1334 return __get_data_block(inode, iblock, bh_result, create,
ce767d9a
JK		 1335 F2FS_GET_BLOCK_DEFAULT, NULL,
1336 rw_hint_to_seg_type(
1337 inode->i_write_hint));
ccfb3000
JK		 1338}
1339
e2b4e2bc 1340static int get_data_block_bmap(struct inode *inode, sector_t iblock,
ccfb3000
JK		 1341 struct buffer_head *bh_result, int create)
1342{
179448bf 1343 /* Block number less than F2FS MAX BLOCKS */
e0afc4d6 1344 if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks))
179448bf
YH		 1345 return -EFBIG;
1346
e2b4e2bc 1347 return __get_data_block(inode, iblock, bh_result, create,
ce767d9a
JK		 1348 F2FS_GET_BLOCK_BMAP, NULL,
1349 NO_CHECK_TYPE);
ccfb3000
JK		 1350}
1351
7f63eb77
JK		 1352static inline sector_t logical_to_blk(struct inode *inode, loff_t offset)
1353{
1354 return (offset >> inode->i_blkbits);
1355}
1356
1357static inline loff_t blk_to_logical(struct inode *inode, sector_t blk)
1358{
1359 return (blk << inode->i_blkbits);
1360}
1361
ce767d9a
JK		 1362static int f2fs_xattr_fiemap(struct inode *inode,
1363 struct fiemap_extent_info *fieinfo)
1364{
1365 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1366 struct page *page;
1367 struct node_info ni;
1368 __u64 phys = 0, len;
1369 __u32 flags;
1370 nid_t xnid = F2FS_I(inode)->i_xattr_nid;
1371 int err = 0;
1372
1373 if (f2fs_has_inline_xattr(inode)) {
1374 int offset;
1375
1376 page = f2fs_grab_cache_page(NODE_MAPPING(sbi),
1377 inode->i_ino, false);
1378 if (!page)
1379 return -ENOMEM;
1380
1381 get_node_info(sbi, inode->i_ino, &ni);
1382
1383 phys = (__u64)blk_to_logical(inode, ni.blk_addr);
1384 offset = offsetof(struct f2fs_inode, i_addr) +
1385 sizeof(__le32) * (DEF_ADDRS_PER_INODE -
1386 get_inline_xattr_addrs(inode));
1387
1388 phys += offset;
1389 len = inline_xattr_size(inode);
1390
1391 f2fs_put_page(page, 1);
1392
1393 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED;
1394
1395 if (!xnid)
1396 flags |= FIEMAP_EXTENT_LAST;
1397
1398 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
1399 if (err || err == 1)
1400 return err;
1401 }
1402
1403 if (xnid) {
1404 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), xnid, false);
1405 if (!page)
1406 return -ENOMEM;
1407
1408 get_node_info(sbi, xnid, &ni);
1409
1410 phys = (__u64)blk_to_logical(inode, ni.blk_addr);
1411 len = inode->i_sb->s_blocksize;
1412
1413 f2fs_put_page(page, 1);
1414
1415 flags = FIEMAP_EXTENT_LAST;
1416 }
1417
1418 if (phys)
1419 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
1420
1421 return (err < 0 ? err : 0);
1422}
1423
9ab70134
JK		 1424int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
1425 u64 start, u64 len)
1426{
7f63eb77
JK		 1427 struct buffer_head map_bh;
1428 sector_t start_blk, last_blk;
da85985c 1429 pgoff_t next_pgofs;
7f63eb77
JK		 1430 u64 logical = 0, phys = 0, size = 0;
1431 u32 flags = 0;
7f63eb77
JK		 1432 int ret = 0;
1433
ce767d9a
JK		 1434 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
1435 ret = f2fs_precache_extents(inode);
1436 if (ret)
1437 return ret;
1438 }
1439
1440 ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC | FIEMAP_FLAG_XATTR);
7f63eb77
JK		 1441 if (ret)
1442 return ret;
1443
ce767d9a
JK		 1444 inode_lock(inode);
1445
1446 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
1447 ret = f2fs_xattr_fiemap(inode, fieinfo);
1448 goto out;
1449 }
1450
67f8cf3c
JK		 1451 if (f2fs_has_inline_data(inode)) {
1452 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
1453 if (ret != -EAGAIN)
ce767d9a 1454 goto out;
67f8cf3c
JK		 1455 }
1456
7f63eb77
JK		 1457 if (logical_to_blk(inode, len) == 0)
1458 len = blk_to_logical(inode, 1);
1459
1460 start_blk = logical_to_blk(inode, start);
1461 last_blk = logical_to_blk(inode, start + len - 1);
9a950d52 1462
7f63eb77
JK		 1463next:
1464 memset(&map_bh, 0, sizeof(struct buffer_head));
1465 map_bh.b_size = len;
1466
e2b4e2bc 1467 ret = get_data_block(inode, start_blk, &map_bh, 0,
da85985c 1468 F2FS_GET_BLOCK_FIEMAP, &next_pgofs);
7f63eb77
JK		 1469 if (ret)
1470 goto out;
1471
1472 /* HOLE */
1473 if (!buffer_mapped(&map_bh)) {
da85985c 1474 start_blk = next_pgofs;
58736fa6
CY		 1475
1476 if (blk_to_logical(inode, start_blk) < blk_to_logical(inode,
1477 F2FS_I_SB(inode)->max_file_blocks))
9a950d52 1478 goto prep_next;
58736fa6 1479
9a950d52
FL		 1480 flags |= FIEMAP_EXTENT_LAST;
1481 }
7f63eb77 1482
da5af127
CY		 1483 if (size) {
1484 if (f2fs_encrypted_inode(inode))
1485 flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;
1486
9a950d52
FL		 1487 ret = fiemap_fill_next_extent(fieinfo, logical,
1488 phys, size, flags);
da5af127 1489 }
7f63eb77 1490
9a950d52
FL		 1491 if (start_blk > last_blk || ret)
1492 goto out;
7f63eb77 1493
9a950d52
FL		 1494 logical = blk_to_logical(inode, start_blk);
1495 phys = blk_to_logical(inode, map_bh.b_blocknr);
1496 size = map_bh.b_size;
1497 flags = 0;
1498 if (buffer_unwritten(&map_bh))
1499 flags = FIEMAP_EXTENT_UNWRITTEN;
7f63eb77 1500
9a950d52 1501 start_blk += logical_to_blk(inode, size);
7f63eb77 1502
9a950d52 1503prep_next:
7f63eb77
JK		 1504 cond_resched();
1505 if (fatal_signal_pending(current))
1506 ret = -EINTR;
1507 else
1508 goto next;
1509out:
1510 if (ret == 1)
1511 ret = 0;
1512
5955102c 1513 inode_unlock(inode);
7f63eb77 1514 return ret;
9ab70134
JK		 1515}
1516
f1e88660
JK		 1517/*
1518 * This function was originally taken from fs/mpage.c, and customized for f2fs.
1519 * Major change was from block_size == page_size in f2fs by default.
1520 */
1521static int f2fs_mpage_readpages(struct address_space *mapping,
1522 struct list_head *pages, struct page *page,
1523 unsigned nr_pages)
1524{
1525 struct bio *bio = NULL;
f1e88660
JK		 1526 sector_t last_block_in_bio = 0;
1527 struct inode *inode = mapping->host;
1528 const unsigned blkbits = inode->i_blkbits;
1529 const unsigned blocksize = 1 << blkbits;
1530 sector_t block_in_file;
1531 sector_t last_block;
1532 sector_t last_block_in_file;
1533 sector_t block_nr;
f1e88660 1534 struct f2fs_map_blocks map;
51cc6215 1535 bool bio_encrypted;
49e3bbe3 1536 u64 dun;
f1e88660
JK		 1537
1538 map.m_pblk = 0;
1539 map.m_lblk = 0;
1540 map.m_len = 0;
1541 map.m_flags = 0;
da85985c 1542 map.m_next_pgofs = NULL;
ce767d9a
JK		 1543 map.m_next_extent = NULL;
1544 map.m_seg_type = NO_CHECK_TYPE;
f1e88660 1545
ce767d9a 1546 for (; nr_pages; nr_pages--) {
f1e88660 1547 if (pages) {
939afa94 1548 page = list_last_entry(pages, struct page, lru);
a83d50bc
KM		 1549
1550 prefetchw(&page->flags);
f1e88660
JK		 1551 list_del(&page->lru);
1552 if (add_to_page_cache_lru(page, mapping,
8a5c743e
MH		 1553 page->index,
1554 readahead_gfp_mask(mapping)))
f1e88660
JK		 1555 goto next_page;
1556 }
1557
1558 block_in_file = (sector_t)page->index;
1559 last_block = block_in_file + nr_pages;
1560 last_block_in_file = (i_size_read(inode) + blocksize - 1) >>
1561 blkbits;
1562 if (last_block > last_block_in_file)
1563 last_block = last_block_in_file;
1564
1565 /*
1566 * Map blocks using the previous result first.
1567 */
1568 if ((map.m_flags & F2FS_MAP_MAPPED) &&
1569 block_in_file > map.m_lblk &&
1570 block_in_file < (map.m_lblk + map.m_len))
1571 goto got_it;
1572
1573 /*
1574 * Then do more f2fs_map_blocks() calls until we are
1575 * done with this page.
1576 */
1577 map.m_flags = 0;
1578
1579 if (block_in_file < last_block) {
1580 map.m_lblk = block_in_file;
1581 map.m_len = last_block - block_in_file;
1582
46c9e141 1583 if (f2fs_map_blocks(inode, &map, 0,
f2220c7f 1584 F2FS_GET_BLOCK_DEFAULT))
f1e88660
JK		 1585 goto set_error_page;
1586 }
1587got_it:
1588 if ((map.m_flags & F2FS_MAP_MAPPED)) {
1589 block_nr = map.m_pblk + block_in_file - map.m_lblk;
1590 SetPageMappedToDisk(page);
1591
1592 if (!PageUptodate(page) && !cleancache_get_page(page)) {
1593 SetPageUptodate(page);
1594 goto confused;
1595 }
ad19d1e7
CY		 1596
1597 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr,
1598 DATA_GENERIC))
1599 goto set_error_page;
f1e88660 1600 } else {
09cbfeaf 1601 zero_user_segment(page, 0, PAGE_SIZE);
237c0790
JK		 1602 if (!PageUptodate(page))
1603 SetPageUptodate(page);
f1e88660
JK		 1604 unlock_page(page);
1605 goto next_page;
1606 }
1607
1608 /*
1609 * This page will go to BIO. Do we need to send this
1610 * BIO off first?
1611 */
3c62be17
JK		 1612 if (bio && (last_block_in_bio != block_nr - 1 ||
1613 !__same_bdev(F2FS_I_SB(inode), block_nr, bio))) {
f1e88660 1614submit_and_realloc:
4fc29c1a 1615 __submit_bio(F2FS_I_SB(inode), bio, DATA);
f1e88660
JK		 1616 bio = NULL;
1617 }
51cc6215 1618
49e3bbe3 1619 dun = PG_DUN(inode, page);
51cc6215 1620 bio_encrypted = f2fs_may_encrypt_bio(inode, NULL);
49e3bbe3 1621 if (!f2fs_mergeable_bio(bio, dun, fscrypt_get_diskcipher(inode), bio_encrypted)) {
51cc6215
BK		 1622 __submit_bio(F2FS_I_SB(inode), bio, DATA);
1623 bio = NULL;
1624 }
1625
f1e88660 1626 if (bio == NULL) {
13ba41e3 1627 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages);
1d353eb7
JK		 1628 if (IS_ERR(bio)) {
1629 bio = NULL;
f1e88660 1630 goto set_error_page;
4375a336 1631 }
51cc6215 1632 if (f2fs_may_encrypt_bio(inode, NULL))
49e3bbe3 1633 fscrypt_set_bio(inode, bio, dun);
f1e88660
JK		 1634 }
1635
f9ef288b
JK		 1636 /*
1637 * If the page is under writeback, we need to wait for
1638 * its completion to see the correct decrypted data.
1639 */
1640 f2fs_wait_on_block_writeback(inode, block_nr);
1641
f1e88660
JK		 1642 if (bio_add_page(bio, page, blocksize, 0) < blocksize)
1643 goto submit_and_realloc;
1644
1645 last_block_in_bio = block_nr;
1646 goto next_page;
1647set_error_page:
1648 SetPageError(page);
09cbfeaf 1649 zero_user_segment(page, 0, PAGE_SIZE);
f1e88660
JK		 1650 unlock_page(page);
1651 goto next_page;
1652confused:
1653 if (bio) {
4fc29c1a 1654 __submit_bio(F2FS_I_SB(inode), bio, DATA);
f1e88660
JK		 1655 bio = NULL;
1656 }
1657 unlock_page(page);
1658next_page:
1659 if (pages)
09cbfeaf 1660 put_page(page);
f1e88660
JK		 1661 }
1662 BUG_ON(pages && !list_empty(pages));
1663 if (bio)
4fc29c1a 1664 __submit_bio(F2FS_I_SB(inode), bio, DATA);
f1e88660
JK		 1665 return 0;
1666}
1667
eb47b800
JK		 1668static int f2fs_read_data_page(struct file *file, struct page *page)
1669{
9ffe0fb5 1670 struct inode *inode = page->mapping->host;
b3d208f9 1671 int ret = -EAGAIN;
9ffe0fb5 1672
c20e89cd
CY		 1673 trace_f2fs_readpage(page, DATA);
1674
e1c42045 1675 /* If the file has inline data, try to read it directly */
9ffe0fb5
HL		 1676 if (f2fs_has_inline_data(inode))
1677 ret = f2fs_read_inline_data(inode, page);
b3d208f9 1678 if (ret == -EAGAIN)
f1e88660 1679 ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1);
9ffe0fb5 1680 return ret;
eb47b800
JK		 1681}
1682
1683static int f2fs_read_data_pages(struct file *file,
1684 struct address_space *mapping,
1685 struct list_head *pages, unsigned nr_pages)
1686{
7fb3f3b2 1687 struct inode *inode = mapping->host;
939afa94 1688 struct page *page = list_last_entry(pages, struct page, lru);
b8c29400
CY		 1689
1690 trace_f2fs_readpages(inode, page, nr_pages);
9ffe0fb5
HL		 1691
1692 /* If the file has inline data, skip readpages */
1693 if (f2fs_has_inline_data(inode))
1694 return 0;
1695
f1e88660 1696 return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages);
eb47b800
JK		 1697}
1698
7eab0c0d
HP		 1699static int encrypt_one_page(struct f2fs_io_info *fio)
1700{
1701 struct inode *inode = fio->page->mapping->host;
1702 gfp_t gfp_flags = GFP_NOFS;
1703
1958593e 1704 if (!f2fs_encrypted_file(inode))
7eab0c0d
HP		 1705 return 0;
1706
c6e638ae 1707 /* wait for GCed page writeback via META_MAPPING */
f9ef288b 1708 f2fs_wait_on_block_writeback(inode, fio->old_blkaddr);
7eab0c0d
HP		 1709
1710retry_encrypt:
51cc6215
BK		 1711 if (fscrypt_disk_encrypted(inode))
1712 return 0;
1713
7eab0c0d
HP		 1714 fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page,
1715 PAGE_SIZE, 0, fio->page->index, gfp_flags);
1716 if (!IS_ERR(fio->encrypted_page))
1717 return 0;
1718
1719 /* flush pending IOs and wait for a while in the ENOMEM case */
1720 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
b9109b0e 1721 f2fs_flush_merged_writes(fio->sbi);
7eab0c0d
HP		 1722 congestion_wait(BLK_RW_ASYNC, HZ/50);
1723 gfp_flags |= __GFP_NOFAIL;
1724 goto retry_encrypt;
1725 }
1726 return PTR_ERR(fio->encrypted_page);
1727}
1728
ce767d9a
JK		 1729static inline bool check_inplace_update_policy(struct inode *inode,
1730 struct f2fs_io_info *fio)
1731{
1732 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1733 unsigned int policy = SM_I(sbi)->ipu_policy;
1734
1735 if (policy & (0x1 << F2FS_IPU_FORCE))
1736 return true;
1737 if (policy & (0x1 << F2FS_IPU_SSR) && need_SSR(sbi))
1738 return true;
1739 if (policy & (0x1 << F2FS_IPU_UTIL) &&
1740 utilization(sbi) > SM_I(sbi)->min_ipu_util)
1741 return true;
1742 if (policy & (0x1 << F2FS_IPU_SSR_UTIL) && need_SSR(sbi) &&
1743 utilization(sbi) > SM_I(sbi)->min_ipu_util)
1744 return true;
1745
1746 /*
1747 * IPU for rewrite async pages
1748 */
1749 if (policy & (0x1 << F2FS_IPU_ASYNC) &&
1750 fio && fio->op == REQ_OP_WRITE &&
1751 !(fio->op_flags & REQ_SYNC) &&
1752 !f2fs_encrypted_inode(inode))
1753 return true;
1754
1755 /* this is only set during fdatasync */
1756 if (policy & (0x1 << F2FS_IPU_FSYNC) &&
1757 is_inode_flag_set(inode, FI_NEED_IPU))
1758 return true;
1759
1760 return false;
1761}
1762
1763bool should_update_inplace(struct inode *inode, struct f2fs_io_info *fio)
1764{
1765 if (f2fs_is_pinned_file(inode))
1766 return true;
1767
1768 /* if this is cold file, we should overwrite to avoid fragmentation */
1769 if (file_is_cold(inode))
1770 return true;
1771
1772 return check_inplace_update_policy(inode, fio);
1773}
1774
1775bool should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)
1776{
1777 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1778
1779 if (test_opt(sbi, LFS))
1780 return true;
1781 if (S_ISDIR(inode->i_mode))
1782 return true;
1783 if (f2fs_is_atomic_file(inode))
1784 return true;
1785 if (fio) {
1786 if (is_cold_data(fio->page))
1787 return true;
1788 if (IS_ATOMIC_WRITTEN_PAGE(fio->page))
1789 return true;
1790 }
1791 return false;
1792}
1793
7eab0c0d
HP		 1794static inline bool need_inplace_update(struct f2fs_io_info *fio)
1795{
1796 struct inode *inode = fio->page->mapping->host;
1797
ce767d9a 1798 if (should_update_outplace(inode, fio))
7eab0c0d
HP		 1799 return false;
1800
ce767d9a 1801 return should_update_inplace(inode, fio);
7eab0c0d
HP		 1802}
1803
05ca3632 1804int do_write_data_page(struct f2fs_io_info *fio)
eb47b800 1805{
05ca3632 1806 struct page *page = fio->page;
eb47b800 1807 struct inode *inode = page->mapping->host;
eb47b800 1808 struct dnode_of_data dn;
e959c8f5
HP		 1809 struct extent_info ei = {0,0,0};
1810 bool ipu_force = false;
eb47b800
JK		 1811 int err = 0;
1812
1813 set_new_dnode(&dn, inode, NULL, NULL, 0);
e959c8f5
HP		 1814 if (need_inplace_update(fio) &&
1815 f2fs_lookup_extent_cache(inode, page->index, &ei)) {
1816 fio->old_blkaddr = ei.blk + page->index - ei.fofs;
a817737e 1817
ad19d1e7
CY		 1818 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
1819 DATA_GENERIC))
1820 return -EFAULT;
1821
1822 ipu_force = true;
1823 fio->need_lock = LOCK_DONE;
1824 goto got_it;
e959c8f5 1825 }
279d6df2 1826
d29460e5
JK		 1827 /* Deadlock due to between page->lock and f2fs_lock_op */
1828 if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi))
1829 return -EAGAIN;
279d6df2 1830
266e97a8 1831 err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
eb47b800 1832 if (err)
279d6df2 1833 goto out;
eb47b800 1834
28bc106b 1835 fio->old_blkaddr = dn.data_blkaddr;
eb47b800
JK		 1836
1837 /* This page is already truncated */
7a9d7548 1838 if (fio->old_blkaddr == NULL_ADDR) {
2bca1e23 1839 ClearPageUptodate(page);
eb47b800 1840 goto out_writepage;
2bca1e23 1841 }
e959c8f5 1842got_it:
ad19d1e7
CY		 1843 if (__is_valid_data_blkaddr(fio->old_blkaddr) &&
1844 !f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
1845 DATA_GENERIC)) {
1846 err = -EFAULT;
1847 goto out_writepage;
1848 }
eb47b800
JK		 1849 /*
1850 * If current allocation needs SSR,
1851 * it had better in-place writes for updated data.
1852 */
eea71570 1853 if (ipu_force || (is_valid_data_blkaddr(fio->sbi, fio->old_blkaddr) &&
9e6c4a85 1854 need_inplace_update(fio))) {
cc15620b
JK		 1855 err = encrypt_one_page(fio);
1856 if (err)
1857 goto out_writepage;
1858
1859 set_page_writeback(page);
f8385a15 1860 ClearPageError(page);
279d6df2 1861 f2fs_put_dnode(&dn);
cc15620b 1862 if (fio->need_lock == LOCK_REQ)
279d6df2 1863 f2fs_unlock_op(fio->sbi);
d1b3e72d 1864 err = rewrite_data_page(fio);
7eab0c0d 1865 trace_f2fs_do_write_data_page(fio->page, IPU);
91942321 1866 set_inode_flag(inode, FI_UPDATE_WRITE);
279d6df2 1867 return err;
eb47b800 1868 }
279d6df2 1869
cc15620b
JK		 1870 if (fio->need_lock == LOCK_RETRY) {
1871 if (!f2fs_trylock_op(fio->sbi)) {
1872 err = -EAGAIN;
1873 goto out_writepage;
1874 }
1875 fio->need_lock = LOCK_REQ;
1876 }
1877
1878 err = encrypt_one_page(fio);
1879 if (err)
1880 goto out_writepage;
1881
1882 set_page_writeback(page);
f8385a15 1883 ClearPageError(page);
cc15620b 1884
279d6df2
HP		 1885 /* LFS mode write path */
1886 write_data_page(&dn, fio);
1887 trace_f2fs_do_write_data_page(page, OPU);
1888 set_inode_flag(inode, FI_APPEND_WRITE);
1889 if (page->index == 0)
1890 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
eb47b800
JK		 1891out_writepage:
1892 f2fs_put_dnode(&dn);
279d6df2 1893out:
cc15620b 1894 if (fio->need_lock == LOCK_REQ)
279d6df2 1895 f2fs_unlock_op(fio->sbi);
eb47b800
JK		 1896 return err;
1897}
1898
d68f735b 1899static int __write_data_page(struct page *page, bool *submitted,
b0af6d49
CY		 1900 struct writeback_control *wbc,
1901 enum iostat_type io_type)
eb47b800
JK		 1902{
1903 struct inode *inode = page->mapping->host;
4081363f 1904 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
eb47b800
JK		 1905 loff_t i_size = i_size_read(inode);
1906 const pgoff_t end_index = ((unsigned long long) i_size)
09cbfeaf 1907 >> PAGE_SHIFT;
26de9b11 1908 loff_t psize = (page->index + 1) << PAGE_SHIFT;
9ffe0fb5 1909 unsigned offset = 0;
39936837 1910 bool need_balance_fs = false;
eb47b800 1911 int err = 0;
458e6197 1912 struct f2fs_io_info fio = {
05ca3632 1913 .sbi = sbi,
7fb3f3b2 1914 .ino = inode->i_ino,
458e6197 1915 .type = DATA,
04d328de 1916 .op = REQ_OP_WRITE,
7637241e 1917 .op_flags = wbc_to_write_flags(wbc),
e959c8f5 1918 .old_blkaddr = NULL_ADDR,
05ca3632 1919 .page = page,
4375a336 1920 .encrypted_page = NULL,
d68f735b 1921 .submitted = false,
cc15620b 1922 .need_lock = LOCK_RETRY,
b0af6d49 1923 .io_type = io_type,
ce767d9a 1924 .io_wbc = wbc,
458e6197 1925 };
eb47b800 1926
ecda0de3
CY		 1927 trace_f2fs_writepage(page, DATA);
1928
ce767d9a
JK		 1929 /* we should bypass data pages to proceed the kworkder jobs */
1930 if (unlikely(f2fs_cp_error(sbi))) {
1931 mapping_set_error(page->mapping, -EIO);
1932 goto out;
1933 }
1934
0771fcc7
CY		 1935 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1936 goto redirty_out;
1937
eb47b800 1938 if (page->index < end_index)
39936837 1939 goto write;
eb47b800
JK		 1940
1941 /*
1942 * If the offset is out-of-range of file size,
1943 * this page does not have to be written to disk.
1944 */
09cbfeaf 1945 offset = i_size & (PAGE_SIZE - 1);
76f60268 1946 if ((page->index >= end_index + 1) || !offset)
39936837 1947 goto out;
eb47b800 1948
09cbfeaf 1949 zero_user_segment(page, offset, PAGE_SIZE);
39936837 1950write:
1e84371f
JK		 1951 if (f2fs_is_drop_cache(inode))
1952 goto out;
e6e5f561
JK		 1953 /* we should not write 0'th page having journal header */
1954 if (f2fs_is_volatile_file(inode) && (!page->index ||
1955 (!wbc->for_reclaim &&
1956 available_free_memory(sbi, BASE_CHECK))))
1e84371f 1957 goto redirty_out;
eb47b800 1958
39936837 1959 /* Dentry blocks are controlled by checkpoint */
eb47b800 1960 if (S_ISDIR(inode->i_mode)) {
cc15620b 1961 fio.need_lock = LOCK_DONE;
05ca3632 1962 err = do_write_data_page(&fio);
8618b881
JK		 1963 goto done;
1964 }
9ffe0fb5 1965
8618b881 1966 if (!wbc->for_reclaim)
39936837 1967 need_balance_fs = true;
7f3037a5 1968 else if (has_not_enough_free_secs(sbi, 0, 0))
39936837 1969 goto redirty_out;
ef095d19
JK		 1970 else
1971 set_inode_flag(inode, FI_HOT_DATA);
eb47b800 1972
b3d208f9 1973 err = -EAGAIN;
dd7b2333 1974 if (f2fs_has_inline_data(inode)) {
b3d208f9 1975 err = f2fs_write_inline_data(inode, page);
dd7b2333
YH		 1976 if (!err)
1977 goto out;
1978 }
279d6df2 1979
cc15620b 1980 if (err == -EAGAIN) {
05ca3632 1981 err = do_write_data_page(&fio);
cc15620b
JK		 1982 if (err == -EAGAIN) {
1983 fio.need_lock = LOCK_REQ;
1984 err = do_write_data_page(&fio);
1985 }
1986 }
7fb3f3b2 1987
ce767d9a
JK		 1988 if (err) {
1989 file_set_keep_isize(inode);
1990 } else {
1991 down_write(&F2FS_I(inode)->i_sem);
1992 if (F2FS_I(inode)->last_disk_size < psize)
1993 F2FS_I(inode)->last_disk_size = psize;
1994 up_write(&F2FS_I(inode)->i_sem);
1995 }
279d6df2 1996
8618b881
JK		 1997done:
1998 if (err && err != -ENOENT)
1999 goto redirty_out;
eb47b800 2000
39936837 2001out:
a7ffdbe2 2002 inode_dec_dirty_pages(inode);
2bca1e23
JK		 2003 if (err)
2004 ClearPageUptodate(page);
0c3a5797
CY		 2005
2006 if (wbc->for_reclaim) {
b9109b0e 2007 f2fs_submit_merged_write_cond(sbi, inode, 0, page->index, DATA);
ef095d19 2008 clear_inode_flag(inode, FI_HOT_DATA);
0c3a5797 2009 remove_dirty_inode(inode);
d68f735b 2010 submitted = NULL;
0c3a5797
CY		 2011 }
2012
eb47b800 2013 unlock_page(page);
a7881893
JK		 2014 if (!S_ISDIR(inode->i_mode))
2015 f2fs_balance_fs(sbi, need_balance_fs);
0c3a5797 2016
d68f735b 2017 if (unlikely(f2fs_cp_error(sbi))) {
b9109b0e 2018 f2fs_submit_merged_write(sbi, DATA);
d68f735b
JK		 2019 submitted = NULL;
2020 }
2021
2022 if (submitted)
2023 *submitted = fio.submitted;
0c3a5797 2024
eb47b800
JK		 2025 return 0;
2026
eb47b800 2027redirty_out:
76f60268 2028 redirty_page_for_writepage(wbc, page);
2ae46065
JK		 2029 /*
2030 * pageout() in MM traslates EAGAIN, so calls handle_write_error()
2031 * -> mapping_set_error() -> set_bit(AS_EIO, ...).
2032 * file_write_and_wait_range() will see EIO error, which is critical
2033 * to return value of fsync() followed by atomic_write failure to user.
2034 */
2035 if (!err || wbc->for_reclaim)
0002b61b 2036 return AOP_WRITEPAGE_ACTIVATE;
b230e6ca
JK		 2037 unlock_page(page);
2038 return err;
fa9150a8
NJ		 2039}
2040
f566bae8
JK		 2041static int f2fs_write_data_page(struct page *page,
2042 struct writeback_control *wbc)
2043{
b0af6d49 2044 return __write_data_page(page, NULL, wbc, FS_DATA_IO);
f566bae8
JK		 2045}
2046
8f46dcae
CY		 2047/*
2048 * This function was copied from write_cche_pages from mm/page-writeback.c.
2049 * The major change is making write step of cold data page separately from
2050 * warm/hot data page.
2051 */
2052static int f2fs_write_cache_pages(struct address_space *mapping,
b0af6d49
CY		 2053 struct writeback_control *wbc,
2054 enum iostat_type io_type)
8f46dcae
CY		 2055{
2056 int ret = 0;
2057 int done = 0;
2058 struct pagevec pvec;
2059 int nr_pages;
2060 pgoff_t uninitialized_var(writeback_index);
2061 pgoff_t index;
2062 pgoff_t end; /* Inclusive */
2063 pgoff_t done_index;
942fd319 2064 pgoff_t last_idx = ULONG_MAX;
8f46dcae
CY		 2065 int cycled;
2066 int range_whole = 0;
2067 int tag;
8f46dcae
CY		 2068
2069 pagevec_init(&pvec, 0);
46ae957f 2070
ef095d19
JK		 2071 if (get_dirty_pages(mapping->host) <=
2072 SM_I(F2FS_M_SB(mapping))->min_hot_blocks)
2073 set_inode_flag(mapping->host, FI_HOT_DATA);
2074 else
2075 clear_inode_flag(mapping->host, FI_HOT_DATA);
2076
8f46dcae
CY		 2077 if (wbc->range_cyclic) {
2078 writeback_index = mapping->writeback_index; /* prev offset */
2079 index = writeback_index;
2080 if (index == 0)
2081 cycled = 1;
2082 else
2083 cycled = 0;
2084 end = -1;
2085 } else {
09cbfeaf
KS		 2086 index = wbc->range_start >> PAGE_SHIFT;
2087 end = wbc->range_end >> PAGE_SHIFT;
8f46dcae
CY		 2088 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2089 range_whole = 1;
2090 cycled = 1; /* ignore range_cyclic tests */
2091 }
2092 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2093 tag = PAGECACHE_TAG_TOWRITE;
2094 else
2095 tag = PAGECACHE_TAG_DIRTY;
2096retry:
2097 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2098 tag_pages_for_writeback(mapping, index, end);
2099 done_index = index;
2100 while (!done && (index <= end)) {
2101 int i;
2102
2103 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
2104 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1);
2105 if (nr_pages == 0)
2106 break;
2107
2108 for (i = 0; i < nr_pages; i++) {
2109 struct page *page = pvec.pages[i];
d68f735b 2110 bool submitted = false;
8f46dcae
CY		 2111
2112 if (page->index > end) {
2113 done = 1;
2114 break;
2115 }
2116
2117 done_index = page->index;
d29460e5 2118retry_write:
8f46dcae
CY		 2119 lock_page(page);
2120
2121 if (unlikely(page->mapping != mapping)) {
2122continue_unlock:
2123 unlock_page(page);
2124 continue;
2125 }
2126
2127 if (!PageDirty(page)) {
2128 /* someone wrote it for us */
2129 goto continue_unlock;
2130 }
2131
8f46dcae
CY		 2132 if (PageWriteback(page)) {
2133 if (wbc->sync_mode != WB_SYNC_NONE)
fec1d657
JK		 2134 f2fs_wait_on_page_writeback(page,
2135 DATA, true);
8f46dcae
CY		 2136 else
2137 goto continue_unlock;
2138 }
2139
2140 BUG_ON(PageWriteback(page));
2141 if (!clear_page_dirty_for_io(page))
2142 goto continue_unlock;
2143
b0af6d49 2144 ret = __write_data_page(page, &submitted, wbc, io_type);
8f46dcae 2145 if (unlikely(ret)) {
0002b61b
CY		 2146 /*
2147 * keep nr_to_write, since vfs uses this to
2148 * get # of written pages.
2149 */
2150 if (ret == AOP_WRITEPAGE_ACTIVATE) {
2151 unlock_page(page);
2152 ret = 0;
2153 continue;
d29460e5
JK		 2154 } else if (ret == -EAGAIN) {
2155 ret = 0;
2156 if (wbc->sync_mode == WB_SYNC_ALL) {
2157 cond_resched();
2158 congestion_wait(BLK_RW_ASYNC,
2159 HZ/50);
2160 goto retry_write;
2161 }
2162 continue;
0002b61b 2163 }
b230e6ca
JK		 2164 done_index = page->index + 1;
2165 done = 1;
2166 break;
d68f735b 2167 } else if (submitted) {
942fd319 2168 last_idx = page->index;
8f46dcae
CY		 2169 }
2170
687de7f1
JK		 2171 /* give a priority to WB_SYNC threads */
2172 if ((atomic_read(&F2FS_M_SB(mapping)->wb_sync_req) ||
2173 --wbc->nr_to_write <= 0) &&
2174 wbc->sync_mode == WB_SYNC_NONE) {
8f46dcae
CY		 2175 done = 1;
2176 break;
2177 }
2178 }
2179 pagevec_release(&pvec);
2180 cond_resched();
2181 }
2182
8f46dcae
CY		 2183 if (!cycled && !done) {
2184 cycled = 1;
2185 index = 0;
2186 end = writeback_index - 1;
2187 goto retry;
2188 }
2189 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
2190 mapping->writeback_index = done_index;
2191
942fd319 2192 if (last_idx != ULONG_MAX)
b9109b0e
JK		 2193 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host,
2194 0, last_idx, DATA);
6ca56ca4 2195
8f46dcae
CY		 2196 return ret;
2197}
2198
b0af6d49
CY		 2199int __f2fs_write_data_pages(struct address_space *mapping,
2200 struct writeback_control *wbc,
2201 enum iostat_type io_type)
eb47b800
JK		 2202{
2203 struct inode *inode = mapping->host;
4081363f 2204 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
9dfa1baf 2205 struct blk_plug plug;
eb47b800 2206 int ret;
eb47b800 2207
cfb185a1 2208 /* deal with chardevs and other special file */
2209 if (!mapping->a_ops->writepage)
2210 return 0;
2211
6a290544
CY		 2212 /* skip writing if there is no dirty page in this inode */
2213 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
2214 return 0;
2215
0771fcc7
CY		 2216 /* during POR, we don't need to trigger writepage at all. */
2217 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
2218 goto skip_write;
2219
a1257023
JK		 2220 if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE &&
2221 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
2222 available_free_memory(sbi, DIRTY_DENTS))
2223 goto skip_write;
2224
d323d005 2225 /* skip writing during file defragment */
91942321 2226 if (is_inode_flag_set(inode, FI_DO_DEFRAG))
d323d005
CY		 2227 goto skip_write;
2228
d31c7c3f
YH		 2229 trace_f2fs_writepages(mapping->host, wbc, DATA);
2230
687de7f1
JK		 2231 /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */
2232 if (wbc->sync_mode == WB_SYNC_ALL)
2233 atomic_inc(&sbi->wb_sync_req);
2234 else if (atomic_read(&sbi->wb_sync_req))
2235 goto skip_write;
2236
9dfa1baf 2237 blk_start_plug(&plug);
b0af6d49 2238 ret = f2fs_write_cache_pages(mapping, wbc, io_type);
9dfa1baf 2239 blk_finish_plug(&plug);
687de7f1
JK		 2240
2241 if (wbc->sync_mode == WB_SYNC_ALL)
2242 atomic_dec(&sbi->wb_sync_req);
28ea6162
JK		 2243 /*
2244 * if some pages were truncated, we cannot guarantee its mapping->host
2245 * to detect pending bios.
2246 */
458e6197 2247
c227f912 2248 remove_dirty_inode(inode);
eb47b800 2249 return ret;
d3baf95d
JK		 2250
2251skip_write:
a7ffdbe2 2252 wbc->pages_skipped += get_dirty_pages(inode);
d31c7c3f 2253 trace_f2fs_writepages(mapping->host, wbc, DATA);
d3baf95d 2254 return 0;
eb47b800
JK		 2255}
2256
b0af6d49
CY		 2257static int f2fs_write_data_pages(struct address_space *mapping,
2258 struct writeback_control *wbc)
2259{
2260 struct inode *inode = mapping->host;
2261
2262 return __f2fs_write_data_pages(mapping, wbc,
2263 F2FS_I(inode)->cp_task == current ?
2264 FS_CP_DATA_IO : FS_DATA_IO);
2265}
2266
3aab8f82
CY		 2267static void f2fs_write_failed(struct address_space *mapping, loff_t to)
2268{
2269 struct inode *inode = mapping->host;
819d9153 2270 loff_t i_size = i_size_read(inode);
3aab8f82 2271
819d9153 2272 if (to > i_size) {
5a3a2d83 2273 down_write(&F2FS_I(inode)->i_mmap_sem);
819d9153
JK		 2274 truncate_pagecache(inode, i_size);
2275 truncate_blocks(inode, i_size, true);
5a3a2d83 2276 up_write(&F2FS_I(inode)->i_mmap_sem);
3aab8f82
CY		 2277 }
2278}
2279
2aadac08
JK		 2280static int prepare_write_begin(struct f2fs_sb_info *sbi,
2281 struct page *page, loff_t pos, unsigned len,
2282 block_t *blk_addr, bool *node_changed)
2283{
2284 struct inode *inode = page->mapping->host;
2285 pgoff_t index = page->index;
2286 struct dnode_of_data dn;
2287 struct page *ipage;
b4d07a3e 2288 bool locked = false;
e15882b6 2289 struct extent_info ei = {0,0,0};
2aadac08 2290 int err = 0;
58282978 2291 int flag;
2aadac08 2292
24b84912
JK		 2293 /*
2294 * we already allocated all the blocks, so we don't need to get
2295 * the block addresses when there is no need to fill the page.
2296 */
dc91de78
JK		 2297 if (!f2fs_has_inline_data(inode) && len == PAGE_SIZE &&
2298 !is_inode_flag_set(inode, FI_NO_PREALLOC))
24b84912
JK		 2299 return 0;
2300
58282978
SY		 2301 /* f2fs_lock_op avoids race between write CP and convert_inline_page */
2302 if (f2fs_has_inline_data(inode) && pos + len > MAX_INLINE_DATA(inode))
2303 flag = F2FS_GET_BLOCK_DEFAULT;
2304 else
2305 flag = F2FS_GET_BLOCK_PRE_AIO;
2306
b4d07a3e 2307 if (f2fs_has_inline_data(inode) ||
09cbfeaf 2308 (pos & PAGE_MASK) >= i_size_read(inode)) {
58282978 2309 __do_map_lock(sbi, flag, true);
b4d07a3e
JK		 2310 locked = true;
2311 }
2312restart:
2aadac08
JK		 2313 /* check inline_data */
2314 ipage = get_node_page(sbi, inode->i_ino);
2315 if (IS_ERR(ipage)) {
2316 err = PTR_ERR(ipage);
2317 goto unlock_out;
2318 }
2319
2320 set_new_dnode(&dn, inode, ipage, ipage, 0);
2321
2322 if (f2fs_has_inline_data(inode)) {
f2470371 2323 if (pos + len <= MAX_INLINE_DATA(inode)) {
2aadac08 2324 read_inline_data(page, ipage);
91942321 2325 set_inode_flag(inode, FI_DATA_EXIST);
ab47036d
CY		 2326 if (inode->i_nlink)
2327 set_inline_node(ipage);
2aadac08
JK		 2328 } else {
2329 err = f2fs_convert_inline_page(&dn, page);
2330 if (err)
b4d07a3e
JK		 2331 goto out;
2332 if (dn.data_blkaddr == NULL_ADDR)
2333 err = f2fs_get_block(&dn, index);
2334 }
2335 } else if (locked) {
2336 err = f2fs_get_block(&dn, index);
2337 } else {
2338 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
2339 dn.data_blkaddr = ei.blk + index - ei.fofs;
2340 } else {
b4d07a3e
JK		 2341 /* hole case */
2342 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
4da7bf5a 2343 if (err || dn.data_blkaddr == NULL_ADDR) {
b4d07a3e 2344 f2fs_put_dnode(&dn);
59c9081b
YH		 2345 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO,
2346 true);
58282978 2347 WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO);
b4d07a3e
JK		 2348 locked = true;
2349 goto restart;
2350 }
2aadac08
JK		 2351 }
2352 }
b4d07a3e 2353
2aadac08
JK		 2354 /* convert_inline_page can make node_changed */
2355 *blk_addr = dn.data_blkaddr;
2356 *node_changed = dn.node_changed;
b4d07a3e 2357out:
2aadac08
JK		 2358 f2fs_put_dnode(&dn);
2359unlock_out:
b4d07a3e 2360 if (locked)
58282978 2361 __do_map_lock(sbi, flag, false);
2aadac08
JK		 2362 return err;
2363}
2364
eb47b800
JK		 2365static int f2fs_write_begin(struct file *file, struct address_space *mapping,
2366 loff_t pos, unsigned len, unsigned flags,
2367 struct page **pagep, void **fsdata)
2368{
2369 struct inode *inode = mapping->host;
4081363f 2370 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
86531d6b 2371 struct page *page = NULL;
09cbfeaf 2372 pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT;
ce767d9a 2373 bool need_balance = false, drop_atomic = false;
2aadac08 2374 block_t blkaddr = NULL_ADDR;
eb47b800
JK		 2375 int err = 0;
2376
a59d8732
MS		 2377 if (trace_android_fs_datawrite_start_enabled()) {
2378 char *path, pathbuf[MAX_TRACE_PATHBUF_LEN];
2379
2380 path = android_fstrace_get_pathname(pathbuf,
2381 MAX_TRACE_PATHBUF_LEN,
2382 inode);
2383 trace_android_fs_datawrite_start(inode, pos, len,
2384 current->pid, path,
2385 current->comm);
2386 }
62aed044
CY		 2387 trace_f2fs_write_begin(inode, pos, len, flags);
2388
7fb3f3b2
JK		 2389 if (f2fs_is_atomic_file(inode) &&
2390 !available_free_memory(sbi, INMEM_PAGES)) {
2391 err = -ENOMEM;
ce767d9a 2392 drop_atomic = true;
7fb3f3b2
JK		 2393 goto fail;
2394 }
2395
5f727395
JK		 2396 /*
2397 * We should check this at this moment to avoid deadlock on inode page
2398 * and #0 page. The locking rule for inline_data conversion should be:
2399 * lock_page(page #0) -> lock_page(inode_page)
2400 */
2401 if (index != 0) {
2402 err = f2fs_convert_inline_inode(inode);
2403 if (err)
2404 goto fail;
2405 }
afcb7ca0 2406repeat:
86d54795
JK		 2407 /*
2408 * Do not use grab_cache_page_write_begin() to avoid deadlock due to
2409 * wait_for_stable_page. Will wait that below with our IO control.
2410 */
7fb3f3b2 2411 page = f2fs_pagecache_get_page(mapping, index,
86d54795 2412 FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS);
3aab8f82
CY		 2413 if (!page) {
2414 err = -ENOMEM;
2415 goto fail;
2416 }
d5f66990 2417
eb47b800
JK		 2418 *pagep = page;
2419
2aadac08
JK		 2420 err = prepare_write_begin(sbi, page, pos, len,
2421 &blkaddr, &need_balance);
9ba69cf9 2422 if (err)
2aadac08 2423 goto fail;
9ba69cf9 2424
7f3037a5 2425 if (need_balance && has_not_enough_free_secs(sbi, 0, 0)) {
2a340760 2426 unlock_page(page);
2c4db1a6 2427 f2fs_balance_fs(sbi, true);
2a340760
JK		 2428 lock_page(page);
2429 if (page->mapping != mapping) {
2430 /* The page got truncated from under us */
2431 f2fs_put_page(page, 1);
2432 goto repeat;
2433 }
2434 }
2435
fec1d657 2436 f2fs_wait_on_page_writeback(page, DATA, false);
b3d208f9 2437
649d7df2
JK		 2438 if (len == PAGE_SIZE || PageUptodate(page))
2439 return 0;
eb47b800 2440
746e2403
YH		 2441 if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode)) {
2442 zero_user_segment(page, len, PAGE_SIZE);
2443 return 0;
2444 }
2445
2aadac08 2446 if (blkaddr == NEW_ADDR) {
09cbfeaf 2447 zero_user_segment(page, 0, PAGE_SIZE);
649d7df2 2448 SetPageUptodate(page);
eb47b800 2449 } else {
13ba41e3
JK		 2450 err = f2fs_submit_page_read(inode, page, blkaddr);
2451 if (err)
78682f79 2452 goto fail;
d54c795b 2453
393ff91f 2454 lock_page(page);
6bacf52f 2455 if (unlikely(page->mapping != mapping)) {
afcb7ca0
JK		 2456 f2fs_put_page(page, 1);
2457 goto repeat;
eb47b800 2458 }
1563ac75
CY		 2459 if (unlikely(!PageUptodate(page))) {
2460 err = -EIO;
2461 goto fail;
4375a336 2462 }
eb47b800 2463 }
eb47b800 2464 return 0;
9ba69cf9 2465
3aab8f82 2466fail:
86531d6b 2467 f2fs_put_page(page, 1);
3aab8f82 2468 f2fs_write_failed(mapping, pos + len);
ce767d9a 2469 if (drop_atomic)
7fb3f3b2 2470 drop_inmem_pages_all(sbi);
3aab8f82 2471 return err;
eb47b800
JK		 2472}
2473
a1dd3c13
JK		 2474static int f2fs_write_end(struct file *file,
2475 struct address_space *mapping,
2476 loff_t pos, unsigned len, unsigned copied,
2477 struct page *page, void *fsdata)
2478{
2479 struct inode *inode = page->mapping->host;
2480
45560fee 2481 trace_android_fs_datawrite_end(inode, pos, len);
dfb2bf38
CY		 2482 trace_f2fs_write_end(inode, pos, len, copied);
2483
649d7df2
JK		 2484 /*
2485 * This should be come from len == PAGE_SIZE, and we expect copied
2486 * should be PAGE_SIZE. Otherwise, we treat it with zero copied and
2487 * let generic_perform_write() try to copy data again through copied=0.
2488 */
2489 if (!PageUptodate(page)) {
746e2403 2490 if (unlikely(copied != len))
649d7df2
JK		 2491 copied = 0;
2492 else
2493 SetPageUptodate(page);
2494 }
2495 if (!copied)
2496 goto unlock_out;
2497
34ba94ba 2498 set_page_dirty(page);
a1dd3c13 2499
fc9581c8
JK		 2500 if (pos + copied > i_size_read(inode))
2501 f2fs_i_size_write(inode, pos + copied);
649d7df2 2502unlock_out:
3024c9a1 2503 f2fs_put_page(page, 1);
d0239e1b 2504 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
a1dd3c13
JK		 2505 return copied;
2506}
2507
6f673763
OS		 2508static int check_direct_IO(struct inode *inode, struct iov_iter *iter,
2509 loff_t offset)
944fcfc1
JK		 2510{
2511 unsigned blocksize_mask = inode->i_sb->s_blocksize - 1;
944fcfc1 2512
944fcfc1
JK		 2513 if (offset & blocksize_mask)
2514 return -EINVAL;
2515
5b46f25d
AV		 2516 if (iov_iter_alignment(iter) & blocksize_mask)
2517 return -EINVAL;
2518
944fcfc1
JK		 2519 return 0;
2520}
2521
c8b8e32d 2522static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
eb47b800 2523{
b439b103 2524 struct address_space *mapping = iocb->ki_filp->f_mapping;
3aab8f82 2525 struct inode *inode = mapping->host;
a2a9032a 2526 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3aab8f82 2527 size_t count = iov_iter_count(iter);
c8b8e32d 2528 loff_t offset = iocb->ki_pos;
82e0a5aa 2529 int rw = iov_iter_rw(iter);
3aab8f82 2530 int err;
a2a9032a
JK		 2531 enum rw_hint hint = iocb->ki_hint;
2532 int whint_mode = F2FS_OPTION(sbi).whint_mode;
944fcfc1 2533
b439b103 2534 err = check_direct_IO(inode, iter, offset);
b9d777b8
JK		 2535 if (err)
2536 return err;
9ffe0fb5 2537
a2a9032a 2538 if (f2fs_force_buffered_io(inode, rw))
36abef4e 2539 return 0;
fcc85a4d 2540
5302fb00 2541 trace_f2fs_direct_IO_enter(inode, offset, count, rw);
70407fad 2542
45560fee 2543 if (trace_android_fs_dataread_start_enabled() &&
a59d8732
MS		 2544 (rw == READ)) {
2545 char *path, pathbuf[MAX_TRACE_PATHBUF_LEN];
2546
2547 path = android_fstrace_get_pathname(pathbuf,
2548 MAX_TRACE_PATHBUF_LEN,
2549 inode);
45560fee 2550 trace_android_fs_dataread_start(inode, offset,
a59d8732 2551 count, current->pid, path,
45560fee 2552 current->comm);
a59d8732 2553 }
45560fee 2554 if (trace_android_fs_datawrite_start_enabled() &&
a59d8732
MS		 2555 (rw == WRITE)) {
2556 char *path, pathbuf[MAX_TRACE_PATHBUF_LEN];
2557
2558 path = android_fstrace_get_pathname(pathbuf,
2559 MAX_TRACE_PATHBUF_LEN,
2560 inode);
45560fee 2561 trace_android_fs_datawrite_start(inode, offset, count,
a59d8732
MS		 2562 current->pid, path,
2563 current->comm);
2564 }
a2a9032a
JK		 2565 if (rw == WRITE && whint_mode == WHINT_MODE_OFF)
2566 iocb->ki_hint = WRITE_LIFE_NOT_SET;
2567
2568 if (!down_read_trylock(&F2FS_I(inode)->dio_rwsem[rw])) {
2569 if (iocb->ki_flags & IOCB_NOWAIT) {
2570 iocb->ki_hint = hint;
2571 err = -EAGAIN;
2572 goto out;
2573 }
2574 down_read(&F2FS_I(inode)->dio_rwsem[rw]);
2575 }
45560fee 2576
c8b8e32d 2577 err = blockdev_direct_IO(iocb, inode, iter, get_data_block_dio);
82e0a5aa
CY		 2578 up_read(&F2FS_I(inode)->dio_rwsem[rw]);
2579
2580 if (rw == WRITE) {
a2a9032a
JK		 2581 if (whint_mode == WHINT_MODE_OFF)
2582 iocb->ki_hint = hint;
b0af6d49
CY		 2583 if (err > 0) {
2584 f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_IO,
2585 err);
91942321 2586 set_inode_flag(inode, FI_UPDATE_WRITE);
b0af6d49 2587 } else if (err < 0) {
6bfc4919 2588 f2fs_write_failed(mapping, offset + count);
b0af6d49 2589 }
6bfc4919 2590 }
a2a9032a 2591out:
45560fee
MS		 2592 if (trace_android_fs_dataread_start_enabled() &&
2593 (rw == READ))
2594 trace_android_fs_dataread_end(inode, offset, count);
2595 if (trace_android_fs_datawrite_start_enabled() &&
2596 (rw == WRITE))
2597 trace_android_fs_datawrite_end(inode, offset, count);
2598
5302fb00 2599 trace_f2fs_direct_IO_exit(inode, offset, count, rw, err);
70407fad 2600
3aab8f82 2601 return err;
eb47b800
JK		 2602}
2603
487261f3
CY		 2604void f2fs_invalidate_page(struct page *page, unsigned int offset,
2605 unsigned int length)
eb47b800
JK		 2606{
2607 struct inode *inode = page->mapping->host;
487261f3 2608 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
a7ffdbe2 2609
487261f3 2610 if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
09cbfeaf 2611 (offset % PAGE_SIZE || length != PAGE_SIZE))
a7ffdbe2
JK		 2612 return;
2613
487261f3 2614 if (PageDirty(page)) {
933439c8 2615 if (inode->i_ino == F2FS_META_INO(sbi)) {
487261f3 2616 dec_page_count(sbi, F2FS_DIRTY_META);
933439c8 2617 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
487261f3 2618 dec_page_count(sbi, F2FS_DIRTY_NODES);
933439c8 2619 } else {
487261f3 2620 inode_dec_dirty_pages(inode);
933439c8
CY		 2621 remove_dirty_inode(inode);
2622 }
487261f3 2623 }
decd36b6
CY		 2624
2625 /* This is atomic written page, keep Private */
2626 if (IS_ATOMIC_WRITTEN_PAGE(page))
8c242db9 2627 return drop_inmem_page(inode, page);
decd36b6 2628
23dc974e 2629 set_page_private(page, 0);
eb47b800
JK		 2630 ClearPagePrivate(page);
2631}
2632
487261f3 2633int f2fs_release_page(struct page *page, gfp_t wait)
eb47b800 2634{
f68daeeb
JK		 2635 /* If this is dirty page, keep PagePrivate */
2636 if (PageDirty(page))
2637 return 0;
2638
decd36b6
CY		 2639 /* This is atomic written page, keep Private */
2640 if (IS_ATOMIC_WRITTEN_PAGE(page))
2641 return 0;
2642
23dc974e 2643 set_page_private(page, 0);
eb47b800 2644 ClearPagePrivate(page);
c3850aa1 2645 return 1;
eb47b800
JK		 2646}
2647
2648static int f2fs_set_data_page_dirty(struct page *page)
2649{
2650 struct address_space *mapping = page->mapping;
2651 struct inode *inode = mapping->host;
2652
26c6b887
JK		 2653 trace_f2fs_set_page_dirty(page, DATA);
2654
237c0790
JK		 2655 if (!PageUptodate(page))
2656 SetPageUptodate(page);
34ba94ba 2657
5fe45743 2658 if (f2fs_is_atomic_file(inode) && !f2fs_is_commit_atomic_write(inode)) {
decd36b6
CY		 2659 if (!IS_ATOMIC_WRITTEN_PAGE(page)) {
2660 register_inmem_page(inode, page);
2661 return 1;
2662 }
2663 /*
2664 * Previously, this page has been registered, we just
2665 * return here.
2666 */
2667 return 0;
34ba94ba
JK		 2668 }
2669
eb47b800 2670 if (!PageDirty(page)) {
3eb22bff 2671 __set_page_dirty_nobuffers(page);
a7ffdbe2 2672 update_dirty_page(inode, page);
eb47b800
JK		 2673 return 1;
2674 }
2675 return 0;
2676}
2677
c01e54b7
JK		 2678static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
2679{
454ae7e5
CY		 2680 struct inode *inode = mapping->host;
2681
1d373a0e
JK		 2682 if (f2fs_has_inline_data(inode))
2683 return 0;
2684
2685 /* make sure allocating whole blocks */
2686 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2687 filemap_write_and_wait(mapping);
2688
e2b4e2bc 2689 return generic_block_bmap(mapping, block, get_data_block_bmap);
429511cd
CY		 2690}
2691
5b7a487c
WG		 2692#ifdef CONFIG_MIGRATION
2693#include <linux/migrate.h>
2694
2695int f2fs_migrate_page(struct address_space *mapping,
2696 struct page *newpage, struct page *page, enum migrate_mode mode)
2697{
2698 int rc, extra_count;
2699 struct f2fs_inode_info *fi = F2FS_I(mapping->host);
2700 bool atomic_written = IS_ATOMIC_WRITTEN_PAGE(page);
2701
2702 BUG_ON(PageWriteback(page));
2703
2704 /* migrating an atomic written page is safe with the inmem_lock hold */
ff1048e7
JK		 2705 if (atomic_written) {
2706 if (mode != MIGRATE_SYNC)
2707 return -EBUSY;
2708 if (!mutex_trylock(&fi->inmem_lock))
2709 return -EAGAIN;
2710 }
5b7a487c
WG		 2711
2712 /*
2713 * A reference is expected if PagePrivate set when move mapping,
2714 * however F2FS breaks this for maintaining dirty page counts when
2715 * truncating pages. So here adjusting the 'extra_count' make it work.
2716 */
2717 extra_count = (atomic_written ? 1 : 0) - page_has_private(page);
2718 rc = migrate_page_move_mapping(mapping, newpage,
2719 page, NULL, mode, extra_count);
2720 if (rc != MIGRATEPAGE_SUCCESS) {
2721 if (atomic_written)
2722 mutex_unlock(&fi->inmem_lock);
2723 return rc;
2724 }
2725
2726 if (atomic_written) {
2727 struct inmem_pages *cur;
2728 list_for_each_entry(cur, &fi->inmem_pages, list)
2729 if (cur->page == page) {
2730 cur->page = newpage;
2731 break;
2732 }
2733 mutex_unlock(&fi->inmem_lock);
2734 put_page(page);
2735 get_page(newpage);
2736 }
2737
2738 if (PagePrivate(page))
2739 SetPagePrivate(newpage);
2740 set_page_private(newpage, page_private(page));
2741
2916ecc0
JG		 2742 if (mode != MIGRATE_SYNC_NO_COPY)
2743 migrate_page_copy(newpage, page);
2744 else
2745 migrate_page_states(newpage, page);
5b7a487c
WG		 2746
2747 return MIGRATEPAGE_SUCCESS;
2748}
2749#endif
2750
eb47b800
JK		 2751const struct address_space_operations f2fs_dblock_aops = {
2752 .readpage = f2fs_read_data_page,
2753 .readpages = f2fs_read_data_pages,
2754 .writepage = f2fs_write_data_page,
2755 .writepages = f2fs_write_data_pages,
2756 .write_begin = f2fs_write_begin,
a1dd3c13 2757 .write_end = f2fs_write_end,
eb47b800 2758 .set_page_dirty = f2fs_set_data_page_dirty,
487261f3
CY		 2759 .invalidatepage = f2fs_invalidate_page,
2760 .releasepage = f2fs_release_page,
eb47b800 2761 .direct_IO = f2fs_direct_IO,
c01e54b7 2762 .bmap = f2fs_bmap,
5b7a487c
WG		 2763#ifdef CONFIG_MIGRATION
2764 .migratepage = f2fs_migrate_page,
2765#endif
eb47b800 2766};
c6e638ae
EB		 2767
2768int __init f2fs_init_post_read_processing(void)
2769{
2770 bio_post_read_ctx_cache = KMEM_CACHE(bio_post_read_ctx, 0);
2771 if (!bio_post_read_ctx_cache)
2772 goto fail;
2773 bio_post_read_ctx_pool =
2774 mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS,
2775 bio_post_read_ctx_cache);
2776 if (!bio_post_read_ctx_pool)
2777 goto fail_free_cache;
2778 return 0;
2779
2780fail_free_cache:
2781 kmem_cache_destroy(bio_post_read_ctx_cache);
2782fail:
2783 return -ENOMEM;
2784}
2785
2786void __exit f2fs_destroy_post_read_processing(void)
2787{
2788 mempool_destroy(bio_post_read_ctx_pool);
2789 kmem_cache_destroy(bio_post_read_ctx_cache);
2790}
Motorola kernel-slsi