4 * Write file data over NFS.
6 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
9 #include <linux/types.h>
10 #include <linux/slab.h>
12 #include <linux/pagemap.h>
13 #include <linux/file.h>
14 #include <linux/writeback.h>
15 #include <linux/swap.h>
16 #include <linux/migrate.h>
18 #include <linux/sunrpc/clnt.h>
19 #include <linux/nfs_fs.h>
20 #include <linux/nfs_mount.h>
21 #include <linux/nfs_page.h>
22 #include <linux/backing-dev.h>
23 #include <linux/export.h>
25 #include <asm/uaccess.h>
27 #include "delegation.h"
34 #define NFSDBG_FACILITY NFSDBG_PAGECACHE
36 #define MIN_POOL_WRITE (32)
37 #define MIN_POOL_COMMIT (4)
40 * Local function declarations
42 static void nfs_redirty_request(struct nfs_page
*req
);
43 static const struct rpc_call_ops nfs_write_common_ops
;
44 static const struct rpc_call_ops nfs_commit_ops
;
45 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops
;
46 static const struct nfs_commit_completion_ops nfs_commit_completion_ops
;
48 static struct kmem_cache
*nfs_wdata_cachep
;
49 static mempool_t
*nfs_wdata_mempool
;
50 static struct kmem_cache
*nfs_cdata_cachep
;
51 static mempool_t
*nfs_commit_mempool
;
53 struct nfs_commit_data
*nfs_commitdata_alloc(void)
55 struct nfs_commit_data
*p
= mempool_alloc(nfs_commit_mempool
, GFP_NOIO
);
58 memset(p
, 0, sizeof(*p
));
59 INIT_LIST_HEAD(&p
->pages
);
63 EXPORT_SYMBOL_GPL(nfs_commitdata_alloc
);
65 void nfs_commit_free(struct nfs_commit_data
*p
)
67 mempool_free(p
, nfs_commit_mempool
);
69 EXPORT_SYMBOL_GPL(nfs_commit_free
);
71 struct nfs_write_header
*nfs_writehdr_alloc(void)
73 struct nfs_write_header
*p
= mempool_alloc(nfs_wdata_mempool
, GFP_NOIO
);
76 struct nfs_pgio_header
*hdr
= &p
->header
;
78 memset(p
, 0, sizeof(*p
));
79 INIT_LIST_HEAD(&hdr
->pages
);
80 INIT_LIST_HEAD(&hdr
->rpc_list
);
81 spin_lock_init(&hdr
->lock
);
82 atomic_set(&hdr
->refcnt
, 0);
87 EXPORT_SYMBOL_GPL(nfs_writehdr_alloc
);
89 static struct nfs_write_data
*nfs_writedata_alloc(struct nfs_pgio_header
*hdr
,
90 unsigned int pagecount
)
92 struct nfs_write_data
*data
, *prealloc
;
94 prealloc
= &container_of(hdr
, struct nfs_write_header
, header
)->rpc_data
;
95 if (prealloc
->header
== NULL
)
98 data
= kzalloc(sizeof(*data
), GFP_KERNEL
);
102 if (nfs_pgarray_set(&data
->pages
, pagecount
)) {
104 atomic_inc(&hdr
->refcnt
);
106 if (data
!= prealloc
)
114 void nfs_writehdr_free(struct nfs_pgio_header
*hdr
)
116 struct nfs_write_header
*whdr
= container_of(hdr
, struct nfs_write_header
, header
);
117 mempool_free(whdr
, nfs_wdata_mempool
);
119 EXPORT_SYMBOL_GPL(nfs_writehdr_free
);
121 void nfs_writedata_release(struct nfs_write_data
*wdata
)
123 struct nfs_pgio_header
*hdr
= wdata
->header
;
124 struct nfs_write_header
*write_header
= container_of(hdr
, struct nfs_write_header
, header
);
126 put_nfs_open_context(wdata
->args
.context
);
127 if (wdata
->pages
.pagevec
!= wdata
->pages
.page_array
)
128 kfree(wdata
->pages
.pagevec
);
129 if (wdata
!= &write_header
->rpc_data
)
132 wdata
->header
= NULL
;
133 if (atomic_dec_and_test(&hdr
->refcnt
))
134 hdr
->completion_ops
->completion(hdr
);
136 EXPORT_SYMBOL_GPL(nfs_writedata_release
);
138 static void nfs_context_set_write_error(struct nfs_open_context
*ctx
, int error
)
142 set_bit(NFS_CONTEXT_ERROR_WRITE
, &ctx
->flags
);
145 static struct nfs_page
*
146 nfs_page_find_request_locked(struct nfs_inode
*nfsi
, struct page
*page
)
148 struct nfs_page
*req
= NULL
;
150 if (PagePrivate(page
))
151 req
= (struct nfs_page
*)page_private(page
);
152 else if (unlikely(PageSwapCache(page
))) {
153 struct nfs_page
*freq
, *t
;
155 /* Linearly search the commit list for the correct req */
156 list_for_each_entry_safe(freq
, t
, &nfsi
->commit_info
.list
, wb_list
) {
157 if (freq
->wb_page
== page
) {
165 kref_get(&req
->wb_kref
);
170 static struct nfs_page
*nfs_page_find_request(struct page
*page
)
172 struct inode
*inode
= page_file_mapping(page
)->host
;
173 struct nfs_page
*req
= NULL
;
175 spin_lock(&inode
->i_lock
);
176 req
= nfs_page_find_request_locked(NFS_I(inode
), page
);
177 spin_unlock(&inode
->i_lock
);
181 /* Adjust the file length if we're writing beyond the end */
182 static void nfs_grow_file(struct page
*page
, unsigned int offset
, unsigned int count
)
184 struct inode
*inode
= page_file_mapping(page
)->host
;
188 spin_lock(&inode
->i_lock
);
189 i_size
= i_size_read(inode
);
190 end_index
= (i_size
- 1) >> PAGE_CACHE_SHIFT
;
191 if (i_size
> 0 && page_file_index(page
) < end_index
)
193 end
= page_file_offset(page
) + ((loff_t
)offset
+count
);
196 i_size_write(inode
, end
);
197 nfs_inc_stats(inode
, NFSIOS_EXTENDWRITE
);
199 spin_unlock(&inode
->i_lock
);
202 /* A writeback failed: mark the page as bad, and invalidate the page cache */
203 static void nfs_set_pageerror(struct page
*page
)
205 nfs_zap_mapping(page_file_mapping(page
)->host
, page_file_mapping(page
));
208 /* We can set the PG_uptodate flag if we see that a write request
209 * covers the full page.
211 static void nfs_mark_uptodate(struct page
*page
, unsigned int base
, unsigned int count
)
213 if (PageUptodate(page
))
217 if (count
!= nfs_page_length(page
))
219 SetPageUptodate(page
);
222 static int wb_priority(struct writeback_control
*wbc
)
224 if (wbc
->for_reclaim
)
225 return FLUSH_HIGHPRI
| FLUSH_STABLE
;
226 if (wbc
->for_kupdate
|| wbc
->for_background
)
227 return FLUSH_LOWPRI
| FLUSH_COND_STABLE
;
228 return FLUSH_COND_STABLE
;
232 * NFS congestion control
235 int nfs_congestion_kb
;
237 #define NFS_CONGESTION_ON_THRESH (nfs_congestion_kb >> (PAGE_SHIFT-10))
238 #define NFS_CONGESTION_OFF_THRESH \
239 (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))
241 static void nfs_set_page_writeback(struct page
*page
)
243 struct nfs_server
*nfss
= NFS_SERVER(page_file_mapping(page
)->host
);
244 int ret
= test_set_page_writeback(page
);
246 WARN_ON_ONCE(ret
!= 0);
248 if (atomic_long_inc_return(&nfss
->writeback
) >
249 NFS_CONGESTION_ON_THRESH
) {
250 set_bdi_congested(&nfss
->backing_dev_info
,
255 static void nfs_end_page_writeback(struct page
*page
)
257 struct inode
*inode
= page_file_mapping(page
)->host
;
258 struct nfs_server
*nfss
= NFS_SERVER(inode
);
260 end_page_writeback(page
);
261 if (atomic_long_dec_return(&nfss
->writeback
) < NFS_CONGESTION_OFF_THRESH
)
262 clear_bdi_congested(&nfss
->backing_dev_info
, BLK_RW_ASYNC
);
265 static struct nfs_page
*nfs_find_and_lock_request(struct page
*page
, bool nonblock
)
267 struct inode
*inode
= page_file_mapping(page
)->host
;
268 struct nfs_page
*req
;
271 spin_lock(&inode
->i_lock
);
273 req
= nfs_page_find_request_locked(NFS_I(inode
), page
);
276 if (nfs_lock_request(req
))
278 /* Note: If we hold the page lock, as is the case in nfs_writepage,
279 * then the call to nfs_lock_request() will always
280 * succeed provided that someone hasn't already marked the
281 * request as dirty (in which case we don't care).
283 spin_unlock(&inode
->i_lock
);
285 ret
= nfs_wait_on_request(req
);
288 nfs_release_request(req
);
291 spin_lock(&inode
->i_lock
);
293 spin_unlock(&inode
->i_lock
);
298 * Find an associated nfs write request, and prepare to flush it out
299 * May return an error if the user signalled nfs_wait_on_request().
301 static int nfs_page_async_flush(struct nfs_pageio_descriptor
*pgio
,
302 struct page
*page
, bool nonblock
)
304 struct nfs_page
*req
;
307 req
= nfs_find_and_lock_request(page
, nonblock
);
314 nfs_set_page_writeback(page
);
315 WARN_ON_ONCE(test_bit(PG_CLEAN
, &req
->wb_flags
));
318 if (!nfs_pageio_add_request(pgio
, req
)) {
319 nfs_redirty_request(req
);
320 ret
= pgio
->pg_error
;
326 static int nfs_do_writepage(struct page
*page
, struct writeback_control
*wbc
, struct nfs_pageio_descriptor
*pgio
)
328 struct inode
*inode
= page_file_mapping(page
)->host
;
331 nfs_inc_stats(inode
, NFSIOS_VFSWRITEPAGE
);
332 nfs_add_stats(inode
, NFSIOS_WRITEPAGES
, 1);
334 nfs_pageio_cond_complete(pgio
, page_file_index(page
));
335 ret
= nfs_page_async_flush(pgio
, page
, wbc
->sync_mode
== WB_SYNC_NONE
);
336 if (ret
== -EAGAIN
) {
337 redirty_page_for_writepage(wbc
, page
);
344 * Write an mmapped page to the server.
346 static int nfs_writepage_locked(struct page
*page
, struct writeback_control
*wbc
)
348 struct nfs_pageio_descriptor pgio
;
351 NFS_PROTO(page_file_mapping(page
)->host
)->write_pageio_init(&pgio
,
354 &nfs_async_write_completion_ops
);
355 err
= nfs_do_writepage(page
, wbc
, &pgio
);
356 nfs_pageio_complete(&pgio
);
359 if (pgio
.pg_error
< 0)
360 return pgio
.pg_error
;
364 int nfs_writepage(struct page
*page
, struct writeback_control
*wbc
)
368 ret
= nfs_writepage_locked(page
, wbc
);
373 static int nfs_writepages_callback(struct page
*page
, struct writeback_control
*wbc
, void *data
)
377 ret
= nfs_do_writepage(page
, wbc
, data
);
382 int nfs_writepages(struct address_space
*mapping
, struct writeback_control
*wbc
)
384 struct inode
*inode
= mapping
->host
;
385 unsigned long *bitlock
= &NFS_I(inode
)->flags
;
386 struct nfs_pageio_descriptor pgio
;
389 /* Stop dirtying of new pages while we sync */
390 err
= wait_on_bit_lock(bitlock
, NFS_INO_FLUSHING
,
391 nfs_wait_bit_killable
, TASK_KILLABLE
);
395 nfs_inc_stats(inode
, NFSIOS_VFSWRITEPAGES
);
397 NFS_PROTO(inode
)->write_pageio_init(&pgio
, inode
, wb_priority(wbc
), &nfs_async_write_completion_ops
);
398 err
= write_cache_pages(mapping
, wbc
, nfs_writepages_callback
, &pgio
);
399 nfs_pageio_complete(&pgio
);
401 clear_bit_unlock(NFS_INO_FLUSHING
, bitlock
);
402 smp_mb__after_clear_bit();
403 wake_up_bit(bitlock
, NFS_INO_FLUSHING
);
416 * Insert a write request into an inode
418 static void nfs_inode_add_request(struct inode
*inode
, struct nfs_page
*req
)
420 struct nfs_inode
*nfsi
= NFS_I(inode
);
422 /* Lock the request! */
423 nfs_lock_request(req
);
425 spin_lock(&inode
->i_lock
);
426 if (!nfsi
->npages
&& NFS_PROTO(inode
)->have_delegation(inode
, FMODE_WRITE
))
429 * Swap-space should not get truncated. Hence no need to plug the race
430 * with invalidate/truncate.
432 if (likely(!PageSwapCache(req
->wb_page
))) {
433 set_bit(PG_MAPPED
, &req
->wb_flags
);
434 SetPagePrivate(req
->wb_page
);
435 set_page_private(req
->wb_page
, (unsigned long)req
);
438 kref_get(&req
->wb_kref
);
439 spin_unlock(&inode
->i_lock
);
443 * Remove a write request from an inode
445 static void nfs_inode_remove_request(struct nfs_page
*req
)
447 struct inode
*inode
= req
->wb_context
->dentry
->d_inode
;
448 struct nfs_inode
*nfsi
= NFS_I(inode
);
450 spin_lock(&inode
->i_lock
);
451 if (likely(!PageSwapCache(req
->wb_page
))) {
452 set_page_private(req
->wb_page
, 0);
453 ClearPagePrivate(req
->wb_page
);
454 clear_bit(PG_MAPPED
, &req
->wb_flags
);
457 spin_unlock(&inode
->i_lock
);
458 nfs_release_request(req
);
462 nfs_mark_request_dirty(struct nfs_page
*req
)
464 __set_page_dirty_nobuffers(req
->wb_page
);
467 #if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
469 * nfs_request_add_commit_list - add request to a commit list
470 * @req: pointer to a struct nfs_page
471 * @dst: commit list head
472 * @cinfo: holds list lock and accounting info
474 * This sets the PG_CLEAN bit, updates the cinfo count of
475 * number of outstanding requests requiring a commit as well as
478 * The caller must _not_ hold the cinfo->lock, but must be
479 * holding the nfs_page lock.
482 nfs_request_add_commit_list(struct nfs_page
*req
, struct list_head
*dst
,
483 struct nfs_commit_info
*cinfo
)
485 set_bit(PG_CLEAN
, &(req
)->wb_flags
);
486 spin_lock(cinfo
->lock
);
487 nfs_list_add_request(req
, dst
);
488 cinfo
->mds
->ncommit
++;
489 spin_unlock(cinfo
->lock
);
491 inc_zone_page_state(req
->wb_page
, NR_UNSTABLE_NFS
);
492 inc_bdi_stat(page_file_mapping(req
->wb_page
)->backing_dev_info
,
494 __mark_inode_dirty(req
->wb_context
->dentry
->d_inode
,
498 EXPORT_SYMBOL_GPL(nfs_request_add_commit_list
);
501 * nfs_request_remove_commit_list - Remove request from a commit list
502 * @req: pointer to a nfs_page
503 * @cinfo: holds list lock and accounting info
505 * This clears the PG_CLEAN bit, and updates the cinfo's count of
506 * number of outstanding requests requiring a commit
507 * It does not update the MM page stats.
509 * The caller _must_ hold the cinfo->lock and the nfs_page lock.
512 nfs_request_remove_commit_list(struct nfs_page
*req
,
513 struct nfs_commit_info
*cinfo
)
515 if (!test_and_clear_bit(PG_CLEAN
, &(req
)->wb_flags
))
517 nfs_list_remove_request(req
);
518 cinfo
->mds
->ncommit
--;
520 EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list
);
522 static void nfs_init_cinfo_from_inode(struct nfs_commit_info
*cinfo
,
525 cinfo
->lock
= &inode
->i_lock
;
526 cinfo
->mds
= &NFS_I(inode
)->commit_info
;
527 cinfo
->ds
= pnfs_get_ds_info(inode
);
529 cinfo
->completion_ops
= &nfs_commit_completion_ops
;
532 void nfs_init_cinfo(struct nfs_commit_info
*cinfo
,
534 struct nfs_direct_req
*dreq
)
537 nfs_init_cinfo_from_dreq(cinfo
, dreq
);
539 nfs_init_cinfo_from_inode(cinfo
, inode
);
541 EXPORT_SYMBOL_GPL(nfs_init_cinfo
);
544 * Add a request to the inode's commit list.
547 nfs_mark_request_commit(struct nfs_page
*req
, struct pnfs_layout_segment
*lseg
,
548 struct nfs_commit_info
*cinfo
)
550 if (pnfs_mark_request_commit(req
, lseg
, cinfo
))
552 nfs_request_add_commit_list(req
, &cinfo
->mds
->list
, cinfo
);
556 nfs_clear_page_commit(struct page
*page
)
558 dec_zone_page_state(page
, NR_UNSTABLE_NFS
);
559 dec_bdi_stat(page_file_mapping(page
)->backing_dev_info
, BDI_RECLAIMABLE
);
563 nfs_clear_request_commit(struct nfs_page
*req
)
565 if (test_bit(PG_CLEAN
, &req
->wb_flags
)) {
566 struct inode
*inode
= req
->wb_context
->dentry
->d_inode
;
567 struct nfs_commit_info cinfo
;
569 nfs_init_cinfo_from_inode(&cinfo
, inode
);
570 if (!pnfs_clear_request_commit(req
, &cinfo
)) {
571 spin_lock(cinfo
.lock
);
572 nfs_request_remove_commit_list(req
, &cinfo
);
573 spin_unlock(cinfo
.lock
);
575 nfs_clear_page_commit(req
->wb_page
);
580 int nfs_write_need_commit(struct nfs_write_data
*data
)
582 if (data
->verf
.committed
== NFS_DATA_SYNC
)
583 return data
->header
->lseg
== NULL
;
584 return data
->verf
.committed
!= NFS_FILE_SYNC
;
588 static void nfs_init_cinfo_from_inode(struct nfs_commit_info
*cinfo
,
593 void nfs_init_cinfo(struct nfs_commit_info
*cinfo
,
595 struct nfs_direct_req
*dreq
)
600 nfs_mark_request_commit(struct nfs_page
*req
, struct pnfs_layout_segment
*lseg
,
601 struct nfs_commit_info
*cinfo
)
606 nfs_clear_request_commit(struct nfs_page
*req
)
611 int nfs_write_need_commit(struct nfs_write_data
*data
)
618 static void nfs_write_completion(struct nfs_pgio_header
*hdr
)
620 struct nfs_commit_info cinfo
;
621 unsigned long bytes
= 0;
623 if (test_bit(NFS_IOHDR_REDO
, &hdr
->flags
))
625 nfs_init_cinfo_from_inode(&cinfo
, hdr
->inode
);
626 while (!list_empty(&hdr
->pages
)) {
627 struct nfs_page
*req
= nfs_list_entry(hdr
->pages
.next
);
629 bytes
+= req
->wb_bytes
;
630 nfs_list_remove_request(req
);
631 if (test_bit(NFS_IOHDR_ERROR
, &hdr
->flags
) &&
632 (hdr
->good_bytes
< bytes
)) {
633 nfs_set_pageerror(req
->wb_page
);
634 nfs_context_set_write_error(req
->wb_context
, hdr
->error
);
637 if (test_bit(NFS_IOHDR_NEED_RESCHED
, &hdr
->flags
)) {
638 nfs_mark_request_dirty(req
);
641 if (test_bit(NFS_IOHDR_NEED_COMMIT
, &hdr
->flags
)) {
642 memcpy(&req
->wb_verf
, &hdr
->verf
->verifier
, sizeof(req
->wb_verf
));
643 nfs_mark_request_commit(req
, hdr
->lseg
, &cinfo
);
647 nfs_inode_remove_request(req
);
649 nfs_unlock_request(req
);
650 nfs_end_page_writeback(req
->wb_page
);
651 nfs_release_request(req
);
657 #if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
659 nfs_reqs_to_commit(struct nfs_commit_info
*cinfo
)
661 return cinfo
->mds
->ncommit
;
664 /* cinfo->lock held by caller */
666 nfs_scan_commit_list(struct list_head
*src
, struct list_head
*dst
,
667 struct nfs_commit_info
*cinfo
, int max
)
669 struct nfs_page
*req
, *tmp
;
672 list_for_each_entry_safe(req
, tmp
, src
, wb_list
) {
673 if (!nfs_lock_request(req
))
675 kref_get(&req
->wb_kref
);
676 if (cond_resched_lock(cinfo
->lock
))
677 list_safe_reset_next(req
, tmp
, wb_list
);
678 nfs_request_remove_commit_list(req
, cinfo
);
679 nfs_list_add_request(req
, dst
);
681 if ((ret
== max
) && !cinfo
->dreq
)
688 * nfs_scan_commit - Scan an inode for commit requests
689 * @inode: NFS inode to scan
690 * @dst: mds destination list
691 * @cinfo: mds and ds lists of reqs ready to commit
693 * Moves requests from the inode's 'commit' request list.
694 * The requests are *not* checked to ensure that they form a contiguous set.
697 nfs_scan_commit(struct inode
*inode
, struct list_head
*dst
,
698 struct nfs_commit_info
*cinfo
)
702 spin_lock(cinfo
->lock
);
703 if (cinfo
->mds
->ncommit
> 0) {
704 const int max
= INT_MAX
;
706 ret
= nfs_scan_commit_list(&cinfo
->mds
->list
, dst
,
708 ret
+= pnfs_scan_commit_lists(inode
, cinfo
, max
- ret
);
710 spin_unlock(cinfo
->lock
);
715 static unsigned long nfs_reqs_to_commit(struct nfs_commit_info
*cinfo
)
720 int nfs_scan_commit(struct inode
*inode
, struct list_head
*dst
,
721 struct nfs_commit_info
*cinfo
)
728 * Search for an existing write request, and attempt to update
729 * it to reflect a new dirty region on a given page.
731 * If the attempt fails, then the existing request is flushed out
734 static struct nfs_page
*nfs_try_to_update_request(struct inode
*inode
,
739 struct nfs_page
*req
;
744 if (!PagePrivate(page
))
747 end
= offset
+ bytes
;
748 spin_lock(&inode
->i_lock
);
751 req
= nfs_page_find_request_locked(NFS_I(inode
), page
);
755 rqend
= req
->wb_offset
+ req
->wb_bytes
;
757 * Tell the caller to flush out the request if
758 * the offsets are non-contiguous.
759 * Note: nfs_flush_incompatible() will already
760 * have flushed out requests having wrong owners.
763 || end
< req
->wb_offset
)
766 if (nfs_lock_request(req
))
769 /* The request is locked, so wait and then retry */
770 spin_unlock(&inode
->i_lock
);
771 error
= nfs_wait_on_request(req
);
772 nfs_release_request(req
);
775 spin_lock(&inode
->i_lock
);
778 /* Okay, the request matches. Update the region */
779 if (offset
< req
->wb_offset
) {
780 req
->wb_offset
= offset
;
781 req
->wb_pgbase
= offset
;
784 req
->wb_bytes
= end
- req
->wb_offset
;
786 req
->wb_bytes
= rqend
- req
->wb_offset
;
788 spin_unlock(&inode
->i_lock
);
790 nfs_clear_request_commit(req
);
793 spin_unlock(&inode
->i_lock
);
794 nfs_release_request(req
);
795 error
= nfs_wb_page(inode
, page
);
797 return ERR_PTR(error
);
801 * Try to update an existing write request, or create one if there is none.
803 * Note: Should always be called with the Page Lock held to prevent races
804 * if we have to add a new request. Also assumes that the caller has
805 * already called nfs_flush_incompatible() if necessary.
807 static struct nfs_page
* nfs_setup_write_request(struct nfs_open_context
* ctx
,
808 struct page
*page
, unsigned int offset
, unsigned int bytes
)
810 struct inode
*inode
= page_file_mapping(page
)->host
;
811 struct nfs_page
*req
;
813 req
= nfs_try_to_update_request(inode
, page
, offset
, bytes
);
816 req
= nfs_create_request(ctx
, inode
, page
, offset
, bytes
);
819 nfs_inode_add_request(inode
, req
);
824 static int nfs_writepage_setup(struct nfs_open_context
*ctx
, struct page
*page
,
825 unsigned int offset
, unsigned int count
)
827 struct nfs_page
*req
;
829 req
= nfs_setup_write_request(ctx
, page
, offset
, count
);
832 /* Update file length */
833 nfs_grow_file(page
, offset
, count
);
834 nfs_mark_uptodate(page
, req
->wb_pgbase
, req
->wb_bytes
);
835 nfs_mark_request_dirty(req
);
836 nfs_unlock_and_release_request(req
);
840 int nfs_flush_incompatible(struct file
*file
, struct page
*page
)
842 struct nfs_open_context
*ctx
= nfs_file_open_context(file
);
843 struct nfs_lock_context
*l_ctx
;
844 struct nfs_page
*req
;
845 int do_flush
, status
;
847 * Look for a request corresponding to this page. If there
848 * is one, and it belongs to another file, we flush it out
849 * before we try to copy anything into the page. Do this
850 * due to the lack of an ACCESS-type call in NFSv2.
851 * Also do the same if we find a request from an existing
855 req
= nfs_page_find_request(page
);
858 l_ctx
= req
->wb_lock_context
;
859 do_flush
= req
->wb_page
!= page
|| req
->wb_context
!= ctx
;
861 do_flush
|= l_ctx
->lockowner
.l_owner
!= current
->files
862 || l_ctx
->lockowner
.l_pid
!= current
->tgid
;
864 nfs_release_request(req
);
867 status
= nfs_wb_page(page_file_mapping(page
)->host
, page
);
868 } while (status
== 0);
873 * If the page cache is marked as unsafe or invalid, then we can't rely on
874 * the PageUptodate() flag. In this case, we will need to turn off
875 * write optimisations that depend on the page contents being correct.
877 static bool nfs_write_pageuptodate(struct page
*page
, struct inode
*inode
)
879 if (nfs_have_delegated_attributes(inode
))
881 if (NFS_I(inode
)->cache_validity
& (NFS_INO_INVALID_DATA
|NFS_INO_REVAL_PAGECACHE
))
884 return PageUptodate(page
) != 0;
888 * Update and possibly write a cached page of an NFS file.
890 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
891 * things with a page scheduled for an RPC call (e.g. invalidate it).
893 int nfs_updatepage(struct file
*file
, struct page
*page
,
894 unsigned int offset
, unsigned int count
)
896 struct nfs_open_context
*ctx
= nfs_file_open_context(file
);
897 struct inode
*inode
= page_file_mapping(page
)->host
;
900 nfs_inc_stats(inode
, NFSIOS_VFSUPDATEPAGE
);
902 dprintk("NFS: nfs_updatepage(%s/%s %d@%lld)\n",
903 file
->f_path
.dentry
->d_parent
->d_name
.name
,
904 file
->f_path
.dentry
->d_name
.name
, count
,
905 (long long)(page_file_offset(page
) + offset
));
907 /* If we're not using byte range locks, and we know the page
908 * is up to date, it may be more efficient to extend the write
909 * to cover the entire page in order to avoid fragmentation
912 if (nfs_write_pageuptodate(page
, inode
) &&
913 inode
->i_flock
== NULL
&&
914 !(file
->f_flags
& O_DSYNC
)) {
915 count
= max(count
+ offset
, nfs_page_length(page
));
919 status
= nfs_writepage_setup(ctx
, page
, offset
, count
);
921 nfs_set_pageerror(page
);
923 __set_page_dirty_nobuffers(page
);
925 dprintk("NFS: nfs_updatepage returns %d (isize %lld)\n",
926 status
, (long long)i_size_read(inode
));
930 static int flush_task_priority(int how
)
932 switch (how
& (FLUSH_HIGHPRI
|FLUSH_LOWPRI
)) {
934 return RPC_PRIORITY_HIGH
;
936 return RPC_PRIORITY_LOW
;
938 return RPC_PRIORITY_NORMAL
;
941 int nfs_initiate_write(struct rpc_clnt
*clnt
,
942 struct nfs_write_data
*data
,
943 const struct rpc_call_ops
*call_ops
,
946 struct inode
*inode
= data
->header
->inode
;
947 int priority
= flush_task_priority(how
);
948 struct rpc_task
*task
;
949 struct rpc_message msg
= {
950 .rpc_argp
= &data
->args
,
951 .rpc_resp
= &data
->res
,
952 .rpc_cred
= data
->header
->cred
,
954 struct rpc_task_setup task_setup_data
= {
958 .callback_ops
= call_ops
,
959 .callback_data
= data
,
960 .workqueue
= nfsiod_workqueue
,
961 .flags
= RPC_TASK_ASYNC
| flags
,
962 .priority
= priority
,
966 /* Set up the initial task struct. */
967 NFS_PROTO(inode
)->write_setup(data
, &msg
);
969 dprintk("NFS: %5u initiated write call "
970 "(req %s/%lld, %u bytes @ offset %llu)\n",
973 (long long)NFS_FILEID(inode
),
975 (unsigned long long)data
->args
.offset
);
977 task
= rpc_run_task(&task_setup_data
);
982 if (how
& FLUSH_SYNC
) {
983 ret
= rpc_wait_for_completion_task(task
);
985 ret
= task
->tk_status
;
991 EXPORT_SYMBOL_GPL(nfs_initiate_write
);
994 * Set up the argument/result storage required for the RPC call.
996 static void nfs_write_rpcsetup(struct nfs_write_data
*data
,
997 unsigned int count
, unsigned int offset
,
998 int how
, struct nfs_commit_info
*cinfo
)
1000 struct nfs_page
*req
= data
->header
->req
;
1002 /* Set up the RPC argument and reply structs
1003 * NB: take care not to mess about with data->commit et al. */
1005 data
->args
.fh
= NFS_FH(data
->header
->inode
);
1006 data
->args
.offset
= req_offset(req
) + offset
;
1007 /* pnfs_set_layoutcommit needs this */
1008 data
->mds_offset
= data
->args
.offset
;
1009 data
->args
.pgbase
= req
->wb_pgbase
+ offset
;
1010 data
->args
.pages
= data
->pages
.pagevec
;
1011 data
->args
.count
= count
;
1012 data
->args
.context
= get_nfs_open_context(req
->wb_context
);
1013 data
->args
.lock_context
= req
->wb_lock_context
;
1014 data
->args
.stable
= NFS_UNSTABLE
;
1015 switch (how
& (FLUSH_STABLE
| FLUSH_COND_STABLE
)) {
1018 case FLUSH_COND_STABLE
:
1019 if (nfs_reqs_to_commit(cinfo
))
1022 data
->args
.stable
= NFS_FILE_SYNC
;
1025 data
->res
.fattr
= &data
->fattr
;
1026 data
->res
.count
= count
;
1027 data
->res
.verf
= &data
->verf
;
1028 nfs_fattr_init(&data
->fattr
);
1031 static int nfs_do_write(struct nfs_write_data
*data
,
1032 const struct rpc_call_ops
*call_ops
,
1035 struct inode
*inode
= data
->header
->inode
;
1037 return nfs_initiate_write(NFS_CLIENT(inode
), data
, call_ops
, how
, 0);
1040 static int nfs_do_multiple_writes(struct list_head
*head
,
1041 const struct rpc_call_ops
*call_ops
,
1044 struct nfs_write_data
*data
;
1047 while (!list_empty(head
)) {
1050 data
= list_first_entry(head
, struct nfs_write_data
, list
);
1051 list_del_init(&data
->list
);
1053 ret2
= nfs_do_write(data
, call_ops
, how
);
1060 /* If a nfs_flush_* function fails, it should remove reqs from @head and
1061 * call this on each, which will prepare them to be retried on next
1062 * writeback using standard nfs.
1064 static void nfs_redirty_request(struct nfs_page
*req
)
1066 nfs_mark_request_dirty(req
);
1067 nfs_unlock_request(req
);
1068 nfs_end_page_writeback(req
->wb_page
);
1069 nfs_release_request(req
);
1072 static void nfs_async_write_error(struct list_head
*head
)
1074 struct nfs_page
*req
;
1076 while (!list_empty(head
)) {
1077 req
= nfs_list_entry(head
->next
);
1078 nfs_list_remove_request(req
);
1079 nfs_redirty_request(req
);
1083 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops
= {
1084 .error_cleanup
= nfs_async_write_error
,
1085 .completion
= nfs_write_completion
,
1088 static void nfs_flush_error(struct nfs_pageio_descriptor
*desc
,
1089 struct nfs_pgio_header
*hdr
)
1091 set_bit(NFS_IOHDR_REDO
, &hdr
->flags
);
1092 while (!list_empty(&hdr
->rpc_list
)) {
1093 struct nfs_write_data
*data
= list_first_entry(&hdr
->rpc_list
,
1094 struct nfs_write_data
, list
);
1095 list_del(&data
->list
);
1096 nfs_writedata_release(data
);
1098 desc
->pg_completion_ops
->error_cleanup(&desc
->pg_list
);
1102 * Generate multiple small requests to write out a single
1103 * contiguous dirty area on one page.
1105 static int nfs_flush_multi(struct nfs_pageio_descriptor
*desc
,
1106 struct nfs_pgio_header
*hdr
)
1108 struct nfs_page
*req
= hdr
->req
;
1109 struct page
*page
= req
->wb_page
;
1110 struct nfs_write_data
*data
;
1111 size_t wsize
= desc
->pg_bsize
, nbytes
;
1112 unsigned int offset
;
1114 struct nfs_commit_info cinfo
;
1116 nfs_init_cinfo(&cinfo
, desc
->pg_inode
, desc
->pg_dreq
);
1118 if ((desc
->pg_ioflags
& FLUSH_COND_STABLE
) &&
1119 (desc
->pg_moreio
|| nfs_reqs_to_commit(&cinfo
) ||
1120 desc
->pg_count
> wsize
))
1121 desc
->pg_ioflags
&= ~FLUSH_COND_STABLE
;
1125 nbytes
= desc
->pg_count
;
1127 size_t len
= min(nbytes
, wsize
);
1129 data
= nfs_writedata_alloc(hdr
, 1);
1131 nfs_flush_error(desc
, hdr
);
1134 data
->pages
.pagevec
[0] = page
;
1135 nfs_write_rpcsetup(data
, len
, offset
, desc
->pg_ioflags
, &cinfo
);
1136 list_add(&data
->list
, &hdr
->rpc_list
);
1140 } while (nbytes
!= 0);
1141 nfs_list_remove_request(req
);
1142 nfs_list_add_request(req
, &hdr
->pages
);
1143 desc
->pg_rpc_callops
= &nfs_write_common_ops
;
1148 * Create an RPC task for the given write request and kick it.
1149 * The page must have been locked by the caller.
1151 * It may happen that the page we're passed is not marked dirty.
1152 * This is the case if nfs_updatepage detects a conflicting request
1153 * that has been written but not committed.
1155 static int nfs_flush_one(struct nfs_pageio_descriptor
*desc
,
1156 struct nfs_pgio_header
*hdr
)
1158 struct nfs_page
*req
;
1159 struct page
**pages
;
1160 struct nfs_write_data
*data
;
1161 struct list_head
*head
= &desc
->pg_list
;
1162 struct nfs_commit_info cinfo
;
1164 data
= nfs_writedata_alloc(hdr
, nfs_page_array_len(desc
->pg_base
,
1167 nfs_flush_error(desc
, hdr
);
1171 nfs_init_cinfo(&cinfo
, desc
->pg_inode
, desc
->pg_dreq
);
1172 pages
= data
->pages
.pagevec
;
1173 while (!list_empty(head
)) {
1174 req
= nfs_list_entry(head
->next
);
1175 nfs_list_remove_request(req
);
1176 nfs_list_add_request(req
, &hdr
->pages
);
1177 *pages
++ = req
->wb_page
;
1180 if ((desc
->pg_ioflags
& FLUSH_COND_STABLE
) &&
1181 (desc
->pg_moreio
|| nfs_reqs_to_commit(&cinfo
)))
1182 desc
->pg_ioflags
&= ~FLUSH_COND_STABLE
;
1184 /* Set up the argument struct */
1185 nfs_write_rpcsetup(data
, desc
->pg_count
, 0, desc
->pg_ioflags
, &cinfo
);
1186 list_add(&data
->list
, &hdr
->rpc_list
);
1187 desc
->pg_rpc_callops
= &nfs_write_common_ops
;
1191 int nfs_generic_flush(struct nfs_pageio_descriptor
*desc
,
1192 struct nfs_pgio_header
*hdr
)
1194 if (desc
->pg_bsize
< PAGE_CACHE_SIZE
)
1195 return nfs_flush_multi(desc
, hdr
);
1196 return nfs_flush_one(desc
, hdr
);
1198 EXPORT_SYMBOL_GPL(nfs_generic_flush
);
1200 static int nfs_generic_pg_writepages(struct nfs_pageio_descriptor
*desc
)
1202 struct nfs_write_header
*whdr
;
1203 struct nfs_pgio_header
*hdr
;
1206 whdr
= nfs_writehdr_alloc();
1208 desc
->pg_completion_ops
->error_cleanup(&desc
->pg_list
);
1211 hdr
= &whdr
->header
;
1212 nfs_pgheader_init(desc
, hdr
, nfs_writehdr_free
);
1213 atomic_inc(&hdr
->refcnt
);
1214 ret
= nfs_generic_flush(desc
, hdr
);
1216 ret
= nfs_do_multiple_writes(&hdr
->rpc_list
,
1217 desc
->pg_rpc_callops
,
1219 if (atomic_dec_and_test(&hdr
->refcnt
))
1220 hdr
->completion_ops
->completion(hdr
);
1224 static const struct nfs_pageio_ops nfs_pageio_write_ops
= {
1225 .pg_test
= nfs_generic_pg_test
,
1226 .pg_doio
= nfs_generic_pg_writepages
,
1229 void nfs_pageio_init_write(struct nfs_pageio_descriptor
*pgio
,
1230 struct inode
*inode
, int ioflags
,
1231 const struct nfs_pgio_completion_ops
*compl_ops
)
1233 nfs_pageio_init(pgio
, inode
, &nfs_pageio_write_ops
, compl_ops
,
1234 NFS_SERVER(inode
)->wsize
, ioflags
);
1236 EXPORT_SYMBOL_GPL(nfs_pageio_init_write
);
1238 void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor
*pgio
)
1240 pgio
->pg_ops
= &nfs_pageio_write_ops
;
1241 pgio
->pg_bsize
= NFS_SERVER(pgio
->pg_inode
)->wsize
;
1243 EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds
);
1246 void nfs_write_prepare(struct rpc_task
*task
, void *calldata
)
1248 struct nfs_write_data
*data
= calldata
;
1249 NFS_PROTO(data
->header
->inode
)->write_rpc_prepare(task
, data
);
1252 void nfs_commit_prepare(struct rpc_task
*task
, void *calldata
)
1254 struct nfs_commit_data
*data
= calldata
;
1256 NFS_PROTO(data
->inode
)->commit_rpc_prepare(task
, data
);
1260 * Handle a write reply that flushes a whole page.
1262 * FIXME: There is an inherent race with invalidate_inode_pages and
1263 * writebacks since the page->count is kept > 1 for as long
1264 * as the page has a write request pending.
1266 static void nfs_writeback_done_common(struct rpc_task
*task
, void *calldata
)
1268 struct nfs_write_data
*data
= calldata
;
1270 nfs_writeback_done(task
, data
);
1273 static void nfs_writeback_release_common(void *calldata
)
1275 struct nfs_write_data
*data
= calldata
;
1276 struct nfs_pgio_header
*hdr
= data
->header
;
1277 int status
= data
->task
.tk_status
;
1279 if ((status
>= 0) && nfs_write_need_commit(data
)) {
1280 spin_lock(&hdr
->lock
);
1281 if (test_bit(NFS_IOHDR_NEED_RESCHED
, &hdr
->flags
))
1283 else if (!test_and_set_bit(NFS_IOHDR_NEED_COMMIT
, &hdr
->flags
))
1284 memcpy(hdr
->verf
, &data
->verf
, sizeof(*hdr
->verf
));
1285 else if (memcmp(hdr
->verf
, &data
->verf
, sizeof(*hdr
->verf
)))
1286 set_bit(NFS_IOHDR_NEED_RESCHED
, &hdr
->flags
);
1287 spin_unlock(&hdr
->lock
);
1289 nfs_writedata_release(data
);
1292 static const struct rpc_call_ops nfs_write_common_ops
= {
1293 .rpc_call_prepare
= nfs_write_prepare
,
1294 .rpc_call_done
= nfs_writeback_done_common
,
1295 .rpc_release
= nfs_writeback_release_common
,
1300 * This function is called when the WRITE call is complete.
1302 void nfs_writeback_done(struct rpc_task
*task
, struct nfs_write_data
*data
)
1304 struct nfs_writeargs
*argp
= &data
->args
;
1305 struct nfs_writeres
*resp
= &data
->res
;
1306 struct inode
*inode
= data
->header
->inode
;
1309 dprintk("NFS: %5u nfs_writeback_done (status %d)\n",
1310 task
->tk_pid
, task
->tk_status
);
1313 * ->write_done will attempt to use post-op attributes to detect
1314 * conflicting writes by other clients. A strict interpretation
1315 * of close-to-open would allow us to continue caching even if
1316 * another writer had changed the file, but some applications
1317 * depend on tighter cache coherency when writing.
1319 status
= NFS_PROTO(inode
)->write_done(task
, data
);
1322 nfs_add_stats(inode
, NFSIOS_SERVERWRITTENBYTES
, resp
->count
);
1324 #if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
1325 if (resp
->verf
->committed
< argp
->stable
&& task
->tk_status
>= 0) {
1326 /* We tried a write call, but the server did not
1327 * commit data to stable storage even though we
1329 * Note: There is a known bug in Tru64 < 5.0 in which
1330 * the server reports NFS_DATA_SYNC, but performs
1331 * NFS_FILE_SYNC. We therefore implement this checking
1332 * as a dprintk() in order to avoid filling syslog.
1334 static unsigned long complain
;
1336 /* Note this will print the MDS for a DS write */
1337 if (time_before(complain
, jiffies
)) {
1338 dprintk("NFS: faulty NFS server %s:"
1339 " (committed = %d) != (stable = %d)\n",
1340 NFS_SERVER(inode
)->nfs_client
->cl_hostname
,
1341 resp
->verf
->committed
, argp
->stable
);
1342 complain
= jiffies
+ 300 * HZ
;
1346 if (task
->tk_status
< 0)
1347 nfs_set_pgio_error(data
->header
, task
->tk_status
, argp
->offset
);
1348 else if (resp
->count
< argp
->count
) {
1349 static unsigned long complain
;
1351 /* This a short write! */
1352 nfs_inc_stats(inode
, NFSIOS_SHORTWRITE
);
1354 /* Has the server at least made some progress? */
1355 if (resp
->count
== 0) {
1356 if (time_before(complain
, jiffies
)) {
1358 "NFS: Server wrote zero bytes, expected %u.\n",
1360 complain
= jiffies
+ 300 * HZ
;
1362 nfs_set_pgio_error(data
->header
, -EIO
, argp
->offset
);
1363 task
->tk_status
= -EIO
;
1366 /* Was this an NFSv2 write or an NFSv3 stable write? */
1367 if (resp
->verf
->committed
!= NFS_UNSTABLE
) {
1368 /* Resend from where the server left off */
1369 data
->mds_offset
+= resp
->count
;
1370 argp
->offset
+= resp
->count
;
1371 argp
->pgbase
+= resp
->count
;
1372 argp
->count
-= resp
->count
;
1374 /* Resend as a stable write in order to avoid
1375 * headaches in the case of a server crash.
1377 argp
->stable
= NFS_FILE_SYNC
;
1379 rpc_restart_call_prepare(task
);
1384 #if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
1385 static int nfs_commit_set_lock(struct nfs_inode
*nfsi
, int may_wait
)
1389 if (!test_and_set_bit(NFS_INO_COMMIT
, &nfsi
->flags
))
1393 ret
= out_of_line_wait_on_bit_lock(&nfsi
->flags
,
1395 nfs_wait_bit_killable
,
1397 return (ret
< 0) ? ret
: 1;
1400 static void nfs_commit_clear_lock(struct nfs_inode
*nfsi
)
1402 clear_bit(NFS_INO_COMMIT
, &nfsi
->flags
);
1403 smp_mb__after_clear_bit();
1404 wake_up_bit(&nfsi
->flags
, NFS_INO_COMMIT
);
1407 void nfs_commitdata_release(struct nfs_commit_data
*data
)
1409 put_nfs_open_context(data
->context
);
1410 nfs_commit_free(data
);
1412 EXPORT_SYMBOL_GPL(nfs_commitdata_release
);
1414 int nfs_initiate_commit(struct rpc_clnt
*clnt
, struct nfs_commit_data
*data
,
1415 const struct rpc_call_ops
*call_ops
,
1418 struct rpc_task
*task
;
1419 int priority
= flush_task_priority(how
);
1420 struct rpc_message msg
= {
1421 .rpc_argp
= &data
->args
,
1422 .rpc_resp
= &data
->res
,
1423 .rpc_cred
= data
->cred
,
1425 struct rpc_task_setup task_setup_data
= {
1426 .task
= &data
->task
,
1428 .rpc_message
= &msg
,
1429 .callback_ops
= call_ops
,
1430 .callback_data
= data
,
1431 .workqueue
= nfsiod_workqueue
,
1432 .flags
= RPC_TASK_ASYNC
| flags
,
1433 .priority
= priority
,
1435 /* Set up the initial task struct. */
1436 NFS_PROTO(data
->inode
)->commit_setup(data
, &msg
);
1438 dprintk("NFS: %5u initiated commit call\n", data
->task
.tk_pid
);
1440 task
= rpc_run_task(&task_setup_data
);
1442 return PTR_ERR(task
);
1443 if (how
& FLUSH_SYNC
)
1444 rpc_wait_for_completion_task(task
);
1448 EXPORT_SYMBOL_GPL(nfs_initiate_commit
);
1451 * Set up the argument/result storage required for the RPC call.
1453 void nfs_init_commit(struct nfs_commit_data
*data
,
1454 struct list_head
*head
,
1455 struct pnfs_layout_segment
*lseg
,
1456 struct nfs_commit_info
*cinfo
)
1458 struct nfs_page
*first
= nfs_list_entry(head
->next
);
1459 struct inode
*inode
= first
->wb_context
->dentry
->d_inode
;
1461 /* Set up the RPC argument and reply structs
1462 * NB: take care not to mess about with data->commit et al. */
1464 list_splice_init(head
, &data
->pages
);
1466 data
->inode
= inode
;
1467 data
->cred
= first
->wb_context
->cred
;
1468 data
->lseg
= lseg
; /* reference transferred */
1469 data
->mds_ops
= &nfs_commit_ops
;
1470 data
->completion_ops
= cinfo
->completion_ops
;
1471 data
->dreq
= cinfo
->dreq
;
1473 data
->args
.fh
= NFS_FH(data
->inode
);
1474 /* Note: we always request a commit of the entire inode */
1475 data
->args
.offset
= 0;
1476 data
->args
.count
= 0;
1477 data
->context
= get_nfs_open_context(first
->wb_context
);
1478 data
->res
.fattr
= &data
->fattr
;
1479 data
->res
.verf
= &data
->verf
;
1480 nfs_fattr_init(&data
->fattr
);
1482 EXPORT_SYMBOL_GPL(nfs_init_commit
);
1484 void nfs_retry_commit(struct list_head
*page_list
,
1485 struct pnfs_layout_segment
*lseg
,
1486 struct nfs_commit_info
*cinfo
)
1488 struct nfs_page
*req
;
1490 while (!list_empty(page_list
)) {
1491 req
= nfs_list_entry(page_list
->next
);
1492 nfs_list_remove_request(req
);
1493 nfs_mark_request_commit(req
, lseg
, cinfo
);
1495 dec_zone_page_state(req
->wb_page
, NR_UNSTABLE_NFS
);
1496 dec_bdi_stat(page_file_mapping(req
->wb_page
)->backing_dev_info
,
1499 nfs_unlock_and_release_request(req
);
1502 EXPORT_SYMBOL_GPL(nfs_retry_commit
);
1505 * Commit dirty pages
1508 nfs_commit_list(struct inode
*inode
, struct list_head
*head
, int how
,
1509 struct nfs_commit_info
*cinfo
)
1511 struct nfs_commit_data
*data
;
1513 data
= nfs_commitdata_alloc();
1518 /* Set up the argument struct */
1519 nfs_init_commit(data
, head
, NULL
, cinfo
);
1520 atomic_inc(&cinfo
->mds
->rpcs_out
);
1521 return nfs_initiate_commit(NFS_CLIENT(inode
), data
, data
->mds_ops
,
1524 nfs_retry_commit(head
, NULL
, cinfo
);
1525 cinfo
->completion_ops
->error_cleanup(NFS_I(inode
));
1530 * COMMIT call returned
1532 static void nfs_commit_done(struct rpc_task
*task
, void *calldata
)
1534 struct nfs_commit_data
*data
= calldata
;
1536 dprintk("NFS: %5u nfs_commit_done (status %d)\n",
1537 task
->tk_pid
, task
->tk_status
);
1539 /* Call the NFS version-specific code */
1540 NFS_PROTO(data
->inode
)->commit_done(task
, data
);
1543 static void nfs_commit_release_pages(struct nfs_commit_data
*data
)
1545 struct nfs_page
*req
;
1546 int status
= data
->task
.tk_status
;
1547 struct nfs_commit_info cinfo
;
1549 while (!list_empty(&data
->pages
)) {
1550 req
= nfs_list_entry(data
->pages
.next
);
1551 nfs_list_remove_request(req
);
1552 nfs_clear_page_commit(req
->wb_page
);
1554 dprintk("NFS: commit (%s/%lld %d@%lld)",
1555 req
->wb_context
->dentry
->d_sb
->s_id
,
1556 (long long)NFS_FILEID(req
->wb_context
->dentry
->d_inode
),
1558 (long long)req_offset(req
));
1560 nfs_context_set_write_error(req
->wb_context
, status
);
1561 nfs_inode_remove_request(req
);
1562 dprintk(", error = %d\n", status
);
1566 /* Okay, COMMIT succeeded, apparently. Check the verifier
1567 * returned by the server against all stored verfs. */
1568 if (!memcmp(&req
->wb_verf
, &data
->verf
.verifier
, sizeof(req
->wb_verf
))) {
1569 /* We have a match */
1570 nfs_inode_remove_request(req
);
1574 /* We have a mismatch. Write the page again */
1575 dprintk(" mismatch\n");
1576 nfs_mark_request_dirty(req
);
1577 set_bit(NFS_CONTEXT_RESEND_WRITES
, &req
->wb_context
->flags
);
1579 nfs_unlock_and_release_request(req
);
1581 nfs_init_cinfo(&cinfo
, data
->inode
, data
->dreq
);
1582 if (atomic_dec_and_test(&cinfo
.mds
->rpcs_out
))
1583 nfs_commit_clear_lock(NFS_I(data
->inode
));
1586 static void nfs_commit_release(void *calldata
)
1588 struct nfs_commit_data
*data
= calldata
;
1590 data
->completion_ops
->completion(data
);
1591 nfs_commitdata_release(calldata
);
1594 static const struct rpc_call_ops nfs_commit_ops
= {
1595 .rpc_call_prepare
= nfs_commit_prepare
,
1596 .rpc_call_done
= nfs_commit_done
,
1597 .rpc_release
= nfs_commit_release
,
1600 static const struct nfs_commit_completion_ops nfs_commit_completion_ops
= {
1601 .completion
= nfs_commit_release_pages
,
1602 .error_cleanup
= nfs_commit_clear_lock
,
1605 int nfs_generic_commit_list(struct inode
*inode
, struct list_head
*head
,
1606 int how
, struct nfs_commit_info
*cinfo
)
1610 status
= pnfs_commit_list(inode
, head
, how
, cinfo
);
1611 if (status
== PNFS_NOT_ATTEMPTED
)
1612 status
= nfs_commit_list(inode
, head
, how
, cinfo
);
1616 int nfs_commit_inode(struct inode
*inode
, int how
)
1619 struct nfs_commit_info cinfo
;
1620 int may_wait
= how
& FLUSH_SYNC
;
1623 res
= nfs_commit_set_lock(NFS_I(inode
), may_wait
);
1625 goto out_mark_dirty
;
1626 nfs_init_cinfo_from_inode(&cinfo
, inode
);
1627 res
= nfs_scan_commit(inode
, &head
, &cinfo
);
1631 error
= nfs_generic_commit_list(inode
, &head
, how
, &cinfo
);
1635 goto out_mark_dirty
;
1636 error
= wait_on_bit(&NFS_I(inode
)->flags
,
1638 nfs_wait_bit_killable
,
1643 nfs_commit_clear_lock(NFS_I(inode
));
1645 /* Note: If we exit without ensuring that the commit is complete,
1646 * we must mark the inode as dirty. Otherwise, future calls to
1647 * sync_inode() with the WB_SYNC_ALL flag set will fail to ensure
1648 * that the data is on the disk.
1651 __mark_inode_dirty(inode
, I_DIRTY_DATASYNC
);
1655 static int nfs_commit_unstable_pages(struct inode
*inode
, struct writeback_control
*wbc
)
1657 struct nfs_inode
*nfsi
= NFS_I(inode
);
1658 int flags
= FLUSH_SYNC
;
1661 /* no commits means nothing needs to be done */
1662 if (!nfsi
->commit_info
.ncommit
)
1665 if (wbc
->sync_mode
== WB_SYNC_NONE
) {
1666 /* Don't commit yet if this is a non-blocking flush and there
1667 * are a lot of outstanding writes for this mapping.
1669 if (nfsi
->commit_info
.ncommit
<= (nfsi
->npages
>> 1))
1670 goto out_mark_dirty
;
1672 /* don't wait for the COMMIT response */
1676 ret
= nfs_commit_inode(inode
, flags
);
1678 if (wbc
->sync_mode
== WB_SYNC_NONE
) {
1679 if (ret
< wbc
->nr_to_write
)
1680 wbc
->nr_to_write
-= ret
;
1682 wbc
->nr_to_write
= 0;
1687 __mark_inode_dirty(inode
, I_DIRTY_DATASYNC
);
1691 static int nfs_commit_unstable_pages(struct inode
*inode
, struct writeback_control
*wbc
)
1697 int nfs_write_inode(struct inode
*inode
, struct writeback_control
*wbc
)
1699 return nfs_commit_unstable_pages(inode
, wbc
);
1701 EXPORT_SYMBOL_GPL(nfs_write_inode
);
1704 * flush the inode to disk.
1706 int nfs_wb_all(struct inode
*inode
)
1708 struct writeback_control wbc
= {
1709 .sync_mode
= WB_SYNC_ALL
,
1710 .nr_to_write
= LONG_MAX
,
1712 .range_end
= LLONG_MAX
,
1715 return sync_inode(inode
, &wbc
);
1717 EXPORT_SYMBOL_GPL(nfs_wb_all
);
1719 int nfs_wb_page_cancel(struct inode
*inode
, struct page
*page
)
1721 struct nfs_page
*req
;
1725 wait_on_page_writeback(page
);
1726 req
= nfs_page_find_request(page
);
1729 if (nfs_lock_request(req
)) {
1730 nfs_clear_request_commit(req
);
1731 nfs_inode_remove_request(req
);
1733 * In case nfs_inode_remove_request has marked the
1734 * page as being dirty
1736 cancel_dirty_page(page
, PAGE_CACHE_SIZE
);
1737 nfs_unlock_and_release_request(req
);
1740 ret
= nfs_wait_on_request(req
);
1741 nfs_release_request(req
);
1749 * Write back all requests on one page - we do this before reading it.
1751 int nfs_wb_page(struct inode
*inode
, struct page
*page
)
1753 loff_t range_start
= page_file_offset(page
);
1754 loff_t range_end
= range_start
+ (loff_t
)(PAGE_CACHE_SIZE
- 1);
1755 struct writeback_control wbc
= {
1756 .sync_mode
= WB_SYNC_ALL
,
1758 .range_start
= range_start
,
1759 .range_end
= range_end
,
1764 wait_on_page_writeback(page
);
1765 if (clear_page_dirty_for_io(page
)) {
1766 ret
= nfs_writepage_locked(page
, &wbc
);
1771 if (!PagePrivate(page
))
1773 ret
= nfs_commit_inode(inode
, FLUSH_SYNC
);
1782 #ifdef CONFIG_MIGRATION
1783 int nfs_migrate_page(struct address_space
*mapping
, struct page
*newpage
,
1784 struct page
*page
, enum migrate_mode mode
)
1787 * If PagePrivate is set, then the page is currently associated with
1788 * an in-progress read or write request. Don't try to migrate it.
1790 * FIXME: we could do this in principle, but we'll need a way to ensure
1791 * that we can safely release the inode reference while holding
1794 if (PagePrivate(page
))
1797 if (!nfs_fscache_release_page(page
, GFP_KERNEL
))
1800 return migrate_page(mapping
, newpage
, page
, mode
);
1804 int __init
nfs_init_writepagecache(void)
1806 nfs_wdata_cachep
= kmem_cache_create("nfs_write_data",
1807 sizeof(struct nfs_write_header
),
1808 0, SLAB_HWCACHE_ALIGN
,
1810 if (nfs_wdata_cachep
== NULL
)
1813 nfs_wdata_mempool
= mempool_create_slab_pool(MIN_POOL_WRITE
,
1815 if (nfs_wdata_mempool
== NULL
)
1816 goto out_destroy_write_cache
;
1818 nfs_cdata_cachep
= kmem_cache_create("nfs_commit_data",
1819 sizeof(struct nfs_commit_data
),
1820 0, SLAB_HWCACHE_ALIGN
,
1822 if (nfs_cdata_cachep
== NULL
)
1823 goto out_destroy_write_mempool
;
1825 nfs_commit_mempool
= mempool_create_slab_pool(MIN_POOL_COMMIT
,
1827 if (nfs_commit_mempool
== NULL
)
1828 goto out_destroy_commit_cache
;
1831 * NFS congestion size, scale with available memory.
1843 * This allows larger machines to have larger/more transfers.
1844 * Limit the default to 256M
1846 nfs_congestion_kb
= (16*int_sqrt(totalram_pages
)) << (PAGE_SHIFT
-10);
1847 if (nfs_congestion_kb
> 256*1024)
1848 nfs_congestion_kb
= 256*1024;
1852 out_destroy_commit_cache
:
1853 kmem_cache_destroy(nfs_cdata_cachep
);
1854 out_destroy_write_mempool
:
1855 mempool_destroy(nfs_wdata_mempool
);
1856 out_destroy_write_cache
:
1857 kmem_cache_destroy(nfs_wdata_cachep
);
1861 void nfs_destroy_writepagecache(void)
1863 mempool_destroy(nfs_commit_mempool
);
1864 kmem_cache_destroy(nfs_cdata_cachep
);
1865 mempool_destroy(nfs_wdata_mempool
);
1866 kmem_cache_destroy(nfs_wdata_cachep
);