1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * linux/fs/nfs/write.c
4 *
5 * Write file data over NFS.
6 *
7 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
8 */
9
10 #include <linux/types.h>
11 #include <linux/slab.h>
12 #include <linux/mm.h>
13 #include <linux/pagemap.h>
14 #include <linux/file.h>
15 #include <linux/writeback.h>
16 #include <linux/swap.h>
17 #include <linux/migrate.h>
18
19 #include <linux/sunrpc/clnt.h>
20 #include <linux/nfs_fs.h>
21 #include <linux/nfs_mount.h>
22 #include <linux/nfs_page.h>
23 #include <linux/backing-dev.h>
24 #include <linux/export.h>
25 #include <linux/freezer.h>
26 #include <linux/wait.h>
27 #include <linux/iversion.h>
28 #include <linux/filelock.h>
29
30 #include <linux/uaccess.h>
31 #include <linux/sched/mm.h>
32
33 #include "delegation.h"
34 #include "internal.h"
35 #include "iostat.h"
36 #include "nfs4_fs.h"
37 #include "fscache.h"
38 #include "pnfs.h"
39
40 #include "nfstrace.h"
41
42 #define NFSDBG_FACILITY NFSDBG_PAGECACHE
43
44 #define MIN_POOL_WRITE (32)
45 #define MIN_POOL_COMMIT (4)
46
47 struct nfs_io_completion {
48 void (*complete)(void *data);
49 void *data;
50 struct kref refcount;
51 };
52
53 /*
54 * Local function declarations
55 */
56 static void nfs_redirty_request(struct nfs_page *req);
57 static const struct rpc_call_ops nfs_commit_ops;
58 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops;
59 static const struct nfs_commit_completion_ops nfs_commit_completion_ops;
60 static const struct nfs_rw_ops nfs_rw_write_ops;
61 static void nfs_inode_remove_request(struct nfs_page *req);
62 static void nfs_clear_request_commit(struct nfs_commit_info *cinfo,
63 struct nfs_page *req);
64 static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
65 struct inode *inode);
66
67 static struct kmem_cache *nfs_wdata_cachep;
68 static mempool_t *nfs_wdata_mempool;
69 static struct kmem_cache *nfs_cdata_cachep;
70 static mempool_t *nfs_commit_mempool;
71
72 struct nfs_commit_data *nfs_commitdata_alloc(void)
73 {
74 struct nfs_commit_data *p;
75
76 p = kmem_cache_zalloc(nfs_cdata_cachep, nfs_io_gfp_mask());
77 if (!p) {
78 p = mempool_alloc(nfs_commit_mempool, GFP_NOWAIT);
79 if (!p)
80 return NULL;
81 memset(p, 0, sizeof(*p));
82 }
83 INIT_LIST_HEAD(&p->pages);
84 return p;
85 }
86 EXPORT_SYMBOL_GPL(nfs_commitdata_alloc);
87
88 void nfs_commit_free(struct nfs_commit_data *p)
89 {
90 mempool_free(p, nfs_commit_mempool);
91 }
92 EXPORT_SYMBOL_GPL(nfs_commit_free);
93
94 static struct nfs_pgio_header *nfs_writehdr_alloc(void)
95 {
96 struct nfs_pgio_header *p;
97
98 p = kmem_cache_zalloc(nfs_wdata_cachep, nfs_io_gfp_mask());
99 if (!p) {
100 p = mempool_alloc(nfs_wdata_mempool, GFP_NOWAIT);
101 if (!p)
102 return NULL;
103 memset(p, 0, sizeof(*p));
104 }
105 p->rw_mode = FMODE_WRITE;
106 return p;
107 }
108
109 static void nfs_writehdr_free(struct nfs_pgio_header *hdr)
110 {
111 mempool_free(hdr, nfs_wdata_mempool);
112 }
113
114 static struct nfs_io_completion *nfs_io_completion_alloc(gfp_t gfp_flags)
115 {
116 return kmalloc(sizeof(struct nfs_io_completion), gfp_flags);
117 }
118
119 static void nfs_io_completion_init(struct nfs_io_completion *ioc,
120 void (*complete)(void *), void *data)
121 {
122 ioc->complete = complete;
123 ioc->data = data;
124 kref_init(&ioc->refcount);
125 }
126
127 static void nfs_io_completion_release(struct kref *kref)
128 {
129 struct nfs_io_completion *ioc = container_of(kref,
130 struct nfs_io_completion, refcount);
131 ioc->complete(ioc->data);
132 kfree(ioc);
133 }
134
135 static void nfs_io_completion_get(struct nfs_io_completion *ioc)
136 {
137 if (ioc != NULL)
138 kref_get(&ioc->refcount);
139 }
140
141 static void nfs_io_completion_put(struct nfs_io_completion *ioc)
142 {
143 if (ioc != NULL)
144 kref_put(&ioc->refcount, nfs_io_completion_release);
145 }
146
147 /**
148 * nfs_folio_find_head_request - find head request associated with a folio
149 * @folio: pointer to folio
150 *
151 * must be called while holding the inode lock.
152 *
153 * returns matching head request with reference held, or NULL if not found.
154 */
155 static struct nfs_page *nfs_folio_find_head_request(struct folio *folio)
156 {
157 struct address_space *mapping = folio->mapping;
158 struct nfs_page *req;
159
160 if (!folio_test_private(folio))
161 return NULL;
162 spin_lock(&mapping->i_private_lock);
163 req = folio->private;
164 if (req) {
165 WARN_ON_ONCE(req->wb_head != req);
166 kref_get(&req->wb_kref);
167 }
168 spin_unlock(&mapping->i_private_lock);
169 return req;
170 }
171
172 /* Adjust the file length if we're writing beyond the end */
173 static void nfs_grow_file(struct folio *folio, unsigned int offset,
174 unsigned int count)
175 {
176 struct inode *inode = folio->mapping->host;
177 loff_t end, i_size;
178 pgoff_t end_index;
179
180 spin_lock(&inode->i_lock);
181 i_size = i_size_read(inode);
182 end_index = ((i_size - 1) >> folio_shift(folio)) << folio_order(folio);
183 if (i_size > 0 && folio->index < end_index)
184 goto out;
185 end = folio_pos(folio) + (loff_t)offset + (loff_t)count;
186 if (i_size >= end)
187 goto out;
188 trace_nfs_size_grow(inode, end);
189 i_size_write(inode, end);
190 NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE;
191 nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);
192 out:
193 /* Atomically update timestamps if they are delegated to us. */
194 nfs_update_delegated_mtime_locked(inode);
195 spin_unlock(&inode->i_lock);
196 nfs_fscache_invalidate(inode, 0);
197 }
198
199 /* A writeback failed: mark the page as bad, and invalidate the page cache */
200 static void nfs_set_pageerror(struct address_space *mapping)
201 {
202 struct inode *inode = mapping->host;
203
204 nfs_zap_mapping(mapping->host, mapping);
205 /* Force file size revalidation */
206 spin_lock(&inode->i_lock);
207 nfs_set_cache_invalid(inode, NFS_INO_REVAL_FORCED |
208 NFS_INO_INVALID_CHANGE |
209 NFS_INO_INVALID_SIZE);
210 spin_unlock(&inode->i_lock);
211 }
212
213 static void nfs_mapping_set_error(struct folio *folio, int error)
214 {
215 struct address_space *mapping = folio->mapping;
216
217 filemap_set_wb_err(mapping, error);
218 if (mapping->host)
219 errseq_set(&mapping->host->i_sb->s_wb_err,
220 error == -ENOSPC ? -ENOSPC : -EIO);
221 nfs_set_pageerror(mapping);
222 }
223
224 /*
225 * nfs_page_group_search_locked
226 * @head - head request of page group
227 * @page_offset - offset into page
228 *
229 * Search page group with head @head to find a request that contains the
230 * page offset @page_offset.
231 *
232 * Returns a pointer to the first matching nfs request, or NULL if no
233 * match is found.
234 *
235 * Must be called with the page group lock held
236 */
237 static struct nfs_page *
238 nfs_page_group_search_locked(struct nfs_page *head, unsigned int page_offset)
239 {
240 struct nfs_page *req;
241
242 req = head;
243 do {
244 if (page_offset >= req->wb_pgbase &&
245 page_offset < (req->wb_pgbase + req->wb_bytes))
246 return req;
247
248 req = req->wb_this_page;
249 } while (req != head);
250
251 return NULL;
252 }
253
254 /*
255 * nfs_page_group_covers_page
256 * @head - head request of page group
257 *
258 * Return true if the page group with head @head covers the whole page,
259 * returns false otherwise
260 */
261 static bool nfs_page_group_covers_page(struct nfs_page *req)
262 {
263 unsigned int len = nfs_folio_length(nfs_page_to_folio(req));
264 struct nfs_page *tmp;
265 unsigned int pos = 0;
266
267 nfs_page_group_lock(req);
268
269 for (;;) {
270 tmp = nfs_page_group_search_locked(req->wb_head, pos);
271 if (!tmp)
272 break;
273 pos = tmp->wb_pgbase + tmp->wb_bytes;
274 }
275
276 nfs_page_group_unlock(req);
277 return pos >= len;
278 }
279
280 /* We can set the PG_uptodate flag if we see that a write request
281 * covers the full page.
282 */
283 static void nfs_mark_uptodate(struct nfs_page *req)
284 {
285 struct folio *folio = nfs_page_to_folio(req);
286
287 if (folio_test_uptodate(folio))
288 return;
289 if (!nfs_page_group_covers_page(req))
290 return;
291 folio_mark_uptodate(folio);
292 }
293
294 static int wb_priority(struct writeback_control *wbc)
295 {
296 int ret = 0;
297
298 if (wbc->sync_mode == WB_SYNC_ALL)
299 ret = FLUSH_COND_STABLE;
300 return ret;
301 }
302
303 /*
304 * NFS congestion control
305 */
306
307 int nfs_congestion_kb;
308
309 #define NFS_CONGESTION_ON_THRESH (nfs_congestion_kb >> (PAGE_SHIFT-10))
310 #define NFS_CONGESTION_OFF_THRESH \
311 (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))
312
313 static void nfs_folio_set_writeback(struct folio *folio)
314 {
315 struct nfs_server *nfss = NFS_SERVER(folio->mapping->host);
316
317 folio_start_writeback(folio);
318 if (atomic_long_inc_return(&nfss->writeback) > NFS_CONGESTION_ON_THRESH)
319 nfss->write_congested = 1;
320 }
321
322 static void nfs_folio_end_writeback(struct folio *folio)
323 {
324 struct nfs_server *nfss = NFS_SERVER(folio->mapping->host);
325
326 folio_end_writeback(folio);
327 if (atomic_long_dec_return(&nfss->writeback) <
328 NFS_CONGESTION_OFF_THRESH) {
329 nfss->write_congested = 0;
330 wake_up_all(&nfss->write_congestion_wait);
331 }
332 }
333
334 static void nfs_page_end_writeback(struct nfs_page *req)
335 {
336 if (nfs_page_group_sync_on_bit(req, PG_WB_END)) {
337 nfs_unlock_request(req);
338 nfs_folio_end_writeback(nfs_page_to_folio(req));
339 } else
340 nfs_unlock_request(req);
341 }
342
343 /*
344 * nfs_destroy_unlinked_subrequests - destroy recently unlinked subrequests
345 *
346 * @destroy_list - request list (using wb_this_page) terminated by @old_head
347 * @old_head - the old head of the list
348 *
349 * All subrequests must be locked and removed from all lists, so at this point
350 * they are only "active" in this function, and possibly in nfs_wait_on_request
351 * with a reference held by some other context.
352 */
353 static void
354 nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list,
355 struct nfs_page *old_head,
356 struct inode *inode)
357 {
358 while (destroy_list) {
359 struct nfs_page *subreq = destroy_list;
360
361 destroy_list = (subreq->wb_this_page == old_head) ?
362 NULL : subreq->wb_this_page;
363
364 /* Note: lock subreq in order to change subreq->wb_head */
365 nfs_page_set_headlock(subreq);
366 WARN_ON_ONCE(old_head != subreq->wb_head);
367
368 /* make sure old group is not used */
369 subreq->wb_this_page = subreq;
370 subreq->wb_head = subreq;
371
372 clear_bit(PG_REMOVE, &subreq->wb_flags);
373
374 /* Note: races with nfs_page_group_destroy() */
375 if (!kref_read(&subreq->wb_kref)) {
376 /* Check if we raced with nfs_page_group_destroy() */
377 if (test_and_clear_bit(PG_TEARDOWN, &subreq->wb_flags)) {
378 nfs_page_clear_headlock(subreq);
379 nfs_free_request(subreq);
380 } else
381 nfs_page_clear_headlock(subreq);
382 continue;
383 }
384 nfs_page_clear_headlock(subreq);
385
386 nfs_release_request(old_head);
387
388 if (test_and_clear_bit(PG_INODE_REF, &subreq->wb_flags)) {
389 nfs_release_request(subreq);
390 atomic_long_dec(&NFS_I(inode)->nrequests);
391 }
392
393 /* subreq is now totally disconnected from page group or any
394 * write / commit lists. last chance to wake any waiters */
395 nfs_unlock_and_release_request(subreq);
396 }
397 }
398
399 /*
400 * nfs_join_page_group - destroy subrequests of the head req
401 * @head: the page used to lookup the "page group" of nfs_page structures
402 * @inode: Inode to which the request belongs.
403 *
404 * This function joins all sub requests to the head request by first
405 * locking all requests in the group, cancelling any pending operations
406 * and finally updating the head request to cover the whole range covered by
407 * the (former) group. All subrequests are removed from any write or commit
408 * lists, unlinked from the group and destroyed.
409 */
410 void nfs_join_page_group(struct nfs_page *head, struct nfs_commit_info *cinfo,
411 struct inode *inode)
412 {
413 struct nfs_page *subreq;
414 struct nfs_page *destroy_list = NULL;
415 unsigned int pgbase, off, bytes;
416
417 pgbase = head->wb_pgbase;
418 bytes = head->wb_bytes;
419 off = head->wb_offset;
420 for (subreq = head->wb_this_page; subreq != head;
421 subreq = subreq->wb_this_page) {
422 /* Subrequests should always form a contiguous range */
423 if (pgbase > subreq->wb_pgbase) {
424 off -= pgbase - subreq->wb_pgbase;
425 bytes += pgbase - subreq->wb_pgbase;
426 pgbase = subreq->wb_pgbase;
427 }
428 bytes = max(subreq->wb_pgbase + subreq->wb_bytes
429 - pgbase, bytes);
430 }
431
432 /* Set the head request's range to cover the former page group */
433 head->wb_pgbase = pgbase;
434 head->wb_bytes = bytes;
435 head->wb_offset = off;
436
437 /* Now that all requests are locked, make sure they aren't on any list.
438 * Commit list removal accounting is done after locks are dropped */
439 subreq = head;
440 do {
441 nfs_clear_request_commit(cinfo, subreq);
442 subreq = subreq->wb_this_page;
443 } while (subreq != head);
444
445 /* unlink subrequests from head, destroy them later */
446 if (head->wb_this_page != head) {
447 /* destroy list will be terminated by head */
448 destroy_list = head->wb_this_page;
449 head->wb_this_page = head;
450 }
451
452 nfs_destroy_unlinked_subrequests(destroy_list, head, inode);
453 }
454
455 /**
456 * nfs_wait_on_request - Wait for a request to complete.
457 * @req: request to wait upon.
458 *
459 * Interruptible by fatal signals only.
460 * The user is responsible for holding a count on the request.
461 */
462 static int nfs_wait_on_request(struct nfs_page *req)
463 {
464 if (!test_bit(PG_BUSY, &req->wb_flags))
465 return 0;
466 set_bit(PG_CONTENDED2, &req->wb_flags);
467 smp_mb__after_atomic();
468 return wait_on_bit_io(&req->wb_flags, PG_BUSY,
469 TASK_UNINTERRUPTIBLE);
470 }
471
472 /*
473 * nfs_unroll_locks - unlock all newly locked reqs and wait on @req
474 * @head: head request of page group, must be holding head lock
475 * @req: request that couldn't lock and needs to wait on the req bit lock
476 *
477 * This is a helper function for nfs_lock_and_join_requests
478 * returns 0 on success, < 0 on error.
479 */
480 static void
481 nfs_unroll_locks(struct nfs_page *head, struct nfs_page *req)
482 {
483 struct nfs_page *tmp;
484
485 /* relinquish all the locks successfully grabbed this run */
486 for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) {
487 if (!kref_read(&tmp->wb_kref))
488 continue;
489 nfs_unlock_and_release_request(tmp);
490 }
491 }
492
493 /*
494 * nfs_page_group_lock_subreq - try to lock a subrequest
495 * @head: head request of page group
496 * @subreq: request to lock
497 *
498 * This is a helper function for nfs_lock_and_join_requests which
499 * must be called with the head request and page group both locked.
500 * On error, it returns with the page group unlocked.
501 */
502 static int
503 nfs_page_group_lock_subreq(struct nfs_page *head, struct nfs_page *subreq)
504 {
505 int ret;
506
507 if (!kref_get_unless_zero(&subreq->wb_kref))
508 return 0;
509 while (!nfs_lock_request(subreq)) {
510 nfs_page_group_unlock(head);
511 ret = nfs_wait_on_request(subreq);
512 if (!ret)
513 ret = nfs_page_group_lock(head);
514 if (ret < 0) {
515 nfs_unroll_locks(head, subreq);
516 nfs_release_request(subreq);
517 return ret;
518 }
519 }
520 return 0;
521 }
522
523 /*
524 * nfs_lock_and_join_requests - join all subreqs to the head req
525 * @folio: the folio used to lookup the "page group" of nfs_page structures
526 *
527 * This function joins all sub requests to the head request by first
528 * locking all requests in the group, cancelling any pending operations
529 * and finally updating the head request to cover the whole range covered by
530 * the (former) group. All subrequests are removed from any write or commit
531 * lists, unlinked from the group and destroyed.
532 *
533 * Returns a locked, referenced pointer to the head request - which after
534 * this call is guaranteed to be the only request associated with the page.
535 * Returns NULL if no requests are found for @folio, or a ERR_PTR if an
536 * error was encountered.
537 */
538 static struct nfs_page *nfs_lock_and_join_requests(struct folio *folio)
539 {
540 struct inode *inode = folio->mapping->host;
541 struct nfs_page *head, *subreq;
542 struct nfs_commit_info cinfo;
543 bool removed;
544 int ret;
545
546 /*
547 * A reference is taken only on the head request which acts as a
548 * reference to the whole page group - the group will not be destroyed
549 * until the head reference is released.
550 */
551 retry:
552 head = nfs_folio_find_head_request(folio);
553 if (!head)
554 return NULL;
555
556 while (!nfs_lock_request(head)) {
557 ret = nfs_wait_on_request(head);
558 if (ret < 0)
559 return ERR_PTR(ret);
560 }
561
562 /* Ensure that nobody removed the request before we locked it */
563 if (head != folio->private) {
564 nfs_unlock_and_release_request(head);
565 goto retry;
566 }
567
568 ret = nfs_page_group_lock(head);
569 if (ret < 0)
570 goto out_unlock;
571
572 removed = test_bit(PG_REMOVE, &head->wb_flags);
573
574 /* lock each request in the page group */
575 for (subreq = head->wb_this_page;
576 subreq != head;
577 subreq = subreq->wb_this_page) {
578 if (test_bit(PG_REMOVE, &subreq->wb_flags))
579 removed = true;
580 ret = nfs_page_group_lock_subreq(head, subreq);
581 if (ret < 0)
582 goto out_unlock;
583 }
584
585 nfs_page_group_unlock(head);
586
587 /*
588 * If PG_REMOVE is set on any request, I/O on that request has
589 * completed, but some requests were still under I/O at the time
590 * we locked the head request.
591 *
592 * In that case the above wait for all requests means that all I/O
593 * has now finished, and we can restart from a clean slate. Let the
594 * old requests go away and start from scratch instead.
595 */
596 if (removed) {
597 nfs_unroll_locks(head, head);
598 nfs_unlock_and_release_request(head);
599 goto retry;
600 }
601
602 nfs_init_cinfo_from_inode(&cinfo, inode);
603 nfs_join_page_group(head, &cinfo, inode);
604 return head;
605
606 out_unlock:
607 nfs_unlock_and_release_request(head);
608 return ERR_PTR(ret);
609 }
610
611 static void nfs_write_error(struct nfs_page *req, int error)
612 {
613 trace_nfs_write_error(nfs_page_to_inode(req), req, error);
614 nfs_mapping_set_error(nfs_page_to_folio(req), error);
615 nfs_inode_remove_request(req);
616 nfs_page_end_writeback(req);
617 nfs_release_request(req);
618 }
619
620 /*
621 * Find an associated nfs write request, and prepare to flush it out
622 * May return an error if the user signalled nfs_wait_on_request().
623 */
624 static int nfs_page_async_flush(struct folio *folio,
625 struct writeback_control *wbc,
626 struct nfs_pageio_descriptor *pgio)
627 {
628 struct nfs_page *req;
629 int ret = 0;
630
631 req = nfs_lock_and_join_requests(folio);
632 if (!req)
633 goto out;
634 ret = PTR_ERR(req);
635 if (IS_ERR(req))
636 goto out;
637
638 nfs_folio_set_writeback(folio);
639 WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags));
640
641 /* If there is a fatal error that covers this write, just exit */
642 ret = pgio->pg_error;
643 if (nfs_error_is_fatal_on_server(ret))
644 goto out_launder;
645
646 ret = 0;
647 if (!nfs_pageio_add_request(pgio, req)) {
648 ret = pgio->pg_error;
649 /*
650 * Remove the problematic req upon fatal errors on the server
651 */
652 if (nfs_error_is_fatal_on_server(ret))
653 goto out_launder;
654 if (wbc->sync_mode == WB_SYNC_NONE)
655 ret = AOP_WRITEPAGE_ACTIVATE;
656 folio_redirty_for_writepage(wbc, folio);
657 nfs_redirty_request(req);
658 pgio->pg_error = 0;
659 } else
660 nfs_add_stats(folio->mapping->host,
661 NFSIOS_WRITEPAGES, 1);
662 out:
663 return ret;
664 out_launder:
665 nfs_write_error(req, ret);
666 return 0;
667 }
668
669 static int nfs_do_writepage(struct folio *folio, struct writeback_control *wbc,
670 struct nfs_pageio_descriptor *pgio)
671 {
672 nfs_pageio_cond_complete(pgio, folio->index);
673 return nfs_page_async_flush(folio, wbc, pgio);
674 }
675
676 /*
677 * Write an mmapped page to the server.
678 */
679 static int nfs_writepage_locked(struct folio *folio,
680 struct writeback_control *wbc)
681 {
682 struct nfs_pageio_descriptor pgio;
683 struct inode *inode = folio->mapping->host;
684 int err;
685
686 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
687 nfs_pageio_init_write(&pgio, inode, 0, false,
688 &nfs_async_write_completion_ops);
689 err = nfs_do_writepage(folio, wbc, &pgio);
690 pgio.pg_error = 0;
691 nfs_pageio_complete(&pgio);
692 return err;
693 }
694
695 static int nfs_writepages_callback(struct folio *folio,
696 struct writeback_control *wbc, void *data)
697 {
698 int ret;
699
700 ret = nfs_do_writepage(folio, wbc, data);
701 if (ret != AOP_WRITEPAGE_ACTIVATE)
702 folio_unlock(folio);
703 return ret;
704 }
705
706 static void nfs_io_completion_commit(void *inode)
707 {
708 nfs_commit_inode(inode, 0);
709 }
710
711 int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
712 {
713 struct inode *inode = mapping->host;
714 struct nfs_pageio_descriptor pgio;
715 struct nfs_io_completion *ioc = NULL;
716 unsigned int mntflags = NFS_SERVER(inode)->flags;
717 struct nfs_server *nfss = NFS_SERVER(inode);
718 int priority = 0;
719 int err;
720
721 /* Wait with writeback until write congestion eases */
722 if (wbc->sync_mode == WB_SYNC_NONE && nfss->write_congested) {
723 err = wait_event_killable(nfss->write_congestion_wait,
724 nfss->write_congested == 0);
725 if (err)
726 return err;
727 }
728
729 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES);
730
731 if (!(mntflags & NFS_MOUNT_WRITE_EAGER) || wbc->for_kupdate ||
732 wbc->for_background || wbc->for_sync || wbc->for_reclaim) {
733 ioc = nfs_io_completion_alloc(GFP_KERNEL);
734 if (ioc)
735 nfs_io_completion_init(ioc, nfs_io_completion_commit,
736 inode);
737 priority = wb_priority(wbc);
738 }
739
740 do {
741 nfs_pageio_init_write(&pgio, inode, priority, false,
742 &nfs_async_write_completion_ops);
743 pgio.pg_io_completion = ioc;
744 err = write_cache_pages(mapping, wbc, nfs_writepages_callback,
745 &pgio);
746 pgio.pg_error = 0;
747 nfs_pageio_complete(&pgio);
748 if (err == -EAGAIN && mntflags & NFS_MOUNT_SOFTERR)
749 break;
750 } while (err < 0 && !nfs_error_is_fatal(err));
751 nfs_io_completion_put(ioc);
752
753 if (err < 0)
754 goto out_err;
755 return 0;
756 out_err:
757 return err;
758 }
759
760 /*
761 * Insert a write request into an inode
762 */
763 static void nfs_inode_add_request(struct nfs_page *req)
764 {
765 struct folio *folio = nfs_page_to_folio(req);
766 struct address_space *mapping = folio->mapping;
767 struct nfs_inode *nfsi = NFS_I(mapping->host);
768
769 WARN_ON_ONCE(req->wb_this_page != req);
770
771 /* Lock the request! */
772 nfs_lock_request(req);
773 spin_lock(&mapping->i_private_lock);
774 set_bit(PG_MAPPED, &req->wb_flags);
775 folio_set_private(folio);
776 folio->private = req;
777 spin_unlock(&mapping->i_private_lock);
778 atomic_long_inc(&nfsi->nrequests);
779 /* this a head request for a page group - mark it as having an
780 * extra reference so sub groups can follow suit.
781 * This flag also informs pgio layer when to bump nrequests when
782 * adding subrequests. */
783 WARN_ON(test_and_set_bit(PG_INODE_REF, &req->wb_flags));
784 kref_get(&req->wb_kref);
785 }
786
787 /*
788 * Remove a write request from an inode
789 */
790 static void nfs_inode_remove_request(struct nfs_page *req)
791 {
792 struct nfs_inode *nfsi = NFS_I(nfs_page_to_inode(req));
793
794 if (nfs_page_group_sync_on_bit(req, PG_REMOVE)) {
795 struct folio *folio = nfs_page_to_folio(req->wb_head);
796 struct address_space *mapping = folio->mapping;
797
798 spin_lock(&mapping->i_private_lock);
799 if (likely(folio)) {
800 folio->private = NULL;
801 folio_clear_private(folio);
802 clear_bit(PG_MAPPED, &req->wb_head->wb_flags);
803 }
804 spin_unlock(&mapping->i_private_lock);
805 }
806
807 if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags)) {
808 atomic_long_dec(&nfsi->nrequests);
809 nfs_release_request(req);
810 }
811 }
812
813 static void nfs_mark_request_dirty(struct nfs_page *req)
814 {
815 struct folio *folio = nfs_page_to_folio(req);
816 if (folio)
817 filemap_dirty_folio(folio_mapping(folio), folio);
818 }
819
820 /**
821 * nfs_request_add_commit_list_locked - add request to a commit list
822 * @req: pointer to a struct nfs_page
823 * @dst: commit list head
824 * @cinfo: holds list lock and accounting info
825 *
826 * This sets the PG_CLEAN bit, updates the cinfo count of
827 * number of outstanding requests requiring a commit as well as
828 * the MM page stats.
829 *
830 * The caller must hold NFS_I(cinfo->inode)->commit_mutex, and the
831 * nfs_page lock.
832 */
833 void
834 nfs_request_add_commit_list_locked(struct nfs_page *req, struct list_head *dst,
835 struct nfs_commit_info *cinfo)
836 {
837 set_bit(PG_CLEAN, &req->wb_flags);
838 nfs_list_add_request(req, dst);
839 atomic_long_inc(&cinfo->mds->ncommit);
840 }
841 EXPORT_SYMBOL_GPL(nfs_request_add_commit_list_locked);
842
843 /**
844 * nfs_request_add_commit_list - add request to a commit list
845 * @req: pointer to a struct nfs_page
846 * @cinfo: holds list lock and accounting info
847 *
848 * This sets the PG_CLEAN bit, updates the cinfo count of
849 * number of outstanding requests requiring a commit as well as
850 * the MM page stats.
851 *
852 * The caller must _not_ hold the cinfo->lock, but must be
853 * holding the nfs_page lock.
854 */
855 void
856 nfs_request_add_commit_list(struct nfs_page *req, struct nfs_commit_info *cinfo)
857 {
858 mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
859 nfs_request_add_commit_list_locked(req, &cinfo->mds->list, cinfo);
860 mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
861 nfs_folio_mark_unstable(nfs_page_to_folio(req), cinfo);
862 }
863 EXPORT_SYMBOL_GPL(nfs_request_add_commit_list);
864
865 /**
866 * nfs_request_remove_commit_list - Remove request from a commit list
867 * @req: pointer to a nfs_page
868 * @cinfo: holds list lock and accounting info
869 *
870 * This clears the PG_CLEAN bit, and updates the cinfo's count of
871 * number of outstanding requests requiring a commit
872 * It does not update the MM page stats.
873 *
874 * The caller _must_ hold the cinfo->lock and the nfs_page lock.
875 */
876 void
877 nfs_request_remove_commit_list(struct nfs_page *req,
878 struct nfs_commit_info *cinfo)
879 {
880 if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags))
881 return;
882 nfs_list_remove_request(req);
883 atomic_long_dec(&cinfo->mds->ncommit);
884 }
885 EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list);
886
887 static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
888 struct inode *inode)
889 {
890 cinfo->inode = inode;
891 cinfo->mds = &NFS_I(inode)->commit_info;
892 cinfo->ds = pnfs_get_ds_info(inode);
893 cinfo->dreq = NULL;
894 cinfo->completion_ops = &nfs_commit_completion_ops;
895 }
896
897 void nfs_init_cinfo(struct nfs_commit_info *cinfo,
898 struct inode *inode,
899 struct nfs_direct_req *dreq)
900 {
901 if (dreq)
902 nfs_init_cinfo_from_dreq(cinfo, dreq);
903 else
904 nfs_init_cinfo_from_inode(cinfo, inode);
905 }
906 EXPORT_SYMBOL_GPL(nfs_init_cinfo);
907
908 /*
909 * Add a request to the inode's commit list.
910 */
911 void
912 nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
913 struct nfs_commit_info *cinfo, u32 ds_commit_idx)
914 {
915 if (pnfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx))
916 return;
917 nfs_request_add_commit_list(req, cinfo);
918 }
919
920 static void nfs_folio_clear_commit(struct folio *folio)
921 {
922 if (folio) {
923 long nr = folio_nr_pages(folio);
924
925 node_stat_mod_folio(folio, NR_WRITEBACK, -nr);
926 wb_stat_mod(&inode_to_bdi(folio->mapping->host)->wb,
927 WB_WRITEBACK, -nr);
928 }
929 }
930
931 /* Called holding the request lock on @req */
932 static void nfs_clear_request_commit(struct nfs_commit_info *cinfo,
933 struct nfs_page *req)
934 {
935 if (test_bit(PG_CLEAN, &req->wb_flags)) {
936 struct nfs_open_context *ctx = nfs_req_openctx(req);
937 struct inode *inode = d_inode(ctx->dentry);
938
939 mutex_lock(&NFS_I(inode)->commit_mutex);
940 if (!pnfs_clear_request_commit(req, cinfo)) {
941 nfs_request_remove_commit_list(req, cinfo);
942 }
943 mutex_unlock(&NFS_I(inode)->commit_mutex);
944 nfs_folio_clear_commit(nfs_page_to_folio(req));
945 }
946 }
947
948 int nfs_write_need_commit(struct nfs_pgio_header *hdr)
949 {
950 if (hdr->verf.committed == NFS_DATA_SYNC)
951 return hdr->lseg == NULL;
952 return hdr->verf.committed != NFS_FILE_SYNC;
953 }
954
955 static void nfs_async_write_init(struct nfs_pgio_header *hdr)
956 {
957 nfs_io_completion_get(hdr->io_completion);
958 }
959
960 static void nfs_write_completion(struct nfs_pgio_header *hdr)
961 {
962 struct nfs_commit_info cinfo;
963 unsigned long bytes = 0;
964
965 if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
966 goto out;
967 nfs_init_cinfo_from_inode(&cinfo, hdr->inode);
968 while (!list_empty(&hdr->pages)) {
969 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
970
971 bytes += req->wb_bytes;
972 nfs_list_remove_request(req);
973 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) &&
974 (hdr->good_bytes < bytes)) {
975 trace_nfs_comp_error(hdr->inode, req, hdr->error);
976 nfs_mapping_set_error(nfs_page_to_folio(req),
977 hdr->error);
978 goto remove_req;
979 }
980 if (nfs_write_need_commit(hdr)) {
981 /* Reset wb_nio, since the write was successful. */
982 req->wb_nio = 0;
983 memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf));
984 nfs_mark_request_commit(req, hdr->lseg, &cinfo,
985 hdr->pgio_mirror_idx);
986 goto next;
987 }
988 remove_req:
989 nfs_inode_remove_request(req);
990 next:
991 nfs_page_end_writeback(req);
992 nfs_release_request(req);
993 }
994 out:
995 nfs_io_completion_put(hdr->io_completion);
996 hdr->release(hdr);
997 }
998
999 unsigned long
1000 nfs_reqs_to_commit(struct nfs_commit_info *cinfo)
1001 {
1002 return atomic_long_read(&cinfo->mds->ncommit);
1003 }
1004
1005 /* NFS_I(cinfo->inode)->commit_mutex held by caller */
1006 int
1007 nfs_scan_commit_list(struct list_head *src, struct list_head *dst,
1008 struct nfs_commit_info *cinfo, int max)
1009 {
1010 struct nfs_page *req, *tmp;
1011 int ret = 0;
1012
1013 list_for_each_entry_safe(req, tmp, src, wb_list) {
1014 kref_get(&req->wb_kref);
1015 if (!nfs_lock_request(req)) {
1016 nfs_release_request(req);
1017 continue;
1018 }
1019 nfs_request_remove_commit_list(req, cinfo);
1020 clear_bit(PG_COMMIT_TO_DS, &req->wb_flags);
1021 nfs_list_add_request(req, dst);
1022 ret++;
1023 if ((ret == max) && !cinfo->dreq)
1024 break;
1025 cond_resched();
1026 }
1027 return ret;
1028 }
1029 EXPORT_SYMBOL_GPL(nfs_scan_commit_list);
1030
1031 /*
1032 * nfs_scan_commit - Scan an inode for commit requests
1033 * @inode: NFS inode to scan
1034 * @dst: mds destination list
1035 * @cinfo: mds and ds lists of reqs ready to commit
1036 *
1037 * Moves requests from the inode's 'commit' request list.
1038 * The requests are *not* checked to ensure that they form a contiguous set.
1039 */
1040 int
1041 nfs_scan_commit(struct inode *inode, struct list_head *dst,
1042 struct nfs_commit_info *cinfo)
1043 {
1044 int ret = 0;
1045
1046 if (!atomic_long_read(&cinfo->mds->ncommit))
1047 return 0;
1048 mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
1049 if (atomic_long_read(&cinfo->mds->ncommit) > 0) {
1050 const int max = INT_MAX;
1051
1052 ret = nfs_scan_commit_list(&cinfo->mds->list, dst,
1053 cinfo, max);
1054 ret += pnfs_scan_commit_lists(inode, cinfo, max - ret);
1055 }
1056 mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
1057 return ret;
1058 }
1059
1060 /*
1061 * Search for an existing write request, and attempt to update
1062 * it to reflect a new dirty region on a given page.
1063 *
1064 * If the attempt fails, then the existing request is flushed out
1065 * to disk.
1066 */
1067 static struct nfs_page *nfs_try_to_update_request(struct folio *folio,
1068 unsigned int offset,
1069 unsigned int bytes)
1070 {
1071 struct nfs_page *req;
1072 unsigned int rqend;
1073 unsigned int end;
1074 int error;
1075
1076 end = offset + bytes;
1077
1078 req = nfs_lock_and_join_requests(folio);
1079 if (IS_ERR_OR_NULL(req))
1080 return req;
1081
1082 rqend = req->wb_offset + req->wb_bytes;
1083 /*
1084 * Tell the caller to flush out the request if
1085 * the offsets are non-contiguous.
1086 * Note: nfs_flush_incompatible() will already
1087 * have flushed out requests having wrong owners.
1088 */
1089 if (offset > rqend || end < req->wb_offset)
1090 goto out_flushme;
1091
1092 /* Okay, the request matches. Update the region */
1093 if (offset < req->wb_offset) {
1094 req->wb_offset = offset;
1095 req->wb_pgbase = offset;
1096 }
1097 if (end > rqend)
1098 req->wb_bytes = end - req->wb_offset;
1099 else
1100 req->wb_bytes = rqend - req->wb_offset;
1101 req->wb_nio = 0;
1102 return req;
1103 out_flushme:
1104 /*
1105 * Note: we mark the request dirty here because
1106 * nfs_lock_and_join_requests() cannot preserve
1107 * commit flags, so we have to replay the write.
1108 */
1109 nfs_mark_request_dirty(req);
1110 nfs_unlock_and_release_request(req);
1111 error = nfs_wb_folio(folio->mapping->host, folio);
1112 return (error < 0) ? ERR_PTR(error) : NULL;
1113 }
1114
1115 /*
1116 * Try to update an existing write request, or create one if there is none.
1117 *
1118 * Note: Should always be called with the Page Lock held to prevent races
1119 * if we have to add a new request. Also assumes that the caller has
1120 * already called nfs_flush_incompatible() if necessary.
1121 */
1122 static struct nfs_page *nfs_setup_write_request(struct nfs_open_context *ctx,
1123 struct folio *folio,
1124 unsigned int offset,
1125 unsigned int bytes)
1126 {
1127 struct nfs_page *req;
1128
1129 req = nfs_try_to_update_request(folio, offset, bytes);
1130 if (req != NULL)
1131 goto out;
1132 req = nfs_page_create_from_folio(ctx, folio, offset, bytes);
1133 if (IS_ERR(req))
1134 goto out;
1135 nfs_inode_add_request(req);
1136 out:
1137 return req;
1138 }
1139
1140 static int nfs_writepage_setup(struct nfs_open_context *ctx,
1141 struct folio *folio, unsigned int offset,
1142 unsigned int count)
1143 {
1144 struct nfs_page *req;
1145
1146 req = nfs_setup_write_request(ctx, folio, offset, count);
1147 if (IS_ERR(req))
1148 return PTR_ERR(req);
1149 /* Update file length */
1150 nfs_grow_file(folio, offset, count);
1151 nfs_mark_uptodate(req);
1152 nfs_mark_request_dirty(req);
1153 nfs_unlock_and_release_request(req);
1154 return 0;
1155 }
1156
1157 int nfs_flush_incompatible(struct file *file, struct folio *folio)
1158 {
1159 struct nfs_open_context *ctx = nfs_file_open_context(file);
1160 struct nfs_lock_context *l_ctx;
1161 struct file_lock_context *flctx = locks_inode_context(file_inode(file));
1162 struct nfs_page *req;
1163 int do_flush, status;
1164 /*
1165 * Look for a request corresponding to this page. If there
1166 * is one, and it belongs to another file, we flush it out
1167 * before we try to copy anything into the page. Do this
1168 * due to the lack of an ACCESS-type call in NFSv2.
1169 * Also do the same if we find a request from an existing
1170 * dropped page.
1171 */
1172 do {
1173 req = nfs_folio_find_head_request(folio);
1174 if (req == NULL)
1175 return 0;
1176 l_ctx = req->wb_lock_context;
1177 do_flush = nfs_page_to_folio(req) != folio ||
1178 !nfs_match_open_context(nfs_req_openctx(req), ctx);
1179 if (l_ctx && flctx &&
1180 !(list_empty_careful(&flctx->flc_posix) &&
1181 list_empty_careful(&flctx->flc_flock))) {
1182 do_flush |= l_ctx->lockowner != current->files;
1183 }
1184 nfs_release_request(req);
1185 if (!do_flush)
1186 return 0;
1187 status = nfs_wb_folio(folio->mapping->host, folio);
1188 } while (status == 0);
1189 return status;
1190 }
1191
1192 /*
1193 * Avoid buffered writes when a open context credential's key would
1194 * expire soon.
1195 *
1196 * Returns -EACCES if the key will expire within RPC_KEY_EXPIRE_FAIL.
1197 *
1198 * Return 0 and set a credential flag which triggers the inode to flush
1199 * and performs NFS_FILE_SYNC writes if the key will expired within
1200 * RPC_KEY_EXPIRE_TIMEO.
1201 */
1202 int
1203 nfs_key_timeout_notify(struct file *filp, struct inode *inode)
1204 {
1205 struct nfs_open_context *ctx = nfs_file_open_context(filp);
1206
1207 if (nfs_ctx_key_to_expire(ctx, inode) &&
1208 !rcu_access_pointer(ctx->ll_cred))
1209 /* Already expired! */
1210 return -EACCES;
1211 return 0;
1212 }
1213
1214 /*
1215 * Test if the open context credential key is marked to expire soon.
1216 */
1217 bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx, struct inode *inode)
1218 {
1219 struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth;
1220 struct rpc_cred *cred, *new, *old = NULL;
1221 struct auth_cred acred = {
1222 .cred = ctx->cred,
1223 };
1224 bool ret = false;
1225
1226 rcu_read_lock();
1227 cred = rcu_dereference(ctx->ll_cred);
1228 if (cred && !(cred->cr_ops->crkey_timeout &&
1229 cred->cr_ops->crkey_timeout(cred)))
1230 goto out;
1231 rcu_read_unlock();
1232
1233 new = auth->au_ops->lookup_cred(auth, &acred, 0);
1234 if (new == cred) {
1235 put_rpccred(new);
1236 return true;
1237 }
1238 if (IS_ERR_OR_NULL(new)) {
1239 new = NULL;
1240 ret = true;
1241 } else if (new->cr_ops->crkey_timeout &&
1242 new->cr_ops->crkey_timeout(new))
1243 ret = true;
1244
1245 rcu_read_lock();
1246 old = rcu_dereference_protected(xchg(&ctx->ll_cred,
1247 RCU_INITIALIZER(new)), 1);
1248 out:
1249 rcu_read_unlock();
1250 put_rpccred(old);
1251 return ret;
1252 }
1253
1254 /*
1255 * If the page cache is marked as unsafe or invalid, then we can't rely on
1256 * the PageUptodate() flag. In this case, we will need to turn off
1257 * write optimisations that depend on the page contents being correct.
1258 */
1259 static bool nfs_folio_write_uptodate(struct folio *folio, unsigned int pagelen)
1260 {
1261 struct inode *inode = folio->mapping->host;
1262 struct nfs_inode *nfsi = NFS_I(inode);
1263
1264 if (nfs_have_delegated_attributes(inode))
1265 goto out;
1266 if (nfsi->cache_validity &
1267 (NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_SIZE))
1268 return false;
1269 smp_rmb();
1270 if (test_bit(NFS_INO_INVALIDATING, &nfsi->flags) && pagelen != 0)
1271 return false;
1272 out:
1273 if (nfsi->cache_validity & NFS_INO_INVALID_DATA && pagelen != 0)
1274 return false;
1275 return folio_test_uptodate(folio) != 0;
1276 }
1277
1278 static bool
1279 is_whole_file_wrlock(struct file_lock *fl)
1280 {
1281 return fl->fl_start == 0 && fl->fl_end == OFFSET_MAX &&
1282 lock_is_write(fl);
1283 }
1284
1285 /* If we know the page is up to date, and we're not using byte range locks (or
1286 * if we have the whole file locked for writing), it may be more efficient to
1287 * extend the write to cover the entire page in order to avoid fragmentation
1288 * inefficiencies.
1289 *
1290 * If the file is opened for synchronous writes then we can just skip the rest
1291 * of the checks.
1292 */
1293 static int nfs_can_extend_write(struct file *file, struct folio *folio,
1294 unsigned int pagelen)
1295 {
1296 struct inode *inode = file_inode(file);
1297 struct file_lock_context *flctx = locks_inode_context(inode);
1298 struct file_lock *fl;
1299 int ret;
1300
1301 if (file->f_flags & O_DSYNC)
1302 return 0;
1303 if (!nfs_folio_write_uptodate(folio, pagelen))
1304 return 0;
1305 if (nfs_have_write_delegation(inode))
1306 return 1;
1307 if (!flctx || (list_empty_careful(&flctx->flc_flock) &&
1308 list_empty_careful(&flctx->flc_posix)))
1309 return 1;
1310
1311 /* Check to see if there are whole file write locks */
1312 ret = 0;
1313 spin_lock(&flctx->flc_lock);
1314 if (!list_empty(&flctx->flc_posix)) {
1315 fl = list_first_entry(&flctx->flc_posix, struct file_lock,
1316 c.flc_list);
1317 if (is_whole_file_wrlock(fl))
1318 ret = 1;
1319 } else if (!list_empty(&flctx->flc_flock)) {
1320 fl = list_first_entry(&flctx->flc_flock, struct file_lock,
1321 c.flc_list);
1322 if (lock_is_write(fl))
1323 ret = 1;
1324 }
1325 spin_unlock(&flctx->flc_lock);
1326 return ret;
1327 }
1328
1329 /*
1330 * Update and possibly write a cached page of an NFS file.
1331 *
1332 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
1333 * things with a page scheduled for an RPC call (e.g. invalidate it).
1334 */
1335 int nfs_update_folio(struct file *file, struct folio *folio,
1336 unsigned int offset, unsigned int count)
1337 {
1338 struct nfs_open_context *ctx = nfs_file_open_context(file);
1339 struct address_space *mapping = folio->mapping;
1340 struct inode *inode = mapping->host;
1341 unsigned int pagelen = nfs_folio_length(folio);
1342 int status = 0;
1343
1344 nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE);
1345
1346 dprintk("NFS: nfs_update_folio(%pD2 %d@%lld)\n", file, count,
1347 (long long)(folio_pos(folio) + offset));
1348
1349 if (!count)
1350 goto out;
1351
1352 if (nfs_can_extend_write(file, folio, pagelen)) {
1353 unsigned int end = count + offset;
1354
1355 offset = round_down(offset, PAGE_SIZE);
1356 if (end < pagelen)
1357 end = min(round_up(end, PAGE_SIZE), pagelen);
1358 count = end - offset;
1359 }
1360
1361 status = nfs_writepage_setup(ctx, folio, offset, count);
1362 if (status < 0)
1363 nfs_set_pageerror(mapping);
1364 out:
1365 dprintk("NFS: nfs_update_folio returns %d (isize %lld)\n",
1366 status, (long long)i_size_read(inode));
1367 return status;
1368 }
1369
1370 static int flush_task_priority(int how)
1371 {
1372 switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) {
1373 case FLUSH_HIGHPRI:
1374 return RPC_PRIORITY_HIGH;
1375 case FLUSH_LOWPRI:
1376 return RPC_PRIORITY_LOW;
1377 }
1378 return RPC_PRIORITY_NORMAL;
1379 }
1380
1381 static void nfs_initiate_write(struct nfs_pgio_header *hdr,
1382 struct rpc_message *msg,
1383 const struct nfs_rpc_ops *rpc_ops,
1384 struct rpc_task_setup *task_setup_data, int how)
1385 {
1386 int priority = flush_task_priority(how);
1387
1388 if (IS_SWAPFILE(hdr->inode))
1389 task_setup_data->flags |= RPC_TASK_SWAPPER;
1390 task_setup_data->priority = priority;
1391 rpc_ops->write_setup(hdr, msg, &task_setup_data->rpc_client);
1392 trace_nfs_initiate_write(hdr);
1393 }
1394
1395 /* If a nfs_flush_* function fails, it should remove reqs from @head and
1396 * call this on each, which will prepare them to be retried on next
1397 * writeback using standard nfs.
1398 */
1399 static void nfs_redirty_request(struct nfs_page *req)
1400 {
1401 struct nfs_inode *nfsi = NFS_I(nfs_page_to_inode(req));
1402
1403 /* Bump the transmission count */
1404 req->wb_nio++;
1405 nfs_mark_request_dirty(req);
1406 atomic_long_inc(&nfsi->redirtied_pages);
1407 nfs_page_end_writeback(req);
1408 nfs_release_request(req);
1409 }
1410
1411 static void nfs_async_write_error(struct list_head *head, int error)
1412 {
1413 struct nfs_page *req;
1414
1415 while (!list_empty(head)) {
1416 req = nfs_list_entry(head->next);
1417 nfs_list_remove_request(req);
1418 if (nfs_error_is_fatal_on_server(error))
1419 nfs_write_error(req, error);
1420 else
1421 nfs_redirty_request(req);
1422 }
1423 }
1424
1425 static void nfs_async_write_reschedule_io(struct nfs_pgio_header *hdr)
1426 {
1427 nfs_async_write_error(&hdr->pages, 0);
1428 }
1429
1430 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = {
1431 .init_hdr = nfs_async_write_init,
1432 .error_cleanup = nfs_async_write_error,
1433 .completion = nfs_write_completion,
1434 .reschedule_io = nfs_async_write_reschedule_io,
1435 };
1436
1437 void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio,
1438 struct inode *inode, int ioflags, bool force_mds,
1439 const struct nfs_pgio_completion_ops *compl_ops)
1440 {
1441 struct nfs_server *server = NFS_SERVER(inode);
1442 const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops;
1443
1444 #ifdef CONFIG_NFS_V4_1
1445 if (server->pnfs_curr_ld && !force_mds)
1446 pg_ops = server->pnfs_curr_ld->pg_write_ops;
1447 #endif
1448 nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_write_ops,
1449 server->wsize, ioflags);
1450 }
1451 EXPORT_SYMBOL_GPL(nfs_pageio_init_write);
1452
1453 void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio)
1454 {
1455 struct nfs_pgio_mirror *mirror;
1456
1457 if (pgio->pg_ops && pgio->pg_ops->pg_cleanup)
1458 pgio->pg_ops->pg_cleanup(pgio);
1459
1460 pgio->pg_ops = &nfs_pgio_rw_ops;
1461
1462 nfs_pageio_stop_mirroring(pgio);
1463
1464 mirror = &pgio->pg_mirrors[0];
1465 mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize;
1466 }
1467 EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds);
1468
1469
1470 void nfs_commit_prepare(struct rpc_task *task, void *calldata)
1471 {
1472 struct nfs_commit_data *data = calldata;
1473
1474 NFS_PROTO(data->inode)->commit_rpc_prepare(task, data);
1475 }
1476
1477 static void nfs_writeback_check_extend(struct nfs_pgio_header *hdr,
1478 struct nfs_fattr *fattr)
1479 {
1480 struct nfs_pgio_args *argp = &hdr->args;
1481 struct nfs_pgio_res *resp = &hdr->res;
1482 u64 size = argp->offset + resp->count;
1483
1484 if (!(fattr->valid & NFS_ATTR_FATTR_SIZE))
1485 fattr->size = size;
1486 if (nfs_size_to_loff_t(fattr->size) < i_size_read(hdr->inode)) {
1487 fattr->valid &= ~NFS_ATTR_FATTR_SIZE;
1488 return;
1489 }
1490 if (size != fattr->size)
1491 return;
1492 /* Set attribute barrier */
1493 nfs_fattr_set_barrier(fattr);
1494 /* ...and update size */
1495 fattr->valid |= NFS_ATTR_FATTR_SIZE;
1496 }
1497
1498 void nfs_writeback_update_inode(struct nfs_pgio_header *hdr)
1499 {
1500 struct nfs_fattr *fattr = &hdr->fattr;
1501 struct inode *inode = hdr->inode;
1502
1503 if (nfs_have_delegated_mtime(inode)) {
1504 spin_lock(&inode->i_lock);
1505 nfs_set_cache_invalid(inode, NFS_INO_INVALID_BLOCKS);
1506 spin_unlock(&inode->i_lock);
1507 return;
1508 }
1509
1510 spin_lock(&inode->i_lock);
1511 nfs_writeback_check_extend(hdr, fattr);
1512 nfs_post_op_update_inode_force_wcc_locked(inode, fattr);
1513 spin_unlock(&inode->i_lock);
1514 }
1515 EXPORT_SYMBOL_GPL(nfs_writeback_update_inode);
1516
1517 /*
1518 * This function is called when the WRITE call is complete.
1519 */
1520 static int nfs_writeback_done(struct rpc_task *task,
1521 struct nfs_pgio_header *hdr,
1522 struct inode *inode)
1523 {
1524 int status;
1525
1526 /*
1527 * ->write_done will attempt to use post-op attributes to detect
1528 * conflicting writes by other clients. A strict interpretation
1529 * of close-to-open would allow us to continue caching even if
1530 * another writer had changed the file, but some applications
1531 * depend on tighter cache coherency when writing.
1532 */
1533 status = NFS_PROTO(inode)->write_done(task, hdr);
1534 if (status != 0)
1535 return status;
1536
1537 nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, hdr->res.count);
1538 trace_nfs_writeback_done(task, hdr);
1539
1540 if (task->tk_status >= 0) {
1541 enum nfs3_stable_how committed = hdr->res.verf->committed;
1542
1543 if (committed == NFS_UNSTABLE) {
1544 /*
1545 * We have some uncommitted data on the server at
1546 * this point, so ensure that we keep track of that
1547 * fact irrespective of what later writes do.
1548 */
1549 set_bit(NFS_IOHDR_UNSTABLE_WRITES, &hdr->flags);
1550 }
1551
1552 if (committed < hdr->args.stable) {
1553 /* We tried a write call, but the server did not
1554 * commit data to stable storage even though we
1555 * requested it.
1556 * Note: There is a known bug in Tru64 < 5.0 in which
1557 * the server reports NFS_DATA_SYNC, but performs
1558 * NFS_FILE_SYNC. We therefore implement this checking
1559 * as a dprintk() in order to avoid filling syslog.
1560 */
1561 static unsigned long complain;
1562
1563 /* Note this will print the MDS for a DS write */
1564 if (time_before(complain, jiffies)) {
1565 dprintk("NFS: faulty NFS server %s:"
1566 " (committed = %d) != (stable = %d)\n",
1567 NFS_SERVER(inode)->nfs_client->cl_hostname,
1568 committed, hdr->args.stable);
1569 complain = jiffies + 300 * HZ;
1570 }
1571 }
1572 }
1573
1574 /* Deal with the suid/sgid bit corner case */
1575 if (nfs_should_remove_suid(inode)) {
1576 spin_lock(&inode->i_lock);
1577 nfs_set_cache_invalid(inode, NFS_INO_INVALID_MODE);
1578 spin_unlock(&inode->i_lock);
1579 }
1580 return 0;
1581 }
1582
1583 /*
1584 * This function is called when the WRITE call is complete.
1585 */
1586 static void nfs_writeback_result(struct rpc_task *task,
1587 struct nfs_pgio_header *hdr)
1588 {
1589 struct nfs_pgio_args *argp = &hdr->args;
1590 struct nfs_pgio_res *resp = &hdr->res;
1591
1592 if (resp->count < argp->count) {
1593 static unsigned long complain;
1594
1595 /* This a short write! */
1596 nfs_inc_stats(hdr->inode, NFSIOS_SHORTWRITE);
1597
1598 /* Has the server at least made some progress? */
1599 if (resp->count == 0) {
1600 if (time_before(complain, jiffies)) {
1601 printk(KERN_WARNING
1602 "NFS: Server wrote zero bytes, expected %u.\n",
1603 argp->count);
1604 complain = jiffies + 300 * HZ;
1605 }
1606 nfs_set_pgio_error(hdr, -EIO, argp->offset);
1607 task->tk_status = -EIO;
1608 return;
1609 }
1610
1611 /* For non rpc-based layout drivers, retry-through-MDS */
1612 if (!task->tk_ops) {
1613 hdr->pnfs_error = -EAGAIN;
1614 return;
1615 }
1616
1617 /* Was this an NFSv2 write or an NFSv3 stable write? */
1618 if (resp->verf->committed != NFS_UNSTABLE) {
1619 /* Resend from where the server left off */
1620 hdr->mds_offset += resp->count;
1621 argp->offset += resp->count;
1622 argp->pgbase += resp->count;
1623 argp->count -= resp->count;
1624 } else {
1625 /* Resend as a stable write in order to avoid
1626 * headaches in the case of a server crash.
1627 */
1628 argp->stable = NFS_FILE_SYNC;
1629 }
1630 resp->count = 0;
1631 resp->verf->committed = 0;
1632 rpc_restart_call_prepare(task);
1633 }
1634 }
1635
1636 static int wait_on_commit(struct nfs_mds_commit_info *cinfo)
1637 {
1638 return wait_var_event_killable(&cinfo->rpcs_out,
1639 !atomic_read(&cinfo->rpcs_out));
1640 }
1641
1642 void nfs_commit_begin(struct nfs_mds_commit_info *cinfo)
1643 {
1644 atomic_inc(&cinfo->rpcs_out);
1645 }
1646
1647 bool nfs_commit_end(struct nfs_mds_commit_info *cinfo)
1648 {
1649 if (atomic_dec_and_test(&cinfo->rpcs_out)) {
1650 wake_up_var(&cinfo->rpcs_out);
1651 return true;
1652 }
1653 return false;
1654 }
1655
1656 void nfs_commitdata_release(struct nfs_commit_data *data)
1657 {
1658 put_nfs_open_context(data->context);
1659 nfs_commit_free(data);
1660 }
1661 EXPORT_SYMBOL_GPL(nfs_commitdata_release);
1662
1663 int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data,
1664 const struct nfs_rpc_ops *nfs_ops,
1665 const struct rpc_call_ops *call_ops,
1666 int how, int flags)
1667 {
1668 struct rpc_task *task;
1669 int priority = flush_task_priority(how);
1670 struct rpc_message msg = {
1671 .rpc_argp = &data->args,
1672 .rpc_resp = &data->res,
1673 .rpc_cred = data->cred,
1674 };
1675 struct rpc_task_setup task_setup_data = {
1676 .task = &data->task,
1677 .rpc_client = clnt,
1678 .rpc_message = &msg,
1679 .callback_ops = call_ops,
1680 .callback_data = data,
1681 .workqueue = nfsiod_workqueue,
1682 .flags = RPC_TASK_ASYNC | flags,
1683 .priority = priority,
1684 };
1685
1686 if (nfs_server_capable(data->inode, NFS_CAP_MOVEABLE))
1687 task_setup_data.flags |= RPC_TASK_MOVEABLE;
1688
1689 /* Set up the initial task struct. */
1690 nfs_ops->commit_setup(data, &msg, &task_setup_data.rpc_client);
1691 trace_nfs_initiate_commit(data);
1692
1693 dprintk("NFS: initiated commit call\n");
1694
1695 task = rpc_run_task(&task_setup_data);
1696 if (IS_ERR(task))
1697 return PTR_ERR(task);
1698 if (how & FLUSH_SYNC)
1699 rpc_wait_for_completion_task(task);
1700 rpc_put_task(task);
1701 return 0;
1702 }
1703 EXPORT_SYMBOL_GPL(nfs_initiate_commit);
1704
1705 static loff_t nfs_get_lwb(struct list_head *head)
1706 {
1707 loff_t lwb = 0;
1708 struct nfs_page *req;
1709
1710 list_for_each_entry(req, head, wb_list)
1711 if (lwb < (req_offset(req) + req->wb_bytes))
1712 lwb = req_offset(req) + req->wb_bytes;
1713
1714 return lwb;
1715 }
1716
1717 /*
1718 * Set up the argument/result storage required for the RPC call.
1719 */
1720 void nfs_init_commit(struct nfs_commit_data *data,
1721 struct list_head *head,
1722 struct pnfs_layout_segment *lseg,
1723 struct nfs_commit_info *cinfo)
1724 {
1725 struct nfs_page *first;
1726 struct nfs_open_context *ctx;
1727 struct inode *inode;
1728
1729 /* Set up the RPC argument and reply structs
1730 * NB: take care not to mess about with data->commit et al. */
1731
1732 if (head)
1733 list_splice_init(head, &data->pages);
1734
1735 first = nfs_list_entry(data->pages.next);
1736 ctx = nfs_req_openctx(first);
1737 inode = d_inode(ctx->dentry);
1738
1739 data->inode = inode;
1740 data->cred = ctx->cred;
1741 data->lseg = lseg; /* reference transferred */
1742 /* only set lwb for pnfs commit */
1743 if (lseg)
1744 data->lwb = nfs_get_lwb(&data->pages);
1745 data->mds_ops = &nfs_commit_ops;
1746 data->completion_ops = cinfo->completion_ops;
1747 data->dreq = cinfo->dreq;
1748
1749 data->args.fh = NFS_FH(data->inode);
1750 /* Note: we always request a commit of the entire inode */
1751 data->args.offset = 0;
1752 data->args.count = 0;
1753 data->context = get_nfs_open_context(ctx);
1754 data->res.fattr = &data->fattr;
1755 data->res.verf = &data->verf;
1756 nfs_fattr_init(&data->fattr);
1757 nfs_commit_begin(cinfo->mds);
1758 }
1759 EXPORT_SYMBOL_GPL(nfs_init_commit);
1760
1761 void nfs_retry_commit(struct list_head *page_list,
1762 struct pnfs_layout_segment *lseg,
1763 struct nfs_commit_info *cinfo,
1764 u32 ds_commit_idx)
1765 {
1766 struct nfs_page *req;
1767
1768 while (!list_empty(page_list)) {
1769 req = nfs_list_entry(page_list->next);
1770 nfs_list_remove_request(req);
1771 nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx);
1772 nfs_folio_clear_commit(nfs_page_to_folio(req));
1773 nfs_unlock_and_release_request(req);
1774 }
1775 }
1776 EXPORT_SYMBOL_GPL(nfs_retry_commit);
1777
1778 static void nfs_commit_resched_write(struct nfs_commit_info *cinfo,
1779 struct nfs_page *req)
1780 {
1781 struct folio *folio = nfs_page_to_folio(req);
1782
1783 filemap_dirty_folio(folio_mapping(folio), folio);
1784 }
1785
1786 /*
1787 * Commit dirty pages
1788 */
1789 static int
1790 nfs_commit_list(struct inode *inode, struct list_head *head, int how,
1791 struct nfs_commit_info *cinfo)
1792 {
1793 struct nfs_commit_data *data;
1794 unsigned short task_flags = 0;
1795
1796 /* another commit raced with us */
1797 if (list_empty(head))
1798 return 0;
1799
1800 data = nfs_commitdata_alloc();
1801 if (!data) {
1802 nfs_retry_commit(head, NULL, cinfo, -1);
1803 return -ENOMEM;
1804 }
1805
1806 /* Set up the argument struct */
1807 nfs_init_commit(data, head, NULL, cinfo);
1808 if (NFS_SERVER(inode)->nfs_client->cl_minorversion)
1809 task_flags = RPC_TASK_MOVEABLE;
1810 return nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(inode),
1811 data->mds_ops, how,
1812 RPC_TASK_CRED_NOREF | task_flags);
1813 }
1814
1815 /*
1816 * COMMIT call returned
1817 */
1818 static void nfs_commit_done(struct rpc_task *task, void *calldata)
1819 {
1820 struct nfs_commit_data *data = calldata;
1821
1822 /* Call the NFS version-specific code */
1823 NFS_PROTO(data->inode)->commit_done(task, data);
1824 trace_nfs_commit_done(task, data);
1825 }
1826
1827 static void nfs_commit_release_pages(struct nfs_commit_data *data)
1828 {
1829 const struct nfs_writeverf *verf = data->res.verf;
1830 struct nfs_page *req;
1831 int status = data->task.tk_status;
1832 struct nfs_commit_info cinfo;
1833 struct folio *folio;
1834
1835 while (!list_empty(&data->pages)) {
1836 req = nfs_list_entry(data->pages.next);
1837 nfs_list_remove_request(req);
1838 folio = nfs_page_to_folio(req);
1839 nfs_folio_clear_commit(folio);
1840
1841 dprintk("NFS: commit (%s/%llu %d@%lld)",
1842 nfs_req_openctx(req)->dentry->d_sb->s_id,
1843 (unsigned long long)NFS_FILEID(d_inode(nfs_req_openctx(req)->dentry)),
1844 req->wb_bytes,
1845 (long long)req_offset(req));
1846 if (status < 0) {
1847 if (folio) {
1848 trace_nfs_commit_error(data->inode, req,
1849 status);
1850 nfs_mapping_set_error(folio, status);
1851 nfs_inode_remove_request(req);
1852 }
1853 dprintk_cont(", error = %d\n", status);
1854 goto next;
1855 }
1856
1857 /* Okay, COMMIT succeeded, apparently. Check the verifier
1858 * returned by the server against all stored verfs. */
1859 if (nfs_write_match_verf(verf, req)) {
1860 /* We have a match */
1861 if (folio)
1862 nfs_inode_remove_request(req);
1863 dprintk_cont(" OK\n");
1864 goto next;
1865 }
1866 /* We have a mismatch. Write the page again */
1867 dprintk_cont(" mismatch\n");
1868 nfs_mark_request_dirty(req);
1869 atomic_long_inc(&NFS_I(data->inode)->redirtied_pages);
1870 next:
1871 nfs_unlock_and_release_request(req);
1872 /* Latency breaker */
1873 cond_resched();
1874 }
1875
1876 nfs_init_cinfo(&cinfo, data->inode, data->dreq);
1877 nfs_commit_end(cinfo.mds);
1878 }
1879
1880 static void nfs_commit_release(void *calldata)
1881 {
1882 struct nfs_commit_data *data = calldata;
1883
1884 data->completion_ops->completion(data);
1885 nfs_commitdata_release(calldata);
1886 }
1887
1888 static const struct rpc_call_ops nfs_commit_ops = {
1889 .rpc_call_prepare = nfs_commit_prepare,
1890 .rpc_call_done = nfs_commit_done,
1891 .rpc_release = nfs_commit_release,
1892 };
1893
1894 static const struct nfs_commit_completion_ops nfs_commit_completion_ops = {
1895 .completion = nfs_commit_release_pages,
1896 .resched_write = nfs_commit_resched_write,
1897 };
1898
1899 int nfs_generic_commit_list(struct inode *inode, struct list_head *head,
1900 int how, struct nfs_commit_info *cinfo)
1901 {
1902 int status;
1903
1904 status = pnfs_commit_list(inode, head, how, cinfo);
1905 if (status == PNFS_NOT_ATTEMPTED)
1906 status = nfs_commit_list(inode, head, how, cinfo);
1907 return status;
1908 }
1909
1910 static int __nfs_commit_inode(struct inode *inode, int how,
1911 struct writeback_control *wbc)
1912 {
1913 LIST_HEAD(head);
1914 struct nfs_commit_info cinfo;
1915 int may_wait = how & FLUSH_SYNC;
1916 int ret, nscan;
1917
1918 how &= ~FLUSH_SYNC;
1919 nfs_init_cinfo_from_inode(&cinfo, inode);
1920 nfs_commit_begin(cinfo.mds);
1921 for (;;) {
1922 ret = nscan = nfs_scan_commit(inode, &head, &cinfo);
1923 if (ret <= 0)
1924 break;
1925 ret = nfs_generic_commit_list(inode, &head, how, &cinfo);
1926 if (ret < 0)
1927 break;
1928 ret = 0;
1929 if (wbc && wbc->sync_mode == WB_SYNC_NONE) {
1930 if (nscan < wbc->nr_to_write)
1931 wbc->nr_to_write -= nscan;
1932 else
1933 wbc->nr_to_write = 0;
1934 }
1935 if (nscan < INT_MAX)
1936 break;
1937 cond_resched();
1938 }
1939 nfs_commit_end(cinfo.mds);
1940 if (ret || !may_wait)
1941 return ret;
1942 return wait_on_commit(cinfo.mds);
1943 }
1944
1945 int nfs_commit_inode(struct inode *inode, int how)
1946 {
1947 return __nfs_commit_inode(inode, how, NULL);
1948 }
1949 EXPORT_SYMBOL_GPL(nfs_commit_inode);
1950
1951 int nfs_write_inode(struct inode *inode, struct writeback_control *wbc)
1952 {
1953 struct nfs_inode *nfsi = NFS_I(inode);
1954 int flags = FLUSH_SYNC;
1955 int ret = 0;
1956
1957 if (wbc->sync_mode == WB_SYNC_NONE) {
1958 /* no commits means nothing needs to be done */
1959 if (!atomic_long_read(&nfsi->commit_info.ncommit))
1960 goto check_requests_outstanding;
1961
1962 /* Don't commit yet if this is a non-blocking flush and there
1963 * are a lot of outstanding writes for this mapping.
1964 */
1965 if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK))
1966 goto out_mark_dirty;
1967
1968 /* don't wait for the COMMIT response */
1969 flags = 0;
1970 }
1971
1972 ret = __nfs_commit_inode(inode, flags, wbc);
1973 if (!ret) {
1974 if (flags & FLUSH_SYNC)
1975 return 0;
1976 } else if (atomic_long_read(&nfsi->commit_info.ncommit))
1977 goto out_mark_dirty;
1978
1979 check_requests_outstanding:
1980 if (!atomic_read(&nfsi->commit_info.rpcs_out))
1981 return ret;
1982 out_mark_dirty:
1983 __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
1984 return ret;
1985 }
1986 EXPORT_SYMBOL_GPL(nfs_write_inode);
1987
1988 /*
1989 * Wrapper for filemap_write_and_wait_range()
1990 *
1991 * Needed for pNFS in order to ensure data becomes visible to the
1992 * client.
1993 */
1994 int nfs_filemap_write_and_wait_range(struct address_space *mapping,
1995 loff_t lstart, loff_t lend)
1996 {
1997 int ret;
1998
1999 ret = filemap_write_and_wait_range(mapping, lstart, lend);
2000 if (ret == 0)
2001 ret = pnfs_sync_inode(mapping->host, true);
2002 return ret;
2003 }
2004 EXPORT_SYMBOL_GPL(nfs_filemap_write_and_wait_range);
2005
2006 /*
2007 * flush the inode to disk.
2008 */
2009 int nfs_wb_all(struct inode *inode)
2010 {
2011 int ret;
2012
2013 trace_nfs_writeback_inode_enter(inode);
2014
2015 ret = filemap_write_and_wait(inode->i_mapping);
2016 if (ret)
2017 goto out;
2018 ret = nfs_commit_inode(inode, FLUSH_SYNC);
2019 if (ret < 0)
2020 goto out;
2021 pnfs_sync_inode(inode, true);
2022 ret = 0;
2023
2024 out:
2025 trace_nfs_writeback_inode_exit(inode, ret);
2026 return ret;
2027 }
2028 EXPORT_SYMBOL_GPL(nfs_wb_all);
2029
2030 int nfs_wb_folio_cancel(struct inode *inode, struct folio *folio)
2031 {
2032 struct nfs_page *req;
2033 int ret = 0;
2034
2035 folio_wait_writeback(folio);
2036
2037 /* blocking call to cancel all requests and join to a single (head)
2038 * request */
2039 req = nfs_lock_and_join_requests(folio);
2040
2041 if (IS_ERR(req)) {
2042 ret = PTR_ERR(req);
2043 } else if (req) {
2044 /* all requests from this folio have been cancelled by
2045 * nfs_lock_and_join_requests, so just remove the head
2046 * request from the inode / page_private pointer and
2047 * release it */
2048 nfs_inode_remove_request(req);
2049 nfs_unlock_and_release_request(req);
2050 }
2051
2052 return ret;
2053 }
2054
2055 /**
2056 * nfs_wb_folio - Write back all requests on one page
2057 * @inode: pointer to page
2058 * @folio: pointer to folio
2059 *
2060 * Assumes that the folio has been locked by the caller, and will
2061 * not unlock it.
2062 */
2063 int nfs_wb_folio(struct inode *inode, struct folio *folio)
2064 {
2065 loff_t range_start = folio_pos(folio);
2066 size_t len = folio_size(folio);
2067 struct writeback_control wbc = {
2068 .sync_mode = WB_SYNC_ALL,
2069 .nr_to_write = 0,
2070 .range_start = range_start,
2071 .range_end = range_start + len - 1,
2072 };
2073 int ret;
2074
2075 trace_nfs_writeback_folio(inode, range_start, len);
2076
2077 for (;;) {
2078 folio_wait_writeback(folio);
2079 if (folio_clear_dirty_for_io(folio)) {
2080 ret = nfs_writepage_locked(folio, &wbc);
2081 if (ret < 0)
2082 goto out_error;
2083 continue;
2084 }
2085 ret = 0;
2086 if (!folio_test_private(folio))
2087 break;
2088 ret = nfs_commit_inode(inode, FLUSH_SYNC);
2089 if (ret < 0)
2090 goto out_error;
2091 }
2092 out_error:
2093 trace_nfs_writeback_folio_done(inode, range_start, len, ret);
2094 return ret;
2095 }
2096
2097 #ifdef CONFIG_MIGRATION
2098 int nfs_migrate_folio(struct address_space *mapping, struct folio *dst,
2099 struct folio *src, enum migrate_mode mode)
2100 {
2101 /*
2102 * If the private flag is set, the folio is currently associated with
2103 * an in-progress read or write request. Don't try to migrate it.
2104 *
2105 * FIXME: we could do this in principle, but we'll need a way to ensure
2106 * that we can safely release the inode reference while holding
2107 * the folio lock.
2108 */
2109 if (folio_test_private(src))
2110 return -EBUSY;
2111
2112 if (folio_test_private_2(src)) { /* [DEPRECATED] */
2113 if (mode == MIGRATE_ASYNC)
2114 return -EBUSY;
2115 folio_wait_private_2(src);
2116 }
2117
2118 return migrate_folio(mapping, dst, src, mode);
2119 }
2120 #endif
2121
2122 int __init nfs_init_writepagecache(void)
2123 {
2124 nfs_wdata_cachep = kmem_cache_create("nfs_write_data",
2125 sizeof(struct nfs_pgio_header),
2126 0, SLAB_HWCACHE_ALIGN,
2127 NULL);
2128 if (nfs_wdata_cachep == NULL)
2129 return -ENOMEM;
2130
2131 nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE,
2132 nfs_wdata_cachep);
2133 if (nfs_wdata_mempool == NULL)
2134 goto out_destroy_write_cache;
2135
2136 nfs_cdata_cachep = kmem_cache_create("nfs_commit_data",
2137 sizeof(struct nfs_commit_data),
2138 0, SLAB_HWCACHE_ALIGN,
2139 NULL);
2140 if (nfs_cdata_cachep == NULL)
2141 goto out_destroy_write_mempool;
2142
2143 nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT,
2144 nfs_cdata_cachep);
2145 if (nfs_commit_mempool == NULL)
2146 goto out_destroy_commit_cache;
2147
2148 /*
2149 * NFS congestion size, scale with available memory.
2150 *
2151 * 64MB: 8192k
2152 * 128MB: 11585k
2153 * 256MB: 16384k
2154 * 512MB: 23170k
2155 * 1GB: 32768k
2156 * 2GB: 46340k
2157 * 4GB: 65536k
2158 * 8GB: 92681k
2159 * 16GB: 131072k
2160 *
2161 * This allows larger machines to have larger/more transfers.
2162 * Limit the default to 256M
2163 */
2164 nfs_congestion_kb = (16*int_sqrt(totalram_pages())) << (PAGE_SHIFT-10);
2165 if (nfs_congestion_kb > 256*1024)
2166 nfs_congestion_kb = 256*1024;
2167
2168 return 0;
2169
2170 out_destroy_commit_cache:
2171 kmem_cache_destroy(nfs_cdata_cachep);
2172 out_destroy_write_mempool:
2173 mempool_destroy(nfs_wdata_mempool);
2174 out_destroy_write_cache:
2175 kmem_cache_destroy(nfs_wdata_cachep);
2176 return -ENOMEM;
2177 }
2178
2179 void nfs_destroy_writepagecache(void)
2180 {
2181 mempool_destroy(nfs_commit_mempool);
2182 kmem_cache_destroy(nfs_cdata_cachep);
2183 mempool_destroy(nfs_wdata_mempool);
2184 kmem_cache_destroy(nfs_wdata_cachep);
2185 }
2186
2187 static const struct nfs_rw_ops nfs_rw_write_ops = {
2188 .rw_alloc_header = nfs_writehdr_alloc,
2189 .rw_free_header = nfs_writehdr_free,
2190 .rw_done = nfs_writeback_done,
2191 .rw_result = nfs_writeback_result,
2192 .rw_initiate = nfs_initiate_write,
2193 };
2194
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.