1 // SPDX-License-Identifier: GPL-2.0-only 1
2 /*
3 * NFS client support for local clients to byp
4 *
5 * Copyright (C) 2014 Weston Andros Adamson <d
6 * Copyright (C) 2019 Trond Myklebust <trond.m
7 * Copyright (C) 2024 Mike Snitzer <snitzer@ha
8 * Copyright (C) 2024 NeilBrown <neilb@suse.de
9 */
10
11 #include <linux/module.h>
12 #include <linux/errno.h>
13 #include <linux/vfs.h>
14 #include <linux/file.h>
15 #include <linux/inet.h>
16 #include <linux/sunrpc/addr.h>
17 #include <linux/inetdevice.h>
18 #include <net/addrconf.h>
19 #include <linux/nfs_common.h>
20 #include <linux/nfslocalio.h>
21 #include <linux/bvec.h>
22
23 #include <linux/nfs.h>
24 #include <linux/nfs_fs.h>
25 #include <linux/nfs_xdr.h>
26
27 #include "internal.h"
28 #include "pnfs.h"
29 #include "nfstrace.h"
30
31 #define NFSDBG_FACILITY NFSDBG_VFS
32
33 struct nfs_local_kiocb {
34 struct kiocb kiocb;
35 struct bio_vec *bvec;
36 struct nfs_pgio_header *hdr;
37 struct work_struct work;
38 struct nfsd_file *localio;
39 };
40
41 struct nfs_local_fsync_ctx {
42 struct nfsd_file *localio;
43 struct nfs_commit_data *data;
44 struct work_struct work;
45 struct kref kref;
46 struct completion *done;
47 };
48 static void nfs_local_fsync_work(struct work_s
49
50 static bool localio_enabled __read_mostly = tr
51 module_param(localio_enabled, bool, 0644);
52
53 static inline bool nfs_client_is_local(const s
54 {
55 return !!test_bit(NFS_CS_LOCAL_IO, &cl
56 }
57
58 bool nfs_server_is_local(const struct nfs_clie
59 {
60 return nfs_client_is_local(clp) && loc
61 }
62 EXPORT_SYMBOL_GPL(nfs_server_is_local);
63
64 /*
65 * UUID_IS_LOCAL XDR functions
66 */
67
68 static void localio_xdr_enc_uuidargs(struct rp
69 struct xd
70 const voi
71 {
72 const u8 *uuid = data;
73
74 encode_opaque_fixed(xdr, uuid, UUID_SI
75 }
76
77 static int localio_xdr_dec_uuidres(struct rpc_
78 struct xdr_
79 void *resul
80 {
81 /* void return */
82 return 0;
83 }
84
85 static const struct rpc_procinfo nfs_localio_p
86 [LOCALIOPROC_UUID_IS_LOCAL] = {
87 .p_proc = LOCALIOPROC_UUID_IS_
88 .p_encode = localio_xdr_enc_uu
89 .p_decode = localio_xdr_dec_uu
90 .p_arglen = XDR_QUADLEN(UUID_S
91 .p_replen = 0,
92 .p_statidx = LOCALIOPROC_UUID_
93 .p_name = "UUID_IS_LOCAL",
94 },
95 };
96
97 static unsigned int nfs_localio_counts[ARRAY_S
98 static const struct rpc_version nfslocalio_ver
99 .number = 1,
100 .nrprocs = ARRAY_SIZE(n
101 .procs = nfs_localio_
102 .counts = nfs_localio_
103 };
104
105 static const struct rpc_version *nfslocalio_ve
106 [1] = &nfslocalio_
107 };
108
109 extern const struct rpc_program nfslocalio_pro
110 static struct rpc_stat nfslocalio_rpc
111
112 const struct rpc_program nfslocalio_program =
113 .name = "nfslocalio"
114 .number = NFS_LOCALIO_
115 .nrvers = ARRAY_SIZE(n
116 .version = nfslocalio_v
117 .stats = &nfslocalio_
118 };
119
120 /*
121 * nfs_local_enable - enable local i/o for an
122 */
123 static void nfs_local_enable(struct nfs_client
124 {
125 spin_lock(&clp->cl_localio_lock);
126 set_bit(NFS_CS_LOCAL_IO, &clp->cl_flag
127 trace_nfs_local_enable(clp);
128 spin_unlock(&clp->cl_localio_lock);
129 }
130
131 /*
132 * nfs_local_disable - disable local i/o for a
133 */
134 void nfs_local_disable(struct nfs_client *clp)
135 {
136 spin_lock(&clp->cl_localio_lock);
137 if (test_and_clear_bit(NFS_CS_LOCAL_IO
138 trace_nfs_local_disable(clp);
139 nfs_uuid_invalidate_one_client
140 }
141 spin_unlock(&clp->cl_localio_lock);
142 }
143
144 /*
145 * nfs_init_localioclient - Initialise an NFS
146 */
147 static struct rpc_clnt *nfs_init_localioclient
148 {
149 struct rpc_clnt *rpcclient_localio;
150
151 rpcclient_localio = rpc_bind_new_progr
152
153
154 dprintk_rcu("%s: server (%s) %s NFS LO
155 __func__, rpc_peeraddr2str(clp
156 (IS_ERR(rpcclient_localio) ? "
157
158 return rpcclient_localio;
159 }
160
161 static bool nfs_server_uuid_is_local(struct nf
162 {
163 u8 uuid[UUID_SIZE];
164 struct rpc_message msg = {
165 .rpc_argp = &uuid,
166 };
167 struct rpc_clnt *rpcclient_localio;
168 int status;
169
170 rpcclient_localio = nfs_init_localiocl
171 if (IS_ERR(rpcclient_localio))
172 return false;
173
174 export_uuid(uuid, &clp->cl_uuid.uuid);
175
176 msg.rpc_proc = &nfs_localio_procedures
177 status = rpc_call_sync(rpcclient_local
178 dprintk("%s: NFS reply UUID_IS_LOCAL:
179 __func__, status);
180 rpc_shutdown_client(rpcclient_localio)
181
182 /* Server is only local if it initiali
183 if (status || !clp->cl_uuid.net || !cl
184 return false;
185
186 return true;
187 }
188
189 /*
190 * nfs_local_probe - probe local i/o support f
191 * - called after alloc_client and init_client
192 * - this function is idempotent, it can be ca
193 */
194 void nfs_local_probe(struct nfs_client *clp)
195 {
196 /* Disallow localio if disabled via sy
197 if (!localio_enabled ||
198 clp->cl_rpcclient->cl_auth->au_fla
199 nfs_local_disable(clp);
200 return;
201 }
202
203 if (nfs_client_is_local(clp)) {
204 /* If already enabled, disable
205 nfs_local_disable(clp);
206 }
207
208 if (!nfs_uuid_begin(&clp->cl_uuid))
209 return;
210 if (nfs_server_uuid_is_local(clp))
211 nfs_local_enable(clp);
212 nfs_uuid_end(&clp->cl_uuid);
213 }
214 EXPORT_SYMBOL_GPL(nfs_local_probe);
215
216 /*
217 * nfs_local_open_fh - open a local filehandle
218 *
219 * Returns a pointer to a struct nfsd_file or
220 */
221 struct nfsd_file *
222 nfs_local_open_fh(struct nfs_client *clp, cons
223 struct nfs_fh *fh, const fmo
224 {
225 struct nfsd_file *localio;
226 int status;
227
228 if (!nfs_server_is_local(clp))
229 return NULL;
230 if (mode & ~(FMODE_READ | FMODE_WRITE)
231 return NULL;
232
233 localio = nfs_open_local_fh(&clp->cl_u
234 cred, fh,
235 if (IS_ERR(localio)) {
236 status = PTR_ERR(localio);
237 trace_nfs_local_open_fh(fh, mo
238 switch (status) {
239 case -ENOMEM:
240 case -ENXIO:
241 case -ENOENT:
242 /* Revalidate localio,
243 nfs_local_probe(clp);
244 }
245 return NULL;
246 }
247 return localio;
248 }
249 EXPORT_SYMBOL_GPL(nfs_local_open_fh);
250
251 static struct bio_vec *
252 nfs_bvec_alloc_and_import_pagevec(struct page
253 unsigned int npages, gfp_t fla
254 {
255 struct bio_vec *bvec, *p;
256
257 bvec = kmalloc_array(npages, sizeof(*b
258 if (bvec != NULL) {
259 for (p = bvec; npages > 0; p++
260 p->bv_page = *pagevec;
261 p->bv_len = PAGE_SIZE;
262 p->bv_offset = 0;
263 }
264 }
265 return bvec;
266 }
267
268 static void
269 nfs_local_iocb_free(struct nfs_local_kiocb *io
270 {
271 kfree(iocb->bvec);
272 kfree(iocb);
273 }
274
275 static struct nfs_local_kiocb *
276 nfs_local_iocb_alloc(struct nfs_pgio_header *h
277 struct nfsd_file *localio
278 {
279 struct nfs_local_kiocb *iocb;
280
281 iocb = kmalloc(sizeof(*iocb), flags);
282 if (iocb == NULL)
283 return NULL;
284 iocb->bvec = nfs_bvec_alloc_and_import
285 hdr->page_array.npages
286 if (iocb->bvec == NULL) {
287 kfree(iocb);
288 return NULL;
289 }
290 init_sync_kiocb(&iocb->kiocb, nfs_to->
291 iocb->kiocb.ki_pos = hdr->args.offset;
292 iocb->localio = localio;
293 iocb->hdr = hdr;
294 iocb->kiocb.ki_flags &= ~IOCB_APPEND;
295 return iocb;
296 }
297
298 static void
299 nfs_local_iter_init(struct iov_iter *i, struct
300 {
301 struct nfs_pgio_header *hdr = iocb->hd
302
303 iov_iter_bvec(i, dir, iocb->bvec, hdr-
304 hdr->args.count + hdr->a
305 if (hdr->args.pgbase != 0)
306 iov_iter_advance(i, hdr->args.
307 }
308
309 static void
310 nfs_local_hdr_release(struct nfs_pgio_header *
311 const struct rpc_call_ops *cal
312 {
313 call_ops->rpc_call_done(&hdr->task, hd
314 call_ops->rpc_release(hdr);
315 }
316
317 static void
318 nfs_local_pgio_init(struct nfs_pgio_header *hd
319 const struct rpc_call_ops *cal
320 {
321 hdr->task.tk_ops = call_ops;
322 if (!hdr->task.tk_start)
323 hdr->task.tk_start = ktime_get
324 }
325
326 static void
327 nfs_local_pgio_done(struct nfs_pgio_header *hd
328 {
329 if (status >= 0) {
330 hdr->res.count = status;
331 hdr->res.op_status = NFS4_OK;
332 hdr->task.tk_status = 0;
333 } else {
334 hdr->res.op_status = nfs4_stat
335 hdr->task.tk_status = status;
336 }
337 }
338
339 static void
340 nfs_local_pgio_release(struct nfs_local_kiocb
341 {
342 struct nfs_pgio_header *hdr = iocb->hd
343
344 nfs_to_nfsd_file_put_local(iocb->local
345 nfs_local_iocb_free(iocb);
346 nfs_local_hdr_release(hdr, hdr->task.t
347 }
348
349 static void
350 nfs_local_read_done(struct nfs_local_kiocb *io
351 {
352 struct nfs_pgio_header *hdr = iocb->hd
353 struct file *filp = iocb->kiocb.ki_fil
354
355 nfs_local_pgio_done(hdr, status);
356
357 if (hdr->res.count != hdr->args.count
358 hdr->args.offset + hdr->res.count
359 hdr->res.eof = true;
360
361 dprintk("%s: read %ld bytes eof %d.\n"
362 status > 0 ? status :
363 }
364
365 static void nfs_local_call_read(struct work_st
366 {
367 struct nfs_local_kiocb *iocb =
368 container_of(work, struct nfs_
369 struct file *filp = iocb->kiocb.ki_fil
370 const struct cred *save_cred;
371 struct iov_iter iter;
372 ssize_t status;
373
374 save_cred = override_creds(filp->f_cre
375
376 nfs_local_iter_init(&iter, iocb, READ)
377
378 status = filp->f_op->read_iter(&iocb->
379 WARN_ON_ONCE(status == -EIOCBQUEUED);
380
381 nfs_local_read_done(iocb, status);
382 nfs_local_pgio_release(iocb);
383
384 revert_creds(save_cred);
385 }
386
387 static int
388 nfs_do_local_read(struct nfs_pgio_header *hdr,
389 struct nfsd_file *localio,
390 const struct rpc_call_ops *c
391 {
392 struct nfs_local_kiocb *iocb;
393
394 dprintk("%s: vfs_read count=%u pos=%ll
395 __func__, hdr->args.count, hdr
396
397 iocb = nfs_local_iocb_alloc(hdr, local
398 if (iocb == NULL)
399 return -ENOMEM;
400
401 nfs_local_pgio_init(hdr, call_ops);
402 hdr->res.eof = false;
403
404 INIT_WORK(&iocb->work, nfs_local_call_
405 queue_work(nfslocaliod_workqueue, &ioc
406
407 return 0;
408 }
409
410 static void
411 nfs_copy_boot_verifier(struct nfs_write_verifi
412 {
413 struct nfs_client *clp = NFS_SERVER(in
414 u32 *verf = (u32 *)verifier->data;
415 int seq = 0;
416
417 do {
418 read_seqbegin_or_lock(&clp->cl
419 verf[0] = (u32)clp->cl_nfssvc_
420 verf[1] = (u32)clp->cl_nfssvc_
421 } while (need_seqretry(&clp->cl_boot_l
422 done_seqretry(&clp->cl_boot_lock, seq)
423 }
424
425 static void
426 nfs_reset_boot_verifier(struct inode *inode)
427 {
428 struct nfs_client *clp = NFS_SERVER(in
429
430 write_seqlock(&clp->cl_boot_lock);
431 ktime_get_real_ts64(&clp->cl_nfssvc_bo
432 write_sequnlock(&clp->cl_boot_lock);
433 }
434
435 static void
436 nfs_set_local_verifier(struct inode *inode,
437 struct nfs_writeverf *verf,
438 enum nfs3_stable_how how)
439 {
440 nfs_copy_boot_verifier(&verf->verifier
441 verf->committed = how;
442 }
443
444 /* Factored out from fs/nfsd/vfs.h:fh_getattr(
445 static int __vfs_getattr(struct path *p, struc
446 {
447 u32 request_mask = STATX_BASIC_STATS;
448
449 if (version == 4)
450 request_mask |= (STATX_BTIME |
451 return vfs_getattr(p, stat, request_ma
452 }
453
454 /* Copied from fs/nfsd/nfsfh.c:nfsd4_change_at
455 static u64 __nfsd4_change_attribute(const stru
456 const stru
457 {
458 u64 chattr;
459
460 if (stat->result_mask & STATX_CHANGE_C
461 chattr = stat->change_cookie;
462 if (S_ISREG(inode->i_mode) &&
463 !(stat->attributes & STATX
464 chattr += (u64)stat->c
465 chattr += stat->ctime.
466 }
467 } else {
468 chattr = time_to_chattr(&stat-
469 }
470 return chattr;
471 }
472
473 static void nfs_local_vfs_getattr(struct nfs_l
474 {
475 struct kstat stat;
476 struct file *filp = iocb->kiocb.ki_fil
477 struct nfs_pgio_header *hdr = iocb->hd
478 struct nfs_fattr *fattr = hdr->res.fat
479 int version = NFS_PROTO(hdr->inode)->v
480
481 if (unlikely(!fattr) || __vfs_getattr(
482 return;
483
484 fattr->valid = (NFS_ATTR_FATTR_FILEID
485 NFS_ATTR_FATTR_CHANGE
486 NFS_ATTR_FATTR_SIZE |
487 NFS_ATTR_FATTR_ATIME |
488 NFS_ATTR_FATTR_MTIME |
489 NFS_ATTR_FATTR_CTIME |
490 NFS_ATTR_FATTR_SPACE_U
491
492 fattr->fileid = stat.ino;
493 fattr->size = stat.size;
494 fattr->atime = stat.atime;
495 fattr->mtime = stat.mtime;
496 fattr->ctime = stat.ctime;
497 if (version == 4) {
498 fattr->change_attr =
499 __nfsd4_change_attribu
500 } else
501 fattr->change_attr = nfs_times
502 fattr->du.nfs3.used = stat.blocks << 9
503 }
504
505 static void
506 nfs_local_write_done(struct nfs_local_kiocb *i
507 {
508 struct nfs_pgio_header *hdr = iocb->hd
509 struct inode *inode = hdr->inode;
510
511 dprintk("%s: wrote %ld bytes.\n", __fu
512
513 /* Handle short writes as if they are
514 if (status > 0 && status < hdr->args.c
515 hdr->mds_offset += status;
516 hdr->args.offset += status;
517 hdr->args.pgbase += status;
518 hdr->args.count -= status;
519 nfs_set_pgio_error(hdr, -ENOSP
520 status = -ENOSPC;
521 }
522 if (status < 0)
523 nfs_reset_boot_verifier(inode)
524 else if (nfs_should_remove_suid(inode)
525 /* Deal with the suid/sgid bit
526 spin_lock(&inode->i_lock);
527 nfs_set_cache_invalid(inode, N
528 spin_unlock(&inode->i_lock);
529 }
530 nfs_local_pgio_done(hdr, status);
531 }
532
533 static void nfs_local_call_write(struct work_s
534 {
535 struct nfs_local_kiocb *iocb =
536 container_of(work, struct nfs_
537 struct file *filp = iocb->kiocb.ki_fil
538 unsigned long old_flags = current->fla
539 const struct cred *save_cred;
540 struct iov_iter iter;
541 ssize_t status;
542
543 current->flags |= PF_LOCAL_THROTTLE |
544 save_cred = override_creds(filp->f_cre
545
546 nfs_local_iter_init(&iter, iocb, WRITE
547
548 file_start_write(filp);
549 status = filp->f_op->write_iter(&iocb-
550 file_end_write(filp);
551 WARN_ON_ONCE(status == -EIOCBQUEUED);
552
553 nfs_local_write_done(iocb, status);
554 nfs_local_vfs_getattr(iocb);
555 nfs_local_pgio_release(iocb);
556
557 revert_creds(save_cred);
558 current->flags = old_flags;
559 }
560
561 static int
562 nfs_do_local_write(struct nfs_pgio_header *hdr
563 struct nfsd_file *localio,
564 const struct rpc_call_ops *
565 {
566 struct nfs_local_kiocb *iocb;
567
568 dprintk("%s: vfs_write count=%u pos=%l
569 __func__, hdr->args.count, hdr
570 (hdr->args.stable == NFS_UNSTA
571
572 iocb = nfs_local_iocb_alloc(hdr, local
573 if (iocb == NULL)
574 return -ENOMEM;
575
576 switch (hdr->args.stable) {
577 default:
578 break;
579 case NFS_DATA_SYNC:
580 iocb->kiocb.ki_flags |= IOCB_D
581 break;
582 case NFS_FILE_SYNC:
583 iocb->kiocb.ki_flags |= IOCB_D
584 }
585 nfs_local_pgio_init(hdr, call_ops);
586
587 nfs_set_local_verifier(hdr->inode, hdr
588
589 INIT_WORK(&iocb->work, nfs_local_call_
590 queue_work(nfslocaliod_workqueue, &ioc
591
592 return 0;
593 }
594
595 int nfs_local_doio(struct nfs_client *clp, str
596 struct nfs_pgio_header *hdr
597 const struct rpc_call_ops *
598 {
599 int status = 0;
600 struct file *filp = nfs_to->nfsd_file_
601
602 if (!hdr->args.count)
603 return 0;
604 /* Don't support filesystems without r
605 if (!filp->f_op->read_iter || !filp->f
606 nfs_local_disable(clp);
607 status = -EAGAIN;
608 goto out;
609 }
610
611 switch (hdr->rw_mode) {
612 case FMODE_READ:
613 status = nfs_do_local_read(hdr
614 break;
615 case FMODE_WRITE:
616 status = nfs_do_local_write(hd
617 break;
618 default:
619 dprintk("%s: invalid mode: %d\
620 hdr->rw_mode);
621 status = -EINVAL;
622 }
623 out:
624 if (status != 0) {
625 nfs_to_nfsd_file_put_local(loc
626 hdr->task.tk_status = status;
627 nfs_local_hdr_release(hdr, cal
628 }
629 return status;
630 }
631
632 static void
633 nfs_local_init_commit(struct nfs_commit_data *
634 const struct rpc_call_ops *cal
635 {
636 data->task.tk_ops = call_ops;
637 }
638
639 static int
640 nfs_local_run_commit(struct file *filp, struct
641 {
642 loff_t start = data->args.offset;
643 loff_t end = LLONG_MAX;
644
645 if (data->args.count > 0) {
646 end = start + data->args.count
647 if (end < start)
648 end = LLONG_MAX;
649 }
650
651 dprintk("%s: commit %llu - %llu\n", __
652 return vfs_fsync_range(filp, start, en
653 }
654
655 static void
656 nfs_local_commit_done(struct nfs_commit_data *
657 {
658 if (status >= 0) {
659 nfs_set_local_verifier(data->i
660 data->res.verf
661 NFS_FILE_SYNC)
662 data->res.op_status = NFS4_OK;
663 data->task.tk_status = 0;
664 } else {
665 nfs_reset_boot_verifier(data->
666 data->res.op_status = nfs4_sta
667 data->task.tk_status = status;
668 }
669 }
670
671 static void
672 nfs_local_release_commit_data(struct nfsd_file
673 struct nfs_commit_data *data,
674 const struct rpc_call_ops *cal
675 {
676 nfs_to_nfsd_file_put_local(localio);
677 call_ops->rpc_call_done(&data->task, d
678 call_ops->rpc_release(data);
679 }
680
681 static struct nfs_local_fsync_ctx *
682 nfs_local_fsync_ctx_alloc(struct nfs_commit_da
683 struct nfsd_file *lo
684 {
685 struct nfs_local_fsync_ctx *ctx = kmal
686
687 if (ctx != NULL) {
688 ctx->localio = localio;
689 ctx->data = data;
690 INIT_WORK(&ctx->work, nfs_loca
691 kref_init(&ctx->kref);
692 ctx->done = NULL;
693 }
694 return ctx;
695 }
696
697 static void
698 nfs_local_fsync_ctx_kref_free(struct kref *kre
699 {
700 kfree(container_of(kref, struct nfs_lo
701 }
702
703 static void
704 nfs_local_fsync_ctx_put(struct nfs_local_fsync
705 {
706 kref_put(&ctx->kref, nfs_local_fsync_c
707 }
708
709 static void
710 nfs_local_fsync_ctx_free(struct nfs_local_fsyn
711 {
712 nfs_local_release_commit_data(ctx->loc
713 ctx->dat
714 nfs_local_fsync_ctx_put(ctx);
715 }
716
717 static void
718 nfs_local_fsync_work(struct work_struct *work)
719 {
720 struct nfs_local_fsync_ctx *ctx;
721 int status;
722
723 ctx = container_of(work, struct nfs_lo
724
725 status = nfs_local_run_commit(nfs_to->
726 ctx->dat
727 nfs_local_commit_done(ctx->data, statu
728 if (ctx->done != NULL)
729 complete(ctx->done);
730 nfs_local_fsync_ctx_free(ctx);
731 }
732
733 int nfs_local_commit(struct nfsd_file *localio
734 struct nfs_commit_data *d
735 const struct rpc_call_ops
736 {
737 struct nfs_local_fsync_ctx *ctx;
738
739 ctx = nfs_local_fsync_ctx_alloc(data,
740 if (!ctx) {
741 nfs_local_commit_done(data, -E
742 nfs_local_release_commit_data(
743 return -ENOMEM;
744 }
745
746 nfs_local_init_commit(data, call_ops);
747 kref_get(&ctx->kref);
748 if (how & FLUSH_SYNC) {
749 DECLARE_COMPLETION_ONSTACK(don
750 ctx->done = &done;
751 queue_work(nfsiod_workqueue, &
752 wait_for_completion(&done);
753 } else
754 queue_work(nfsiod_workqueue, &
755 nfs_local_fsync_ctx_put(ctx);
756 return 0;
757 }
758
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.