1 // SPDX-License-Identifier: GPL-2.0-only 1 // SPDX-License-Identifier: GPL-2.0-only
2 /* 2 /*
3 * NFS client support for local clients to byp 3 * NFS client support for local clients to bypass network stack
4 * 4 *
5 * Copyright (C) 2014 Weston Andros Adamson <d 5 * Copyright (C) 2014 Weston Andros Adamson <dros@primarydata.com>
6 * Copyright (C) 2019 Trond Myklebust <trond.m 6 * Copyright (C) 2019 Trond Myklebust <trond.myklebust@hammerspace.com>
7 * Copyright (C) 2024 Mike Snitzer <snitzer@ha 7 * Copyright (C) 2024 Mike Snitzer <snitzer@hammerspace.com>
8 * Copyright (C) 2024 NeilBrown <neilb@suse.de 8 * Copyright (C) 2024 NeilBrown <neilb@suse.de>
9 */ 9 */
10 10
11 #include <linux/module.h> 11 #include <linux/module.h>
12 #include <linux/errno.h> 12 #include <linux/errno.h>
13 #include <linux/vfs.h> 13 #include <linux/vfs.h>
14 #include <linux/file.h> 14 #include <linux/file.h>
15 #include <linux/inet.h> 15 #include <linux/inet.h>
16 #include <linux/sunrpc/addr.h> 16 #include <linux/sunrpc/addr.h>
17 #include <linux/inetdevice.h> 17 #include <linux/inetdevice.h>
18 #include <net/addrconf.h> 18 #include <net/addrconf.h>
19 #include <linux/nfs_common.h> 19 #include <linux/nfs_common.h>
20 #include <linux/nfslocalio.h> 20 #include <linux/nfslocalio.h>
21 #include <linux/bvec.h> 21 #include <linux/bvec.h>
22 22
23 #include <linux/nfs.h> 23 #include <linux/nfs.h>
24 #include <linux/nfs_fs.h> 24 #include <linux/nfs_fs.h>
25 #include <linux/nfs_xdr.h> 25 #include <linux/nfs_xdr.h>
26 26
27 #include "internal.h" 27 #include "internal.h"
28 #include "pnfs.h" 28 #include "pnfs.h"
29 #include "nfstrace.h" 29 #include "nfstrace.h"
30 30
31 #define NFSDBG_FACILITY NFSDBG_VFS 31 #define NFSDBG_FACILITY NFSDBG_VFS
32 32
33 struct nfs_local_kiocb { 33 struct nfs_local_kiocb {
34 struct kiocb kiocb; 34 struct kiocb kiocb;
35 struct bio_vec *bvec; 35 struct bio_vec *bvec;
36 struct nfs_pgio_header *hdr; 36 struct nfs_pgio_header *hdr;
37 struct work_struct work; 37 struct work_struct work;
38 struct nfsd_file *localio; 38 struct nfsd_file *localio;
39 }; 39 };
40 40
41 struct nfs_local_fsync_ctx { 41 struct nfs_local_fsync_ctx {
42 struct nfsd_file *localio; 42 struct nfsd_file *localio;
43 struct nfs_commit_data *data; 43 struct nfs_commit_data *data;
44 struct work_struct work; 44 struct work_struct work;
45 struct kref kref; 45 struct kref kref;
46 struct completion *done; 46 struct completion *done;
47 }; 47 };
48 static void nfs_local_fsync_work(struct work_s 48 static void nfs_local_fsync_work(struct work_struct *work);
49 49
50 static bool localio_enabled __read_mostly = tr 50 static bool localio_enabled __read_mostly = true;
51 module_param(localio_enabled, bool, 0644); 51 module_param(localio_enabled, bool, 0644);
52 52
53 static inline bool nfs_client_is_local(const s 53 static inline bool nfs_client_is_local(const struct nfs_client *clp)
54 { 54 {
55 return !!test_bit(NFS_CS_LOCAL_IO, &cl 55 return !!test_bit(NFS_CS_LOCAL_IO, &clp->cl_flags);
56 } 56 }
57 57
58 bool nfs_server_is_local(const struct nfs_clie 58 bool nfs_server_is_local(const struct nfs_client *clp)
59 { 59 {
60 return nfs_client_is_local(clp) && loc 60 return nfs_client_is_local(clp) && localio_enabled;
61 } 61 }
62 EXPORT_SYMBOL_GPL(nfs_server_is_local); 62 EXPORT_SYMBOL_GPL(nfs_server_is_local);
63 63
64 /* 64 /*
65 * UUID_IS_LOCAL XDR functions 65 * UUID_IS_LOCAL XDR functions
66 */ 66 */
67 67
68 static void localio_xdr_enc_uuidargs(struct rp 68 static void localio_xdr_enc_uuidargs(struct rpc_rqst *req,
69 struct xd 69 struct xdr_stream *xdr,
70 const voi 70 const void *data)
71 { 71 {
72 const u8 *uuid = data; 72 const u8 *uuid = data;
73 73
74 encode_opaque_fixed(xdr, uuid, UUID_SI 74 encode_opaque_fixed(xdr, uuid, UUID_SIZE);
75 } 75 }
76 76
77 static int localio_xdr_dec_uuidres(struct rpc_ 77 static int localio_xdr_dec_uuidres(struct rpc_rqst *req,
78 struct xdr_ 78 struct xdr_stream *xdr,
79 void *resul 79 void *result)
80 { 80 {
81 /* void return */ 81 /* void return */
82 return 0; 82 return 0;
83 } 83 }
84 84
85 static const struct rpc_procinfo nfs_localio_p 85 static const struct rpc_procinfo nfs_localio_procedures[] = {
86 [LOCALIOPROC_UUID_IS_LOCAL] = { 86 [LOCALIOPROC_UUID_IS_LOCAL] = {
87 .p_proc = LOCALIOPROC_UUID_IS_ 87 .p_proc = LOCALIOPROC_UUID_IS_LOCAL,
88 .p_encode = localio_xdr_enc_uu 88 .p_encode = localio_xdr_enc_uuidargs,
89 .p_decode = localio_xdr_dec_uu 89 .p_decode = localio_xdr_dec_uuidres,
90 .p_arglen = XDR_QUADLEN(UUID_S 90 .p_arglen = XDR_QUADLEN(UUID_SIZE),
91 .p_replen = 0, 91 .p_replen = 0,
92 .p_statidx = LOCALIOPROC_UUID_ 92 .p_statidx = LOCALIOPROC_UUID_IS_LOCAL,
93 .p_name = "UUID_IS_LOCAL", 93 .p_name = "UUID_IS_LOCAL",
94 }, 94 },
95 }; 95 };
96 96
97 static unsigned int nfs_localio_counts[ARRAY_S 97 static unsigned int nfs_localio_counts[ARRAY_SIZE(nfs_localio_procedures)];
98 static const struct rpc_version nfslocalio_ver 98 static const struct rpc_version nfslocalio_version1 = {
99 .number = 1, 99 .number = 1,
100 .nrprocs = ARRAY_SIZE(n 100 .nrprocs = ARRAY_SIZE(nfs_localio_procedures),
101 .procs = nfs_localio_ 101 .procs = nfs_localio_procedures,
102 .counts = nfs_localio_ 102 .counts = nfs_localio_counts,
103 }; 103 };
104 104
105 static const struct rpc_version *nfslocalio_ve 105 static const struct rpc_version *nfslocalio_version[] = {
106 [1] = &nfslocalio_ 106 [1] = &nfslocalio_version1,
107 }; 107 };
108 108
109 extern const struct rpc_program nfslocalio_pro 109 extern const struct rpc_program nfslocalio_program;
110 static struct rpc_stat nfslocalio_rpc 110 static struct rpc_stat nfslocalio_rpcstat = { &nfslocalio_program };
111 111
112 const struct rpc_program nfslocalio_program = 112 const struct rpc_program nfslocalio_program = {
113 .name = "nfslocalio" 113 .name = "nfslocalio",
114 .number = NFS_LOCALIO_ 114 .number = NFS_LOCALIO_PROGRAM,
115 .nrvers = ARRAY_SIZE(n 115 .nrvers = ARRAY_SIZE(nfslocalio_version),
116 .version = nfslocalio_v 116 .version = nfslocalio_version,
117 .stats = &nfslocalio_ 117 .stats = &nfslocalio_rpcstat,
118 }; 118 };
119 119
120 /* 120 /*
121 * nfs_local_enable - enable local i/o for an 121 * nfs_local_enable - enable local i/o for an nfs_client
122 */ 122 */
123 static void nfs_local_enable(struct nfs_client 123 static void nfs_local_enable(struct nfs_client *clp)
124 { 124 {
125 spin_lock(&clp->cl_localio_lock); 125 spin_lock(&clp->cl_localio_lock);
126 set_bit(NFS_CS_LOCAL_IO, &clp->cl_flag 126 set_bit(NFS_CS_LOCAL_IO, &clp->cl_flags);
127 trace_nfs_local_enable(clp); 127 trace_nfs_local_enable(clp);
128 spin_unlock(&clp->cl_localio_lock); 128 spin_unlock(&clp->cl_localio_lock);
129 } 129 }
130 130
131 /* 131 /*
132 * nfs_local_disable - disable local i/o for a 132 * nfs_local_disable - disable local i/o for an nfs_client
133 */ 133 */
134 void nfs_local_disable(struct nfs_client *clp) 134 void nfs_local_disable(struct nfs_client *clp)
135 { 135 {
136 spin_lock(&clp->cl_localio_lock); 136 spin_lock(&clp->cl_localio_lock);
137 if (test_and_clear_bit(NFS_CS_LOCAL_IO 137 if (test_and_clear_bit(NFS_CS_LOCAL_IO, &clp->cl_flags)) {
138 trace_nfs_local_disable(clp); 138 trace_nfs_local_disable(clp);
139 nfs_uuid_invalidate_one_client 139 nfs_uuid_invalidate_one_client(&clp->cl_uuid);
140 } 140 }
141 spin_unlock(&clp->cl_localio_lock); 141 spin_unlock(&clp->cl_localio_lock);
142 } 142 }
143 143
144 /* 144 /*
145 * nfs_init_localioclient - Initialise an NFS 145 * nfs_init_localioclient - Initialise an NFS localio client connection
146 */ 146 */
147 static struct rpc_clnt *nfs_init_localioclient 147 static struct rpc_clnt *nfs_init_localioclient(struct nfs_client *clp)
148 { 148 {
149 struct rpc_clnt *rpcclient_localio; 149 struct rpc_clnt *rpcclient_localio;
150 150
151 rpcclient_localio = rpc_bind_new_progr 151 rpcclient_localio = rpc_bind_new_program(clp->cl_rpcclient,
152 152 &nfslocalio_program, 1);
153 153
154 dprintk_rcu("%s: server (%s) %s NFS LO 154 dprintk_rcu("%s: server (%s) %s NFS LOCALIO.\n",
155 __func__, rpc_peeraddr2str(clp 155 __func__, rpc_peeraddr2str(clp->cl_rpcclient, RPC_DISPLAY_ADDR),
156 (IS_ERR(rpcclient_localio) ? " 156 (IS_ERR(rpcclient_localio) ? "does not support" : "supports"));
157 157
158 return rpcclient_localio; 158 return rpcclient_localio;
159 } 159 }
160 160
161 static bool nfs_server_uuid_is_local(struct nf 161 static bool nfs_server_uuid_is_local(struct nfs_client *clp)
162 { 162 {
163 u8 uuid[UUID_SIZE]; 163 u8 uuid[UUID_SIZE];
164 struct rpc_message msg = { 164 struct rpc_message msg = {
165 .rpc_argp = &uuid, 165 .rpc_argp = &uuid,
166 }; 166 };
167 struct rpc_clnt *rpcclient_localio; 167 struct rpc_clnt *rpcclient_localio;
168 int status; 168 int status;
169 169
170 rpcclient_localio = nfs_init_localiocl 170 rpcclient_localio = nfs_init_localioclient(clp);
171 if (IS_ERR(rpcclient_localio)) 171 if (IS_ERR(rpcclient_localio))
172 return false; 172 return false;
173 173
174 export_uuid(uuid, &clp->cl_uuid.uuid); 174 export_uuid(uuid, &clp->cl_uuid.uuid);
175 175
176 msg.rpc_proc = &nfs_localio_procedures 176 msg.rpc_proc = &nfs_localio_procedures[LOCALIOPROC_UUID_IS_LOCAL];
177 status = rpc_call_sync(rpcclient_local 177 status = rpc_call_sync(rpcclient_localio, &msg, 0);
178 dprintk("%s: NFS reply UUID_IS_LOCAL: 178 dprintk("%s: NFS reply UUID_IS_LOCAL: status=%d\n",
179 __func__, status); 179 __func__, status);
180 rpc_shutdown_client(rpcclient_localio) 180 rpc_shutdown_client(rpcclient_localio);
181 181
182 /* Server is only local if it initiali 182 /* Server is only local if it initialized required struct members */
183 if (status || !clp->cl_uuid.net || !cl 183 if (status || !clp->cl_uuid.net || !clp->cl_uuid.dom)
184 return false; 184 return false;
185 185
186 return true; 186 return true;
187 } 187 }
188 188
189 /* 189 /*
190 * nfs_local_probe - probe local i/o support f 190 * nfs_local_probe - probe local i/o support for an nfs_server and nfs_client
191 * - called after alloc_client and init_client 191 * - called after alloc_client and init_client (so cl_rpcclient exists)
192 * - this function is idempotent, it can be ca 192 * - this function is idempotent, it can be called for old or new clients
193 */ 193 */
194 void nfs_local_probe(struct nfs_client *clp) 194 void nfs_local_probe(struct nfs_client *clp)
195 { 195 {
196 /* Disallow localio if disabled via sy 196 /* Disallow localio if disabled via sysfs or AUTH_SYS isn't used */
197 if (!localio_enabled || 197 if (!localio_enabled ||
198 clp->cl_rpcclient->cl_auth->au_fla 198 clp->cl_rpcclient->cl_auth->au_flavor != RPC_AUTH_UNIX) {
199 nfs_local_disable(clp); 199 nfs_local_disable(clp);
200 return; 200 return;
201 } 201 }
202 202
203 if (nfs_client_is_local(clp)) { 203 if (nfs_client_is_local(clp)) {
204 /* If already enabled, disable 204 /* If already enabled, disable and re-enable */
205 nfs_local_disable(clp); 205 nfs_local_disable(clp);
206 } 206 }
207 207
208 if (!nfs_uuid_begin(&clp->cl_uuid)) 208 if (!nfs_uuid_begin(&clp->cl_uuid))
209 return; 209 return;
210 if (nfs_server_uuid_is_local(clp)) 210 if (nfs_server_uuid_is_local(clp))
211 nfs_local_enable(clp); 211 nfs_local_enable(clp);
212 nfs_uuid_end(&clp->cl_uuid); 212 nfs_uuid_end(&clp->cl_uuid);
213 } 213 }
214 EXPORT_SYMBOL_GPL(nfs_local_probe); 214 EXPORT_SYMBOL_GPL(nfs_local_probe);
215 215
216 /* 216 /*
217 * nfs_local_open_fh - open a local filehandle 217 * nfs_local_open_fh - open a local filehandle in terms of nfsd_file
218 * 218 *
219 * Returns a pointer to a struct nfsd_file or 219 * Returns a pointer to a struct nfsd_file or NULL
220 */ 220 */
221 struct nfsd_file * 221 struct nfsd_file *
222 nfs_local_open_fh(struct nfs_client *clp, cons 222 nfs_local_open_fh(struct nfs_client *clp, const struct cred *cred,
223 struct nfs_fh *fh, const fmo 223 struct nfs_fh *fh, const fmode_t mode)
224 { 224 {
225 struct nfsd_file *localio; 225 struct nfsd_file *localio;
226 int status; 226 int status;
227 227
228 if (!nfs_server_is_local(clp)) 228 if (!nfs_server_is_local(clp))
229 return NULL; 229 return NULL;
230 if (mode & ~(FMODE_READ | FMODE_WRITE) 230 if (mode & ~(FMODE_READ | FMODE_WRITE))
231 return NULL; 231 return NULL;
232 232
233 localio = nfs_open_local_fh(&clp->cl_u 233 localio = nfs_open_local_fh(&clp->cl_uuid, clp->cl_rpcclient,
234 cred, fh, 234 cred, fh, mode);
235 if (IS_ERR(localio)) { 235 if (IS_ERR(localio)) {
236 status = PTR_ERR(localio); 236 status = PTR_ERR(localio);
237 trace_nfs_local_open_fh(fh, mo 237 trace_nfs_local_open_fh(fh, mode, status);
238 switch (status) { 238 switch (status) {
239 case -ENOMEM: 239 case -ENOMEM:
240 case -ENXIO: 240 case -ENXIO:
241 case -ENOENT: 241 case -ENOENT:
242 /* Revalidate localio, 242 /* Revalidate localio, will disable if unsupported */
243 nfs_local_probe(clp); 243 nfs_local_probe(clp);
244 } 244 }
245 return NULL; 245 return NULL;
246 } 246 }
247 return localio; 247 return localio;
248 } 248 }
249 EXPORT_SYMBOL_GPL(nfs_local_open_fh); 249 EXPORT_SYMBOL_GPL(nfs_local_open_fh);
250 250
251 static struct bio_vec * 251 static struct bio_vec *
252 nfs_bvec_alloc_and_import_pagevec(struct page 252 nfs_bvec_alloc_and_import_pagevec(struct page **pagevec,
253 unsigned int npages, gfp_t fla 253 unsigned int npages, gfp_t flags)
254 { 254 {
255 struct bio_vec *bvec, *p; 255 struct bio_vec *bvec, *p;
256 256
257 bvec = kmalloc_array(npages, sizeof(*b 257 bvec = kmalloc_array(npages, sizeof(*bvec), flags);
258 if (bvec != NULL) { 258 if (bvec != NULL) {
259 for (p = bvec; npages > 0; p++ 259 for (p = bvec; npages > 0; p++, pagevec++, npages--) {
260 p->bv_page = *pagevec; 260 p->bv_page = *pagevec;
261 p->bv_len = PAGE_SIZE; 261 p->bv_len = PAGE_SIZE;
262 p->bv_offset = 0; 262 p->bv_offset = 0;
263 } 263 }
264 } 264 }
265 return bvec; 265 return bvec;
266 } 266 }
267 267
268 static void 268 static void
269 nfs_local_iocb_free(struct nfs_local_kiocb *io 269 nfs_local_iocb_free(struct nfs_local_kiocb *iocb)
270 { 270 {
271 kfree(iocb->bvec); 271 kfree(iocb->bvec);
272 kfree(iocb); 272 kfree(iocb);
273 } 273 }
274 274
275 static struct nfs_local_kiocb * 275 static struct nfs_local_kiocb *
276 nfs_local_iocb_alloc(struct nfs_pgio_header *h 276 nfs_local_iocb_alloc(struct nfs_pgio_header *hdr,
277 struct nfsd_file *localio 277 struct nfsd_file *localio, gfp_t flags)
278 { 278 {
279 struct nfs_local_kiocb *iocb; 279 struct nfs_local_kiocb *iocb;
280 280
281 iocb = kmalloc(sizeof(*iocb), flags); 281 iocb = kmalloc(sizeof(*iocb), flags);
282 if (iocb == NULL) 282 if (iocb == NULL)
283 return NULL; 283 return NULL;
284 iocb->bvec = nfs_bvec_alloc_and_import 284 iocb->bvec = nfs_bvec_alloc_and_import_pagevec(hdr->page_array.pagevec,
285 hdr->page_array.npages 285 hdr->page_array.npages, flags);
286 if (iocb->bvec == NULL) { 286 if (iocb->bvec == NULL) {
287 kfree(iocb); 287 kfree(iocb);
288 return NULL; 288 return NULL;
289 } 289 }
290 init_sync_kiocb(&iocb->kiocb, nfs_to-> 290 init_sync_kiocb(&iocb->kiocb, nfs_to->nfsd_file_file(localio));
291 iocb->kiocb.ki_pos = hdr->args.offset; 291 iocb->kiocb.ki_pos = hdr->args.offset;
292 iocb->localio = localio; 292 iocb->localio = localio;
293 iocb->hdr = hdr; 293 iocb->hdr = hdr;
294 iocb->kiocb.ki_flags &= ~IOCB_APPEND; 294 iocb->kiocb.ki_flags &= ~IOCB_APPEND;
295 return iocb; 295 return iocb;
296 } 296 }
297 297
298 static void 298 static void
299 nfs_local_iter_init(struct iov_iter *i, struct 299 nfs_local_iter_init(struct iov_iter *i, struct nfs_local_kiocb *iocb, int dir)
300 { 300 {
301 struct nfs_pgio_header *hdr = iocb->hd 301 struct nfs_pgio_header *hdr = iocb->hdr;
302 302
303 iov_iter_bvec(i, dir, iocb->bvec, hdr- 303 iov_iter_bvec(i, dir, iocb->bvec, hdr->page_array.npages,
304 hdr->args.count + hdr->a 304 hdr->args.count + hdr->args.pgbase);
305 if (hdr->args.pgbase != 0) 305 if (hdr->args.pgbase != 0)
306 iov_iter_advance(i, hdr->args. 306 iov_iter_advance(i, hdr->args.pgbase);
307 } 307 }
308 308
309 static void 309 static void
310 nfs_local_hdr_release(struct nfs_pgio_header * 310 nfs_local_hdr_release(struct nfs_pgio_header *hdr,
311 const struct rpc_call_ops *cal 311 const struct rpc_call_ops *call_ops)
312 { 312 {
313 call_ops->rpc_call_done(&hdr->task, hd 313 call_ops->rpc_call_done(&hdr->task, hdr);
314 call_ops->rpc_release(hdr); 314 call_ops->rpc_release(hdr);
315 } 315 }
316 316
317 static void 317 static void
318 nfs_local_pgio_init(struct nfs_pgio_header *hd 318 nfs_local_pgio_init(struct nfs_pgio_header *hdr,
319 const struct rpc_call_ops *cal 319 const struct rpc_call_ops *call_ops)
320 { 320 {
321 hdr->task.tk_ops = call_ops; 321 hdr->task.tk_ops = call_ops;
322 if (!hdr->task.tk_start) 322 if (!hdr->task.tk_start)
323 hdr->task.tk_start = ktime_get 323 hdr->task.tk_start = ktime_get();
324 } 324 }
325 325
326 static void 326 static void
327 nfs_local_pgio_done(struct nfs_pgio_header *hd 327 nfs_local_pgio_done(struct nfs_pgio_header *hdr, long status)
328 { 328 {
329 if (status >= 0) { 329 if (status >= 0) {
330 hdr->res.count = status; 330 hdr->res.count = status;
331 hdr->res.op_status = NFS4_OK; 331 hdr->res.op_status = NFS4_OK;
332 hdr->task.tk_status = 0; 332 hdr->task.tk_status = 0;
333 } else { 333 } else {
334 hdr->res.op_status = nfs4_stat 334 hdr->res.op_status = nfs4_stat_to_errno(status);
335 hdr->task.tk_status = status; 335 hdr->task.tk_status = status;
336 } 336 }
337 } 337 }
338 338
339 static void 339 static void
340 nfs_local_pgio_release(struct nfs_local_kiocb 340 nfs_local_pgio_release(struct nfs_local_kiocb *iocb)
341 { 341 {
342 struct nfs_pgio_header *hdr = iocb->hd 342 struct nfs_pgio_header *hdr = iocb->hdr;
343 343
344 nfs_to_nfsd_file_put_local(iocb->local 344 nfs_to_nfsd_file_put_local(iocb->localio);
345 nfs_local_iocb_free(iocb); 345 nfs_local_iocb_free(iocb);
346 nfs_local_hdr_release(hdr, hdr->task.t 346 nfs_local_hdr_release(hdr, hdr->task.tk_ops);
347 } 347 }
348 348
349 static void 349 static void
350 nfs_local_read_done(struct nfs_local_kiocb *io 350 nfs_local_read_done(struct nfs_local_kiocb *iocb, long status)
351 { 351 {
352 struct nfs_pgio_header *hdr = iocb->hd 352 struct nfs_pgio_header *hdr = iocb->hdr;
353 struct file *filp = iocb->kiocb.ki_fil 353 struct file *filp = iocb->kiocb.ki_filp;
354 354
355 nfs_local_pgio_done(hdr, status); 355 nfs_local_pgio_done(hdr, status);
356 356
357 if (hdr->res.count != hdr->args.count 357 if (hdr->res.count != hdr->args.count ||
358 hdr->args.offset + hdr->res.count 358 hdr->args.offset + hdr->res.count >= i_size_read(file_inode(filp)))
359 hdr->res.eof = true; 359 hdr->res.eof = true;
360 360
361 dprintk("%s: read %ld bytes eof %d.\n" 361 dprintk("%s: read %ld bytes eof %d.\n", __func__,
362 status > 0 ? status : 362 status > 0 ? status : 0, hdr->res.eof);
363 } 363 }
364 364
365 static void nfs_local_call_read(struct work_st 365 static void nfs_local_call_read(struct work_struct *work)
366 { 366 {
367 struct nfs_local_kiocb *iocb = 367 struct nfs_local_kiocb *iocb =
368 container_of(work, struct nfs_ 368 container_of(work, struct nfs_local_kiocb, work);
369 struct file *filp = iocb->kiocb.ki_fil 369 struct file *filp = iocb->kiocb.ki_filp;
370 const struct cred *save_cred; 370 const struct cred *save_cred;
371 struct iov_iter iter; 371 struct iov_iter iter;
372 ssize_t status; 372 ssize_t status;
373 373
374 save_cred = override_creds(filp->f_cre 374 save_cred = override_creds(filp->f_cred);
375 375
376 nfs_local_iter_init(&iter, iocb, READ) 376 nfs_local_iter_init(&iter, iocb, READ);
377 377
378 status = filp->f_op->read_iter(&iocb-> 378 status = filp->f_op->read_iter(&iocb->kiocb, &iter);
379 WARN_ON_ONCE(status == -EIOCBQUEUED); 379 WARN_ON_ONCE(status == -EIOCBQUEUED);
380 380
381 nfs_local_read_done(iocb, status); 381 nfs_local_read_done(iocb, status);
382 nfs_local_pgio_release(iocb); 382 nfs_local_pgio_release(iocb);
383 383
384 revert_creds(save_cred); 384 revert_creds(save_cred);
385 } 385 }
386 386
387 static int 387 static int
388 nfs_do_local_read(struct nfs_pgio_header *hdr, 388 nfs_do_local_read(struct nfs_pgio_header *hdr,
389 struct nfsd_file *localio, 389 struct nfsd_file *localio,
390 const struct rpc_call_ops *c 390 const struct rpc_call_ops *call_ops)
391 { 391 {
392 struct nfs_local_kiocb *iocb; 392 struct nfs_local_kiocb *iocb;
393 393
394 dprintk("%s: vfs_read count=%u pos=%ll 394 dprintk("%s: vfs_read count=%u pos=%llu\n",
395 __func__, hdr->args.count, hdr 395 __func__, hdr->args.count, hdr->args.offset);
396 396
397 iocb = nfs_local_iocb_alloc(hdr, local 397 iocb = nfs_local_iocb_alloc(hdr, localio, GFP_KERNEL);
398 if (iocb == NULL) 398 if (iocb == NULL)
399 return -ENOMEM; 399 return -ENOMEM;
400 400
401 nfs_local_pgio_init(hdr, call_ops); 401 nfs_local_pgio_init(hdr, call_ops);
402 hdr->res.eof = false; 402 hdr->res.eof = false;
403 403
404 INIT_WORK(&iocb->work, nfs_local_call_ 404 INIT_WORK(&iocb->work, nfs_local_call_read);
405 queue_work(nfslocaliod_workqueue, &ioc 405 queue_work(nfslocaliod_workqueue, &iocb->work);
406 406
407 return 0; 407 return 0;
408 } 408 }
409 409
410 static void 410 static void
411 nfs_copy_boot_verifier(struct nfs_write_verifi 411 nfs_copy_boot_verifier(struct nfs_write_verifier *verifier, struct inode *inode)
412 { 412 {
413 struct nfs_client *clp = NFS_SERVER(in 413 struct nfs_client *clp = NFS_SERVER(inode)->nfs_client;
414 u32 *verf = (u32 *)verifier->data; 414 u32 *verf = (u32 *)verifier->data;
415 int seq = 0; 415 int seq = 0;
416 416
417 do { 417 do {
418 read_seqbegin_or_lock(&clp->cl 418 read_seqbegin_or_lock(&clp->cl_boot_lock, &seq);
419 verf[0] = (u32)clp->cl_nfssvc_ 419 verf[0] = (u32)clp->cl_nfssvc_boot.tv_sec;
420 verf[1] = (u32)clp->cl_nfssvc_ 420 verf[1] = (u32)clp->cl_nfssvc_boot.tv_nsec;
421 } while (need_seqretry(&clp->cl_boot_l 421 } while (need_seqretry(&clp->cl_boot_lock, seq));
422 done_seqretry(&clp->cl_boot_lock, seq) 422 done_seqretry(&clp->cl_boot_lock, seq);
423 } 423 }
424 424
425 static void 425 static void
426 nfs_reset_boot_verifier(struct inode *inode) 426 nfs_reset_boot_verifier(struct inode *inode)
427 { 427 {
428 struct nfs_client *clp = NFS_SERVER(in 428 struct nfs_client *clp = NFS_SERVER(inode)->nfs_client;
429 429
430 write_seqlock(&clp->cl_boot_lock); 430 write_seqlock(&clp->cl_boot_lock);
431 ktime_get_real_ts64(&clp->cl_nfssvc_bo 431 ktime_get_real_ts64(&clp->cl_nfssvc_boot);
432 write_sequnlock(&clp->cl_boot_lock); 432 write_sequnlock(&clp->cl_boot_lock);
433 } 433 }
434 434
435 static void 435 static void
436 nfs_set_local_verifier(struct inode *inode, 436 nfs_set_local_verifier(struct inode *inode,
437 struct nfs_writeverf *verf, 437 struct nfs_writeverf *verf,
438 enum nfs3_stable_how how) 438 enum nfs3_stable_how how)
439 { 439 {
440 nfs_copy_boot_verifier(&verf->verifier 440 nfs_copy_boot_verifier(&verf->verifier, inode);
441 verf->committed = how; 441 verf->committed = how;
442 } 442 }
443 443
444 /* Factored out from fs/nfsd/vfs.h:fh_getattr( 444 /* Factored out from fs/nfsd/vfs.h:fh_getattr() */
445 static int __vfs_getattr(struct path *p, struc 445 static int __vfs_getattr(struct path *p, struct kstat *stat, int version)
446 { 446 {
447 u32 request_mask = STATX_BASIC_STATS; 447 u32 request_mask = STATX_BASIC_STATS;
448 448
449 if (version == 4) 449 if (version == 4)
450 request_mask |= (STATX_BTIME | 450 request_mask |= (STATX_BTIME | STATX_CHANGE_COOKIE);
451 return vfs_getattr(p, stat, request_ma 451 return vfs_getattr(p, stat, request_mask, AT_STATX_SYNC_AS_STAT);
452 } 452 }
453 453
454 /* Copied from fs/nfsd/nfsfh.c:nfsd4_change_at 454 /* Copied from fs/nfsd/nfsfh.c:nfsd4_change_attribute() */
455 static u64 __nfsd4_change_attribute(const stru 455 static u64 __nfsd4_change_attribute(const struct kstat *stat,
456 const stru 456 const struct inode *inode)
457 { 457 {
458 u64 chattr; 458 u64 chattr;
459 459
460 if (stat->result_mask & STATX_CHANGE_C 460 if (stat->result_mask & STATX_CHANGE_COOKIE) {
461 chattr = stat->change_cookie; 461 chattr = stat->change_cookie;
462 if (S_ISREG(inode->i_mode) && 462 if (S_ISREG(inode->i_mode) &&
463 !(stat->attributes & STATX 463 !(stat->attributes & STATX_ATTR_CHANGE_MONOTONIC)) {
464 chattr += (u64)stat->c 464 chattr += (u64)stat->ctime.tv_sec << 30;
465 chattr += stat->ctime. 465 chattr += stat->ctime.tv_nsec;
466 } 466 }
467 } else { 467 } else {
468 chattr = time_to_chattr(&stat- 468 chattr = time_to_chattr(&stat->ctime);
469 } 469 }
470 return chattr; 470 return chattr;
471 } 471 }
472 472
473 static void nfs_local_vfs_getattr(struct nfs_l 473 static void nfs_local_vfs_getattr(struct nfs_local_kiocb *iocb)
474 { 474 {
475 struct kstat stat; 475 struct kstat stat;
476 struct file *filp = iocb->kiocb.ki_fil 476 struct file *filp = iocb->kiocb.ki_filp;
477 struct nfs_pgio_header *hdr = iocb->hd 477 struct nfs_pgio_header *hdr = iocb->hdr;
478 struct nfs_fattr *fattr = hdr->res.fat 478 struct nfs_fattr *fattr = hdr->res.fattr;
479 int version = NFS_PROTO(hdr->inode)->v 479 int version = NFS_PROTO(hdr->inode)->version;
480 480
481 if (unlikely(!fattr) || __vfs_getattr( 481 if (unlikely(!fattr) || __vfs_getattr(&filp->f_path, &stat, version))
482 return; 482 return;
483 483
484 fattr->valid = (NFS_ATTR_FATTR_FILEID 484 fattr->valid = (NFS_ATTR_FATTR_FILEID |
485 NFS_ATTR_FATTR_CHANGE 485 NFS_ATTR_FATTR_CHANGE |
486 NFS_ATTR_FATTR_SIZE | 486 NFS_ATTR_FATTR_SIZE |
487 NFS_ATTR_FATTR_ATIME | 487 NFS_ATTR_FATTR_ATIME |
488 NFS_ATTR_FATTR_MTIME | 488 NFS_ATTR_FATTR_MTIME |
489 NFS_ATTR_FATTR_CTIME | 489 NFS_ATTR_FATTR_CTIME |
490 NFS_ATTR_FATTR_SPACE_U 490 NFS_ATTR_FATTR_SPACE_USED);
491 491
492 fattr->fileid = stat.ino; 492 fattr->fileid = stat.ino;
493 fattr->size = stat.size; 493 fattr->size = stat.size;
494 fattr->atime = stat.atime; 494 fattr->atime = stat.atime;
495 fattr->mtime = stat.mtime; 495 fattr->mtime = stat.mtime;
496 fattr->ctime = stat.ctime; 496 fattr->ctime = stat.ctime;
497 if (version == 4) { 497 if (version == 4) {
498 fattr->change_attr = 498 fattr->change_attr =
499 __nfsd4_change_attribu 499 __nfsd4_change_attribute(&stat, file_inode(filp));
500 } else 500 } else
501 fattr->change_attr = nfs_times 501 fattr->change_attr = nfs_timespec_to_change_attr(&fattr->ctime);
502 fattr->du.nfs3.used = stat.blocks << 9 502 fattr->du.nfs3.used = stat.blocks << 9;
503 } 503 }
504 504
505 static void 505 static void
506 nfs_local_write_done(struct nfs_local_kiocb *i 506 nfs_local_write_done(struct nfs_local_kiocb *iocb, long status)
507 { 507 {
508 struct nfs_pgio_header *hdr = iocb->hd 508 struct nfs_pgio_header *hdr = iocb->hdr;
509 struct inode *inode = hdr->inode; 509 struct inode *inode = hdr->inode;
510 510
511 dprintk("%s: wrote %ld bytes.\n", __fu 511 dprintk("%s: wrote %ld bytes.\n", __func__, status > 0 ? status : 0);
512 512
513 /* Handle short writes as if they are 513 /* Handle short writes as if they are ENOSPC */
514 if (status > 0 && status < hdr->args.c 514 if (status > 0 && status < hdr->args.count) {
515 hdr->mds_offset += status; 515 hdr->mds_offset += status;
516 hdr->args.offset += status; 516 hdr->args.offset += status;
517 hdr->args.pgbase += status; 517 hdr->args.pgbase += status;
518 hdr->args.count -= status; 518 hdr->args.count -= status;
519 nfs_set_pgio_error(hdr, -ENOSP 519 nfs_set_pgio_error(hdr, -ENOSPC, hdr->args.offset);
520 status = -ENOSPC; 520 status = -ENOSPC;
521 } 521 }
522 if (status < 0) 522 if (status < 0)
523 nfs_reset_boot_verifier(inode) 523 nfs_reset_boot_verifier(inode);
524 else if (nfs_should_remove_suid(inode) 524 else if (nfs_should_remove_suid(inode)) {
525 /* Deal with the suid/sgid bit 525 /* Deal with the suid/sgid bit corner case */
526 spin_lock(&inode->i_lock); 526 spin_lock(&inode->i_lock);
527 nfs_set_cache_invalid(inode, N 527 nfs_set_cache_invalid(inode, NFS_INO_INVALID_MODE);
528 spin_unlock(&inode->i_lock); 528 spin_unlock(&inode->i_lock);
529 } 529 }
530 nfs_local_pgio_done(hdr, status); 530 nfs_local_pgio_done(hdr, status);
531 } 531 }
532 532
533 static void nfs_local_call_write(struct work_s 533 static void nfs_local_call_write(struct work_struct *work)
534 { 534 {
535 struct nfs_local_kiocb *iocb = 535 struct nfs_local_kiocb *iocb =
536 container_of(work, struct nfs_ 536 container_of(work, struct nfs_local_kiocb, work);
537 struct file *filp = iocb->kiocb.ki_fil 537 struct file *filp = iocb->kiocb.ki_filp;
538 unsigned long old_flags = current->fla 538 unsigned long old_flags = current->flags;
539 const struct cred *save_cred; 539 const struct cred *save_cred;
540 struct iov_iter iter; 540 struct iov_iter iter;
541 ssize_t status; 541 ssize_t status;
542 542
543 current->flags |= PF_LOCAL_THROTTLE | 543 current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO;
544 save_cred = override_creds(filp->f_cre 544 save_cred = override_creds(filp->f_cred);
545 545
546 nfs_local_iter_init(&iter, iocb, WRITE 546 nfs_local_iter_init(&iter, iocb, WRITE);
547 547
548 file_start_write(filp); 548 file_start_write(filp);
549 status = filp->f_op->write_iter(&iocb- 549 status = filp->f_op->write_iter(&iocb->kiocb, &iter);
550 file_end_write(filp); 550 file_end_write(filp);
551 WARN_ON_ONCE(status == -EIOCBQUEUED); 551 WARN_ON_ONCE(status == -EIOCBQUEUED);
552 552
553 nfs_local_write_done(iocb, status); 553 nfs_local_write_done(iocb, status);
554 nfs_local_vfs_getattr(iocb); 554 nfs_local_vfs_getattr(iocb);
555 nfs_local_pgio_release(iocb); 555 nfs_local_pgio_release(iocb);
556 556
557 revert_creds(save_cred); 557 revert_creds(save_cred);
558 current->flags = old_flags; 558 current->flags = old_flags;
559 } 559 }
560 560
561 static int 561 static int
562 nfs_do_local_write(struct nfs_pgio_header *hdr 562 nfs_do_local_write(struct nfs_pgio_header *hdr,
563 struct nfsd_file *localio, 563 struct nfsd_file *localio,
564 const struct rpc_call_ops * 564 const struct rpc_call_ops *call_ops)
565 { 565 {
566 struct nfs_local_kiocb *iocb; 566 struct nfs_local_kiocb *iocb;
567 567
568 dprintk("%s: vfs_write count=%u pos=%l 568 dprintk("%s: vfs_write count=%u pos=%llu %s\n",
569 __func__, hdr->args.count, hdr 569 __func__, hdr->args.count, hdr->args.offset,
570 (hdr->args.stable == NFS_UNSTA 570 (hdr->args.stable == NFS_UNSTABLE) ? "unstable" : "stable");
571 571
572 iocb = nfs_local_iocb_alloc(hdr, local 572 iocb = nfs_local_iocb_alloc(hdr, localio, GFP_NOIO);
573 if (iocb == NULL) 573 if (iocb == NULL)
574 return -ENOMEM; 574 return -ENOMEM;
575 575
576 switch (hdr->args.stable) { 576 switch (hdr->args.stable) {
577 default: 577 default:
578 break; 578 break;
579 case NFS_DATA_SYNC: 579 case NFS_DATA_SYNC:
580 iocb->kiocb.ki_flags |= IOCB_D 580 iocb->kiocb.ki_flags |= IOCB_DSYNC;
581 break; 581 break;
582 case NFS_FILE_SYNC: 582 case NFS_FILE_SYNC:
583 iocb->kiocb.ki_flags |= IOCB_D 583 iocb->kiocb.ki_flags |= IOCB_DSYNC|IOCB_SYNC;
584 } 584 }
585 nfs_local_pgio_init(hdr, call_ops); 585 nfs_local_pgio_init(hdr, call_ops);
586 586
587 nfs_set_local_verifier(hdr->inode, hdr 587 nfs_set_local_verifier(hdr->inode, hdr->res.verf, hdr->args.stable);
588 588
589 INIT_WORK(&iocb->work, nfs_local_call_ 589 INIT_WORK(&iocb->work, nfs_local_call_write);
590 queue_work(nfslocaliod_workqueue, &ioc 590 queue_work(nfslocaliod_workqueue, &iocb->work);
591 591
592 return 0; 592 return 0;
593 } 593 }
594 594
595 int nfs_local_doio(struct nfs_client *clp, str 595 int nfs_local_doio(struct nfs_client *clp, struct nfsd_file *localio,
596 struct nfs_pgio_header *hdr 596 struct nfs_pgio_header *hdr,
597 const struct rpc_call_ops * 597 const struct rpc_call_ops *call_ops)
598 { 598 {
599 int status = 0; 599 int status = 0;
600 struct file *filp = nfs_to->nfsd_file_ 600 struct file *filp = nfs_to->nfsd_file_file(localio);
601 601
602 if (!hdr->args.count) 602 if (!hdr->args.count)
603 return 0; 603 return 0;
604 /* Don't support filesystems without r 604 /* Don't support filesystems without read_iter/write_iter */
605 if (!filp->f_op->read_iter || !filp->f 605 if (!filp->f_op->read_iter || !filp->f_op->write_iter) {
606 nfs_local_disable(clp); 606 nfs_local_disable(clp);
607 status = -EAGAIN; 607 status = -EAGAIN;
608 goto out; 608 goto out;
609 } 609 }
610 610
611 switch (hdr->rw_mode) { 611 switch (hdr->rw_mode) {
612 case FMODE_READ: 612 case FMODE_READ:
613 status = nfs_do_local_read(hdr 613 status = nfs_do_local_read(hdr, localio, call_ops);
614 break; 614 break;
615 case FMODE_WRITE: 615 case FMODE_WRITE:
616 status = nfs_do_local_write(hd 616 status = nfs_do_local_write(hdr, localio, call_ops);
617 break; 617 break;
618 default: 618 default:
619 dprintk("%s: invalid mode: %d\ 619 dprintk("%s: invalid mode: %d\n", __func__,
620 hdr->rw_mode); 620 hdr->rw_mode);
621 status = -EINVAL; 621 status = -EINVAL;
622 } 622 }
623 out: 623 out:
624 if (status != 0) { 624 if (status != 0) {
625 nfs_to_nfsd_file_put_local(loc 625 nfs_to_nfsd_file_put_local(localio);
626 hdr->task.tk_status = status; 626 hdr->task.tk_status = status;
627 nfs_local_hdr_release(hdr, cal 627 nfs_local_hdr_release(hdr, call_ops);
628 } 628 }
629 return status; 629 return status;
630 } 630 }
631 631
632 static void 632 static void
633 nfs_local_init_commit(struct nfs_commit_data * 633 nfs_local_init_commit(struct nfs_commit_data *data,
634 const struct rpc_call_ops *cal 634 const struct rpc_call_ops *call_ops)
635 { 635 {
636 data->task.tk_ops = call_ops; 636 data->task.tk_ops = call_ops;
637 } 637 }
638 638
639 static int 639 static int
640 nfs_local_run_commit(struct file *filp, struct 640 nfs_local_run_commit(struct file *filp, struct nfs_commit_data *data)
641 { 641 {
642 loff_t start = data->args.offset; 642 loff_t start = data->args.offset;
643 loff_t end = LLONG_MAX; 643 loff_t end = LLONG_MAX;
644 644
645 if (data->args.count > 0) { 645 if (data->args.count > 0) {
646 end = start + data->args.count 646 end = start + data->args.count - 1;
647 if (end < start) 647 if (end < start)
648 end = LLONG_MAX; 648 end = LLONG_MAX;
649 } 649 }
650 650
651 dprintk("%s: commit %llu - %llu\n", __ 651 dprintk("%s: commit %llu - %llu\n", __func__, start, end);
652 return vfs_fsync_range(filp, start, en 652 return vfs_fsync_range(filp, start, end, 0);
653 } 653 }
654 654
655 static void 655 static void
656 nfs_local_commit_done(struct nfs_commit_data * 656 nfs_local_commit_done(struct nfs_commit_data *data, int status)
657 { 657 {
658 if (status >= 0) { 658 if (status >= 0) {
659 nfs_set_local_verifier(data->i 659 nfs_set_local_verifier(data->inode,
660 data->res.verf 660 data->res.verf,
661 NFS_FILE_SYNC) 661 NFS_FILE_SYNC);
662 data->res.op_status = NFS4_OK; 662 data->res.op_status = NFS4_OK;
663 data->task.tk_status = 0; 663 data->task.tk_status = 0;
664 } else { 664 } else {
665 nfs_reset_boot_verifier(data-> 665 nfs_reset_boot_verifier(data->inode);
666 data->res.op_status = nfs4_sta 666 data->res.op_status = nfs4_stat_to_errno(status);
667 data->task.tk_status = status; 667 data->task.tk_status = status;
668 } 668 }
669 } 669 }
670 670
671 static void 671 static void
672 nfs_local_release_commit_data(struct nfsd_file 672 nfs_local_release_commit_data(struct nfsd_file *localio,
673 struct nfs_commit_data *data, 673 struct nfs_commit_data *data,
674 const struct rpc_call_ops *cal 674 const struct rpc_call_ops *call_ops)
675 { 675 {
676 nfs_to_nfsd_file_put_local(localio); 676 nfs_to_nfsd_file_put_local(localio);
677 call_ops->rpc_call_done(&data->task, d 677 call_ops->rpc_call_done(&data->task, data);
678 call_ops->rpc_release(data); 678 call_ops->rpc_release(data);
679 } 679 }
680 680
681 static struct nfs_local_fsync_ctx * 681 static struct nfs_local_fsync_ctx *
682 nfs_local_fsync_ctx_alloc(struct nfs_commit_da 682 nfs_local_fsync_ctx_alloc(struct nfs_commit_data *data,
683 struct nfsd_file *lo 683 struct nfsd_file *localio, gfp_t flags)
684 { 684 {
685 struct nfs_local_fsync_ctx *ctx = kmal 685 struct nfs_local_fsync_ctx *ctx = kmalloc(sizeof(*ctx), flags);
686 686
687 if (ctx != NULL) { 687 if (ctx != NULL) {
688 ctx->localio = localio; 688 ctx->localio = localio;
689 ctx->data = data; 689 ctx->data = data;
690 INIT_WORK(&ctx->work, nfs_loca 690 INIT_WORK(&ctx->work, nfs_local_fsync_work);
691 kref_init(&ctx->kref); 691 kref_init(&ctx->kref);
692 ctx->done = NULL; 692 ctx->done = NULL;
693 } 693 }
694 return ctx; 694 return ctx;
695 } 695 }
696 696
697 static void 697 static void
698 nfs_local_fsync_ctx_kref_free(struct kref *kre 698 nfs_local_fsync_ctx_kref_free(struct kref *kref)
699 { 699 {
700 kfree(container_of(kref, struct nfs_lo 700 kfree(container_of(kref, struct nfs_local_fsync_ctx, kref));
701 } 701 }
702 702
703 static void 703 static void
704 nfs_local_fsync_ctx_put(struct nfs_local_fsync 704 nfs_local_fsync_ctx_put(struct nfs_local_fsync_ctx *ctx)
705 { 705 {
706 kref_put(&ctx->kref, nfs_local_fsync_c 706 kref_put(&ctx->kref, nfs_local_fsync_ctx_kref_free);
707 } 707 }
708 708
709 static void 709 static void
710 nfs_local_fsync_ctx_free(struct nfs_local_fsyn 710 nfs_local_fsync_ctx_free(struct nfs_local_fsync_ctx *ctx)
711 { 711 {
712 nfs_local_release_commit_data(ctx->loc 712 nfs_local_release_commit_data(ctx->localio, ctx->data,
713 ctx->dat 713 ctx->data->task.tk_ops);
714 nfs_local_fsync_ctx_put(ctx); 714 nfs_local_fsync_ctx_put(ctx);
715 } 715 }
716 716
717 static void 717 static void
718 nfs_local_fsync_work(struct work_struct *work) 718 nfs_local_fsync_work(struct work_struct *work)
719 { 719 {
720 struct nfs_local_fsync_ctx *ctx; 720 struct nfs_local_fsync_ctx *ctx;
721 int status; 721 int status;
722 722
723 ctx = container_of(work, struct nfs_lo 723 ctx = container_of(work, struct nfs_local_fsync_ctx, work);
724 724
725 status = nfs_local_run_commit(nfs_to-> 725 status = nfs_local_run_commit(nfs_to->nfsd_file_file(ctx->localio),
726 ctx->dat 726 ctx->data);
727 nfs_local_commit_done(ctx->data, statu 727 nfs_local_commit_done(ctx->data, status);
728 if (ctx->done != NULL) 728 if (ctx->done != NULL)
729 complete(ctx->done); 729 complete(ctx->done);
730 nfs_local_fsync_ctx_free(ctx); 730 nfs_local_fsync_ctx_free(ctx);
731 } 731 }
732 732
733 int nfs_local_commit(struct nfsd_file *localio 733 int nfs_local_commit(struct nfsd_file *localio,
734 struct nfs_commit_data *d 734 struct nfs_commit_data *data,
735 const struct rpc_call_ops 735 const struct rpc_call_ops *call_ops, int how)
736 { 736 {
737 struct nfs_local_fsync_ctx *ctx; 737 struct nfs_local_fsync_ctx *ctx;
738 738
739 ctx = nfs_local_fsync_ctx_alloc(data, 739 ctx = nfs_local_fsync_ctx_alloc(data, localio, GFP_KERNEL);
740 if (!ctx) { 740 if (!ctx) {
741 nfs_local_commit_done(data, -E 741 nfs_local_commit_done(data, -ENOMEM);
742 nfs_local_release_commit_data( 742 nfs_local_release_commit_data(localio, data, call_ops);
743 return -ENOMEM; 743 return -ENOMEM;
744 } 744 }
745 745
746 nfs_local_init_commit(data, call_ops); 746 nfs_local_init_commit(data, call_ops);
747 kref_get(&ctx->kref); 747 kref_get(&ctx->kref);
748 if (how & FLUSH_SYNC) { 748 if (how & FLUSH_SYNC) {
749 DECLARE_COMPLETION_ONSTACK(don 749 DECLARE_COMPLETION_ONSTACK(done);
750 ctx->done = &done; 750 ctx->done = &done;
751 queue_work(nfsiod_workqueue, & 751 queue_work(nfsiod_workqueue, &ctx->work);
752 wait_for_completion(&done); 752 wait_for_completion(&done);
753 } else 753 } else
754 queue_work(nfsiod_workqueue, & 754 queue_work(nfsiod_workqueue, &ctx->work);
755 nfs_local_fsync_ctx_put(ctx); 755 nfs_local_fsync_ctx_put(ctx);
756 return 0; 756 return 0;
757 } 757 }
758 758
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.