1 // SPDX-License-Identifier: GPL-2.0-only
2 /******************************************************************************
3
4 (c) 2007 Network Appliance, Inc. All Rights Reserved.
5 (c) 2009 NetApp. All Rights Reserved.
6
7
8 ******************************************************************************/
9
10 #include <linux/tcp.h>
11 #include <linux/slab.h>
12 #include <linux/sunrpc/xprt.h>
13 #include <linux/export.h>
14 #include <linux/sunrpc/bc_xprt.h>
15
16 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
17 #define RPCDBG_FACILITY RPCDBG_TRANS
18 #endif
19
20 #define BC_MAX_SLOTS 64U
21
22 unsigned int xprt_bc_max_slots(struct rpc_xprt *xprt)
23 {
24 return BC_MAX_SLOTS;
25 }
26
27 /*
28 * Helper routines that track the number of preallocation elements
29 * on the transport.
30 */
31 static inline int xprt_need_to_requeue(struct rpc_xprt *xprt)
32 {
33 return xprt->bc_alloc_count < xprt->bc_alloc_max;
34 }
35
36 /*
37 * Free the preallocated rpc_rqst structure and the memory
38 * buffers hanging off of it.
39 */
40 static void xprt_free_allocation(struct rpc_rqst *req)
41 {
42 struct xdr_buf *xbufp;
43
44 dprintk("RPC: free allocations for req= %p\n", req);
45 WARN_ON_ONCE(test_bit(RPC_BC_PA_IN_USE, &req->rq_bc_pa_state));
46 xbufp = &req->rq_rcv_buf;
47 free_page((unsigned long)xbufp->head[0].iov_base);
48 xbufp = &req->rq_snd_buf;
49 free_page((unsigned long)xbufp->head[0].iov_base);
50 kfree(req);
51 }
52
53 static void xprt_bc_reinit_xdr_buf(struct xdr_buf *buf)
54 {
55 buf->head[0].iov_len = PAGE_SIZE;
56 buf->tail[0].iov_len = 0;
57 buf->pages = NULL;
58 buf->page_len = 0;
59 buf->flags = 0;
60 buf->len = 0;
61 buf->buflen = PAGE_SIZE;
62 }
63
64 static int xprt_alloc_xdr_buf(struct xdr_buf *buf, gfp_t gfp_flags)
65 {
66 struct page *page;
67 /* Preallocate one XDR receive buffer */
68 page = alloc_page(gfp_flags);
69 if (page == NULL)
70 return -ENOMEM;
71 xdr_buf_init(buf, page_address(page), PAGE_SIZE);
72 return 0;
73 }
74
75 static struct rpc_rqst *xprt_alloc_bc_req(struct rpc_xprt *xprt)
76 {
77 gfp_t gfp_flags = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN;
78 struct rpc_rqst *req;
79
80 /* Pre-allocate one backchannel rpc_rqst */
81 req = kzalloc(sizeof(*req), gfp_flags);
82 if (req == NULL)
83 return NULL;
84
85 req->rq_xprt = xprt;
86
87 /* Preallocate one XDR receive buffer */
88 if (xprt_alloc_xdr_buf(&req->rq_rcv_buf, gfp_flags) < 0) {
89 printk(KERN_ERR "Failed to create bc receive xbuf\n");
90 goto out_free;
91 }
92 req->rq_rcv_buf.len = PAGE_SIZE;
93
94 /* Preallocate one XDR send buffer */
95 if (xprt_alloc_xdr_buf(&req->rq_snd_buf, gfp_flags) < 0) {
96 printk(KERN_ERR "Failed to create bc snd xbuf\n");
97 goto out_free;
98 }
99 return req;
100 out_free:
101 xprt_free_allocation(req);
102 return NULL;
103 }
104
105 /*
106 * Preallocate up to min_reqs structures and related buffers for use
107 * by the backchannel. This function can be called multiple times
108 * when creating new sessions that use the same rpc_xprt. The
109 * preallocated buffers are added to the pool of resources used by
110 * the rpc_xprt. Any one of these resources may be used by an
111 * incoming callback request. It's up to the higher levels in the
112 * stack to enforce that the maximum number of session slots is not
113 * being exceeded.
114 *
115 * Some callback arguments can be large. For example, a pNFS server
116 * using multiple deviceids. The list can be unbound, but the client
117 * has the ability to tell the server the maximum size of the callback
118 * requests. Each deviceID is 16 bytes, so allocate one page
119 * for the arguments to have enough room to receive a number of these
120 * deviceIDs. The NFS client indicates to the pNFS server that its
121 * callback requests can be up to 4096 bytes in size.
122 */
123 int xprt_setup_backchannel(struct rpc_xprt *xprt, unsigned int min_reqs)
124 {
125 if (!xprt->ops->bc_setup)
126 return 0;
127 return xprt->ops->bc_setup(xprt, min_reqs);
128 }
129 EXPORT_SYMBOL_GPL(xprt_setup_backchannel);
130
131 int xprt_setup_bc(struct rpc_xprt *xprt, unsigned int min_reqs)
132 {
133 struct rpc_rqst *req;
134 struct list_head tmp_list;
135 int i;
136
137 dprintk("RPC: setup backchannel transport\n");
138
139 if (min_reqs > BC_MAX_SLOTS)
140 min_reqs = BC_MAX_SLOTS;
141
142 /*
143 * We use a temporary list to keep track of the preallocated
144 * buffers. Once we're done building the list we splice it
145 * into the backchannel preallocation list off of the rpc_xprt
146 * struct. This helps minimize the amount of time the list
147 * lock is held on the rpc_xprt struct. It also makes cleanup
148 * easier in case of memory allocation errors.
149 */
150 INIT_LIST_HEAD(&tmp_list);
151 for (i = 0; i < min_reqs; i++) {
152 /* Pre-allocate one backchannel rpc_rqst */
153 req = xprt_alloc_bc_req(xprt);
154 if (req == NULL) {
155 printk(KERN_ERR "Failed to create bc rpc_rqst\n");
156 goto out_free;
157 }
158
159 /* Add the allocated buffer to the tmp list */
160 dprintk("RPC: adding req= %p\n", req);
161 list_add(&req->rq_bc_pa_list, &tmp_list);
162 }
163
164 /*
165 * Add the temporary list to the backchannel preallocation list
166 */
167 spin_lock(&xprt->bc_pa_lock);
168 list_splice(&tmp_list, &xprt->bc_pa_list);
169 xprt->bc_alloc_count += min_reqs;
170 xprt->bc_alloc_max += min_reqs;
171 atomic_add(min_reqs, &xprt->bc_slot_count);
172 spin_unlock(&xprt->bc_pa_lock);
173
174 dprintk("RPC: setup backchannel transport done\n");
175 return 0;
176
177 out_free:
178 /*
179 * Memory allocation failed, free the temporary list
180 */
181 while (!list_empty(&tmp_list)) {
182 req = list_first_entry(&tmp_list,
183 struct rpc_rqst,
184 rq_bc_pa_list);
185 list_del(&req->rq_bc_pa_list);
186 xprt_free_allocation(req);
187 }
188
189 dprintk("RPC: setup backchannel transport failed\n");
190 return -ENOMEM;
191 }
192
193 /**
194 * xprt_destroy_backchannel - Destroys the backchannel preallocated structures.
195 * @xprt: the transport holding the preallocated strucures
196 * @max_reqs: the maximum number of preallocated structures to destroy
197 *
198 * Since these structures may have been allocated by multiple calls
199 * to xprt_setup_backchannel, we only destroy up to the maximum number
200 * of reqs specified by the caller.
201 */
202 void xprt_destroy_backchannel(struct rpc_xprt *xprt, unsigned int max_reqs)
203 {
204 if (xprt->ops->bc_destroy)
205 xprt->ops->bc_destroy(xprt, max_reqs);
206 }
207 EXPORT_SYMBOL_GPL(xprt_destroy_backchannel);
208
209 void xprt_destroy_bc(struct rpc_xprt *xprt, unsigned int max_reqs)
210 {
211 struct rpc_rqst *req = NULL, *tmp = NULL;
212
213 dprintk("RPC: destroy backchannel transport\n");
214
215 if (max_reqs == 0)
216 goto out;
217
218 spin_lock_bh(&xprt->bc_pa_lock);
219 xprt->bc_alloc_max -= min(max_reqs, xprt->bc_alloc_max);
220 list_for_each_entry_safe(req, tmp, &xprt->bc_pa_list, rq_bc_pa_list) {
221 dprintk("RPC: req=%p\n", req);
222 list_del(&req->rq_bc_pa_list);
223 xprt_free_allocation(req);
224 xprt->bc_alloc_count--;
225 atomic_dec(&xprt->bc_slot_count);
226 if (--max_reqs == 0)
227 break;
228 }
229 spin_unlock_bh(&xprt->bc_pa_lock);
230
231 out:
232 dprintk("RPC: backchannel list empty= %s\n",
233 list_empty(&xprt->bc_pa_list) ? "true" : "false");
234 }
235
236 static struct rpc_rqst *xprt_get_bc_request(struct rpc_xprt *xprt, __be32 xid,
237 struct rpc_rqst *new)
238 {
239 struct rpc_rqst *req = NULL;
240
241 dprintk("RPC: allocate a backchannel request\n");
242 if (list_empty(&xprt->bc_pa_list)) {
243 if (!new)
244 goto not_found;
245 if (atomic_read(&xprt->bc_slot_count) >= BC_MAX_SLOTS)
246 goto not_found;
247 list_add_tail(&new->rq_bc_pa_list, &xprt->bc_pa_list);
248 xprt->bc_alloc_count++;
249 atomic_inc(&xprt->bc_slot_count);
250 }
251 req = list_first_entry(&xprt->bc_pa_list, struct rpc_rqst,
252 rq_bc_pa_list);
253 req->rq_reply_bytes_recvd = 0;
254 memcpy(&req->rq_private_buf, &req->rq_rcv_buf,
255 sizeof(req->rq_private_buf));
256 req->rq_xid = xid;
257 req->rq_connect_cookie = xprt->connect_cookie;
258 dprintk("RPC: backchannel req=%p\n", req);
259 not_found:
260 return req;
261 }
262
263 /*
264 * Return the preallocated rpc_rqst structure and XDR buffers
265 * associated with this rpc_task.
266 */
267 void xprt_free_bc_request(struct rpc_rqst *req)
268 {
269 struct rpc_xprt *xprt = req->rq_xprt;
270
271 xprt->ops->bc_free_rqst(req);
272 }
273
274 void xprt_free_bc_rqst(struct rpc_rqst *req)
275 {
276 struct rpc_xprt *xprt = req->rq_xprt;
277
278 dprintk("RPC: free backchannel req=%p\n", req);
279
280 req->rq_connect_cookie = xprt->connect_cookie - 1;
281 smp_mb__before_atomic();
282 clear_bit(RPC_BC_PA_IN_USE, &req->rq_bc_pa_state);
283 smp_mb__after_atomic();
284
285 /*
286 * Return it to the list of preallocations so that it
287 * may be reused by a new callback request.
288 */
289 spin_lock_bh(&xprt->bc_pa_lock);
290 if (xprt_need_to_requeue(xprt)) {
291 xprt_bc_reinit_xdr_buf(&req->rq_snd_buf);
292 xprt_bc_reinit_xdr_buf(&req->rq_rcv_buf);
293 req->rq_rcv_buf.len = PAGE_SIZE;
294 list_add_tail(&req->rq_bc_pa_list, &xprt->bc_pa_list);
295 xprt->bc_alloc_count++;
296 atomic_inc(&xprt->bc_slot_count);
297 req = NULL;
298 }
299 spin_unlock_bh(&xprt->bc_pa_lock);
300 if (req != NULL) {
301 /*
302 * The last remaining session was destroyed while this
303 * entry was in use. Free the entry and don't attempt
304 * to add back to the list because there is no need to
305 * have anymore preallocated entries.
306 */
307 dprintk("RPC: Last session removed req=%p\n", req);
308 xprt_free_allocation(req);
309 }
310 xprt_put(xprt);
311 }
312
313 /*
314 * One or more rpc_rqst structure have been preallocated during the
315 * backchannel setup. Buffer space for the send and private XDR buffers
316 * has been preallocated as well. Use xprt_alloc_bc_request to allocate
317 * to this request. Use xprt_free_bc_request to return it.
318 *
319 * We know that we're called in soft interrupt context, grab the spin_lock
320 * since there is no need to grab the bottom half spin_lock.
321 *
322 * Return an available rpc_rqst, otherwise NULL if non are available.
323 */
324 struct rpc_rqst *xprt_lookup_bc_request(struct rpc_xprt *xprt, __be32 xid)
325 {
326 struct rpc_rqst *req, *new = NULL;
327
328 do {
329 spin_lock(&xprt->bc_pa_lock);
330 list_for_each_entry(req, &xprt->bc_pa_list, rq_bc_pa_list) {
331 if (req->rq_connect_cookie != xprt->connect_cookie)
332 continue;
333 if (req->rq_xid == xid)
334 goto found;
335 }
336 req = xprt_get_bc_request(xprt, xid, new);
337 found:
338 spin_unlock(&xprt->bc_pa_lock);
339 if (new) {
340 if (req != new)
341 xprt_free_allocation(new);
342 break;
343 } else if (req)
344 break;
345 new = xprt_alloc_bc_req(xprt);
346 } while (new);
347 return req;
348 }
349
350 /*
351 * Add callback request to callback list. Wake a thread
352 * on the first pool (usually the only pool) to handle it.
353 */
354 void xprt_complete_bc_request(struct rpc_rqst *req, uint32_t copied)
355 {
356 struct rpc_xprt *xprt = req->rq_xprt;
357 struct svc_serv *bc_serv = xprt->bc_serv;
358
359 spin_lock(&xprt->bc_pa_lock);
360 list_del(&req->rq_bc_pa_list);
361 xprt->bc_alloc_count--;
362 spin_unlock(&xprt->bc_pa_lock);
363
364 req->rq_private_buf.len = copied;
365 set_bit(RPC_BC_PA_IN_USE, &req->rq_bc_pa_state);
366
367 dprintk("RPC: add callback request to list\n");
368 xprt_get(xprt);
369 lwq_enqueue(&req->rq_bc_list, &bc_serv->sv_cb_list);
370 svc_pool_wake_idle_thread(&bc_serv->sv_pools[0]);
371 }
372
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.