1 // SPDX-License-Identifier: GPL-2.0-or-later 1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Client connection-specific management code. 2 /* Client connection-specific management code.
3 * 3 *
4 * Copyright (C) 2016, 2020 Red Hat, Inc. All 4 * Copyright (C) 2016, 2020 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.c 5 * Written by David Howells (dhowells@redhat.com)
6 * 6 *
7 * Client connections need to be cached for a 7 * Client connections need to be cached for a little while after they've made a
8 * call so as to handle retransmitted DATA pac 8 * call so as to handle retransmitted DATA packets in case the server didn't
9 * receive the final ACK or terminating ABORT 9 * receive the final ACK or terminating ABORT we sent it.
10 * 10 *
11 * There are flags of relevance to the cache: 11 * There are flags of relevance to the cache:
12 * 12 *
13 * (2) DONT_REUSE - The connection should be 13 * (2) DONT_REUSE - The connection should be discarded as soon as possible and
14 * should not be reused. This is set whe 14 * should not be reused. This is set when an exclusive connection is used
15 * or a call ID counter overflows. 15 * or a call ID counter overflows.
16 * 16 *
17 * The caching state may only be changed if th 17 * The caching state may only be changed if the cache lock is held.
18 * 18 *
19 * There are two idle client connection expiry 19 * There are two idle client connection expiry durations. If the total number
20 * of connections is below the reap threshold, 20 * of connections is below the reap threshold, we use the normal duration; if
21 * it's above, we use the fast duration. 21 * it's above, we use the fast duration.
22 */ 22 */
23 23
24 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
25 25
26 #include <linux/slab.h> 26 #include <linux/slab.h>
27 #include <linux/idr.h> 27 #include <linux/idr.h>
28 #include <linux/timer.h> 28 #include <linux/timer.h>
29 #include <linux/sched/signal.h> 29 #include <linux/sched/signal.h>
30 30
31 #include "ar-internal.h" 31 #include "ar-internal.h"
32 32
33 __read_mostly unsigned int rxrpc_reap_client_c 33 __read_mostly unsigned int rxrpc_reap_client_connections = 900;
34 __read_mostly unsigned long rxrpc_conn_idle_cl 34 __read_mostly unsigned long rxrpc_conn_idle_client_expiry = 2 * 60 * HZ;
35 __read_mostly unsigned long rxrpc_conn_idle_cl 35 __read_mostly unsigned long rxrpc_conn_idle_client_fast_expiry = 2 * HZ;
36 36
37 static void rxrpc_activate_bundle(struct rxrpc !! 37 /*
>> 38 * We use machine-unique IDs for our client connections.
>> 39 */
>> 40 DEFINE_IDR(rxrpc_client_conn_ids);
>> 41 static DEFINE_SPINLOCK(rxrpc_conn_id_lock);
>> 42
>> 43 /*
>> 44 * Get a connection ID and epoch for a client connection from the global pool.
>> 45 * The connection struct pointer is then recorded in the idr radix tree. The
>> 46 * epoch doesn't change until the client is rebooted (or, at least, unless the
>> 47 * module is unloaded).
>> 48 */
>> 49 static int rxrpc_get_client_connection_id(struct rxrpc_connection *conn,
>> 50 gfp_t gfp)
38 { 51 {
39 atomic_inc(&bundle->active); !! 52 struct rxrpc_net *rxnet = conn->params.local->rxnet;
>> 53 int id;
>> 54
>> 55 _enter("");
>> 56
>> 57 idr_preload(gfp);
>> 58 spin_lock(&rxrpc_conn_id_lock);
>> 59
>> 60 id = idr_alloc_cyclic(&rxrpc_client_conn_ids, conn,
>> 61 1, 0x40000000, GFP_NOWAIT);
>> 62 if (id < 0)
>> 63 goto error;
>> 64
>> 65 spin_unlock(&rxrpc_conn_id_lock);
>> 66 idr_preload_end();
>> 67
>> 68 conn->proto.epoch = rxnet->epoch;
>> 69 conn->proto.cid = id << RXRPC_CIDSHIFT;
>> 70 set_bit(RXRPC_CONN_HAS_IDR, &conn->flags);
>> 71 _leave(" [CID %x]", conn->proto.cid);
>> 72 return 0;
>> 73
>> 74 error:
>> 75 spin_unlock(&rxrpc_conn_id_lock);
>> 76 idr_preload_end();
>> 77 _leave(" = %d", id);
>> 78 return id;
40 } 79 }
41 80
42 /* 81 /*
43 * Release a connection ID for a client connec !! 82 * Release a connection ID for a client connection from the global pool.
44 */ 83 */
45 static void rxrpc_put_client_connection_id(str !! 84 static void rxrpc_put_client_connection_id(struct rxrpc_connection *conn)
46 str <<
47 { 85 {
48 idr_remove(&local->conn_ids, conn->pro !! 86 if (test_bit(RXRPC_CONN_HAS_IDR, &conn->flags)) {
>> 87 spin_lock(&rxrpc_conn_id_lock);
>> 88 idr_remove(&rxrpc_client_conn_ids,
>> 89 conn->proto.cid >> RXRPC_CIDSHIFT);
>> 90 spin_unlock(&rxrpc_conn_id_lock);
>> 91 }
49 } 92 }
50 93
51 /* 94 /*
52 * Destroy the client connection ID tree. 95 * Destroy the client connection ID tree.
53 */ 96 */
54 static void rxrpc_destroy_client_conn_ids(stru !! 97 void rxrpc_destroy_client_conn_ids(void)
55 { 98 {
56 struct rxrpc_connection *conn; 99 struct rxrpc_connection *conn;
57 int id; 100 int id;
58 101
59 if (!idr_is_empty(&local->conn_ids)) { !! 102 if (!idr_is_empty(&rxrpc_client_conn_ids)) {
60 idr_for_each_entry(&local->con !! 103 idr_for_each_entry(&rxrpc_client_conn_ids, conn, id) {
61 pr_err("AF_RXRPC: Leak 104 pr_err("AF_RXRPC: Leaked client conn %p {%d}\n",
62 conn, refcount_ 105 conn, refcount_read(&conn->ref));
63 } 106 }
64 BUG(); 107 BUG();
65 } 108 }
66 109
67 idr_destroy(&local->conn_ids); !! 110 idr_destroy(&rxrpc_client_conn_ids);
68 } 111 }
69 112
70 /* 113 /*
71 * Allocate a connection bundle. 114 * Allocate a connection bundle.
72 */ 115 */
73 static struct rxrpc_bundle *rxrpc_alloc_bundle !! 116 static struct rxrpc_bundle *rxrpc_alloc_bundle(struct rxrpc_conn_parameters *cp,
74 117 gfp_t gfp)
75 { 118 {
76 static atomic_t rxrpc_bundle_id; <<
77 struct rxrpc_bundle *bundle; 119 struct rxrpc_bundle *bundle;
78 120
79 bundle = kzalloc(sizeof(*bundle), gfp) 121 bundle = kzalloc(sizeof(*bundle), gfp);
80 if (bundle) { 122 if (bundle) {
81 bundle->local = call !! 123 bundle->params = *cp;
82 bundle->peer = rxrp !! 124 rxrpc_get_peer(bundle->params.peer);
83 bundle->key = key_ <<
84 bundle->security = call <<
85 bundle->exclusive = test <<
86 bundle->upgrade = test <<
87 bundle->service_id = call <<
88 bundle->security_level = call <<
89 bundle->debug_id = atom <<
90 refcount_set(&bundle->ref, 1); 125 refcount_set(&bundle->ref, 1);
91 atomic_set(&bundle->active, 1) !! 126 spin_lock_init(&bundle->channel_lock);
92 INIT_LIST_HEAD(&bundle->waitin 127 INIT_LIST_HEAD(&bundle->waiting_calls);
93 trace_rxrpc_bundle(bundle->deb <<
94 <<
95 write_lock(&bundle->local->rxn <<
96 list_add_tail(&bundle->proc_li <<
97 write_unlock(&bundle->local->r <<
98 } 128 }
99 return bundle; 129 return bundle;
100 } 130 }
101 131
102 struct rxrpc_bundle *rxrpc_get_bundle(struct r !! 132 struct rxrpc_bundle *rxrpc_get_bundle(struct rxrpc_bundle *bundle)
103 enum rxr <<
104 { 133 {
105 int r; !! 134 refcount_inc(&bundle->ref);
106 <<
107 __refcount_inc(&bundle->ref, &r); <<
108 trace_rxrpc_bundle(bundle->debug_id, r <<
109 return bundle; 135 return bundle;
110 } 136 }
111 137
112 static void rxrpc_free_bundle(struct rxrpc_bun 138 static void rxrpc_free_bundle(struct rxrpc_bundle *bundle)
113 { 139 {
114 trace_rxrpc_bundle(bundle->debug_id, r !! 140 rxrpc_put_peer(bundle->params.peer);
115 rxrpc_bundle_free); <<
116 write_lock(&bundle->local->rxnet->conn <<
117 list_del(&bundle->proc_link); <<
118 write_unlock(&bundle->local->rxnet->co <<
119 rxrpc_put_peer(bundle->peer, rxrpc_pee <<
120 key_put(bundle->key); <<
121 kfree(bundle); 141 kfree(bundle);
122 } 142 }
123 143
124 void rxrpc_put_bundle(struct rxrpc_bundle *bun !! 144 void rxrpc_put_bundle(struct rxrpc_bundle *bundle)
125 { 145 {
126 unsigned int id; !! 146 unsigned int d = bundle->debug_id;
127 bool dead; 147 bool dead;
128 int r; 148 int r;
129 149
130 if (bundle) { !! 150 dead = __refcount_dec_and_test(&bundle->ref, &r);
131 id = bundle->debug_id; <<
132 dead = __refcount_dec_and_test <<
133 trace_rxrpc_bundle(id, r - 1, <<
134 if (dead) <<
135 rxrpc_free_bundle(bund <<
136 } <<
137 } <<
138 151
139 /* !! 152 _debug("PUT B=%x %d", d, r);
140 * Get rid of outstanding client connection pr !! 153 if (dead)
141 * endpoint is destroyed. !! 154 rxrpc_free_bundle(bundle);
142 */ <<
143 void rxrpc_purge_client_connections(struct rxr <<
144 { <<
145 rxrpc_destroy_client_conn_ids(local); <<
146 } 155 }
147 156
148 /* 157 /*
149 * Allocate a client connection. 158 * Allocate a client connection.
150 */ 159 */
151 static struct rxrpc_connection * 160 static struct rxrpc_connection *
152 rxrpc_alloc_client_connection(struct rxrpc_bun !! 161 rxrpc_alloc_client_connection(struct rxrpc_bundle *bundle, gfp_t gfp)
153 { 162 {
154 struct rxrpc_connection *conn; 163 struct rxrpc_connection *conn;
155 struct rxrpc_local *local = bundle->lo !! 164 struct rxrpc_net *rxnet = bundle->params.local->rxnet;
156 struct rxrpc_net *rxnet = local->rxnet !! 165 int ret;
157 int id; <<
158 166
159 _enter(""); 167 _enter("");
160 168
161 conn = rxrpc_alloc_connection(rxnet, G !! 169 conn = rxrpc_alloc_connection(gfp);
162 if (!conn) !! 170 if (!conn) {
>> 171 _leave(" = -ENOMEM");
163 return ERR_PTR(-ENOMEM); 172 return ERR_PTR(-ENOMEM);
164 <<
165 id = idr_alloc_cyclic(&local->conn_ids <<
166 GFP_ATOMIC | __G <<
167 if (id < 0) { <<
168 kfree(conn); <<
169 return ERR_PTR(id); <<
170 } 173 }
171 174
172 refcount_set(&conn->ref, 1); 175 refcount_set(&conn->ref, 1);
173 conn->proto.cid = id << RXRPC_ !! 176 conn->bundle = bundle;
174 conn->proto.epoch = local->rxnet !! 177 conn->params = bundle->params;
175 conn->out_clientflag = RXRPC_CLIENT 178 conn->out_clientflag = RXRPC_CLIENT_INITIATED;
176 conn->bundle = rxrpc_get_bu !! 179 conn->state = RXRPC_CONN_CLIENT;
177 conn->local = rxrpc_get_lo !! 180 conn->service_id = conn->params.service_id;
178 conn->peer = rxrpc_get_pe <<
179 conn->key = key_get(bund <<
180 conn->security = bundle->secu <<
181 conn->exclusive = bundle->excl <<
182 conn->upgrade = bundle->upgr <<
183 conn->orig_service_id = bundle->serv <<
184 conn->security_level = bundle->secu <<
185 conn->state = RXRPC_CONN_C <<
186 conn->service_id = conn->orig_s <<
187 181
188 if (conn->security == &rxrpc_no_securi !! 182 ret = rxrpc_get_client_connection_id(conn, gfp);
189 conn->state = RXRPC_CONN_C !! 183 if (ret < 0)
>> 184 goto error_0;
>> 185
>> 186 ret = rxrpc_init_client_conn_security(conn);
>> 187 if (ret < 0)
>> 188 goto error_1;
190 189
191 atomic_inc(&rxnet->nr_conns); 190 atomic_inc(&rxnet->nr_conns);
192 write_lock(&rxnet->conn_lock); 191 write_lock(&rxnet->conn_lock);
193 list_add_tail(&conn->proc_link, &rxnet 192 list_add_tail(&conn->proc_link, &rxnet->conn_proc_list);
194 write_unlock(&rxnet->conn_lock); 193 write_unlock(&rxnet->conn_lock);
195 194
196 rxrpc_see_connection(conn, rxrpc_conn_ !! 195 rxrpc_get_bundle(bundle);
>> 196 rxrpc_get_peer(conn->params.peer);
>> 197 rxrpc_get_local(conn->params.local);
>> 198 key_get(conn->params.key);
>> 199
>> 200 trace_rxrpc_conn(conn->debug_id, rxrpc_conn_new_client,
>> 201 refcount_read(&conn->ref),
>> 202 __builtin_return_address(0));
197 203
198 atomic_inc(&rxnet->nr_client_conns); 204 atomic_inc(&rxnet->nr_client_conns);
199 trace_rxrpc_client(conn, -1, rxrpc_cli 205 trace_rxrpc_client(conn, -1, rxrpc_client_alloc);
>> 206 _leave(" = %p", conn);
200 return conn; 207 return conn;
>> 208
>> 209 error_1:
>> 210 rxrpc_put_client_connection_id(conn);
>> 211 error_0:
>> 212 kfree(conn);
>> 213 _leave(" = %d", ret);
>> 214 return ERR_PTR(ret);
201 } 215 }
202 216
203 /* 217 /*
204 * Determine if a connection may be reused. 218 * Determine if a connection may be reused.
205 */ 219 */
206 static bool rxrpc_may_reuse_conn(struct rxrpc_ 220 static bool rxrpc_may_reuse_conn(struct rxrpc_connection *conn)
207 { 221 {
208 struct rxrpc_net *rxnet; 222 struct rxrpc_net *rxnet;
209 int id_cursor, id, distance, limit; 223 int id_cursor, id, distance, limit;
210 224
211 if (!conn) 225 if (!conn)
212 goto dont_reuse; 226 goto dont_reuse;
213 227
214 rxnet = conn->rxnet; !! 228 rxnet = conn->params.local->rxnet;
215 if (test_bit(RXRPC_CONN_DONT_REUSE, &c 229 if (test_bit(RXRPC_CONN_DONT_REUSE, &conn->flags))
216 goto dont_reuse; 230 goto dont_reuse;
217 231
218 if ((conn->state != RXRPC_CONN_CLIENT_ !! 232 if (conn->state != RXRPC_CONN_CLIENT ||
219 conn->state != RXRPC_CONN_CLIENT) <<
220 conn->proto.epoch != rxnet->epoch) 233 conn->proto.epoch != rxnet->epoch)
221 goto mark_dont_reuse; 234 goto mark_dont_reuse;
222 235
223 /* The IDR tree gets very expensive on 236 /* The IDR tree gets very expensive on memory if the connection IDs are
224 * widely scattered throughout the num 237 * widely scattered throughout the number space, so we shall want to
225 * kill off connections that, say, hav 238 * kill off connections that, say, have an ID more than about four
226 * times the maximum number of client 239 * times the maximum number of client conns away from the current
227 * allocation point to try and keep th 240 * allocation point to try and keep the IDs concentrated.
228 */ 241 */
229 id_cursor = idr_get_cursor(&conn->loca !! 242 id_cursor = idr_get_cursor(&rxrpc_client_conn_ids);
230 id = conn->proto.cid >> RXRPC_CIDSHIFT 243 id = conn->proto.cid >> RXRPC_CIDSHIFT;
231 distance = id - id_cursor; 244 distance = id - id_cursor;
232 if (distance < 0) 245 if (distance < 0)
233 distance = -distance; 246 distance = -distance;
234 limit = max_t(unsigned long, atomic_re 247 limit = max_t(unsigned long, atomic_read(&rxnet->nr_conns) * 4, 1024);
235 if (distance > limit) 248 if (distance > limit)
236 goto mark_dont_reuse; 249 goto mark_dont_reuse;
237 250
238 return true; 251 return true;
239 252
240 mark_dont_reuse: 253 mark_dont_reuse:
241 set_bit(RXRPC_CONN_DONT_REUSE, &conn-> 254 set_bit(RXRPC_CONN_DONT_REUSE, &conn->flags);
242 dont_reuse: 255 dont_reuse:
243 return false; 256 return false;
244 } 257 }
245 258
246 /* 259 /*
247 * Look up the conn bundle that matches the co 260 * Look up the conn bundle that matches the connection parameters, adding it if
248 * it doesn't yet exist. 261 * it doesn't yet exist.
249 */ 262 */
250 int rxrpc_look_up_bundle(struct rxrpc_call *ca !! 263 static struct rxrpc_bundle *rxrpc_look_up_bundle(struct rxrpc_conn_parameters *cp,
>> 264 gfp_t gfp)
251 { 265 {
>> 266 static atomic_t rxrpc_bundle_id;
252 struct rxrpc_bundle *bundle, *candidat 267 struct rxrpc_bundle *bundle, *candidate;
253 struct rxrpc_local *local = call->loca !! 268 struct rxrpc_local *local = cp->local;
254 struct rb_node *p, **pp, *parent; 269 struct rb_node *p, **pp, *parent;
255 long diff; 270 long diff;
256 bool upgrade = test_bit(RXRPC_CALL_UPG <<
257 271
258 _enter("{%px,%x,%u,%u}", 272 _enter("{%px,%x,%u,%u}",
259 call->peer, key_serial(call->ke !! 273 cp->peer, key_serial(cp->key), cp->security_level, cp->upgrade);
260 upgrade); <<
261 274
262 if (test_bit(RXRPC_CALL_EXCLUSIVE, &ca !! 275 if (cp->exclusive)
263 call->bundle = rxrpc_alloc_bun !! 276 return rxrpc_alloc_bundle(cp, gfp);
264 return call->bundle ? 0 : -ENO <<
265 } <<
266 277
267 /* First, see if the bundle is already 278 /* First, see if the bundle is already there. */
268 _debug("search 1"); 279 _debug("search 1");
269 spin_lock(&local->client_bundles_lock) 280 spin_lock(&local->client_bundles_lock);
270 p = local->client_bundles.rb_node; 281 p = local->client_bundles.rb_node;
271 while (p) { 282 while (p) {
272 bundle = rb_entry(p, struct rx 283 bundle = rb_entry(p, struct rxrpc_bundle, local_node);
273 284
274 #define cmp(X, Y) ((long)(X) - (long)(Y)) !! 285 #define cmp(X) ((long)bundle->params.X - (long)cp->X)
275 diff = (cmp(bundle->peer, call !! 286 diff = (cmp(peer) ?:
276 cmp(bundle->key, call- !! 287 cmp(key) ?:
277 cmp(bundle->security_l !! 288 cmp(security_level) ?:
278 cmp(bundle->upgrade, u !! 289 cmp(upgrade));
279 #undef cmp 290 #undef cmp
280 if (diff < 0) 291 if (diff < 0)
281 p = p->rb_left; 292 p = p->rb_left;
282 else if (diff > 0) 293 else if (diff > 0)
283 p = p->rb_right; 294 p = p->rb_right;
284 else 295 else
285 goto found_bundle; 296 goto found_bundle;
286 } 297 }
287 spin_unlock(&local->client_bundles_loc 298 spin_unlock(&local->client_bundles_lock);
288 _debug("not found"); 299 _debug("not found");
289 300
290 /* It wasn't. We need to add one. */ 301 /* It wasn't. We need to add one. */
291 candidate = rxrpc_alloc_bundle(call, g !! 302 candidate = rxrpc_alloc_bundle(cp, gfp);
292 if (!candidate) 303 if (!candidate)
293 return -ENOMEM; !! 304 return NULL;
294 305
295 _debug("search 2"); 306 _debug("search 2");
296 spin_lock(&local->client_bundles_lock) 307 spin_lock(&local->client_bundles_lock);
297 pp = &local->client_bundles.rb_node; 308 pp = &local->client_bundles.rb_node;
298 parent = NULL; 309 parent = NULL;
299 while (*pp) { 310 while (*pp) {
300 parent = *pp; 311 parent = *pp;
301 bundle = rb_entry(parent, stru 312 bundle = rb_entry(parent, struct rxrpc_bundle, local_node);
302 313
303 #define cmp(X, Y) ((long)(X) - (long)(Y)) !! 314 #define cmp(X) ((long)bundle->params.X - (long)cp->X)
304 diff = (cmp(bundle->peer, call !! 315 diff = (cmp(peer) ?:
305 cmp(bundle->key, call- !! 316 cmp(key) ?:
306 cmp(bundle->security_l !! 317 cmp(security_level) ?:
307 cmp(bundle->upgrade, u !! 318 cmp(upgrade));
308 #undef cmp 319 #undef cmp
309 if (diff < 0) 320 if (diff < 0)
310 pp = &(*pp)->rb_left; 321 pp = &(*pp)->rb_left;
311 else if (diff > 0) 322 else if (diff > 0)
312 pp = &(*pp)->rb_right; 323 pp = &(*pp)->rb_right;
313 else 324 else
314 goto found_bundle_free 325 goto found_bundle_free;
315 } 326 }
316 327
317 _debug("new bundle"); 328 _debug("new bundle");
>> 329 candidate->debug_id = atomic_inc_return(&rxrpc_bundle_id);
318 rb_link_node(&candidate->local_node, p 330 rb_link_node(&candidate->local_node, parent, pp);
319 rb_insert_color(&candidate->local_node 331 rb_insert_color(&candidate->local_node, &local->client_bundles);
320 call->bundle = rxrpc_get_bundle(candid !! 332 rxrpc_get_bundle(candidate);
321 spin_unlock(&local->client_bundles_loc 333 spin_unlock(&local->client_bundles_lock);
322 _leave(" = B=%u [new]", call->bundle-> !! 334 _leave(" = %u [new]", candidate->debug_id);
323 return 0; !! 335 return candidate;
324 336
325 found_bundle_free: 337 found_bundle_free:
326 rxrpc_free_bundle(candidate); 338 rxrpc_free_bundle(candidate);
327 found_bundle: 339 found_bundle:
328 call->bundle = rxrpc_get_bundle(bundle !! 340 rxrpc_get_bundle(bundle);
329 rxrpc_activate_bundle(bundle); <<
330 spin_unlock(&local->client_bundles_loc 341 spin_unlock(&local->client_bundles_lock);
331 _leave(" = B=%u [found]", call->bundle !! 342 _leave(" = %u [found]", bundle->debug_id);
332 return 0; !! 343 return bundle;
>> 344 }
>> 345
>> 346 /*
>> 347 * Create or find a client bundle to use for a call.
>> 348 *
>> 349 * If we return with a connection, the call will be on its waiting list. It's
>> 350 * left to the caller to assign a channel and wake up the call.
>> 351 */
>> 352 static struct rxrpc_bundle *rxrpc_prep_call(struct rxrpc_sock *rx,
>> 353 struct rxrpc_call *call,
>> 354 struct rxrpc_conn_parameters *cp,
>> 355 struct sockaddr_rxrpc *srx,
>> 356 gfp_t gfp)
>> 357 {
>> 358 struct rxrpc_bundle *bundle;
>> 359
>> 360 _enter("{%d,%lx},", call->debug_id, call->user_call_ID);
>> 361
>> 362 cp->peer = rxrpc_lookup_peer(rx, cp->local, srx, gfp);
>> 363 if (!cp->peer)
>> 364 goto error;
>> 365
>> 366 call->cong_cwnd = cp->peer->cong_cwnd;
>> 367 if (call->cong_cwnd >= call->cong_ssthresh)
>> 368 call->cong_mode = RXRPC_CALL_CONGEST_AVOIDANCE;
>> 369 else
>> 370 call->cong_mode = RXRPC_CALL_SLOW_START;
>> 371 if (cp->upgrade)
>> 372 __set_bit(RXRPC_CALL_UPGRADE, &call->flags);
>> 373
>> 374 /* Find the client connection bundle. */
>> 375 bundle = rxrpc_look_up_bundle(cp, gfp);
>> 376 if (!bundle)
>> 377 goto error;
>> 378
>> 379 /* Get this call queued. Someone else may activate it whilst we're
>> 380 * lining up a new connection, but that's fine.
>> 381 */
>> 382 spin_lock(&bundle->channel_lock);
>> 383 list_add_tail(&call->chan_wait_link, &bundle->waiting_calls);
>> 384 spin_unlock(&bundle->channel_lock);
>> 385
>> 386 _leave(" = [B=%x]", bundle->debug_id);
>> 387 return bundle;
>> 388
>> 389 error:
>> 390 _leave(" = -ENOMEM");
>> 391 return ERR_PTR(-ENOMEM);
333 } 392 }
334 393
335 /* 394 /*
336 * Allocate a new connection and add it into a 395 * Allocate a new connection and add it into a bundle.
337 */ 396 */
338 static bool rxrpc_add_conn_to_bundle(struct rx !! 397 static void rxrpc_add_conn_to_bundle(struct rxrpc_bundle *bundle, gfp_t gfp)
339 unsigned !! 398 __releases(bundle->channel_lock)
340 { 399 {
341 struct rxrpc_connection *conn, *old; !! 400 struct rxrpc_connection *candidate = NULL, *old = NULL;
342 unsigned int shift = slot * RXRPC_MAXC !! 401 bool conflict;
343 unsigned int i; !! 402 int i;
344 !! 403
345 old = bundle->conns[slot]; !! 404 _enter("");
346 if (old) { !! 405
347 bundle->conns[slot] = NULL; !! 406 conflict = bundle->alloc_conn;
348 bundle->conn_ids[slot] = 0; !! 407 if (!conflict)
349 trace_rxrpc_client(old, -1, rx !! 408 bundle->alloc_conn = true;
350 rxrpc_put_connection(old, rxrp !! 409 spin_unlock(&bundle->channel_lock);
351 } !! 410 if (conflict) {
352 !! 411 _leave(" [conf]");
353 conn = rxrpc_alloc_client_connection(b !! 412 return;
354 if (IS_ERR(conn)) { !! 413 }
355 bundle->alloc_error = PTR_ERR( !! 414
356 return false; !! 415 candidate = rxrpc_alloc_client_connection(bundle, gfp);
357 } !! 416
358 !! 417 spin_lock(&bundle->channel_lock);
359 rxrpc_activate_bundle(bundle); !! 418 bundle->alloc_conn = false;
360 conn->bundle_shift = shift; !! 419
361 bundle->conns[slot] = conn; !! 420 if (IS_ERR(candidate)) {
362 bundle->conn_ids[slot] = conn->debug_i !! 421 bundle->alloc_error = PTR_ERR(candidate);
363 for (i = 0; i < RXRPC_MAXCALLS; i++) !! 422 spin_unlock(&bundle->channel_lock);
364 set_bit(shift + i, &bundle->av !! 423 _leave(" [err %ld]", PTR_ERR(candidate));
365 return true; !! 424 return;
>> 425 }
>> 426
>> 427 bundle->alloc_error = 0;
>> 428
>> 429 for (i = 0; i < ARRAY_SIZE(bundle->conns); i++) {
>> 430 unsigned int shift = i * RXRPC_MAXCALLS;
>> 431 int j;
>> 432
>> 433 old = bundle->conns[i];
>> 434 if (!rxrpc_may_reuse_conn(old)) {
>> 435 if (old)
>> 436 trace_rxrpc_client(old, -1, rxrpc_client_replace);
>> 437 candidate->bundle_shift = shift;
>> 438 bundle->conns[i] = candidate;
>> 439 for (j = 0; j < RXRPC_MAXCALLS; j++)
>> 440 set_bit(shift + j, &bundle->avail_chans);
>> 441 candidate = NULL;
>> 442 break;
>> 443 }
>> 444
>> 445 old = NULL;
>> 446 }
>> 447
>> 448 spin_unlock(&bundle->channel_lock);
>> 449
>> 450 if (candidate) {
>> 451 _debug("discard C=%x", candidate->debug_id);
>> 452 trace_rxrpc_client(candidate, -1, rxrpc_client_duplicate);
>> 453 rxrpc_put_connection(candidate);
>> 454 }
>> 455
>> 456 rxrpc_put_connection(old);
>> 457 _leave("");
366 } 458 }
367 459
368 /* 460 /*
369 * Add a connection to a bundle if there are n 461 * Add a connection to a bundle if there are no usable connections or we have
370 * connections waiting for extra capacity. 462 * connections waiting for extra capacity.
371 */ 463 */
372 static bool rxrpc_bundle_has_space(struct rxrp !! 464 static void rxrpc_maybe_add_conn(struct rxrpc_bundle *bundle, gfp_t gfp)
373 { 465 {
374 int slot = -1, i, usable; !! 466 struct rxrpc_call *call;
>> 467 int i, usable;
375 468
376 _enter(""); 469 _enter("");
377 470
378 bundle->alloc_error = 0; !! 471 spin_lock(&bundle->channel_lock);
379 472
380 /* See if there are any usable connect 473 /* See if there are any usable connections. */
381 usable = 0; 474 usable = 0;
382 for (i = 0; i < ARRAY_SIZE(bundle->con !! 475 for (i = 0; i < ARRAY_SIZE(bundle->conns); i++)
383 if (rxrpc_may_reuse_conn(bundl 476 if (rxrpc_may_reuse_conn(bundle->conns[i]))
384 usable++; 477 usable++;
385 else if (slot == -1) <<
386 slot = i; <<
387 } <<
388 478
389 if (!usable && bundle->upgrade) !! 479 if (!usable && !list_empty(&bundle->waiting_calls)) {
390 bundle->try_upgrade = true; !! 480 call = list_first_entry(&bundle->waiting_calls,
>> 481 struct rxrpc_call, chan_wait_link);
>> 482 if (test_bit(RXRPC_CALL_UPGRADE, &call->flags))
>> 483 bundle->try_upgrade = true;
>> 484 }
391 485
392 if (!usable) 486 if (!usable)
393 goto alloc_conn; 487 goto alloc_conn;
394 488
395 if (!bundle->avail_chans && 489 if (!bundle->avail_chans &&
396 !bundle->try_upgrade && 490 !bundle->try_upgrade &&
>> 491 !list_empty(&bundle->waiting_calls) &&
397 usable < ARRAY_SIZE(bundle->conns) 492 usable < ARRAY_SIZE(bundle->conns))
398 goto alloc_conn; 493 goto alloc_conn;
399 494
>> 495 spin_unlock(&bundle->channel_lock);
400 _leave(""); 496 _leave("");
401 return usable; !! 497 return;
402 498
403 alloc_conn: 499 alloc_conn:
404 return slot >= 0 ? rxrpc_add_conn_to_b !! 500 return rxrpc_add_conn_to_bundle(bundle, gfp);
405 } 501 }
406 502
407 /* 503 /*
408 * Assign a channel to the call at the front o 504 * Assign a channel to the call at the front of the queue and wake the call up.
409 * We don't increment the callNumber counter u 505 * We don't increment the callNumber counter until this number has been exposed
410 * to the world. 506 * to the world.
411 */ 507 */
412 static void rxrpc_activate_one_channel(struct 508 static void rxrpc_activate_one_channel(struct rxrpc_connection *conn,
413 unsigne 509 unsigned int channel)
414 { 510 {
415 struct rxrpc_channel *chan = &conn->ch 511 struct rxrpc_channel *chan = &conn->channels[channel];
416 struct rxrpc_bundle *bundle = conn->bu 512 struct rxrpc_bundle *bundle = conn->bundle;
417 struct rxrpc_call *call = list_entry(b 513 struct rxrpc_call *call = list_entry(bundle->waiting_calls.next,
418 s !! 514 struct rxrpc_call, chan_wait_link);
419 u32 call_id = chan->call_counter + 1; 515 u32 call_id = chan->call_counter + 1;
420 516
421 _enter("C=%x,%u", conn->debug_id, chan 517 _enter("C=%x,%u", conn->debug_id, channel);
422 518
423 list_del_init(&call->wait_link); <<
424 <<
425 trace_rxrpc_client(conn, channel, rxrp 519 trace_rxrpc_client(conn, channel, rxrpc_client_chan_activate);
426 520
427 /* Cancel the final ACK on the previou 521 /* Cancel the final ACK on the previous call if it hasn't been sent yet
428 * as the DATA packet will implicitly 522 * as the DATA packet will implicitly ACK it.
429 */ 523 */
430 clear_bit(RXRPC_CONN_FINAL_ACK_0 + cha 524 clear_bit(RXRPC_CONN_FINAL_ACK_0 + channel, &conn->flags);
431 clear_bit(conn->bundle_shift + channel 525 clear_bit(conn->bundle_shift + channel, &bundle->avail_chans);
432 526
433 rxrpc_see_call(call, rxrpc_call_see_ac !! 527 rxrpc_see_call(call);
434 call->conn = rxrpc_get_connection !! 528 list_del_init(&call->chan_wait_link);
>> 529 call->peer = rxrpc_get_peer(conn->params.peer);
>> 530 call->conn = rxrpc_get_connection(conn);
435 call->cid = conn->proto.cid | ch 531 call->cid = conn->proto.cid | channel;
436 call->call_id = call_id; 532 call->call_id = call_id;
437 call->dest_srx.srx_service = conn->ser !! 533 call->security = conn->security;
438 call->cong_ssthresh = call->peer->cong !! 534 call->security_ix = conn->security_ix;
439 if (call->cong_cwnd >= call->cong_ssth !! 535 call->service_id = conn->service_id;
440 call->cong_mode = RXRPC_CALL_C !! 536
441 else !! 537 trace_rxrpc_connect_call(call);
442 call->cong_mode = RXRPC_CALL_S !! 538 _net("CONNECT call %08x:%08x as call %d on conn %d",
>> 539 call->cid, call->call_id, call->debug_id, conn->debug_id);
>> 540
>> 541 write_lock_bh(&call->state_lock);
>> 542 call->state = RXRPC_CALL_CLIENT_SEND_REQUEST;
>> 543 write_unlock_bh(&call->state_lock);
>> 544
>> 545 /* Paired with the read barrier in rxrpc_connect_call(). This orders
>> 546 * cid and epoch in the connection wrt to call_id without the need to
>> 547 * take the channel_lock.
>> 548 *
>> 549 * We provisionally assign a callNumber at this point, but we don't
>> 550 * confirm it until the call is about to be exposed.
>> 551 *
>> 552 * TODO: Pair with a barrier in the data_ready handler when that looks
>> 553 * at the call ID through a connection channel.
>> 554 */
>> 555 smp_wmb();
443 556
444 chan->call_id = call_id; 557 chan->call_id = call_id;
445 chan->call_debug_id = call->debug_ 558 chan->call_debug_id = call->debug_id;
446 chan->call = call; !! 559 rcu_assign_pointer(chan->call, call);
447 <<
448 rxrpc_see_call(call, rxrpc_call_see_co <<
449 trace_rxrpc_connect_call(call); <<
450 call->tx_last_sent = ktime_get_real(); <<
451 rxrpc_start_call_timer(call); <<
452 rxrpc_set_call_state(call, RXRPC_CALL_ <<
453 wake_up(&call->waitq); 560 wake_up(&call->waitq);
454 } 561 }
455 562
456 /* 563 /*
457 * Remove a connection from the idle list if i 564 * Remove a connection from the idle list if it's on it.
458 */ 565 */
459 static void rxrpc_unidle_conn(struct rxrpc_con !! 566 static void rxrpc_unidle_conn(struct rxrpc_bundle *bundle, struct rxrpc_connection *conn)
460 { 567 {
>> 568 struct rxrpc_net *rxnet = bundle->params.local->rxnet;
>> 569 bool drop_ref;
>> 570
461 if (!list_empty(&conn->cache_link)) { 571 if (!list_empty(&conn->cache_link)) {
462 list_del_init(&conn->cache_lin !! 572 drop_ref = false;
463 rxrpc_put_connection(conn, rxr !! 573 spin_lock(&rxnet->client_conn_cache_lock);
>> 574 if (!list_empty(&conn->cache_link)) {
>> 575 list_del_init(&conn->cache_link);
>> 576 drop_ref = true;
>> 577 }
>> 578 spin_unlock(&rxnet->client_conn_cache_lock);
>> 579 if (drop_ref)
>> 580 rxrpc_put_connection(conn);
464 } 581 }
465 } 582 }
466 583
467 /* 584 /*
468 * Assign channels and callNumbers to waiting !! 585 * Assign channels and callNumbers to waiting calls with channel_lock
>> 586 * held by caller.
469 */ 587 */
470 static void rxrpc_activate_channels(struct rxr !! 588 static void rxrpc_activate_channels_locked(struct rxrpc_bundle *bundle)
471 { 589 {
472 struct rxrpc_connection *conn; 590 struct rxrpc_connection *conn;
473 unsigned long avail, mask; 591 unsigned long avail, mask;
474 unsigned int channel, slot; 592 unsigned int channel, slot;
475 593
476 trace_rxrpc_client(NULL, -1, rxrpc_cli <<
477 <<
478 if (bundle->try_upgrade) 594 if (bundle->try_upgrade)
479 mask = 1; 595 mask = 1;
480 else 596 else
481 mask = ULONG_MAX; 597 mask = ULONG_MAX;
482 598
483 while (!list_empty(&bundle->waiting_ca 599 while (!list_empty(&bundle->waiting_calls)) {
484 avail = bundle->avail_chans & 600 avail = bundle->avail_chans & mask;
485 if (!avail) 601 if (!avail)
486 break; 602 break;
487 channel = __ffs(avail); 603 channel = __ffs(avail);
488 clear_bit(channel, &bundle->av 604 clear_bit(channel, &bundle->avail_chans);
489 605
490 slot = channel / RXRPC_MAXCALL 606 slot = channel / RXRPC_MAXCALLS;
491 conn = bundle->conns[slot]; 607 conn = bundle->conns[slot];
492 if (!conn) 608 if (!conn)
493 break; 609 break;
494 610
495 if (bundle->try_upgrade) 611 if (bundle->try_upgrade)
496 set_bit(RXRPC_CONN_PRO 612 set_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags);
497 rxrpc_unidle_conn(conn); !! 613 rxrpc_unidle_conn(bundle, conn);
498 614
499 channel &= (RXRPC_MAXCALLS - 1 615 channel &= (RXRPC_MAXCALLS - 1);
500 conn->act_chans |= 1 << channe 616 conn->act_chans |= 1 << channel;
501 rxrpc_activate_one_channel(con 617 rxrpc_activate_one_channel(conn, channel);
502 } 618 }
503 } 619 }
504 620
505 /* 621 /*
506 * Connect waiting channels (called from the I !! 622 * Assign channels and callNumbers to waiting calls.
507 */ 623 */
508 void rxrpc_connect_client_calls(struct rxrpc_l !! 624 static void rxrpc_activate_channels(struct rxrpc_bundle *bundle)
509 { 625 {
510 struct rxrpc_call *call; !! 626 _enter("B=%x", bundle->debug_id);
>> 627
>> 628 trace_rxrpc_client(NULL, -1, rxrpc_client_activate_chans);
>> 629
>> 630 if (!bundle->avail_chans)
>> 631 return;
>> 632
>> 633 spin_lock(&bundle->channel_lock);
>> 634 rxrpc_activate_channels_locked(bundle);
>> 635 spin_unlock(&bundle->channel_lock);
>> 636 _leave("");
>> 637 }
511 638
512 while ((call = list_first_entry_or_nul !! 639 /*
513 !! 640 * Wait for a callNumber and a channel to be granted to a call.
514 ) { !! 641 */
515 struct rxrpc_bundle *bundle = !! 642 static int rxrpc_wait_for_channel(struct rxrpc_bundle *bundle,
516 !! 643 struct rxrpc_call *call, gfp_t gfp)
517 spin_lock(&local->client_call_ !! 644 {
518 list_move_tail(&call->wait_lin !! 645 DECLARE_WAITQUEUE(myself, current);
519 rxrpc_see_call(call, rxrpc_cal !! 646 int ret = 0;
520 spin_unlock(&local->client_cal <<
521 647
522 if (rxrpc_bundle_has_space(bun !! 648 _enter("%d", call->debug_id);
523 rxrpc_activate_channel !! 649
>> 650 if (!gfpflags_allow_blocking(gfp)) {
>> 651 rxrpc_maybe_add_conn(bundle, gfp);
>> 652 rxrpc_activate_channels(bundle);
>> 653 ret = bundle->alloc_error ?: -EAGAIN;
>> 654 goto out;
524 } 655 }
>> 656
>> 657 add_wait_queue_exclusive(&call->waitq, &myself);
>> 658 for (;;) {
>> 659 rxrpc_maybe_add_conn(bundle, gfp);
>> 660 rxrpc_activate_channels(bundle);
>> 661 ret = bundle->alloc_error;
>> 662 if (ret < 0)
>> 663 break;
>> 664
>> 665 switch (call->interruptibility) {
>> 666 case RXRPC_INTERRUPTIBLE:
>> 667 case RXRPC_PREINTERRUPTIBLE:
>> 668 set_current_state(TASK_INTERRUPTIBLE);
>> 669 break;
>> 670 case RXRPC_UNINTERRUPTIBLE:
>> 671 default:
>> 672 set_current_state(TASK_UNINTERRUPTIBLE);
>> 673 break;
>> 674 }
>> 675 if (READ_ONCE(call->state) != RXRPC_CALL_CLIENT_AWAIT_CONN)
>> 676 break;
>> 677 if ((call->interruptibility == RXRPC_INTERRUPTIBLE ||
>> 678 call->interruptibility == RXRPC_PREINTERRUPTIBLE) &&
>> 679 signal_pending(current)) {
>> 680 ret = -ERESTARTSYS;
>> 681 break;
>> 682 }
>> 683 schedule();
>> 684 }
>> 685 remove_wait_queue(&call->waitq, &myself);
>> 686 __set_current_state(TASK_RUNNING);
>> 687
>> 688 out:
>> 689 _leave(" = %d", ret);
>> 690 return ret;
>> 691 }
>> 692
>> 693 /*
>> 694 * find a connection for a call
>> 695 * - called in process context with IRQs enabled
>> 696 */
>> 697 int rxrpc_connect_call(struct rxrpc_sock *rx,
>> 698 struct rxrpc_call *call,
>> 699 struct rxrpc_conn_parameters *cp,
>> 700 struct sockaddr_rxrpc *srx,
>> 701 gfp_t gfp)
>> 702 {
>> 703 struct rxrpc_bundle *bundle;
>> 704 struct rxrpc_net *rxnet = cp->local->rxnet;
>> 705 int ret = 0;
>> 706
>> 707 _enter("{%d,%lx},", call->debug_id, call->user_call_ID);
>> 708
>> 709 rxrpc_discard_expired_client_conns(&rxnet->client_conn_reaper);
>> 710
>> 711 bundle = rxrpc_prep_call(rx, call, cp, srx, gfp);
>> 712 if (IS_ERR(bundle)) {
>> 713 ret = PTR_ERR(bundle);
>> 714 goto out;
>> 715 }
>> 716
>> 717 if (call->state == RXRPC_CALL_CLIENT_AWAIT_CONN) {
>> 718 ret = rxrpc_wait_for_channel(bundle, call, gfp);
>> 719 if (ret < 0)
>> 720 goto wait_failed;
>> 721 }
>> 722
>> 723 granted_channel:
>> 724 /* Paired with the write barrier in rxrpc_activate_one_channel(). */
>> 725 smp_rmb();
>> 726
>> 727 out_put_bundle:
>> 728 rxrpc_put_bundle(bundle);
>> 729 out:
>> 730 _leave(" = %d", ret);
>> 731 return ret;
>> 732
>> 733 wait_failed:
>> 734 spin_lock(&bundle->channel_lock);
>> 735 list_del_init(&call->chan_wait_link);
>> 736 spin_unlock(&bundle->channel_lock);
>> 737
>> 738 if (call->state != RXRPC_CALL_CLIENT_AWAIT_CONN) {
>> 739 ret = 0;
>> 740 goto granted_channel;
>> 741 }
>> 742
>> 743 trace_rxrpc_client(call->conn, ret, rxrpc_client_chan_wait_failed);
>> 744 rxrpc_set_call_completion(call, RXRPC_CALL_LOCAL_ERROR, 0, ret);
>> 745 rxrpc_disconnect_client_call(bundle, call);
>> 746 goto out_put_bundle;
525 } 747 }
526 748
527 /* 749 /*
528 * Note that a call, and thus a connection, is 750 * Note that a call, and thus a connection, is about to be exposed to the
529 * world. 751 * world.
530 */ 752 */
531 void rxrpc_expose_client_call(struct rxrpc_cal 753 void rxrpc_expose_client_call(struct rxrpc_call *call)
532 { 754 {
533 unsigned int channel = call->cid & RXR 755 unsigned int channel = call->cid & RXRPC_CHANNELMASK;
534 struct rxrpc_connection *conn = call-> 756 struct rxrpc_connection *conn = call->conn;
535 struct rxrpc_channel *chan = &conn->ch 757 struct rxrpc_channel *chan = &conn->channels[channel];
536 758
537 if (!test_and_set_bit(RXRPC_CALL_EXPOS 759 if (!test_and_set_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
538 /* Mark the call ID as being u 760 /* Mark the call ID as being used. If the callNumber counter
539 * exceeds ~2 billion, we kill 761 * exceeds ~2 billion, we kill the connection after its
540 * outstanding calls have fini 762 * outstanding calls have finished so that the counter doesn't
541 * wrap. 763 * wrap.
542 */ 764 */
543 chan->call_counter++; 765 chan->call_counter++;
544 if (chan->call_counter >= INT_ 766 if (chan->call_counter >= INT_MAX)
545 set_bit(RXRPC_CONN_DON 767 set_bit(RXRPC_CONN_DONT_REUSE, &conn->flags);
546 trace_rxrpc_client(conn, chann 768 trace_rxrpc_client(conn, channel, rxrpc_client_exposed);
547 <<
548 spin_lock(&call->peer->lock); <<
549 hlist_add_head(&call->error_li <<
550 spin_unlock(&call->peer->lock) <<
551 } 769 }
552 } 770 }
553 771
554 /* 772 /*
555 * Set the reap timer. 773 * Set the reap timer.
556 */ 774 */
557 static void rxrpc_set_client_reap_timer(struct !! 775 static void rxrpc_set_client_reap_timer(struct rxrpc_net *rxnet)
558 { 776 {
559 if (!local->kill_all_client_conns) { !! 777 if (!rxnet->kill_all_client_conns) {
560 unsigned long now = jiffies; 778 unsigned long now = jiffies;
561 unsigned long reap_at = now + 779 unsigned long reap_at = now + rxrpc_conn_idle_client_expiry;
562 780
563 if (local->rxnet->live) !! 781 if (rxnet->live)
564 timer_reduce(&local->c !! 782 timer_reduce(&rxnet->client_conn_reap_timer, reap_at);
565 } 783 }
566 } 784 }
567 785
568 /* 786 /*
569 * Disconnect a client call. 787 * Disconnect a client call.
570 */ 788 */
571 void rxrpc_disconnect_client_call(struct rxrpc 789 void rxrpc_disconnect_client_call(struct rxrpc_bundle *bundle, struct rxrpc_call *call)
572 { 790 {
573 struct rxrpc_connection *conn; 791 struct rxrpc_connection *conn;
574 struct rxrpc_channel *chan = NULL; 792 struct rxrpc_channel *chan = NULL;
575 struct rxrpc_local *local = bundle->lo !! 793 struct rxrpc_net *rxnet = bundle->params.local->rxnet;
576 unsigned int channel; 794 unsigned int channel;
577 bool may_reuse; 795 bool may_reuse;
578 u32 cid; 796 u32 cid;
579 797
580 _enter("c=%x", call->debug_id); 798 _enter("c=%x", call->debug_id);
581 799
>> 800 spin_lock(&bundle->channel_lock);
>> 801 set_bit(RXRPC_CALL_DISCONNECTED, &call->flags);
>> 802
582 /* Calls that have never actually been 803 /* Calls that have never actually been assigned a channel can simply be
583 * discarded. 804 * discarded.
584 */ 805 */
585 conn = call->conn; 806 conn = call->conn;
586 if (!conn) { 807 if (!conn) {
587 _debug("call is waiting"); 808 _debug("call is waiting");
588 ASSERTCMP(call->call_id, ==, 0 809 ASSERTCMP(call->call_id, ==, 0);
589 ASSERT(!test_bit(RXRPC_CALL_EX 810 ASSERT(!test_bit(RXRPC_CALL_EXPOSED, &call->flags));
590 /* May still be on ->new_clien !! 811 list_del_init(&call->chan_wait_link);
591 spin_lock(&local->client_call_ !! 812 goto out;
592 list_del_init(&call->wait_link <<
593 spin_unlock(&local->client_cal <<
594 return; <<
595 } 813 }
596 814
597 cid = call->cid; 815 cid = call->cid;
598 channel = cid & RXRPC_CHANNELMASK; 816 channel = cid & RXRPC_CHANNELMASK;
599 chan = &conn->channels[channel]; 817 chan = &conn->channels[channel];
600 trace_rxrpc_client(conn, channel, rxrp 818 trace_rxrpc_client(conn, channel, rxrpc_client_chan_disconnect);
601 819
602 if (WARN_ON(chan->call != call)) !! 820 if (rcu_access_pointer(chan->call) != call) {
603 return; !! 821 spin_unlock(&bundle->channel_lock);
>> 822 BUG();
>> 823 }
604 824
605 may_reuse = rxrpc_may_reuse_conn(conn) 825 may_reuse = rxrpc_may_reuse_conn(conn);
606 826
607 /* If a client call was exposed to the 827 /* If a client call was exposed to the world, we save the result for
608 * retransmission. 828 * retransmission.
609 * 829 *
610 * We use a barrier here so that the c 830 * We use a barrier here so that the call number and abort code can be
611 * read without needing to take a lock 831 * read without needing to take a lock.
612 * 832 *
613 * TODO: Make the incoming packet hand 833 * TODO: Make the incoming packet handler check this and handle
614 * terminal retransmission without req 834 * terminal retransmission without requiring access to the call.
615 */ 835 */
616 if (test_bit(RXRPC_CALL_EXPOSED, &call 836 if (test_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
617 _debug("exposed %u,%u", call-> 837 _debug("exposed %u,%u", call->call_id, call->abort_code);
618 __rxrpc_disconnect_call(conn, 838 __rxrpc_disconnect_call(conn, call);
619 839
620 if (test_and_clear_bit(RXRPC_C 840 if (test_and_clear_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags)) {
621 trace_rxrpc_client(con 841 trace_rxrpc_client(conn, channel, rxrpc_client_to_active);
622 bundle->try_upgrade = 842 bundle->try_upgrade = false;
623 if (may_reuse) 843 if (may_reuse)
624 rxrpc_activate !! 844 rxrpc_activate_channels_locked(bundle);
625 } 845 }
>> 846
626 } 847 }
627 848
628 /* See if we can pass the channel dire 849 /* See if we can pass the channel directly to another call. */
629 if (may_reuse && !list_empty(&bundle-> 850 if (may_reuse && !list_empty(&bundle->waiting_calls)) {
630 trace_rxrpc_client(conn, chann 851 trace_rxrpc_client(conn, channel, rxrpc_client_chan_pass);
631 rxrpc_activate_one_channel(con 852 rxrpc_activate_one_channel(conn, channel);
632 return; !! 853 goto out;
633 } 854 }
634 855
635 /* Schedule the final ACK to be transm 856 /* Schedule the final ACK to be transmitted in a short while so that it
636 * can be skipped if we find a follow- 857 * can be skipped if we find a follow-on call. The first DATA packet
637 * of the follow on call will implicit 858 * of the follow on call will implicitly ACK this call.
638 */ 859 */
639 if (call->completion == RXRPC_CALL_SUC 860 if (call->completion == RXRPC_CALL_SUCCEEDED &&
640 test_bit(RXRPC_CALL_EXPOSED, &call 861 test_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
641 unsigned long final_ack_at = j 862 unsigned long final_ack_at = jiffies + 2;
642 863
643 chan->final_ack_at = final_ack !! 864 WRITE_ONCE(chan->final_ack_at, final_ack_at);
644 smp_wmb(); /* vs rxrpc_process 865 smp_wmb(); /* vs rxrpc_process_delayed_final_acks() */
645 set_bit(RXRPC_CONN_FINAL_ACK_0 866 set_bit(RXRPC_CONN_FINAL_ACK_0 + channel, &conn->flags);
646 rxrpc_reduce_conn_timer(conn, 867 rxrpc_reduce_conn_timer(conn, final_ack_at);
647 } 868 }
648 869
649 /* Deactivate the channel. */ 870 /* Deactivate the channel. */
650 chan->call = NULL; !! 871 rcu_assign_pointer(chan->call, NULL);
651 set_bit(conn->bundle_shift + channel, 872 set_bit(conn->bundle_shift + channel, &conn->bundle->avail_chans);
652 conn->act_chans &= ~(1 << channel); 873 conn->act_chans &= ~(1 << channel);
653 874
654 /* If no channels remain active, then 875 /* If no channels remain active, then put the connection on the idle
655 * list for a short while. Give it a 876 * list for a short while. Give it a ref to stop it going away if it
656 * becomes unbundled. 877 * becomes unbundled.
657 */ 878 */
658 if (!conn->act_chans) { 879 if (!conn->act_chans) {
659 trace_rxrpc_client(conn, chann 880 trace_rxrpc_client(conn, channel, rxrpc_client_to_idle);
660 conn->idle_timestamp = jiffies 881 conn->idle_timestamp = jiffies;
661 882
662 rxrpc_get_connection(conn, rxr !! 883 rxrpc_get_connection(conn);
663 list_move_tail(&conn->cache_li !! 884 spin_lock(&rxnet->client_conn_cache_lock);
>> 885 list_move_tail(&conn->cache_link, &rxnet->idle_client_conns);
>> 886 spin_unlock(&rxnet->client_conn_cache_lock);
664 887
665 rxrpc_set_client_reap_timer(lo !! 888 rxrpc_set_client_reap_timer(rxnet);
666 } 889 }
>> 890
>> 891 out:
>> 892 spin_unlock(&bundle->channel_lock);
>> 893 _leave("");
>> 894 return;
667 } 895 }
668 896
669 /* 897 /*
670 * Remove a connection from a bundle. 898 * Remove a connection from a bundle.
671 */ 899 */
672 static void rxrpc_unbundle_conn(struct rxrpc_c 900 static void rxrpc_unbundle_conn(struct rxrpc_connection *conn)
673 { 901 {
674 struct rxrpc_bundle *bundle = conn->bu 902 struct rxrpc_bundle *bundle = conn->bundle;
>> 903 struct rxrpc_local *local = bundle->params.local;
675 unsigned int bindex; 904 unsigned int bindex;
>> 905 bool need_drop = false, need_put = false;
676 int i; 906 int i;
677 907
678 _enter("C=%x", conn->debug_id); 908 _enter("C=%x", conn->debug_id);
679 909
680 if (conn->flags & RXRPC_CONN_FINAL_ACK 910 if (conn->flags & RXRPC_CONN_FINAL_ACK_MASK)
681 rxrpc_process_delayed_final_ac 911 rxrpc_process_delayed_final_acks(conn, true);
682 912
>> 913 spin_lock(&bundle->channel_lock);
683 bindex = conn->bundle_shift / RXRPC_MA 914 bindex = conn->bundle_shift / RXRPC_MAXCALLS;
684 if (bundle->conns[bindex] == conn) { 915 if (bundle->conns[bindex] == conn) {
685 _debug("clear slot %u", bindex 916 _debug("clear slot %u", bindex);
686 bundle->conns[bindex] = NULL; 917 bundle->conns[bindex] = NULL;
687 bundle->conn_ids[bindex] = 0; <<
688 for (i = 0; i < RXRPC_MAXCALLS 918 for (i = 0; i < RXRPC_MAXCALLS; i++)
689 clear_bit(conn->bundle 919 clear_bit(conn->bundle_shift + i, &bundle->avail_chans);
690 rxrpc_put_client_connection_id !! 920 need_drop = true;
691 rxrpc_deactivate_bundle(bundle <<
692 rxrpc_put_connection(conn, rxr <<
693 } 921 }
694 } !! 922 spin_unlock(&bundle->channel_lock);
695 <<
696 /* <<
697 * Drop the active count on a bundle. <<
698 */ <<
699 void rxrpc_deactivate_bundle(struct rxrpc_bund <<
700 { <<
701 struct rxrpc_local *local; <<
702 bool need_put = false; <<
703 <<
704 if (!bundle) <<
705 return; <<
706 923
707 local = bundle->local; !! 924 /* If there are no more connections, remove the bundle */
708 if (atomic_dec_and_lock(&bundle->activ !! 925 if (!bundle->avail_chans) {
709 if (!bundle->exclusive) { !! 926 _debug("maybe unbundle");
>> 927 spin_lock(&local->client_bundles_lock);
>> 928
>> 929 for (i = 0; i < ARRAY_SIZE(bundle->conns); i++)
>> 930 if (bundle->conns[i])
>> 931 break;
>> 932 if (i == ARRAY_SIZE(bundle->conns) && !bundle->params.exclusive) {
710 _debug("erase bundle") 933 _debug("erase bundle");
711 rb_erase(&bundle->loca 934 rb_erase(&bundle->local_node, &local->client_bundles);
712 need_put = true; 935 need_put = true;
713 } 936 }
714 937
715 spin_unlock(&local->client_bun 938 spin_unlock(&local->client_bundles_lock);
716 if (need_put) 939 if (need_put)
717 rxrpc_put_bundle(bundl !! 940 rxrpc_put_bundle(bundle);
718 } 941 }
>> 942
>> 943 if (need_drop)
>> 944 rxrpc_put_connection(conn);
>> 945 _leave("");
719 } 946 }
720 947
721 /* 948 /*
722 * Clean up a dead client connection. 949 * Clean up a dead client connection.
723 */ 950 */
724 void rxrpc_kill_client_conn(struct rxrpc_conne !! 951 static void rxrpc_kill_client_conn(struct rxrpc_connection *conn)
725 { 952 {
726 struct rxrpc_local *local = conn->loca !! 953 struct rxrpc_local *local = conn->params.local;
727 struct rxrpc_net *rxnet = local->rxnet 954 struct rxrpc_net *rxnet = local->rxnet;
728 955
729 _enter("C=%x", conn->debug_id); 956 _enter("C=%x", conn->debug_id);
730 957
731 trace_rxrpc_client(conn, -1, rxrpc_cli 958 trace_rxrpc_client(conn, -1, rxrpc_client_cleanup);
732 atomic_dec(&rxnet->nr_client_conns); 959 atomic_dec(&rxnet->nr_client_conns);
733 960
734 rxrpc_put_client_connection_id(local, !! 961 rxrpc_put_client_connection_id(conn);
>> 962 rxrpc_kill_connection(conn);
>> 963 }
>> 964
>> 965 /*
>> 966 * Clean up a dead client connections.
>> 967 */
>> 968 void rxrpc_put_client_conn(struct rxrpc_connection *conn)
>> 969 {
>> 970 const void *here = __builtin_return_address(0);
>> 971 unsigned int debug_id = conn->debug_id;
>> 972 bool dead;
>> 973 int r;
>> 974
>> 975 dead = __refcount_dec_and_test(&conn->ref, &r);
>> 976 trace_rxrpc_conn(debug_id, rxrpc_conn_put_client, r - 1, here);
>> 977 if (dead)
>> 978 rxrpc_kill_client_conn(conn);
735 } 979 }
736 980
737 /* 981 /*
738 * Discard expired client connections from the 982 * Discard expired client connections from the idle list. Each conn in the
739 * idle list has been exposed and holds an ext 983 * idle list has been exposed and holds an extra ref because of that.
740 * 984 *
741 * This may be called from conn setup or from 985 * This may be called from conn setup or from a work item so cannot be
742 * considered non-reentrant. 986 * considered non-reentrant.
743 */ 987 */
744 void rxrpc_discard_expired_client_conns(struct !! 988 void rxrpc_discard_expired_client_conns(struct work_struct *work)
745 { 989 {
746 struct rxrpc_connection *conn; 990 struct rxrpc_connection *conn;
>> 991 struct rxrpc_net *rxnet =
>> 992 container_of(work, struct rxrpc_net, client_conn_reaper);
747 unsigned long expiry, conn_expires_at, 993 unsigned long expiry, conn_expires_at, now;
748 unsigned int nr_conns; 994 unsigned int nr_conns;
749 995
750 _enter(""); 996 _enter("");
751 997
>> 998 if (list_empty(&rxnet->idle_client_conns)) {
>> 999 _leave(" [empty]");
>> 1000 return;
>> 1001 }
>> 1002
>> 1003 /* Don't double up on the discarding */
>> 1004 if (!spin_trylock(&rxnet->client_conn_discard_lock)) {
>> 1005 _leave(" [already]");
>> 1006 return;
>> 1007 }
>> 1008
752 /* We keep an estimate of what the num 1009 /* We keep an estimate of what the number of conns ought to be after
753 * we've discarded some so that we don 1010 * we've discarded some so that we don't overdo the discarding.
754 */ 1011 */
755 nr_conns = atomic_read(&local->rxnet-> !! 1012 nr_conns = atomic_read(&rxnet->nr_client_conns);
756 1013
757 next: 1014 next:
758 conn = list_first_entry_or_null(&local !! 1015 spin_lock(&rxnet->client_conn_cache_lock);
759 struct !! 1016
760 if (!conn) !! 1017 if (list_empty(&rxnet->idle_client_conns))
761 return; !! 1018 goto out;
762 1019
763 if (!local->kill_all_client_conns) { !! 1020 conn = list_entry(rxnet->idle_client_conns.next,
>> 1021 struct rxrpc_connection, cache_link);
>> 1022
>> 1023 if (!rxnet->kill_all_client_conns) {
764 /* If the number of connection 1024 /* If the number of connections is over the reap limit, we
765 * expedite discard by reducin 1025 * expedite discard by reducing the expiry timeout. We must,
766 * however, have at least a sh 1026 * however, have at least a short grace period to be able to do
767 * final-ACK or ABORT retransm 1027 * final-ACK or ABORT retransmission.
768 */ 1028 */
769 expiry = rxrpc_conn_idle_clien 1029 expiry = rxrpc_conn_idle_client_expiry;
770 if (nr_conns > rxrpc_reap_clie 1030 if (nr_conns > rxrpc_reap_client_connections)
771 expiry = rxrpc_conn_id 1031 expiry = rxrpc_conn_idle_client_fast_expiry;
772 if (conn->local->service_close !! 1032 if (conn->params.local->service_closed)
773 expiry = rxrpc_closed_ 1033 expiry = rxrpc_closed_conn_expiry * HZ;
774 1034
775 conn_expires_at = conn->idle_t 1035 conn_expires_at = conn->idle_timestamp + expiry;
776 1036
777 now = jiffies; !! 1037 now = READ_ONCE(jiffies);
778 if (time_after(conn_expires_at 1038 if (time_after(conn_expires_at, now))
779 goto not_yet_expired; 1039 goto not_yet_expired;
780 } 1040 }
781 1041
782 atomic_dec(&conn->active); <<
783 trace_rxrpc_client(conn, -1, rxrpc_cli 1042 trace_rxrpc_client(conn, -1, rxrpc_client_discard);
784 list_del_init(&conn->cache_link); 1043 list_del_init(&conn->cache_link);
785 1044
>> 1045 spin_unlock(&rxnet->client_conn_cache_lock);
>> 1046
786 rxrpc_unbundle_conn(conn); 1047 rxrpc_unbundle_conn(conn);
787 /* Drop the ->cache_link ref */ !! 1048 rxrpc_put_connection(conn); /* Drop the ->cache_link ref */
788 rxrpc_put_connection(conn, rxrpc_conn_ <<
789 1049
790 nr_conns--; 1050 nr_conns--;
791 goto next; 1051 goto next;
792 1052
793 not_yet_expired: 1053 not_yet_expired:
794 /* The connection at the front of the 1054 /* The connection at the front of the queue hasn't yet expired, so
795 * schedule the work item for that poi 1055 * schedule the work item for that point if we discarded something.
796 * 1056 *
797 * We don't worry if the work item is 1057 * We don't worry if the work item is already scheduled - it can look
798 * after rescheduling itself at a late 1058 * after rescheduling itself at a later time. We could cancel it, but
799 * then things get messier. 1059 * then things get messier.
800 */ 1060 */
801 _debug("not yet"); 1061 _debug("not yet");
802 if (!local->kill_all_client_conns) !! 1062 if (!rxnet->kill_all_client_conns)
803 timer_reduce(&local->client_co !! 1063 timer_reduce(&rxnet->client_conn_reap_timer, conn_expires_at);
>> 1064
>> 1065 out:
>> 1066 spin_unlock(&rxnet->client_conn_cache_lock);
>> 1067 spin_unlock(&rxnet->client_conn_discard_lock);
>> 1068 _leave("");
>> 1069 }
>> 1070
>> 1071 /*
>> 1072 * Preemptively destroy all the client connection records rather than waiting
>> 1073 * for them to time out
>> 1074 */
>> 1075 void rxrpc_destroy_all_client_connections(struct rxrpc_net *rxnet)
>> 1076 {
>> 1077 _enter("");
>> 1078
>> 1079 spin_lock(&rxnet->client_conn_cache_lock);
>> 1080 rxnet->kill_all_client_conns = true;
>> 1081 spin_unlock(&rxnet->client_conn_cache_lock);
>> 1082
>> 1083 del_timer_sync(&rxnet->client_conn_reap_timer);
>> 1084
>> 1085 if (!rxrpc_queue_work(&rxnet->client_conn_reaper))
>> 1086 _debug("destroy: queue failed");
804 1087
805 _leave(""); 1088 _leave("");
806 } 1089 }
807 1090
808 /* 1091 /*
809 * Clean up the client connections on a local 1092 * Clean up the client connections on a local endpoint.
810 */ 1093 */
811 void rxrpc_clean_up_local_conns(struct rxrpc_l 1094 void rxrpc_clean_up_local_conns(struct rxrpc_local *local)
812 { 1095 {
813 struct rxrpc_connection *conn; !! 1096 struct rxrpc_connection *conn, *tmp;
>> 1097 struct rxrpc_net *rxnet = local->rxnet;
>> 1098 LIST_HEAD(graveyard);
814 1099
815 _enter(""); 1100 _enter("");
816 1101
817 local->kill_all_client_conns = true; !! 1102 spin_lock(&rxnet->client_conn_cache_lock);
>> 1103
>> 1104 list_for_each_entry_safe(conn, tmp, &rxnet->idle_client_conns,
>> 1105 cache_link) {
>> 1106 if (conn->params.local == local) {
>> 1107 trace_rxrpc_client(conn, -1, rxrpc_client_discard);
>> 1108 list_move(&conn->cache_link, &graveyard);
>> 1109 }
>> 1110 }
818 1111
819 del_timer_sync(&local->client_conn_rea !! 1112 spin_unlock(&rxnet->client_conn_cache_lock);
820 1113
821 while ((conn = list_first_entry_or_nul !! 1114 while (!list_empty(&graveyard)) {
822 !! 1115 conn = list_entry(graveyard.next,
>> 1116 struct rxrpc_connection, cache_link);
823 list_del_init(&conn->cache_lin 1117 list_del_init(&conn->cache_link);
824 atomic_dec(&conn->active); <<
825 trace_rxrpc_client(conn, -1, r <<
826 rxrpc_unbundle_conn(conn); 1118 rxrpc_unbundle_conn(conn);
827 rxrpc_put_connection(conn, rxr !! 1119 rxrpc_put_connection(conn);
828 } 1120 }
829 1121
830 _leave(" [culled]"); 1122 _leave(" [culled]");
831 } 1123 }
832 1124
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.