1 // SPDX-License-Identifier: GPL-2.0-or-later 1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Client connection-specific management code. 2 /* Client connection-specific management code.
3 * 3 *
4 * Copyright (C) 2016, 2020 Red Hat, Inc. All 4 * Copyright (C) 2016, 2020 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.c 5 * Written by David Howells (dhowells@redhat.com)
6 * 6 *
7 * Client connections need to be cached for a 7 * Client connections need to be cached for a little while after they've made a
8 * call so as to handle retransmitted DATA pac 8 * call so as to handle retransmitted DATA packets in case the server didn't
9 * receive the final ACK or terminating ABORT 9 * receive the final ACK or terminating ABORT we sent it.
10 * 10 *
11 * There are flags of relevance to the cache: 11 * There are flags of relevance to the cache:
12 * 12 *
13 * (2) DONT_REUSE - The connection should be 13 * (2) DONT_REUSE - The connection should be discarded as soon as possible and
14 * should not be reused. This is set whe 14 * should not be reused. This is set when an exclusive connection is used
15 * or a call ID counter overflows. 15 * or a call ID counter overflows.
16 * 16 *
17 * The caching state may only be changed if th 17 * The caching state may only be changed if the cache lock is held.
18 * 18 *
19 * There are two idle client connection expiry 19 * There are two idle client connection expiry durations. If the total number
20 * of connections is below the reap threshold, 20 * of connections is below the reap threshold, we use the normal duration; if
21 * it's above, we use the fast duration. 21 * it's above, we use the fast duration.
22 */ 22 */
23 23
24 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
25 25
26 #include <linux/slab.h> 26 #include <linux/slab.h>
27 #include <linux/idr.h> 27 #include <linux/idr.h>
28 #include <linux/timer.h> 28 #include <linux/timer.h>
29 #include <linux/sched/signal.h> 29 #include <linux/sched/signal.h>
30 30
31 #include "ar-internal.h" 31 #include "ar-internal.h"
32 32
33 __read_mostly unsigned int rxrpc_reap_client_c 33 __read_mostly unsigned int rxrpc_reap_client_connections = 900;
34 __read_mostly unsigned long rxrpc_conn_idle_cl 34 __read_mostly unsigned long rxrpc_conn_idle_client_expiry = 2 * 60 * HZ;
35 __read_mostly unsigned long rxrpc_conn_idle_cl 35 __read_mostly unsigned long rxrpc_conn_idle_client_fast_expiry = 2 * HZ;
36 36
37 static void rxrpc_activate_bundle(struct rxrpc !! 37 /*
>> 38 * We use machine-unique IDs for our client connections.
>> 39 */
>> 40 DEFINE_IDR(rxrpc_client_conn_ids);
>> 41 static DEFINE_SPINLOCK(rxrpc_conn_id_lock);
>> 42
>> 43 static void rxrpc_deactivate_bundle(struct rxrpc_bundle *bundle);
>> 44
>> 45 /*
>> 46 * Get a connection ID and epoch for a client connection from the global pool.
>> 47 * The connection struct pointer is then recorded in the idr radix tree. The
>> 48 * epoch doesn't change until the client is rebooted (or, at least, unless the
>> 49 * module is unloaded).
>> 50 */
>> 51 static int rxrpc_get_client_connection_id(struct rxrpc_connection *conn,
>> 52 gfp_t gfp)
38 { 53 {
39 atomic_inc(&bundle->active); !! 54 struct rxrpc_net *rxnet = conn->params.local->rxnet;
>> 55 int id;
>> 56
>> 57 _enter("");
>> 58
>> 59 idr_preload(gfp);
>> 60 spin_lock(&rxrpc_conn_id_lock);
>> 61
>> 62 id = idr_alloc_cyclic(&rxrpc_client_conn_ids, conn,
>> 63 1, 0x40000000, GFP_NOWAIT);
>> 64 if (id < 0)
>> 65 goto error;
>> 66
>> 67 spin_unlock(&rxrpc_conn_id_lock);
>> 68 idr_preload_end();
>> 69
>> 70 conn->proto.epoch = rxnet->epoch;
>> 71 conn->proto.cid = id << RXRPC_CIDSHIFT;
>> 72 set_bit(RXRPC_CONN_HAS_IDR, &conn->flags);
>> 73 _leave(" [CID %x]", conn->proto.cid);
>> 74 return 0;
>> 75
>> 76 error:
>> 77 spin_unlock(&rxrpc_conn_id_lock);
>> 78 idr_preload_end();
>> 79 _leave(" = %d", id);
>> 80 return id;
40 } 81 }
41 82
42 /* 83 /*
43 * Release a connection ID for a client connec !! 84 * Release a connection ID for a client connection from the global pool.
44 */ 85 */
45 static void rxrpc_put_client_connection_id(str !! 86 static void rxrpc_put_client_connection_id(struct rxrpc_connection *conn)
46 str <<
47 { 87 {
48 idr_remove(&local->conn_ids, conn->pro !! 88 if (test_bit(RXRPC_CONN_HAS_IDR, &conn->flags)) {
>> 89 spin_lock(&rxrpc_conn_id_lock);
>> 90 idr_remove(&rxrpc_client_conn_ids,
>> 91 conn->proto.cid >> RXRPC_CIDSHIFT);
>> 92 spin_unlock(&rxrpc_conn_id_lock);
>> 93 }
49 } 94 }
50 95
51 /* 96 /*
52 * Destroy the client connection ID tree. 97 * Destroy the client connection ID tree.
53 */ 98 */
54 static void rxrpc_destroy_client_conn_ids(stru !! 99 void rxrpc_destroy_client_conn_ids(void)
55 { 100 {
56 struct rxrpc_connection *conn; 101 struct rxrpc_connection *conn;
57 int id; 102 int id;
58 103
59 if (!idr_is_empty(&local->conn_ids)) { !! 104 if (!idr_is_empty(&rxrpc_client_conn_ids)) {
60 idr_for_each_entry(&local->con !! 105 idr_for_each_entry(&rxrpc_client_conn_ids, conn, id) {
61 pr_err("AF_RXRPC: Leak 106 pr_err("AF_RXRPC: Leaked client conn %p {%d}\n",
62 conn, refcount_ 107 conn, refcount_read(&conn->ref));
63 } 108 }
64 BUG(); 109 BUG();
65 } 110 }
66 111
67 idr_destroy(&local->conn_ids); !! 112 idr_destroy(&rxrpc_client_conn_ids);
68 } 113 }
69 114
70 /* 115 /*
71 * Allocate a connection bundle. 116 * Allocate a connection bundle.
72 */ 117 */
73 static struct rxrpc_bundle *rxrpc_alloc_bundle !! 118 static struct rxrpc_bundle *rxrpc_alloc_bundle(struct rxrpc_conn_parameters *cp,
74 119 gfp_t gfp)
75 { 120 {
76 static atomic_t rxrpc_bundle_id; <<
77 struct rxrpc_bundle *bundle; 121 struct rxrpc_bundle *bundle;
78 122
79 bundle = kzalloc(sizeof(*bundle), gfp) 123 bundle = kzalloc(sizeof(*bundle), gfp);
80 if (bundle) { 124 if (bundle) {
81 bundle->local = call !! 125 bundle->params = *cp;
82 bundle->peer = rxrp !! 126 rxrpc_get_peer(bundle->params.peer);
83 bundle->key = key_ <<
84 bundle->security = call <<
85 bundle->exclusive = test <<
86 bundle->upgrade = test <<
87 bundle->service_id = call <<
88 bundle->security_level = call <<
89 bundle->debug_id = atom <<
90 refcount_set(&bundle->ref, 1); 127 refcount_set(&bundle->ref, 1);
91 atomic_set(&bundle->active, 1) 128 atomic_set(&bundle->active, 1);
>> 129 spin_lock_init(&bundle->channel_lock);
92 INIT_LIST_HEAD(&bundle->waitin 130 INIT_LIST_HEAD(&bundle->waiting_calls);
93 trace_rxrpc_bundle(bundle->deb <<
94 <<
95 write_lock(&bundle->local->rxn <<
96 list_add_tail(&bundle->proc_li <<
97 write_unlock(&bundle->local->r <<
98 } 131 }
99 return bundle; 132 return bundle;
100 } 133 }
101 134
102 struct rxrpc_bundle *rxrpc_get_bundle(struct r !! 135 struct rxrpc_bundle *rxrpc_get_bundle(struct rxrpc_bundle *bundle)
103 enum rxr <<
104 { 136 {
105 int r; !! 137 refcount_inc(&bundle->ref);
106 <<
107 __refcount_inc(&bundle->ref, &r); <<
108 trace_rxrpc_bundle(bundle->debug_id, r <<
109 return bundle; 138 return bundle;
110 } 139 }
111 140
112 static void rxrpc_free_bundle(struct rxrpc_bun 141 static void rxrpc_free_bundle(struct rxrpc_bundle *bundle)
113 { 142 {
114 trace_rxrpc_bundle(bundle->debug_id, r !! 143 rxrpc_put_peer(bundle->params.peer);
115 rxrpc_bundle_free); <<
116 write_lock(&bundle->local->rxnet->conn <<
117 list_del(&bundle->proc_link); <<
118 write_unlock(&bundle->local->rxnet->co <<
119 rxrpc_put_peer(bundle->peer, rxrpc_pee <<
120 key_put(bundle->key); <<
121 kfree(bundle); 144 kfree(bundle);
122 } 145 }
123 146
124 void rxrpc_put_bundle(struct rxrpc_bundle *bun !! 147 void rxrpc_put_bundle(struct rxrpc_bundle *bundle)
125 { 148 {
126 unsigned int id; !! 149 unsigned int d = bundle->debug_id;
127 bool dead; 150 bool dead;
128 int r; 151 int r;
129 152
130 if (bundle) { !! 153 dead = __refcount_dec_and_test(&bundle->ref, &r);
131 id = bundle->debug_id; <<
132 dead = __refcount_dec_and_test <<
133 trace_rxrpc_bundle(id, r - 1, <<
134 if (dead) <<
135 rxrpc_free_bundle(bund <<
136 } <<
137 } <<
138 154
139 /* !! 155 _debug("PUT B=%x %d", d, r - 1);
140 * Get rid of outstanding client connection pr !! 156 if (dead)
141 * endpoint is destroyed. !! 157 rxrpc_free_bundle(bundle);
142 */ <<
143 void rxrpc_purge_client_connections(struct rxr <<
144 { <<
145 rxrpc_destroy_client_conn_ids(local); <<
146 } 158 }
147 159
148 /* 160 /*
149 * Allocate a client connection. 161 * Allocate a client connection.
150 */ 162 */
151 static struct rxrpc_connection * 163 static struct rxrpc_connection *
152 rxrpc_alloc_client_connection(struct rxrpc_bun !! 164 rxrpc_alloc_client_connection(struct rxrpc_bundle *bundle, gfp_t gfp)
153 { 165 {
154 struct rxrpc_connection *conn; 166 struct rxrpc_connection *conn;
155 struct rxrpc_local *local = bundle->lo !! 167 struct rxrpc_net *rxnet = bundle->params.local->rxnet;
156 struct rxrpc_net *rxnet = local->rxnet !! 168 int ret;
157 int id; <<
158 169
159 _enter(""); 170 _enter("");
160 171
161 conn = rxrpc_alloc_connection(rxnet, G !! 172 conn = rxrpc_alloc_connection(gfp);
162 if (!conn) !! 173 if (!conn) {
>> 174 _leave(" = -ENOMEM");
163 return ERR_PTR(-ENOMEM); 175 return ERR_PTR(-ENOMEM);
164 <<
165 id = idr_alloc_cyclic(&local->conn_ids <<
166 GFP_ATOMIC | __G <<
167 if (id < 0) { <<
168 kfree(conn); <<
169 return ERR_PTR(id); <<
170 } 176 }
171 177
172 refcount_set(&conn->ref, 1); 178 refcount_set(&conn->ref, 1);
173 conn->proto.cid = id << RXRPC_ !! 179 conn->bundle = bundle;
174 conn->proto.epoch = local->rxnet !! 180 conn->params = bundle->params;
175 conn->out_clientflag = RXRPC_CLIENT 181 conn->out_clientflag = RXRPC_CLIENT_INITIATED;
176 conn->bundle = rxrpc_get_bu !! 182 conn->state = RXRPC_CONN_CLIENT;
177 conn->local = rxrpc_get_lo !! 183 conn->service_id = conn->params.service_id;
178 conn->peer = rxrpc_get_pe <<
179 conn->key = key_get(bund <<
180 conn->security = bundle->secu <<
181 conn->exclusive = bundle->excl <<
182 conn->upgrade = bundle->upgr <<
183 conn->orig_service_id = bundle->serv <<
184 conn->security_level = bundle->secu <<
185 conn->state = RXRPC_CONN_C <<
186 conn->service_id = conn->orig_s <<
187 184
188 if (conn->security == &rxrpc_no_securi !! 185 ret = rxrpc_get_client_connection_id(conn, gfp);
189 conn->state = RXRPC_CONN_C !! 186 if (ret < 0)
>> 187 goto error_0;
>> 188
>> 189 ret = rxrpc_init_client_conn_security(conn);
>> 190 if (ret < 0)
>> 191 goto error_1;
190 192
191 atomic_inc(&rxnet->nr_conns); 193 atomic_inc(&rxnet->nr_conns);
192 write_lock(&rxnet->conn_lock); 194 write_lock(&rxnet->conn_lock);
193 list_add_tail(&conn->proc_link, &rxnet 195 list_add_tail(&conn->proc_link, &rxnet->conn_proc_list);
194 write_unlock(&rxnet->conn_lock); 196 write_unlock(&rxnet->conn_lock);
195 197
196 rxrpc_see_connection(conn, rxrpc_conn_ !! 198 rxrpc_get_bundle(bundle);
>> 199 rxrpc_get_peer(conn->params.peer);
>> 200 rxrpc_get_local(conn->params.local);
>> 201 key_get(conn->params.key);
>> 202
>> 203 trace_rxrpc_conn(conn->debug_id, rxrpc_conn_new_client,
>> 204 refcount_read(&conn->ref),
>> 205 __builtin_return_address(0));
197 206
198 atomic_inc(&rxnet->nr_client_conns); 207 atomic_inc(&rxnet->nr_client_conns);
199 trace_rxrpc_client(conn, -1, rxrpc_cli 208 trace_rxrpc_client(conn, -1, rxrpc_client_alloc);
>> 209 _leave(" = %p", conn);
200 return conn; 210 return conn;
>> 211
>> 212 error_1:
>> 213 rxrpc_put_client_connection_id(conn);
>> 214 error_0:
>> 215 kfree(conn);
>> 216 _leave(" = %d", ret);
>> 217 return ERR_PTR(ret);
201 } 218 }
202 219
203 /* 220 /*
204 * Determine if a connection may be reused. 221 * Determine if a connection may be reused.
205 */ 222 */
206 static bool rxrpc_may_reuse_conn(struct rxrpc_ 223 static bool rxrpc_may_reuse_conn(struct rxrpc_connection *conn)
207 { 224 {
208 struct rxrpc_net *rxnet; 225 struct rxrpc_net *rxnet;
209 int id_cursor, id, distance, limit; 226 int id_cursor, id, distance, limit;
210 227
211 if (!conn) 228 if (!conn)
212 goto dont_reuse; 229 goto dont_reuse;
213 230
214 rxnet = conn->rxnet; !! 231 rxnet = conn->params.local->rxnet;
215 if (test_bit(RXRPC_CONN_DONT_REUSE, &c 232 if (test_bit(RXRPC_CONN_DONT_REUSE, &conn->flags))
216 goto dont_reuse; 233 goto dont_reuse;
217 234
218 if ((conn->state != RXRPC_CONN_CLIENT_ !! 235 if (conn->state != RXRPC_CONN_CLIENT ||
219 conn->state != RXRPC_CONN_CLIENT) <<
220 conn->proto.epoch != rxnet->epoch) 236 conn->proto.epoch != rxnet->epoch)
221 goto mark_dont_reuse; 237 goto mark_dont_reuse;
222 238
223 /* The IDR tree gets very expensive on 239 /* The IDR tree gets very expensive on memory if the connection IDs are
224 * widely scattered throughout the num 240 * widely scattered throughout the number space, so we shall want to
225 * kill off connections that, say, hav 241 * kill off connections that, say, have an ID more than about four
226 * times the maximum number of client 242 * times the maximum number of client conns away from the current
227 * allocation point to try and keep th 243 * allocation point to try and keep the IDs concentrated.
228 */ 244 */
229 id_cursor = idr_get_cursor(&conn->loca !! 245 id_cursor = idr_get_cursor(&rxrpc_client_conn_ids);
230 id = conn->proto.cid >> RXRPC_CIDSHIFT 246 id = conn->proto.cid >> RXRPC_CIDSHIFT;
231 distance = id - id_cursor; 247 distance = id - id_cursor;
232 if (distance < 0) 248 if (distance < 0)
233 distance = -distance; 249 distance = -distance;
234 limit = max_t(unsigned long, atomic_re 250 limit = max_t(unsigned long, atomic_read(&rxnet->nr_conns) * 4, 1024);
235 if (distance > limit) 251 if (distance > limit)
236 goto mark_dont_reuse; 252 goto mark_dont_reuse;
237 253
238 return true; 254 return true;
239 255
240 mark_dont_reuse: 256 mark_dont_reuse:
241 set_bit(RXRPC_CONN_DONT_REUSE, &conn-> 257 set_bit(RXRPC_CONN_DONT_REUSE, &conn->flags);
242 dont_reuse: 258 dont_reuse:
243 return false; 259 return false;
244 } 260 }
245 261
246 /* 262 /*
247 * Look up the conn bundle that matches the co 263 * Look up the conn bundle that matches the connection parameters, adding it if
248 * it doesn't yet exist. 264 * it doesn't yet exist.
249 */ 265 */
250 int rxrpc_look_up_bundle(struct rxrpc_call *ca !! 266 static struct rxrpc_bundle *rxrpc_look_up_bundle(struct rxrpc_conn_parameters *cp,
>> 267 gfp_t gfp)
251 { 268 {
>> 269 static atomic_t rxrpc_bundle_id;
252 struct rxrpc_bundle *bundle, *candidat 270 struct rxrpc_bundle *bundle, *candidate;
253 struct rxrpc_local *local = call->loca !! 271 struct rxrpc_local *local = cp->local;
254 struct rb_node *p, **pp, *parent; 272 struct rb_node *p, **pp, *parent;
255 long diff; 273 long diff;
256 bool upgrade = test_bit(RXRPC_CALL_UPG <<
257 274
258 _enter("{%px,%x,%u,%u}", 275 _enter("{%px,%x,%u,%u}",
259 call->peer, key_serial(call->ke !! 276 cp->peer, key_serial(cp->key), cp->security_level, cp->upgrade);
260 upgrade); <<
261 277
262 if (test_bit(RXRPC_CALL_EXCLUSIVE, &ca !! 278 if (cp->exclusive)
263 call->bundle = rxrpc_alloc_bun !! 279 return rxrpc_alloc_bundle(cp, gfp);
264 return call->bundle ? 0 : -ENO <<
265 } <<
266 280
267 /* First, see if the bundle is already 281 /* First, see if the bundle is already there. */
268 _debug("search 1"); 282 _debug("search 1");
269 spin_lock(&local->client_bundles_lock) 283 spin_lock(&local->client_bundles_lock);
270 p = local->client_bundles.rb_node; 284 p = local->client_bundles.rb_node;
271 while (p) { 285 while (p) {
272 bundle = rb_entry(p, struct rx 286 bundle = rb_entry(p, struct rxrpc_bundle, local_node);
273 287
274 #define cmp(X, Y) ((long)(X) - (long)(Y)) !! 288 #define cmp(X) ((long)bundle->params.X - (long)cp->X)
275 diff = (cmp(bundle->peer, call !! 289 diff = (cmp(peer) ?:
276 cmp(bundle->key, call- !! 290 cmp(key) ?:
277 cmp(bundle->security_l !! 291 cmp(security_level) ?:
278 cmp(bundle->upgrade, u !! 292 cmp(upgrade));
279 #undef cmp 293 #undef cmp
280 if (diff < 0) 294 if (diff < 0)
281 p = p->rb_left; 295 p = p->rb_left;
282 else if (diff > 0) 296 else if (diff > 0)
283 p = p->rb_right; 297 p = p->rb_right;
284 else 298 else
285 goto found_bundle; 299 goto found_bundle;
286 } 300 }
287 spin_unlock(&local->client_bundles_loc 301 spin_unlock(&local->client_bundles_lock);
288 _debug("not found"); 302 _debug("not found");
289 303
290 /* It wasn't. We need to add one. */ 304 /* It wasn't. We need to add one. */
291 candidate = rxrpc_alloc_bundle(call, g !! 305 candidate = rxrpc_alloc_bundle(cp, gfp);
292 if (!candidate) 306 if (!candidate)
293 return -ENOMEM; !! 307 return NULL;
294 308
295 _debug("search 2"); 309 _debug("search 2");
296 spin_lock(&local->client_bundles_lock) 310 spin_lock(&local->client_bundles_lock);
297 pp = &local->client_bundles.rb_node; 311 pp = &local->client_bundles.rb_node;
298 parent = NULL; 312 parent = NULL;
299 while (*pp) { 313 while (*pp) {
300 parent = *pp; 314 parent = *pp;
301 bundle = rb_entry(parent, stru 315 bundle = rb_entry(parent, struct rxrpc_bundle, local_node);
302 316
303 #define cmp(X, Y) ((long)(X) - (long)(Y)) !! 317 #define cmp(X) ((long)bundle->params.X - (long)cp->X)
304 diff = (cmp(bundle->peer, call !! 318 diff = (cmp(peer) ?:
305 cmp(bundle->key, call- !! 319 cmp(key) ?:
306 cmp(bundle->security_l !! 320 cmp(security_level) ?:
307 cmp(bundle->upgrade, u !! 321 cmp(upgrade));
308 #undef cmp 322 #undef cmp
309 if (diff < 0) 323 if (diff < 0)
310 pp = &(*pp)->rb_left; 324 pp = &(*pp)->rb_left;
311 else if (diff > 0) 325 else if (diff > 0)
312 pp = &(*pp)->rb_right; 326 pp = &(*pp)->rb_right;
313 else 327 else
314 goto found_bundle_free 328 goto found_bundle_free;
315 } 329 }
316 330
317 _debug("new bundle"); 331 _debug("new bundle");
>> 332 candidate->debug_id = atomic_inc_return(&rxrpc_bundle_id);
318 rb_link_node(&candidate->local_node, p 333 rb_link_node(&candidate->local_node, parent, pp);
319 rb_insert_color(&candidate->local_node 334 rb_insert_color(&candidate->local_node, &local->client_bundles);
320 call->bundle = rxrpc_get_bundle(candid !! 335 rxrpc_get_bundle(candidate);
321 spin_unlock(&local->client_bundles_loc 336 spin_unlock(&local->client_bundles_lock);
322 _leave(" = B=%u [new]", call->bundle-> !! 337 _leave(" = %u [new]", candidate->debug_id);
323 return 0; !! 338 return candidate;
324 339
325 found_bundle_free: 340 found_bundle_free:
326 rxrpc_free_bundle(candidate); 341 rxrpc_free_bundle(candidate);
327 found_bundle: 342 found_bundle:
328 call->bundle = rxrpc_get_bundle(bundle !! 343 rxrpc_get_bundle(bundle);
329 rxrpc_activate_bundle(bundle); !! 344 atomic_inc(&bundle->active);
330 spin_unlock(&local->client_bundles_loc 345 spin_unlock(&local->client_bundles_lock);
331 _leave(" = B=%u [found]", call->bundle !! 346 _leave(" = %u [found]", bundle->debug_id);
332 return 0; !! 347 return bundle;
>> 348 }
>> 349
>> 350 /*
>> 351 * Create or find a client bundle to use for a call.
>> 352 *
>> 353 * If we return with a connection, the call will be on its waiting list. It's
>> 354 * left to the caller to assign a channel and wake up the call.
>> 355 */
>> 356 static struct rxrpc_bundle *rxrpc_prep_call(struct rxrpc_sock *rx,
>> 357 struct rxrpc_call *call,
>> 358 struct rxrpc_conn_parameters *cp,
>> 359 struct sockaddr_rxrpc *srx,
>> 360 gfp_t gfp)
>> 361 {
>> 362 struct rxrpc_bundle *bundle;
>> 363
>> 364 _enter("{%d,%lx},", call->debug_id, call->user_call_ID);
>> 365
>> 366 cp->peer = rxrpc_lookup_peer(rx, cp->local, srx, gfp);
>> 367 if (!cp->peer)
>> 368 goto error;
>> 369
>> 370 call->cong_cwnd = cp->peer->cong_cwnd;
>> 371 if (call->cong_cwnd >= call->cong_ssthresh)
>> 372 call->cong_mode = RXRPC_CALL_CONGEST_AVOIDANCE;
>> 373 else
>> 374 call->cong_mode = RXRPC_CALL_SLOW_START;
>> 375 if (cp->upgrade)
>> 376 __set_bit(RXRPC_CALL_UPGRADE, &call->flags);
>> 377
>> 378 /* Find the client connection bundle. */
>> 379 bundle = rxrpc_look_up_bundle(cp, gfp);
>> 380 if (!bundle)
>> 381 goto error;
>> 382
>> 383 /* Get this call queued. Someone else may activate it whilst we're
>> 384 * lining up a new connection, but that's fine.
>> 385 */
>> 386 spin_lock(&bundle->channel_lock);
>> 387 list_add_tail(&call->chan_wait_link, &bundle->waiting_calls);
>> 388 spin_unlock(&bundle->channel_lock);
>> 389
>> 390 _leave(" = [B=%x]", bundle->debug_id);
>> 391 return bundle;
>> 392
>> 393 error:
>> 394 _leave(" = -ENOMEM");
>> 395 return ERR_PTR(-ENOMEM);
333 } 396 }
334 397
335 /* 398 /*
336 * Allocate a new connection and add it into a 399 * Allocate a new connection and add it into a bundle.
337 */ 400 */
338 static bool rxrpc_add_conn_to_bundle(struct rx !! 401 static void rxrpc_add_conn_to_bundle(struct rxrpc_bundle *bundle, gfp_t gfp)
339 unsigned !! 402 __releases(bundle->channel_lock)
340 { 403 {
341 struct rxrpc_connection *conn, *old; !! 404 struct rxrpc_connection *candidate = NULL, *old = NULL;
342 unsigned int shift = slot * RXRPC_MAXC !! 405 bool conflict;
343 unsigned int i; !! 406 int i;
344 !! 407
345 old = bundle->conns[slot]; !! 408 _enter("");
346 if (old) { !! 409
347 bundle->conns[slot] = NULL; !! 410 conflict = bundle->alloc_conn;
348 bundle->conn_ids[slot] = 0; !! 411 if (!conflict)
349 trace_rxrpc_client(old, -1, rx !! 412 bundle->alloc_conn = true;
350 rxrpc_put_connection(old, rxrp !! 413 spin_unlock(&bundle->channel_lock);
351 } !! 414 if (conflict) {
352 !! 415 _leave(" [conf]");
353 conn = rxrpc_alloc_client_connection(b !! 416 return;
354 if (IS_ERR(conn)) { !! 417 }
355 bundle->alloc_error = PTR_ERR( !! 418
356 return false; !! 419 candidate = rxrpc_alloc_client_connection(bundle, gfp);
357 } !! 420
358 !! 421 spin_lock(&bundle->channel_lock);
359 rxrpc_activate_bundle(bundle); !! 422 bundle->alloc_conn = false;
360 conn->bundle_shift = shift; !! 423
361 bundle->conns[slot] = conn; !! 424 if (IS_ERR(candidate)) {
362 bundle->conn_ids[slot] = conn->debug_i !! 425 bundle->alloc_error = PTR_ERR(candidate);
363 for (i = 0; i < RXRPC_MAXCALLS; i++) !! 426 spin_unlock(&bundle->channel_lock);
364 set_bit(shift + i, &bundle->av !! 427 _leave(" [err %ld]", PTR_ERR(candidate));
365 return true; !! 428 return;
>> 429 }
>> 430
>> 431 bundle->alloc_error = 0;
>> 432
>> 433 for (i = 0; i < ARRAY_SIZE(bundle->conns); i++) {
>> 434 unsigned int shift = i * RXRPC_MAXCALLS;
>> 435 int j;
>> 436
>> 437 old = bundle->conns[i];
>> 438 if (!rxrpc_may_reuse_conn(old)) {
>> 439 if (old)
>> 440 trace_rxrpc_client(old, -1, rxrpc_client_replace);
>> 441 candidate->bundle_shift = shift;
>> 442 atomic_inc(&bundle->active);
>> 443 bundle->conns[i] = candidate;
>> 444 for (j = 0; j < RXRPC_MAXCALLS; j++)
>> 445 set_bit(shift + j, &bundle->avail_chans);
>> 446 candidate = NULL;
>> 447 break;
>> 448 }
>> 449
>> 450 old = NULL;
>> 451 }
>> 452
>> 453 spin_unlock(&bundle->channel_lock);
>> 454
>> 455 if (candidate) {
>> 456 _debug("discard C=%x", candidate->debug_id);
>> 457 trace_rxrpc_client(candidate, -1, rxrpc_client_duplicate);
>> 458 rxrpc_put_connection(candidate);
>> 459 }
>> 460
>> 461 rxrpc_put_connection(old);
>> 462 _leave("");
366 } 463 }
367 464
368 /* 465 /*
369 * Add a connection to a bundle if there are n 466 * Add a connection to a bundle if there are no usable connections or we have
370 * connections waiting for extra capacity. 467 * connections waiting for extra capacity.
371 */ 468 */
372 static bool rxrpc_bundle_has_space(struct rxrp !! 469 static void rxrpc_maybe_add_conn(struct rxrpc_bundle *bundle, gfp_t gfp)
373 { 470 {
374 int slot = -1, i, usable; !! 471 struct rxrpc_call *call;
>> 472 int i, usable;
375 473
376 _enter(""); 474 _enter("");
377 475
378 bundle->alloc_error = 0; !! 476 spin_lock(&bundle->channel_lock);
379 477
380 /* See if there are any usable connect 478 /* See if there are any usable connections. */
381 usable = 0; 479 usable = 0;
382 for (i = 0; i < ARRAY_SIZE(bundle->con !! 480 for (i = 0; i < ARRAY_SIZE(bundle->conns); i++)
383 if (rxrpc_may_reuse_conn(bundl 481 if (rxrpc_may_reuse_conn(bundle->conns[i]))
384 usable++; 482 usable++;
385 else if (slot == -1) <<
386 slot = i; <<
387 } <<
388 483
389 if (!usable && bundle->upgrade) !! 484 if (!usable && !list_empty(&bundle->waiting_calls)) {
390 bundle->try_upgrade = true; !! 485 call = list_first_entry(&bundle->waiting_calls,
>> 486 struct rxrpc_call, chan_wait_link);
>> 487 if (test_bit(RXRPC_CALL_UPGRADE, &call->flags))
>> 488 bundle->try_upgrade = true;
>> 489 }
391 490
392 if (!usable) 491 if (!usable)
393 goto alloc_conn; 492 goto alloc_conn;
394 493
395 if (!bundle->avail_chans && 494 if (!bundle->avail_chans &&
396 !bundle->try_upgrade && 495 !bundle->try_upgrade &&
>> 496 !list_empty(&bundle->waiting_calls) &&
397 usable < ARRAY_SIZE(bundle->conns) 497 usable < ARRAY_SIZE(bundle->conns))
398 goto alloc_conn; 498 goto alloc_conn;
399 499
>> 500 spin_unlock(&bundle->channel_lock);
400 _leave(""); 501 _leave("");
401 return usable; !! 502 return;
402 503
403 alloc_conn: 504 alloc_conn:
404 return slot >= 0 ? rxrpc_add_conn_to_b !! 505 return rxrpc_add_conn_to_bundle(bundle, gfp);
405 } 506 }
406 507
407 /* 508 /*
408 * Assign a channel to the call at the front o 509 * Assign a channel to the call at the front of the queue and wake the call up.
409 * We don't increment the callNumber counter u 510 * We don't increment the callNumber counter until this number has been exposed
410 * to the world. 511 * to the world.
411 */ 512 */
412 static void rxrpc_activate_one_channel(struct 513 static void rxrpc_activate_one_channel(struct rxrpc_connection *conn,
413 unsigne 514 unsigned int channel)
414 { 515 {
415 struct rxrpc_channel *chan = &conn->ch 516 struct rxrpc_channel *chan = &conn->channels[channel];
416 struct rxrpc_bundle *bundle = conn->bu 517 struct rxrpc_bundle *bundle = conn->bundle;
417 struct rxrpc_call *call = list_entry(b 518 struct rxrpc_call *call = list_entry(bundle->waiting_calls.next,
418 s !! 519 struct rxrpc_call, chan_wait_link);
419 u32 call_id = chan->call_counter + 1; 520 u32 call_id = chan->call_counter + 1;
420 521
421 _enter("C=%x,%u", conn->debug_id, chan 522 _enter("C=%x,%u", conn->debug_id, channel);
422 523
423 list_del_init(&call->wait_link); <<
424 <<
425 trace_rxrpc_client(conn, channel, rxrp 524 trace_rxrpc_client(conn, channel, rxrpc_client_chan_activate);
426 525
427 /* Cancel the final ACK on the previou 526 /* Cancel the final ACK on the previous call if it hasn't been sent yet
428 * as the DATA packet will implicitly 527 * as the DATA packet will implicitly ACK it.
429 */ 528 */
430 clear_bit(RXRPC_CONN_FINAL_ACK_0 + cha 529 clear_bit(RXRPC_CONN_FINAL_ACK_0 + channel, &conn->flags);
431 clear_bit(conn->bundle_shift + channel 530 clear_bit(conn->bundle_shift + channel, &bundle->avail_chans);
432 531
433 rxrpc_see_call(call, rxrpc_call_see_ac !! 532 rxrpc_see_call(call);
434 call->conn = rxrpc_get_connection !! 533 list_del_init(&call->chan_wait_link);
>> 534 call->peer = rxrpc_get_peer(conn->params.peer);
>> 535 call->conn = rxrpc_get_connection(conn);
435 call->cid = conn->proto.cid | ch 536 call->cid = conn->proto.cid | channel;
436 call->call_id = call_id; 537 call->call_id = call_id;
437 call->dest_srx.srx_service = conn->ser !! 538 call->security = conn->security;
438 call->cong_ssthresh = call->peer->cong !! 539 call->security_ix = conn->security_ix;
439 if (call->cong_cwnd >= call->cong_ssth !! 540 call->service_id = conn->service_id;
440 call->cong_mode = RXRPC_CALL_C !! 541
441 else !! 542 trace_rxrpc_connect_call(call);
442 call->cong_mode = RXRPC_CALL_S !! 543 _net("CONNECT call %08x:%08x as call %d on conn %d",
>> 544 call->cid, call->call_id, call->debug_id, conn->debug_id);
>> 545
>> 546 write_lock_bh(&call->state_lock);
>> 547 call->state = RXRPC_CALL_CLIENT_SEND_REQUEST;
>> 548 write_unlock_bh(&call->state_lock);
>> 549
>> 550 /* Paired with the read barrier in rxrpc_connect_call(). This orders
>> 551 * cid and epoch in the connection wrt to call_id without the need to
>> 552 * take the channel_lock.
>> 553 *
>> 554 * We provisionally assign a callNumber at this point, but we don't
>> 555 * confirm it until the call is about to be exposed.
>> 556 *
>> 557 * TODO: Pair with a barrier in the data_ready handler when that looks
>> 558 * at the call ID through a connection channel.
>> 559 */
>> 560 smp_wmb();
443 561
444 chan->call_id = call_id; 562 chan->call_id = call_id;
445 chan->call_debug_id = call->debug_ 563 chan->call_debug_id = call->debug_id;
446 chan->call = call; !! 564 rcu_assign_pointer(chan->call, call);
447 <<
448 rxrpc_see_call(call, rxrpc_call_see_co <<
449 trace_rxrpc_connect_call(call); <<
450 call->tx_last_sent = ktime_get_real(); <<
451 rxrpc_start_call_timer(call); <<
452 rxrpc_set_call_state(call, RXRPC_CALL_ <<
453 wake_up(&call->waitq); 565 wake_up(&call->waitq);
454 } 566 }
455 567
456 /* 568 /*
457 * Remove a connection from the idle list if i 569 * Remove a connection from the idle list if it's on it.
458 */ 570 */
459 static void rxrpc_unidle_conn(struct rxrpc_con !! 571 static void rxrpc_unidle_conn(struct rxrpc_bundle *bundle, struct rxrpc_connection *conn)
460 { 572 {
>> 573 struct rxrpc_net *rxnet = bundle->params.local->rxnet;
>> 574 bool drop_ref;
>> 575
461 if (!list_empty(&conn->cache_link)) { 576 if (!list_empty(&conn->cache_link)) {
462 list_del_init(&conn->cache_lin !! 577 drop_ref = false;
463 rxrpc_put_connection(conn, rxr !! 578 spin_lock(&rxnet->client_conn_cache_lock);
>> 579 if (!list_empty(&conn->cache_link)) {
>> 580 list_del_init(&conn->cache_link);
>> 581 drop_ref = true;
>> 582 }
>> 583 spin_unlock(&rxnet->client_conn_cache_lock);
>> 584 if (drop_ref)
>> 585 rxrpc_put_connection(conn);
464 } 586 }
465 } 587 }
466 588
467 /* 589 /*
468 * Assign channels and callNumbers to waiting !! 590 * Assign channels and callNumbers to waiting calls with channel_lock
>> 591 * held by caller.
469 */ 592 */
470 static void rxrpc_activate_channels(struct rxr !! 593 static void rxrpc_activate_channels_locked(struct rxrpc_bundle *bundle)
471 { 594 {
472 struct rxrpc_connection *conn; 595 struct rxrpc_connection *conn;
473 unsigned long avail, mask; 596 unsigned long avail, mask;
474 unsigned int channel, slot; 597 unsigned int channel, slot;
475 598
476 trace_rxrpc_client(NULL, -1, rxrpc_cli <<
477 <<
478 if (bundle->try_upgrade) 599 if (bundle->try_upgrade)
479 mask = 1; 600 mask = 1;
480 else 601 else
481 mask = ULONG_MAX; 602 mask = ULONG_MAX;
482 603
483 while (!list_empty(&bundle->waiting_ca 604 while (!list_empty(&bundle->waiting_calls)) {
484 avail = bundle->avail_chans & 605 avail = bundle->avail_chans & mask;
485 if (!avail) 606 if (!avail)
486 break; 607 break;
487 channel = __ffs(avail); 608 channel = __ffs(avail);
488 clear_bit(channel, &bundle->av 609 clear_bit(channel, &bundle->avail_chans);
489 610
490 slot = channel / RXRPC_MAXCALL 611 slot = channel / RXRPC_MAXCALLS;
491 conn = bundle->conns[slot]; 612 conn = bundle->conns[slot];
492 if (!conn) 613 if (!conn)
493 break; 614 break;
494 615
495 if (bundle->try_upgrade) 616 if (bundle->try_upgrade)
496 set_bit(RXRPC_CONN_PRO 617 set_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags);
497 rxrpc_unidle_conn(conn); !! 618 rxrpc_unidle_conn(bundle, conn);
498 619
499 channel &= (RXRPC_MAXCALLS - 1 620 channel &= (RXRPC_MAXCALLS - 1);
500 conn->act_chans |= 1 << channe 621 conn->act_chans |= 1 << channel;
501 rxrpc_activate_one_channel(con 622 rxrpc_activate_one_channel(conn, channel);
502 } 623 }
503 } 624 }
504 625
505 /* 626 /*
506 * Connect waiting channels (called from the I !! 627 * Assign channels and callNumbers to waiting calls.
507 */ 628 */
508 void rxrpc_connect_client_calls(struct rxrpc_l !! 629 static void rxrpc_activate_channels(struct rxrpc_bundle *bundle)
509 { 630 {
510 struct rxrpc_call *call; !! 631 _enter("B=%x", bundle->debug_id);
>> 632
>> 633 trace_rxrpc_client(NULL, -1, rxrpc_client_activate_chans);
511 634
512 while ((call = list_first_entry_or_nul !! 635 if (!bundle->avail_chans)
513 !! 636 return;
514 ) { !! 637
515 struct rxrpc_bundle *bundle = !! 638 spin_lock(&bundle->channel_lock);
516 !! 639 rxrpc_activate_channels_locked(bundle);
517 spin_lock(&local->client_call_ !! 640 spin_unlock(&bundle->channel_lock);
518 list_move_tail(&call->wait_lin !! 641 _leave("");
519 rxrpc_see_call(call, rxrpc_cal !! 642 }
520 spin_unlock(&local->client_cal <<
521 643
522 if (rxrpc_bundle_has_space(bun !! 644 /*
523 rxrpc_activate_channel !! 645 * Wait for a callNumber and a channel to be granted to a call.
>> 646 */
>> 647 static int rxrpc_wait_for_channel(struct rxrpc_bundle *bundle,
>> 648 struct rxrpc_call *call, gfp_t gfp)
>> 649 {
>> 650 DECLARE_WAITQUEUE(myself, current);
>> 651 int ret = 0;
>> 652
>> 653 _enter("%d", call->debug_id);
>> 654
>> 655 if (!gfpflags_allow_blocking(gfp)) {
>> 656 rxrpc_maybe_add_conn(bundle, gfp);
>> 657 rxrpc_activate_channels(bundle);
>> 658 ret = bundle->alloc_error ?: -EAGAIN;
>> 659 goto out;
524 } 660 }
>> 661
>> 662 add_wait_queue_exclusive(&call->waitq, &myself);
>> 663 for (;;) {
>> 664 rxrpc_maybe_add_conn(bundle, gfp);
>> 665 rxrpc_activate_channels(bundle);
>> 666 ret = bundle->alloc_error;
>> 667 if (ret < 0)
>> 668 break;
>> 669
>> 670 switch (call->interruptibility) {
>> 671 case RXRPC_INTERRUPTIBLE:
>> 672 case RXRPC_PREINTERRUPTIBLE:
>> 673 set_current_state(TASK_INTERRUPTIBLE);
>> 674 break;
>> 675 case RXRPC_UNINTERRUPTIBLE:
>> 676 default:
>> 677 set_current_state(TASK_UNINTERRUPTIBLE);
>> 678 break;
>> 679 }
>> 680 if (READ_ONCE(call->state) != RXRPC_CALL_CLIENT_AWAIT_CONN)
>> 681 break;
>> 682 if ((call->interruptibility == RXRPC_INTERRUPTIBLE ||
>> 683 call->interruptibility == RXRPC_PREINTERRUPTIBLE) &&
>> 684 signal_pending(current)) {
>> 685 ret = -ERESTARTSYS;
>> 686 break;
>> 687 }
>> 688 schedule();
>> 689 }
>> 690 remove_wait_queue(&call->waitq, &myself);
>> 691 __set_current_state(TASK_RUNNING);
>> 692
>> 693 out:
>> 694 _leave(" = %d", ret);
>> 695 return ret;
>> 696 }
>> 697
>> 698 /*
>> 699 * find a connection for a call
>> 700 * - called in process context with IRQs enabled
>> 701 */
>> 702 int rxrpc_connect_call(struct rxrpc_sock *rx,
>> 703 struct rxrpc_call *call,
>> 704 struct rxrpc_conn_parameters *cp,
>> 705 struct sockaddr_rxrpc *srx,
>> 706 gfp_t gfp)
>> 707 {
>> 708 struct rxrpc_bundle *bundle;
>> 709 struct rxrpc_net *rxnet = cp->local->rxnet;
>> 710 int ret = 0;
>> 711
>> 712 _enter("{%d,%lx},", call->debug_id, call->user_call_ID);
>> 713
>> 714 rxrpc_discard_expired_client_conns(&rxnet->client_conn_reaper);
>> 715
>> 716 bundle = rxrpc_prep_call(rx, call, cp, srx, gfp);
>> 717 if (IS_ERR(bundle)) {
>> 718 ret = PTR_ERR(bundle);
>> 719 goto out;
>> 720 }
>> 721
>> 722 if (call->state == RXRPC_CALL_CLIENT_AWAIT_CONN) {
>> 723 ret = rxrpc_wait_for_channel(bundle, call, gfp);
>> 724 if (ret < 0)
>> 725 goto wait_failed;
>> 726 }
>> 727
>> 728 granted_channel:
>> 729 /* Paired with the write barrier in rxrpc_activate_one_channel(). */
>> 730 smp_rmb();
>> 731
>> 732 out_put_bundle:
>> 733 rxrpc_deactivate_bundle(bundle);
>> 734 rxrpc_put_bundle(bundle);
>> 735 out:
>> 736 _leave(" = %d", ret);
>> 737 return ret;
>> 738
>> 739 wait_failed:
>> 740 spin_lock(&bundle->channel_lock);
>> 741 list_del_init(&call->chan_wait_link);
>> 742 spin_unlock(&bundle->channel_lock);
>> 743
>> 744 if (call->state != RXRPC_CALL_CLIENT_AWAIT_CONN) {
>> 745 ret = 0;
>> 746 goto granted_channel;
>> 747 }
>> 748
>> 749 trace_rxrpc_client(call->conn, ret, rxrpc_client_chan_wait_failed);
>> 750 rxrpc_set_call_completion(call, RXRPC_CALL_LOCAL_ERROR, 0, ret);
>> 751 rxrpc_disconnect_client_call(bundle, call);
>> 752 goto out_put_bundle;
525 } 753 }
526 754
527 /* 755 /*
528 * Note that a call, and thus a connection, is 756 * Note that a call, and thus a connection, is about to be exposed to the
529 * world. 757 * world.
530 */ 758 */
531 void rxrpc_expose_client_call(struct rxrpc_cal 759 void rxrpc_expose_client_call(struct rxrpc_call *call)
532 { 760 {
533 unsigned int channel = call->cid & RXR 761 unsigned int channel = call->cid & RXRPC_CHANNELMASK;
534 struct rxrpc_connection *conn = call-> 762 struct rxrpc_connection *conn = call->conn;
535 struct rxrpc_channel *chan = &conn->ch 763 struct rxrpc_channel *chan = &conn->channels[channel];
536 764
537 if (!test_and_set_bit(RXRPC_CALL_EXPOS 765 if (!test_and_set_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
538 /* Mark the call ID as being u 766 /* Mark the call ID as being used. If the callNumber counter
539 * exceeds ~2 billion, we kill 767 * exceeds ~2 billion, we kill the connection after its
540 * outstanding calls have fini 768 * outstanding calls have finished so that the counter doesn't
541 * wrap. 769 * wrap.
542 */ 770 */
543 chan->call_counter++; 771 chan->call_counter++;
544 if (chan->call_counter >= INT_ 772 if (chan->call_counter >= INT_MAX)
545 set_bit(RXRPC_CONN_DON 773 set_bit(RXRPC_CONN_DONT_REUSE, &conn->flags);
546 trace_rxrpc_client(conn, chann 774 trace_rxrpc_client(conn, channel, rxrpc_client_exposed);
547 <<
548 spin_lock(&call->peer->lock); <<
549 hlist_add_head(&call->error_li <<
550 spin_unlock(&call->peer->lock) <<
551 } 775 }
552 } 776 }
553 777
554 /* 778 /*
555 * Set the reap timer. 779 * Set the reap timer.
556 */ 780 */
557 static void rxrpc_set_client_reap_timer(struct !! 781 static void rxrpc_set_client_reap_timer(struct rxrpc_net *rxnet)
558 { 782 {
559 if (!local->kill_all_client_conns) { !! 783 if (!rxnet->kill_all_client_conns) {
560 unsigned long now = jiffies; 784 unsigned long now = jiffies;
561 unsigned long reap_at = now + 785 unsigned long reap_at = now + rxrpc_conn_idle_client_expiry;
562 786
563 if (local->rxnet->live) !! 787 if (rxnet->live)
564 timer_reduce(&local->c !! 788 timer_reduce(&rxnet->client_conn_reap_timer, reap_at);
565 } 789 }
566 } 790 }
567 791
568 /* 792 /*
569 * Disconnect a client call. 793 * Disconnect a client call.
570 */ 794 */
571 void rxrpc_disconnect_client_call(struct rxrpc 795 void rxrpc_disconnect_client_call(struct rxrpc_bundle *bundle, struct rxrpc_call *call)
572 { 796 {
573 struct rxrpc_connection *conn; 797 struct rxrpc_connection *conn;
574 struct rxrpc_channel *chan = NULL; 798 struct rxrpc_channel *chan = NULL;
575 struct rxrpc_local *local = bundle->lo !! 799 struct rxrpc_net *rxnet = bundle->params.local->rxnet;
576 unsigned int channel; 800 unsigned int channel;
577 bool may_reuse; 801 bool may_reuse;
578 u32 cid; 802 u32 cid;
579 803
580 _enter("c=%x", call->debug_id); 804 _enter("c=%x", call->debug_id);
581 805
>> 806 spin_lock(&bundle->channel_lock);
>> 807 set_bit(RXRPC_CALL_DISCONNECTED, &call->flags);
>> 808
582 /* Calls that have never actually been 809 /* Calls that have never actually been assigned a channel can simply be
583 * discarded. 810 * discarded.
584 */ 811 */
585 conn = call->conn; 812 conn = call->conn;
586 if (!conn) { 813 if (!conn) {
587 _debug("call is waiting"); 814 _debug("call is waiting");
588 ASSERTCMP(call->call_id, ==, 0 815 ASSERTCMP(call->call_id, ==, 0);
589 ASSERT(!test_bit(RXRPC_CALL_EX 816 ASSERT(!test_bit(RXRPC_CALL_EXPOSED, &call->flags));
590 /* May still be on ->new_clien !! 817 list_del_init(&call->chan_wait_link);
591 spin_lock(&local->client_call_ !! 818 goto out;
592 list_del_init(&call->wait_link <<
593 spin_unlock(&local->client_cal <<
594 return; <<
595 } 819 }
596 820
597 cid = call->cid; 821 cid = call->cid;
598 channel = cid & RXRPC_CHANNELMASK; 822 channel = cid & RXRPC_CHANNELMASK;
599 chan = &conn->channels[channel]; 823 chan = &conn->channels[channel];
600 trace_rxrpc_client(conn, channel, rxrp 824 trace_rxrpc_client(conn, channel, rxrpc_client_chan_disconnect);
601 825
602 if (WARN_ON(chan->call != call)) !! 826 if (rcu_access_pointer(chan->call) != call) {
603 return; !! 827 spin_unlock(&bundle->channel_lock);
>> 828 BUG();
>> 829 }
604 830
605 may_reuse = rxrpc_may_reuse_conn(conn) 831 may_reuse = rxrpc_may_reuse_conn(conn);
606 832
607 /* If a client call was exposed to the 833 /* If a client call was exposed to the world, we save the result for
608 * retransmission. 834 * retransmission.
609 * 835 *
610 * We use a barrier here so that the c 836 * We use a barrier here so that the call number and abort code can be
611 * read without needing to take a lock 837 * read without needing to take a lock.
612 * 838 *
613 * TODO: Make the incoming packet hand 839 * TODO: Make the incoming packet handler check this and handle
614 * terminal retransmission without req 840 * terminal retransmission without requiring access to the call.
615 */ 841 */
616 if (test_bit(RXRPC_CALL_EXPOSED, &call 842 if (test_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
617 _debug("exposed %u,%u", call-> 843 _debug("exposed %u,%u", call->call_id, call->abort_code);
618 __rxrpc_disconnect_call(conn, 844 __rxrpc_disconnect_call(conn, call);
619 845
620 if (test_and_clear_bit(RXRPC_C 846 if (test_and_clear_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags)) {
621 trace_rxrpc_client(con 847 trace_rxrpc_client(conn, channel, rxrpc_client_to_active);
622 bundle->try_upgrade = 848 bundle->try_upgrade = false;
623 if (may_reuse) 849 if (may_reuse)
624 rxrpc_activate !! 850 rxrpc_activate_channels_locked(bundle);
625 } 851 }
>> 852
626 } 853 }
627 854
628 /* See if we can pass the channel dire 855 /* See if we can pass the channel directly to another call. */
629 if (may_reuse && !list_empty(&bundle-> 856 if (may_reuse && !list_empty(&bundle->waiting_calls)) {
630 trace_rxrpc_client(conn, chann 857 trace_rxrpc_client(conn, channel, rxrpc_client_chan_pass);
631 rxrpc_activate_one_channel(con 858 rxrpc_activate_one_channel(conn, channel);
632 return; !! 859 goto out;
633 } 860 }
634 861
635 /* Schedule the final ACK to be transm 862 /* Schedule the final ACK to be transmitted in a short while so that it
636 * can be skipped if we find a follow- 863 * can be skipped if we find a follow-on call. The first DATA packet
637 * of the follow on call will implicit 864 * of the follow on call will implicitly ACK this call.
638 */ 865 */
639 if (call->completion == RXRPC_CALL_SUC 866 if (call->completion == RXRPC_CALL_SUCCEEDED &&
640 test_bit(RXRPC_CALL_EXPOSED, &call 867 test_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
641 unsigned long final_ack_at = j 868 unsigned long final_ack_at = jiffies + 2;
642 869
643 chan->final_ack_at = final_ack !! 870 WRITE_ONCE(chan->final_ack_at, final_ack_at);
644 smp_wmb(); /* vs rxrpc_process 871 smp_wmb(); /* vs rxrpc_process_delayed_final_acks() */
645 set_bit(RXRPC_CONN_FINAL_ACK_0 872 set_bit(RXRPC_CONN_FINAL_ACK_0 + channel, &conn->flags);
646 rxrpc_reduce_conn_timer(conn, 873 rxrpc_reduce_conn_timer(conn, final_ack_at);
647 } 874 }
648 875
649 /* Deactivate the channel. */ 876 /* Deactivate the channel. */
650 chan->call = NULL; !! 877 rcu_assign_pointer(chan->call, NULL);
651 set_bit(conn->bundle_shift + channel, 878 set_bit(conn->bundle_shift + channel, &conn->bundle->avail_chans);
652 conn->act_chans &= ~(1 << channel); 879 conn->act_chans &= ~(1 << channel);
653 880
654 /* If no channels remain active, then 881 /* If no channels remain active, then put the connection on the idle
655 * list for a short while. Give it a 882 * list for a short while. Give it a ref to stop it going away if it
656 * becomes unbundled. 883 * becomes unbundled.
657 */ 884 */
658 if (!conn->act_chans) { 885 if (!conn->act_chans) {
659 trace_rxrpc_client(conn, chann 886 trace_rxrpc_client(conn, channel, rxrpc_client_to_idle);
660 conn->idle_timestamp = jiffies 887 conn->idle_timestamp = jiffies;
661 888
662 rxrpc_get_connection(conn, rxr !! 889 rxrpc_get_connection(conn);
663 list_move_tail(&conn->cache_li !! 890 spin_lock(&rxnet->client_conn_cache_lock);
>> 891 list_move_tail(&conn->cache_link, &rxnet->idle_client_conns);
>> 892 spin_unlock(&rxnet->client_conn_cache_lock);
664 893
665 rxrpc_set_client_reap_timer(lo !! 894 rxrpc_set_client_reap_timer(rxnet);
666 } 895 }
>> 896
>> 897 out:
>> 898 spin_unlock(&bundle->channel_lock);
>> 899 _leave("");
>> 900 return;
667 } 901 }
668 902
669 /* 903 /*
670 * Remove a connection from a bundle. 904 * Remove a connection from a bundle.
671 */ 905 */
672 static void rxrpc_unbundle_conn(struct rxrpc_c 906 static void rxrpc_unbundle_conn(struct rxrpc_connection *conn)
673 { 907 {
674 struct rxrpc_bundle *bundle = conn->bu 908 struct rxrpc_bundle *bundle = conn->bundle;
675 unsigned int bindex; 909 unsigned int bindex;
>> 910 bool need_drop = false;
676 int i; 911 int i;
677 912
678 _enter("C=%x", conn->debug_id); 913 _enter("C=%x", conn->debug_id);
679 914
680 if (conn->flags & RXRPC_CONN_FINAL_ACK 915 if (conn->flags & RXRPC_CONN_FINAL_ACK_MASK)
681 rxrpc_process_delayed_final_ac 916 rxrpc_process_delayed_final_acks(conn, true);
682 917
>> 918 spin_lock(&bundle->channel_lock);
683 bindex = conn->bundle_shift / RXRPC_MA 919 bindex = conn->bundle_shift / RXRPC_MAXCALLS;
684 if (bundle->conns[bindex] == conn) { 920 if (bundle->conns[bindex] == conn) {
685 _debug("clear slot %u", bindex 921 _debug("clear slot %u", bindex);
686 bundle->conns[bindex] = NULL; 922 bundle->conns[bindex] = NULL;
687 bundle->conn_ids[bindex] = 0; <<
688 for (i = 0; i < RXRPC_MAXCALLS 923 for (i = 0; i < RXRPC_MAXCALLS; i++)
689 clear_bit(conn->bundle 924 clear_bit(conn->bundle_shift + i, &bundle->avail_chans);
690 rxrpc_put_client_connection_id !! 925 need_drop = true;
>> 926 }
>> 927 spin_unlock(&bundle->channel_lock);
>> 928
>> 929 if (need_drop) {
691 rxrpc_deactivate_bundle(bundle 930 rxrpc_deactivate_bundle(bundle);
692 rxrpc_put_connection(conn, rxr !! 931 rxrpc_put_connection(conn);
693 } 932 }
694 } 933 }
695 934
696 /* 935 /*
697 * Drop the active count on a bundle. 936 * Drop the active count on a bundle.
698 */ 937 */
699 void rxrpc_deactivate_bundle(struct rxrpc_bund !! 938 static void rxrpc_deactivate_bundle(struct rxrpc_bundle *bundle)
700 { 939 {
701 struct rxrpc_local *local; !! 940 struct rxrpc_local *local = bundle->params.local;
702 bool need_put = false; 941 bool need_put = false;
703 942
704 if (!bundle) <<
705 return; <<
706 <<
707 local = bundle->local; <<
708 if (atomic_dec_and_lock(&bundle->activ 943 if (atomic_dec_and_lock(&bundle->active, &local->client_bundles_lock)) {
709 if (!bundle->exclusive) { !! 944 if (!bundle->params.exclusive) {
710 _debug("erase bundle") 945 _debug("erase bundle");
711 rb_erase(&bundle->loca 946 rb_erase(&bundle->local_node, &local->client_bundles);
712 need_put = true; 947 need_put = true;
713 } 948 }
714 949
715 spin_unlock(&local->client_bun 950 spin_unlock(&local->client_bundles_lock);
716 if (need_put) 951 if (need_put)
717 rxrpc_put_bundle(bundl !! 952 rxrpc_put_bundle(bundle);
718 } 953 }
719 } 954 }
720 955
721 /* 956 /*
722 * Clean up a dead client connection. 957 * Clean up a dead client connection.
723 */ 958 */
724 void rxrpc_kill_client_conn(struct rxrpc_conne !! 959 static void rxrpc_kill_client_conn(struct rxrpc_connection *conn)
725 { 960 {
726 struct rxrpc_local *local = conn->loca !! 961 struct rxrpc_local *local = conn->params.local;
727 struct rxrpc_net *rxnet = local->rxnet 962 struct rxrpc_net *rxnet = local->rxnet;
728 963
729 _enter("C=%x", conn->debug_id); 964 _enter("C=%x", conn->debug_id);
730 965
731 trace_rxrpc_client(conn, -1, rxrpc_cli 966 trace_rxrpc_client(conn, -1, rxrpc_client_cleanup);
732 atomic_dec(&rxnet->nr_client_conns); 967 atomic_dec(&rxnet->nr_client_conns);
733 968
734 rxrpc_put_client_connection_id(local, !! 969 rxrpc_put_client_connection_id(conn);
>> 970 rxrpc_kill_connection(conn);
>> 971 }
>> 972
>> 973 /*
>> 974 * Clean up a dead client connections.
>> 975 */
>> 976 void rxrpc_put_client_conn(struct rxrpc_connection *conn)
>> 977 {
>> 978 const void *here = __builtin_return_address(0);
>> 979 unsigned int debug_id = conn->debug_id;
>> 980 bool dead;
>> 981 int r;
>> 982
>> 983 dead = __refcount_dec_and_test(&conn->ref, &r);
>> 984 trace_rxrpc_conn(debug_id, rxrpc_conn_put_client, r - 1, here);
>> 985 if (dead)
>> 986 rxrpc_kill_client_conn(conn);
735 } 987 }
736 988
737 /* 989 /*
738 * Discard expired client connections from the 990 * Discard expired client connections from the idle list. Each conn in the
739 * idle list has been exposed and holds an ext 991 * idle list has been exposed and holds an extra ref because of that.
740 * 992 *
741 * This may be called from conn setup or from 993 * This may be called from conn setup or from a work item so cannot be
742 * considered non-reentrant. 994 * considered non-reentrant.
743 */ 995 */
744 void rxrpc_discard_expired_client_conns(struct !! 996 void rxrpc_discard_expired_client_conns(struct work_struct *work)
745 { 997 {
746 struct rxrpc_connection *conn; 998 struct rxrpc_connection *conn;
>> 999 struct rxrpc_net *rxnet =
>> 1000 container_of(work, struct rxrpc_net, client_conn_reaper);
747 unsigned long expiry, conn_expires_at, 1001 unsigned long expiry, conn_expires_at, now;
748 unsigned int nr_conns; 1002 unsigned int nr_conns;
749 1003
750 _enter(""); 1004 _enter("");
751 1005
>> 1006 if (list_empty(&rxnet->idle_client_conns)) {
>> 1007 _leave(" [empty]");
>> 1008 return;
>> 1009 }
>> 1010
>> 1011 /* Don't double up on the discarding */
>> 1012 if (!spin_trylock(&rxnet->client_conn_discard_lock)) {
>> 1013 _leave(" [already]");
>> 1014 return;
>> 1015 }
>> 1016
752 /* We keep an estimate of what the num 1017 /* We keep an estimate of what the number of conns ought to be after
753 * we've discarded some so that we don 1018 * we've discarded some so that we don't overdo the discarding.
754 */ 1019 */
755 nr_conns = atomic_read(&local->rxnet-> !! 1020 nr_conns = atomic_read(&rxnet->nr_client_conns);
756 1021
757 next: 1022 next:
758 conn = list_first_entry_or_null(&local !! 1023 spin_lock(&rxnet->client_conn_cache_lock);
759 struct !! 1024
760 if (!conn) !! 1025 if (list_empty(&rxnet->idle_client_conns))
761 return; !! 1026 goto out;
762 1027
763 if (!local->kill_all_client_conns) { !! 1028 conn = list_entry(rxnet->idle_client_conns.next,
>> 1029 struct rxrpc_connection, cache_link);
>> 1030
>> 1031 if (!rxnet->kill_all_client_conns) {
764 /* If the number of connection 1032 /* If the number of connections is over the reap limit, we
765 * expedite discard by reducin 1033 * expedite discard by reducing the expiry timeout. We must,
766 * however, have at least a sh 1034 * however, have at least a short grace period to be able to do
767 * final-ACK or ABORT retransm 1035 * final-ACK or ABORT retransmission.
768 */ 1036 */
769 expiry = rxrpc_conn_idle_clien 1037 expiry = rxrpc_conn_idle_client_expiry;
770 if (nr_conns > rxrpc_reap_clie 1038 if (nr_conns > rxrpc_reap_client_connections)
771 expiry = rxrpc_conn_id 1039 expiry = rxrpc_conn_idle_client_fast_expiry;
772 if (conn->local->service_close !! 1040 if (conn->params.local->service_closed)
773 expiry = rxrpc_closed_ 1041 expiry = rxrpc_closed_conn_expiry * HZ;
774 1042
775 conn_expires_at = conn->idle_t 1043 conn_expires_at = conn->idle_timestamp + expiry;
776 1044
777 now = jiffies; !! 1045 now = READ_ONCE(jiffies);
778 if (time_after(conn_expires_at 1046 if (time_after(conn_expires_at, now))
779 goto not_yet_expired; 1047 goto not_yet_expired;
780 } 1048 }
781 1049
782 atomic_dec(&conn->active); <<
783 trace_rxrpc_client(conn, -1, rxrpc_cli 1050 trace_rxrpc_client(conn, -1, rxrpc_client_discard);
784 list_del_init(&conn->cache_link); 1051 list_del_init(&conn->cache_link);
785 1052
>> 1053 spin_unlock(&rxnet->client_conn_cache_lock);
>> 1054
786 rxrpc_unbundle_conn(conn); 1055 rxrpc_unbundle_conn(conn);
787 /* Drop the ->cache_link ref */ !! 1056 rxrpc_put_connection(conn); /* Drop the ->cache_link ref */
788 rxrpc_put_connection(conn, rxrpc_conn_ <<
789 1057
790 nr_conns--; 1058 nr_conns--;
791 goto next; 1059 goto next;
792 1060
793 not_yet_expired: 1061 not_yet_expired:
794 /* The connection at the front of the 1062 /* The connection at the front of the queue hasn't yet expired, so
795 * schedule the work item for that poi 1063 * schedule the work item for that point if we discarded something.
796 * 1064 *
797 * We don't worry if the work item is 1065 * We don't worry if the work item is already scheduled - it can look
798 * after rescheduling itself at a late 1066 * after rescheduling itself at a later time. We could cancel it, but
799 * then things get messier. 1067 * then things get messier.
800 */ 1068 */
801 _debug("not yet"); 1069 _debug("not yet");
802 if (!local->kill_all_client_conns) !! 1070 if (!rxnet->kill_all_client_conns)
803 timer_reduce(&local->client_co !! 1071 timer_reduce(&rxnet->client_conn_reap_timer, conn_expires_at);
>> 1072
>> 1073 out:
>> 1074 spin_unlock(&rxnet->client_conn_cache_lock);
>> 1075 spin_unlock(&rxnet->client_conn_discard_lock);
>> 1076 _leave("");
>> 1077 }
>> 1078
>> 1079 /*
>> 1080 * Preemptively destroy all the client connection records rather than waiting
>> 1081 * for them to time out
>> 1082 */
>> 1083 void rxrpc_destroy_all_client_connections(struct rxrpc_net *rxnet)
>> 1084 {
>> 1085 _enter("");
>> 1086
>> 1087 spin_lock(&rxnet->client_conn_cache_lock);
>> 1088 rxnet->kill_all_client_conns = true;
>> 1089 spin_unlock(&rxnet->client_conn_cache_lock);
>> 1090
>> 1091 del_timer_sync(&rxnet->client_conn_reap_timer);
>> 1092
>> 1093 if (!rxrpc_queue_work(&rxnet->client_conn_reaper))
>> 1094 _debug("destroy: queue failed");
804 1095
805 _leave(""); 1096 _leave("");
806 } 1097 }
807 1098
808 /* 1099 /*
809 * Clean up the client connections on a local 1100 * Clean up the client connections on a local endpoint.
810 */ 1101 */
811 void rxrpc_clean_up_local_conns(struct rxrpc_l 1102 void rxrpc_clean_up_local_conns(struct rxrpc_local *local)
812 { 1103 {
813 struct rxrpc_connection *conn; !! 1104 struct rxrpc_connection *conn, *tmp;
>> 1105 struct rxrpc_net *rxnet = local->rxnet;
>> 1106 LIST_HEAD(graveyard);
814 1107
815 _enter(""); 1108 _enter("");
816 1109
817 local->kill_all_client_conns = true; !! 1110 spin_lock(&rxnet->client_conn_cache_lock);
>> 1111
>> 1112 list_for_each_entry_safe(conn, tmp, &rxnet->idle_client_conns,
>> 1113 cache_link) {
>> 1114 if (conn->params.local == local) {
>> 1115 trace_rxrpc_client(conn, -1, rxrpc_client_discard);
>> 1116 list_move(&conn->cache_link, &graveyard);
>> 1117 }
>> 1118 }
818 1119
819 del_timer_sync(&local->client_conn_rea !! 1120 spin_unlock(&rxnet->client_conn_cache_lock);
820 1121
821 while ((conn = list_first_entry_or_nul !! 1122 while (!list_empty(&graveyard)) {
822 !! 1123 conn = list_entry(graveyard.next,
>> 1124 struct rxrpc_connection, cache_link);
823 list_del_init(&conn->cache_lin 1125 list_del_init(&conn->cache_link);
824 atomic_dec(&conn->active); <<
825 trace_rxrpc_client(conn, -1, r <<
826 rxrpc_unbundle_conn(conn); 1126 rxrpc_unbundle_conn(conn);
827 rxrpc_put_connection(conn, rxr !! 1127 rxrpc_put_connection(conn);
828 } 1128 }
829 1129
830 _leave(" [culled]"); 1130 _leave(" [culled]");
831 } 1131 }
832 1132
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.