1 // SPDX-License-Identifier: GPL-2.0-or-later 1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Client connection-specific management code. 2 /* Client connection-specific management code.
3 * 3 *
4 * Copyright (C) 2016, 2020 Red Hat, Inc. All 4 * Copyright (C) 2016, 2020 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.c 5 * Written by David Howells (dhowells@redhat.com)
6 * 6 *
7 * Client connections need to be cached for a 7 * Client connections need to be cached for a little while after they've made a
8 * call so as to handle retransmitted DATA pac 8 * call so as to handle retransmitted DATA packets in case the server didn't
9 * receive the final ACK or terminating ABORT 9 * receive the final ACK or terminating ABORT we sent it.
10 * 10 *
11 * There are flags of relevance to the cache: 11 * There are flags of relevance to the cache:
12 * 12 *
13 * (2) DONT_REUSE - The connection should be 13 * (2) DONT_REUSE - The connection should be discarded as soon as possible and
14 * should not be reused. This is set whe 14 * should not be reused. This is set when an exclusive connection is used
15 * or a call ID counter overflows. 15 * or a call ID counter overflows.
16 * 16 *
17 * The caching state may only be changed if th 17 * The caching state may only be changed if the cache lock is held.
18 * 18 *
19 * There are two idle client connection expiry 19 * There are two idle client connection expiry durations. If the total number
20 * of connections is below the reap threshold, 20 * of connections is below the reap threshold, we use the normal duration; if
21 * it's above, we use the fast duration. 21 * it's above, we use the fast duration.
22 */ 22 */
23 23
24 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
25 25
26 #include <linux/slab.h> 26 #include <linux/slab.h>
27 #include <linux/idr.h> 27 #include <linux/idr.h>
28 #include <linux/timer.h> 28 #include <linux/timer.h>
29 #include <linux/sched/signal.h> 29 #include <linux/sched/signal.h>
30 30
31 #include "ar-internal.h" 31 #include "ar-internal.h"
32 32
33 __read_mostly unsigned int rxrpc_reap_client_c 33 __read_mostly unsigned int rxrpc_reap_client_connections = 900;
34 __read_mostly unsigned long rxrpc_conn_idle_cl 34 __read_mostly unsigned long rxrpc_conn_idle_client_expiry = 2 * 60 * HZ;
35 __read_mostly unsigned long rxrpc_conn_idle_cl 35 __read_mostly unsigned long rxrpc_conn_idle_client_fast_expiry = 2 * HZ;
36 36
37 static void rxrpc_activate_bundle(struct rxrpc !! 37 /*
>> 38 * We use machine-unique IDs for our client connections.
>> 39 */
>> 40 DEFINE_IDR(rxrpc_client_conn_ids);
>> 41 static DEFINE_SPINLOCK(rxrpc_conn_id_lock);
>> 42
>> 43 /*
>> 44 * Get a connection ID and epoch for a client connection from the global pool.
>> 45 * The connection struct pointer is then recorded in the idr radix tree. The
>> 46 * epoch doesn't change until the client is rebooted (or, at least, unless the
>> 47 * module is unloaded).
>> 48 */
>> 49 static int rxrpc_get_client_connection_id(struct rxrpc_connection *conn,
>> 50 gfp_t gfp)
38 { 51 {
39 atomic_inc(&bundle->active); !! 52 struct rxrpc_net *rxnet = conn->params.local->rxnet;
>> 53 int id;
>> 54
>> 55 _enter("");
>> 56
>> 57 idr_preload(gfp);
>> 58 spin_lock(&rxrpc_conn_id_lock);
>> 59
>> 60 id = idr_alloc_cyclic(&rxrpc_client_conn_ids, conn,
>> 61 1, 0x40000000, GFP_NOWAIT);
>> 62 if (id < 0)
>> 63 goto error;
>> 64
>> 65 spin_unlock(&rxrpc_conn_id_lock);
>> 66 idr_preload_end();
>> 67
>> 68 conn->proto.epoch = rxnet->epoch;
>> 69 conn->proto.cid = id << RXRPC_CIDSHIFT;
>> 70 set_bit(RXRPC_CONN_HAS_IDR, &conn->flags);
>> 71 _leave(" [CID %x]", conn->proto.cid);
>> 72 return 0;
>> 73
>> 74 error:
>> 75 spin_unlock(&rxrpc_conn_id_lock);
>> 76 idr_preload_end();
>> 77 _leave(" = %d", id);
>> 78 return id;
40 } 79 }
41 80
42 /* 81 /*
43 * Release a connection ID for a client connec !! 82 * Release a connection ID for a client connection from the global pool.
44 */ 83 */
45 static void rxrpc_put_client_connection_id(str !! 84 static void rxrpc_put_client_connection_id(struct rxrpc_connection *conn)
46 str <<
47 { 85 {
48 idr_remove(&local->conn_ids, conn->pro !! 86 if (test_bit(RXRPC_CONN_HAS_IDR, &conn->flags)) {
>> 87 spin_lock(&rxrpc_conn_id_lock);
>> 88 idr_remove(&rxrpc_client_conn_ids,
>> 89 conn->proto.cid >> RXRPC_CIDSHIFT);
>> 90 spin_unlock(&rxrpc_conn_id_lock);
>> 91 }
49 } 92 }
50 93
51 /* 94 /*
52 * Destroy the client connection ID tree. 95 * Destroy the client connection ID tree.
53 */ 96 */
54 static void rxrpc_destroy_client_conn_ids(stru !! 97 void rxrpc_destroy_client_conn_ids(void)
55 { 98 {
56 struct rxrpc_connection *conn; 99 struct rxrpc_connection *conn;
57 int id; 100 int id;
58 101
59 if (!idr_is_empty(&local->conn_ids)) { !! 102 if (!idr_is_empty(&rxrpc_client_conn_ids)) {
60 idr_for_each_entry(&local->con !! 103 idr_for_each_entry(&rxrpc_client_conn_ids, conn, id) {
61 pr_err("AF_RXRPC: Leak 104 pr_err("AF_RXRPC: Leaked client conn %p {%d}\n",
62 conn, refcount_ !! 105 conn, atomic_read(&conn->usage));
63 } 106 }
64 BUG(); 107 BUG();
65 } 108 }
66 109
67 idr_destroy(&local->conn_ids); !! 110 idr_destroy(&rxrpc_client_conn_ids);
68 } 111 }
69 112
70 /* 113 /*
71 * Allocate a connection bundle. 114 * Allocate a connection bundle.
72 */ 115 */
73 static struct rxrpc_bundle *rxrpc_alloc_bundle !! 116 static struct rxrpc_bundle *rxrpc_alloc_bundle(struct rxrpc_conn_parameters *cp,
74 117 gfp_t gfp)
75 { 118 {
76 static atomic_t rxrpc_bundle_id; <<
77 struct rxrpc_bundle *bundle; 119 struct rxrpc_bundle *bundle;
78 120
79 bundle = kzalloc(sizeof(*bundle), gfp) 121 bundle = kzalloc(sizeof(*bundle), gfp);
80 if (bundle) { 122 if (bundle) {
81 bundle->local = call !! 123 bundle->params = *cp;
82 bundle->peer = rxrp !! 124 rxrpc_get_peer(bundle->params.peer);
83 bundle->key = key_ !! 125 atomic_set(&bundle->usage, 1);
84 bundle->security = call !! 126 spin_lock_init(&bundle->channel_lock);
85 bundle->exclusive = test <<
86 bundle->upgrade = test <<
87 bundle->service_id = call <<
88 bundle->security_level = call <<
89 bundle->debug_id = atom <<
90 refcount_set(&bundle->ref, 1); <<
91 atomic_set(&bundle->active, 1) <<
92 INIT_LIST_HEAD(&bundle->waitin 127 INIT_LIST_HEAD(&bundle->waiting_calls);
93 trace_rxrpc_bundle(bundle->deb <<
94 <<
95 write_lock(&bundle->local->rxn <<
96 list_add_tail(&bundle->proc_li <<
97 write_unlock(&bundle->local->r <<
98 } 128 }
99 return bundle; 129 return bundle;
100 } 130 }
101 131
102 struct rxrpc_bundle *rxrpc_get_bundle(struct r !! 132 struct rxrpc_bundle *rxrpc_get_bundle(struct rxrpc_bundle *bundle)
103 enum rxr <<
104 { 133 {
105 int r; !! 134 atomic_inc(&bundle->usage);
106 <<
107 __refcount_inc(&bundle->ref, &r); <<
108 trace_rxrpc_bundle(bundle->debug_id, r <<
109 return bundle; 135 return bundle;
110 } 136 }
111 137
112 static void rxrpc_free_bundle(struct rxrpc_bun !! 138 void rxrpc_put_bundle(struct rxrpc_bundle *bundle)
113 { 139 {
114 trace_rxrpc_bundle(bundle->debug_id, r !! 140 unsigned int d = bundle->debug_id;
115 rxrpc_bundle_free); !! 141 unsigned int u = atomic_dec_return(&bundle->usage);
116 write_lock(&bundle->local->rxnet->conn <<
117 list_del(&bundle->proc_link); <<
118 write_unlock(&bundle->local->rxnet->co <<
119 rxrpc_put_peer(bundle->peer, rxrpc_pee <<
120 key_put(bundle->key); <<
121 kfree(bundle); <<
122 } <<
123 <<
124 void rxrpc_put_bundle(struct rxrpc_bundle *bun <<
125 { <<
126 unsigned int id; <<
127 bool dead; <<
128 int r; <<
129 142
130 if (bundle) { !! 143 _debug("PUT B=%x %u", d, u);
131 id = bundle->debug_id; !! 144 if (u == 0) {
132 dead = __refcount_dec_and_test !! 145 rxrpc_put_peer(bundle->params.peer);
133 trace_rxrpc_bundle(id, r - 1, !! 146 kfree(bundle);
134 if (dead) <<
135 rxrpc_free_bundle(bund <<
136 } 147 }
137 } 148 }
138 149
139 /* 150 /*
140 * Get rid of outstanding client connection pr <<
141 * endpoint is destroyed. <<
142 */ <<
143 void rxrpc_purge_client_connections(struct rxr <<
144 { <<
145 rxrpc_destroy_client_conn_ids(local); <<
146 } <<
147 <<
148 /* <<
149 * Allocate a client connection. 151 * Allocate a client connection.
150 */ 152 */
151 static struct rxrpc_connection * 153 static struct rxrpc_connection *
152 rxrpc_alloc_client_connection(struct rxrpc_bun !! 154 rxrpc_alloc_client_connection(struct rxrpc_bundle *bundle, gfp_t gfp)
153 { 155 {
154 struct rxrpc_connection *conn; 156 struct rxrpc_connection *conn;
155 struct rxrpc_local *local = bundle->lo !! 157 struct rxrpc_net *rxnet = bundle->params.local->rxnet;
156 struct rxrpc_net *rxnet = local->rxnet !! 158 int ret;
157 int id; <<
158 159
159 _enter(""); 160 _enter("");
160 161
161 conn = rxrpc_alloc_connection(rxnet, G !! 162 conn = rxrpc_alloc_connection(gfp);
162 if (!conn) !! 163 if (!conn) {
>> 164 _leave(" = -ENOMEM");
163 return ERR_PTR(-ENOMEM); 165 return ERR_PTR(-ENOMEM);
164 <<
165 id = idr_alloc_cyclic(&local->conn_ids <<
166 GFP_ATOMIC | __G <<
167 if (id < 0) { <<
168 kfree(conn); <<
169 return ERR_PTR(id); <<
170 } 166 }
171 167
172 refcount_set(&conn->ref, 1); !! 168 atomic_set(&conn->usage, 1);
173 conn->proto.cid = id << RXRPC_ !! 169 conn->bundle = bundle;
174 conn->proto.epoch = local->rxnet !! 170 conn->params = bundle->params;
175 conn->out_clientflag = RXRPC_CLIENT 171 conn->out_clientflag = RXRPC_CLIENT_INITIATED;
176 conn->bundle = rxrpc_get_bu !! 172 conn->state = RXRPC_CONN_CLIENT;
177 conn->local = rxrpc_get_lo !! 173 conn->service_id = conn->params.service_id;
178 conn->peer = rxrpc_get_pe <<
179 conn->key = key_get(bund <<
180 conn->security = bundle->secu <<
181 conn->exclusive = bundle->excl <<
182 conn->upgrade = bundle->upgr <<
183 conn->orig_service_id = bundle->serv <<
184 conn->security_level = bundle->secu <<
185 conn->state = RXRPC_CONN_C <<
186 conn->service_id = conn->orig_s <<
187 174
188 if (conn->security == &rxrpc_no_securi !! 175 ret = rxrpc_get_client_connection_id(conn, gfp);
189 conn->state = RXRPC_CONN_C !! 176 if (ret < 0)
>> 177 goto error_0;
>> 178
>> 179 ret = rxrpc_init_client_conn_security(conn);
>> 180 if (ret < 0)
>> 181 goto error_1;
190 182
191 atomic_inc(&rxnet->nr_conns); 183 atomic_inc(&rxnet->nr_conns);
192 write_lock(&rxnet->conn_lock); 184 write_lock(&rxnet->conn_lock);
193 list_add_tail(&conn->proc_link, &rxnet 185 list_add_tail(&conn->proc_link, &rxnet->conn_proc_list);
194 write_unlock(&rxnet->conn_lock); 186 write_unlock(&rxnet->conn_lock);
195 187
196 rxrpc_see_connection(conn, rxrpc_conn_ !! 188 rxrpc_get_bundle(bundle);
>> 189 rxrpc_get_peer(conn->params.peer);
>> 190 rxrpc_get_local(conn->params.local);
>> 191 key_get(conn->params.key);
>> 192
>> 193 trace_rxrpc_conn(conn->debug_id, rxrpc_conn_new_client,
>> 194 atomic_read(&conn->usage),
>> 195 __builtin_return_address(0));
197 196
198 atomic_inc(&rxnet->nr_client_conns); 197 atomic_inc(&rxnet->nr_client_conns);
199 trace_rxrpc_client(conn, -1, rxrpc_cli 198 trace_rxrpc_client(conn, -1, rxrpc_client_alloc);
>> 199 _leave(" = %p", conn);
200 return conn; 200 return conn;
>> 201
>> 202 error_1:
>> 203 rxrpc_put_client_connection_id(conn);
>> 204 error_0:
>> 205 kfree(conn);
>> 206 _leave(" = %d", ret);
>> 207 return ERR_PTR(ret);
201 } 208 }
202 209
203 /* 210 /*
204 * Determine if a connection may be reused. 211 * Determine if a connection may be reused.
205 */ 212 */
206 static bool rxrpc_may_reuse_conn(struct rxrpc_ 213 static bool rxrpc_may_reuse_conn(struct rxrpc_connection *conn)
207 { 214 {
208 struct rxrpc_net *rxnet; 215 struct rxrpc_net *rxnet;
209 int id_cursor, id, distance, limit; 216 int id_cursor, id, distance, limit;
210 217
211 if (!conn) 218 if (!conn)
212 goto dont_reuse; 219 goto dont_reuse;
213 220
214 rxnet = conn->rxnet; !! 221 rxnet = conn->params.local->rxnet;
215 if (test_bit(RXRPC_CONN_DONT_REUSE, &c 222 if (test_bit(RXRPC_CONN_DONT_REUSE, &conn->flags))
216 goto dont_reuse; 223 goto dont_reuse;
217 224
218 if ((conn->state != RXRPC_CONN_CLIENT_ !! 225 if (conn->state != RXRPC_CONN_CLIENT ||
219 conn->state != RXRPC_CONN_CLIENT) <<
220 conn->proto.epoch != rxnet->epoch) 226 conn->proto.epoch != rxnet->epoch)
221 goto mark_dont_reuse; 227 goto mark_dont_reuse;
222 228
223 /* The IDR tree gets very expensive on 229 /* The IDR tree gets very expensive on memory if the connection IDs are
224 * widely scattered throughout the num 230 * widely scattered throughout the number space, so we shall want to
225 * kill off connections that, say, hav 231 * kill off connections that, say, have an ID more than about four
226 * times the maximum number of client 232 * times the maximum number of client conns away from the current
227 * allocation point to try and keep th 233 * allocation point to try and keep the IDs concentrated.
228 */ 234 */
229 id_cursor = idr_get_cursor(&conn->loca !! 235 id_cursor = idr_get_cursor(&rxrpc_client_conn_ids);
230 id = conn->proto.cid >> RXRPC_CIDSHIFT 236 id = conn->proto.cid >> RXRPC_CIDSHIFT;
231 distance = id - id_cursor; 237 distance = id - id_cursor;
232 if (distance < 0) 238 if (distance < 0)
233 distance = -distance; 239 distance = -distance;
234 limit = max_t(unsigned long, atomic_re 240 limit = max_t(unsigned long, atomic_read(&rxnet->nr_conns) * 4, 1024);
235 if (distance > limit) 241 if (distance > limit)
236 goto mark_dont_reuse; 242 goto mark_dont_reuse;
237 243
238 return true; 244 return true;
239 245
240 mark_dont_reuse: 246 mark_dont_reuse:
241 set_bit(RXRPC_CONN_DONT_REUSE, &conn-> 247 set_bit(RXRPC_CONN_DONT_REUSE, &conn->flags);
242 dont_reuse: 248 dont_reuse:
243 return false; 249 return false;
244 } 250 }
245 251
246 /* 252 /*
247 * Look up the conn bundle that matches the co 253 * Look up the conn bundle that matches the connection parameters, adding it if
248 * it doesn't yet exist. 254 * it doesn't yet exist.
249 */ 255 */
250 int rxrpc_look_up_bundle(struct rxrpc_call *ca !! 256 static struct rxrpc_bundle *rxrpc_look_up_bundle(struct rxrpc_conn_parameters *cp,
>> 257 gfp_t gfp)
251 { 258 {
>> 259 static atomic_t rxrpc_bundle_id;
252 struct rxrpc_bundle *bundle, *candidat 260 struct rxrpc_bundle *bundle, *candidate;
253 struct rxrpc_local *local = call->loca !! 261 struct rxrpc_local *local = cp->local;
254 struct rb_node *p, **pp, *parent; 262 struct rb_node *p, **pp, *parent;
255 long diff; 263 long diff;
256 bool upgrade = test_bit(RXRPC_CALL_UPG <<
257 264
258 _enter("{%px,%x,%u,%u}", 265 _enter("{%px,%x,%u,%u}",
259 call->peer, key_serial(call->ke !! 266 cp->peer, key_serial(cp->key), cp->security_level, cp->upgrade);
260 upgrade); <<
261 267
262 if (test_bit(RXRPC_CALL_EXCLUSIVE, &ca !! 268 if (cp->exclusive)
263 call->bundle = rxrpc_alloc_bun !! 269 return rxrpc_alloc_bundle(cp, gfp);
264 return call->bundle ? 0 : -ENO <<
265 } <<
266 270
267 /* First, see if the bundle is already 271 /* First, see if the bundle is already there. */
268 _debug("search 1"); 272 _debug("search 1");
269 spin_lock(&local->client_bundles_lock) 273 spin_lock(&local->client_bundles_lock);
270 p = local->client_bundles.rb_node; 274 p = local->client_bundles.rb_node;
271 while (p) { 275 while (p) {
272 bundle = rb_entry(p, struct rx 276 bundle = rb_entry(p, struct rxrpc_bundle, local_node);
273 277
274 #define cmp(X, Y) ((long)(X) - (long)(Y)) !! 278 #define cmp(X) ((long)bundle->params.X - (long)cp->X)
275 diff = (cmp(bundle->peer, call !! 279 diff = (cmp(peer) ?:
276 cmp(bundle->key, call- !! 280 cmp(key) ?:
277 cmp(bundle->security_l !! 281 cmp(security_level) ?:
278 cmp(bundle->upgrade, u !! 282 cmp(upgrade));
279 #undef cmp 283 #undef cmp
280 if (diff < 0) 284 if (diff < 0)
281 p = p->rb_left; 285 p = p->rb_left;
282 else if (diff > 0) 286 else if (diff > 0)
283 p = p->rb_right; 287 p = p->rb_right;
284 else 288 else
285 goto found_bundle; 289 goto found_bundle;
286 } 290 }
287 spin_unlock(&local->client_bundles_loc 291 spin_unlock(&local->client_bundles_lock);
288 _debug("not found"); 292 _debug("not found");
289 293
290 /* It wasn't. We need to add one. */ 294 /* It wasn't. We need to add one. */
291 candidate = rxrpc_alloc_bundle(call, g !! 295 candidate = rxrpc_alloc_bundle(cp, gfp);
292 if (!candidate) 296 if (!candidate)
293 return -ENOMEM; !! 297 return NULL;
294 298
295 _debug("search 2"); 299 _debug("search 2");
296 spin_lock(&local->client_bundles_lock) 300 spin_lock(&local->client_bundles_lock);
297 pp = &local->client_bundles.rb_node; 301 pp = &local->client_bundles.rb_node;
298 parent = NULL; 302 parent = NULL;
299 while (*pp) { 303 while (*pp) {
300 parent = *pp; 304 parent = *pp;
301 bundle = rb_entry(parent, stru 305 bundle = rb_entry(parent, struct rxrpc_bundle, local_node);
302 306
303 #define cmp(X, Y) ((long)(X) - (long)(Y)) !! 307 #define cmp(X) ((long)bundle->params.X - (long)cp->X)
304 diff = (cmp(bundle->peer, call !! 308 diff = (cmp(peer) ?:
305 cmp(bundle->key, call- !! 309 cmp(key) ?:
306 cmp(bundle->security_l !! 310 cmp(security_level) ?:
307 cmp(bundle->upgrade, u !! 311 cmp(upgrade));
308 #undef cmp 312 #undef cmp
309 if (diff < 0) 313 if (diff < 0)
310 pp = &(*pp)->rb_left; 314 pp = &(*pp)->rb_left;
311 else if (diff > 0) 315 else if (diff > 0)
312 pp = &(*pp)->rb_right; 316 pp = &(*pp)->rb_right;
313 else 317 else
314 goto found_bundle_free 318 goto found_bundle_free;
315 } 319 }
316 320
317 _debug("new bundle"); 321 _debug("new bundle");
>> 322 candidate->debug_id = atomic_inc_return(&rxrpc_bundle_id);
318 rb_link_node(&candidate->local_node, p 323 rb_link_node(&candidate->local_node, parent, pp);
319 rb_insert_color(&candidate->local_node 324 rb_insert_color(&candidate->local_node, &local->client_bundles);
320 call->bundle = rxrpc_get_bundle(candid !! 325 rxrpc_get_bundle(candidate);
321 spin_unlock(&local->client_bundles_loc 326 spin_unlock(&local->client_bundles_lock);
322 _leave(" = B=%u [new]", call->bundle-> !! 327 _leave(" = %u [new]", candidate->debug_id);
323 return 0; !! 328 return candidate;
324 329
325 found_bundle_free: 330 found_bundle_free:
326 rxrpc_free_bundle(candidate); !! 331 kfree(candidate);
327 found_bundle: 332 found_bundle:
328 call->bundle = rxrpc_get_bundle(bundle !! 333 rxrpc_get_bundle(bundle);
329 rxrpc_activate_bundle(bundle); <<
330 spin_unlock(&local->client_bundles_loc 334 spin_unlock(&local->client_bundles_lock);
331 _leave(" = B=%u [found]", call->bundle !! 335 _leave(" = %u [found]", bundle->debug_id);
332 return 0; !! 336 return bundle;
>> 337 }
>> 338
>> 339 /*
>> 340 * Create or find a client bundle to use for a call.
>> 341 *
>> 342 * If we return with a connection, the call will be on its waiting list. It's
>> 343 * left to the caller to assign a channel and wake up the call.
>> 344 */
>> 345 static struct rxrpc_bundle *rxrpc_prep_call(struct rxrpc_sock *rx,
>> 346 struct rxrpc_call *call,
>> 347 struct rxrpc_conn_parameters *cp,
>> 348 struct sockaddr_rxrpc *srx,
>> 349 gfp_t gfp)
>> 350 {
>> 351 struct rxrpc_bundle *bundle;
>> 352
>> 353 _enter("{%d,%lx},", call->debug_id, call->user_call_ID);
>> 354
>> 355 cp->peer = rxrpc_lookup_peer(rx, cp->local, srx, gfp);
>> 356 if (!cp->peer)
>> 357 goto error;
>> 358
>> 359 call->cong_cwnd = cp->peer->cong_cwnd;
>> 360 if (call->cong_cwnd >= call->cong_ssthresh)
>> 361 call->cong_mode = RXRPC_CALL_CONGEST_AVOIDANCE;
>> 362 else
>> 363 call->cong_mode = RXRPC_CALL_SLOW_START;
>> 364 if (cp->upgrade)
>> 365 __set_bit(RXRPC_CALL_UPGRADE, &call->flags);
>> 366
>> 367 /* Find the client connection bundle. */
>> 368 bundle = rxrpc_look_up_bundle(cp, gfp);
>> 369 if (!bundle)
>> 370 goto error;
>> 371
>> 372 /* Get this call queued. Someone else may activate it whilst we're
>> 373 * lining up a new connection, but that's fine.
>> 374 */
>> 375 spin_lock(&bundle->channel_lock);
>> 376 list_add_tail(&call->chan_wait_link, &bundle->waiting_calls);
>> 377 spin_unlock(&bundle->channel_lock);
>> 378
>> 379 _leave(" = [B=%x]", bundle->debug_id);
>> 380 return bundle;
>> 381
>> 382 error:
>> 383 _leave(" = -ENOMEM");
>> 384 return ERR_PTR(-ENOMEM);
333 } 385 }
334 386
335 /* 387 /*
336 * Allocate a new connection and add it into a 388 * Allocate a new connection and add it into a bundle.
337 */ 389 */
338 static bool rxrpc_add_conn_to_bundle(struct rx !! 390 static void rxrpc_add_conn_to_bundle(struct rxrpc_bundle *bundle, gfp_t gfp)
339 unsigned !! 391 __releases(bundle->channel_lock)
340 { 392 {
341 struct rxrpc_connection *conn, *old; !! 393 struct rxrpc_connection *candidate = NULL, *old = NULL;
342 unsigned int shift = slot * RXRPC_MAXC !! 394 bool conflict;
343 unsigned int i; !! 395 int i;
344 !! 396
345 old = bundle->conns[slot]; !! 397 _enter("");
346 if (old) { !! 398
347 bundle->conns[slot] = NULL; !! 399 conflict = bundle->alloc_conn;
348 bundle->conn_ids[slot] = 0; !! 400 if (!conflict)
349 trace_rxrpc_client(old, -1, rx !! 401 bundle->alloc_conn = true;
350 rxrpc_put_connection(old, rxrp !! 402 spin_unlock(&bundle->channel_lock);
351 } !! 403 if (conflict) {
352 !! 404 _leave(" [conf]");
353 conn = rxrpc_alloc_client_connection(b !! 405 return;
354 if (IS_ERR(conn)) { !! 406 }
355 bundle->alloc_error = PTR_ERR( !! 407
356 return false; !! 408 candidate = rxrpc_alloc_client_connection(bundle, gfp);
357 } !! 409
358 !! 410 spin_lock(&bundle->channel_lock);
359 rxrpc_activate_bundle(bundle); !! 411 bundle->alloc_conn = false;
360 conn->bundle_shift = shift; !! 412
361 bundle->conns[slot] = conn; !! 413 if (IS_ERR(candidate)) {
362 bundle->conn_ids[slot] = conn->debug_i !! 414 bundle->alloc_error = PTR_ERR(candidate);
363 for (i = 0; i < RXRPC_MAXCALLS; i++) !! 415 spin_unlock(&bundle->channel_lock);
364 set_bit(shift + i, &bundle->av !! 416 _leave(" [err %ld]", PTR_ERR(candidate));
365 return true; !! 417 return;
>> 418 }
>> 419
>> 420 bundle->alloc_error = 0;
>> 421
>> 422 for (i = 0; i < ARRAY_SIZE(bundle->conns); i++) {
>> 423 unsigned int shift = i * RXRPC_MAXCALLS;
>> 424 int j;
>> 425
>> 426 old = bundle->conns[i];
>> 427 if (!rxrpc_may_reuse_conn(old)) {
>> 428 if (old)
>> 429 trace_rxrpc_client(old, -1, rxrpc_client_replace);
>> 430 candidate->bundle_shift = shift;
>> 431 bundle->conns[i] = candidate;
>> 432 for (j = 0; j < RXRPC_MAXCALLS; j++)
>> 433 set_bit(shift + j, &bundle->avail_chans);
>> 434 candidate = NULL;
>> 435 break;
>> 436 }
>> 437
>> 438 old = NULL;
>> 439 }
>> 440
>> 441 spin_unlock(&bundle->channel_lock);
>> 442
>> 443 if (candidate) {
>> 444 _debug("discard C=%x", candidate->debug_id);
>> 445 trace_rxrpc_client(candidate, -1, rxrpc_client_duplicate);
>> 446 rxrpc_put_connection(candidate);
>> 447 }
>> 448
>> 449 rxrpc_put_connection(old);
>> 450 _leave("");
366 } 451 }
367 452
368 /* 453 /*
369 * Add a connection to a bundle if there are n 454 * Add a connection to a bundle if there are no usable connections or we have
370 * connections waiting for extra capacity. 455 * connections waiting for extra capacity.
371 */ 456 */
372 static bool rxrpc_bundle_has_space(struct rxrp !! 457 static void rxrpc_maybe_add_conn(struct rxrpc_bundle *bundle, gfp_t gfp)
373 { 458 {
374 int slot = -1, i, usable; !! 459 struct rxrpc_call *call;
>> 460 int i, usable;
375 461
376 _enter(""); 462 _enter("");
377 463
378 bundle->alloc_error = 0; !! 464 spin_lock(&bundle->channel_lock);
379 465
380 /* See if there are any usable connect 466 /* See if there are any usable connections. */
381 usable = 0; 467 usable = 0;
382 for (i = 0; i < ARRAY_SIZE(bundle->con !! 468 for (i = 0; i < ARRAY_SIZE(bundle->conns); i++)
383 if (rxrpc_may_reuse_conn(bundl 469 if (rxrpc_may_reuse_conn(bundle->conns[i]))
384 usable++; 470 usable++;
385 else if (slot == -1) <<
386 slot = i; <<
387 } <<
388 471
389 if (!usable && bundle->upgrade) !! 472 if (!usable && !list_empty(&bundle->waiting_calls)) {
390 bundle->try_upgrade = true; !! 473 call = list_first_entry(&bundle->waiting_calls,
>> 474 struct rxrpc_call, chan_wait_link);
>> 475 if (test_bit(RXRPC_CALL_UPGRADE, &call->flags))
>> 476 bundle->try_upgrade = true;
>> 477 }
391 478
392 if (!usable) 479 if (!usable)
393 goto alloc_conn; 480 goto alloc_conn;
394 481
395 if (!bundle->avail_chans && 482 if (!bundle->avail_chans &&
396 !bundle->try_upgrade && 483 !bundle->try_upgrade &&
>> 484 !list_empty(&bundle->waiting_calls) &&
397 usable < ARRAY_SIZE(bundle->conns) 485 usable < ARRAY_SIZE(bundle->conns))
398 goto alloc_conn; 486 goto alloc_conn;
399 487
>> 488 spin_unlock(&bundle->channel_lock);
400 _leave(""); 489 _leave("");
401 return usable; !! 490 return;
402 491
403 alloc_conn: 492 alloc_conn:
404 return slot >= 0 ? rxrpc_add_conn_to_b !! 493 return rxrpc_add_conn_to_bundle(bundle, gfp);
405 } 494 }
406 495
407 /* 496 /*
408 * Assign a channel to the call at the front o 497 * Assign a channel to the call at the front of the queue and wake the call up.
409 * We don't increment the callNumber counter u 498 * We don't increment the callNumber counter until this number has been exposed
410 * to the world. 499 * to the world.
411 */ 500 */
412 static void rxrpc_activate_one_channel(struct 501 static void rxrpc_activate_one_channel(struct rxrpc_connection *conn,
413 unsigne 502 unsigned int channel)
414 { 503 {
415 struct rxrpc_channel *chan = &conn->ch 504 struct rxrpc_channel *chan = &conn->channels[channel];
416 struct rxrpc_bundle *bundle = conn->bu 505 struct rxrpc_bundle *bundle = conn->bundle;
417 struct rxrpc_call *call = list_entry(b 506 struct rxrpc_call *call = list_entry(bundle->waiting_calls.next,
418 s !! 507 struct rxrpc_call, chan_wait_link);
419 u32 call_id = chan->call_counter + 1; 508 u32 call_id = chan->call_counter + 1;
420 509
421 _enter("C=%x,%u", conn->debug_id, chan 510 _enter("C=%x,%u", conn->debug_id, channel);
422 511
423 list_del_init(&call->wait_link); <<
424 <<
425 trace_rxrpc_client(conn, channel, rxrp 512 trace_rxrpc_client(conn, channel, rxrpc_client_chan_activate);
426 513
427 /* Cancel the final ACK on the previou 514 /* Cancel the final ACK on the previous call if it hasn't been sent yet
428 * as the DATA packet will implicitly 515 * as the DATA packet will implicitly ACK it.
429 */ 516 */
430 clear_bit(RXRPC_CONN_FINAL_ACK_0 + cha 517 clear_bit(RXRPC_CONN_FINAL_ACK_0 + channel, &conn->flags);
431 clear_bit(conn->bundle_shift + channel 518 clear_bit(conn->bundle_shift + channel, &bundle->avail_chans);
432 519
433 rxrpc_see_call(call, rxrpc_call_see_ac !! 520 rxrpc_see_call(call);
434 call->conn = rxrpc_get_connection !! 521 list_del_init(&call->chan_wait_link);
>> 522 call->peer = rxrpc_get_peer(conn->params.peer);
>> 523 call->conn = rxrpc_get_connection(conn);
435 call->cid = conn->proto.cid | ch 524 call->cid = conn->proto.cid | channel;
436 call->call_id = call_id; 525 call->call_id = call_id;
437 call->dest_srx.srx_service = conn->ser !! 526 call->security = conn->security;
438 call->cong_ssthresh = call->peer->cong !! 527 call->security_ix = conn->security_ix;
439 if (call->cong_cwnd >= call->cong_ssth !! 528 call->service_id = conn->service_id;
440 call->cong_mode = RXRPC_CALL_C !! 529
441 else !! 530 trace_rxrpc_connect_call(call);
442 call->cong_mode = RXRPC_CALL_S !! 531 _net("CONNECT call %08x:%08x as call %d on conn %d",
>> 532 call->cid, call->call_id, call->debug_id, conn->debug_id);
>> 533
>> 534 write_lock_bh(&call->state_lock);
>> 535 call->state = RXRPC_CALL_CLIENT_SEND_REQUEST;
>> 536 write_unlock_bh(&call->state_lock);
>> 537
>> 538 /* Paired with the read barrier in rxrpc_connect_call(). This orders
>> 539 * cid and epoch in the connection wrt to call_id without the need to
>> 540 * take the channel_lock.
>> 541 *
>> 542 * We provisionally assign a callNumber at this point, but we don't
>> 543 * confirm it until the call is about to be exposed.
>> 544 *
>> 545 * TODO: Pair with a barrier in the data_ready handler when that looks
>> 546 * at the call ID through a connection channel.
>> 547 */
>> 548 smp_wmb();
443 549
444 chan->call_id = call_id; 550 chan->call_id = call_id;
445 chan->call_debug_id = call->debug_ 551 chan->call_debug_id = call->debug_id;
446 chan->call = call; !! 552 rcu_assign_pointer(chan->call, call);
447 <<
448 rxrpc_see_call(call, rxrpc_call_see_co <<
449 trace_rxrpc_connect_call(call); <<
450 call->tx_last_sent = ktime_get_real(); <<
451 rxrpc_start_call_timer(call); <<
452 rxrpc_set_call_state(call, RXRPC_CALL_ <<
453 wake_up(&call->waitq); 553 wake_up(&call->waitq);
454 } 554 }
455 555
456 /* 556 /*
457 * Remove a connection from the idle list if i 557 * Remove a connection from the idle list if it's on it.
458 */ 558 */
459 static void rxrpc_unidle_conn(struct rxrpc_con !! 559 static void rxrpc_unidle_conn(struct rxrpc_bundle *bundle, struct rxrpc_connection *conn)
460 { 560 {
>> 561 struct rxrpc_net *rxnet = bundle->params.local->rxnet;
>> 562 bool drop_ref;
>> 563
461 if (!list_empty(&conn->cache_link)) { 564 if (!list_empty(&conn->cache_link)) {
462 list_del_init(&conn->cache_lin !! 565 drop_ref = false;
463 rxrpc_put_connection(conn, rxr !! 566 spin_lock(&rxnet->client_conn_cache_lock);
>> 567 if (!list_empty(&conn->cache_link)) {
>> 568 list_del_init(&conn->cache_link);
>> 569 drop_ref = true;
>> 570 }
>> 571 spin_unlock(&rxnet->client_conn_cache_lock);
>> 572 if (drop_ref)
>> 573 rxrpc_put_connection(conn);
464 } 574 }
465 } 575 }
466 576
467 /* 577 /*
468 * Assign channels and callNumbers to waiting !! 578 * Assign channels and callNumbers to waiting calls with channel_lock
>> 579 * held by caller.
469 */ 580 */
470 static void rxrpc_activate_channels(struct rxr !! 581 static void rxrpc_activate_channels_locked(struct rxrpc_bundle *bundle)
471 { 582 {
472 struct rxrpc_connection *conn; 583 struct rxrpc_connection *conn;
473 unsigned long avail, mask; 584 unsigned long avail, mask;
474 unsigned int channel, slot; 585 unsigned int channel, slot;
475 586
476 trace_rxrpc_client(NULL, -1, rxrpc_cli <<
477 <<
478 if (bundle->try_upgrade) 587 if (bundle->try_upgrade)
479 mask = 1; 588 mask = 1;
480 else 589 else
481 mask = ULONG_MAX; 590 mask = ULONG_MAX;
482 591
483 while (!list_empty(&bundle->waiting_ca 592 while (!list_empty(&bundle->waiting_calls)) {
484 avail = bundle->avail_chans & 593 avail = bundle->avail_chans & mask;
485 if (!avail) 594 if (!avail)
486 break; 595 break;
487 channel = __ffs(avail); 596 channel = __ffs(avail);
488 clear_bit(channel, &bundle->av 597 clear_bit(channel, &bundle->avail_chans);
489 598
490 slot = channel / RXRPC_MAXCALL 599 slot = channel / RXRPC_MAXCALLS;
491 conn = bundle->conns[slot]; 600 conn = bundle->conns[slot];
492 if (!conn) 601 if (!conn)
493 break; 602 break;
494 603
495 if (bundle->try_upgrade) 604 if (bundle->try_upgrade)
496 set_bit(RXRPC_CONN_PRO 605 set_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags);
497 rxrpc_unidle_conn(conn); !! 606 rxrpc_unidle_conn(bundle, conn);
498 607
499 channel &= (RXRPC_MAXCALLS - 1 608 channel &= (RXRPC_MAXCALLS - 1);
500 conn->act_chans |= 1 << channe 609 conn->act_chans |= 1 << channel;
501 rxrpc_activate_one_channel(con 610 rxrpc_activate_one_channel(conn, channel);
502 } 611 }
503 } 612 }
504 613
505 /* 614 /*
506 * Connect waiting channels (called from the I !! 615 * Assign channels and callNumbers to waiting calls.
507 */ 616 */
508 void rxrpc_connect_client_calls(struct rxrpc_l !! 617 static void rxrpc_activate_channels(struct rxrpc_bundle *bundle)
509 { 618 {
510 struct rxrpc_call *call; !! 619 _enter("B=%x", bundle->debug_id);
>> 620
>> 621 trace_rxrpc_client(NULL, -1, rxrpc_client_activate_chans);
>> 622
>> 623 if (!bundle->avail_chans)
>> 624 return;
>> 625
>> 626 spin_lock(&bundle->channel_lock);
>> 627 rxrpc_activate_channels_locked(bundle);
>> 628 spin_unlock(&bundle->channel_lock);
>> 629 _leave("");
>> 630 }
511 631
512 while ((call = list_first_entry_or_nul !! 632 /*
513 !! 633 * Wait for a callNumber and a channel to be granted to a call.
514 ) { !! 634 */
515 struct rxrpc_bundle *bundle = !! 635 static int rxrpc_wait_for_channel(struct rxrpc_bundle *bundle,
516 !! 636 struct rxrpc_call *call, gfp_t gfp)
517 spin_lock(&local->client_call_ !! 637 {
518 list_move_tail(&call->wait_lin !! 638 DECLARE_WAITQUEUE(myself, current);
519 rxrpc_see_call(call, rxrpc_cal !! 639 int ret = 0;
520 spin_unlock(&local->client_cal <<
521 640
522 if (rxrpc_bundle_has_space(bun !! 641 _enter("%d", call->debug_id);
523 rxrpc_activate_channel !! 642
>> 643 if (!gfpflags_allow_blocking(gfp)) {
>> 644 rxrpc_maybe_add_conn(bundle, gfp);
>> 645 rxrpc_activate_channels(bundle);
>> 646 ret = bundle->alloc_error ?: -EAGAIN;
>> 647 goto out;
524 } 648 }
>> 649
>> 650 add_wait_queue_exclusive(&call->waitq, &myself);
>> 651 for (;;) {
>> 652 rxrpc_maybe_add_conn(bundle, gfp);
>> 653 rxrpc_activate_channels(bundle);
>> 654 ret = bundle->alloc_error;
>> 655 if (ret < 0)
>> 656 break;
>> 657
>> 658 switch (call->interruptibility) {
>> 659 case RXRPC_INTERRUPTIBLE:
>> 660 case RXRPC_PREINTERRUPTIBLE:
>> 661 set_current_state(TASK_INTERRUPTIBLE);
>> 662 break;
>> 663 case RXRPC_UNINTERRUPTIBLE:
>> 664 default:
>> 665 set_current_state(TASK_UNINTERRUPTIBLE);
>> 666 break;
>> 667 }
>> 668 if (READ_ONCE(call->state) != RXRPC_CALL_CLIENT_AWAIT_CONN)
>> 669 break;
>> 670 if ((call->interruptibility == RXRPC_INTERRUPTIBLE ||
>> 671 call->interruptibility == RXRPC_PREINTERRUPTIBLE) &&
>> 672 signal_pending(current)) {
>> 673 ret = -ERESTARTSYS;
>> 674 break;
>> 675 }
>> 676 schedule();
>> 677 }
>> 678 remove_wait_queue(&call->waitq, &myself);
>> 679 __set_current_state(TASK_RUNNING);
>> 680
>> 681 out:
>> 682 _leave(" = %d", ret);
>> 683 return ret;
>> 684 }
>> 685
>> 686 /*
>> 687 * find a connection for a call
>> 688 * - called in process context with IRQs enabled
>> 689 */
>> 690 int rxrpc_connect_call(struct rxrpc_sock *rx,
>> 691 struct rxrpc_call *call,
>> 692 struct rxrpc_conn_parameters *cp,
>> 693 struct sockaddr_rxrpc *srx,
>> 694 gfp_t gfp)
>> 695 {
>> 696 struct rxrpc_bundle *bundle;
>> 697 struct rxrpc_net *rxnet = cp->local->rxnet;
>> 698 int ret = 0;
>> 699
>> 700 _enter("{%d,%lx},", call->debug_id, call->user_call_ID);
>> 701
>> 702 rxrpc_discard_expired_client_conns(&rxnet->client_conn_reaper);
>> 703
>> 704 bundle = rxrpc_prep_call(rx, call, cp, srx, gfp);
>> 705 if (IS_ERR(bundle)) {
>> 706 ret = PTR_ERR(bundle);
>> 707 goto out;
>> 708 }
>> 709
>> 710 if (call->state == RXRPC_CALL_CLIENT_AWAIT_CONN) {
>> 711 ret = rxrpc_wait_for_channel(bundle, call, gfp);
>> 712 if (ret < 0)
>> 713 goto wait_failed;
>> 714 }
>> 715
>> 716 granted_channel:
>> 717 /* Paired with the write barrier in rxrpc_activate_one_channel(). */
>> 718 smp_rmb();
>> 719
>> 720 out_put_bundle:
>> 721 rxrpc_put_bundle(bundle);
>> 722 out:
>> 723 _leave(" = %d", ret);
>> 724 return ret;
>> 725
>> 726 wait_failed:
>> 727 spin_lock(&bundle->channel_lock);
>> 728 list_del_init(&call->chan_wait_link);
>> 729 spin_unlock(&bundle->channel_lock);
>> 730
>> 731 if (call->state != RXRPC_CALL_CLIENT_AWAIT_CONN) {
>> 732 ret = 0;
>> 733 goto granted_channel;
>> 734 }
>> 735
>> 736 trace_rxrpc_client(call->conn, ret, rxrpc_client_chan_wait_failed);
>> 737 rxrpc_set_call_completion(call, RXRPC_CALL_LOCAL_ERROR, 0, ret);
>> 738 rxrpc_disconnect_client_call(bundle, call);
>> 739 goto out_put_bundle;
525 } 740 }
526 741
527 /* 742 /*
528 * Note that a call, and thus a connection, is 743 * Note that a call, and thus a connection, is about to be exposed to the
529 * world. 744 * world.
530 */ 745 */
531 void rxrpc_expose_client_call(struct rxrpc_cal 746 void rxrpc_expose_client_call(struct rxrpc_call *call)
532 { 747 {
533 unsigned int channel = call->cid & RXR 748 unsigned int channel = call->cid & RXRPC_CHANNELMASK;
534 struct rxrpc_connection *conn = call-> 749 struct rxrpc_connection *conn = call->conn;
535 struct rxrpc_channel *chan = &conn->ch 750 struct rxrpc_channel *chan = &conn->channels[channel];
536 751
537 if (!test_and_set_bit(RXRPC_CALL_EXPOS 752 if (!test_and_set_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
538 /* Mark the call ID as being u 753 /* Mark the call ID as being used. If the callNumber counter
539 * exceeds ~2 billion, we kill 754 * exceeds ~2 billion, we kill the connection after its
540 * outstanding calls have fini 755 * outstanding calls have finished so that the counter doesn't
541 * wrap. 756 * wrap.
542 */ 757 */
543 chan->call_counter++; 758 chan->call_counter++;
544 if (chan->call_counter >= INT_ 759 if (chan->call_counter >= INT_MAX)
545 set_bit(RXRPC_CONN_DON 760 set_bit(RXRPC_CONN_DONT_REUSE, &conn->flags);
546 trace_rxrpc_client(conn, chann 761 trace_rxrpc_client(conn, channel, rxrpc_client_exposed);
547 <<
548 spin_lock(&call->peer->lock); <<
549 hlist_add_head(&call->error_li <<
550 spin_unlock(&call->peer->lock) <<
551 } 762 }
552 } 763 }
553 764
554 /* 765 /*
555 * Set the reap timer. 766 * Set the reap timer.
556 */ 767 */
557 static void rxrpc_set_client_reap_timer(struct !! 768 static void rxrpc_set_client_reap_timer(struct rxrpc_net *rxnet)
558 { 769 {
559 if (!local->kill_all_client_conns) { !! 770 if (!rxnet->kill_all_client_conns) {
560 unsigned long now = jiffies; 771 unsigned long now = jiffies;
561 unsigned long reap_at = now + 772 unsigned long reap_at = now + rxrpc_conn_idle_client_expiry;
562 773
563 if (local->rxnet->live) !! 774 if (rxnet->live)
564 timer_reduce(&local->c !! 775 timer_reduce(&rxnet->client_conn_reap_timer, reap_at);
565 } 776 }
566 } 777 }
567 778
568 /* 779 /*
569 * Disconnect a client call. 780 * Disconnect a client call.
570 */ 781 */
571 void rxrpc_disconnect_client_call(struct rxrpc 782 void rxrpc_disconnect_client_call(struct rxrpc_bundle *bundle, struct rxrpc_call *call)
572 { 783 {
573 struct rxrpc_connection *conn; 784 struct rxrpc_connection *conn;
574 struct rxrpc_channel *chan = NULL; 785 struct rxrpc_channel *chan = NULL;
575 struct rxrpc_local *local = bundle->lo !! 786 struct rxrpc_net *rxnet = bundle->params.local->rxnet;
576 unsigned int channel; 787 unsigned int channel;
577 bool may_reuse; 788 bool may_reuse;
578 u32 cid; 789 u32 cid;
579 790
580 _enter("c=%x", call->debug_id); 791 _enter("c=%x", call->debug_id);
581 792
>> 793 spin_lock(&bundle->channel_lock);
>> 794 set_bit(RXRPC_CALL_DISCONNECTED, &call->flags);
>> 795
582 /* Calls that have never actually been 796 /* Calls that have never actually been assigned a channel can simply be
583 * discarded. 797 * discarded.
584 */ 798 */
585 conn = call->conn; 799 conn = call->conn;
586 if (!conn) { 800 if (!conn) {
587 _debug("call is waiting"); 801 _debug("call is waiting");
588 ASSERTCMP(call->call_id, ==, 0 802 ASSERTCMP(call->call_id, ==, 0);
589 ASSERT(!test_bit(RXRPC_CALL_EX 803 ASSERT(!test_bit(RXRPC_CALL_EXPOSED, &call->flags));
590 /* May still be on ->new_clien !! 804 list_del_init(&call->chan_wait_link);
591 spin_lock(&local->client_call_ !! 805 goto out;
592 list_del_init(&call->wait_link <<
593 spin_unlock(&local->client_cal <<
594 return; <<
595 } 806 }
596 807
597 cid = call->cid; 808 cid = call->cid;
598 channel = cid & RXRPC_CHANNELMASK; 809 channel = cid & RXRPC_CHANNELMASK;
599 chan = &conn->channels[channel]; 810 chan = &conn->channels[channel];
600 trace_rxrpc_client(conn, channel, rxrp 811 trace_rxrpc_client(conn, channel, rxrpc_client_chan_disconnect);
601 812
602 if (WARN_ON(chan->call != call)) !! 813 if (rcu_access_pointer(chan->call) != call) {
603 return; !! 814 spin_unlock(&bundle->channel_lock);
>> 815 BUG();
>> 816 }
604 817
605 may_reuse = rxrpc_may_reuse_conn(conn) 818 may_reuse = rxrpc_may_reuse_conn(conn);
606 819
607 /* If a client call was exposed to the 820 /* If a client call was exposed to the world, we save the result for
608 * retransmission. 821 * retransmission.
609 * 822 *
610 * We use a barrier here so that the c 823 * We use a barrier here so that the call number and abort code can be
611 * read without needing to take a lock 824 * read without needing to take a lock.
612 * 825 *
613 * TODO: Make the incoming packet hand 826 * TODO: Make the incoming packet handler check this and handle
614 * terminal retransmission without req 827 * terminal retransmission without requiring access to the call.
615 */ 828 */
616 if (test_bit(RXRPC_CALL_EXPOSED, &call 829 if (test_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
617 _debug("exposed %u,%u", call-> 830 _debug("exposed %u,%u", call->call_id, call->abort_code);
618 __rxrpc_disconnect_call(conn, 831 __rxrpc_disconnect_call(conn, call);
619 832
620 if (test_and_clear_bit(RXRPC_C 833 if (test_and_clear_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags)) {
621 trace_rxrpc_client(con 834 trace_rxrpc_client(conn, channel, rxrpc_client_to_active);
622 bundle->try_upgrade = 835 bundle->try_upgrade = false;
623 if (may_reuse) 836 if (may_reuse)
624 rxrpc_activate !! 837 rxrpc_activate_channels_locked(bundle);
625 } 838 }
>> 839
626 } 840 }
627 841
628 /* See if we can pass the channel dire 842 /* See if we can pass the channel directly to another call. */
629 if (may_reuse && !list_empty(&bundle-> 843 if (may_reuse && !list_empty(&bundle->waiting_calls)) {
630 trace_rxrpc_client(conn, chann 844 trace_rxrpc_client(conn, channel, rxrpc_client_chan_pass);
631 rxrpc_activate_one_channel(con 845 rxrpc_activate_one_channel(conn, channel);
632 return; !! 846 goto out;
633 } 847 }
634 848
635 /* Schedule the final ACK to be transm 849 /* Schedule the final ACK to be transmitted in a short while so that it
636 * can be skipped if we find a follow- 850 * can be skipped if we find a follow-on call. The first DATA packet
637 * of the follow on call will implicit 851 * of the follow on call will implicitly ACK this call.
638 */ 852 */
639 if (call->completion == RXRPC_CALL_SUC 853 if (call->completion == RXRPC_CALL_SUCCEEDED &&
640 test_bit(RXRPC_CALL_EXPOSED, &call 854 test_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
641 unsigned long final_ack_at = j 855 unsigned long final_ack_at = jiffies + 2;
642 856
643 chan->final_ack_at = final_ack !! 857 WRITE_ONCE(chan->final_ack_at, final_ack_at);
644 smp_wmb(); /* vs rxrpc_process 858 smp_wmb(); /* vs rxrpc_process_delayed_final_acks() */
645 set_bit(RXRPC_CONN_FINAL_ACK_0 859 set_bit(RXRPC_CONN_FINAL_ACK_0 + channel, &conn->flags);
646 rxrpc_reduce_conn_timer(conn, 860 rxrpc_reduce_conn_timer(conn, final_ack_at);
647 } 861 }
648 862
649 /* Deactivate the channel. */ 863 /* Deactivate the channel. */
650 chan->call = NULL; !! 864 rcu_assign_pointer(chan->call, NULL);
651 set_bit(conn->bundle_shift + channel, 865 set_bit(conn->bundle_shift + channel, &conn->bundle->avail_chans);
652 conn->act_chans &= ~(1 << channel); 866 conn->act_chans &= ~(1 << channel);
653 867
654 /* If no channels remain active, then 868 /* If no channels remain active, then put the connection on the idle
655 * list for a short while. Give it a 869 * list for a short while. Give it a ref to stop it going away if it
656 * becomes unbundled. 870 * becomes unbundled.
657 */ 871 */
658 if (!conn->act_chans) { 872 if (!conn->act_chans) {
659 trace_rxrpc_client(conn, chann 873 trace_rxrpc_client(conn, channel, rxrpc_client_to_idle);
660 conn->idle_timestamp = jiffies 874 conn->idle_timestamp = jiffies;
661 875
662 rxrpc_get_connection(conn, rxr !! 876 rxrpc_get_connection(conn);
663 list_move_tail(&conn->cache_li !! 877 spin_lock(&rxnet->client_conn_cache_lock);
>> 878 list_move_tail(&conn->cache_link, &rxnet->idle_client_conns);
>> 879 spin_unlock(&rxnet->client_conn_cache_lock);
664 880
665 rxrpc_set_client_reap_timer(lo !! 881 rxrpc_set_client_reap_timer(rxnet);
666 } 882 }
>> 883
>> 884 out:
>> 885 spin_unlock(&bundle->channel_lock);
>> 886 _leave("");
>> 887 return;
667 } 888 }
668 889
669 /* 890 /*
670 * Remove a connection from a bundle. 891 * Remove a connection from a bundle.
671 */ 892 */
672 static void rxrpc_unbundle_conn(struct rxrpc_c 893 static void rxrpc_unbundle_conn(struct rxrpc_connection *conn)
673 { 894 {
674 struct rxrpc_bundle *bundle = conn->bu 895 struct rxrpc_bundle *bundle = conn->bundle;
>> 896 struct rxrpc_local *local = bundle->params.local;
675 unsigned int bindex; 897 unsigned int bindex;
>> 898 bool need_drop = false, need_put = false;
676 int i; 899 int i;
677 900
678 _enter("C=%x", conn->debug_id); 901 _enter("C=%x", conn->debug_id);
679 902
680 if (conn->flags & RXRPC_CONN_FINAL_ACK 903 if (conn->flags & RXRPC_CONN_FINAL_ACK_MASK)
681 rxrpc_process_delayed_final_ac 904 rxrpc_process_delayed_final_acks(conn, true);
682 905
>> 906 spin_lock(&bundle->channel_lock);
683 bindex = conn->bundle_shift / RXRPC_MA 907 bindex = conn->bundle_shift / RXRPC_MAXCALLS;
684 if (bundle->conns[bindex] == conn) { 908 if (bundle->conns[bindex] == conn) {
685 _debug("clear slot %u", bindex 909 _debug("clear slot %u", bindex);
686 bundle->conns[bindex] = NULL; 910 bundle->conns[bindex] = NULL;
687 bundle->conn_ids[bindex] = 0; <<
688 for (i = 0; i < RXRPC_MAXCALLS 911 for (i = 0; i < RXRPC_MAXCALLS; i++)
689 clear_bit(conn->bundle 912 clear_bit(conn->bundle_shift + i, &bundle->avail_chans);
690 rxrpc_put_client_connection_id !! 913 need_drop = true;
691 rxrpc_deactivate_bundle(bundle <<
692 rxrpc_put_connection(conn, rxr <<
693 } 914 }
694 } !! 915 spin_unlock(&bundle->channel_lock);
695 916
696 /* !! 917 /* If there are no more connections, remove the bundle */
697 * Drop the active count on a bundle. !! 918 if (!bundle->avail_chans) {
698 */ !! 919 _debug("maybe unbundle");
699 void rxrpc_deactivate_bundle(struct rxrpc_bund !! 920 spin_lock(&local->client_bundles_lock);
700 { !! 921
701 struct rxrpc_local *local; !! 922 for (i = 0; i < ARRAY_SIZE(bundle->conns); i++)
702 bool need_put = false; !! 923 if (bundle->conns[i])
703 !! 924 break;
704 if (!bundle) !! 925 if (i == ARRAY_SIZE(bundle->conns) && !bundle->params.exclusive) {
705 return; <<
706 <<
707 local = bundle->local; <<
708 if (atomic_dec_and_lock(&bundle->activ <<
709 if (!bundle->exclusive) { <<
710 _debug("erase bundle") 926 _debug("erase bundle");
711 rb_erase(&bundle->loca 927 rb_erase(&bundle->local_node, &local->client_bundles);
712 need_put = true; 928 need_put = true;
713 } 929 }
714 930
715 spin_unlock(&local->client_bun 931 spin_unlock(&local->client_bundles_lock);
716 if (need_put) 932 if (need_put)
717 rxrpc_put_bundle(bundl !! 933 rxrpc_put_bundle(bundle);
718 } 934 }
>> 935
>> 936 if (need_drop)
>> 937 rxrpc_put_connection(conn);
>> 938 _leave("");
719 } 939 }
720 940
721 /* 941 /*
722 * Clean up a dead client connection. 942 * Clean up a dead client connection.
723 */ 943 */
724 void rxrpc_kill_client_conn(struct rxrpc_conne !! 944 static void rxrpc_kill_client_conn(struct rxrpc_connection *conn)
725 { 945 {
726 struct rxrpc_local *local = conn->loca !! 946 struct rxrpc_local *local = conn->params.local;
727 struct rxrpc_net *rxnet = local->rxnet 947 struct rxrpc_net *rxnet = local->rxnet;
728 948
729 _enter("C=%x", conn->debug_id); 949 _enter("C=%x", conn->debug_id);
730 950
731 trace_rxrpc_client(conn, -1, rxrpc_cli 951 trace_rxrpc_client(conn, -1, rxrpc_client_cleanup);
732 atomic_dec(&rxnet->nr_client_conns); 952 atomic_dec(&rxnet->nr_client_conns);
733 953
734 rxrpc_put_client_connection_id(local, !! 954 rxrpc_put_client_connection_id(conn);
>> 955 rxrpc_kill_connection(conn);
>> 956 }
>> 957
>> 958 /*
>> 959 * Clean up a dead client connections.
>> 960 */
>> 961 void rxrpc_put_client_conn(struct rxrpc_connection *conn)
>> 962 {
>> 963 const void *here = __builtin_return_address(0);
>> 964 unsigned int debug_id = conn->debug_id;
>> 965 int n;
>> 966
>> 967 n = atomic_dec_return(&conn->usage);
>> 968 trace_rxrpc_conn(debug_id, rxrpc_conn_put_client, n, here);
>> 969 if (n <= 0) {
>> 970 ASSERTCMP(n, >=, 0);
>> 971 rxrpc_kill_client_conn(conn);
>> 972 }
735 } 973 }
736 974
737 /* 975 /*
738 * Discard expired client connections from the 976 * Discard expired client connections from the idle list. Each conn in the
739 * idle list has been exposed and holds an ext 977 * idle list has been exposed and holds an extra ref because of that.
740 * 978 *
741 * This may be called from conn setup or from 979 * This may be called from conn setup or from a work item so cannot be
742 * considered non-reentrant. 980 * considered non-reentrant.
743 */ 981 */
744 void rxrpc_discard_expired_client_conns(struct !! 982 void rxrpc_discard_expired_client_conns(struct work_struct *work)
745 { 983 {
746 struct rxrpc_connection *conn; 984 struct rxrpc_connection *conn;
>> 985 struct rxrpc_net *rxnet =
>> 986 container_of(work, struct rxrpc_net, client_conn_reaper);
747 unsigned long expiry, conn_expires_at, 987 unsigned long expiry, conn_expires_at, now;
748 unsigned int nr_conns; 988 unsigned int nr_conns;
749 989
750 _enter(""); 990 _enter("");
751 991
>> 992 if (list_empty(&rxnet->idle_client_conns)) {
>> 993 _leave(" [empty]");
>> 994 return;
>> 995 }
>> 996
>> 997 /* Don't double up on the discarding */
>> 998 if (!spin_trylock(&rxnet->client_conn_discard_lock)) {
>> 999 _leave(" [already]");
>> 1000 return;
>> 1001 }
>> 1002
752 /* We keep an estimate of what the num 1003 /* We keep an estimate of what the number of conns ought to be after
753 * we've discarded some so that we don 1004 * we've discarded some so that we don't overdo the discarding.
754 */ 1005 */
755 nr_conns = atomic_read(&local->rxnet-> !! 1006 nr_conns = atomic_read(&rxnet->nr_client_conns);
756 1007
757 next: 1008 next:
758 conn = list_first_entry_or_null(&local !! 1009 spin_lock(&rxnet->client_conn_cache_lock);
759 struct !! 1010
760 if (!conn) !! 1011 if (list_empty(&rxnet->idle_client_conns))
761 return; !! 1012 goto out;
762 1013
763 if (!local->kill_all_client_conns) { !! 1014 conn = list_entry(rxnet->idle_client_conns.next,
>> 1015 struct rxrpc_connection, cache_link);
>> 1016
>> 1017 if (!rxnet->kill_all_client_conns) {
764 /* If the number of connection 1018 /* If the number of connections is over the reap limit, we
765 * expedite discard by reducin 1019 * expedite discard by reducing the expiry timeout. We must,
766 * however, have at least a sh 1020 * however, have at least a short grace period to be able to do
767 * final-ACK or ABORT retransm 1021 * final-ACK or ABORT retransmission.
768 */ 1022 */
769 expiry = rxrpc_conn_idle_clien 1023 expiry = rxrpc_conn_idle_client_expiry;
770 if (nr_conns > rxrpc_reap_clie 1024 if (nr_conns > rxrpc_reap_client_connections)
771 expiry = rxrpc_conn_id 1025 expiry = rxrpc_conn_idle_client_fast_expiry;
772 if (conn->local->service_close !! 1026 if (conn->params.local->service_closed)
773 expiry = rxrpc_closed_ 1027 expiry = rxrpc_closed_conn_expiry * HZ;
774 1028
775 conn_expires_at = conn->idle_t 1029 conn_expires_at = conn->idle_timestamp + expiry;
776 1030
777 now = jiffies; !! 1031 now = READ_ONCE(jiffies);
778 if (time_after(conn_expires_at 1032 if (time_after(conn_expires_at, now))
779 goto not_yet_expired; 1033 goto not_yet_expired;
780 } 1034 }
781 1035
782 atomic_dec(&conn->active); <<
783 trace_rxrpc_client(conn, -1, rxrpc_cli 1036 trace_rxrpc_client(conn, -1, rxrpc_client_discard);
784 list_del_init(&conn->cache_link); 1037 list_del_init(&conn->cache_link);
785 1038
>> 1039 spin_unlock(&rxnet->client_conn_cache_lock);
>> 1040
786 rxrpc_unbundle_conn(conn); 1041 rxrpc_unbundle_conn(conn);
787 /* Drop the ->cache_link ref */ !! 1042 rxrpc_put_connection(conn); /* Drop the ->cache_link ref */
788 rxrpc_put_connection(conn, rxrpc_conn_ <<
789 1043
790 nr_conns--; 1044 nr_conns--;
791 goto next; 1045 goto next;
792 1046
793 not_yet_expired: 1047 not_yet_expired:
794 /* The connection at the front of the 1048 /* The connection at the front of the queue hasn't yet expired, so
795 * schedule the work item for that poi 1049 * schedule the work item for that point if we discarded something.
796 * 1050 *
797 * We don't worry if the work item is 1051 * We don't worry if the work item is already scheduled - it can look
798 * after rescheduling itself at a late 1052 * after rescheduling itself at a later time. We could cancel it, but
799 * then things get messier. 1053 * then things get messier.
800 */ 1054 */
801 _debug("not yet"); 1055 _debug("not yet");
802 if (!local->kill_all_client_conns) !! 1056 if (!rxnet->kill_all_client_conns)
803 timer_reduce(&local->client_co !! 1057 timer_reduce(&rxnet->client_conn_reap_timer, conn_expires_at);
>> 1058
>> 1059 out:
>> 1060 spin_unlock(&rxnet->client_conn_cache_lock);
>> 1061 spin_unlock(&rxnet->client_conn_discard_lock);
>> 1062 _leave("");
>> 1063 }
>> 1064
>> 1065 /*
>> 1066 * Preemptively destroy all the client connection records rather than waiting
>> 1067 * for them to time out
>> 1068 */
>> 1069 void rxrpc_destroy_all_client_connections(struct rxrpc_net *rxnet)
>> 1070 {
>> 1071 _enter("");
>> 1072
>> 1073 spin_lock(&rxnet->client_conn_cache_lock);
>> 1074 rxnet->kill_all_client_conns = true;
>> 1075 spin_unlock(&rxnet->client_conn_cache_lock);
>> 1076
>> 1077 del_timer_sync(&rxnet->client_conn_reap_timer);
>> 1078
>> 1079 if (!rxrpc_queue_work(&rxnet->client_conn_reaper))
>> 1080 _debug("destroy: queue failed");
804 1081
805 _leave(""); 1082 _leave("");
806 } 1083 }
807 1084
808 /* 1085 /*
809 * Clean up the client connections on a local 1086 * Clean up the client connections on a local endpoint.
810 */ 1087 */
811 void rxrpc_clean_up_local_conns(struct rxrpc_l 1088 void rxrpc_clean_up_local_conns(struct rxrpc_local *local)
812 { 1089 {
813 struct rxrpc_connection *conn; !! 1090 struct rxrpc_connection *conn, *tmp;
>> 1091 struct rxrpc_net *rxnet = local->rxnet;
>> 1092 LIST_HEAD(graveyard);
814 1093
815 _enter(""); 1094 _enter("");
816 1095
817 local->kill_all_client_conns = true; !! 1096 spin_lock(&rxnet->client_conn_cache_lock);
>> 1097
>> 1098 list_for_each_entry_safe(conn, tmp, &rxnet->idle_client_conns,
>> 1099 cache_link) {
>> 1100 if (conn->params.local == local) {
>> 1101 trace_rxrpc_client(conn, -1, rxrpc_client_discard);
>> 1102 list_move(&conn->cache_link, &graveyard);
>> 1103 }
>> 1104 }
818 1105
819 del_timer_sync(&local->client_conn_rea !! 1106 spin_unlock(&rxnet->client_conn_cache_lock);
820 1107
821 while ((conn = list_first_entry_or_nul !! 1108 while (!list_empty(&graveyard)) {
822 !! 1109 conn = list_entry(graveyard.next,
>> 1110 struct rxrpc_connection, cache_link);
823 list_del_init(&conn->cache_lin 1111 list_del_init(&conn->cache_link);
824 atomic_dec(&conn->active); <<
825 trace_rxrpc_client(conn, -1, r <<
826 rxrpc_unbundle_conn(conn); 1112 rxrpc_unbundle_conn(conn);
827 rxrpc_put_connection(conn, rxr !! 1113 rxrpc_put_connection(conn);
828 } 1114 }
829 1115
830 _leave(" [culled]"); 1116 _leave(" [culled]");
831 } 1117 }
832 1118
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.