conn_client.c 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /* Client connection-specific management code.
  3. *
  4. * Copyright (C) 2016, 2020 Red Hat, Inc. All Rights Reserved.
  5. * Written by David Howells ([email protected])
  6. *
  7. * Client connections need to be cached for a little while after they've made a
  8. * call so as to handle retransmitted DATA packets in case the server didn't
  9. * receive the final ACK or terminating ABORT we sent it.
  10. *
  11. * There are flags of relevance to the cache:
  12. *
  13. * (2) DONT_REUSE - The connection should be discarded as soon as possible and
  14. * should not be reused. This is set when an exclusive connection is used
  15. * or a call ID counter overflows.
  16. *
  17. * The caching state may only be changed if the cache lock is held.
  18. *
  19. * There are two idle client connection expiry durations. If the total number
  20. * of connections is below the reap threshold, we use the normal duration; if
  21. * it's above, we use the fast duration.
  22. */
  23. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  24. #include <linux/slab.h>
  25. #include <linux/idr.h>
  26. #include <linux/timer.h>
  27. #include <linux/sched/signal.h>
  28. #include "ar-internal.h"
  29. __read_mostly unsigned int rxrpc_reap_client_connections = 900;
  30. __read_mostly unsigned long rxrpc_conn_idle_client_expiry = 2 * 60 * HZ;
  31. __read_mostly unsigned long rxrpc_conn_idle_client_fast_expiry = 2 * HZ;
  32. /*
  33. * We use machine-unique IDs for our client connections.
  34. */
  35. DEFINE_IDR(rxrpc_client_conn_ids);
  36. static DEFINE_SPINLOCK(rxrpc_conn_id_lock);
  37. static void rxrpc_deactivate_bundle(struct rxrpc_bundle *bundle);
  38. /*
  39. * Get a connection ID and epoch for a client connection from the global pool.
  40. * The connection struct pointer is then recorded in the idr radix tree. The
  41. * epoch doesn't change until the client is rebooted (or, at least, unless the
  42. * module is unloaded).
  43. */
  44. static int rxrpc_get_client_connection_id(struct rxrpc_connection *conn,
  45. gfp_t gfp)
  46. {
  47. struct rxrpc_net *rxnet = conn->params.local->rxnet;
  48. int id;
  49. _enter("");
  50. idr_preload(gfp);
  51. spin_lock(&rxrpc_conn_id_lock);
  52. id = idr_alloc_cyclic(&rxrpc_client_conn_ids, conn,
  53. 1, 0x40000000, GFP_NOWAIT);
  54. if (id < 0)
  55. goto error;
  56. spin_unlock(&rxrpc_conn_id_lock);
  57. idr_preload_end();
  58. conn->proto.epoch = rxnet->epoch;
  59. conn->proto.cid = id << RXRPC_CIDSHIFT;
  60. set_bit(RXRPC_CONN_HAS_IDR, &conn->flags);
  61. _leave(" [CID %x]", conn->proto.cid);
  62. return 0;
  63. error:
  64. spin_unlock(&rxrpc_conn_id_lock);
  65. idr_preload_end();
  66. _leave(" = %d", id);
  67. return id;
  68. }
  69. /*
  70. * Release a connection ID for a client connection from the global pool.
  71. */
  72. static void rxrpc_put_client_connection_id(struct rxrpc_connection *conn)
  73. {
  74. if (test_bit(RXRPC_CONN_HAS_IDR, &conn->flags)) {
  75. spin_lock(&rxrpc_conn_id_lock);
  76. idr_remove(&rxrpc_client_conn_ids,
  77. conn->proto.cid >> RXRPC_CIDSHIFT);
  78. spin_unlock(&rxrpc_conn_id_lock);
  79. }
  80. }
  81. /*
  82. * Destroy the client connection ID tree.
  83. */
  84. void rxrpc_destroy_client_conn_ids(void)
  85. {
  86. struct rxrpc_connection *conn;
  87. int id;
  88. if (!idr_is_empty(&rxrpc_client_conn_ids)) {
  89. idr_for_each_entry(&rxrpc_client_conn_ids, conn, id) {
  90. pr_err("AF_RXRPC: Leaked client conn %p {%d}\n",
  91. conn, refcount_read(&conn->ref));
  92. }
  93. BUG();
  94. }
  95. idr_destroy(&rxrpc_client_conn_ids);
  96. }
  97. /*
  98. * Allocate a connection bundle.
  99. */
  100. static struct rxrpc_bundle *rxrpc_alloc_bundle(struct rxrpc_conn_parameters *cp,
  101. gfp_t gfp)
  102. {
  103. struct rxrpc_bundle *bundle;
  104. bundle = kzalloc(sizeof(*bundle), gfp);
  105. if (bundle) {
  106. bundle->params = *cp;
  107. rxrpc_get_peer(bundle->params.peer);
  108. refcount_set(&bundle->ref, 1);
  109. atomic_set(&bundle->active, 1);
  110. spin_lock_init(&bundle->channel_lock);
  111. INIT_LIST_HEAD(&bundle->waiting_calls);
  112. }
  113. return bundle;
  114. }
  115. struct rxrpc_bundle *rxrpc_get_bundle(struct rxrpc_bundle *bundle)
  116. {
  117. refcount_inc(&bundle->ref);
  118. return bundle;
  119. }
  120. static void rxrpc_free_bundle(struct rxrpc_bundle *bundle)
  121. {
  122. rxrpc_put_peer(bundle->params.peer);
  123. kfree(bundle);
  124. }
  125. void rxrpc_put_bundle(struct rxrpc_bundle *bundle)
  126. {
  127. unsigned int d = bundle->debug_id;
  128. bool dead;
  129. int r;
  130. dead = __refcount_dec_and_test(&bundle->ref, &r);
  131. _debug("PUT B=%x %d", d, r - 1);
  132. if (dead)
  133. rxrpc_free_bundle(bundle);
  134. }
  135. /*
  136. * Allocate a client connection.
  137. */
  138. static struct rxrpc_connection *
  139. rxrpc_alloc_client_connection(struct rxrpc_bundle *bundle, gfp_t gfp)
  140. {
  141. struct rxrpc_connection *conn;
  142. struct rxrpc_net *rxnet = bundle->params.local->rxnet;
  143. int ret;
  144. _enter("");
  145. conn = rxrpc_alloc_connection(gfp);
  146. if (!conn) {
  147. _leave(" = -ENOMEM");
  148. return ERR_PTR(-ENOMEM);
  149. }
  150. refcount_set(&conn->ref, 1);
  151. conn->bundle = bundle;
  152. conn->params = bundle->params;
  153. conn->out_clientflag = RXRPC_CLIENT_INITIATED;
  154. conn->state = RXRPC_CONN_CLIENT;
  155. conn->service_id = conn->params.service_id;
  156. ret = rxrpc_get_client_connection_id(conn, gfp);
  157. if (ret < 0)
  158. goto error_0;
  159. ret = rxrpc_init_client_conn_security(conn);
  160. if (ret < 0)
  161. goto error_1;
  162. atomic_inc(&rxnet->nr_conns);
  163. write_lock(&rxnet->conn_lock);
  164. list_add_tail(&conn->proc_link, &rxnet->conn_proc_list);
  165. write_unlock(&rxnet->conn_lock);
  166. rxrpc_get_bundle(bundle);
  167. rxrpc_get_peer(conn->params.peer);
  168. rxrpc_get_local(conn->params.local);
  169. key_get(conn->params.key);
  170. trace_rxrpc_conn(conn->debug_id, rxrpc_conn_new_client,
  171. refcount_read(&conn->ref),
  172. __builtin_return_address(0));
  173. atomic_inc(&rxnet->nr_client_conns);
  174. trace_rxrpc_client(conn, -1, rxrpc_client_alloc);
  175. _leave(" = %p", conn);
  176. return conn;
  177. error_1:
  178. rxrpc_put_client_connection_id(conn);
  179. error_0:
  180. kfree(conn);
  181. _leave(" = %d", ret);
  182. return ERR_PTR(ret);
  183. }
  184. /*
  185. * Determine if a connection may be reused.
  186. */
  187. static bool rxrpc_may_reuse_conn(struct rxrpc_connection *conn)
  188. {
  189. struct rxrpc_net *rxnet;
  190. int id_cursor, id, distance, limit;
  191. if (!conn)
  192. goto dont_reuse;
  193. rxnet = conn->params.local->rxnet;
  194. if (test_bit(RXRPC_CONN_DONT_REUSE, &conn->flags))
  195. goto dont_reuse;
  196. if (conn->state != RXRPC_CONN_CLIENT ||
  197. conn->proto.epoch != rxnet->epoch)
  198. goto mark_dont_reuse;
  199. /* The IDR tree gets very expensive on memory if the connection IDs are
  200. * widely scattered throughout the number space, so we shall want to
  201. * kill off connections that, say, have an ID more than about four
  202. * times the maximum number of client conns away from the current
  203. * allocation point to try and keep the IDs concentrated.
  204. */
  205. id_cursor = idr_get_cursor(&rxrpc_client_conn_ids);
  206. id = conn->proto.cid >> RXRPC_CIDSHIFT;
  207. distance = id - id_cursor;
  208. if (distance < 0)
  209. distance = -distance;
  210. limit = max_t(unsigned long, atomic_read(&rxnet->nr_conns) * 4, 1024);
  211. if (distance > limit)
  212. goto mark_dont_reuse;
  213. return true;
  214. mark_dont_reuse:
  215. set_bit(RXRPC_CONN_DONT_REUSE, &conn->flags);
  216. dont_reuse:
  217. return false;
  218. }
  219. /*
  220. * Look up the conn bundle that matches the connection parameters, adding it if
  221. * it doesn't yet exist.
  222. */
  223. static struct rxrpc_bundle *rxrpc_look_up_bundle(struct rxrpc_conn_parameters *cp,
  224. gfp_t gfp)
  225. {
  226. static atomic_t rxrpc_bundle_id;
  227. struct rxrpc_bundle *bundle, *candidate;
  228. struct rxrpc_local *local = cp->local;
  229. struct rb_node *p, **pp, *parent;
  230. long diff;
  231. _enter("{%px,%x,%u,%u}",
  232. cp->peer, key_serial(cp->key), cp->security_level, cp->upgrade);
  233. if (cp->exclusive)
  234. return rxrpc_alloc_bundle(cp, gfp);
  235. /* First, see if the bundle is already there. */
  236. _debug("search 1");
  237. spin_lock(&local->client_bundles_lock);
  238. p = local->client_bundles.rb_node;
  239. while (p) {
  240. bundle = rb_entry(p, struct rxrpc_bundle, local_node);
  241. #define cmp(X) ((long)bundle->params.X - (long)cp->X)
  242. diff = (cmp(peer) ?:
  243. cmp(key) ?:
  244. cmp(security_level) ?:
  245. cmp(upgrade));
  246. #undef cmp
  247. if (diff < 0)
  248. p = p->rb_left;
  249. else if (diff > 0)
  250. p = p->rb_right;
  251. else
  252. goto found_bundle;
  253. }
  254. spin_unlock(&local->client_bundles_lock);
  255. _debug("not found");
  256. /* It wasn't. We need to add one. */
  257. candidate = rxrpc_alloc_bundle(cp, gfp);
  258. if (!candidate)
  259. return NULL;
  260. _debug("search 2");
  261. spin_lock(&local->client_bundles_lock);
  262. pp = &local->client_bundles.rb_node;
  263. parent = NULL;
  264. while (*pp) {
  265. parent = *pp;
  266. bundle = rb_entry(parent, struct rxrpc_bundle, local_node);
  267. #define cmp(X) ((long)bundle->params.X - (long)cp->X)
  268. diff = (cmp(peer) ?:
  269. cmp(key) ?:
  270. cmp(security_level) ?:
  271. cmp(upgrade));
  272. #undef cmp
  273. if (diff < 0)
  274. pp = &(*pp)->rb_left;
  275. else if (diff > 0)
  276. pp = &(*pp)->rb_right;
  277. else
  278. goto found_bundle_free;
  279. }
  280. _debug("new bundle");
  281. candidate->debug_id = atomic_inc_return(&rxrpc_bundle_id);
  282. rb_link_node(&candidate->local_node, parent, pp);
  283. rb_insert_color(&candidate->local_node, &local->client_bundles);
  284. rxrpc_get_bundle(candidate);
  285. spin_unlock(&local->client_bundles_lock);
  286. _leave(" = %u [new]", candidate->debug_id);
  287. return candidate;
  288. found_bundle_free:
  289. rxrpc_free_bundle(candidate);
  290. found_bundle:
  291. rxrpc_get_bundle(bundle);
  292. atomic_inc(&bundle->active);
  293. spin_unlock(&local->client_bundles_lock);
  294. _leave(" = %u [found]", bundle->debug_id);
  295. return bundle;
  296. }
  297. /*
  298. * Create or find a client bundle to use for a call.
  299. *
  300. * If we return with a connection, the call will be on its waiting list. It's
  301. * left to the caller to assign a channel and wake up the call.
  302. */
  303. static struct rxrpc_bundle *rxrpc_prep_call(struct rxrpc_sock *rx,
  304. struct rxrpc_call *call,
  305. struct rxrpc_conn_parameters *cp,
  306. struct sockaddr_rxrpc *srx,
  307. gfp_t gfp)
  308. {
  309. struct rxrpc_bundle *bundle;
  310. _enter("{%d,%lx},", call->debug_id, call->user_call_ID);
  311. cp->peer = rxrpc_lookup_peer(rx, cp->local, srx, gfp);
  312. if (!cp->peer)
  313. goto error;
  314. call->cong_cwnd = cp->peer->cong_cwnd;
  315. if (call->cong_cwnd >= call->cong_ssthresh)
  316. call->cong_mode = RXRPC_CALL_CONGEST_AVOIDANCE;
  317. else
  318. call->cong_mode = RXRPC_CALL_SLOW_START;
  319. if (cp->upgrade)
  320. __set_bit(RXRPC_CALL_UPGRADE, &call->flags);
  321. /* Find the client connection bundle. */
  322. bundle = rxrpc_look_up_bundle(cp, gfp);
  323. if (!bundle)
  324. goto error;
  325. /* Get this call queued. Someone else may activate it whilst we're
  326. * lining up a new connection, but that's fine.
  327. */
  328. spin_lock(&bundle->channel_lock);
  329. list_add_tail(&call->chan_wait_link, &bundle->waiting_calls);
  330. spin_unlock(&bundle->channel_lock);
  331. _leave(" = [B=%x]", bundle->debug_id);
  332. return bundle;
  333. error:
  334. _leave(" = -ENOMEM");
  335. return ERR_PTR(-ENOMEM);
  336. }
  337. /*
  338. * Allocate a new connection and add it into a bundle.
  339. */
  340. static void rxrpc_add_conn_to_bundle(struct rxrpc_bundle *bundle, gfp_t gfp)
  341. __releases(bundle->channel_lock)
  342. {
  343. struct rxrpc_connection *candidate = NULL, *old = NULL;
  344. bool conflict;
  345. int i;
  346. _enter("");
  347. conflict = bundle->alloc_conn;
  348. if (!conflict)
  349. bundle->alloc_conn = true;
  350. spin_unlock(&bundle->channel_lock);
  351. if (conflict) {
  352. _leave(" [conf]");
  353. return;
  354. }
  355. candidate = rxrpc_alloc_client_connection(bundle, gfp);
  356. spin_lock(&bundle->channel_lock);
  357. bundle->alloc_conn = false;
  358. if (IS_ERR(candidate)) {
  359. bundle->alloc_error = PTR_ERR(candidate);
  360. spin_unlock(&bundle->channel_lock);
  361. _leave(" [err %ld]", PTR_ERR(candidate));
  362. return;
  363. }
  364. bundle->alloc_error = 0;
  365. for (i = 0; i < ARRAY_SIZE(bundle->conns); i++) {
  366. unsigned int shift = i * RXRPC_MAXCALLS;
  367. int j;
  368. old = bundle->conns[i];
  369. if (!rxrpc_may_reuse_conn(old)) {
  370. if (old)
  371. trace_rxrpc_client(old, -1, rxrpc_client_replace);
  372. candidate->bundle_shift = shift;
  373. atomic_inc(&bundle->active);
  374. bundle->conns[i] = candidate;
  375. for (j = 0; j < RXRPC_MAXCALLS; j++)
  376. set_bit(shift + j, &bundle->avail_chans);
  377. candidate = NULL;
  378. break;
  379. }
  380. old = NULL;
  381. }
  382. spin_unlock(&bundle->channel_lock);
  383. if (candidate) {
  384. _debug("discard C=%x", candidate->debug_id);
  385. trace_rxrpc_client(candidate, -1, rxrpc_client_duplicate);
  386. rxrpc_put_connection(candidate);
  387. }
  388. rxrpc_put_connection(old);
  389. _leave("");
  390. }
  391. /*
  392. * Add a connection to a bundle if there are no usable connections or we have
  393. * connections waiting for extra capacity.
  394. */
  395. static void rxrpc_maybe_add_conn(struct rxrpc_bundle *bundle, gfp_t gfp)
  396. {
  397. struct rxrpc_call *call;
  398. int i, usable;
  399. _enter("");
  400. spin_lock(&bundle->channel_lock);
  401. /* See if there are any usable connections. */
  402. usable = 0;
  403. for (i = 0; i < ARRAY_SIZE(bundle->conns); i++)
  404. if (rxrpc_may_reuse_conn(bundle->conns[i]))
  405. usable++;
  406. if (!usable && !list_empty(&bundle->waiting_calls)) {
  407. call = list_first_entry(&bundle->waiting_calls,
  408. struct rxrpc_call, chan_wait_link);
  409. if (test_bit(RXRPC_CALL_UPGRADE, &call->flags))
  410. bundle->try_upgrade = true;
  411. }
  412. if (!usable)
  413. goto alloc_conn;
  414. if (!bundle->avail_chans &&
  415. !bundle->try_upgrade &&
  416. !list_empty(&bundle->waiting_calls) &&
  417. usable < ARRAY_SIZE(bundle->conns))
  418. goto alloc_conn;
  419. spin_unlock(&bundle->channel_lock);
  420. _leave("");
  421. return;
  422. alloc_conn:
  423. return rxrpc_add_conn_to_bundle(bundle, gfp);
  424. }
  425. /*
  426. * Assign a channel to the call at the front of the queue and wake the call up.
  427. * We don't increment the callNumber counter until this number has been exposed
  428. * to the world.
  429. */
  430. static void rxrpc_activate_one_channel(struct rxrpc_connection *conn,
  431. unsigned int channel)
  432. {
  433. struct rxrpc_channel *chan = &conn->channels[channel];
  434. struct rxrpc_bundle *bundle = conn->bundle;
  435. struct rxrpc_call *call = list_entry(bundle->waiting_calls.next,
  436. struct rxrpc_call, chan_wait_link);
  437. u32 call_id = chan->call_counter + 1;
  438. _enter("C=%x,%u", conn->debug_id, channel);
  439. trace_rxrpc_client(conn, channel, rxrpc_client_chan_activate);
  440. /* Cancel the final ACK on the previous call if it hasn't been sent yet
  441. * as the DATA packet will implicitly ACK it.
  442. */
  443. clear_bit(RXRPC_CONN_FINAL_ACK_0 + channel, &conn->flags);
  444. clear_bit(conn->bundle_shift + channel, &bundle->avail_chans);
  445. rxrpc_see_call(call);
  446. list_del_init(&call->chan_wait_link);
  447. call->peer = rxrpc_get_peer(conn->params.peer);
  448. call->conn = rxrpc_get_connection(conn);
  449. call->cid = conn->proto.cid | channel;
  450. call->call_id = call_id;
  451. call->security = conn->security;
  452. call->security_ix = conn->security_ix;
  453. call->service_id = conn->service_id;
  454. trace_rxrpc_connect_call(call);
  455. _net("CONNECT call %08x:%08x as call %d on conn %d",
  456. call->cid, call->call_id, call->debug_id, conn->debug_id);
  457. write_lock_bh(&call->state_lock);
  458. call->state = RXRPC_CALL_CLIENT_SEND_REQUEST;
  459. write_unlock_bh(&call->state_lock);
  460. /* Paired with the read barrier in rxrpc_connect_call(). This orders
  461. * cid and epoch in the connection wrt to call_id without the need to
  462. * take the channel_lock.
  463. *
  464. * We provisionally assign a callNumber at this point, but we don't
  465. * confirm it until the call is about to be exposed.
  466. *
  467. * TODO: Pair with a barrier in the data_ready handler when that looks
  468. * at the call ID through a connection channel.
  469. */
  470. smp_wmb();
  471. chan->call_id = call_id;
  472. chan->call_debug_id = call->debug_id;
  473. rcu_assign_pointer(chan->call, call);
  474. wake_up(&call->waitq);
  475. }
  476. /*
  477. * Remove a connection from the idle list if it's on it.
  478. */
  479. static void rxrpc_unidle_conn(struct rxrpc_bundle *bundle, struct rxrpc_connection *conn)
  480. {
  481. struct rxrpc_net *rxnet = bundle->params.local->rxnet;
  482. bool drop_ref;
  483. if (!list_empty(&conn->cache_link)) {
  484. drop_ref = false;
  485. spin_lock(&rxnet->client_conn_cache_lock);
  486. if (!list_empty(&conn->cache_link)) {
  487. list_del_init(&conn->cache_link);
  488. drop_ref = true;
  489. }
  490. spin_unlock(&rxnet->client_conn_cache_lock);
  491. if (drop_ref)
  492. rxrpc_put_connection(conn);
  493. }
  494. }
  495. /*
  496. * Assign channels and callNumbers to waiting calls with channel_lock
  497. * held by caller.
  498. */
  499. static void rxrpc_activate_channels_locked(struct rxrpc_bundle *bundle)
  500. {
  501. struct rxrpc_connection *conn;
  502. unsigned long avail, mask;
  503. unsigned int channel, slot;
  504. if (bundle->try_upgrade)
  505. mask = 1;
  506. else
  507. mask = ULONG_MAX;
  508. while (!list_empty(&bundle->waiting_calls)) {
  509. avail = bundle->avail_chans & mask;
  510. if (!avail)
  511. break;
  512. channel = __ffs(avail);
  513. clear_bit(channel, &bundle->avail_chans);
  514. slot = channel / RXRPC_MAXCALLS;
  515. conn = bundle->conns[slot];
  516. if (!conn)
  517. break;
  518. if (bundle->try_upgrade)
  519. set_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags);
  520. rxrpc_unidle_conn(bundle, conn);
  521. channel &= (RXRPC_MAXCALLS - 1);
  522. conn->act_chans |= 1 << channel;
  523. rxrpc_activate_one_channel(conn, channel);
  524. }
  525. }
  526. /*
  527. * Assign channels and callNumbers to waiting calls.
  528. */
  529. static void rxrpc_activate_channels(struct rxrpc_bundle *bundle)
  530. {
  531. _enter("B=%x", bundle->debug_id);
  532. trace_rxrpc_client(NULL, -1, rxrpc_client_activate_chans);
  533. if (!bundle->avail_chans)
  534. return;
  535. spin_lock(&bundle->channel_lock);
  536. rxrpc_activate_channels_locked(bundle);
  537. spin_unlock(&bundle->channel_lock);
  538. _leave("");
  539. }
  540. /*
  541. * Wait for a callNumber and a channel to be granted to a call.
  542. */
  543. static int rxrpc_wait_for_channel(struct rxrpc_bundle *bundle,
  544. struct rxrpc_call *call, gfp_t gfp)
  545. {
  546. DECLARE_WAITQUEUE(myself, current);
  547. int ret = 0;
  548. _enter("%d", call->debug_id);
  549. if (!gfpflags_allow_blocking(gfp)) {
  550. rxrpc_maybe_add_conn(bundle, gfp);
  551. rxrpc_activate_channels(bundle);
  552. ret = bundle->alloc_error ?: -EAGAIN;
  553. goto out;
  554. }
  555. add_wait_queue_exclusive(&call->waitq, &myself);
  556. for (;;) {
  557. rxrpc_maybe_add_conn(bundle, gfp);
  558. rxrpc_activate_channels(bundle);
  559. ret = bundle->alloc_error;
  560. if (ret < 0)
  561. break;
  562. switch (call->interruptibility) {
  563. case RXRPC_INTERRUPTIBLE:
  564. case RXRPC_PREINTERRUPTIBLE:
  565. set_current_state(TASK_INTERRUPTIBLE);
  566. break;
  567. case RXRPC_UNINTERRUPTIBLE:
  568. default:
  569. set_current_state(TASK_UNINTERRUPTIBLE);
  570. break;
  571. }
  572. if (READ_ONCE(call->state) != RXRPC_CALL_CLIENT_AWAIT_CONN)
  573. break;
  574. if ((call->interruptibility == RXRPC_INTERRUPTIBLE ||
  575. call->interruptibility == RXRPC_PREINTERRUPTIBLE) &&
  576. signal_pending(current)) {
  577. ret = -ERESTARTSYS;
  578. break;
  579. }
  580. schedule();
  581. }
  582. remove_wait_queue(&call->waitq, &myself);
  583. __set_current_state(TASK_RUNNING);
  584. out:
  585. _leave(" = %d", ret);
  586. return ret;
  587. }
  588. /*
  589. * find a connection for a call
  590. * - called in process context with IRQs enabled
  591. */
  592. int rxrpc_connect_call(struct rxrpc_sock *rx,
  593. struct rxrpc_call *call,
  594. struct rxrpc_conn_parameters *cp,
  595. struct sockaddr_rxrpc *srx,
  596. gfp_t gfp)
  597. {
  598. struct rxrpc_bundle *bundle;
  599. struct rxrpc_net *rxnet = cp->local->rxnet;
  600. int ret = 0;
  601. _enter("{%d,%lx},", call->debug_id, call->user_call_ID);
  602. rxrpc_discard_expired_client_conns(&rxnet->client_conn_reaper);
  603. bundle = rxrpc_prep_call(rx, call, cp, srx, gfp);
  604. if (IS_ERR(bundle)) {
  605. ret = PTR_ERR(bundle);
  606. goto out;
  607. }
  608. if (call->state == RXRPC_CALL_CLIENT_AWAIT_CONN) {
  609. ret = rxrpc_wait_for_channel(bundle, call, gfp);
  610. if (ret < 0)
  611. goto wait_failed;
  612. }
  613. granted_channel:
  614. /* Paired with the write barrier in rxrpc_activate_one_channel(). */
  615. smp_rmb();
  616. out_put_bundle:
  617. rxrpc_deactivate_bundle(bundle);
  618. rxrpc_put_bundle(bundle);
  619. out:
  620. _leave(" = %d", ret);
  621. return ret;
  622. wait_failed:
  623. spin_lock(&bundle->channel_lock);
  624. list_del_init(&call->chan_wait_link);
  625. spin_unlock(&bundle->channel_lock);
  626. if (call->state != RXRPC_CALL_CLIENT_AWAIT_CONN) {
  627. ret = 0;
  628. goto granted_channel;
  629. }
  630. trace_rxrpc_client(call->conn, ret, rxrpc_client_chan_wait_failed);
  631. rxrpc_set_call_completion(call, RXRPC_CALL_LOCAL_ERROR, 0, ret);
  632. rxrpc_disconnect_client_call(bundle, call);
  633. goto out_put_bundle;
  634. }
  635. /*
  636. * Note that a call, and thus a connection, is about to be exposed to the
  637. * world.
  638. */
  639. void rxrpc_expose_client_call(struct rxrpc_call *call)
  640. {
  641. unsigned int channel = call->cid & RXRPC_CHANNELMASK;
  642. struct rxrpc_connection *conn = call->conn;
  643. struct rxrpc_channel *chan = &conn->channels[channel];
  644. if (!test_and_set_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
  645. /* Mark the call ID as being used. If the callNumber counter
  646. * exceeds ~2 billion, we kill the connection after its
  647. * outstanding calls have finished so that the counter doesn't
  648. * wrap.
  649. */
  650. chan->call_counter++;
  651. if (chan->call_counter >= INT_MAX)
  652. set_bit(RXRPC_CONN_DONT_REUSE, &conn->flags);
  653. trace_rxrpc_client(conn, channel, rxrpc_client_exposed);
  654. }
  655. }
  656. /*
  657. * Set the reap timer.
  658. */
  659. static void rxrpc_set_client_reap_timer(struct rxrpc_net *rxnet)
  660. {
  661. if (!rxnet->kill_all_client_conns) {
  662. unsigned long now = jiffies;
  663. unsigned long reap_at = now + rxrpc_conn_idle_client_expiry;
  664. if (rxnet->live)
  665. timer_reduce(&rxnet->client_conn_reap_timer, reap_at);
  666. }
  667. }
  668. /*
  669. * Disconnect a client call.
  670. */
  671. void rxrpc_disconnect_client_call(struct rxrpc_bundle *bundle, struct rxrpc_call *call)
  672. {
  673. struct rxrpc_connection *conn;
  674. struct rxrpc_channel *chan = NULL;
  675. struct rxrpc_net *rxnet = bundle->params.local->rxnet;
  676. unsigned int channel;
  677. bool may_reuse;
  678. u32 cid;
  679. _enter("c=%x", call->debug_id);
  680. spin_lock(&bundle->channel_lock);
  681. set_bit(RXRPC_CALL_DISCONNECTED, &call->flags);
  682. /* Calls that have never actually been assigned a channel can simply be
  683. * discarded.
  684. */
  685. conn = call->conn;
  686. if (!conn) {
  687. _debug("call is waiting");
  688. ASSERTCMP(call->call_id, ==, 0);
  689. ASSERT(!test_bit(RXRPC_CALL_EXPOSED, &call->flags));
  690. list_del_init(&call->chan_wait_link);
  691. goto out;
  692. }
  693. cid = call->cid;
  694. channel = cid & RXRPC_CHANNELMASK;
  695. chan = &conn->channels[channel];
  696. trace_rxrpc_client(conn, channel, rxrpc_client_chan_disconnect);
  697. if (rcu_access_pointer(chan->call) != call) {
  698. spin_unlock(&bundle->channel_lock);
  699. BUG();
  700. }
  701. may_reuse = rxrpc_may_reuse_conn(conn);
  702. /* If a client call was exposed to the world, we save the result for
  703. * retransmission.
  704. *
  705. * We use a barrier here so that the call number and abort code can be
  706. * read without needing to take a lock.
  707. *
  708. * TODO: Make the incoming packet handler check this and handle
  709. * terminal retransmission without requiring access to the call.
  710. */
  711. if (test_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
  712. _debug("exposed %u,%u", call->call_id, call->abort_code);
  713. __rxrpc_disconnect_call(conn, call);
  714. if (test_and_clear_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags)) {
  715. trace_rxrpc_client(conn, channel, rxrpc_client_to_active);
  716. bundle->try_upgrade = false;
  717. if (may_reuse)
  718. rxrpc_activate_channels_locked(bundle);
  719. }
  720. }
  721. /* See if we can pass the channel directly to another call. */
  722. if (may_reuse && !list_empty(&bundle->waiting_calls)) {
  723. trace_rxrpc_client(conn, channel, rxrpc_client_chan_pass);
  724. rxrpc_activate_one_channel(conn, channel);
  725. goto out;
  726. }
  727. /* Schedule the final ACK to be transmitted in a short while so that it
  728. * can be skipped if we find a follow-on call. The first DATA packet
  729. * of the follow on call will implicitly ACK this call.
  730. */
  731. if (call->completion == RXRPC_CALL_SUCCEEDED &&
  732. test_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
  733. unsigned long final_ack_at = jiffies + 2;
  734. WRITE_ONCE(chan->final_ack_at, final_ack_at);
  735. smp_wmb(); /* vs rxrpc_process_delayed_final_acks() */
  736. set_bit(RXRPC_CONN_FINAL_ACK_0 + channel, &conn->flags);
  737. rxrpc_reduce_conn_timer(conn, final_ack_at);
  738. }
  739. /* Deactivate the channel. */
  740. rcu_assign_pointer(chan->call, NULL);
  741. set_bit(conn->bundle_shift + channel, &conn->bundle->avail_chans);
  742. conn->act_chans &= ~(1 << channel);
  743. /* If no channels remain active, then put the connection on the idle
  744. * list for a short while. Give it a ref to stop it going away if it
  745. * becomes unbundled.
  746. */
  747. if (!conn->act_chans) {
  748. trace_rxrpc_client(conn, channel, rxrpc_client_to_idle);
  749. conn->idle_timestamp = jiffies;
  750. rxrpc_get_connection(conn);
  751. spin_lock(&rxnet->client_conn_cache_lock);
  752. list_move_tail(&conn->cache_link, &rxnet->idle_client_conns);
  753. spin_unlock(&rxnet->client_conn_cache_lock);
  754. rxrpc_set_client_reap_timer(rxnet);
  755. }
  756. out:
  757. spin_unlock(&bundle->channel_lock);
  758. _leave("");
  759. return;
  760. }
  761. /*
  762. * Remove a connection from a bundle.
  763. */
  764. static void rxrpc_unbundle_conn(struct rxrpc_connection *conn)
  765. {
  766. struct rxrpc_bundle *bundle = conn->bundle;
  767. unsigned int bindex;
  768. bool need_drop = false;
  769. int i;
  770. _enter("C=%x", conn->debug_id);
  771. if (conn->flags & RXRPC_CONN_FINAL_ACK_MASK)
  772. rxrpc_process_delayed_final_acks(conn, true);
  773. spin_lock(&bundle->channel_lock);
  774. bindex = conn->bundle_shift / RXRPC_MAXCALLS;
  775. if (bundle->conns[bindex] == conn) {
  776. _debug("clear slot %u", bindex);
  777. bundle->conns[bindex] = NULL;
  778. for (i = 0; i < RXRPC_MAXCALLS; i++)
  779. clear_bit(conn->bundle_shift + i, &bundle->avail_chans);
  780. need_drop = true;
  781. }
  782. spin_unlock(&bundle->channel_lock);
  783. if (need_drop) {
  784. rxrpc_deactivate_bundle(bundle);
  785. rxrpc_put_connection(conn);
  786. }
  787. }
  788. /*
  789. * Drop the active count on a bundle.
  790. */
  791. static void rxrpc_deactivate_bundle(struct rxrpc_bundle *bundle)
  792. {
  793. struct rxrpc_local *local = bundle->params.local;
  794. bool need_put = false;
  795. if (atomic_dec_and_lock(&bundle->active, &local->client_bundles_lock)) {
  796. if (!bundle->params.exclusive) {
  797. _debug("erase bundle");
  798. rb_erase(&bundle->local_node, &local->client_bundles);
  799. need_put = true;
  800. }
  801. spin_unlock(&local->client_bundles_lock);
  802. if (need_put)
  803. rxrpc_put_bundle(bundle);
  804. }
  805. }
  806. /*
  807. * Clean up a dead client connection.
  808. */
  809. static void rxrpc_kill_client_conn(struct rxrpc_connection *conn)
  810. {
  811. struct rxrpc_local *local = conn->params.local;
  812. struct rxrpc_net *rxnet = local->rxnet;
  813. _enter("C=%x", conn->debug_id);
  814. trace_rxrpc_client(conn, -1, rxrpc_client_cleanup);
  815. atomic_dec(&rxnet->nr_client_conns);
  816. rxrpc_put_client_connection_id(conn);
  817. rxrpc_kill_connection(conn);
  818. }
  819. /*
  820. * Clean up a dead client connections.
  821. */
  822. void rxrpc_put_client_conn(struct rxrpc_connection *conn)
  823. {
  824. const void *here = __builtin_return_address(0);
  825. unsigned int debug_id = conn->debug_id;
  826. bool dead;
  827. int r;
  828. dead = __refcount_dec_and_test(&conn->ref, &r);
  829. trace_rxrpc_conn(debug_id, rxrpc_conn_put_client, r - 1, here);
  830. if (dead)
  831. rxrpc_kill_client_conn(conn);
  832. }
  833. /*
  834. * Discard expired client connections from the idle list. Each conn in the
  835. * idle list has been exposed and holds an extra ref because of that.
  836. *
  837. * This may be called from conn setup or from a work item so cannot be
  838. * considered non-reentrant.
  839. */
  840. void rxrpc_discard_expired_client_conns(struct work_struct *work)
  841. {
  842. struct rxrpc_connection *conn;
  843. struct rxrpc_net *rxnet =
  844. container_of(work, struct rxrpc_net, client_conn_reaper);
  845. unsigned long expiry, conn_expires_at, now;
  846. unsigned int nr_conns;
  847. _enter("");
  848. if (list_empty(&rxnet->idle_client_conns)) {
  849. _leave(" [empty]");
  850. return;
  851. }
  852. /* Don't double up on the discarding */
  853. if (!spin_trylock(&rxnet->client_conn_discard_lock)) {
  854. _leave(" [already]");
  855. return;
  856. }
  857. /* We keep an estimate of what the number of conns ought to be after
  858. * we've discarded some so that we don't overdo the discarding.
  859. */
  860. nr_conns = atomic_read(&rxnet->nr_client_conns);
  861. next:
  862. spin_lock(&rxnet->client_conn_cache_lock);
  863. if (list_empty(&rxnet->idle_client_conns))
  864. goto out;
  865. conn = list_entry(rxnet->idle_client_conns.next,
  866. struct rxrpc_connection, cache_link);
  867. if (!rxnet->kill_all_client_conns) {
  868. /* If the number of connections is over the reap limit, we
  869. * expedite discard by reducing the expiry timeout. We must,
  870. * however, have at least a short grace period to be able to do
  871. * final-ACK or ABORT retransmission.
  872. */
  873. expiry = rxrpc_conn_idle_client_expiry;
  874. if (nr_conns > rxrpc_reap_client_connections)
  875. expiry = rxrpc_conn_idle_client_fast_expiry;
  876. if (conn->params.local->service_closed)
  877. expiry = rxrpc_closed_conn_expiry * HZ;
  878. conn_expires_at = conn->idle_timestamp + expiry;
  879. now = READ_ONCE(jiffies);
  880. if (time_after(conn_expires_at, now))
  881. goto not_yet_expired;
  882. }
  883. trace_rxrpc_client(conn, -1, rxrpc_client_discard);
  884. list_del_init(&conn->cache_link);
  885. spin_unlock(&rxnet->client_conn_cache_lock);
  886. rxrpc_unbundle_conn(conn);
  887. rxrpc_put_connection(conn); /* Drop the ->cache_link ref */
  888. nr_conns--;
  889. goto next;
  890. not_yet_expired:
  891. /* The connection at the front of the queue hasn't yet expired, so
  892. * schedule the work item for that point if we discarded something.
  893. *
  894. * We don't worry if the work item is already scheduled - it can look
  895. * after rescheduling itself at a later time. We could cancel it, but
  896. * then things get messier.
  897. */
  898. _debug("not yet");
  899. if (!rxnet->kill_all_client_conns)
  900. timer_reduce(&rxnet->client_conn_reap_timer, conn_expires_at);
  901. out:
  902. spin_unlock(&rxnet->client_conn_cache_lock);
  903. spin_unlock(&rxnet->client_conn_discard_lock);
  904. _leave("");
  905. }
  906. /*
  907. * Preemptively destroy all the client connection records rather than waiting
  908. * for them to time out
  909. */
  910. void rxrpc_destroy_all_client_connections(struct rxrpc_net *rxnet)
  911. {
  912. _enter("");
  913. spin_lock(&rxnet->client_conn_cache_lock);
  914. rxnet->kill_all_client_conns = true;
  915. spin_unlock(&rxnet->client_conn_cache_lock);
  916. del_timer_sync(&rxnet->client_conn_reap_timer);
  917. if (!rxrpc_queue_work(&rxnet->client_conn_reaper))
  918. _debug("destroy: queue failed");
  919. _leave("");
  920. }
  921. /*
  922. * Clean up the client connections on a local endpoint.
  923. */
  924. void rxrpc_clean_up_local_conns(struct rxrpc_local *local)
  925. {
  926. struct rxrpc_connection *conn, *tmp;
  927. struct rxrpc_net *rxnet = local->rxnet;
  928. LIST_HEAD(graveyard);
  929. _enter("");
  930. spin_lock(&rxnet->client_conn_cache_lock);
  931. list_for_each_entry_safe(conn, tmp, &rxnet->idle_client_conns,
  932. cache_link) {
  933. if (conn->params.local == local) {
  934. trace_rxrpc_client(conn, -1, rxrpc_client_discard);
  935. list_move(&conn->cache_link, &graveyard);
  936. }
  937. }
  938. spin_unlock(&rxnet->client_conn_cache_lock);
  939. while (!list_empty(&graveyard)) {
  940. conn = list_entry(graveyard.next,
  941. struct rxrpc_connection, cache_link);
  942. list_del_init(&conn->cache_link);
  943. rxrpc_unbundle_conn(conn);
  944. rxrpc_put_connection(conn);
  945. }
  946. _leave(" [culled]");
  947. }