tcp.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754
  1. /*
  2. * Copyright (c) 2006, 2018 Oracle and/or its affiliates. All rights reserved.
  3. *
  4. * This software is available to you under a choice of one of two
  5. * licenses. You may choose to be licensed under the terms of the GNU
  6. * General Public License (GPL) Version 2, available from the file
  7. * COPYING in the main directory of this source tree, or the
  8. * OpenIB.org BSD license below:
  9. *
  10. * Redistribution and use in source and binary forms, with or
  11. * without modification, are permitted provided that the following
  12. * conditions are met:
  13. *
  14. * - Redistributions of source code must retain the above
  15. * copyright notice, this list of conditions and the following
  16. * disclaimer.
  17. *
  18. * - Redistributions in binary form must reproduce the above
  19. * copyright notice, this list of conditions and the following
  20. * disclaimer in the documentation and/or other materials
  21. * provided with the distribution.
  22. *
  23. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30. * SOFTWARE.
  31. *
  32. */
  33. #include <linux/kernel.h>
  34. #include <linux/slab.h>
  35. #include <linux/in.h>
  36. #include <linux/module.h>
  37. #include <net/tcp.h>
  38. #include <net/net_namespace.h>
  39. #include <net/netns/generic.h>
  40. #include <net/addrconf.h>
  41. #include "rds.h"
  42. #include "tcp.h"
  43. /* only for info exporting */
  44. static DEFINE_SPINLOCK(rds_tcp_tc_list_lock);
  45. static LIST_HEAD(rds_tcp_tc_list);
  46. /* rds_tcp_tc_count counts only IPv4 connections.
  47. * rds6_tcp_tc_count counts both IPv4 and IPv6 connections.
  48. */
  49. static unsigned int rds_tcp_tc_count;
  50. #if IS_ENABLED(CONFIG_IPV6)
  51. static unsigned int rds6_tcp_tc_count;
  52. #endif
  53. /* Track rds_tcp_connection structs so they can be cleaned up */
  54. static DEFINE_SPINLOCK(rds_tcp_conn_lock);
  55. static LIST_HEAD(rds_tcp_conn_list);
  56. static atomic_t rds_tcp_unloading = ATOMIC_INIT(0);
  57. static struct kmem_cache *rds_tcp_conn_slab;
  58. static int rds_tcp_skbuf_handler(struct ctl_table *ctl, int write,
  59. void *buffer, size_t *lenp, loff_t *fpos);
  60. static int rds_tcp_min_sndbuf = SOCK_MIN_SNDBUF;
  61. static int rds_tcp_min_rcvbuf = SOCK_MIN_RCVBUF;
  62. static struct ctl_table rds_tcp_sysctl_table[] = {
  63. #define RDS_TCP_SNDBUF 0
  64. {
  65. .procname = "rds_tcp_sndbuf",
  66. /* data is per-net pointer */
  67. .maxlen = sizeof(int),
  68. .mode = 0644,
  69. .proc_handler = rds_tcp_skbuf_handler,
  70. .extra1 = &rds_tcp_min_sndbuf,
  71. },
  72. #define RDS_TCP_RCVBUF 1
  73. {
  74. .procname = "rds_tcp_rcvbuf",
  75. /* data is per-net pointer */
  76. .maxlen = sizeof(int),
  77. .mode = 0644,
  78. .proc_handler = rds_tcp_skbuf_handler,
  79. .extra1 = &rds_tcp_min_rcvbuf,
  80. },
  81. { }
  82. };
  83. u32 rds_tcp_write_seq(struct rds_tcp_connection *tc)
  84. {
  85. /* seq# of the last byte of data in tcp send buffer */
  86. return tcp_sk(tc->t_sock->sk)->write_seq;
  87. }
  88. u32 rds_tcp_snd_una(struct rds_tcp_connection *tc)
  89. {
  90. return tcp_sk(tc->t_sock->sk)->snd_una;
  91. }
  92. void rds_tcp_restore_callbacks(struct socket *sock,
  93. struct rds_tcp_connection *tc)
  94. {
  95. rdsdebug("restoring sock %p callbacks from tc %p\n", sock, tc);
  96. write_lock_bh(&sock->sk->sk_callback_lock);
  97. /* done under the callback_lock to serialize with write_space */
  98. spin_lock(&rds_tcp_tc_list_lock);
  99. list_del_init(&tc->t_list_item);
  100. #if IS_ENABLED(CONFIG_IPV6)
  101. rds6_tcp_tc_count--;
  102. #endif
  103. if (!tc->t_cpath->cp_conn->c_isv6)
  104. rds_tcp_tc_count--;
  105. spin_unlock(&rds_tcp_tc_list_lock);
  106. tc->t_sock = NULL;
  107. sock->sk->sk_write_space = tc->t_orig_write_space;
  108. sock->sk->sk_data_ready = tc->t_orig_data_ready;
  109. sock->sk->sk_state_change = tc->t_orig_state_change;
  110. sock->sk->sk_user_data = NULL;
  111. write_unlock_bh(&sock->sk->sk_callback_lock);
  112. }
  113. /*
  114. * rds_tcp_reset_callbacks() switches the to the new sock and
  115. * returns the existing tc->t_sock.
  116. *
  117. * The only functions that set tc->t_sock are rds_tcp_set_callbacks
  118. * and rds_tcp_reset_callbacks. Send and receive trust that
  119. * it is set. The absence of RDS_CONN_UP bit protects those paths
  120. * from being called while it isn't set.
  121. */
  122. void rds_tcp_reset_callbacks(struct socket *sock,
  123. struct rds_conn_path *cp)
  124. {
  125. struct rds_tcp_connection *tc = cp->cp_transport_data;
  126. struct socket *osock = tc->t_sock;
  127. if (!osock)
  128. goto newsock;
  129. /* Need to resolve a duelling SYN between peers.
  130. * We have an outstanding SYN to this peer, which may
  131. * potentially have transitioned to the RDS_CONN_UP state,
  132. * so we must quiesce any send threads before resetting
  133. * cp_transport_data. We quiesce these threads by setting
  134. * cp_state to something other than RDS_CONN_UP, and then
  135. * waiting for any existing threads in rds_send_xmit to
  136. * complete release_in_xmit(). (Subsequent threads entering
  137. * rds_send_xmit() will bail on !rds_conn_up().
  138. *
  139. * However an incoming syn-ack at this point would end up
  140. * marking the conn as RDS_CONN_UP, and would again permit
  141. * rds_send_xmi() threads through, so ideally we would
  142. * synchronize on RDS_CONN_UP after lock_sock(), but cannot
  143. * do that: waiting on !RDS_IN_XMIT after lock_sock() may
  144. * end up deadlocking with tcp_sendmsg(), and the RDS_IN_XMIT
  145. * would not get set. As a result, we set c_state to
  146. * RDS_CONN_RESETTTING, to ensure that rds_tcp_state_change
  147. * cannot mark rds_conn_path_up() in the window before lock_sock()
  148. */
  149. atomic_set(&cp->cp_state, RDS_CONN_RESETTING);
  150. wait_event(cp->cp_waitq, !test_bit(RDS_IN_XMIT, &cp->cp_flags));
  151. /* reset receive side state for rds_tcp_data_recv() for osock */
  152. cancel_delayed_work_sync(&cp->cp_send_w);
  153. cancel_delayed_work_sync(&cp->cp_recv_w);
  154. lock_sock(osock->sk);
  155. if (tc->t_tinc) {
  156. rds_inc_put(&tc->t_tinc->ti_inc);
  157. tc->t_tinc = NULL;
  158. }
  159. tc->t_tinc_hdr_rem = sizeof(struct rds_header);
  160. tc->t_tinc_data_rem = 0;
  161. rds_tcp_restore_callbacks(osock, tc);
  162. release_sock(osock->sk);
  163. sock_release(osock);
  164. newsock:
  165. rds_send_path_reset(cp);
  166. lock_sock(sock->sk);
  167. rds_tcp_set_callbacks(sock, cp);
  168. release_sock(sock->sk);
  169. }
  170. /* Add tc to rds_tcp_tc_list and set tc->t_sock. See comments
  171. * above rds_tcp_reset_callbacks for notes about synchronization
  172. * with data path
  173. */
  174. void rds_tcp_set_callbacks(struct socket *sock, struct rds_conn_path *cp)
  175. {
  176. struct rds_tcp_connection *tc = cp->cp_transport_data;
  177. rdsdebug("setting sock %p callbacks to tc %p\n", sock, tc);
  178. write_lock_bh(&sock->sk->sk_callback_lock);
  179. /* done under the callback_lock to serialize with write_space */
  180. spin_lock(&rds_tcp_tc_list_lock);
  181. list_add_tail(&tc->t_list_item, &rds_tcp_tc_list);
  182. #if IS_ENABLED(CONFIG_IPV6)
  183. rds6_tcp_tc_count++;
  184. #endif
  185. if (!tc->t_cpath->cp_conn->c_isv6)
  186. rds_tcp_tc_count++;
  187. spin_unlock(&rds_tcp_tc_list_lock);
  188. /* accepted sockets need our listen data ready undone */
  189. if (sock->sk->sk_data_ready == rds_tcp_listen_data_ready)
  190. sock->sk->sk_data_ready = sock->sk->sk_user_data;
  191. tc->t_sock = sock;
  192. tc->t_cpath = cp;
  193. tc->t_orig_data_ready = sock->sk->sk_data_ready;
  194. tc->t_orig_write_space = sock->sk->sk_write_space;
  195. tc->t_orig_state_change = sock->sk->sk_state_change;
  196. sock->sk->sk_user_data = cp;
  197. sock->sk->sk_data_ready = rds_tcp_data_ready;
  198. sock->sk->sk_write_space = rds_tcp_write_space;
  199. sock->sk->sk_state_change = rds_tcp_state_change;
  200. write_unlock_bh(&sock->sk->sk_callback_lock);
  201. }
  202. /* Handle RDS_INFO_TCP_SOCKETS socket option. It only returns IPv4
  203. * connections for backward compatibility.
  204. */
  205. static void rds_tcp_tc_info(struct socket *rds_sock, unsigned int len,
  206. struct rds_info_iterator *iter,
  207. struct rds_info_lengths *lens)
  208. {
  209. struct rds_info_tcp_socket tsinfo;
  210. struct rds_tcp_connection *tc;
  211. unsigned long flags;
  212. spin_lock_irqsave(&rds_tcp_tc_list_lock, flags);
  213. if (len / sizeof(tsinfo) < rds_tcp_tc_count)
  214. goto out;
  215. list_for_each_entry(tc, &rds_tcp_tc_list, t_list_item) {
  216. struct inet_sock *inet = inet_sk(tc->t_sock->sk);
  217. if (tc->t_cpath->cp_conn->c_isv6)
  218. continue;
  219. tsinfo.local_addr = inet->inet_saddr;
  220. tsinfo.local_port = inet->inet_sport;
  221. tsinfo.peer_addr = inet->inet_daddr;
  222. tsinfo.peer_port = inet->inet_dport;
  223. tsinfo.hdr_rem = tc->t_tinc_hdr_rem;
  224. tsinfo.data_rem = tc->t_tinc_data_rem;
  225. tsinfo.last_sent_nxt = tc->t_last_sent_nxt;
  226. tsinfo.last_expected_una = tc->t_last_expected_una;
  227. tsinfo.last_seen_una = tc->t_last_seen_una;
  228. tsinfo.tos = tc->t_cpath->cp_conn->c_tos;
  229. rds_info_copy(iter, &tsinfo, sizeof(tsinfo));
  230. }
  231. out:
  232. lens->nr = rds_tcp_tc_count;
  233. lens->each = sizeof(tsinfo);
  234. spin_unlock_irqrestore(&rds_tcp_tc_list_lock, flags);
  235. }
  236. #if IS_ENABLED(CONFIG_IPV6)
  237. /* Handle RDS6_INFO_TCP_SOCKETS socket option. It returns both IPv4 and
  238. * IPv6 connections. IPv4 connection address is returned in an IPv4 mapped
  239. * address.
  240. */
  241. static void rds6_tcp_tc_info(struct socket *sock, unsigned int len,
  242. struct rds_info_iterator *iter,
  243. struct rds_info_lengths *lens)
  244. {
  245. struct rds6_info_tcp_socket tsinfo6;
  246. struct rds_tcp_connection *tc;
  247. unsigned long flags;
  248. spin_lock_irqsave(&rds_tcp_tc_list_lock, flags);
  249. if (len / sizeof(tsinfo6) < rds6_tcp_tc_count)
  250. goto out;
  251. list_for_each_entry(tc, &rds_tcp_tc_list, t_list_item) {
  252. struct sock *sk = tc->t_sock->sk;
  253. struct inet_sock *inet = inet_sk(sk);
  254. tsinfo6.local_addr = sk->sk_v6_rcv_saddr;
  255. tsinfo6.local_port = inet->inet_sport;
  256. tsinfo6.peer_addr = sk->sk_v6_daddr;
  257. tsinfo6.peer_port = inet->inet_dport;
  258. tsinfo6.hdr_rem = tc->t_tinc_hdr_rem;
  259. tsinfo6.data_rem = tc->t_tinc_data_rem;
  260. tsinfo6.last_sent_nxt = tc->t_last_sent_nxt;
  261. tsinfo6.last_expected_una = tc->t_last_expected_una;
  262. tsinfo6.last_seen_una = tc->t_last_seen_una;
  263. rds_info_copy(iter, &tsinfo6, sizeof(tsinfo6));
  264. }
  265. out:
  266. lens->nr = rds6_tcp_tc_count;
  267. lens->each = sizeof(tsinfo6);
  268. spin_unlock_irqrestore(&rds_tcp_tc_list_lock, flags);
  269. }
  270. #endif
  271. int rds_tcp_laddr_check(struct net *net, const struct in6_addr *addr,
  272. __u32 scope_id)
  273. {
  274. struct net_device *dev = NULL;
  275. #if IS_ENABLED(CONFIG_IPV6)
  276. int ret;
  277. #endif
  278. if (ipv6_addr_v4mapped(addr)) {
  279. if (inet_addr_type(net, addr->s6_addr32[3]) == RTN_LOCAL)
  280. return 0;
  281. return -EADDRNOTAVAIL;
  282. }
  283. /* If the scope_id is specified, check only those addresses
  284. * hosted on the specified interface.
  285. */
  286. if (scope_id != 0) {
  287. rcu_read_lock();
  288. dev = dev_get_by_index_rcu(net, scope_id);
  289. /* scope_id is not valid... */
  290. if (!dev) {
  291. rcu_read_unlock();
  292. return -EADDRNOTAVAIL;
  293. }
  294. rcu_read_unlock();
  295. }
  296. #if IS_ENABLED(CONFIG_IPV6)
  297. ret = ipv6_chk_addr(net, addr, dev, 0);
  298. if (ret)
  299. return 0;
  300. #endif
  301. return -EADDRNOTAVAIL;
  302. }
  303. static void rds_tcp_conn_free(void *arg)
  304. {
  305. struct rds_tcp_connection *tc = arg;
  306. unsigned long flags;
  307. rdsdebug("freeing tc %p\n", tc);
  308. spin_lock_irqsave(&rds_tcp_conn_lock, flags);
  309. if (!tc->t_tcp_node_detached)
  310. list_del(&tc->t_tcp_node);
  311. spin_unlock_irqrestore(&rds_tcp_conn_lock, flags);
  312. kmem_cache_free(rds_tcp_conn_slab, tc);
  313. }
  314. static int rds_tcp_conn_alloc(struct rds_connection *conn, gfp_t gfp)
  315. {
  316. struct rds_tcp_connection *tc;
  317. int i, j;
  318. int ret = 0;
  319. for (i = 0; i < RDS_MPATH_WORKERS; i++) {
  320. tc = kmem_cache_alloc(rds_tcp_conn_slab, gfp);
  321. if (!tc) {
  322. ret = -ENOMEM;
  323. goto fail;
  324. }
  325. mutex_init(&tc->t_conn_path_lock);
  326. tc->t_sock = NULL;
  327. tc->t_tinc = NULL;
  328. tc->t_tinc_hdr_rem = sizeof(struct rds_header);
  329. tc->t_tinc_data_rem = 0;
  330. conn->c_path[i].cp_transport_data = tc;
  331. tc->t_cpath = &conn->c_path[i];
  332. tc->t_tcp_node_detached = true;
  333. rdsdebug("rds_conn_path [%d] tc %p\n", i,
  334. conn->c_path[i].cp_transport_data);
  335. }
  336. spin_lock_irq(&rds_tcp_conn_lock);
  337. for (i = 0; i < RDS_MPATH_WORKERS; i++) {
  338. tc = conn->c_path[i].cp_transport_data;
  339. tc->t_tcp_node_detached = false;
  340. list_add_tail(&tc->t_tcp_node, &rds_tcp_conn_list);
  341. }
  342. spin_unlock_irq(&rds_tcp_conn_lock);
  343. fail:
  344. if (ret) {
  345. for (j = 0; j < i; j++)
  346. rds_tcp_conn_free(conn->c_path[j].cp_transport_data);
  347. }
  348. return ret;
  349. }
  350. static bool list_has_conn(struct list_head *list, struct rds_connection *conn)
  351. {
  352. struct rds_tcp_connection *tc, *_tc;
  353. list_for_each_entry_safe(tc, _tc, list, t_tcp_node) {
  354. if (tc->t_cpath->cp_conn == conn)
  355. return true;
  356. }
  357. return false;
  358. }
  359. static void rds_tcp_set_unloading(void)
  360. {
  361. atomic_set(&rds_tcp_unloading, 1);
  362. }
  363. static bool rds_tcp_is_unloading(struct rds_connection *conn)
  364. {
  365. return atomic_read(&rds_tcp_unloading) != 0;
  366. }
  367. static void rds_tcp_destroy_conns(void)
  368. {
  369. struct rds_tcp_connection *tc, *_tc;
  370. LIST_HEAD(tmp_list);
  371. /* avoid calling conn_destroy with irqs off */
  372. spin_lock_irq(&rds_tcp_conn_lock);
  373. list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) {
  374. if (!list_has_conn(&tmp_list, tc->t_cpath->cp_conn))
  375. list_move_tail(&tc->t_tcp_node, &tmp_list);
  376. }
  377. spin_unlock_irq(&rds_tcp_conn_lock);
  378. list_for_each_entry_safe(tc, _tc, &tmp_list, t_tcp_node)
  379. rds_conn_destroy(tc->t_cpath->cp_conn);
  380. }
  381. static void rds_tcp_exit(void);
  382. static u8 rds_tcp_get_tos_map(u8 tos)
  383. {
  384. /* all user tos mapped to default 0 for TCP transport */
  385. return 0;
  386. }
  387. struct rds_transport rds_tcp_transport = {
  388. .laddr_check = rds_tcp_laddr_check,
  389. .xmit_path_prepare = rds_tcp_xmit_path_prepare,
  390. .xmit_path_complete = rds_tcp_xmit_path_complete,
  391. .xmit = rds_tcp_xmit,
  392. .recv_path = rds_tcp_recv_path,
  393. .conn_alloc = rds_tcp_conn_alloc,
  394. .conn_free = rds_tcp_conn_free,
  395. .conn_path_connect = rds_tcp_conn_path_connect,
  396. .conn_path_shutdown = rds_tcp_conn_path_shutdown,
  397. .inc_copy_to_user = rds_tcp_inc_copy_to_user,
  398. .inc_free = rds_tcp_inc_free,
  399. .stats_info_copy = rds_tcp_stats_info_copy,
  400. .exit = rds_tcp_exit,
  401. .get_tos_map = rds_tcp_get_tos_map,
  402. .t_owner = THIS_MODULE,
  403. .t_name = "tcp",
  404. .t_type = RDS_TRANS_TCP,
  405. .t_prefer_loopback = 1,
  406. .t_mp_capable = 1,
  407. .t_unloading = rds_tcp_is_unloading,
  408. };
  409. static unsigned int rds_tcp_netid;
  410. /* per-network namespace private data for this module */
  411. struct rds_tcp_net {
  412. struct socket *rds_tcp_listen_sock;
  413. struct work_struct rds_tcp_accept_w;
  414. struct ctl_table_header *rds_tcp_sysctl;
  415. struct ctl_table *ctl_table;
  416. int sndbuf_size;
  417. int rcvbuf_size;
  418. };
  419. /* All module specific customizations to the RDS-TCP socket should be done in
  420. * rds_tcp_tune() and applied after socket creation.
  421. */
  422. bool rds_tcp_tune(struct socket *sock)
  423. {
  424. struct sock *sk = sock->sk;
  425. struct net *net = sock_net(sk);
  426. struct rds_tcp_net *rtn;
  427. tcp_sock_set_nodelay(sock->sk);
  428. lock_sock(sk);
  429. /* TCP timer functions might access net namespace even after
  430. * a process which created this net namespace terminated.
  431. */
  432. if (!sk->sk_net_refcnt) {
  433. if (!maybe_get_net(net)) {
  434. release_sock(sk);
  435. return false;
  436. }
  437. sk->sk_net_refcnt = 1;
  438. netns_tracker_alloc(net, &sk->ns_tracker, GFP_KERNEL);
  439. sock_inuse_add(net, 1);
  440. }
  441. rtn = net_generic(net, rds_tcp_netid);
  442. if (rtn->sndbuf_size > 0) {
  443. sk->sk_sndbuf = rtn->sndbuf_size;
  444. sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
  445. }
  446. if (rtn->rcvbuf_size > 0) {
  447. sk->sk_rcvbuf = rtn->rcvbuf_size;
  448. sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
  449. }
  450. release_sock(sk);
  451. return true;
  452. }
  453. static void rds_tcp_accept_worker(struct work_struct *work)
  454. {
  455. struct rds_tcp_net *rtn = container_of(work,
  456. struct rds_tcp_net,
  457. rds_tcp_accept_w);
  458. while (rds_tcp_accept_one(rtn->rds_tcp_listen_sock) == 0)
  459. cond_resched();
  460. }
  461. void rds_tcp_accept_work(struct sock *sk)
  462. {
  463. struct net *net = sock_net(sk);
  464. struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
  465. queue_work(rds_wq, &rtn->rds_tcp_accept_w);
  466. }
  467. static __net_init int rds_tcp_init_net(struct net *net)
  468. {
  469. struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
  470. struct ctl_table *tbl;
  471. int err = 0;
  472. memset(rtn, 0, sizeof(*rtn));
  473. /* {snd, rcv}buf_size default to 0, which implies we let the
  474. * stack pick the value, and permit auto-tuning of buffer size.
  475. */
  476. if (net == &init_net) {
  477. tbl = rds_tcp_sysctl_table;
  478. } else {
  479. tbl = kmemdup(rds_tcp_sysctl_table,
  480. sizeof(rds_tcp_sysctl_table), GFP_KERNEL);
  481. if (!tbl) {
  482. pr_warn("could not set allocate sysctl table\n");
  483. return -ENOMEM;
  484. }
  485. rtn->ctl_table = tbl;
  486. }
  487. tbl[RDS_TCP_SNDBUF].data = &rtn->sndbuf_size;
  488. tbl[RDS_TCP_RCVBUF].data = &rtn->rcvbuf_size;
  489. rtn->rds_tcp_sysctl = register_net_sysctl(net, "net/rds/tcp", tbl);
  490. if (!rtn->rds_tcp_sysctl) {
  491. pr_warn("could not register sysctl\n");
  492. err = -ENOMEM;
  493. goto fail;
  494. }
  495. #if IS_ENABLED(CONFIG_IPV6)
  496. rtn->rds_tcp_listen_sock = rds_tcp_listen_init(net, true);
  497. #else
  498. rtn->rds_tcp_listen_sock = rds_tcp_listen_init(net, false);
  499. #endif
  500. if (!rtn->rds_tcp_listen_sock) {
  501. pr_warn("could not set up IPv6 listen sock\n");
  502. #if IS_ENABLED(CONFIG_IPV6)
  503. /* Try IPv4 as some systems disable IPv6 */
  504. rtn->rds_tcp_listen_sock = rds_tcp_listen_init(net, false);
  505. if (!rtn->rds_tcp_listen_sock) {
  506. #endif
  507. unregister_net_sysctl_table(rtn->rds_tcp_sysctl);
  508. rtn->rds_tcp_sysctl = NULL;
  509. err = -EAFNOSUPPORT;
  510. goto fail;
  511. #if IS_ENABLED(CONFIG_IPV6)
  512. }
  513. #endif
  514. }
  515. INIT_WORK(&rtn->rds_tcp_accept_w, rds_tcp_accept_worker);
  516. return 0;
  517. fail:
  518. if (net != &init_net)
  519. kfree(tbl);
  520. return err;
  521. }
  522. static void rds_tcp_kill_sock(struct net *net)
  523. {
  524. struct rds_tcp_connection *tc, *_tc;
  525. LIST_HEAD(tmp_list);
  526. struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
  527. struct socket *lsock = rtn->rds_tcp_listen_sock;
  528. rtn->rds_tcp_listen_sock = NULL;
  529. rds_tcp_listen_stop(lsock, &rtn->rds_tcp_accept_w);
  530. spin_lock_irq(&rds_tcp_conn_lock);
  531. list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) {
  532. struct net *c_net = read_pnet(&tc->t_cpath->cp_conn->c_net);
  533. if (net != c_net)
  534. continue;
  535. if (!list_has_conn(&tmp_list, tc->t_cpath->cp_conn)) {
  536. list_move_tail(&tc->t_tcp_node, &tmp_list);
  537. } else {
  538. list_del(&tc->t_tcp_node);
  539. tc->t_tcp_node_detached = true;
  540. }
  541. }
  542. spin_unlock_irq(&rds_tcp_conn_lock);
  543. list_for_each_entry_safe(tc, _tc, &tmp_list, t_tcp_node)
  544. rds_conn_destroy(tc->t_cpath->cp_conn);
  545. }
  546. static void __net_exit rds_tcp_exit_net(struct net *net)
  547. {
  548. struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
  549. rds_tcp_kill_sock(net);
  550. if (rtn->rds_tcp_sysctl)
  551. unregister_net_sysctl_table(rtn->rds_tcp_sysctl);
  552. if (net != &init_net)
  553. kfree(rtn->ctl_table);
  554. }
  555. static struct pernet_operations rds_tcp_net_ops = {
  556. .init = rds_tcp_init_net,
  557. .exit = rds_tcp_exit_net,
  558. .id = &rds_tcp_netid,
  559. .size = sizeof(struct rds_tcp_net),
  560. };
  561. void *rds_tcp_listen_sock_def_readable(struct net *net)
  562. {
  563. struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
  564. struct socket *lsock = rtn->rds_tcp_listen_sock;
  565. if (!lsock)
  566. return NULL;
  567. return lsock->sk->sk_user_data;
  568. }
  569. /* when sysctl is used to modify some kernel socket parameters,this
  570. * function resets the RDS connections in that netns so that we can
  571. * restart with new parameters. The assumption is that such reset
  572. * events are few and far-between.
  573. */
  574. static void rds_tcp_sysctl_reset(struct net *net)
  575. {
  576. struct rds_tcp_connection *tc, *_tc;
  577. spin_lock_irq(&rds_tcp_conn_lock);
  578. list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) {
  579. struct net *c_net = read_pnet(&tc->t_cpath->cp_conn->c_net);
  580. if (net != c_net || !tc->t_sock)
  581. continue;
  582. /* reconnect with new parameters */
  583. rds_conn_path_drop(tc->t_cpath, false);
  584. }
  585. spin_unlock_irq(&rds_tcp_conn_lock);
  586. }
  587. static int rds_tcp_skbuf_handler(struct ctl_table *ctl, int write,
  588. void *buffer, size_t *lenp, loff_t *fpos)
  589. {
  590. struct net *net = current->nsproxy->net_ns;
  591. int err;
  592. err = proc_dointvec_minmax(ctl, write, buffer, lenp, fpos);
  593. if (err < 0) {
  594. pr_warn("Invalid input. Must be >= %d\n",
  595. *(int *)(ctl->extra1));
  596. return err;
  597. }
  598. if (write)
  599. rds_tcp_sysctl_reset(net);
  600. return 0;
  601. }
  602. static void rds_tcp_exit(void)
  603. {
  604. rds_tcp_set_unloading();
  605. synchronize_rcu();
  606. rds_info_deregister_func(RDS_INFO_TCP_SOCKETS, rds_tcp_tc_info);
  607. #if IS_ENABLED(CONFIG_IPV6)
  608. rds_info_deregister_func(RDS6_INFO_TCP_SOCKETS, rds6_tcp_tc_info);
  609. #endif
  610. unregister_pernet_device(&rds_tcp_net_ops);
  611. rds_tcp_destroy_conns();
  612. rds_trans_unregister(&rds_tcp_transport);
  613. rds_tcp_recv_exit();
  614. kmem_cache_destroy(rds_tcp_conn_slab);
  615. }
  616. module_exit(rds_tcp_exit);
  617. static int __init rds_tcp_init(void)
  618. {
  619. int ret;
  620. rds_tcp_conn_slab = kmem_cache_create("rds_tcp_connection",
  621. sizeof(struct rds_tcp_connection),
  622. 0, 0, NULL);
  623. if (!rds_tcp_conn_slab) {
  624. ret = -ENOMEM;
  625. goto out;
  626. }
  627. ret = rds_tcp_recv_init();
  628. if (ret)
  629. goto out_slab;
  630. ret = register_pernet_device(&rds_tcp_net_ops);
  631. if (ret)
  632. goto out_recv;
  633. rds_trans_register(&rds_tcp_transport);
  634. rds_info_register_func(RDS_INFO_TCP_SOCKETS, rds_tcp_tc_info);
  635. #if IS_ENABLED(CONFIG_IPV6)
  636. rds_info_register_func(RDS6_INFO_TCP_SOCKETS, rds6_tcp_tc_info);
  637. #endif
  638. goto out;
  639. out_recv:
  640. rds_tcp_recv_exit();
  641. out_slab:
  642. kmem_cache_destroy(rds_tcp_conn_slab);
  643. out:
  644. return ret;
  645. }
  646. module_init(rds_tcp_init);
  647. MODULE_AUTHOR("Oracle Corporation <[email protected]>");
  648. MODULE_DESCRIPTION("RDS: TCP transport");
  649. MODULE_LICENSE("Dual BSD/GPL");