rdma.c 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958
  1. /*
  2. * Copyright (c) 2007, 2020 Oracle and/or its affiliates.
  3. *
  4. * This software is available to you under a choice of one of two
  5. * licenses. You may choose to be licensed under the terms of the GNU
  6. * General Public License (GPL) Version 2, available from the file
  7. * COPYING in the main directory of this source tree, or the
  8. * OpenIB.org BSD license below:
  9. *
  10. * Redistribution and use in source and binary forms, with or
  11. * without modification, are permitted provided that the following
  12. * conditions are met:
  13. *
  14. * - Redistributions of source code must retain the above
  15. * copyright notice, this list of conditions and the following
  16. * disclaimer.
  17. *
  18. * - Redistributions in binary form must reproduce the above
  19. * copyright notice, this list of conditions and the following
  20. * disclaimer in the documentation and/or other materials
  21. * provided with the distribution.
  22. *
  23. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30. * SOFTWARE.
  31. *
  32. */
  33. #include <linux/pagemap.h>
  34. #include <linux/slab.h>
  35. #include <linux/rbtree.h>
  36. #include <linux/dma-mapping.h> /* for DMA_*_DEVICE */
  37. #include "rds.h"
  38. /*
  39. * XXX
  40. * - build with sparse
  41. * - should we detect duplicate keys on a socket? hmm.
  42. * - an rdma is an mlock, apply rlimit?
  43. */
  44. /*
  45. * get the number of pages by looking at the page indices that the start and
  46. * end addresses fall in.
  47. *
  48. * Returns 0 if the vec is invalid. It is invalid if the number of bytes
  49. * causes the address to wrap or overflows an unsigned int. This comes
  50. * from being stored in the 'length' member of 'struct scatterlist'.
  51. */
  52. static unsigned int rds_pages_in_vec(struct rds_iovec *vec)
  53. {
  54. if ((vec->addr + vec->bytes <= vec->addr) ||
  55. (vec->bytes > (u64)UINT_MAX))
  56. return 0;
  57. return ((vec->addr + vec->bytes + PAGE_SIZE - 1) >> PAGE_SHIFT) -
  58. (vec->addr >> PAGE_SHIFT);
  59. }
  60. static struct rds_mr *rds_mr_tree_walk(struct rb_root *root, u64 key,
  61. struct rds_mr *insert)
  62. {
  63. struct rb_node **p = &root->rb_node;
  64. struct rb_node *parent = NULL;
  65. struct rds_mr *mr;
  66. while (*p) {
  67. parent = *p;
  68. mr = rb_entry(parent, struct rds_mr, r_rb_node);
  69. if (key < mr->r_key)
  70. p = &(*p)->rb_left;
  71. else if (key > mr->r_key)
  72. p = &(*p)->rb_right;
  73. else
  74. return mr;
  75. }
  76. if (insert) {
  77. rb_link_node(&insert->r_rb_node, parent, p);
  78. rb_insert_color(&insert->r_rb_node, root);
  79. kref_get(&insert->r_kref);
  80. }
  81. return NULL;
  82. }
  83. /*
  84. * Destroy the transport-specific part of a MR.
  85. */
  86. static void rds_destroy_mr(struct rds_mr *mr)
  87. {
  88. struct rds_sock *rs = mr->r_sock;
  89. void *trans_private = NULL;
  90. unsigned long flags;
  91. rdsdebug("RDS: destroy mr key is %x refcnt %u\n",
  92. mr->r_key, kref_read(&mr->r_kref));
  93. spin_lock_irqsave(&rs->rs_rdma_lock, flags);
  94. if (!RB_EMPTY_NODE(&mr->r_rb_node))
  95. rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);
  96. trans_private = mr->r_trans_private;
  97. mr->r_trans_private = NULL;
  98. spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);
  99. if (trans_private)
  100. mr->r_trans->free_mr(trans_private, mr->r_invalidate);
  101. }
  102. void __rds_put_mr_final(struct kref *kref)
  103. {
  104. struct rds_mr *mr = container_of(kref, struct rds_mr, r_kref);
  105. rds_destroy_mr(mr);
  106. kfree(mr);
  107. }
  108. /*
  109. * By the time this is called we can't have any more ioctls called on
  110. * the socket so we don't need to worry about racing with others.
  111. */
  112. void rds_rdma_drop_keys(struct rds_sock *rs)
  113. {
  114. struct rds_mr *mr;
  115. struct rb_node *node;
  116. unsigned long flags;
  117. /* Release any MRs associated with this socket */
  118. spin_lock_irqsave(&rs->rs_rdma_lock, flags);
  119. while ((node = rb_first(&rs->rs_rdma_keys))) {
  120. mr = rb_entry(node, struct rds_mr, r_rb_node);
  121. if (mr->r_trans == rs->rs_transport)
  122. mr->r_invalidate = 0;
  123. rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);
  124. RB_CLEAR_NODE(&mr->r_rb_node);
  125. spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);
  126. kref_put(&mr->r_kref, __rds_put_mr_final);
  127. spin_lock_irqsave(&rs->rs_rdma_lock, flags);
  128. }
  129. spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);
  130. if (rs->rs_transport && rs->rs_transport->flush_mrs)
  131. rs->rs_transport->flush_mrs();
  132. }
  133. /*
  134. * Helper function to pin user pages.
  135. */
  136. static int rds_pin_pages(unsigned long user_addr, unsigned int nr_pages,
  137. struct page **pages, int write)
  138. {
  139. unsigned int gup_flags = FOLL_LONGTERM;
  140. int ret;
  141. if (write)
  142. gup_flags |= FOLL_WRITE;
  143. ret = pin_user_pages_fast(user_addr, nr_pages, gup_flags, pages);
  144. if (ret >= 0 && ret < nr_pages) {
  145. unpin_user_pages(pages, ret);
  146. ret = -EFAULT;
  147. }
  148. return ret;
  149. }
  150. static int __rds_rdma_map(struct rds_sock *rs, struct rds_get_mr_args *args,
  151. u64 *cookie_ret, struct rds_mr **mr_ret,
  152. struct rds_conn_path *cp)
  153. {
  154. struct rds_mr *mr = NULL, *found;
  155. struct scatterlist *sg = NULL;
  156. unsigned int nr_pages;
  157. struct page **pages = NULL;
  158. void *trans_private;
  159. unsigned long flags;
  160. rds_rdma_cookie_t cookie;
  161. unsigned int nents = 0;
  162. int need_odp = 0;
  163. long i;
  164. int ret;
  165. if (ipv6_addr_any(&rs->rs_bound_addr) || !rs->rs_transport) {
  166. ret = -ENOTCONN; /* XXX not a great errno */
  167. goto out;
  168. }
  169. if (!rs->rs_transport->get_mr) {
  170. ret = -EOPNOTSUPP;
  171. goto out;
  172. }
  173. /* If the combination of the addr and size requested for this memory
  174. * region causes an integer overflow, return error.
  175. */
  176. if (((args->vec.addr + args->vec.bytes) < args->vec.addr) ||
  177. PAGE_ALIGN(args->vec.addr + args->vec.bytes) <
  178. (args->vec.addr + args->vec.bytes)) {
  179. ret = -EINVAL;
  180. goto out;
  181. }
  182. if (!can_do_mlock()) {
  183. ret = -EPERM;
  184. goto out;
  185. }
  186. nr_pages = rds_pages_in_vec(&args->vec);
  187. if (nr_pages == 0) {
  188. ret = -EINVAL;
  189. goto out;
  190. }
  191. /* Restrict the size of mr irrespective of underlying transport
  192. * To account for unaligned mr regions, subtract one from nr_pages
  193. */
  194. if ((nr_pages - 1) > (RDS_MAX_MSG_SIZE >> PAGE_SHIFT)) {
  195. ret = -EMSGSIZE;
  196. goto out;
  197. }
  198. rdsdebug("RDS: get_mr addr %llx len %llu nr_pages %u\n",
  199. args->vec.addr, args->vec.bytes, nr_pages);
  200. /* XXX clamp nr_pages to limit the size of this alloc? */
  201. pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);
  202. if (!pages) {
  203. ret = -ENOMEM;
  204. goto out;
  205. }
  206. mr = kzalloc(sizeof(struct rds_mr), GFP_KERNEL);
  207. if (!mr) {
  208. ret = -ENOMEM;
  209. goto out;
  210. }
  211. kref_init(&mr->r_kref);
  212. RB_CLEAR_NODE(&mr->r_rb_node);
  213. mr->r_trans = rs->rs_transport;
  214. mr->r_sock = rs;
  215. if (args->flags & RDS_RDMA_USE_ONCE)
  216. mr->r_use_once = 1;
  217. if (args->flags & RDS_RDMA_INVALIDATE)
  218. mr->r_invalidate = 1;
  219. if (args->flags & RDS_RDMA_READWRITE)
  220. mr->r_write = 1;
  221. /*
  222. * Pin the pages that make up the user buffer and transfer the page
  223. * pointers to the mr's sg array. We check to see if we've mapped
  224. * the whole region after transferring the partial page references
  225. * to the sg array so that we can have one page ref cleanup path.
  226. *
  227. * For now we have no flag that tells us whether the mapping is
  228. * r/o or r/w. We need to assume r/w, or we'll do a lot of RDMA to
  229. * the zero page.
  230. */
  231. ret = rds_pin_pages(args->vec.addr, nr_pages, pages, 1);
  232. if (ret == -EOPNOTSUPP) {
  233. need_odp = 1;
  234. } else if (ret <= 0) {
  235. goto out;
  236. } else {
  237. nents = ret;
  238. sg = kmalloc_array(nents, sizeof(*sg), GFP_KERNEL);
  239. if (!sg) {
  240. ret = -ENOMEM;
  241. goto out;
  242. }
  243. WARN_ON(!nents);
  244. sg_init_table(sg, nents);
  245. /* Stick all pages into the scatterlist */
  246. for (i = 0 ; i < nents; i++)
  247. sg_set_page(&sg[i], pages[i], PAGE_SIZE, 0);
  248. rdsdebug("RDS: trans_private nents is %u\n", nents);
  249. }
  250. /* Obtain a transport specific MR. If this succeeds, the
  251. * s/g list is now owned by the MR.
  252. * Note that dma_map() implies that pending writes are
  253. * flushed to RAM, so no dma_sync is needed here. */
  254. trans_private = rs->rs_transport->get_mr(
  255. sg, nents, rs, &mr->r_key, cp ? cp->cp_conn : NULL,
  256. args->vec.addr, args->vec.bytes,
  257. need_odp ? ODP_ZEROBASED : ODP_NOT_NEEDED);
  258. if (IS_ERR(trans_private)) {
  259. /* In ODP case, we don't GUP pages, so don't need
  260. * to release anything.
  261. */
  262. if (!need_odp) {
  263. unpin_user_pages(pages, nr_pages);
  264. kfree(sg);
  265. }
  266. ret = PTR_ERR(trans_private);
  267. goto out;
  268. }
  269. mr->r_trans_private = trans_private;
  270. rdsdebug("RDS: get_mr put_user key is %x cookie_addr %p\n",
  271. mr->r_key, (void *)(unsigned long) args->cookie_addr);
  272. /* The user may pass us an unaligned address, but we can only
  273. * map page aligned regions. So we keep the offset, and build
  274. * a 64bit cookie containing <R_Key, offset> and pass that
  275. * around. */
  276. if (need_odp)
  277. cookie = rds_rdma_make_cookie(mr->r_key, 0);
  278. else
  279. cookie = rds_rdma_make_cookie(mr->r_key,
  280. args->vec.addr & ~PAGE_MASK);
  281. if (cookie_ret)
  282. *cookie_ret = cookie;
  283. if (args->cookie_addr &&
  284. put_user(cookie, (u64 __user *)(unsigned long)args->cookie_addr)) {
  285. if (!need_odp) {
  286. unpin_user_pages(pages, nr_pages);
  287. kfree(sg);
  288. }
  289. ret = -EFAULT;
  290. goto out;
  291. }
  292. /* Inserting the new MR into the rbtree bumps its
  293. * reference count. */
  294. spin_lock_irqsave(&rs->rs_rdma_lock, flags);
  295. found = rds_mr_tree_walk(&rs->rs_rdma_keys, mr->r_key, mr);
  296. spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);
  297. BUG_ON(found && found != mr);
  298. rdsdebug("RDS: get_mr key is %x\n", mr->r_key);
  299. if (mr_ret) {
  300. kref_get(&mr->r_kref);
  301. *mr_ret = mr;
  302. }
  303. ret = 0;
  304. out:
  305. kfree(pages);
  306. if (mr)
  307. kref_put(&mr->r_kref, __rds_put_mr_final);
  308. return ret;
  309. }
  310. int rds_get_mr(struct rds_sock *rs, sockptr_t optval, int optlen)
  311. {
  312. struct rds_get_mr_args args;
  313. if (optlen != sizeof(struct rds_get_mr_args))
  314. return -EINVAL;
  315. if (copy_from_sockptr(&args, optval, sizeof(struct rds_get_mr_args)))
  316. return -EFAULT;
  317. return __rds_rdma_map(rs, &args, NULL, NULL, NULL);
  318. }
  319. int rds_get_mr_for_dest(struct rds_sock *rs, sockptr_t optval, int optlen)
  320. {
  321. struct rds_get_mr_for_dest_args args;
  322. struct rds_get_mr_args new_args;
  323. if (optlen != sizeof(struct rds_get_mr_for_dest_args))
  324. return -EINVAL;
  325. if (copy_from_sockptr(&args, optval,
  326. sizeof(struct rds_get_mr_for_dest_args)))
  327. return -EFAULT;
  328. /*
  329. * Initially, just behave like get_mr().
  330. * TODO: Implement get_mr as wrapper around this
  331. * and deprecate it.
  332. */
  333. new_args.vec = args.vec;
  334. new_args.cookie_addr = args.cookie_addr;
  335. new_args.flags = args.flags;
  336. return __rds_rdma_map(rs, &new_args, NULL, NULL, NULL);
  337. }
  338. /*
  339. * Free the MR indicated by the given R_Key
  340. */
  341. int rds_free_mr(struct rds_sock *rs, sockptr_t optval, int optlen)
  342. {
  343. struct rds_free_mr_args args;
  344. struct rds_mr *mr;
  345. unsigned long flags;
  346. if (optlen != sizeof(struct rds_free_mr_args))
  347. return -EINVAL;
  348. if (copy_from_sockptr(&args, optval, sizeof(struct rds_free_mr_args)))
  349. return -EFAULT;
  350. /* Special case - a null cookie means flush all unused MRs */
  351. if (args.cookie == 0) {
  352. if (!rs->rs_transport || !rs->rs_transport->flush_mrs)
  353. return -EINVAL;
  354. rs->rs_transport->flush_mrs();
  355. return 0;
  356. }
  357. /* Look up the MR given its R_key and remove it from the rbtree
  358. * so nobody else finds it.
  359. * This should also prevent races with rds_rdma_unuse.
  360. */
  361. spin_lock_irqsave(&rs->rs_rdma_lock, flags);
  362. mr = rds_mr_tree_walk(&rs->rs_rdma_keys, rds_rdma_cookie_key(args.cookie), NULL);
  363. if (mr) {
  364. rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);
  365. RB_CLEAR_NODE(&mr->r_rb_node);
  366. if (args.flags & RDS_RDMA_INVALIDATE)
  367. mr->r_invalidate = 1;
  368. }
  369. spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);
  370. if (!mr)
  371. return -EINVAL;
  372. kref_put(&mr->r_kref, __rds_put_mr_final);
  373. return 0;
  374. }
  375. /*
  376. * This is called when we receive an extension header that
  377. * tells us this MR was used. It allows us to implement
  378. * use_once semantics
  379. */
  380. void rds_rdma_unuse(struct rds_sock *rs, u32 r_key, int force)
  381. {
  382. struct rds_mr *mr;
  383. unsigned long flags;
  384. int zot_me = 0;
  385. spin_lock_irqsave(&rs->rs_rdma_lock, flags);
  386. mr = rds_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL);
  387. if (!mr) {
  388. pr_debug("rds: trying to unuse MR with unknown r_key %u!\n",
  389. r_key);
  390. spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);
  391. return;
  392. }
  393. /* Get a reference so that the MR won't go away before calling
  394. * sync_mr() below.
  395. */
  396. kref_get(&mr->r_kref);
  397. /* If it is going to be freed, remove it from the tree now so
  398. * that no other thread can find it and free it.
  399. */
  400. if (mr->r_use_once || force) {
  401. rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);
  402. RB_CLEAR_NODE(&mr->r_rb_node);
  403. zot_me = 1;
  404. }
  405. spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);
  406. /* May have to issue a dma_sync on this memory region.
  407. * Note we could avoid this if the operation was a RDMA READ,
  408. * but at this point we can't tell. */
  409. if (mr->r_trans->sync_mr)
  410. mr->r_trans->sync_mr(mr->r_trans_private, DMA_FROM_DEVICE);
  411. /* Release the reference held above. */
  412. kref_put(&mr->r_kref, __rds_put_mr_final);
  413. /* If the MR was marked as invalidate, this will
  414. * trigger an async flush. */
  415. if (zot_me)
  416. kref_put(&mr->r_kref, __rds_put_mr_final);
  417. }
  418. void rds_rdma_free_op(struct rm_rdma_op *ro)
  419. {
  420. unsigned int i;
  421. if (ro->op_odp_mr) {
  422. kref_put(&ro->op_odp_mr->r_kref, __rds_put_mr_final);
  423. } else {
  424. for (i = 0; i < ro->op_nents; i++) {
  425. struct page *page = sg_page(&ro->op_sg[i]);
  426. /* Mark page dirty if it was possibly modified, which
  427. * is the case for a RDMA_READ which copies from remote
  428. * to local memory
  429. */
  430. unpin_user_pages_dirty_lock(&page, 1, !ro->op_write);
  431. }
  432. }
  433. kfree(ro->op_notifier);
  434. ro->op_notifier = NULL;
  435. ro->op_active = 0;
  436. ro->op_odp_mr = NULL;
  437. }
  438. void rds_atomic_free_op(struct rm_atomic_op *ao)
  439. {
  440. struct page *page = sg_page(ao->op_sg);
  441. /* Mark page dirty if it was possibly modified, which
  442. * is the case for a RDMA_READ which copies from remote
  443. * to local memory */
  444. unpin_user_pages_dirty_lock(&page, 1, true);
  445. kfree(ao->op_notifier);
  446. ao->op_notifier = NULL;
  447. ao->op_active = 0;
  448. }
  449. /*
  450. * Count the number of pages needed to describe an incoming iovec array.
  451. */
  452. static int rds_rdma_pages(struct rds_iovec iov[], int nr_iovecs)
  453. {
  454. int tot_pages = 0;
  455. unsigned int nr_pages;
  456. unsigned int i;
  457. /* figure out the number of pages in the vector */
  458. for (i = 0; i < nr_iovecs; i++) {
  459. nr_pages = rds_pages_in_vec(&iov[i]);
  460. if (nr_pages == 0)
  461. return -EINVAL;
  462. tot_pages += nr_pages;
  463. /*
  464. * nr_pages for one entry is limited to (UINT_MAX>>PAGE_SHIFT)+1,
  465. * so tot_pages cannot overflow without first going negative.
  466. */
  467. if (tot_pages < 0)
  468. return -EINVAL;
  469. }
  470. return tot_pages;
  471. }
  472. int rds_rdma_extra_size(struct rds_rdma_args *args,
  473. struct rds_iov_vector *iov)
  474. {
  475. struct rds_iovec *vec;
  476. struct rds_iovec __user *local_vec;
  477. int tot_pages = 0;
  478. unsigned int nr_pages;
  479. unsigned int i;
  480. local_vec = (struct rds_iovec __user *)(unsigned long) args->local_vec_addr;
  481. if (args->nr_local == 0)
  482. return -EINVAL;
  483. if (args->nr_local > UIO_MAXIOV)
  484. return -EMSGSIZE;
  485. iov->iov = kcalloc(args->nr_local,
  486. sizeof(struct rds_iovec),
  487. GFP_KERNEL);
  488. if (!iov->iov)
  489. return -ENOMEM;
  490. vec = &iov->iov[0];
  491. if (copy_from_user(vec, local_vec, args->nr_local *
  492. sizeof(struct rds_iovec)))
  493. return -EFAULT;
  494. iov->len = args->nr_local;
  495. /* figure out the number of pages in the vector */
  496. for (i = 0; i < args->nr_local; i++, vec++) {
  497. nr_pages = rds_pages_in_vec(vec);
  498. if (nr_pages == 0)
  499. return -EINVAL;
  500. tot_pages += nr_pages;
  501. /*
  502. * nr_pages for one entry is limited to (UINT_MAX>>PAGE_SHIFT)+1,
  503. * so tot_pages cannot overflow without first going negative.
  504. */
  505. if (tot_pages < 0)
  506. return -EINVAL;
  507. }
  508. return tot_pages * sizeof(struct scatterlist);
  509. }
  510. /*
  511. * The application asks for a RDMA transfer.
  512. * Extract all arguments and set up the rdma_op
  513. */
  514. int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,
  515. struct cmsghdr *cmsg,
  516. struct rds_iov_vector *vec)
  517. {
  518. struct rds_rdma_args *args;
  519. struct rm_rdma_op *op = &rm->rdma;
  520. int nr_pages;
  521. unsigned int nr_bytes;
  522. struct page **pages = NULL;
  523. struct rds_iovec *iovs;
  524. unsigned int i, j;
  525. int ret = 0;
  526. bool odp_supported = true;
  527. if (cmsg->cmsg_len < CMSG_LEN(sizeof(struct rds_rdma_args))
  528. || rm->rdma.op_active)
  529. return -EINVAL;
  530. args = CMSG_DATA(cmsg);
  531. if (ipv6_addr_any(&rs->rs_bound_addr)) {
  532. ret = -ENOTCONN; /* XXX not a great errno */
  533. goto out_ret;
  534. }
  535. if (args->nr_local > UIO_MAXIOV) {
  536. ret = -EMSGSIZE;
  537. goto out_ret;
  538. }
  539. if (vec->len != args->nr_local) {
  540. ret = -EINVAL;
  541. goto out_ret;
  542. }
  543. /* odp-mr is not supported for multiple requests within one message */
  544. if (args->nr_local != 1)
  545. odp_supported = false;
  546. iovs = vec->iov;
  547. nr_pages = rds_rdma_pages(iovs, args->nr_local);
  548. if (nr_pages < 0) {
  549. ret = -EINVAL;
  550. goto out_ret;
  551. }
  552. pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);
  553. if (!pages) {
  554. ret = -ENOMEM;
  555. goto out_ret;
  556. }
  557. op->op_write = !!(args->flags & RDS_RDMA_READWRITE);
  558. op->op_fence = !!(args->flags & RDS_RDMA_FENCE);
  559. op->op_notify = !!(args->flags & RDS_RDMA_NOTIFY_ME);
  560. op->op_silent = !!(args->flags & RDS_RDMA_SILENT);
  561. op->op_active = 1;
  562. op->op_recverr = rs->rs_recverr;
  563. op->op_odp_mr = NULL;
  564. WARN_ON(!nr_pages);
  565. op->op_sg = rds_message_alloc_sgs(rm, nr_pages);
  566. if (IS_ERR(op->op_sg)) {
  567. ret = PTR_ERR(op->op_sg);
  568. goto out_pages;
  569. }
  570. if (op->op_notify || op->op_recverr) {
  571. /* We allocate an uninitialized notifier here, because
  572. * we don't want to do that in the completion handler. We
  573. * would have to use GFP_ATOMIC there, and don't want to deal
  574. * with failed allocations.
  575. */
  576. op->op_notifier = kmalloc(sizeof(struct rds_notifier), GFP_KERNEL);
  577. if (!op->op_notifier) {
  578. ret = -ENOMEM;
  579. goto out_pages;
  580. }
  581. op->op_notifier->n_user_token = args->user_token;
  582. op->op_notifier->n_status = RDS_RDMA_SUCCESS;
  583. }
  584. /* The cookie contains the R_Key of the remote memory region, and
  585. * optionally an offset into it. This is how we implement RDMA into
  586. * unaligned memory.
  587. * When setting up the RDMA, we need to add that offset to the
  588. * destination address (which is really an offset into the MR)
  589. * FIXME: We may want to move this into ib_rdma.c
  590. */
  591. op->op_rkey = rds_rdma_cookie_key(args->cookie);
  592. op->op_remote_addr = args->remote_vec.addr + rds_rdma_cookie_offset(args->cookie);
  593. nr_bytes = 0;
  594. rdsdebug("RDS: rdma prepare nr_local %llu rva %llx rkey %x\n",
  595. (unsigned long long)args->nr_local,
  596. (unsigned long long)args->remote_vec.addr,
  597. op->op_rkey);
  598. for (i = 0; i < args->nr_local; i++) {
  599. struct rds_iovec *iov = &iovs[i];
  600. /* don't need to check, rds_rdma_pages() verified nr will be +nonzero */
  601. unsigned int nr = rds_pages_in_vec(iov);
  602. rs->rs_user_addr = iov->addr;
  603. rs->rs_user_bytes = iov->bytes;
  604. /* If it's a WRITE operation, we want to pin the pages for reading.
  605. * If it's a READ operation, we need to pin the pages for writing.
  606. */
  607. ret = rds_pin_pages(iov->addr, nr, pages, !op->op_write);
  608. if ((!odp_supported && ret <= 0) ||
  609. (odp_supported && ret <= 0 && ret != -EOPNOTSUPP))
  610. goto out_pages;
  611. if (ret == -EOPNOTSUPP) {
  612. struct rds_mr *local_odp_mr;
  613. if (!rs->rs_transport->get_mr) {
  614. ret = -EOPNOTSUPP;
  615. goto out_pages;
  616. }
  617. local_odp_mr =
  618. kzalloc(sizeof(*local_odp_mr), GFP_KERNEL);
  619. if (!local_odp_mr) {
  620. ret = -ENOMEM;
  621. goto out_pages;
  622. }
  623. RB_CLEAR_NODE(&local_odp_mr->r_rb_node);
  624. kref_init(&local_odp_mr->r_kref);
  625. local_odp_mr->r_trans = rs->rs_transport;
  626. local_odp_mr->r_sock = rs;
  627. local_odp_mr->r_trans_private =
  628. rs->rs_transport->get_mr(
  629. NULL, 0, rs, &local_odp_mr->r_key, NULL,
  630. iov->addr, iov->bytes, ODP_VIRTUAL);
  631. if (IS_ERR(local_odp_mr->r_trans_private)) {
  632. ret = PTR_ERR(local_odp_mr->r_trans_private);
  633. rdsdebug("get_mr ret %d %p\"", ret,
  634. local_odp_mr->r_trans_private);
  635. kfree(local_odp_mr);
  636. ret = -EOPNOTSUPP;
  637. goto out_pages;
  638. }
  639. rdsdebug("Need odp; local_odp_mr %p trans_private %p\n",
  640. local_odp_mr, local_odp_mr->r_trans_private);
  641. op->op_odp_mr = local_odp_mr;
  642. op->op_odp_addr = iov->addr;
  643. }
  644. rdsdebug("RDS: nr_bytes %u nr %u iov->bytes %llu iov->addr %llx\n",
  645. nr_bytes, nr, iov->bytes, iov->addr);
  646. nr_bytes += iov->bytes;
  647. for (j = 0; j < nr; j++) {
  648. unsigned int offset = iov->addr & ~PAGE_MASK;
  649. struct scatterlist *sg;
  650. sg = &op->op_sg[op->op_nents + j];
  651. sg_set_page(sg, pages[j],
  652. min_t(unsigned int, iov->bytes, PAGE_SIZE - offset),
  653. offset);
  654. sg_dma_len(sg) = sg->length;
  655. rdsdebug("RDS: sg->offset %x sg->len %x iov->addr %llx iov->bytes %llu\n",
  656. sg->offset, sg->length, iov->addr, iov->bytes);
  657. iov->addr += sg->length;
  658. iov->bytes -= sg->length;
  659. }
  660. op->op_nents += nr;
  661. }
  662. if (nr_bytes > args->remote_vec.bytes) {
  663. rdsdebug("RDS nr_bytes %u remote_bytes %u do not match\n",
  664. nr_bytes,
  665. (unsigned int) args->remote_vec.bytes);
  666. ret = -EINVAL;
  667. goto out_pages;
  668. }
  669. op->op_bytes = nr_bytes;
  670. ret = 0;
  671. out_pages:
  672. kfree(pages);
  673. out_ret:
  674. if (ret)
  675. rds_rdma_free_op(op);
  676. else
  677. rds_stats_inc(s_send_rdma);
  678. return ret;
  679. }
  680. /*
  681. * The application wants us to pass an RDMA destination (aka MR)
  682. * to the remote
  683. */
  684. int rds_cmsg_rdma_dest(struct rds_sock *rs, struct rds_message *rm,
  685. struct cmsghdr *cmsg)
  686. {
  687. unsigned long flags;
  688. struct rds_mr *mr;
  689. u32 r_key;
  690. int err = 0;
  691. if (cmsg->cmsg_len < CMSG_LEN(sizeof(rds_rdma_cookie_t)) ||
  692. rm->m_rdma_cookie != 0)
  693. return -EINVAL;
  694. memcpy(&rm->m_rdma_cookie, CMSG_DATA(cmsg), sizeof(rm->m_rdma_cookie));
  695. /* We are reusing a previously mapped MR here. Most likely, the
  696. * application has written to the buffer, so we need to explicitly
  697. * flush those writes to RAM. Otherwise the HCA may not see them
  698. * when doing a DMA from that buffer.
  699. */
  700. r_key = rds_rdma_cookie_key(rm->m_rdma_cookie);
  701. spin_lock_irqsave(&rs->rs_rdma_lock, flags);
  702. mr = rds_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL);
  703. if (!mr)
  704. err = -EINVAL; /* invalid r_key */
  705. else
  706. kref_get(&mr->r_kref);
  707. spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);
  708. if (mr) {
  709. mr->r_trans->sync_mr(mr->r_trans_private,
  710. DMA_TO_DEVICE);
  711. rm->rdma.op_rdma_mr = mr;
  712. }
  713. return err;
  714. }
  715. /*
  716. * The application passes us an address range it wants to enable RDMA
  717. * to/from. We map the area, and save the <R_Key,offset> pair
  718. * in rm->m_rdma_cookie. This causes it to be sent along to the peer
  719. * in an extension header.
  720. */
  721. int rds_cmsg_rdma_map(struct rds_sock *rs, struct rds_message *rm,
  722. struct cmsghdr *cmsg)
  723. {
  724. if (cmsg->cmsg_len < CMSG_LEN(sizeof(struct rds_get_mr_args)) ||
  725. rm->m_rdma_cookie != 0)
  726. return -EINVAL;
  727. return __rds_rdma_map(rs, CMSG_DATA(cmsg), &rm->m_rdma_cookie,
  728. &rm->rdma.op_rdma_mr, rm->m_conn_path);
  729. }
  730. /*
  731. * Fill in rds_message for an atomic request.
  732. */
  733. int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,
  734. struct cmsghdr *cmsg)
  735. {
  736. struct page *page = NULL;
  737. struct rds_atomic_args *args;
  738. int ret = 0;
  739. if (cmsg->cmsg_len < CMSG_LEN(sizeof(struct rds_atomic_args))
  740. || rm->atomic.op_active)
  741. return -EINVAL;
  742. args = CMSG_DATA(cmsg);
  743. /* Nonmasked & masked cmsg ops converted to masked hw ops */
  744. switch (cmsg->cmsg_type) {
  745. case RDS_CMSG_ATOMIC_FADD:
  746. rm->atomic.op_type = RDS_ATOMIC_TYPE_FADD;
  747. rm->atomic.op_m_fadd.add = args->fadd.add;
  748. rm->atomic.op_m_fadd.nocarry_mask = 0;
  749. break;
  750. case RDS_CMSG_MASKED_ATOMIC_FADD:
  751. rm->atomic.op_type = RDS_ATOMIC_TYPE_FADD;
  752. rm->atomic.op_m_fadd.add = args->m_fadd.add;
  753. rm->atomic.op_m_fadd.nocarry_mask = args->m_fadd.nocarry_mask;
  754. break;
  755. case RDS_CMSG_ATOMIC_CSWP:
  756. rm->atomic.op_type = RDS_ATOMIC_TYPE_CSWP;
  757. rm->atomic.op_m_cswp.compare = args->cswp.compare;
  758. rm->atomic.op_m_cswp.swap = args->cswp.swap;
  759. rm->atomic.op_m_cswp.compare_mask = ~0;
  760. rm->atomic.op_m_cswp.swap_mask = ~0;
  761. break;
  762. case RDS_CMSG_MASKED_ATOMIC_CSWP:
  763. rm->atomic.op_type = RDS_ATOMIC_TYPE_CSWP;
  764. rm->atomic.op_m_cswp.compare = args->m_cswp.compare;
  765. rm->atomic.op_m_cswp.swap = args->m_cswp.swap;
  766. rm->atomic.op_m_cswp.compare_mask = args->m_cswp.compare_mask;
  767. rm->atomic.op_m_cswp.swap_mask = args->m_cswp.swap_mask;
  768. break;
  769. default:
  770. BUG(); /* should never happen */
  771. }
  772. rm->atomic.op_notify = !!(args->flags & RDS_RDMA_NOTIFY_ME);
  773. rm->atomic.op_silent = !!(args->flags & RDS_RDMA_SILENT);
  774. rm->atomic.op_active = 1;
  775. rm->atomic.op_recverr = rs->rs_recverr;
  776. rm->atomic.op_sg = rds_message_alloc_sgs(rm, 1);
  777. if (IS_ERR(rm->atomic.op_sg)) {
  778. ret = PTR_ERR(rm->atomic.op_sg);
  779. goto err;
  780. }
  781. /* verify 8 byte-aligned */
  782. if (args->local_addr & 0x7) {
  783. ret = -EFAULT;
  784. goto err;
  785. }
  786. ret = rds_pin_pages(args->local_addr, 1, &page, 1);
  787. if (ret != 1)
  788. goto err;
  789. ret = 0;
  790. sg_set_page(rm->atomic.op_sg, page, 8, offset_in_page(args->local_addr));
  791. if (rm->atomic.op_notify || rm->atomic.op_recverr) {
  792. /* We allocate an uninitialized notifier here, because
  793. * we don't want to do that in the completion handler. We
  794. * would have to use GFP_ATOMIC there, and don't want to deal
  795. * with failed allocations.
  796. */
  797. rm->atomic.op_notifier = kmalloc(sizeof(*rm->atomic.op_notifier), GFP_KERNEL);
  798. if (!rm->atomic.op_notifier) {
  799. ret = -ENOMEM;
  800. goto err;
  801. }
  802. rm->atomic.op_notifier->n_user_token = args->user_token;
  803. rm->atomic.op_notifier->n_status = RDS_RDMA_SUCCESS;
  804. }
  805. rm->atomic.op_rkey = rds_rdma_cookie_key(args->cookie);
  806. rm->atomic.op_remote_addr = args->remote_addr + rds_rdma_cookie_offset(args->cookie);
  807. return ret;
  808. err:
  809. if (page)
  810. unpin_user_page(page);
  811. rm->atomic.op_active = 0;
  812. kfree(rm->atomic.op_notifier);
  813. return ret;
  814. }