mr.c 61 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430
  1. /*
  2. * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
  3. *
  4. * This software is available to you under a choice of one of two
  5. * licenses. You may choose to be licensed under the terms of the GNU
  6. * General Public License (GPL) Version 2, available from the file
  7. * COPYING in the main directory of this source tree, or the
  8. * OpenIB.org BSD license below:
  9. *
  10. * Redistribution and use in source and binary forms, with or
  11. * without modification, are permitted provided that the following
  12. * conditions are met:
  13. *
  14. * - Redistributions of source code must retain the above
  15. * copyright notice, this list of conditions and the following
  16. * disclaimer.
  17. *
  18. * - Redistributions in binary form must reproduce the above
  19. * copyright notice, this list of conditions and the following
  20. * disclaimer in the documentation and/or other materials
  21. * provided with the distribution.
  22. *
  23. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30. * SOFTWARE.
  31. */
  32. #include <linux/kref.h>
  33. #include <linux/random.h>
  34. #include <linux/debugfs.h>
  35. #include <linux/export.h>
  36. #include <linux/delay.h>
  37. #include <rdma/ib_umem.h>
  38. #include <rdma/ib_umem_odp.h>
  39. #include <rdma/ib_verbs.h>
  40. #include "mlx5_ib.h"
  41. enum {
  42. MAX_PENDING_REG_MR = 8,
  43. };
  44. #define MLX5_UMR_ALIGN 2048
  45. static void
  46. create_mkey_callback(int status, struct mlx5_async_work *context);
  47. static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr,
  48. struct ib_pd *pd)
  49. {
  50. struct mlx5_ib_dev *dev = to_mdev(pd->device);
  51. MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC));
  52. MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE));
  53. MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ));
  54. MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE));
  55. MLX5_SET(mkc, mkc, lr, 1);
  56. if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write))
  57. MLX5_SET(mkc, mkc, relaxed_ordering_write,
  58. !!(acc & IB_ACCESS_RELAXED_ORDERING));
  59. if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read))
  60. MLX5_SET(mkc, mkc, relaxed_ordering_read,
  61. !!(acc & IB_ACCESS_RELAXED_ORDERING));
  62. MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
  63. MLX5_SET(mkc, mkc, qpn, 0xffffff);
  64. MLX5_SET64(mkc, mkc, start_addr, start_addr);
  65. }
  66. static void
  67. assign_mkey_variant(struct mlx5_ib_dev *dev, struct mlx5_core_mkey *mkey,
  68. u32 *in)
  69. {
  70. u8 key = atomic_inc_return(&dev->mkey_var);
  71. void *mkc;
  72. mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
  73. MLX5_SET(mkc, mkc, mkey_7_0, key);
  74. mkey->key = key;
  75. }
  76. static int
  77. mlx5_ib_create_mkey(struct mlx5_ib_dev *dev, struct mlx5_core_mkey *mkey,
  78. u32 *in, int inlen)
  79. {
  80. assign_mkey_variant(dev, mkey, in);
  81. return mlx5_core_create_mkey(dev->mdev, mkey, in, inlen);
  82. }
  83. static int
  84. mlx5_ib_create_mkey_cb(struct mlx5_ib_dev *dev,
  85. struct mlx5_core_mkey *mkey,
  86. struct mlx5_async_ctx *async_ctx,
  87. u32 *in, int inlen, u32 *out, int outlen,
  88. struct mlx5_async_work *context)
  89. {
  90. MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
  91. assign_mkey_variant(dev, mkey, in);
  92. return mlx5_cmd_exec_cb(async_ctx, in, inlen, out, outlen,
  93. create_mkey_callback, context);
  94. }
  95. static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
  96. static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
  97. static int mr_cache_max_order(struct mlx5_ib_dev *dev);
  98. static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent);
  99. static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev)
  100. {
  101. return !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled);
  102. }
  103. static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
  104. {
  105. WARN_ON(xa_load(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key)));
  106. return mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
  107. }
  108. static inline bool mlx5_ib_pas_fits_in_mr(struct mlx5_ib_mr *mr, u64 start,
  109. u64 length)
  110. {
  111. return ((u64)1 << mr->order) * MLX5_ADAPTER_PAGE_SIZE >=
  112. length + (start & (MLX5_ADAPTER_PAGE_SIZE - 1));
  113. }
  114. static void create_mkey_callback(int status, struct mlx5_async_work *context)
  115. {
  116. struct mlx5_ib_mr *mr =
  117. container_of(context, struct mlx5_ib_mr, cb_work);
  118. struct mlx5_ib_dev *dev = mr->dev;
  119. struct mlx5_cache_ent *ent = mr->cache_ent;
  120. unsigned long flags;
  121. if (status) {
  122. mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status);
  123. kfree(mr);
  124. spin_lock_irqsave(&ent->lock, flags);
  125. ent->pending--;
  126. WRITE_ONCE(dev->fill_delay, 1);
  127. spin_unlock_irqrestore(&ent->lock, flags);
  128. mod_timer(&dev->delay_timer, jiffies + HZ);
  129. return;
  130. }
  131. mr->mmkey.type = MLX5_MKEY_MR;
  132. mr->mmkey.key |= mlx5_idx_to_mkey(
  133. MLX5_GET(create_mkey_out, mr->out, mkey_index));
  134. WRITE_ONCE(dev->cache.last_add, jiffies);
  135. spin_lock_irqsave(&ent->lock, flags);
  136. list_add_tail(&mr->list, &ent->head);
  137. ent->available_mrs++;
  138. ent->total_mrs++;
  139. /* If we are doing fill_to_high_water then keep going. */
  140. queue_adjust_cache_locked(ent);
  141. ent->pending--;
  142. spin_unlock_irqrestore(&ent->lock, flags);
  143. }
  144. static struct mlx5_ib_mr *alloc_cache_mr(struct mlx5_cache_ent *ent, void *mkc)
  145. {
  146. struct mlx5_ib_mr *mr;
  147. mr = kzalloc(sizeof(*mr), GFP_KERNEL);
  148. if (!mr)
  149. return NULL;
  150. mr->order = ent->order;
  151. mr->cache_ent = ent;
  152. mr->dev = ent->dev;
  153. set_mkc_access_pd_addr_fields(mkc, 0, 0, ent->dev->umrc.pd);
  154. MLX5_SET(mkc, mkc, free, 1);
  155. MLX5_SET(mkc, mkc, umr_en, 1);
  156. MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3);
  157. MLX5_SET(mkc, mkc, access_mode_4_2, (ent->access_mode >> 2) & 0x7);
  158. MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt);
  159. MLX5_SET(mkc, mkc, log_page_size, ent->page);
  160. return mr;
  161. }
  162. /* Asynchronously schedule new MRs to be populated in the cache. */
  163. static int add_keys(struct mlx5_cache_ent *ent, unsigned int num)
  164. {
  165. size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
  166. struct mlx5_ib_mr *mr;
  167. void *mkc;
  168. u32 *in;
  169. int err = 0;
  170. int i;
  171. in = kzalloc(inlen, GFP_KERNEL);
  172. if (!in)
  173. return -ENOMEM;
  174. mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
  175. for (i = 0; i < num; i++) {
  176. mr = alloc_cache_mr(ent, mkc);
  177. if (!mr) {
  178. err = -ENOMEM;
  179. break;
  180. }
  181. spin_lock_irq(&ent->lock);
  182. if (ent->pending >= MAX_PENDING_REG_MR) {
  183. err = -EAGAIN;
  184. spin_unlock_irq(&ent->lock);
  185. kfree(mr);
  186. break;
  187. }
  188. ent->pending++;
  189. spin_unlock_irq(&ent->lock);
  190. err = mlx5_ib_create_mkey_cb(ent->dev, &mr->mmkey,
  191. &ent->dev->async_ctx, in, inlen,
  192. mr->out, sizeof(mr->out),
  193. &mr->cb_work);
  194. if (err) {
  195. spin_lock_irq(&ent->lock);
  196. ent->pending--;
  197. spin_unlock_irq(&ent->lock);
  198. mlx5_ib_warn(ent->dev, "create mkey failed %d\n", err);
  199. kfree(mr);
  200. break;
  201. }
  202. }
  203. kfree(in);
  204. return err;
  205. }
  206. /* Synchronously create a MR in the cache */
  207. static struct mlx5_ib_mr *create_cache_mr(struct mlx5_cache_ent *ent)
  208. {
  209. size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
  210. struct mlx5_ib_mr *mr;
  211. void *mkc;
  212. u32 *in;
  213. int err;
  214. in = kzalloc(inlen, GFP_KERNEL);
  215. if (!in)
  216. return ERR_PTR(-ENOMEM);
  217. mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
  218. mr = alloc_cache_mr(ent, mkc);
  219. if (!mr) {
  220. err = -ENOMEM;
  221. goto free_in;
  222. }
  223. err = mlx5_core_create_mkey(ent->dev->mdev, &mr->mmkey, in, inlen);
  224. if (err)
  225. goto free_mr;
  226. mr->mmkey.type = MLX5_MKEY_MR;
  227. WRITE_ONCE(ent->dev->cache.last_add, jiffies);
  228. spin_lock_irq(&ent->lock);
  229. ent->total_mrs++;
  230. spin_unlock_irq(&ent->lock);
  231. kfree(in);
  232. return mr;
  233. free_mr:
  234. kfree(mr);
  235. free_in:
  236. kfree(in);
  237. return ERR_PTR(err);
  238. }
  239. static void remove_cache_mr_locked(struct mlx5_cache_ent *ent)
  240. {
  241. struct mlx5_ib_mr *mr;
  242. lockdep_assert_held(&ent->lock);
  243. if (list_empty(&ent->head))
  244. return;
  245. mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
  246. list_del(&mr->list);
  247. ent->available_mrs--;
  248. ent->total_mrs--;
  249. spin_unlock_irq(&ent->lock);
  250. mlx5_core_destroy_mkey(ent->dev->mdev, &mr->mmkey);
  251. kfree(mr);
  252. spin_lock_irq(&ent->lock);
  253. }
  254. static int resize_available_mrs(struct mlx5_cache_ent *ent, unsigned int target,
  255. bool limit_fill)
  256. {
  257. int err;
  258. lockdep_assert_held(&ent->lock);
  259. while (true) {
  260. if (limit_fill)
  261. target = ent->limit * 2;
  262. if (target == ent->available_mrs + ent->pending)
  263. return 0;
  264. if (target > ent->available_mrs + ent->pending) {
  265. u32 todo = target - (ent->available_mrs + ent->pending);
  266. spin_unlock_irq(&ent->lock);
  267. err = add_keys(ent, todo);
  268. if (err == -EAGAIN)
  269. usleep_range(3000, 5000);
  270. spin_lock_irq(&ent->lock);
  271. if (err) {
  272. if (err != -EAGAIN)
  273. return err;
  274. } else
  275. return 0;
  276. } else {
  277. remove_cache_mr_locked(ent);
  278. }
  279. }
  280. }
  281. static ssize_t size_write(struct file *filp, const char __user *buf,
  282. size_t count, loff_t *pos)
  283. {
  284. struct mlx5_cache_ent *ent = filp->private_data;
  285. u32 target;
  286. int err;
  287. err = kstrtou32_from_user(buf, count, 0, &target);
  288. if (err)
  289. return err;
  290. /*
  291. * Target is the new value of total_mrs the user requests, however we
  292. * cannot free MRs that are in use. Compute the target value for
  293. * available_mrs.
  294. */
  295. spin_lock_irq(&ent->lock);
  296. if (target < ent->total_mrs - ent->available_mrs) {
  297. err = -EINVAL;
  298. goto err_unlock;
  299. }
  300. target = target - (ent->total_mrs - ent->available_mrs);
  301. if (target < ent->limit || target > ent->limit*2) {
  302. err = -EINVAL;
  303. goto err_unlock;
  304. }
  305. err = resize_available_mrs(ent, target, false);
  306. if (err)
  307. goto err_unlock;
  308. spin_unlock_irq(&ent->lock);
  309. return count;
  310. err_unlock:
  311. spin_unlock_irq(&ent->lock);
  312. return err;
  313. }
  314. static ssize_t size_read(struct file *filp, char __user *buf, size_t count,
  315. loff_t *pos)
  316. {
  317. struct mlx5_cache_ent *ent = filp->private_data;
  318. char lbuf[20];
  319. int err;
  320. err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->total_mrs);
  321. if (err < 0)
  322. return err;
  323. return simple_read_from_buffer(buf, count, pos, lbuf, err);
  324. }
  325. static const struct file_operations size_fops = {
  326. .owner = THIS_MODULE,
  327. .open = simple_open,
  328. .write = size_write,
  329. .read = size_read,
  330. };
  331. static ssize_t limit_write(struct file *filp, const char __user *buf,
  332. size_t count, loff_t *pos)
  333. {
  334. struct mlx5_cache_ent *ent = filp->private_data;
  335. u32 var;
  336. int err;
  337. err = kstrtou32_from_user(buf, count, 0, &var);
  338. if (err)
  339. return err;
  340. /*
  341. * Upon set we immediately fill the cache to high water mark implied by
  342. * the limit.
  343. */
  344. spin_lock_irq(&ent->lock);
  345. ent->limit = var;
  346. err = resize_available_mrs(ent, 0, true);
  347. spin_unlock_irq(&ent->lock);
  348. if (err)
  349. return err;
  350. return count;
  351. }
  352. static ssize_t limit_read(struct file *filp, char __user *buf, size_t count,
  353. loff_t *pos)
  354. {
  355. struct mlx5_cache_ent *ent = filp->private_data;
  356. char lbuf[20];
  357. int err;
  358. err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit);
  359. if (err < 0)
  360. return err;
  361. return simple_read_from_buffer(buf, count, pos, lbuf, err);
  362. }
  363. static const struct file_operations limit_fops = {
  364. .owner = THIS_MODULE,
  365. .open = simple_open,
  366. .write = limit_write,
  367. .read = limit_read,
  368. };
  369. static bool someone_adding(struct mlx5_mr_cache *cache)
  370. {
  371. unsigned int i;
  372. for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
  373. struct mlx5_cache_ent *ent = &cache->ent[i];
  374. bool ret;
  375. spin_lock_irq(&ent->lock);
  376. ret = ent->available_mrs < ent->limit;
  377. spin_unlock_irq(&ent->lock);
  378. if (ret)
  379. return true;
  380. }
  381. return false;
  382. }
  383. /*
  384. * Check if the bucket is outside the high/low water mark and schedule an async
  385. * update. The cache refill has hysteresis, once the low water mark is hit it is
  386. * refilled up to the high mark.
  387. */
  388. static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent)
  389. {
  390. lockdep_assert_held(&ent->lock);
  391. if (ent->disabled || READ_ONCE(ent->dev->fill_delay))
  392. return;
  393. if (ent->available_mrs < ent->limit) {
  394. ent->fill_to_high_water = true;
  395. queue_work(ent->dev->cache.wq, &ent->work);
  396. } else if (ent->fill_to_high_water &&
  397. ent->available_mrs + ent->pending < 2 * ent->limit) {
  398. /*
  399. * Once we start populating due to hitting a low water mark
  400. * continue until we pass the high water mark.
  401. */
  402. queue_work(ent->dev->cache.wq, &ent->work);
  403. } else if (ent->available_mrs == 2 * ent->limit) {
  404. ent->fill_to_high_water = false;
  405. } else if (ent->available_mrs > 2 * ent->limit) {
  406. /* Queue deletion of excess entries */
  407. ent->fill_to_high_water = false;
  408. if (ent->pending)
  409. queue_delayed_work(ent->dev->cache.wq, &ent->dwork,
  410. msecs_to_jiffies(1000));
  411. else
  412. queue_work(ent->dev->cache.wq, &ent->work);
  413. }
  414. }
  415. static void __cache_work_func(struct mlx5_cache_ent *ent)
  416. {
  417. struct mlx5_ib_dev *dev = ent->dev;
  418. struct mlx5_mr_cache *cache = &dev->cache;
  419. int err;
  420. spin_lock_irq(&ent->lock);
  421. if (ent->disabled)
  422. goto out;
  423. if (ent->fill_to_high_water &&
  424. ent->available_mrs + ent->pending < 2 * ent->limit &&
  425. !READ_ONCE(dev->fill_delay)) {
  426. spin_unlock_irq(&ent->lock);
  427. err = add_keys(ent, 1);
  428. spin_lock_irq(&ent->lock);
  429. if (ent->disabled)
  430. goto out;
  431. if (err) {
  432. /*
  433. * EAGAIN only happens if pending is positive, so we
  434. * will be rescheduled from reg_mr_callback(). The only
  435. * failure path here is ENOMEM.
  436. */
  437. if (err != -EAGAIN) {
  438. mlx5_ib_warn(
  439. dev,
  440. "command failed order %d, err %d\n",
  441. ent->order, err);
  442. queue_delayed_work(cache->wq, &ent->dwork,
  443. msecs_to_jiffies(1000));
  444. }
  445. }
  446. } else if (ent->available_mrs > 2 * ent->limit) {
  447. bool need_delay;
  448. /*
  449. * The remove_cache_mr() logic is performed as garbage
  450. * collection task. Such task is intended to be run when no
  451. * other active processes are running.
  452. *
  453. * The need_resched() will return TRUE if there are user tasks
  454. * to be activated in near future.
  455. *
  456. * In such case, we don't execute remove_cache_mr() and postpone
  457. * the garbage collection work to try to run in next cycle, in
  458. * order to free CPU resources to other tasks.
  459. */
  460. spin_unlock_irq(&ent->lock);
  461. need_delay = need_resched() || someone_adding(cache) ||
  462. time_after(jiffies,
  463. READ_ONCE(cache->last_add) + 300 * HZ);
  464. spin_lock_irq(&ent->lock);
  465. if (ent->disabled)
  466. goto out;
  467. if (need_delay)
  468. queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ);
  469. remove_cache_mr_locked(ent);
  470. queue_adjust_cache_locked(ent);
  471. }
  472. out:
  473. spin_unlock_irq(&ent->lock);
  474. }
  475. static void delayed_cache_work_func(struct work_struct *work)
  476. {
  477. struct mlx5_cache_ent *ent;
  478. ent = container_of(work, struct mlx5_cache_ent, dwork.work);
  479. __cache_work_func(ent);
  480. }
  481. static void cache_work_func(struct work_struct *work)
  482. {
  483. struct mlx5_cache_ent *ent;
  484. ent = container_of(work, struct mlx5_cache_ent, work);
  485. __cache_work_func(ent);
  486. }
  487. /* Allocate a special entry from the cache */
  488. struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
  489. unsigned int entry, int access_flags)
  490. {
  491. struct mlx5_mr_cache *cache = &dev->cache;
  492. struct mlx5_cache_ent *ent;
  493. struct mlx5_ib_mr *mr;
  494. if (WARN_ON(entry <= MR_CACHE_LAST_STD_ENTRY ||
  495. entry >= ARRAY_SIZE(cache->ent)))
  496. return ERR_PTR(-EINVAL);
  497. /* Matches access in alloc_cache_mr() */
  498. if (!mlx5_ib_can_reconfig_with_umr(dev, 0, access_flags))
  499. return ERR_PTR(-EOPNOTSUPP);
  500. ent = &cache->ent[entry];
  501. spin_lock_irq(&ent->lock);
  502. if (list_empty(&ent->head)) {
  503. spin_unlock_irq(&ent->lock);
  504. mr = create_cache_mr(ent);
  505. if (IS_ERR(mr))
  506. return mr;
  507. } else {
  508. mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
  509. list_del(&mr->list);
  510. ent->available_mrs--;
  511. queue_adjust_cache_locked(ent);
  512. spin_unlock_irq(&ent->lock);
  513. }
  514. mr->access_flags = access_flags;
  515. return mr;
  516. }
  517. /* Return a MR already available in the cache */
  518. static struct mlx5_ib_mr *get_cache_mr(struct mlx5_cache_ent *req_ent)
  519. {
  520. struct mlx5_ib_dev *dev = req_ent->dev;
  521. struct mlx5_ib_mr *mr = NULL;
  522. struct mlx5_cache_ent *ent = req_ent;
  523. /* Try larger MR pools from the cache to satisfy the allocation */
  524. for (; ent != &dev->cache.ent[MR_CACHE_LAST_STD_ENTRY + 1]; ent++) {
  525. mlx5_ib_dbg(dev, "order %u, cache index %zu\n", ent->order,
  526. ent - dev->cache.ent);
  527. spin_lock_irq(&ent->lock);
  528. if (!list_empty(&ent->head)) {
  529. mr = list_first_entry(&ent->head, struct mlx5_ib_mr,
  530. list);
  531. list_del(&mr->list);
  532. ent->available_mrs--;
  533. queue_adjust_cache_locked(ent);
  534. spin_unlock_irq(&ent->lock);
  535. break;
  536. }
  537. queue_adjust_cache_locked(ent);
  538. spin_unlock_irq(&ent->lock);
  539. }
  540. if (!mr)
  541. req_ent->miss++;
  542. return mr;
  543. }
  544. static void detach_mr_from_cache(struct mlx5_ib_mr *mr)
  545. {
  546. struct mlx5_cache_ent *ent = mr->cache_ent;
  547. mr->cache_ent = NULL;
  548. spin_lock_irq(&ent->lock);
  549. ent->total_mrs--;
  550. spin_unlock_irq(&ent->lock);
  551. }
  552. void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
  553. {
  554. struct mlx5_cache_ent *ent = mr->cache_ent;
  555. if (!ent)
  556. return;
  557. if (mlx5_mr_cache_invalidate(mr)) {
  558. detach_mr_from_cache(mr);
  559. destroy_mkey(dev, mr);
  560. return;
  561. }
  562. spin_lock_irq(&ent->lock);
  563. list_add_tail(&mr->list, &ent->head);
  564. ent->available_mrs++;
  565. queue_adjust_cache_locked(ent);
  566. spin_unlock_irq(&ent->lock);
  567. }
  568. static void clean_keys(struct mlx5_ib_dev *dev, int c)
  569. {
  570. struct mlx5_mr_cache *cache = &dev->cache;
  571. struct mlx5_cache_ent *ent = &cache->ent[c];
  572. struct mlx5_ib_mr *tmp_mr;
  573. struct mlx5_ib_mr *mr;
  574. LIST_HEAD(del_list);
  575. cancel_delayed_work(&ent->dwork);
  576. while (1) {
  577. spin_lock_irq(&ent->lock);
  578. if (list_empty(&ent->head)) {
  579. spin_unlock_irq(&ent->lock);
  580. break;
  581. }
  582. mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
  583. list_move(&mr->list, &del_list);
  584. ent->available_mrs--;
  585. ent->total_mrs--;
  586. spin_unlock_irq(&ent->lock);
  587. mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
  588. }
  589. list_for_each_entry_safe(mr, tmp_mr, &del_list, list) {
  590. list_del(&mr->list);
  591. kfree(mr);
  592. }
  593. }
  594. static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev)
  595. {
  596. if (!mlx5_debugfs_root || dev->is_rep)
  597. return;
  598. debugfs_remove_recursive(dev->cache.root);
  599. dev->cache.root = NULL;
  600. }
  601. static void mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev)
  602. {
  603. struct mlx5_mr_cache *cache = &dev->cache;
  604. struct mlx5_cache_ent *ent;
  605. struct dentry *dir;
  606. int i;
  607. if (!mlx5_debugfs_root || dev->is_rep)
  608. return;
  609. cache->root = debugfs_create_dir("mr_cache", dev->mdev->priv.dbg_root);
  610. for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
  611. ent = &cache->ent[i];
  612. sprintf(ent->name, "%d", ent->order);
  613. dir = debugfs_create_dir(ent->name, cache->root);
  614. debugfs_create_file("size", 0600, dir, ent, &size_fops);
  615. debugfs_create_file("limit", 0600, dir, ent, &limit_fops);
  616. debugfs_create_u32("cur", 0400, dir, &ent->available_mrs);
  617. debugfs_create_u32("miss", 0600, dir, &ent->miss);
  618. }
  619. }
  620. static void delay_time_func(struct timer_list *t)
  621. {
  622. struct mlx5_ib_dev *dev = from_timer(dev, t, delay_timer);
  623. WRITE_ONCE(dev->fill_delay, 0);
  624. }
  625. int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
  626. {
  627. struct mlx5_mr_cache *cache = &dev->cache;
  628. struct mlx5_cache_ent *ent;
  629. int i;
  630. mutex_init(&dev->slow_path_mutex);
  631. cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM);
  632. if (!cache->wq) {
  633. mlx5_ib_warn(dev, "failed to create work queue\n");
  634. return -ENOMEM;
  635. }
  636. mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx);
  637. timer_setup(&dev->delay_timer, delay_time_func, 0);
  638. for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
  639. ent = &cache->ent[i];
  640. INIT_LIST_HEAD(&ent->head);
  641. spin_lock_init(&ent->lock);
  642. ent->order = i + 2;
  643. ent->dev = dev;
  644. ent->limit = 0;
  645. INIT_WORK(&ent->work, cache_work_func);
  646. INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
  647. if (i > MR_CACHE_LAST_STD_ENTRY) {
  648. mlx5_odp_init_mr_cache_entry(ent);
  649. continue;
  650. }
  651. if (ent->order > mr_cache_max_order(dev))
  652. continue;
  653. ent->page = PAGE_SHIFT;
  654. ent->xlt = (1 << ent->order) * sizeof(struct mlx5_mtt) /
  655. MLX5_IB_UMR_OCTOWORD;
  656. ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
  657. if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) &&
  658. !dev->is_rep && mlx5_core_is_pf(dev->mdev) &&
  659. mlx5_ib_can_load_pas_with_umr(dev, 0))
  660. ent->limit = dev->mdev->profile->mr_cache[i].limit;
  661. else
  662. ent->limit = 0;
  663. spin_lock_irq(&ent->lock);
  664. queue_adjust_cache_locked(ent);
  665. spin_unlock_irq(&ent->lock);
  666. }
  667. mlx5_mr_cache_debugfs_init(dev);
  668. return 0;
  669. }
  670. int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)
  671. {
  672. unsigned int i;
  673. if (!dev->cache.wq)
  674. return 0;
  675. for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
  676. struct mlx5_cache_ent *ent = &dev->cache.ent[i];
  677. spin_lock_irq(&ent->lock);
  678. ent->disabled = true;
  679. spin_unlock_irq(&ent->lock);
  680. cancel_work_sync(&ent->work);
  681. cancel_delayed_work_sync(&ent->dwork);
  682. }
  683. mlx5_mr_cache_debugfs_cleanup(dev);
  684. mlx5_cmd_cleanup_async_ctx(&dev->async_ctx);
  685. for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++)
  686. clean_keys(dev, i);
  687. destroy_workqueue(dev->cache.wq);
  688. del_timer_sync(&dev->delay_timer);
  689. return 0;
  690. }
  691. struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc)
  692. {
  693. struct mlx5_ib_dev *dev = to_mdev(pd->device);
  694. int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
  695. struct mlx5_ib_mr *mr;
  696. void *mkc;
  697. u32 *in;
  698. int err;
  699. mr = kzalloc(sizeof(*mr), GFP_KERNEL);
  700. if (!mr)
  701. return ERR_PTR(-ENOMEM);
  702. in = kzalloc(inlen, GFP_KERNEL);
  703. if (!in) {
  704. err = -ENOMEM;
  705. goto err_free;
  706. }
  707. mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
  708. MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
  709. MLX5_SET(mkc, mkc, length64, 1);
  710. set_mkc_access_pd_addr_fields(mkc, acc, 0, pd);
  711. err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
  712. if (err)
  713. goto err_in;
  714. kfree(in);
  715. mr->mmkey.type = MLX5_MKEY_MR;
  716. mr->ibmr.lkey = mr->mmkey.key;
  717. mr->ibmr.rkey = mr->mmkey.key;
  718. mr->umem = NULL;
  719. return &mr->ibmr;
  720. err_in:
  721. kfree(in);
  722. err_free:
  723. kfree(mr);
  724. return ERR_PTR(err);
  725. }
  726. static int get_octo_len(u64 addr, u64 len, int page_shift)
  727. {
  728. u64 page_size = 1ULL << page_shift;
  729. u64 offset;
  730. int npages;
  731. offset = addr & (page_size - 1);
  732. npages = ALIGN(len + offset, page_size) >> page_shift;
  733. return (npages + 1) / 2;
  734. }
  735. static int mr_cache_max_order(struct mlx5_ib_dev *dev)
  736. {
  737. if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))
  738. return MR_CACHE_LAST_STD_ENTRY + 2;
  739. return MLX5_MAX_UMR_SHIFT;
  740. }
  741. static int mr_umem_get(struct mlx5_ib_dev *dev, u64 start, u64 length,
  742. int access_flags, struct ib_umem **umem, int *npages,
  743. int *page_shift, int *ncont, int *order)
  744. {
  745. struct ib_umem *u;
  746. *umem = NULL;
  747. if (access_flags & IB_ACCESS_ON_DEMAND) {
  748. struct ib_umem_odp *odp;
  749. odp = ib_umem_odp_get(&dev->ib_dev, start, length, access_flags,
  750. &mlx5_mn_ops);
  751. if (IS_ERR(odp)) {
  752. mlx5_ib_dbg(dev, "umem get failed (%ld)\n",
  753. PTR_ERR(odp));
  754. return PTR_ERR(odp);
  755. }
  756. u = &odp->umem;
  757. *page_shift = odp->page_shift;
  758. *ncont = ib_umem_odp_num_pages(odp);
  759. *npages = *ncont << (*page_shift - PAGE_SHIFT);
  760. if (order)
  761. *order = ilog2(roundup_pow_of_two(*ncont));
  762. } else {
  763. u = ib_umem_get(&dev->ib_dev, start, length, access_flags);
  764. if (IS_ERR(u)) {
  765. mlx5_ib_dbg(dev, "umem get failed (%ld)\n", PTR_ERR(u));
  766. return PTR_ERR(u);
  767. }
  768. mlx5_ib_cont_pages(u, start, MLX5_MKEY_PAGE_SHIFT_MASK, npages,
  769. page_shift, ncont, order);
  770. }
  771. if (!*npages) {
  772. mlx5_ib_warn(dev, "avoid zero region\n");
  773. ib_umem_release(u);
  774. return -EINVAL;
  775. }
  776. *umem = u;
  777. mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n",
  778. *npages, *ncont, *order, *page_shift);
  779. return 0;
  780. }
  781. static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc)
  782. {
  783. struct mlx5_ib_umr_context *context =
  784. container_of(wc->wr_cqe, struct mlx5_ib_umr_context, cqe);
  785. context->status = wc->status;
  786. complete(&context->done);
  787. }
  788. static inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context *context)
  789. {
  790. context->cqe.done = mlx5_ib_umr_done;
  791. context->status = -1;
  792. init_completion(&context->done);
  793. }
  794. static int mlx5_ib_post_send_wait(struct mlx5_ib_dev *dev,
  795. struct mlx5_umr_wr *umrwr)
  796. {
  797. struct umr_common *umrc = &dev->umrc;
  798. const struct ib_send_wr *bad;
  799. int err;
  800. struct mlx5_ib_umr_context umr_context;
  801. mlx5_ib_init_umr_context(&umr_context);
  802. umrwr->wr.wr_cqe = &umr_context.cqe;
  803. down(&umrc->sem);
  804. err = ib_post_send(umrc->qp, &umrwr->wr, &bad);
  805. if (err) {
  806. mlx5_ib_warn(dev, "UMR post send failed, err %d\n", err);
  807. } else {
  808. wait_for_completion(&umr_context.done);
  809. if (umr_context.status != IB_WC_SUCCESS) {
  810. mlx5_ib_warn(dev, "reg umr failed (%u)\n",
  811. umr_context.status);
  812. err = -EFAULT;
  813. }
  814. }
  815. up(&umrc->sem);
  816. return err;
  817. }
  818. static struct mlx5_cache_ent *mr_cache_ent_from_order(struct mlx5_ib_dev *dev,
  819. unsigned int order)
  820. {
  821. struct mlx5_mr_cache *cache = &dev->cache;
  822. if (order < cache->ent[0].order)
  823. return &cache->ent[0];
  824. order = order - cache->ent[0].order;
  825. if (order > MR_CACHE_LAST_STD_ENTRY)
  826. return NULL;
  827. return &cache->ent[order];
  828. }
  829. static struct mlx5_ib_mr *
  830. alloc_mr_from_cache(struct ib_pd *pd, struct ib_umem *umem, u64 virt_addr,
  831. u64 len, int npages, int page_shift, unsigned int order,
  832. int access_flags)
  833. {
  834. struct mlx5_ib_dev *dev = to_mdev(pd->device);
  835. struct mlx5_cache_ent *ent = mr_cache_ent_from_order(dev, order);
  836. struct mlx5_ib_mr *mr;
  837. if (!ent)
  838. return ERR_PTR(-E2BIG);
  839. /* Matches access in alloc_cache_mr() */
  840. if (!mlx5_ib_can_reconfig_with_umr(dev, 0, access_flags))
  841. return ERR_PTR(-EOPNOTSUPP);
  842. mr = get_cache_mr(ent);
  843. if (!mr) {
  844. mr = create_cache_mr(ent);
  845. if (IS_ERR(mr))
  846. return mr;
  847. }
  848. mr->ibmr.pd = pd;
  849. mr->umem = umem;
  850. mr->access_flags = access_flags;
  851. mr->desc_size = sizeof(struct mlx5_mtt);
  852. mr->mmkey.iova = virt_addr;
  853. mr->mmkey.size = len;
  854. mr->mmkey.pd = to_mpd(pd)->pdn;
  855. return mr;
  856. }
  857. #define MLX5_MAX_UMR_CHUNK ((1 << (MLX5_MAX_UMR_SHIFT + 4)) - \
  858. MLX5_UMR_MTT_ALIGNMENT)
  859. #define MLX5_SPARE_UMR_CHUNK 0x10000
  860. int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
  861. int page_shift, int flags)
  862. {
  863. struct mlx5_ib_dev *dev = mr->dev;
  864. struct device *ddev = dev->ib_dev.dev.parent;
  865. int size;
  866. void *xlt;
  867. dma_addr_t dma;
  868. struct mlx5_umr_wr wr;
  869. struct ib_sge sg;
  870. int err = 0;
  871. int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT)
  872. ? sizeof(struct mlx5_klm)
  873. : sizeof(struct mlx5_mtt);
  874. const int page_align = MLX5_UMR_MTT_ALIGNMENT / desc_size;
  875. const int page_mask = page_align - 1;
  876. size_t pages_mapped = 0;
  877. size_t pages_to_map = 0;
  878. size_t pages_iter = 0;
  879. size_t size_to_map = 0;
  880. gfp_t gfp;
  881. bool use_emergency_page = false;
  882. if ((flags & MLX5_IB_UPD_XLT_INDIRECT) &&
  883. !umr_can_use_indirect_mkey(dev))
  884. return -EPERM;
  885. /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes,
  886. * so we need to align the offset and length accordingly
  887. */
  888. if (idx & page_mask) {
  889. npages += idx & page_mask;
  890. idx &= ~page_mask;
  891. }
  892. gfp = flags & MLX5_IB_UPD_XLT_ATOMIC ? GFP_ATOMIC : GFP_KERNEL;
  893. gfp |= __GFP_ZERO | __GFP_NOWARN;
  894. pages_to_map = ALIGN(npages, page_align);
  895. size = desc_size * pages_to_map;
  896. size = min_t(int, size, MLX5_MAX_UMR_CHUNK);
  897. xlt = (void *)__get_free_pages(gfp, get_order(size));
  898. if (!xlt && size > MLX5_SPARE_UMR_CHUNK) {
  899. mlx5_ib_dbg(dev, "Failed to allocate %d bytes of order %d. fallback to spare UMR allocation od %d bytes\n",
  900. size, get_order(size), MLX5_SPARE_UMR_CHUNK);
  901. size = MLX5_SPARE_UMR_CHUNK;
  902. xlt = (void *)__get_free_pages(gfp, get_order(size));
  903. }
  904. if (!xlt) {
  905. mlx5_ib_warn(dev, "Using XLT emergency buffer\n");
  906. xlt = (void *)mlx5_ib_get_xlt_emergency_page();
  907. size = PAGE_SIZE;
  908. memset(xlt, 0, size);
  909. use_emergency_page = true;
  910. }
  911. pages_iter = size / desc_size;
  912. dma = dma_map_single(ddev, xlt, size, DMA_TO_DEVICE);
  913. if (dma_mapping_error(ddev, dma)) {
  914. mlx5_ib_err(dev, "unable to map DMA during XLT update.\n");
  915. err = -ENOMEM;
  916. goto free_xlt;
  917. }
  918. if (mr->umem->is_odp) {
  919. if (!(flags & MLX5_IB_UPD_XLT_INDIRECT)) {
  920. struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
  921. size_t max_pages = ib_umem_odp_num_pages(odp) - idx;
  922. pages_to_map = min_t(size_t, pages_to_map, max_pages);
  923. }
  924. }
  925. sg.addr = dma;
  926. sg.lkey = dev->umrc.pd->local_dma_lkey;
  927. memset(&wr, 0, sizeof(wr));
  928. wr.wr.send_flags = MLX5_IB_SEND_UMR_UPDATE_XLT;
  929. if (!(flags & MLX5_IB_UPD_XLT_ENABLE))
  930. wr.wr.send_flags |= MLX5_IB_SEND_UMR_FAIL_IF_FREE;
  931. wr.wr.sg_list = &sg;
  932. wr.wr.num_sge = 1;
  933. wr.wr.opcode = MLX5_IB_WR_UMR;
  934. wr.pd = mr->ibmr.pd;
  935. wr.mkey = mr->mmkey.key;
  936. wr.length = mr->mmkey.size;
  937. wr.virt_addr = mr->mmkey.iova;
  938. wr.access_flags = mr->access_flags;
  939. wr.page_shift = page_shift;
  940. for (pages_mapped = 0;
  941. pages_mapped < pages_to_map && !err;
  942. pages_mapped += pages_iter, idx += pages_iter) {
  943. npages = min_t(int, pages_iter, pages_to_map - pages_mapped);
  944. size_to_map = npages * desc_size;
  945. dma_sync_single_for_cpu(ddev, dma, size, DMA_TO_DEVICE);
  946. if (mr->umem->is_odp) {
  947. mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags);
  948. } else {
  949. __mlx5_ib_populate_pas(dev, mr->umem, page_shift, idx,
  950. npages, xlt,
  951. MLX5_IB_MTT_PRESENT);
  952. /* Clear padding after the pages
  953. * brought from the umem.
  954. */
  955. memset(xlt + size_to_map, 0, size - size_to_map);
  956. }
  957. dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE);
  958. sg.length = ALIGN(size_to_map, MLX5_UMR_MTT_ALIGNMENT);
  959. if (pages_mapped + pages_iter >= pages_to_map) {
  960. if (flags & MLX5_IB_UPD_XLT_ENABLE)
  961. wr.wr.send_flags |=
  962. MLX5_IB_SEND_UMR_ENABLE_MR |
  963. MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS |
  964. MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
  965. if (flags & MLX5_IB_UPD_XLT_PD ||
  966. flags & MLX5_IB_UPD_XLT_ACCESS)
  967. wr.wr.send_flags |=
  968. MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
  969. if (flags & MLX5_IB_UPD_XLT_ADDR)
  970. wr.wr.send_flags |=
  971. MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
  972. }
  973. wr.offset = idx * desc_size;
  974. wr.xlt_size = sg.length;
  975. err = mlx5_ib_post_send_wait(dev, &wr);
  976. }
  977. dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE);
  978. free_xlt:
  979. if (use_emergency_page)
  980. mlx5_ib_put_xlt_emergency_page();
  981. else
  982. free_pages((unsigned long)xlt, get_order(size));
  983. return err;
  984. }
  985. /*
  986. * If ibmr is NULL it will be allocated by reg_create.
  987. * Else, the given ibmr will be used.
  988. */
  989. static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
  990. u64 virt_addr, u64 length,
  991. struct ib_umem *umem, int npages,
  992. int page_shift, int access_flags,
  993. bool populate)
  994. {
  995. struct mlx5_ib_dev *dev = to_mdev(pd->device);
  996. struct mlx5_ib_mr *mr;
  997. __be64 *pas;
  998. void *mkc;
  999. int inlen;
  1000. u32 *in;
  1001. int err;
  1002. bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg));
  1003. mr = ibmr ? to_mmr(ibmr) : kzalloc(sizeof(*mr), GFP_KERNEL);
  1004. if (!mr)
  1005. return ERR_PTR(-ENOMEM);
  1006. mr->ibmr.pd = pd;
  1007. mr->access_flags = access_flags;
  1008. inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
  1009. if (populate)
  1010. inlen += sizeof(*pas) * roundup(npages, 2);
  1011. in = kvzalloc(inlen, GFP_KERNEL);
  1012. if (!in) {
  1013. err = -ENOMEM;
  1014. goto err_1;
  1015. }
  1016. pas = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
  1017. if (populate) {
  1018. if (WARN_ON(access_flags & IB_ACCESS_ON_DEMAND)) {
  1019. err = -EINVAL;
  1020. goto err_2;
  1021. }
  1022. mlx5_ib_populate_pas(dev, umem, page_shift, pas,
  1023. pg_cap ? MLX5_IB_MTT_PRESENT : 0);
  1024. }
  1025. /* The pg_access bit allows setting the access flags
  1026. * in the page list submitted with the command. */
  1027. MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap));
  1028. mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
  1029. set_mkc_access_pd_addr_fields(mkc, access_flags, virt_addr,
  1030. populate ? pd : dev->umrc.pd);
  1031. MLX5_SET(mkc, mkc, free, !populate);
  1032. MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
  1033. MLX5_SET(mkc, mkc, umr_en, 1);
  1034. MLX5_SET64(mkc, mkc, len, length);
  1035. MLX5_SET(mkc, mkc, bsf_octword_size, 0);
  1036. MLX5_SET(mkc, mkc, translations_octword_size,
  1037. get_octo_len(virt_addr, length, page_shift));
  1038. MLX5_SET(mkc, mkc, log_page_size, page_shift);
  1039. if (populate) {
  1040. MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
  1041. get_octo_len(virt_addr, length, page_shift));
  1042. }
  1043. err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
  1044. if (err) {
  1045. mlx5_ib_warn(dev, "create mkey failed\n");
  1046. goto err_2;
  1047. }
  1048. mr->mmkey.type = MLX5_MKEY_MR;
  1049. mr->desc_size = sizeof(struct mlx5_mtt);
  1050. mr->dev = dev;
  1051. kvfree(in);
  1052. mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key);
  1053. return mr;
  1054. err_2:
  1055. kvfree(in);
  1056. err_1:
  1057. if (!ibmr)
  1058. kfree(mr);
  1059. return ERR_PTR(err);
  1060. }
  1061. static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
  1062. int npages, u64 length, int access_flags)
  1063. {
  1064. mr->npages = npages;
  1065. atomic_add(npages, &dev->mdev->priv.reg_pages);
  1066. mr->ibmr.lkey = mr->mmkey.key;
  1067. mr->ibmr.rkey = mr->mmkey.key;
  1068. mr->ibmr.length = length;
  1069. mr->access_flags = access_flags;
  1070. }
  1071. static struct ib_mr *mlx5_ib_get_dm_mr(struct ib_pd *pd, u64 start_addr,
  1072. u64 length, int acc, int mode)
  1073. {
  1074. struct mlx5_ib_dev *dev = to_mdev(pd->device);
  1075. int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
  1076. struct mlx5_ib_mr *mr;
  1077. void *mkc;
  1078. u32 *in;
  1079. int err;
  1080. mr = kzalloc(sizeof(*mr), GFP_KERNEL);
  1081. if (!mr)
  1082. return ERR_PTR(-ENOMEM);
  1083. in = kzalloc(inlen, GFP_KERNEL);
  1084. if (!in) {
  1085. err = -ENOMEM;
  1086. goto err_free;
  1087. }
  1088. mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
  1089. MLX5_SET(mkc, mkc, access_mode_1_0, mode & 0x3);
  1090. MLX5_SET(mkc, mkc, access_mode_4_2, (mode >> 2) & 0x7);
  1091. MLX5_SET64(mkc, mkc, len, length);
  1092. set_mkc_access_pd_addr_fields(mkc, acc, start_addr, pd);
  1093. err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
  1094. if (err)
  1095. goto err_in;
  1096. kfree(in);
  1097. mr->umem = NULL;
  1098. set_mr_fields(dev, mr, 0, length, acc);
  1099. return &mr->ibmr;
  1100. err_in:
  1101. kfree(in);
  1102. err_free:
  1103. kfree(mr);
  1104. return ERR_PTR(err);
  1105. }
  1106. int mlx5_ib_advise_mr(struct ib_pd *pd,
  1107. enum ib_uverbs_advise_mr_advice advice,
  1108. u32 flags,
  1109. struct ib_sge *sg_list,
  1110. u32 num_sge,
  1111. struct uverbs_attr_bundle *attrs)
  1112. {
  1113. if (advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH &&
  1114. advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE &&
  1115. advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT)
  1116. return -EOPNOTSUPP;
  1117. return mlx5_ib_advise_mr_prefetch(pd, advice, flags,
  1118. sg_list, num_sge);
  1119. }
  1120. struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,
  1121. struct ib_dm_mr_attr *attr,
  1122. struct uverbs_attr_bundle *attrs)
  1123. {
  1124. struct mlx5_ib_dm *mdm = to_mdm(dm);
  1125. struct mlx5_core_dev *dev = to_mdev(dm->device)->mdev;
  1126. u64 start_addr = mdm->dev_addr + attr->offset;
  1127. int mode;
  1128. switch (mdm->type) {
  1129. case MLX5_IB_UAPI_DM_TYPE_MEMIC:
  1130. if (attr->access_flags & ~MLX5_IB_DM_MEMIC_ALLOWED_ACCESS)
  1131. return ERR_PTR(-EINVAL);
  1132. mode = MLX5_MKC_ACCESS_MODE_MEMIC;
  1133. start_addr -= pci_resource_start(dev->pdev, 0);
  1134. break;
  1135. case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
  1136. case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
  1137. if (attr->access_flags & ~MLX5_IB_DM_SW_ICM_ALLOWED_ACCESS)
  1138. return ERR_PTR(-EINVAL);
  1139. mode = MLX5_MKC_ACCESS_MODE_SW_ICM;
  1140. break;
  1141. default:
  1142. return ERR_PTR(-EINVAL);
  1143. }
  1144. return mlx5_ib_get_dm_mr(pd, start_addr, attr->length,
  1145. attr->access_flags, mode);
  1146. }
  1147. struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
  1148. u64 virt_addr, int access_flags,
  1149. struct ib_udata *udata)
  1150. {
  1151. struct mlx5_ib_dev *dev = to_mdev(pd->device);
  1152. struct mlx5_ib_mr *mr = NULL;
  1153. bool xlt_with_umr;
  1154. struct ib_umem *umem;
  1155. int page_shift;
  1156. int npages;
  1157. int ncont;
  1158. int order;
  1159. int err;
  1160. if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM))
  1161. return ERR_PTR(-EOPNOTSUPP);
  1162. mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
  1163. start, virt_addr, length, access_flags);
  1164. xlt_with_umr = mlx5_ib_can_load_pas_with_umr(dev, length);
  1165. /* ODP requires xlt update via umr to work. */
  1166. if (!xlt_with_umr && (access_flags & IB_ACCESS_ON_DEMAND))
  1167. return ERR_PTR(-EINVAL);
  1168. if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && !start &&
  1169. length == U64_MAX) {
  1170. if (virt_addr != start)
  1171. return ERR_PTR(-EINVAL);
  1172. if (!(access_flags & IB_ACCESS_ON_DEMAND) ||
  1173. !(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))
  1174. return ERR_PTR(-EINVAL);
  1175. mr = mlx5_ib_alloc_implicit_mr(to_mpd(pd), udata, access_flags);
  1176. if (IS_ERR(mr))
  1177. return ERR_CAST(mr);
  1178. return &mr->ibmr;
  1179. }
  1180. err = mr_umem_get(dev, start, length, access_flags, &umem,
  1181. &npages, &page_shift, &ncont, &order);
  1182. if (err < 0)
  1183. return ERR_PTR(err);
  1184. if (xlt_with_umr) {
  1185. mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont,
  1186. page_shift, order, access_flags);
  1187. if (IS_ERR(mr))
  1188. mr = NULL;
  1189. }
  1190. if (!mr) {
  1191. mutex_lock(&dev->slow_path_mutex);
  1192. mr = reg_create(NULL, pd, virt_addr, length, umem, ncont,
  1193. page_shift, access_flags, !xlt_with_umr);
  1194. mutex_unlock(&dev->slow_path_mutex);
  1195. }
  1196. if (IS_ERR(mr)) {
  1197. err = PTR_ERR(mr);
  1198. goto error;
  1199. }
  1200. mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key);
  1201. mr->umem = umem;
  1202. set_mr_fields(dev, mr, npages, length, access_flags);
  1203. if (xlt_with_umr && !(access_flags & IB_ACCESS_ON_DEMAND)) {
  1204. /*
  1205. * If the MR was created with reg_create then it will be
  1206. * configured properly but left disabled. It is safe to go ahead
  1207. * and configure it again via UMR while enabling it.
  1208. */
  1209. int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE;
  1210. err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift,
  1211. update_xlt_flags);
  1212. if (err) {
  1213. dereg_mr(dev, mr);
  1214. return ERR_PTR(err);
  1215. }
  1216. }
  1217. if (is_odp_mr(mr)) {
  1218. to_ib_umem_odp(mr->umem)->private = mr;
  1219. init_waitqueue_head(&mr->q_deferred_work);
  1220. atomic_set(&mr->num_deferred_work, 0);
  1221. err = xa_err(xa_store(&dev->odp_mkeys,
  1222. mlx5_base_mkey(mr->mmkey.key), &mr->mmkey,
  1223. GFP_KERNEL));
  1224. if (err) {
  1225. dereg_mr(dev, mr);
  1226. return ERR_PTR(err);
  1227. }
  1228. err = mlx5_ib_init_odp_mr(mr, xlt_with_umr);
  1229. if (err) {
  1230. dereg_mr(dev, mr);
  1231. return ERR_PTR(err);
  1232. }
  1233. }
  1234. return &mr->ibmr;
  1235. error:
  1236. ib_umem_release(umem);
  1237. return ERR_PTR(err);
  1238. }
  1239. /**
  1240. * mlx5_mr_cache_invalidate - Fence all DMA on the MR
  1241. * @mr: The MR to fence
  1242. *
  1243. * Upon return the NIC will not be doing any DMA to the pages under the MR,
  1244. * and any DMA inprogress will be completed. Failure of this function
  1245. * indicates the HW has failed catastrophically.
  1246. */
  1247. int mlx5_mr_cache_invalidate(struct mlx5_ib_mr *mr)
  1248. {
  1249. struct mlx5_umr_wr umrwr = {};
  1250. if (mr->dev->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
  1251. return 0;
  1252. umrwr.wr.send_flags = MLX5_IB_SEND_UMR_DISABLE_MR |
  1253. MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
  1254. umrwr.wr.opcode = MLX5_IB_WR_UMR;
  1255. umrwr.pd = mr->dev->umrc.pd;
  1256. umrwr.mkey = mr->mmkey.key;
  1257. umrwr.ignore_free_state = 1;
  1258. return mlx5_ib_post_send_wait(mr->dev, &umrwr);
  1259. }
  1260. static int rereg_umr(struct ib_pd *pd, struct mlx5_ib_mr *mr,
  1261. int access_flags, int flags)
  1262. {
  1263. struct mlx5_ib_dev *dev = to_mdev(pd->device);
  1264. struct mlx5_umr_wr umrwr = {};
  1265. int err;
  1266. umrwr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE;
  1267. umrwr.wr.opcode = MLX5_IB_WR_UMR;
  1268. umrwr.mkey = mr->mmkey.key;
  1269. if (flags & IB_MR_REREG_PD || flags & IB_MR_REREG_ACCESS) {
  1270. umrwr.pd = pd;
  1271. umrwr.access_flags = access_flags;
  1272. umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
  1273. }
  1274. err = mlx5_ib_post_send_wait(dev, &umrwr);
  1275. return err;
  1276. }
  1277. int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
  1278. u64 length, u64 virt_addr, int new_access_flags,
  1279. struct ib_pd *new_pd, struct ib_udata *udata)
  1280. {
  1281. struct mlx5_ib_dev *dev = to_mdev(ib_mr->device);
  1282. struct mlx5_ib_mr *mr = to_mmr(ib_mr);
  1283. struct ib_pd *pd = (flags & IB_MR_REREG_PD) ? new_pd : ib_mr->pd;
  1284. int access_flags = flags & IB_MR_REREG_ACCESS ?
  1285. new_access_flags :
  1286. mr->access_flags;
  1287. int page_shift = 0;
  1288. int upd_flags = 0;
  1289. int npages = 0;
  1290. int ncont = 0;
  1291. int order = 0;
  1292. u64 addr, len;
  1293. int err;
  1294. mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
  1295. start, virt_addr, length, access_flags);
  1296. atomic_sub(mr->npages, &dev->mdev->priv.reg_pages);
  1297. if (!mr->umem)
  1298. return -EINVAL;
  1299. if (is_odp_mr(mr))
  1300. return -EOPNOTSUPP;
  1301. if (flags & IB_MR_REREG_TRANS) {
  1302. addr = virt_addr;
  1303. len = length;
  1304. } else {
  1305. addr = mr->umem->address;
  1306. len = mr->umem->length;
  1307. }
  1308. if (flags != IB_MR_REREG_PD) {
  1309. /*
  1310. * Replace umem. This needs to be done whether or not UMR is
  1311. * used.
  1312. */
  1313. flags |= IB_MR_REREG_TRANS;
  1314. ib_umem_release(mr->umem);
  1315. mr->umem = NULL;
  1316. err = mr_umem_get(dev, addr, len, access_flags, &mr->umem,
  1317. &npages, &page_shift, &ncont, &order);
  1318. if (err)
  1319. goto err;
  1320. }
  1321. if (!mlx5_ib_can_reconfig_with_umr(dev, mr->access_flags,
  1322. access_flags) ||
  1323. !mlx5_ib_can_load_pas_with_umr(dev, len) ||
  1324. (flags & IB_MR_REREG_TRANS &&
  1325. !mlx5_ib_pas_fits_in_mr(mr, addr, len))) {
  1326. /*
  1327. * UMR can't be used - MKey needs to be replaced.
  1328. */
  1329. if (mr->cache_ent)
  1330. detach_mr_from_cache(mr);
  1331. err = destroy_mkey(dev, mr);
  1332. if (err)
  1333. goto err;
  1334. mr = reg_create(ib_mr, pd, addr, len, mr->umem, ncont,
  1335. page_shift, access_flags, true);
  1336. if (IS_ERR(mr)) {
  1337. err = PTR_ERR(mr);
  1338. mr = to_mmr(ib_mr);
  1339. goto err;
  1340. }
  1341. } else {
  1342. /*
  1343. * Send a UMR WQE
  1344. */
  1345. mr->ibmr.pd = pd;
  1346. mr->access_flags = access_flags;
  1347. mr->mmkey.iova = addr;
  1348. mr->mmkey.size = len;
  1349. mr->mmkey.pd = to_mpd(pd)->pdn;
  1350. if (flags & IB_MR_REREG_TRANS) {
  1351. upd_flags = MLX5_IB_UPD_XLT_ADDR;
  1352. if (flags & IB_MR_REREG_PD)
  1353. upd_flags |= MLX5_IB_UPD_XLT_PD;
  1354. if (flags & IB_MR_REREG_ACCESS)
  1355. upd_flags |= MLX5_IB_UPD_XLT_ACCESS;
  1356. err = mlx5_ib_update_xlt(mr, 0, npages, page_shift,
  1357. upd_flags);
  1358. } else {
  1359. err = rereg_umr(pd, mr, access_flags, flags);
  1360. }
  1361. if (err)
  1362. goto err;
  1363. }
  1364. set_mr_fields(dev, mr, npages, len, access_flags);
  1365. return 0;
  1366. err:
  1367. ib_umem_release(mr->umem);
  1368. mr->umem = NULL;
  1369. clean_mr(dev, mr);
  1370. return err;
  1371. }
  1372. static int
  1373. mlx5_alloc_priv_descs(struct ib_device *device,
  1374. struct mlx5_ib_mr *mr,
  1375. int ndescs,
  1376. int desc_size)
  1377. {
  1378. int size = ndescs * desc_size;
  1379. int add_size;
  1380. int ret;
  1381. add_size = max_t(int, MLX5_UMR_ALIGN - ARCH_KMALLOC_MINALIGN, 0);
  1382. mr->descs_alloc = kzalloc(size + add_size, GFP_KERNEL);
  1383. if (!mr->descs_alloc)
  1384. return -ENOMEM;
  1385. mr->descs = PTR_ALIGN(mr->descs_alloc, MLX5_UMR_ALIGN);
  1386. mr->desc_map = dma_map_single(device->dev.parent, mr->descs,
  1387. size, DMA_TO_DEVICE);
  1388. if (dma_mapping_error(device->dev.parent, mr->desc_map)) {
  1389. ret = -ENOMEM;
  1390. goto err;
  1391. }
  1392. return 0;
  1393. err:
  1394. kfree(mr->descs_alloc);
  1395. return ret;
  1396. }
  1397. static void
  1398. mlx5_free_priv_descs(struct mlx5_ib_mr *mr)
  1399. {
  1400. if (mr->descs) {
  1401. struct ib_device *device = mr->ibmr.device;
  1402. int size = mr->max_descs * mr->desc_size;
  1403. dma_unmap_single(device->dev.parent, mr->desc_map,
  1404. size, DMA_TO_DEVICE);
  1405. kfree(mr->descs_alloc);
  1406. mr->descs = NULL;
  1407. }
  1408. }
  1409. static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
  1410. {
  1411. if (mr->sig) {
  1412. if (mlx5_core_destroy_psv(dev->mdev,
  1413. mr->sig->psv_memory.psv_idx))
  1414. mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
  1415. mr->sig->psv_memory.psv_idx);
  1416. if (mlx5_core_destroy_psv(dev->mdev,
  1417. mr->sig->psv_wire.psv_idx))
  1418. mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
  1419. mr->sig->psv_wire.psv_idx);
  1420. xa_erase(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key));
  1421. kfree(mr->sig);
  1422. mr->sig = NULL;
  1423. }
  1424. if (!mr->cache_ent) {
  1425. destroy_mkey(dev, mr);
  1426. mlx5_free_priv_descs(mr);
  1427. }
  1428. }
  1429. static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
  1430. {
  1431. int npages = mr->npages;
  1432. struct ib_umem *umem = mr->umem;
  1433. /* Stop all DMA */
  1434. if (is_odp_mr(mr))
  1435. mlx5_ib_fence_odp_mr(mr);
  1436. else
  1437. clean_mr(dev, mr);
  1438. if (mr->cache_ent)
  1439. mlx5_mr_cache_free(dev, mr);
  1440. else
  1441. kfree(mr);
  1442. ib_umem_release(umem);
  1443. atomic_sub(npages, &dev->mdev->priv.reg_pages);
  1444. }
  1445. int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
  1446. {
  1447. struct mlx5_ib_mr *mmr = to_mmr(ibmr);
  1448. if (ibmr->type == IB_MR_TYPE_INTEGRITY) {
  1449. dereg_mr(to_mdev(mmr->mtt_mr->ibmr.device), mmr->mtt_mr);
  1450. dereg_mr(to_mdev(mmr->klm_mr->ibmr.device), mmr->klm_mr);
  1451. }
  1452. if (is_odp_mr(mmr) && to_ib_umem_odp(mmr->umem)->is_implicit_odp) {
  1453. mlx5_ib_free_implicit_mr(mmr);
  1454. return 0;
  1455. }
  1456. dereg_mr(to_mdev(ibmr->device), mmr);
  1457. return 0;
  1458. }
  1459. static void mlx5_set_umr_free_mkey(struct ib_pd *pd, u32 *in, int ndescs,
  1460. int access_mode, int page_shift)
  1461. {
  1462. void *mkc;
  1463. mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
  1464. /* This is only used from the kernel, so setting the PD is OK. */
  1465. set_mkc_access_pd_addr_fields(mkc, 0, 0, pd);
  1466. MLX5_SET(mkc, mkc, free, 1);
  1467. MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
  1468. MLX5_SET(mkc, mkc, access_mode_1_0, access_mode & 0x3);
  1469. MLX5_SET(mkc, mkc, access_mode_4_2, (access_mode >> 2) & 0x7);
  1470. MLX5_SET(mkc, mkc, umr_en, 1);
  1471. MLX5_SET(mkc, mkc, log_page_size, page_shift);
  1472. }
  1473. static int _mlx5_alloc_mkey_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
  1474. int ndescs, int desc_size, int page_shift,
  1475. int access_mode, u32 *in, int inlen)
  1476. {
  1477. struct mlx5_ib_dev *dev = to_mdev(pd->device);
  1478. int err;
  1479. mr->access_mode = access_mode;
  1480. mr->desc_size = desc_size;
  1481. mr->max_descs = ndescs;
  1482. err = mlx5_alloc_priv_descs(pd->device, mr, ndescs, desc_size);
  1483. if (err)
  1484. return err;
  1485. mlx5_set_umr_free_mkey(pd, in, ndescs, access_mode, page_shift);
  1486. err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
  1487. if (err)
  1488. goto err_free_descs;
  1489. mr->mmkey.type = MLX5_MKEY_MR;
  1490. mr->ibmr.lkey = mr->mmkey.key;
  1491. mr->ibmr.rkey = mr->mmkey.key;
  1492. return 0;
  1493. err_free_descs:
  1494. mlx5_free_priv_descs(mr);
  1495. return err;
  1496. }
  1497. static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd,
  1498. u32 max_num_sg, u32 max_num_meta_sg,
  1499. int desc_size, int access_mode)
  1500. {
  1501. int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
  1502. int ndescs = ALIGN(max_num_sg + max_num_meta_sg, 4);
  1503. int page_shift = 0;
  1504. struct mlx5_ib_mr *mr;
  1505. u32 *in;
  1506. int err;
  1507. mr = kzalloc(sizeof(*mr), GFP_KERNEL);
  1508. if (!mr)
  1509. return ERR_PTR(-ENOMEM);
  1510. mr->ibmr.pd = pd;
  1511. mr->ibmr.device = pd->device;
  1512. in = kzalloc(inlen, GFP_KERNEL);
  1513. if (!in) {
  1514. err = -ENOMEM;
  1515. goto err_free;
  1516. }
  1517. if (access_mode == MLX5_MKC_ACCESS_MODE_MTT)
  1518. page_shift = PAGE_SHIFT;
  1519. err = _mlx5_alloc_mkey_descs(pd, mr, ndescs, desc_size, page_shift,
  1520. access_mode, in, inlen);
  1521. if (err)
  1522. goto err_free_in;
  1523. mr->umem = NULL;
  1524. kfree(in);
  1525. return mr;
  1526. err_free_in:
  1527. kfree(in);
  1528. err_free:
  1529. kfree(mr);
  1530. return ERR_PTR(err);
  1531. }
  1532. static int mlx5_alloc_mem_reg_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
  1533. int ndescs, u32 *in, int inlen)
  1534. {
  1535. return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_mtt),
  1536. PAGE_SHIFT, MLX5_MKC_ACCESS_MODE_MTT, in,
  1537. inlen);
  1538. }
  1539. static int mlx5_alloc_sg_gaps_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
  1540. int ndescs, u32 *in, int inlen)
  1541. {
  1542. return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_klm),
  1543. 0, MLX5_MKC_ACCESS_MODE_KLMS, in, inlen);
  1544. }
  1545. static int mlx5_alloc_integrity_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
  1546. int max_num_sg, int max_num_meta_sg,
  1547. u32 *in, int inlen)
  1548. {
  1549. struct mlx5_ib_dev *dev = to_mdev(pd->device);
  1550. u32 psv_index[2];
  1551. void *mkc;
  1552. int err;
  1553. mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL);
  1554. if (!mr->sig)
  1555. return -ENOMEM;
  1556. /* create mem & wire PSVs */
  1557. err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, 2, psv_index);
  1558. if (err)
  1559. goto err_free_sig;
  1560. mr->sig->psv_memory.psv_idx = psv_index[0];
  1561. mr->sig->psv_wire.psv_idx = psv_index[1];
  1562. mr->sig->sig_status_checked = true;
  1563. mr->sig->sig_err_exists = false;
  1564. /* Next UMR, Arm SIGERR */
  1565. ++mr->sig->sigerr_count;
  1566. mr->klm_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg,
  1567. sizeof(struct mlx5_klm),
  1568. MLX5_MKC_ACCESS_MODE_KLMS);
  1569. if (IS_ERR(mr->klm_mr)) {
  1570. err = PTR_ERR(mr->klm_mr);
  1571. goto err_destroy_psv;
  1572. }
  1573. mr->mtt_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg,
  1574. sizeof(struct mlx5_mtt),
  1575. MLX5_MKC_ACCESS_MODE_MTT);
  1576. if (IS_ERR(mr->mtt_mr)) {
  1577. err = PTR_ERR(mr->mtt_mr);
  1578. goto err_free_klm_mr;
  1579. }
  1580. /* Set bsf descriptors for mkey */
  1581. mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
  1582. MLX5_SET(mkc, mkc, bsf_en, 1);
  1583. MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE);
  1584. err = _mlx5_alloc_mkey_descs(pd, mr, 4, sizeof(struct mlx5_klm), 0,
  1585. MLX5_MKC_ACCESS_MODE_KLMS, in, inlen);
  1586. if (err)
  1587. goto err_free_mtt_mr;
  1588. err = xa_err(xa_store(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key),
  1589. mr->sig, GFP_KERNEL));
  1590. if (err)
  1591. goto err_free_descs;
  1592. return 0;
  1593. err_free_descs:
  1594. destroy_mkey(dev, mr);
  1595. mlx5_free_priv_descs(mr);
  1596. err_free_mtt_mr:
  1597. dereg_mr(to_mdev(mr->mtt_mr->ibmr.device), mr->mtt_mr);
  1598. mr->mtt_mr = NULL;
  1599. err_free_klm_mr:
  1600. dereg_mr(to_mdev(mr->klm_mr->ibmr.device), mr->klm_mr);
  1601. mr->klm_mr = NULL;
  1602. err_destroy_psv:
  1603. if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_memory.psv_idx))
  1604. mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
  1605. mr->sig->psv_memory.psv_idx);
  1606. if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_wire.psv_idx))
  1607. mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
  1608. mr->sig->psv_wire.psv_idx);
  1609. err_free_sig:
  1610. kfree(mr->sig);
  1611. return err;
  1612. }
  1613. static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd,
  1614. enum ib_mr_type mr_type, u32 max_num_sg,
  1615. u32 max_num_meta_sg)
  1616. {
  1617. struct mlx5_ib_dev *dev = to_mdev(pd->device);
  1618. int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
  1619. int ndescs = ALIGN(max_num_sg, 4);
  1620. struct mlx5_ib_mr *mr;
  1621. u32 *in;
  1622. int err;
  1623. mr = kzalloc(sizeof(*mr), GFP_KERNEL);
  1624. if (!mr)
  1625. return ERR_PTR(-ENOMEM);
  1626. in = kzalloc(inlen, GFP_KERNEL);
  1627. if (!in) {
  1628. err = -ENOMEM;
  1629. goto err_free;
  1630. }
  1631. mr->ibmr.device = pd->device;
  1632. mr->umem = NULL;
  1633. switch (mr_type) {
  1634. case IB_MR_TYPE_MEM_REG:
  1635. err = mlx5_alloc_mem_reg_descs(pd, mr, ndescs, in, inlen);
  1636. break;
  1637. case IB_MR_TYPE_SG_GAPS:
  1638. err = mlx5_alloc_sg_gaps_descs(pd, mr, ndescs, in, inlen);
  1639. break;
  1640. case IB_MR_TYPE_INTEGRITY:
  1641. err = mlx5_alloc_integrity_descs(pd, mr, max_num_sg,
  1642. max_num_meta_sg, in, inlen);
  1643. break;
  1644. default:
  1645. mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type);
  1646. err = -EINVAL;
  1647. }
  1648. if (err)
  1649. goto err_free_in;
  1650. kfree(in);
  1651. return &mr->ibmr;
  1652. err_free_in:
  1653. kfree(in);
  1654. err_free:
  1655. kfree(mr);
  1656. return ERR_PTR(err);
  1657. }
  1658. struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
  1659. u32 max_num_sg)
  1660. {
  1661. return __mlx5_ib_alloc_mr(pd, mr_type, max_num_sg, 0);
  1662. }
  1663. struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd,
  1664. u32 max_num_sg, u32 max_num_meta_sg)
  1665. {
  1666. return __mlx5_ib_alloc_mr(pd, IB_MR_TYPE_INTEGRITY, max_num_sg,
  1667. max_num_meta_sg);
  1668. }
  1669. int mlx5_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
  1670. {
  1671. struct mlx5_ib_dev *dev = to_mdev(ibmw->device);
  1672. int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
  1673. struct mlx5_ib_mw *mw = to_mmw(ibmw);
  1674. u32 *in = NULL;
  1675. void *mkc;
  1676. int ndescs;
  1677. int err;
  1678. struct mlx5_ib_alloc_mw req = {};
  1679. struct {
  1680. __u32 comp_mask;
  1681. __u32 response_length;
  1682. } resp = {};
  1683. err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
  1684. if (err)
  1685. return err;
  1686. if (req.comp_mask || req.reserved1 || req.reserved2)
  1687. return -EOPNOTSUPP;
  1688. if (udata->inlen > sizeof(req) &&
  1689. !ib_is_udata_cleared(udata, sizeof(req),
  1690. udata->inlen - sizeof(req)))
  1691. return -EOPNOTSUPP;
  1692. ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4);
  1693. in = kzalloc(inlen, GFP_KERNEL);
  1694. if (!in) {
  1695. err = -ENOMEM;
  1696. goto free;
  1697. }
  1698. mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
  1699. MLX5_SET(mkc, mkc, free, 1);
  1700. MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
  1701. MLX5_SET(mkc, mkc, pd, to_mpd(ibmw->pd)->pdn);
  1702. MLX5_SET(mkc, mkc, umr_en, 1);
  1703. MLX5_SET(mkc, mkc, lr, 1);
  1704. MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS);
  1705. MLX5_SET(mkc, mkc, en_rinval, !!((ibmw->type == IB_MW_TYPE_2)));
  1706. MLX5_SET(mkc, mkc, qpn, 0xffffff);
  1707. err = mlx5_ib_create_mkey(dev, &mw->mmkey, in, inlen);
  1708. if (err)
  1709. goto free;
  1710. mw->mmkey.type = MLX5_MKEY_MW;
  1711. ibmw->rkey = mw->mmkey.key;
  1712. mw->ndescs = ndescs;
  1713. resp.response_length =
  1714. min(offsetofend(typeof(resp), response_length), udata->outlen);
  1715. if (resp.response_length) {
  1716. err = ib_copy_to_udata(udata, &resp, resp.response_length);
  1717. if (err)
  1718. goto free_mkey;
  1719. }
  1720. if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
  1721. err = xa_err(xa_store(&dev->odp_mkeys,
  1722. mlx5_base_mkey(mw->mmkey.key), &mw->mmkey,
  1723. GFP_KERNEL));
  1724. if (err)
  1725. goto free_mkey;
  1726. }
  1727. kfree(in);
  1728. return 0;
  1729. free_mkey:
  1730. mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey);
  1731. free:
  1732. kfree(in);
  1733. return err;
  1734. }
  1735. int mlx5_ib_dealloc_mw(struct ib_mw *mw)
  1736. {
  1737. struct mlx5_ib_dev *dev = to_mdev(mw->device);
  1738. struct mlx5_ib_mw *mmw = to_mmw(mw);
  1739. if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
  1740. xa_erase(&dev->odp_mkeys, mlx5_base_mkey(mmw->mmkey.key));
  1741. /*
  1742. * pagefault_single_data_segment() may be accessing mmw under
  1743. * SRCU if the user bound an ODP MR to this MW.
  1744. */
  1745. synchronize_srcu(&dev->odp_srcu);
  1746. }
  1747. return mlx5_core_destroy_mkey(dev->mdev, &mmw->mmkey);
  1748. }
  1749. int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
  1750. struct ib_mr_status *mr_status)
  1751. {
  1752. struct mlx5_ib_mr *mmr = to_mmr(ibmr);
  1753. int ret = 0;
  1754. if (check_mask & ~IB_MR_CHECK_SIG_STATUS) {
  1755. pr_err("Invalid status check mask\n");
  1756. ret = -EINVAL;
  1757. goto done;
  1758. }
  1759. mr_status->fail_status = 0;
  1760. if (check_mask & IB_MR_CHECK_SIG_STATUS) {
  1761. if (!mmr->sig) {
  1762. ret = -EINVAL;
  1763. pr_err("signature status check requested on a non-signature enabled MR\n");
  1764. goto done;
  1765. }
  1766. mmr->sig->sig_status_checked = true;
  1767. if (!mmr->sig->sig_err_exists)
  1768. goto done;
  1769. if (ibmr->lkey == mmr->sig->err_item.key)
  1770. memcpy(&mr_status->sig_err, &mmr->sig->err_item,
  1771. sizeof(mr_status->sig_err));
  1772. else {
  1773. mr_status->sig_err.err_type = IB_SIG_BAD_GUARD;
  1774. mr_status->sig_err.sig_err_offset = 0;
  1775. mr_status->sig_err.key = mmr->sig->err_item.key;
  1776. }
  1777. mmr->sig->sig_err_exists = false;
  1778. mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS;
  1779. }
  1780. done:
  1781. return ret;
  1782. }
  1783. static int
  1784. mlx5_ib_map_pa_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
  1785. int data_sg_nents, unsigned int *data_sg_offset,
  1786. struct scatterlist *meta_sg, int meta_sg_nents,
  1787. unsigned int *meta_sg_offset)
  1788. {
  1789. struct mlx5_ib_mr *mr = to_mmr(ibmr);
  1790. unsigned int sg_offset = 0;
  1791. int n = 0;
  1792. mr->meta_length = 0;
  1793. if (data_sg_nents == 1) {
  1794. n++;
  1795. mr->ndescs = 1;
  1796. if (data_sg_offset)
  1797. sg_offset = *data_sg_offset;
  1798. mr->data_length = sg_dma_len(data_sg) - sg_offset;
  1799. mr->data_iova = sg_dma_address(data_sg) + sg_offset;
  1800. if (meta_sg_nents == 1) {
  1801. n++;
  1802. mr->meta_ndescs = 1;
  1803. if (meta_sg_offset)
  1804. sg_offset = *meta_sg_offset;
  1805. else
  1806. sg_offset = 0;
  1807. mr->meta_length = sg_dma_len(meta_sg) - sg_offset;
  1808. mr->pi_iova = sg_dma_address(meta_sg) + sg_offset;
  1809. }
  1810. ibmr->length = mr->data_length + mr->meta_length;
  1811. }
  1812. return n;
  1813. }
  1814. static int
  1815. mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,
  1816. struct scatterlist *sgl,
  1817. unsigned short sg_nents,
  1818. unsigned int *sg_offset_p,
  1819. struct scatterlist *meta_sgl,
  1820. unsigned short meta_sg_nents,
  1821. unsigned int *meta_sg_offset_p)
  1822. {
  1823. struct scatterlist *sg = sgl;
  1824. struct mlx5_klm *klms = mr->descs;
  1825. unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
  1826. u32 lkey = mr->ibmr.pd->local_dma_lkey;
  1827. int i, j = 0;
  1828. mr->ibmr.iova = sg_dma_address(sg) + sg_offset;
  1829. mr->ibmr.length = 0;
  1830. for_each_sg(sgl, sg, sg_nents, i) {
  1831. if (unlikely(i >= mr->max_descs))
  1832. break;
  1833. klms[i].va = cpu_to_be64(sg_dma_address(sg) + sg_offset);
  1834. klms[i].bcount = cpu_to_be32(sg_dma_len(sg) - sg_offset);
  1835. klms[i].key = cpu_to_be32(lkey);
  1836. mr->ibmr.length += sg_dma_len(sg) - sg_offset;
  1837. sg_offset = 0;
  1838. }
  1839. if (sg_offset_p)
  1840. *sg_offset_p = sg_offset;
  1841. mr->ndescs = i;
  1842. mr->data_length = mr->ibmr.length;
  1843. if (meta_sg_nents) {
  1844. sg = meta_sgl;
  1845. sg_offset = meta_sg_offset_p ? *meta_sg_offset_p : 0;
  1846. for_each_sg(meta_sgl, sg, meta_sg_nents, j) {
  1847. if (unlikely(i + j >= mr->max_descs))
  1848. break;
  1849. klms[i + j].va = cpu_to_be64(sg_dma_address(sg) +
  1850. sg_offset);
  1851. klms[i + j].bcount = cpu_to_be32(sg_dma_len(sg) -
  1852. sg_offset);
  1853. klms[i + j].key = cpu_to_be32(lkey);
  1854. mr->ibmr.length += sg_dma_len(sg) - sg_offset;
  1855. sg_offset = 0;
  1856. }
  1857. if (meta_sg_offset_p)
  1858. *meta_sg_offset_p = sg_offset;
  1859. mr->meta_ndescs = j;
  1860. mr->meta_length = mr->ibmr.length - mr->data_length;
  1861. }
  1862. return i + j;
  1863. }
  1864. static int mlx5_set_page(struct ib_mr *ibmr, u64 addr)
  1865. {
  1866. struct mlx5_ib_mr *mr = to_mmr(ibmr);
  1867. __be64 *descs;
  1868. if (unlikely(mr->ndescs == mr->max_descs))
  1869. return -ENOMEM;
  1870. descs = mr->descs;
  1871. descs[mr->ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR);
  1872. return 0;
  1873. }
  1874. static int mlx5_set_page_pi(struct ib_mr *ibmr, u64 addr)
  1875. {
  1876. struct mlx5_ib_mr *mr = to_mmr(ibmr);
  1877. __be64 *descs;
  1878. if (unlikely(mr->ndescs + mr->meta_ndescs == mr->max_descs))
  1879. return -ENOMEM;
  1880. descs = mr->descs;
  1881. descs[mr->ndescs + mr->meta_ndescs++] =
  1882. cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR);
  1883. return 0;
  1884. }
  1885. static int
  1886. mlx5_ib_map_mtt_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
  1887. int data_sg_nents, unsigned int *data_sg_offset,
  1888. struct scatterlist *meta_sg, int meta_sg_nents,
  1889. unsigned int *meta_sg_offset)
  1890. {
  1891. struct mlx5_ib_mr *mr = to_mmr(ibmr);
  1892. struct mlx5_ib_mr *pi_mr = mr->mtt_mr;
  1893. int n;
  1894. pi_mr->ndescs = 0;
  1895. pi_mr->meta_ndescs = 0;
  1896. pi_mr->meta_length = 0;
  1897. ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map,
  1898. pi_mr->desc_size * pi_mr->max_descs,
  1899. DMA_TO_DEVICE);
  1900. pi_mr->ibmr.page_size = ibmr->page_size;
  1901. n = ib_sg_to_pages(&pi_mr->ibmr, data_sg, data_sg_nents, data_sg_offset,
  1902. mlx5_set_page);
  1903. if (n != data_sg_nents)
  1904. return n;
  1905. pi_mr->data_iova = pi_mr->ibmr.iova;
  1906. pi_mr->data_length = pi_mr->ibmr.length;
  1907. pi_mr->ibmr.length = pi_mr->data_length;
  1908. ibmr->length = pi_mr->data_length;
  1909. if (meta_sg_nents) {
  1910. u64 page_mask = ~((u64)ibmr->page_size - 1);
  1911. u64 iova = pi_mr->data_iova;
  1912. n += ib_sg_to_pages(&pi_mr->ibmr, meta_sg, meta_sg_nents,
  1913. meta_sg_offset, mlx5_set_page_pi);
  1914. pi_mr->meta_length = pi_mr->ibmr.length;
  1915. /*
  1916. * PI address for the HW is the offset of the metadata address
  1917. * relative to the first data page address.
  1918. * It equals to first data page address + size of data pages +
  1919. * metadata offset at the first metadata page
  1920. */
  1921. pi_mr->pi_iova = (iova & page_mask) +
  1922. pi_mr->ndescs * ibmr->page_size +
  1923. (pi_mr->ibmr.iova & ~page_mask);
  1924. /*
  1925. * In order to use one MTT MR for data and metadata, we register
  1926. * also the gaps between the end of the data and the start of
  1927. * the metadata (the sig MR will verify that the HW will access
  1928. * to right addresses). This mapping is safe because we use
  1929. * internal mkey for the registration.
  1930. */
  1931. pi_mr->ibmr.length = pi_mr->pi_iova + pi_mr->meta_length - iova;
  1932. pi_mr->ibmr.iova = iova;
  1933. ibmr->length += pi_mr->meta_length;
  1934. }
  1935. ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map,
  1936. pi_mr->desc_size * pi_mr->max_descs,
  1937. DMA_TO_DEVICE);
  1938. return n;
  1939. }
  1940. static int
  1941. mlx5_ib_map_klm_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
  1942. int data_sg_nents, unsigned int *data_sg_offset,
  1943. struct scatterlist *meta_sg, int meta_sg_nents,
  1944. unsigned int *meta_sg_offset)
  1945. {
  1946. struct mlx5_ib_mr *mr = to_mmr(ibmr);
  1947. struct mlx5_ib_mr *pi_mr = mr->klm_mr;
  1948. int n;
  1949. pi_mr->ndescs = 0;
  1950. pi_mr->meta_ndescs = 0;
  1951. pi_mr->meta_length = 0;
  1952. ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map,
  1953. pi_mr->desc_size * pi_mr->max_descs,
  1954. DMA_TO_DEVICE);
  1955. n = mlx5_ib_sg_to_klms(pi_mr, data_sg, data_sg_nents, data_sg_offset,
  1956. meta_sg, meta_sg_nents, meta_sg_offset);
  1957. ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map,
  1958. pi_mr->desc_size * pi_mr->max_descs,
  1959. DMA_TO_DEVICE);
  1960. /* This is zero-based memory region */
  1961. pi_mr->data_iova = 0;
  1962. pi_mr->ibmr.iova = 0;
  1963. pi_mr->pi_iova = pi_mr->data_length;
  1964. ibmr->length = pi_mr->ibmr.length;
  1965. return n;
  1966. }
  1967. int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
  1968. int data_sg_nents, unsigned int *data_sg_offset,
  1969. struct scatterlist *meta_sg, int meta_sg_nents,
  1970. unsigned int *meta_sg_offset)
  1971. {
  1972. struct mlx5_ib_mr *mr = to_mmr(ibmr);
  1973. struct mlx5_ib_mr *pi_mr = NULL;
  1974. int n;
  1975. WARN_ON(ibmr->type != IB_MR_TYPE_INTEGRITY);
  1976. mr->ndescs = 0;
  1977. mr->data_length = 0;
  1978. mr->data_iova = 0;
  1979. mr->meta_ndescs = 0;
  1980. mr->pi_iova = 0;
  1981. /*
  1982. * As a performance optimization, if possible, there is no need to
  1983. * perform UMR operation to register the data/metadata buffers.
  1984. * First try to map the sg lists to PA descriptors with local_dma_lkey.
  1985. * Fallback to UMR only in case of a failure.
  1986. */
  1987. n = mlx5_ib_map_pa_mr_sg_pi(ibmr, data_sg, data_sg_nents,
  1988. data_sg_offset, meta_sg, meta_sg_nents,
  1989. meta_sg_offset);
  1990. if (n == data_sg_nents + meta_sg_nents)
  1991. goto out;
  1992. /*
  1993. * As a performance optimization, if possible, there is no need to map
  1994. * the sg lists to KLM descriptors. First try to map the sg lists to MTT
  1995. * descriptors and fallback to KLM only in case of a failure.
  1996. * It's more efficient for the HW to work with MTT descriptors
  1997. * (especially in high load).
  1998. * Use KLM (indirect access) only if it's mandatory.
  1999. */
  2000. pi_mr = mr->mtt_mr;
  2001. n = mlx5_ib_map_mtt_mr_sg_pi(ibmr, data_sg, data_sg_nents,
  2002. data_sg_offset, meta_sg, meta_sg_nents,
  2003. meta_sg_offset);
  2004. if (n == data_sg_nents + meta_sg_nents)
  2005. goto out;
  2006. pi_mr = mr->klm_mr;
  2007. n = mlx5_ib_map_klm_mr_sg_pi(ibmr, data_sg, data_sg_nents,
  2008. data_sg_offset, meta_sg, meta_sg_nents,
  2009. meta_sg_offset);
  2010. if (unlikely(n != data_sg_nents + meta_sg_nents))
  2011. return -ENOMEM;
  2012. out:
  2013. /* This is zero-based memory region */
  2014. ibmr->iova = 0;
  2015. mr->pi_mr = pi_mr;
  2016. if (pi_mr)
  2017. ibmr->sig_attrs->meta_length = pi_mr->meta_length;
  2018. else
  2019. ibmr->sig_attrs->meta_length = mr->meta_length;
  2020. return 0;
  2021. }
  2022. int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
  2023. unsigned int *sg_offset)
  2024. {
  2025. struct mlx5_ib_mr *mr = to_mmr(ibmr);
  2026. int n;
  2027. mr->ndescs = 0;
  2028. ib_dma_sync_single_for_cpu(ibmr->device, mr->desc_map,
  2029. mr->desc_size * mr->max_descs,
  2030. DMA_TO_DEVICE);
  2031. if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
  2032. n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset, NULL, 0,
  2033. NULL);
  2034. else
  2035. n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset,
  2036. mlx5_set_page);
  2037. ib_dma_sync_single_for_device(ibmr->device, mr->desc_map,
  2038. mr->desc_size * mr->max_descs,
  2039. DMA_TO_DEVICE);
  2040. return n;
  2041. }