vfio_ccw_cp.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * channel program interfaces
  4. *
  5. * Copyright IBM Corp. 2017
  6. *
  7. * Author(s): Dong Jia Shi <[email protected]>
  8. * Xiao Feng Ren <[email protected]>
  9. */
  10. #include <linux/ratelimit.h>
  11. #include <linux/mm.h>
  12. #include <linux/slab.h>
  13. #include <linux/highmem.h>
  14. #include <linux/iommu.h>
  15. #include <linux/vfio.h>
  16. #include <asm/idals.h>
  17. #include "vfio_ccw_cp.h"
  18. #include "vfio_ccw_private.h"
  19. struct page_array {
  20. /* Array that stores pages need to pin. */
  21. dma_addr_t *pa_iova;
  22. /* Array that receives the pinned pages. */
  23. struct page **pa_page;
  24. /* Number of pages pinned from @pa_iova. */
  25. int pa_nr;
  26. };
  27. struct ccwchain {
  28. struct list_head next;
  29. struct ccw1 *ch_ccw;
  30. /* Guest physical address of the current chain. */
  31. u64 ch_iova;
  32. /* Count of the valid ccws in chain. */
  33. int ch_len;
  34. /* Pinned PAGEs for the original data. */
  35. struct page_array *ch_pa;
  36. };
  37. /*
  38. * page_array_alloc() - alloc memory for page array
  39. * @pa: page_array on which to perform the operation
  40. * @iova: target guest physical address
  41. * @len: number of bytes that should be pinned from @iova
  42. *
  43. * Attempt to allocate memory for page array.
  44. *
  45. * Usage of page_array:
  46. * We expect (pa_nr == 0) and (pa_iova == NULL), any field in
  47. * this structure will be filled in by this function.
  48. *
  49. * Returns:
  50. * 0 if page array is allocated
  51. * -EINVAL if pa->pa_nr is not initially zero, or pa->pa_iova is not NULL
  52. * -ENOMEM if alloc failed
  53. */
  54. static int page_array_alloc(struct page_array *pa, u64 iova, unsigned int len)
  55. {
  56. int i;
  57. if (pa->pa_nr || pa->pa_iova)
  58. return -EINVAL;
  59. pa->pa_nr = ((iova & ~PAGE_MASK) + len + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
  60. if (!pa->pa_nr)
  61. return -EINVAL;
  62. pa->pa_iova = kcalloc(pa->pa_nr,
  63. sizeof(*pa->pa_iova) + sizeof(*pa->pa_page),
  64. GFP_KERNEL);
  65. if (unlikely(!pa->pa_iova)) {
  66. pa->pa_nr = 0;
  67. return -ENOMEM;
  68. }
  69. pa->pa_page = (struct page **)&pa->pa_iova[pa->pa_nr];
  70. pa->pa_iova[0] = iova;
  71. pa->pa_page[0] = NULL;
  72. for (i = 1; i < pa->pa_nr; i++) {
  73. pa->pa_iova[i] = pa->pa_iova[i - 1] + PAGE_SIZE;
  74. pa->pa_page[i] = NULL;
  75. }
  76. return 0;
  77. }
  78. /*
  79. * page_array_unpin() - Unpin user pages in memory
  80. * @pa: page_array on which to perform the operation
  81. * @vdev: the vfio device to perform the operation
  82. * @pa_nr: number of user pages to unpin
  83. *
  84. * Only unpin if any pages were pinned to begin with, i.e. pa_nr > 0,
  85. * otherwise only clear pa->pa_nr
  86. */
  87. static void page_array_unpin(struct page_array *pa,
  88. struct vfio_device *vdev, int pa_nr)
  89. {
  90. int unpinned = 0, npage = 1;
  91. while (unpinned < pa_nr) {
  92. dma_addr_t *first = &pa->pa_iova[unpinned];
  93. dma_addr_t *last = &first[npage];
  94. if (unpinned + npage < pa_nr &&
  95. *first + npage * PAGE_SIZE == *last) {
  96. npage++;
  97. continue;
  98. }
  99. vfio_unpin_pages(vdev, *first, npage);
  100. unpinned += npage;
  101. npage = 1;
  102. }
  103. pa->pa_nr = 0;
  104. }
  105. /*
  106. * page_array_pin() - Pin user pages in memory
  107. * @pa: page_array on which to perform the operation
  108. * @mdev: the mediated device to perform pin operations
  109. *
  110. * Returns number of pages pinned upon success.
  111. * If the pin request partially succeeds, or fails completely,
  112. * all pages are left unpinned and a negative error value is returned.
  113. */
  114. static int page_array_pin(struct page_array *pa, struct vfio_device *vdev)
  115. {
  116. int pinned = 0, npage = 1;
  117. int ret = 0;
  118. while (pinned < pa->pa_nr) {
  119. dma_addr_t *first = &pa->pa_iova[pinned];
  120. dma_addr_t *last = &first[npage];
  121. if (pinned + npage < pa->pa_nr &&
  122. *first + npage * PAGE_SIZE == *last) {
  123. npage++;
  124. continue;
  125. }
  126. ret = vfio_pin_pages(vdev, *first, npage,
  127. IOMMU_READ | IOMMU_WRITE,
  128. &pa->pa_page[pinned]);
  129. if (ret < 0) {
  130. goto err_out;
  131. } else if (ret > 0 && ret != npage) {
  132. pinned += ret;
  133. ret = -EINVAL;
  134. goto err_out;
  135. }
  136. pinned += npage;
  137. npage = 1;
  138. }
  139. return ret;
  140. err_out:
  141. page_array_unpin(pa, vdev, pinned);
  142. return ret;
  143. }
  144. /* Unpin the pages before releasing the memory. */
  145. static void page_array_unpin_free(struct page_array *pa, struct vfio_device *vdev)
  146. {
  147. page_array_unpin(pa, vdev, pa->pa_nr);
  148. kfree(pa->pa_iova);
  149. }
  150. static bool page_array_iova_pinned(struct page_array *pa, u64 iova, u64 length)
  151. {
  152. u64 iova_pfn_start = iova >> PAGE_SHIFT;
  153. u64 iova_pfn_end = (iova + length - 1) >> PAGE_SHIFT;
  154. u64 pfn;
  155. int i;
  156. for (i = 0; i < pa->pa_nr; i++) {
  157. pfn = pa->pa_iova[i] >> PAGE_SHIFT;
  158. if (pfn >= iova_pfn_start && pfn <= iova_pfn_end)
  159. return true;
  160. }
  161. return false;
  162. }
  163. /* Create the list of IDAL words for a page_array. */
  164. static inline void page_array_idal_create_words(struct page_array *pa,
  165. unsigned long *idaws)
  166. {
  167. int i;
  168. /*
  169. * Idal words (execept the first one) rely on the memory being 4k
  170. * aligned. If a user virtual address is 4K aligned, then it's
  171. * corresponding kernel physical address will also be 4K aligned. Thus
  172. * there will be no problem here to simply use the phys to create an
  173. * idaw.
  174. */
  175. for (i = 0; i < pa->pa_nr; i++)
  176. idaws[i] = page_to_phys(pa->pa_page[i]);
  177. /* Adjust the first IDAW, since it may not start on a page boundary */
  178. idaws[0] += pa->pa_iova[0] & (PAGE_SIZE - 1);
  179. }
  180. static void convert_ccw0_to_ccw1(struct ccw1 *source, unsigned long len)
  181. {
  182. struct ccw0 ccw0;
  183. struct ccw1 *pccw1 = source;
  184. int i;
  185. for (i = 0; i < len; i++) {
  186. ccw0 = *(struct ccw0 *)pccw1;
  187. if ((pccw1->cmd_code & 0x0f) == CCW_CMD_TIC) {
  188. pccw1->cmd_code = CCW_CMD_TIC;
  189. pccw1->flags = 0;
  190. pccw1->count = 0;
  191. } else {
  192. pccw1->cmd_code = ccw0.cmd_code;
  193. pccw1->flags = ccw0.flags;
  194. pccw1->count = ccw0.count;
  195. }
  196. pccw1->cda = ccw0.cda;
  197. pccw1++;
  198. }
  199. }
  200. /*
  201. * Within the domain (@mdev), copy @n bytes from a guest physical
  202. * address (@iova) to a host physical address (@to).
  203. */
  204. static long copy_from_iova(struct vfio_device *vdev, void *to, u64 iova,
  205. unsigned long n)
  206. {
  207. struct page_array pa = {0};
  208. int i, ret;
  209. unsigned long l, m;
  210. ret = page_array_alloc(&pa, iova, n);
  211. if (ret < 0)
  212. return ret;
  213. ret = page_array_pin(&pa, vdev);
  214. if (ret < 0) {
  215. page_array_unpin_free(&pa, vdev);
  216. return ret;
  217. }
  218. l = n;
  219. for (i = 0; i < pa.pa_nr; i++) {
  220. void *from = kmap_local_page(pa.pa_page[i]);
  221. m = PAGE_SIZE;
  222. if (i == 0) {
  223. from += iova & (PAGE_SIZE - 1);
  224. m -= iova & (PAGE_SIZE - 1);
  225. }
  226. m = min(l, m);
  227. memcpy(to + (n - l), from, m);
  228. kunmap_local(from);
  229. l -= m;
  230. if (l == 0)
  231. break;
  232. }
  233. page_array_unpin_free(&pa, vdev);
  234. return l;
  235. }
  236. /*
  237. * Helpers to operate ccwchain.
  238. */
  239. #define ccw_is_read(_ccw) (((_ccw)->cmd_code & 0x03) == 0x02)
  240. #define ccw_is_read_backward(_ccw) (((_ccw)->cmd_code & 0x0F) == 0x0C)
  241. #define ccw_is_sense(_ccw) (((_ccw)->cmd_code & 0x0F) == CCW_CMD_BASIC_SENSE)
  242. #define ccw_is_noop(_ccw) ((_ccw)->cmd_code == CCW_CMD_NOOP)
  243. #define ccw_is_tic(_ccw) ((_ccw)->cmd_code == CCW_CMD_TIC)
  244. #define ccw_is_idal(_ccw) ((_ccw)->flags & CCW_FLAG_IDA)
  245. #define ccw_is_skip(_ccw) ((_ccw)->flags & CCW_FLAG_SKIP)
  246. #define ccw_is_chain(_ccw) ((_ccw)->flags & (CCW_FLAG_CC | CCW_FLAG_DC))
  247. /*
  248. * ccw_does_data_transfer()
  249. *
  250. * Determine whether a CCW will move any data, such that the guest pages
  251. * would need to be pinned before performing the I/O.
  252. *
  253. * Returns 1 if yes, 0 if no.
  254. */
  255. static inline int ccw_does_data_transfer(struct ccw1 *ccw)
  256. {
  257. /* If the count field is zero, then no data will be transferred */
  258. if (ccw->count == 0)
  259. return 0;
  260. /* If the command is a NOP, then no data will be transferred */
  261. if (ccw_is_noop(ccw))
  262. return 0;
  263. /* If the skip flag is off, then data will be transferred */
  264. if (!ccw_is_skip(ccw))
  265. return 1;
  266. /*
  267. * If the skip flag is on, it is only meaningful if the command
  268. * code is a read, read backward, sense, or sense ID. In those
  269. * cases, no data will be transferred.
  270. */
  271. if (ccw_is_read(ccw) || ccw_is_read_backward(ccw))
  272. return 0;
  273. if (ccw_is_sense(ccw))
  274. return 0;
  275. /* The skip flag is on, but it is ignored for this command code. */
  276. return 1;
  277. }
  278. /*
  279. * is_cpa_within_range()
  280. *
  281. * @cpa: channel program address being questioned
  282. * @head: address of the beginning of a CCW chain
  283. * @len: number of CCWs within the chain
  284. *
  285. * Determine whether the address of a CCW (whether a new chain,
  286. * or the target of a TIC) falls within a range (including the end points).
  287. *
  288. * Returns 1 if yes, 0 if no.
  289. */
  290. static inline int is_cpa_within_range(u32 cpa, u32 head, int len)
  291. {
  292. u32 tail = head + (len - 1) * sizeof(struct ccw1);
  293. return (head <= cpa && cpa <= tail);
  294. }
  295. static inline int is_tic_within_range(struct ccw1 *ccw, u32 head, int len)
  296. {
  297. if (!ccw_is_tic(ccw))
  298. return 0;
  299. return is_cpa_within_range(ccw->cda, head, len);
  300. }
  301. static struct ccwchain *ccwchain_alloc(struct channel_program *cp, int len)
  302. {
  303. struct ccwchain *chain;
  304. void *data;
  305. size_t size;
  306. /* Make ccw address aligned to 8. */
  307. size = ((sizeof(*chain) + 7L) & -8L) +
  308. sizeof(*chain->ch_ccw) * len +
  309. sizeof(*chain->ch_pa) * len;
  310. chain = kzalloc(size, GFP_DMA | GFP_KERNEL);
  311. if (!chain)
  312. return NULL;
  313. data = (u8 *)chain + ((sizeof(*chain) + 7L) & -8L);
  314. chain->ch_ccw = (struct ccw1 *)data;
  315. data = (u8 *)(chain->ch_ccw) + sizeof(*chain->ch_ccw) * len;
  316. chain->ch_pa = (struct page_array *)data;
  317. chain->ch_len = len;
  318. list_add_tail(&chain->next, &cp->ccwchain_list);
  319. return chain;
  320. }
  321. static void ccwchain_free(struct ccwchain *chain)
  322. {
  323. list_del(&chain->next);
  324. kfree(chain);
  325. }
  326. /* Free resource for a ccw that allocated memory for its cda. */
  327. static void ccwchain_cda_free(struct ccwchain *chain, int idx)
  328. {
  329. struct ccw1 *ccw = chain->ch_ccw + idx;
  330. if (ccw_is_tic(ccw))
  331. return;
  332. kfree((void *)(u64)ccw->cda);
  333. }
  334. /**
  335. * ccwchain_calc_length - calculate the length of the ccw chain.
  336. * @iova: guest physical address of the target ccw chain
  337. * @cp: channel_program on which to perform the operation
  338. *
  339. * This is the chain length not considering any TICs.
  340. * You need to do a new round for each TIC target.
  341. *
  342. * The program is also validated for absence of not yet supported
  343. * indirect data addressing scenarios.
  344. *
  345. * Returns: the length of the ccw chain or -errno.
  346. */
  347. static int ccwchain_calc_length(u64 iova, struct channel_program *cp)
  348. {
  349. struct ccw1 *ccw = cp->guest_cp;
  350. int cnt = 0;
  351. do {
  352. cnt++;
  353. /*
  354. * As we don't want to fail direct addressing even if the
  355. * orb specified one of the unsupported formats, we defer
  356. * checking for IDAWs in unsupported formats to here.
  357. */
  358. if ((!cp->orb.cmd.c64 || cp->orb.cmd.i2k) && ccw_is_idal(ccw))
  359. return -EOPNOTSUPP;
  360. /*
  361. * We want to keep counting if the current CCW has the
  362. * command-chaining flag enabled, or if it is a TIC CCW
  363. * that loops back into the current chain. The latter
  364. * is used for device orientation, where the CCW PRIOR to
  365. * the TIC can either jump to the TIC or a CCW immediately
  366. * after the TIC, depending on the results of its operation.
  367. */
  368. if (!ccw_is_chain(ccw) && !is_tic_within_range(ccw, iova, cnt))
  369. break;
  370. ccw++;
  371. } while (cnt < CCWCHAIN_LEN_MAX + 1);
  372. if (cnt == CCWCHAIN_LEN_MAX + 1)
  373. cnt = -EINVAL;
  374. return cnt;
  375. }
  376. static int tic_target_chain_exists(struct ccw1 *tic, struct channel_program *cp)
  377. {
  378. struct ccwchain *chain;
  379. u32 ccw_head;
  380. list_for_each_entry(chain, &cp->ccwchain_list, next) {
  381. ccw_head = chain->ch_iova;
  382. if (is_cpa_within_range(tic->cda, ccw_head, chain->ch_len))
  383. return 1;
  384. }
  385. return 0;
  386. }
  387. static int ccwchain_loop_tic(struct ccwchain *chain,
  388. struct channel_program *cp);
  389. static int ccwchain_handle_ccw(u32 cda, struct channel_program *cp)
  390. {
  391. struct vfio_device *vdev =
  392. &container_of(cp, struct vfio_ccw_private, cp)->vdev;
  393. struct ccwchain *chain;
  394. int len, ret;
  395. /* Copy 2K (the most we support today) of possible CCWs */
  396. len = copy_from_iova(vdev, cp->guest_cp, cda,
  397. CCWCHAIN_LEN_MAX * sizeof(struct ccw1));
  398. if (len)
  399. return len;
  400. /* Convert any Format-0 CCWs to Format-1 */
  401. if (!cp->orb.cmd.fmt)
  402. convert_ccw0_to_ccw1(cp->guest_cp, CCWCHAIN_LEN_MAX);
  403. /* Count the CCWs in the current chain */
  404. len = ccwchain_calc_length(cda, cp);
  405. if (len < 0)
  406. return len;
  407. /* Need alloc a new chain for this one. */
  408. chain = ccwchain_alloc(cp, len);
  409. if (!chain)
  410. return -ENOMEM;
  411. chain->ch_iova = cda;
  412. /* Copy the actual CCWs into the new chain */
  413. memcpy(chain->ch_ccw, cp->guest_cp, len * sizeof(struct ccw1));
  414. /* Loop for tics on this new chain. */
  415. ret = ccwchain_loop_tic(chain, cp);
  416. if (ret)
  417. ccwchain_free(chain);
  418. return ret;
  419. }
  420. /* Loop for TICs. */
  421. static int ccwchain_loop_tic(struct ccwchain *chain, struct channel_program *cp)
  422. {
  423. struct ccw1 *tic;
  424. int i, ret;
  425. for (i = 0; i < chain->ch_len; i++) {
  426. tic = chain->ch_ccw + i;
  427. if (!ccw_is_tic(tic))
  428. continue;
  429. /* May transfer to an existing chain. */
  430. if (tic_target_chain_exists(tic, cp))
  431. continue;
  432. /* Build a ccwchain for the next segment */
  433. ret = ccwchain_handle_ccw(tic->cda, cp);
  434. if (ret)
  435. return ret;
  436. }
  437. return 0;
  438. }
  439. static int ccwchain_fetch_tic(struct ccwchain *chain,
  440. int idx,
  441. struct channel_program *cp)
  442. {
  443. struct ccw1 *ccw = chain->ch_ccw + idx;
  444. struct ccwchain *iter;
  445. u32 ccw_head;
  446. list_for_each_entry(iter, &cp->ccwchain_list, next) {
  447. ccw_head = iter->ch_iova;
  448. if (is_cpa_within_range(ccw->cda, ccw_head, iter->ch_len)) {
  449. ccw->cda = (__u32) (addr_t) (((char *)iter->ch_ccw) +
  450. (ccw->cda - ccw_head));
  451. return 0;
  452. }
  453. }
  454. return -EFAULT;
  455. }
  456. static int ccwchain_fetch_direct(struct ccwchain *chain,
  457. int idx,
  458. struct channel_program *cp)
  459. {
  460. struct vfio_device *vdev =
  461. &container_of(cp, struct vfio_ccw_private, cp)->vdev;
  462. struct ccw1 *ccw;
  463. struct page_array *pa;
  464. u64 iova;
  465. unsigned long *idaws;
  466. int ret;
  467. int bytes = 1;
  468. int idaw_nr, idal_len;
  469. int i;
  470. ccw = chain->ch_ccw + idx;
  471. if (ccw->count)
  472. bytes = ccw->count;
  473. /* Calculate size of IDAL */
  474. if (ccw_is_idal(ccw)) {
  475. /* Read first IDAW to see if it's 4K-aligned or not. */
  476. /* All subsequent IDAws will be 4K-aligned. */
  477. ret = copy_from_iova(vdev, &iova, ccw->cda, sizeof(iova));
  478. if (ret)
  479. return ret;
  480. } else {
  481. iova = ccw->cda;
  482. }
  483. idaw_nr = idal_nr_words((void *)iova, bytes);
  484. idal_len = idaw_nr * sizeof(*idaws);
  485. /* Allocate an IDAL from host storage */
  486. idaws = kcalloc(idaw_nr, sizeof(*idaws), GFP_DMA | GFP_KERNEL);
  487. if (!idaws) {
  488. ret = -ENOMEM;
  489. goto out_init;
  490. }
  491. /*
  492. * Allocate an array of pages to pin/translate.
  493. * The number of pages is actually the count of the idaws
  494. * required for the data transfer, since we only only support
  495. * 4K IDAWs today.
  496. */
  497. pa = chain->ch_pa + idx;
  498. ret = page_array_alloc(pa, iova, bytes);
  499. if (ret < 0)
  500. goto out_free_idaws;
  501. if (ccw_is_idal(ccw)) {
  502. /* Copy guest IDAL into host IDAL */
  503. ret = copy_from_iova(vdev, idaws, ccw->cda, idal_len);
  504. if (ret)
  505. goto out_unpin;
  506. /*
  507. * Copy guest IDAWs into page_array, in case the memory they
  508. * occupy is not contiguous.
  509. */
  510. for (i = 0; i < idaw_nr; i++)
  511. pa->pa_iova[i] = idaws[i];
  512. } else {
  513. /*
  514. * No action is required here; the iova addresses in page_array
  515. * were initialized sequentially in page_array_alloc() beginning
  516. * with the contents of ccw->cda.
  517. */
  518. }
  519. if (ccw_does_data_transfer(ccw)) {
  520. ret = page_array_pin(pa, vdev);
  521. if (ret < 0)
  522. goto out_unpin;
  523. } else {
  524. pa->pa_nr = 0;
  525. }
  526. ccw->cda = (__u32) virt_to_phys(idaws);
  527. ccw->flags |= CCW_FLAG_IDA;
  528. /* Populate the IDAL with pinned/translated addresses from page */
  529. page_array_idal_create_words(pa, idaws);
  530. return 0;
  531. out_unpin:
  532. page_array_unpin_free(pa, vdev);
  533. out_free_idaws:
  534. kfree(idaws);
  535. out_init:
  536. ccw->cda = 0;
  537. return ret;
  538. }
  539. /*
  540. * Fetch one ccw.
  541. * To reduce memory copy, we'll pin the cda page in memory,
  542. * and to get rid of the cda 2G limitiaion of ccw1, we'll translate
  543. * direct ccws to idal ccws.
  544. */
  545. static int ccwchain_fetch_one(struct ccwchain *chain,
  546. int idx,
  547. struct channel_program *cp)
  548. {
  549. struct ccw1 *ccw = chain->ch_ccw + idx;
  550. if (ccw_is_tic(ccw))
  551. return ccwchain_fetch_tic(chain, idx, cp);
  552. return ccwchain_fetch_direct(chain, idx, cp);
  553. }
  554. /**
  555. * cp_init() - allocate ccwchains for a channel program.
  556. * @cp: channel_program on which to perform the operation
  557. * @mdev: the mediated device to perform pin/unpin operations
  558. * @orb: control block for the channel program from the guest
  559. *
  560. * This creates one or more ccwchain(s), and copies the raw data of
  561. * the target channel program from @orb->cmd.iova to the new ccwchain(s).
  562. *
  563. * Limitations:
  564. * 1. Supports idal(c64) ccw chaining.
  565. * 2. Supports 4k idaw.
  566. *
  567. * Returns:
  568. * %0 on success and a negative error value on failure.
  569. */
  570. int cp_init(struct channel_program *cp, union orb *orb)
  571. {
  572. struct vfio_device *vdev =
  573. &container_of(cp, struct vfio_ccw_private, cp)->vdev;
  574. /* custom ratelimit used to avoid flood during guest IPL */
  575. static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 1);
  576. int ret;
  577. /* this is an error in the caller */
  578. if (cp->initialized)
  579. return -EBUSY;
  580. /*
  581. * We only support prefetching the channel program. We assume all channel
  582. * programs executed by supported guests likewise support prefetching.
  583. * Executing a channel program that does not specify prefetching will
  584. * typically not cause an error, but a warning is issued to help identify
  585. * the problem if something does break.
  586. */
  587. if (!orb->cmd.pfch && __ratelimit(&ratelimit_state))
  588. dev_warn(
  589. vdev->dev,
  590. "Prefetching channel program even though prefetch not specified in ORB");
  591. INIT_LIST_HEAD(&cp->ccwchain_list);
  592. memcpy(&cp->orb, orb, sizeof(*orb));
  593. /* Build a ccwchain for the first CCW segment */
  594. ret = ccwchain_handle_ccw(orb->cmd.cpa, cp);
  595. if (!ret) {
  596. cp->initialized = true;
  597. /* It is safe to force: if it was not set but idals used
  598. * ccwchain_calc_length would have returned an error.
  599. */
  600. cp->orb.cmd.c64 = 1;
  601. }
  602. return ret;
  603. }
  604. /**
  605. * cp_free() - free resources for channel program.
  606. * @cp: channel_program on which to perform the operation
  607. *
  608. * This unpins the memory pages and frees the memory space occupied by
  609. * @cp, which must have been returned by a previous call to cp_init().
  610. * Otherwise, undefined behavior occurs.
  611. */
  612. void cp_free(struct channel_program *cp)
  613. {
  614. struct vfio_device *vdev =
  615. &container_of(cp, struct vfio_ccw_private, cp)->vdev;
  616. struct ccwchain *chain, *temp;
  617. int i;
  618. if (!cp->initialized)
  619. return;
  620. cp->initialized = false;
  621. list_for_each_entry_safe(chain, temp, &cp->ccwchain_list, next) {
  622. for (i = 0; i < chain->ch_len; i++) {
  623. page_array_unpin_free(chain->ch_pa + i, vdev);
  624. ccwchain_cda_free(chain, i);
  625. }
  626. ccwchain_free(chain);
  627. }
  628. }
  629. /**
  630. * cp_prefetch() - translate a guest physical address channel program to
  631. * a real-device runnable channel program.
  632. * @cp: channel_program on which to perform the operation
  633. *
  634. * This function translates the guest-physical-address channel program
  635. * and stores the result to ccwchain list. @cp must have been
  636. * initialized by a previous call with cp_init(). Otherwise, undefined
  637. * behavior occurs.
  638. * For each chain composing the channel program:
  639. * - On entry ch_len holds the count of CCWs to be translated.
  640. * - On exit ch_len is adjusted to the count of successfully translated CCWs.
  641. * This allows cp_free to find in ch_len the count of CCWs to free in a chain.
  642. *
  643. * The S/390 CCW Translation APIS (prefixed by 'cp_') are introduced
  644. * as helpers to do ccw chain translation inside the kernel. Basically
  645. * they accept a channel program issued by a virtual machine, and
  646. * translate the channel program to a real-device runnable channel
  647. * program.
  648. *
  649. * These APIs will copy the ccws into kernel-space buffers, and update
  650. * the guest phsical addresses with their corresponding host physical
  651. * addresses. Then channel I/O device drivers could issue the
  652. * translated channel program to real devices to perform an I/O
  653. * operation.
  654. *
  655. * These interfaces are designed to support translation only for
  656. * channel programs, which are generated and formatted by a
  657. * guest. Thus this will make it possible for things like VFIO to
  658. * leverage the interfaces to passthrough a channel I/O mediated
  659. * device in QEMU.
  660. *
  661. * We support direct ccw chaining by translating them to idal ccws.
  662. *
  663. * Returns:
  664. * %0 on success and a negative error value on failure.
  665. */
  666. int cp_prefetch(struct channel_program *cp)
  667. {
  668. struct ccwchain *chain;
  669. int len, idx, ret;
  670. /* this is an error in the caller */
  671. if (!cp->initialized)
  672. return -EINVAL;
  673. list_for_each_entry(chain, &cp->ccwchain_list, next) {
  674. len = chain->ch_len;
  675. for (idx = 0; idx < len; idx++) {
  676. ret = ccwchain_fetch_one(chain, idx, cp);
  677. if (ret)
  678. goto out_err;
  679. }
  680. }
  681. return 0;
  682. out_err:
  683. /* Only cleanup the chain elements that were actually translated. */
  684. chain->ch_len = idx;
  685. list_for_each_entry_continue(chain, &cp->ccwchain_list, next) {
  686. chain->ch_len = 0;
  687. }
  688. return ret;
  689. }
  690. /**
  691. * cp_get_orb() - get the orb of the channel program
  692. * @cp: channel_program on which to perform the operation
  693. * @intparm: new intparm for the returned orb
  694. * @lpm: candidate value of the logical-path mask for the returned orb
  695. *
  696. * This function returns the address of the updated orb of the channel
  697. * program. Channel I/O device drivers could use this orb to issue a
  698. * ssch.
  699. */
  700. union orb *cp_get_orb(struct channel_program *cp, u32 intparm, u8 lpm)
  701. {
  702. union orb *orb;
  703. struct ccwchain *chain;
  704. struct ccw1 *cpa;
  705. /* this is an error in the caller */
  706. if (!cp->initialized)
  707. return NULL;
  708. orb = &cp->orb;
  709. orb->cmd.intparm = intparm;
  710. orb->cmd.fmt = 1;
  711. orb->cmd.key = PAGE_DEFAULT_KEY >> 4;
  712. if (orb->cmd.lpm == 0)
  713. orb->cmd.lpm = lpm;
  714. chain = list_first_entry(&cp->ccwchain_list, struct ccwchain, next);
  715. cpa = chain->ch_ccw;
  716. orb->cmd.cpa = (__u32) __pa(cpa);
  717. return orb;
  718. }
  719. /**
  720. * cp_update_scsw() - update scsw for a channel program.
  721. * @cp: channel_program on which to perform the operation
  722. * @scsw: I/O results of the channel program and also the target to be
  723. * updated
  724. *
  725. * @scsw contains the I/O results of the channel program that pointed
  726. * to by @cp. However what @scsw->cpa stores is a host physical
  727. * address, which is meaningless for the guest, which is waiting for
  728. * the I/O results.
  729. *
  730. * This function updates @scsw->cpa to its coressponding guest physical
  731. * address.
  732. */
  733. void cp_update_scsw(struct channel_program *cp, union scsw *scsw)
  734. {
  735. struct ccwchain *chain;
  736. u32 cpa = scsw->cmd.cpa;
  737. u32 ccw_head;
  738. if (!cp->initialized)
  739. return;
  740. /*
  741. * LATER:
  742. * For now, only update the cmd.cpa part. We may need to deal with
  743. * other portions of the schib as well, even if we don't return them
  744. * in the ioctl directly. Path status changes etc.
  745. */
  746. list_for_each_entry(chain, &cp->ccwchain_list, next) {
  747. ccw_head = (u32)(u64)chain->ch_ccw;
  748. /*
  749. * On successful execution, cpa points just beyond the end
  750. * of the chain.
  751. */
  752. if (is_cpa_within_range(cpa, ccw_head, chain->ch_len + 1)) {
  753. /*
  754. * (cpa - ccw_head) is the offset value of the host
  755. * physical ccw to its chain head.
  756. * Adding this value to the guest physical ccw chain
  757. * head gets us the guest cpa.
  758. */
  759. cpa = chain->ch_iova + (cpa - ccw_head);
  760. break;
  761. }
  762. }
  763. scsw->cmd.cpa = cpa;
  764. }
  765. /**
  766. * cp_iova_pinned() - check if an iova is pinned for a ccw chain.
  767. * @cp: channel_program on which to perform the operation
  768. * @iova: the iova to check
  769. * @length: the length to check from @iova
  770. *
  771. * If the @iova is currently pinned for the ccw chain, return true;
  772. * else return false.
  773. */
  774. bool cp_iova_pinned(struct channel_program *cp, u64 iova, u64 length)
  775. {
  776. struct ccwchain *chain;
  777. int i;
  778. if (!cp->initialized)
  779. return false;
  780. list_for_each_entry(chain, &cp->ccwchain_list, next) {
  781. for (i = 0; i < chain->ch_len; i++)
  782. if (page_array_iova_pinned(chain->ch_pa + i, iova, length))
  783. return true;
  784. }
  785. return false;
  786. }