virt-pci.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2020 Intel Corporation
  4. * Author: Johannes Berg <[email protected]>
  5. */
  6. #include <linux/module.h>
  7. #include <linux/pci.h>
  8. #include <linux/virtio.h>
  9. #include <linux/virtio_config.h>
  10. #include <linux/logic_iomem.h>
  11. #include <linux/irqdomain.h>
  12. #include <linux/virtio_pcidev.h>
  13. #include <linux/virtio-uml.h>
  14. #include <linux/delay.h>
  15. #include <linux/msi.h>
  16. #include <asm/unaligned.h>
  17. #include <irq_kern.h>
  18. #define MAX_DEVICES 8
  19. #define MAX_MSI_VECTORS 32
  20. #define CFG_SPACE_SIZE 4096
  21. /* for MSI-X we have a 32-bit payload */
  22. #define MAX_IRQ_MSG_SIZE (sizeof(struct virtio_pcidev_msg) + sizeof(u32))
  23. #define NUM_IRQ_MSGS 10
  24. #define HANDLE_NO_FREE(ptr) ((void *)((unsigned long)(ptr) | 1))
  25. #define HANDLE_IS_NO_FREE(ptr) ((unsigned long)(ptr) & 1)
  26. struct um_pci_device {
  27. struct virtio_device *vdev;
  28. /* for now just standard BARs */
  29. u8 resptr[PCI_STD_NUM_BARS];
  30. struct virtqueue *cmd_vq, *irq_vq;
  31. #define UM_PCI_STAT_WAITING 0
  32. unsigned long status;
  33. int irq;
  34. };
  35. struct um_pci_device_reg {
  36. struct um_pci_device *dev;
  37. void __iomem *iomem;
  38. };
  39. static struct pci_host_bridge *bridge;
  40. static DEFINE_MUTEX(um_pci_mtx);
  41. static struct um_pci_device_reg um_pci_devices[MAX_DEVICES];
  42. static struct fwnode_handle *um_pci_fwnode;
  43. static struct irq_domain *um_pci_inner_domain;
  44. static struct irq_domain *um_pci_msi_domain;
  45. static unsigned long um_pci_msi_used[BITS_TO_LONGS(MAX_MSI_VECTORS)];
  46. #define UM_VIRT_PCI_MAXDELAY 40000
  47. struct um_pci_message_buffer {
  48. struct virtio_pcidev_msg hdr;
  49. u8 data[8];
  50. };
  51. static struct um_pci_message_buffer __percpu *um_pci_msg_bufs;
  52. static int um_pci_send_cmd(struct um_pci_device *dev,
  53. struct virtio_pcidev_msg *cmd,
  54. unsigned int cmd_size,
  55. const void *extra, unsigned int extra_size,
  56. void *out, unsigned int out_size)
  57. {
  58. struct scatterlist out_sg, extra_sg, in_sg;
  59. struct scatterlist *sgs_list[] = {
  60. [0] = &out_sg,
  61. [1] = extra ? &extra_sg : &in_sg,
  62. [2] = extra ? &in_sg : NULL,
  63. };
  64. struct um_pci_message_buffer *buf;
  65. int delay_count = 0;
  66. int ret, len;
  67. bool posted;
  68. if (WARN_ON(cmd_size < sizeof(*cmd) || cmd_size > sizeof(*buf)))
  69. return -EINVAL;
  70. switch (cmd->op) {
  71. case VIRTIO_PCIDEV_OP_CFG_WRITE:
  72. case VIRTIO_PCIDEV_OP_MMIO_WRITE:
  73. case VIRTIO_PCIDEV_OP_MMIO_MEMSET:
  74. /* in PCI, writes are posted, so don't wait */
  75. posted = !out;
  76. WARN_ON(!posted);
  77. break;
  78. default:
  79. posted = false;
  80. break;
  81. }
  82. buf = get_cpu_var(um_pci_msg_bufs);
  83. if (buf)
  84. memcpy(buf, cmd, cmd_size);
  85. if (posted) {
  86. u8 *ncmd = kmalloc(cmd_size + extra_size, GFP_ATOMIC);
  87. if (ncmd) {
  88. memcpy(ncmd, cmd, cmd_size);
  89. if (extra)
  90. memcpy(ncmd + cmd_size, extra, extra_size);
  91. cmd = (void *)ncmd;
  92. cmd_size += extra_size;
  93. extra = NULL;
  94. extra_size = 0;
  95. } else {
  96. /* try without allocating memory */
  97. posted = false;
  98. cmd = (void *)buf;
  99. }
  100. } else {
  101. cmd = (void *)buf;
  102. }
  103. sg_init_one(&out_sg, cmd, cmd_size);
  104. if (extra)
  105. sg_init_one(&extra_sg, extra, extra_size);
  106. if (out)
  107. sg_init_one(&in_sg, out, out_size);
  108. /* add to internal virtio queue */
  109. ret = virtqueue_add_sgs(dev->cmd_vq, sgs_list,
  110. extra ? 2 : 1,
  111. out ? 1 : 0,
  112. posted ? cmd : HANDLE_NO_FREE(cmd),
  113. GFP_ATOMIC);
  114. if (ret) {
  115. if (posted)
  116. kfree(cmd);
  117. goto out;
  118. }
  119. if (posted) {
  120. virtqueue_kick(dev->cmd_vq);
  121. ret = 0;
  122. goto out;
  123. }
  124. /* kick and poll for getting a response on the queue */
  125. set_bit(UM_PCI_STAT_WAITING, &dev->status);
  126. virtqueue_kick(dev->cmd_vq);
  127. while (1) {
  128. void *completed = virtqueue_get_buf(dev->cmd_vq, &len);
  129. if (completed == HANDLE_NO_FREE(cmd))
  130. break;
  131. if (completed && !HANDLE_IS_NO_FREE(completed))
  132. kfree(completed);
  133. if (WARN_ONCE(virtqueue_is_broken(dev->cmd_vq) ||
  134. ++delay_count > UM_VIRT_PCI_MAXDELAY,
  135. "um virt-pci delay: %d", delay_count)) {
  136. ret = -EIO;
  137. break;
  138. }
  139. udelay(1);
  140. }
  141. clear_bit(UM_PCI_STAT_WAITING, &dev->status);
  142. out:
  143. put_cpu_var(um_pci_msg_bufs);
  144. return ret;
  145. }
  146. static unsigned long um_pci_cfgspace_read(void *priv, unsigned int offset,
  147. int size)
  148. {
  149. struct um_pci_device_reg *reg = priv;
  150. struct um_pci_device *dev = reg->dev;
  151. struct virtio_pcidev_msg hdr = {
  152. .op = VIRTIO_PCIDEV_OP_CFG_READ,
  153. .size = size,
  154. .addr = offset,
  155. };
  156. /* buf->data is maximum size - we may only use parts of it */
  157. struct um_pci_message_buffer *buf;
  158. u8 *data;
  159. unsigned long ret = ULONG_MAX;
  160. size_t bytes = sizeof(buf->data);
  161. if (!dev)
  162. return ULONG_MAX;
  163. buf = get_cpu_var(um_pci_msg_bufs);
  164. data = buf->data;
  165. if (buf)
  166. memset(data, 0xff, bytes);
  167. switch (size) {
  168. case 1:
  169. case 2:
  170. case 4:
  171. #ifdef CONFIG_64BIT
  172. case 8:
  173. #endif
  174. break;
  175. default:
  176. WARN(1, "invalid config space read size %d\n", size);
  177. goto out;
  178. }
  179. if (um_pci_send_cmd(dev, &hdr, sizeof(hdr), NULL, 0, data, bytes))
  180. goto out;
  181. switch (size) {
  182. case 1:
  183. ret = data[0];
  184. break;
  185. case 2:
  186. ret = le16_to_cpup((void *)data);
  187. break;
  188. case 4:
  189. ret = le32_to_cpup((void *)data);
  190. break;
  191. #ifdef CONFIG_64BIT
  192. case 8:
  193. ret = le64_to_cpup((void *)data);
  194. break;
  195. #endif
  196. default:
  197. break;
  198. }
  199. out:
  200. put_cpu_var(um_pci_msg_bufs);
  201. return ret;
  202. }
  203. static void um_pci_cfgspace_write(void *priv, unsigned int offset, int size,
  204. unsigned long val)
  205. {
  206. struct um_pci_device_reg *reg = priv;
  207. struct um_pci_device *dev = reg->dev;
  208. struct {
  209. struct virtio_pcidev_msg hdr;
  210. /* maximum size - we may only use parts of it */
  211. u8 data[8];
  212. } msg = {
  213. .hdr = {
  214. .op = VIRTIO_PCIDEV_OP_CFG_WRITE,
  215. .size = size,
  216. .addr = offset,
  217. },
  218. };
  219. if (!dev)
  220. return;
  221. switch (size) {
  222. case 1:
  223. msg.data[0] = (u8)val;
  224. break;
  225. case 2:
  226. put_unaligned_le16(val, (void *)msg.data);
  227. break;
  228. case 4:
  229. put_unaligned_le32(val, (void *)msg.data);
  230. break;
  231. #ifdef CONFIG_64BIT
  232. case 8:
  233. put_unaligned_le64(val, (void *)msg.data);
  234. break;
  235. #endif
  236. default:
  237. WARN(1, "invalid config space write size %d\n", size);
  238. return;
  239. }
  240. WARN_ON(um_pci_send_cmd(dev, &msg.hdr, sizeof(msg), NULL, 0, NULL, 0));
  241. }
  242. static const struct logic_iomem_ops um_pci_device_cfgspace_ops = {
  243. .read = um_pci_cfgspace_read,
  244. .write = um_pci_cfgspace_write,
  245. };
  246. static void um_pci_bar_copy_from(void *priv, void *buffer,
  247. unsigned int offset, int size)
  248. {
  249. u8 *resptr = priv;
  250. struct um_pci_device *dev = container_of(resptr - *resptr,
  251. struct um_pci_device,
  252. resptr[0]);
  253. struct virtio_pcidev_msg hdr = {
  254. .op = VIRTIO_PCIDEV_OP_MMIO_READ,
  255. .bar = *resptr,
  256. .size = size,
  257. .addr = offset,
  258. };
  259. memset(buffer, 0xff, size);
  260. um_pci_send_cmd(dev, &hdr, sizeof(hdr), NULL, 0, buffer, size);
  261. }
  262. static unsigned long um_pci_bar_read(void *priv, unsigned int offset,
  263. int size)
  264. {
  265. /* buf->data is maximum size - we may only use parts of it */
  266. struct um_pci_message_buffer *buf;
  267. u8 *data;
  268. unsigned long ret = ULONG_MAX;
  269. buf = get_cpu_var(um_pci_msg_bufs);
  270. data = buf->data;
  271. switch (size) {
  272. case 1:
  273. case 2:
  274. case 4:
  275. #ifdef CONFIG_64BIT
  276. case 8:
  277. #endif
  278. break;
  279. default:
  280. WARN(1, "invalid config space read size %d\n", size);
  281. goto out;
  282. }
  283. um_pci_bar_copy_from(priv, data, offset, size);
  284. switch (size) {
  285. case 1:
  286. ret = data[0];
  287. break;
  288. case 2:
  289. ret = le16_to_cpup((void *)data);
  290. break;
  291. case 4:
  292. ret = le32_to_cpup((void *)data);
  293. break;
  294. #ifdef CONFIG_64BIT
  295. case 8:
  296. ret = le64_to_cpup((void *)data);
  297. break;
  298. #endif
  299. default:
  300. break;
  301. }
  302. out:
  303. put_cpu_var(um_pci_msg_bufs);
  304. return ret;
  305. }
  306. static void um_pci_bar_copy_to(void *priv, unsigned int offset,
  307. const void *buffer, int size)
  308. {
  309. u8 *resptr = priv;
  310. struct um_pci_device *dev = container_of(resptr - *resptr,
  311. struct um_pci_device,
  312. resptr[0]);
  313. struct virtio_pcidev_msg hdr = {
  314. .op = VIRTIO_PCIDEV_OP_MMIO_WRITE,
  315. .bar = *resptr,
  316. .size = size,
  317. .addr = offset,
  318. };
  319. um_pci_send_cmd(dev, &hdr, sizeof(hdr), buffer, size, NULL, 0);
  320. }
  321. static void um_pci_bar_write(void *priv, unsigned int offset, int size,
  322. unsigned long val)
  323. {
  324. /* maximum size - we may only use parts of it */
  325. u8 data[8];
  326. switch (size) {
  327. case 1:
  328. data[0] = (u8)val;
  329. break;
  330. case 2:
  331. put_unaligned_le16(val, (void *)data);
  332. break;
  333. case 4:
  334. put_unaligned_le32(val, (void *)data);
  335. break;
  336. #ifdef CONFIG_64BIT
  337. case 8:
  338. put_unaligned_le64(val, (void *)data);
  339. break;
  340. #endif
  341. default:
  342. WARN(1, "invalid config space write size %d\n", size);
  343. return;
  344. }
  345. um_pci_bar_copy_to(priv, offset, data, size);
  346. }
  347. static void um_pci_bar_set(void *priv, unsigned int offset, u8 value, int size)
  348. {
  349. u8 *resptr = priv;
  350. struct um_pci_device *dev = container_of(resptr - *resptr,
  351. struct um_pci_device,
  352. resptr[0]);
  353. struct {
  354. struct virtio_pcidev_msg hdr;
  355. u8 data;
  356. } msg = {
  357. .hdr = {
  358. .op = VIRTIO_PCIDEV_OP_CFG_WRITE,
  359. .bar = *resptr,
  360. .size = size,
  361. .addr = offset,
  362. },
  363. .data = value,
  364. };
  365. um_pci_send_cmd(dev, &msg.hdr, sizeof(msg), NULL, 0, NULL, 0);
  366. }
  367. static const struct logic_iomem_ops um_pci_device_bar_ops = {
  368. .read = um_pci_bar_read,
  369. .write = um_pci_bar_write,
  370. .set = um_pci_bar_set,
  371. .copy_from = um_pci_bar_copy_from,
  372. .copy_to = um_pci_bar_copy_to,
  373. };
  374. static void __iomem *um_pci_map_bus(struct pci_bus *bus, unsigned int devfn,
  375. int where)
  376. {
  377. struct um_pci_device_reg *dev;
  378. unsigned int busn = bus->number;
  379. if (busn > 0)
  380. return NULL;
  381. /* not allowing functions for now ... */
  382. if (devfn % 8)
  383. return NULL;
  384. if (devfn / 8 >= ARRAY_SIZE(um_pci_devices))
  385. return NULL;
  386. dev = &um_pci_devices[devfn / 8];
  387. if (!dev)
  388. return NULL;
  389. return (void __iomem *)((unsigned long)dev->iomem + where);
  390. }
  391. static struct pci_ops um_pci_ops = {
  392. .map_bus = um_pci_map_bus,
  393. .read = pci_generic_config_read,
  394. .write = pci_generic_config_write,
  395. };
  396. static void um_pci_rescan(void)
  397. {
  398. pci_lock_rescan_remove();
  399. pci_rescan_bus(bridge->bus);
  400. pci_unlock_rescan_remove();
  401. }
  402. static void um_pci_irq_vq_addbuf(struct virtqueue *vq, void *buf, bool kick)
  403. {
  404. struct scatterlist sg[1];
  405. sg_init_one(sg, buf, MAX_IRQ_MSG_SIZE);
  406. if (virtqueue_add_inbuf(vq, sg, 1, buf, GFP_ATOMIC))
  407. kfree(buf);
  408. else if (kick)
  409. virtqueue_kick(vq);
  410. }
  411. static void um_pci_handle_irq_message(struct virtqueue *vq,
  412. struct virtio_pcidev_msg *msg)
  413. {
  414. struct virtio_device *vdev = vq->vdev;
  415. struct um_pci_device *dev = vdev->priv;
  416. /* we should properly chain interrupts, but on ARCH=um we don't care */
  417. switch (msg->op) {
  418. case VIRTIO_PCIDEV_OP_INT:
  419. generic_handle_irq(dev->irq);
  420. break;
  421. case VIRTIO_PCIDEV_OP_MSI:
  422. /* our MSI message is just the interrupt number */
  423. if (msg->size == sizeof(u32))
  424. generic_handle_irq(le32_to_cpup((void *)msg->data));
  425. else
  426. generic_handle_irq(le16_to_cpup((void *)msg->data));
  427. break;
  428. case VIRTIO_PCIDEV_OP_PME:
  429. /* nothing to do - we already woke up due to the message */
  430. break;
  431. default:
  432. dev_err(&vdev->dev, "unexpected virt-pci message %d\n", msg->op);
  433. break;
  434. }
  435. }
  436. static void um_pci_cmd_vq_cb(struct virtqueue *vq)
  437. {
  438. struct virtio_device *vdev = vq->vdev;
  439. struct um_pci_device *dev = vdev->priv;
  440. void *cmd;
  441. int len;
  442. if (test_bit(UM_PCI_STAT_WAITING, &dev->status))
  443. return;
  444. while ((cmd = virtqueue_get_buf(vq, &len))) {
  445. if (WARN_ON(HANDLE_IS_NO_FREE(cmd)))
  446. continue;
  447. kfree(cmd);
  448. }
  449. }
  450. static void um_pci_irq_vq_cb(struct virtqueue *vq)
  451. {
  452. struct virtio_pcidev_msg *msg;
  453. int len;
  454. while ((msg = virtqueue_get_buf(vq, &len))) {
  455. if (len >= sizeof(*msg))
  456. um_pci_handle_irq_message(vq, msg);
  457. /* recycle the message buffer */
  458. um_pci_irq_vq_addbuf(vq, msg, true);
  459. }
  460. }
  461. static int um_pci_init_vqs(struct um_pci_device *dev)
  462. {
  463. struct virtqueue *vqs[2];
  464. static const char *const names[2] = { "cmd", "irq" };
  465. vq_callback_t *cbs[2] = { um_pci_cmd_vq_cb, um_pci_irq_vq_cb };
  466. int err, i;
  467. err = virtio_find_vqs(dev->vdev, 2, vqs, cbs, names, NULL);
  468. if (err)
  469. return err;
  470. dev->cmd_vq = vqs[0];
  471. dev->irq_vq = vqs[1];
  472. virtio_device_ready(dev->vdev);
  473. for (i = 0; i < NUM_IRQ_MSGS; i++) {
  474. void *msg = kzalloc(MAX_IRQ_MSG_SIZE, GFP_KERNEL);
  475. if (msg)
  476. um_pci_irq_vq_addbuf(dev->irq_vq, msg, false);
  477. }
  478. virtqueue_kick(dev->irq_vq);
  479. return 0;
  480. }
  481. static int um_pci_virtio_probe(struct virtio_device *vdev)
  482. {
  483. struct um_pci_device *dev;
  484. int i, free = -1;
  485. int err = -ENOSPC;
  486. dev = kzalloc(sizeof(*dev), GFP_KERNEL);
  487. if (!dev)
  488. return -ENOMEM;
  489. dev->vdev = vdev;
  490. vdev->priv = dev;
  491. mutex_lock(&um_pci_mtx);
  492. for (i = 0; i < MAX_DEVICES; i++) {
  493. if (um_pci_devices[i].dev)
  494. continue;
  495. free = i;
  496. break;
  497. }
  498. if (free < 0)
  499. goto error;
  500. err = um_pci_init_vqs(dev);
  501. if (err)
  502. goto error;
  503. dev->irq = irq_alloc_desc(numa_node_id());
  504. if (dev->irq < 0) {
  505. err = dev->irq;
  506. goto err_reset;
  507. }
  508. um_pci_devices[free].dev = dev;
  509. vdev->priv = dev;
  510. mutex_unlock(&um_pci_mtx);
  511. device_set_wakeup_enable(&vdev->dev, true);
  512. /*
  513. * In order to do suspend-resume properly, don't allow VQs
  514. * to be suspended.
  515. */
  516. virtio_uml_set_no_vq_suspend(vdev, true);
  517. um_pci_rescan();
  518. return 0;
  519. err_reset:
  520. virtio_reset_device(vdev);
  521. vdev->config->del_vqs(vdev);
  522. error:
  523. mutex_unlock(&um_pci_mtx);
  524. kfree(dev);
  525. return err;
  526. }
  527. static void um_pci_virtio_remove(struct virtio_device *vdev)
  528. {
  529. struct um_pci_device *dev = vdev->priv;
  530. int i;
  531. device_set_wakeup_enable(&vdev->dev, false);
  532. mutex_lock(&um_pci_mtx);
  533. for (i = 0; i < MAX_DEVICES; i++) {
  534. if (um_pci_devices[i].dev != dev)
  535. continue;
  536. um_pci_devices[i].dev = NULL;
  537. irq_free_desc(dev->irq);
  538. break;
  539. }
  540. mutex_unlock(&um_pci_mtx);
  541. if (i < MAX_DEVICES) {
  542. struct pci_dev *pci_dev;
  543. pci_dev = pci_get_slot(bridge->bus, i);
  544. if (pci_dev)
  545. pci_stop_and_remove_bus_device_locked(pci_dev);
  546. }
  547. /* Stop all virtqueues */
  548. virtio_reset_device(vdev);
  549. dev->cmd_vq = NULL;
  550. dev->irq_vq = NULL;
  551. vdev->config->del_vqs(vdev);
  552. kfree(dev);
  553. }
  554. static struct virtio_device_id id_table[] = {
  555. { CONFIG_UML_PCI_OVER_VIRTIO_DEVICE_ID, VIRTIO_DEV_ANY_ID },
  556. { 0 },
  557. };
  558. MODULE_DEVICE_TABLE(virtio, id_table);
  559. static struct virtio_driver um_pci_virtio_driver = {
  560. .driver.name = "virtio-pci",
  561. .driver.owner = THIS_MODULE,
  562. .id_table = id_table,
  563. .probe = um_pci_virtio_probe,
  564. .remove = um_pci_virtio_remove,
  565. };
  566. static struct resource virt_cfgspace_resource = {
  567. .name = "PCI config space",
  568. .start = 0xf0000000 - MAX_DEVICES * CFG_SPACE_SIZE,
  569. .end = 0xf0000000 - 1,
  570. .flags = IORESOURCE_MEM,
  571. };
  572. static long um_pci_map_cfgspace(unsigned long offset, size_t size,
  573. const struct logic_iomem_ops **ops,
  574. void **priv)
  575. {
  576. if (WARN_ON(size > CFG_SPACE_SIZE || offset % CFG_SPACE_SIZE))
  577. return -EINVAL;
  578. if (offset / CFG_SPACE_SIZE < MAX_DEVICES) {
  579. *ops = &um_pci_device_cfgspace_ops;
  580. *priv = &um_pci_devices[offset / CFG_SPACE_SIZE];
  581. return 0;
  582. }
  583. WARN(1, "cannot map offset 0x%lx/0x%zx\n", offset, size);
  584. return -ENOENT;
  585. }
  586. static const struct logic_iomem_region_ops um_pci_cfgspace_ops = {
  587. .map = um_pci_map_cfgspace,
  588. };
  589. static struct resource virt_iomem_resource = {
  590. .name = "PCI iomem",
  591. .start = 0xf0000000,
  592. .end = 0xffffffff,
  593. .flags = IORESOURCE_MEM,
  594. };
  595. struct um_pci_map_iomem_data {
  596. unsigned long offset;
  597. size_t size;
  598. const struct logic_iomem_ops **ops;
  599. void **priv;
  600. long ret;
  601. };
  602. static int um_pci_map_iomem_walk(struct pci_dev *pdev, void *_data)
  603. {
  604. struct um_pci_map_iomem_data *data = _data;
  605. struct um_pci_device_reg *reg = &um_pci_devices[pdev->devfn / 8];
  606. struct um_pci_device *dev;
  607. int i;
  608. if (!reg->dev)
  609. return 0;
  610. for (i = 0; i < ARRAY_SIZE(dev->resptr); i++) {
  611. struct resource *r = &pdev->resource[i];
  612. if ((r->flags & IORESOURCE_TYPE_BITS) != IORESOURCE_MEM)
  613. continue;
  614. /*
  615. * must be the whole or part of the resource,
  616. * not allowed to only overlap
  617. */
  618. if (data->offset < r->start || data->offset > r->end)
  619. continue;
  620. if (data->offset + data->size - 1 > r->end)
  621. continue;
  622. dev = reg->dev;
  623. *data->ops = &um_pci_device_bar_ops;
  624. dev->resptr[i] = i;
  625. *data->priv = &dev->resptr[i];
  626. data->ret = data->offset - r->start;
  627. /* no need to continue */
  628. return 1;
  629. }
  630. return 0;
  631. }
  632. static long um_pci_map_iomem(unsigned long offset, size_t size,
  633. const struct logic_iomem_ops **ops,
  634. void **priv)
  635. {
  636. struct um_pci_map_iomem_data data = {
  637. /* we want the full address here */
  638. .offset = offset + virt_iomem_resource.start,
  639. .size = size,
  640. .ops = ops,
  641. .priv = priv,
  642. .ret = -ENOENT,
  643. };
  644. pci_walk_bus(bridge->bus, um_pci_map_iomem_walk, &data);
  645. return data.ret;
  646. }
  647. static const struct logic_iomem_region_ops um_pci_iomem_ops = {
  648. .map = um_pci_map_iomem,
  649. };
  650. static void um_pci_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
  651. {
  652. /*
  653. * This is a very low address and not actually valid 'physical' memory
  654. * in UML, so we can simply map MSI(-X) vectors to there, it cannot be
  655. * legitimately written to by the device in any other way.
  656. * We use the (virtual) IRQ number here as the message to simplify the
  657. * code that receives the message, where for now we simply trust the
  658. * device to send the correct message.
  659. */
  660. msg->address_hi = 0;
  661. msg->address_lo = 0xa0000;
  662. msg->data = data->irq;
  663. }
  664. static struct irq_chip um_pci_msi_bottom_irq_chip = {
  665. .name = "UM virtio MSI",
  666. .irq_compose_msi_msg = um_pci_compose_msi_msg,
  667. };
  668. static int um_pci_inner_domain_alloc(struct irq_domain *domain,
  669. unsigned int virq, unsigned int nr_irqs,
  670. void *args)
  671. {
  672. unsigned long bit;
  673. WARN_ON(nr_irqs != 1);
  674. mutex_lock(&um_pci_mtx);
  675. bit = find_first_zero_bit(um_pci_msi_used, MAX_MSI_VECTORS);
  676. if (bit >= MAX_MSI_VECTORS) {
  677. mutex_unlock(&um_pci_mtx);
  678. return -ENOSPC;
  679. }
  680. set_bit(bit, um_pci_msi_used);
  681. mutex_unlock(&um_pci_mtx);
  682. irq_domain_set_info(domain, virq, bit, &um_pci_msi_bottom_irq_chip,
  683. domain->host_data, handle_simple_irq,
  684. NULL, NULL);
  685. return 0;
  686. }
  687. static void um_pci_inner_domain_free(struct irq_domain *domain,
  688. unsigned int virq, unsigned int nr_irqs)
  689. {
  690. struct irq_data *d = irq_domain_get_irq_data(domain, virq);
  691. mutex_lock(&um_pci_mtx);
  692. if (!test_bit(d->hwirq, um_pci_msi_used))
  693. pr_err("trying to free unused MSI#%lu\n", d->hwirq);
  694. else
  695. __clear_bit(d->hwirq, um_pci_msi_used);
  696. mutex_unlock(&um_pci_mtx);
  697. }
  698. static const struct irq_domain_ops um_pci_inner_domain_ops = {
  699. .alloc = um_pci_inner_domain_alloc,
  700. .free = um_pci_inner_domain_free,
  701. };
  702. static struct irq_chip um_pci_msi_irq_chip = {
  703. .name = "UM virtio PCIe MSI",
  704. .irq_mask = pci_msi_mask_irq,
  705. .irq_unmask = pci_msi_unmask_irq,
  706. };
  707. static struct msi_domain_info um_pci_msi_domain_info = {
  708. .flags = MSI_FLAG_USE_DEF_DOM_OPS |
  709. MSI_FLAG_USE_DEF_CHIP_OPS |
  710. MSI_FLAG_PCI_MSIX,
  711. .chip = &um_pci_msi_irq_chip,
  712. };
  713. static struct resource busn_resource = {
  714. .name = "PCI busn",
  715. .start = 0,
  716. .end = 0,
  717. .flags = IORESOURCE_BUS,
  718. };
  719. static int um_pci_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin)
  720. {
  721. struct um_pci_device_reg *reg = &um_pci_devices[pdev->devfn / 8];
  722. if (WARN_ON(!reg->dev))
  723. return -EINVAL;
  724. /* Yes, we map all pins to the same IRQ ... doesn't matter for now. */
  725. return reg->dev->irq;
  726. }
  727. void *pci_root_bus_fwnode(struct pci_bus *bus)
  728. {
  729. return um_pci_fwnode;
  730. }
  731. static int __init um_pci_init(void)
  732. {
  733. int err, i;
  734. WARN_ON(logic_iomem_add_region(&virt_cfgspace_resource,
  735. &um_pci_cfgspace_ops));
  736. WARN_ON(logic_iomem_add_region(&virt_iomem_resource,
  737. &um_pci_iomem_ops));
  738. if (WARN(CONFIG_UML_PCI_OVER_VIRTIO_DEVICE_ID < 0,
  739. "No virtio device ID configured for PCI - no PCI support\n"))
  740. return 0;
  741. um_pci_msg_bufs = alloc_percpu(struct um_pci_message_buffer);
  742. if (!um_pci_msg_bufs)
  743. return -ENOMEM;
  744. bridge = pci_alloc_host_bridge(0);
  745. if (!bridge) {
  746. err = -ENOMEM;
  747. goto free;
  748. }
  749. um_pci_fwnode = irq_domain_alloc_named_fwnode("um-pci");
  750. if (!um_pci_fwnode) {
  751. err = -ENOMEM;
  752. goto free;
  753. }
  754. um_pci_inner_domain = __irq_domain_add(um_pci_fwnode, MAX_MSI_VECTORS,
  755. MAX_MSI_VECTORS, 0,
  756. &um_pci_inner_domain_ops, NULL);
  757. if (!um_pci_inner_domain) {
  758. err = -ENOMEM;
  759. goto free;
  760. }
  761. um_pci_msi_domain = pci_msi_create_irq_domain(um_pci_fwnode,
  762. &um_pci_msi_domain_info,
  763. um_pci_inner_domain);
  764. if (!um_pci_msi_domain) {
  765. err = -ENOMEM;
  766. goto free;
  767. }
  768. pci_add_resource(&bridge->windows, &virt_iomem_resource);
  769. pci_add_resource(&bridge->windows, &busn_resource);
  770. bridge->ops = &um_pci_ops;
  771. bridge->map_irq = um_pci_map_irq;
  772. for (i = 0; i < MAX_DEVICES; i++) {
  773. resource_size_t start;
  774. start = virt_cfgspace_resource.start + i * CFG_SPACE_SIZE;
  775. um_pci_devices[i].iomem = ioremap(start, CFG_SPACE_SIZE);
  776. if (WARN(!um_pci_devices[i].iomem, "failed to map %d\n", i)) {
  777. err = -ENOMEM;
  778. goto free;
  779. }
  780. }
  781. err = pci_host_probe(bridge);
  782. if (err)
  783. goto free;
  784. err = register_virtio_driver(&um_pci_virtio_driver);
  785. if (err)
  786. goto free;
  787. return 0;
  788. free:
  789. if (um_pci_inner_domain)
  790. irq_domain_remove(um_pci_inner_domain);
  791. if (um_pci_fwnode)
  792. irq_domain_free_fwnode(um_pci_fwnode);
  793. if (bridge) {
  794. pci_free_resource_list(&bridge->windows);
  795. pci_free_host_bridge(bridge);
  796. }
  797. free_percpu(um_pci_msg_bufs);
  798. return err;
  799. }
  800. module_init(um_pci_init);
  801. static void __exit um_pci_exit(void)
  802. {
  803. unregister_virtio_driver(&um_pci_virtio_driver);
  804. irq_domain_remove(um_pci_msi_domain);
  805. irq_domain_remove(um_pci_inner_domain);
  806. pci_free_resource_list(&bridge->windows);
  807. pci_free_host_bridge(bridge);
  808. free_percpu(um_pci_msg_bufs);
  809. }
  810. module_exit(um_pci_exit);