virtio_vdpa.c 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * VIRTIO based driver for vDPA device
  4. *
  5. * Copyright (c) 2020, Red Hat. All rights reserved.
  6. * Author: Jason Wang <[email protected]>
  7. *
  8. */
  9. #include <linux/init.h>
  10. #include <linux/module.h>
  11. #include <linux/device.h>
  12. #include <linux/kernel.h>
  13. #include <linux/slab.h>
  14. #include <linux/uuid.h>
  15. #include <linux/virtio.h>
  16. #include <linux/vdpa.h>
  17. #include <linux/virtio_config.h>
  18. #include <linux/virtio_ring.h>
  19. #define MOD_VERSION "0.1"
  20. #define MOD_AUTHOR "Jason Wang <[email protected]>"
  21. #define MOD_DESC "vDPA bus driver for virtio devices"
  22. #define MOD_LICENSE "GPL v2"
  23. struct virtio_vdpa_device {
  24. struct virtio_device vdev;
  25. struct vdpa_device *vdpa;
  26. u64 features;
  27. /* The lock to protect virtqueue list */
  28. spinlock_t lock;
  29. /* List of virtio_vdpa_vq_info */
  30. struct list_head virtqueues;
  31. };
  32. struct virtio_vdpa_vq_info {
  33. /* the actual virtqueue */
  34. struct virtqueue *vq;
  35. /* the list node for the virtqueues list */
  36. struct list_head node;
  37. };
  38. static inline struct virtio_vdpa_device *
  39. to_virtio_vdpa_device(struct virtio_device *dev)
  40. {
  41. return container_of(dev, struct virtio_vdpa_device, vdev);
  42. }
  43. static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev)
  44. {
  45. return to_virtio_vdpa_device(vdev)->vdpa;
  46. }
  47. static void virtio_vdpa_get(struct virtio_device *vdev, unsigned int offset,
  48. void *buf, unsigned int len)
  49. {
  50. struct vdpa_device *vdpa = vd_get_vdpa(vdev);
  51. vdpa_get_config(vdpa, offset, buf, len);
  52. }
  53. static void virtio_vdpa_set(struct virtio_device *vdev, unsigned int offset,
  54. const void *buf, unsigned int len)
  55. {
  56. struct vdpa_device *vdpa = vd_get_vdpa(vdev);
  57. vdpa_set_config(vdpa, offset, buf, len);
  58. }
  59. static u32 virtio_vdpa_generation(struct virtio_device *vdev)
  60. {
  61. struct vdpa_device *vdpa = vd_get_vdpa(vdev);
  62. const struct vdpa_config_ops *ops = vdpa->config;
  63. if (ops->get_generation)
  64. return ops->get_generation(vdpa);
  65. return 0;
  66. }
  67. static u8 virtio_vdpa_get_status(struct virtio_device *vdev)
  68. {
  69. struct vdpa_device *vdpa = vd_get_vdpa(vdev);
  70. const struct vdpa_config_ops *ops = vdpa->config;
  71. return ops->get_status(vdpa);
  72. }
  73. static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status)
  74. {
  75. struct vdpa_device *vdpa = vd_get_vdpa(vdev);
  76. return vdpa_set_status(vdpa, status);
  77. }
  78. static void virtio_vdpa_reset(struct virtio_device *vdev)
  79. {
  80. struct vdpa_device *vdpa = vd_get_vdpa(vdev);
  81. vdpa_reset(vdpa);
  82. }
  83. static bool virtio_vdpa_notify(struct virtqueue *vq)
  84. {
  85. struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev);
  86. const struct vdpa_config_ops *ops = vdpa->config;
  87. ops->kick_vq(vdpa, vq->index);
  88. return true;
  89. }
  90. static irqreturn_t virtio_vdpa_config_cb(void *private)
  91. {
  92. struct virtio_vdpa_device *vd_dev = private;
  93. virtio_config_changed(&vd_dev->vdev);
  94. return IRQ_HANDLED;
  95. }
  96. static irqreturn_t virtio_vdpa_virtqueue_cb(void *private)
  97. {
  98. struct virtio_vdpa_vq_info *info = private;
  99. return vring_interrupt(0, info->vq);
  100. }
  101. static struct virtqueue *
  102. virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
  103. void (*callback)(struct virtqueue *vq),
  104. const char *name, bool ctx)
  105. {
  106. struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
  107. struct vdpa_device *vdpa = vd_get_vdpa(vdev);
  108. const struct vdpa_config_ops *ops = vdpa->config;
  109. struct virtio_vdpa_vq_info *info;
  110. struct vdpa_callback cb;
  111. struct virtqueue *vq;
  112. u64 desc_addr, driver_addr, device_addr;
  113. /* Assume split virtqueue, switch to packed if necessary */
  114. struct vdpa_vq_state state = {0};
  115. unsigned long flags;
  116. u32 align, max_num, min_num = 1;
  117. bool may_reduce_num = true;
  118. int err;
  119. if (!name)
  120. return NULL;
  121. if (index >= vdpa->nvqs)
  122. return ERR_PTR(-ENOENT);
  123. /* Queue shouldn't already be set up. */
  124. if (ops->get_vq_ready(vdpa, index))
  125. return ERR_PTR(-ENOENT);
  126. /* Allocate and fill out our active queue description */
  127. info = kmalloc(sizeof(*info), GFP_KERNEL);
  128. if (!info)
  129. return ERR_PTR(-ENOMEM);
  130. max_num = ops->get_vq_num_max(vdpa);
  131. if (max_num == 0) {
  132. err = -ENOENT;
  133. goto error_new_virtqueue;
  134. }
  135. if (ops->get_vq_num_min)
  136. min_num = ops->get_vq_num_min(vdpa);
  137. may_reduce_num = (max_num == min_num) ? false : true;
  138. /* Create the vring */
  139. align = ops->get_vq_align(vdpa);
  140. vq = vring_create_virtqueue(index, max_num, align, vdev,
  141. true, may_reduce_num, ctx,
  142. virtio_vdpa_notify, callback, name);
  143. if (!vq) {
  144. err = -ENOMEM;
  145. goto error_new_virtqueue;
  146. }
  147. vq->num_max = max_num;
  148. /* Setup virtqueue callback */
  149. cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL;
  150. cb.private = info;
  151. ops->set_vq_cb(vdpa, index, &cb);
  152. ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq));
  153. desc_addr = virtqueue_get_desc_addr(vq);
  154. driver_addr = virtqueue_get_avail_addr(vq);
  155. device_addr = virtqueue_get_used_addr(vq);
  156. if (ops->set_vq_address(vdpa, index,
  157. desc_addr, driver_addr,
  158. device_addr)) {
  159. err = -EINVAL;
  160. goto err_vq;
  161. }
  162. /* reset virtqueue state index */
  163. if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
  164. struct vdpa_vq_state_packed *s = &state.packed;
  165. s->last_avail_counter = 1;
  166. s->last_avail_idx = 0;
  167. s->last_used_counter = 1;
  168. s->last_used_idx = 0;
  169. }
  170. err = ops->set_vq_state(vdpa, index, &state);
  171. if (err)
  172. goto err_vq;
  173. ops->set_vq_ready(vdpa, index, 1);
  174. vq->priv = info;
  175. info->vq = vq;
  176. spin_lock_irqsave(&vd_dev->lock, flags);
  177. list_add(&info->node, &vd_dev->virtqueues);
  178. spin_unlock_irqrestore(&vd_dev->lock, flags);
  179. return vq;
  180. err_vq:
  181. vring_del_virtqueue(vq);
  182. error_new_virtqueue:
  183. ops->set_vq_ready(vdpa, index, 0);
  184. /* VDPA driver should make sure vq is stopeed here */
  185. WARN_ON(ops->get_vq_ready(vdpa, index));
  186. kfree(info);
  187. return ERR_PTR(err);
  188. }
  189. static void virtio_vdpa_del_vq(struct virtqueue *vq)
  190. {
  191. struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev);
  192. struct vdpa_device *vdpa = vd_dev->vdpa;
  193. const struct vdpa_config_ops *ops = vdpa->config;
  194. struct virtio_vdpa_vq_info *info = vq->priv;
  195. unsigned int index = vq->index;
  196. unsigned long flags;
  197. spin_lock_irqsave(&vd_dev->lock, flags);
  198. list_del(&info->node);
  199. spin_unlock_irqrestore(&vd_dev->lock, flags);
  200. /* Select and deactivate the queue (best effort) */
  201. ops->set_vq_ready(vdpa, index, 0);
  202. vring_del_virtqueue(vq);
  203. kfree(info);
  204. }
  205. static void virtio_vdpa_del_vqs(struct virtio_device *vdev)
  206. {
  207. struct virtqueue *vq, *n;
  208. list_for_each_entry_safe(vq, n, &vdev->vqs, list)
  209. virtio_vdpa_del_vq(vq);
  210. }
  211. static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
  212. struct virtqueue *vqs[],
  213. vq_callback_t *callbacks[],
  214. const char * const names[],
  215. const bool *ctx,
  216. struct irq_affinity *desc)
  217. {
  218. struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
  219. struct vdpa_device *vdpa = vd_get_vdpa(vdev);
  220. const struct vdpa_config_ops *ops = vdpa->config;
  221. struct vdpa_callback cb;
  222. int i, err, queue_idx = 0;
  223. for (i = 0; i < nvqs; ++i) {
  224. if (!names[i]) {
  225. vqs[i] = NULL;
  226. continue;
  227. }
  228. vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++,
  229. callbacks[i], names[i], ctx ?
  230. ctx[i] : false);
  231. if (IS_ERR(vqs[i])) {
  232. err = PTR_ERR(vqs[i]);
  233. goto err_setup_vq;
  234. }
  235. }
  236. cb.callback = virtio_vdpa_config_cb;
  237. cb.private = vd_dev;
  238. ops->set_config_cb(vdpa, &cb);
  239. return 0;
  240. err_setup_vq:
  241. virtio_vdpa_del_vqs(vdev);
  242. return err;
  243. }
  244. static u64 virtio_vdpa_get_features(struct virtio_device *vdev)
  245. {
  246. struct vdpa_device *vdpa = vd_get_vdpa(vdev);
  247. const struct vdpa_config_ops *ops = vdpa->config;
  248. return ops->get_device_features(vdpa);
  249. }
  250. static int virtio_vdpa_finalize_features(struct virtio_device *vdev)
  251. {
  252. struct vdpa_device *vdpa = vd_get_vdpa(vdev);
  253. /* Give virtio_ring a chance to accept features. */
  254. vring_transport_features(vdev);
  255. return vdpa_set_features(vdpa, vdev->features);
  256. }
  257. static const char *virtio_vdpa_bus_name(struct virtio_device *vdev)
  258. {
  259. struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
  260. struct vdpa_device *vdpa = vd_dev->vdpa;
  261. return dev_name(&vdpa->dev);
  262. }
  263. static const struct virtio_config_ops virtio_vdpa_config_ops = {
  264. .get = virtio_vdpa_get,
  265. .set = virtio_vdpa_set,
  266. .generation = virtio_vdpa_generation,
  267. .get_status = virtio_vdpa_get_status,
  268. .set_status = virtio_vdpa_set_status,
  269. .reset = virtio_vdpa_reset,
  270. .find_vqs = virtio_vdpa_find_vqs,
  271. .del_vqs = virtio_vdpa_del_vqs,
  272. .get_features = virtio_vdpa_get_features,
  273. .finalize_features = virtio_vdpa_finalize_features,
  274. .bus_name = virtio_vdpa_bus_name,
  275. };
  276. static void virtio_vdpa_release_dev(struct device *_d)
  277. {
  278. struct virtio_device *vdev =
  279. container_of(_d, struct virtio_device, dev);
  280. struct virtio_vdpa_device *vd_dev =
  281. container_of(vdev, struct virtio_vdpa_device, vdev);
  282. kfree(vd_dev);
  283. }
  284. static int virtio_vdpa_probe(struct vdpa_device *vdpa)
  285. {
  286. const struct vdpa_config_ops *ops = vdpa->config;
  287. struct virtio_vdpa_device *vd_dev, *reg_dev = NULL;
  288. int ret = -EINVAL;
  289. vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL);
  290. if (!vd_dev)
  291. return -ENOMEM;
  292. vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdpa);
  293. vd_dev->vdev.dev.release = virtio_vdpa_release_dev;
  294. vd_dev->vdev.config = &virtio_vdpa_config_ops;
  295. vd_dev->vdpa = vdpa;
  296. INIT_LIST_HEAD(&vd_dev->virtqueues);
  297. spin_lock_init(&vd_dev->lock);
  298. vd_dev->vdev.id.device = ops->get_device_id(vdpa);
  299. if (vd_dev->vdev.id.device == 0)
  300. goto err;
  301. vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa);
  302. ret = register_virtio_device(&vd_dev->vdev);
  303. reg_dev = vd_dev;
  304. if (ret)
  305. goto err;
  306. vdpa_set_drvdata(vdpa, vd_dev);
  307. return 0;
  308. err:
  309. if (reg_dev)
  310. put_device(&vd_dev->vdev.dev);
  311. else
  312. kfree(vd_dev);
  313. return ret;
  314. }
  315. static void virtio_vdpa_remove(struct vdpa_device *vdpa)
  316. {
  317. struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdpa);
  318. unregister_virtio_device(&vd_dev->vdev);
  319. }
  320. static struct vdpa_driver virtio_vdpa_driver = {
  321. .driver = {
  322. .name = "virtio_vdpa",
  323. },
  324. .probe = virtio_vdpa_probe,
  325. .remove = virtio_vdpa_remove,
  326. };
  327. module_vdpa_driver(virtio_vdpa_driver);
  328. MODULE_VERSION(MOD_VERSION);
  329. MODULE_LICENSE(MOD_LICENSE);
  330. MODULE_AUTHOR(MOD_AUTHOR);
  331. MODULE_DESCRIPTION(MOD_DESC);