virtio_scsi.c 27 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Virtio SCSI HBA driver
  4. *
  5. * Copyright IBM Corp. 2010
  6. * Copyright Red Hat, Inc. 2011
  7. *
  8. * Authors:
  9. * Stefan Hajnoczi <[email protected]>
  10. * Paolo Bonzini <[email protected]>
  11. */
  12. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  13. #include <linux/module.h>
  14. #include <linux/slab.h>
  15. #include <linux/mempool.h>
  16. #include <linux/interrupt.h>
  17. #include <linux/virtio.h>
  18. #include <linux/virtio_ids.h>
  19. #include <linux/virtio_config.h>
  20. #include <linux/virtio_scsi.h>
  21. #include <linux/cpu.h>
  22. #include <linux/blkdev.h>
  23. #include <linux/blk-integrity.h>
  24. #include <scsi/scsi_host.h>
  25. #include <scsi/scsi_device.h>
  26. #include <scsi/scsi_cmnd.h>
  27. #include <scsi/scsi_tcq.h>
  28. #include <scsi/scsi_devinfo.h>
  29. #include <linux/seqlock.h>
  30. #include <linux/blk-mq-virtio.h>
  31. #include "sd.h"
  32. #define VIRTIO_SCSI_MEMPOOL_SZ 64
  33. #define VIRTIO_SCSI_EVENT_LEN 8
  34. #define VIRTIO_SCSI_VQ_BASE 2
  35. /* Command queue element */
  36. struct virtio_scsi_cmd {
  37. struct scsi_cmnd *sc;
  38. struct completion *comp;
  39. union {
  40. struct virtio_scsi_cmd_req cmd;
  41. struct virtio_scsi_cmd_req_pi cmd_pi;
  42. struct virtio_scsi_ctrl_tmf_req tmf;
  43. struct virtio_scsi_ctrl_an_req an;
  44. } req;
  45. union {
  46. struct virtio_scsi_cmd_resp cmd;
  47. struct virtio_scsi_ctrl_tmf_resp tmf;
  48. struct virtio_scsi_ctrl_an_resp an;
  49. struct virtio_scsi_event evt;
  50. } resp;
  51. } ____cacheline_aligned_in_smp;
  52. struct virtio_scsi_event_node {
  53. struct virtio_scsi *vscsi;
  54. struct virtio_scsi_event event;
  55. struct work_struct work;
  56. };
  57. struct virtio_scsi_vq {
  58. /* Protects vq */
  59. spinlock_t vq_lock;
  60. struct virtqueue *vq;
  61. };
  62. /* Driver instance state */
  63. struct virtio_scsi {
  64. struct virtio_device *vdev;
  65. /* Get some buffers ready for event vq */
  66. struct virtio_scsi_event_node event_list[VIRTIO_SCSI_EVENT_LEN];
  67. u32 num_queues;
  68. struct hlist_node node;
  69. /* Protected by event_vq lock */
  70. bool stop_events;
  71. struct virtio_scsi_vq ctrl_vq;
  72. struct virtio_scsi_vq event_vq;
  73. struct virtio_scsi_vq req_vqs[];
  74. };
  75. static struct kmem_cache *virtscsi_cmd_cache;
  76. static mempool_t *virtscsi_cmd_pool;
  77. static inline struct Scsi_Host *virtio_scsi_host(struct virtio_device *vdev)
  78. {
  79. return vdev->priv;
  80. }
  81. static void virtscsi_compute_resid(struct scsi_cmnd *sc, u32 resid)
  82. {
  83. if (resid)
  84. scsi_set_resid(sc, min(resid, scsi_bufflen(sc)));
  85. }
  86. /*
  87. * virtscsi_complete_cmd - finish a scsi_cmd and invoke scsi_done
  88. *
  89. * Called with vq_lock held.
  90. */
  91. static void virtscsi_complete_cmd(struct virtio_scsi *vscsi, void *buf)
  92. {
  93. struct virtio_scsi_cmd *cmd = buf;
  94. struct scsi_cmnd *sc = cmd->sc;
  95. struct virtio_scsi_cmd_resp *resp = &cmd->resp.cmd;
  96. dev_dbg(&sc->device->sdev_gendev,
  97. "cmd %p response %u status %#02x sense_len %u\n",
  98. sc, resp->response, resp->status, resp->sense_len);
  99. sc->result = resp->status;
  100. virtscsi_compute_resid(sc, virtio32_to_cpu(vscsi->vdev, resp->resid));
  101. switch (resp->response) {
  102. case VIRTIO_SCSI_S_OK:
  103. set_host_byte(sc, DID_OK);
  104. break;
  105. case VIRTIO_SCSI_S_OVERRUN:
  106. set_host_byte(sc, DID_ERROR);
  107. break;
  108. case VIRTIO_SCSI_S_ABORTED:
  109. set_host_byte(sc, DID_ABORT);
  110. break;
  111. case VIRTIO_SCSI_S_BAD_TARGET:
  112. set_host_byte(sc, DID_BAD_TARGET);
  113. break;
  114. case VIRTIO_SCSI_S_RESET:
  115. set_host_byte(sc, DID_RESET);
  116. break;
  117. case VIRTIO_SCSI_S_BUSY:
  118. set_host_byte(sc, DID_BUS_BUSY);
  119. break;
  120. case VIRTIO_SCSI_S_TRANSPORT_FAILURE:
  121. set_host_byte(sc, DID_TRANSPORT_DISRUPTED);
  122. break;
  123. case VIRTIO_SCSI_S_TARGET_FAILURE:
  124. set_host_byte(sc, DID_BAD_TARGET);
  125. break;
  126. case VIRTIO_SCSI_S_NEXUS_FAILURE:
  127. set_status_byte(sc, SAM_STAT_RESERVATION_CONFLICT);
  128. break;
  129. default:
  130. scmd_printk(KERN_WARNING, sc, "Unknown response %d",
  131. resp->response);
  132. fallthrough;
  133. case VIRTIO_SCSI_S_FAILURE:
  134. set_host_byte(sc, DID_ERROR);
  135. break;
  136. }
  137. WARN_ON(virtio32_to_cpu(vscsi->vdev, resp->sense_len) >
  138. VIRTIO_SCSI_SENSE_SIZE);
  139. if (resp->sense_len) {
  140. memcpy(sc->sense_buffer, resp->sense,
  141. min_t(u32,
  142. virtio32_to_cpu(vscsi->vdev, resp->sense_len),
  143. VIRTIO_SCSI_SENSE_SIZE));
  144. }
  145. scsi_done(sc);
  146. }
  147. static void virtscsi_vq_done(struct virtio_scsi *vscsi,
  148. struct virtio_scsi_vq *virtscsi_vq,
  149. void (*fn)(struct virtio_scsi *vscsi, void *buf))
  150. {
  151. void *buf;
  152. unsigned int len;
  153. unsigned long flags;
  154. struct virtqueue *vq = virtscsi_vq->vq;
  155. spin_lock_irqsave(&virtscsi_vq->vq_lock, flags);
  156. do {
  157. virtqueue_disable_cb(vq);
  158. while ((buf = virtqueue_get_buf(vq, &len)) != NULL)
  159. fn(vscsi, buf);
  160. if (unlikely(virtqueue_is_broken(vq)))
  161. break;
  162. } while (!virtqueue_enable_cb(vq));
  163. spin_unlock_irqrestore(&virtscsi_vq->vq_lock, flags);
  164. }
  165. static void virtscsi_req_done(struct virtqueue *vq)
  166. {
  167. struct Scsi_Host *sh = virtio_scsi_host(vq->vdev);
  168. struct virtio_scsi *vscsi = shost_priv(sh);
  169. int index = vq->index - VIRTIO_SCSI_VQ_BASE;
  170. struct virtio_scsi_vq *req_vq = &vscsi->req_vqs[index];
  171. virtscsi_vq_done(vscsi, req_vq, virtscsi_complete_cmd);
  172. };
  173. static void virtscsi_poll_requests(struct virtio_scsi *vscsi)
  174. {
  175. int i, num_vqs;
  176. num_vqs = vscsi->num_queues;
  177. for (i = 0; i < num_vqs; i++)
  178. virtscsi_vq_done(vscsi, &vscsi->req_vqs[i],
  179. virtscsi_complete_cmd);
  180. }
  181. static void virtscsi_complete_free(struct virtio_scsi *vscsi, void *buf)
  182. {
  183. struct virtio_scsi_cmd *cmd = buf;
  184. if (cmd->comp)
  185. complete(cmd->comp);
  186. }
  187. static void virtscsi_ctrl_done(struct virtqueue *vq)
  188. {
  189. struct Scsi_Host *sh = virtio_scsi_host(vq->vdev);
  190. struct virtio_scsi *vscsi = shost_priv(sh);
  191. virtscsi_vq_done(vscsi, &vscsi->ctrl_vq, virtscsi_complete_free);
  192. };
  193. static void virtscsi_handle_event(struct work_struct *work);
  194. static int virtscsi_kick_event(struct virtio_scsi *vscsi,
  195. struct virtio_scsi_event_node *event_node)
  196. {
  197. int err;
  198. struct scatterlist sg;
  199. unsigned long flags;
  200. INIT_WORK(&event_node->work, virtscsi_handle_event);
  201. sg_init_one(&sg, &event_node->event, sizeof(struct virtio_scsi_event));
  202. spin_lock_irqsave(&vscsi->event_vq.vq_lock, flags);
  203. err = virtqueue_add_inbuf(vscsi->event_vq.vq, &sg, 1, event_node,
  204. GFP_ATOMIC);
  205. if (!err)
  206. virtqueue_kick(vscsi->event_vq.vq);
  207. spin_unlock_irqrestore(&vscsi->event_vq.vq_lock, flags);
  208. return err;
  209. }
  210. static int virtscsi_kick_event_all(struct virtio_scsi *vscsi)
  211. {
  212. int i;
  213. for (i = 0; i < VIRTIO_SCSI_EVENT_LEN; i++) {
  214. vscsi->event_list[i].vscsi = vscsi;
  215. virtscsi_kick_event(vscsi, &vscsi->event_list[i]);
  216. }
  217. return 0;
  218. }
  219. static void virtscsi_cancel_event_work(struct virtio_scsi *vscsi)
  220. {
  221. int i;
  222. /* Stop scheduling work before calling cancel_work_sync. */
  223. spin_lock_irq(&vscsi->event_vq.vq_lock);
  224. vscsi->stop_events = true;
  225. spin_unlock_irq(&vscsi->event_vq.vq_lock);
  226. for (i = 0; i < VIRTIO_SCSI_EVENT_LEN; i++)
  227. cancel_work_sync(&vscsi->event_list[i].work);
  228. }
  229. static void virtscsi_handle_transport_reset(struct virtio_scsi *vscsi,
  230. struct virtio_scsi_event *event)
  231. {
  232. struct scsi_device *sdev;
  233. struct Scsi_Host *shost = virtio_scsi_host(vscsi->vdev);
  234. unsigned int target = event->lun[1];
  235. unsigned int lun = (event->lun[2] << 8) | event->lun[3];
  236. switch (virtio32_to_cpu(vscsi->vdev, event->reason)) {
  237. case VIRTIO_SCSI_EVT_RESET_RESCAN:
  238. if (lun == 0) {
  239. scsi_scan_target(&shost->shost_gendev, 0, target,
  240. SCAN_WILD_CARD, SCSI_SCAN_INITIAL);
  241. } else {
  242. scsi_add_device(shost, 0, target, lun);
  243. }
  244. break;
  245. case VIRTIO_SCSI_EVT_RESET_REMOVED:
  246. sdev = scsi_device_lookup(shost, 0, target, lun);
  247. if (sdev) {
  248. scsi_remove_device(sdev);
  249. scsi_device_put(sdev);
  250. } else {
  251. pr_err("SCSI device %d 0 %d %d not found\n",
  252. shost->host_no, target, lun);
  253. }
  254. break;
  255. default:
  256. pr_info("Unsupported virtio scsi event reason %x\n", event->reason);
  257. }
  258. }
  259. static void virtscsi_handle_param_change(struct virtio_scsi *vscsi,
  260. struct virtio_scsi_event *event)
  261. {
  262. struct scsi_device *sdev;
  263. struct Scsi_Host *shost = virtio_scsi_host(vscsi->vdev);
  264. unsigned int target = event->lun[1];
  265. unsigned int lun = (event->lun[2] << 8) | event->lun[3];
  266. u8 asc = virtio32_to_cpu(vscsi->vdev, event->reason) & 255;
  267. u8 ascq = virtio32_to_cpu(vscsi->vdev, event->reason) >> 8;
  268. sdev = scsi_device_lookup(shost, 0, target, lun);
  269. if (!sdev) {
  270. pr_err("SCSI device %d 0 %d %d not found\n",
  271. shost->host_no, target, lun);
  272. return;
  273. }
  274. /* Handle "Parameters changed", "Mode parameters changed", and
  275. "Capacity data has changed". */
  276. if (asc == 0x2a && (ascq == 0x00 || ascq == 0x01 || ascq == 0x09))
  277. scsi_rescan_device(sdev);
  278. scsi_device_put(sdev);
  279. }
  280. static void virtscsi_rescan_hotunplug(struct virtio_scsi *vscsi)
  281. {
  282. struct scsi_device *sdev;
  283. struct Scsi_Host *shost = virtio_scsi_host(vscsi->vdev);
  284. unsigned char scsi_cmd[MAX_COMMAND_SIZE];
  285. int result, inquiry_len, inq_result_len = 256;
  286. char *inq_result = kmalloc(inq_result_len, GFP_KERNEL);
  287. shost_for_each_device(sdev, shost) {
  288. inquiry_len = sdev->inquiry_len ? sdev->inquiry_len : 36;
  289. memset(scsi_cmd, 0, sizeof(scsi_cmd));
  290. scsi_cmd[0] = INQUIRY;
  291. scsi_cmd[4] = (unsigned char) inquiry_len;
  292. memset(inq_result, 0, inq_result_len);
  293. result = scsi_execute_cmd(sdev, scsi_cmd, REQ_OP_DRV_IN,
  294. inq_result, inquiry_len,
  295. SD_TIMEOUT, SD_MAX_RETRIES, NULL);
  296. if (result == 0 && inq_result[0] >> 5) {
  297. /* PQ indicates the LUN is not attached */
  298. scsi_remove_device(sdev);
  299. } else if (result > 0 && host_byte(result) == DID_BAD_TARGET) {
  300. /*
  301. * If all LUNs of a virtio-scsi device are unplugged
  302. * it will respond with BAD TARGET on any INQUIRY
  303. * command.
  304. * Remove the device in this case as well.
  305. */
  306. scsi_remove_device(sdev);
  307. }
  308. }
  309. kfree(inq_result);
  310. }
  311. static void virtscsi_handle_event(struct work_struct *work)
  312. {
  313. struct virtio_scsi_event_node *event_node =
  314. container_of(work, struct virtio_scsi_event_node, work);
  315. struct virtio_scsi *vscsi = event_node->vscsi;
  316. struct virtio_scsi_event *event = &event_node->event;
  317. if (event->event &
  318. cpu_to_virtio32(vscsi->vdev, VIRTIO_SCSI_T_EVENTS_MISSED)) {
  319. event->event &= ~cpu_to_virtio32(vscsi->vdev,
  320. VIRTIO_SCSI_T_EVENTS_MISSED);
  321. virtscsi_rescan_hotunplug(vscsi);
  322. scsi_scan_host(virtio_scsi_host(vscsi->vdev));
  323. }
  324. switch (virtio32_to_cpu(vscsi->vdev, event->event)) {
  325. case VIRTIO_SCSI_T_NO_EVENT:
  326. break;
  327. case VIRTIO_SCSI_T_TRANSPORT_RESET:
  328. virtscsi_handle_transport_reset(vscsi, event);
  329. break;
  330. case VIRTIO_SCSI_T_PARAM_CHANGE:
  331. virtscsi_handle_param_change(vscsi, event);
  332. break;
  333. default:
  334. pr_err("Unsupported virtio scsi event %x\n", event->event);
  335. }
  336. virtscsi_kick_event(vscsi, event_node);
  337. }
  338. static void virtscsi_complete_event(struct virtio_scsi *vscsi, void *buf)
  339. {
  340. struct virtio_scsi_event_node *event_node = buf;
  341. if (!vscsi->stop_events)
  342. queue_work(system_freezable_wq, &event_node->work);
  343. }
  344. static void virtscsi_event_done(struct virtqueue *vq)
  345. {
  346. struct Scsi_Host *sh = virtio_scsi_host(vq->vdev);
  347. struct virtio_scsi *vscsi = shost_priv(sh);
  348. virtscsi_vq_done(vscsi, &vscsi->event_vq, virtscsi_complete_event);
  349. };
  350. static int __virtscsi_add_cmd(struct virtqueue *vq,
  351. struct virtio_scsi_cmd *cmd,
  352. size_t req_size, size_t resp_size)
  353. {
  354. struct scsi_cmnd *sc = cmd->sc;
  355. struct scatterlist *sgs[6], req, resp;
  356. struct sg_table *out, *in;
  357. unsigned out_num = 0, in_num = 0;
  358. out = in = NULL;
  359. if (sc && sc->sc_data_direction != DMA_NONE) {
  360. if (sc->sc_data_direction != DMA_FROM_DEVICE)
  361. out = &sc->sdb.table;
  362. if (sc->sc_data_direction != DMA_TO_DEVICE)
  363. in = &sc->sdb.table;
  364. }
  365. /* Request header. */
  366. sg_init_one(&req, &cmd->req, req_size);
  367. sgs[out_num++] = &req;
  368. /* Data-out buffer. */
  369. if (out) {
  370. /* Place WRITE protection SGLs before Data OUT payload */
  371. if (scsi_prot_sg_count(sc))
  372. sgs[out_num++] = scsi_prot_sglist(sc);
  373. sgs[out_num++] = out->sgl;
  374. }
  375. /* Response header. */
  376. sg_init_one(&resp, &cmd->resp, resp_size);
  377. sgs[out_num + in_num++] = &resp;
  378. /* Data-in buffer */
  379. if (in) {
  380. /* Place READ protection SGLs before Data IN payload */
  381. if (scsi_prot_sg_count(sc))
  382. sgs[out_num + in_num++] = scsi_prot_sglist(sc);
  383. sgs[out_num + in_num++] = in->sgl;
  384. }
  385. return virtqueue_add_sgs(vq, sgs, out_num, in_num, cmd, GFP_ATOMIC);
  386. }
  387. static void virtscsi_kick_vq(struct virtio_scsi_vq *vq)
  388. {
  389. bool needs_kick;
  390. unsigned long flags;
  391. spin_lock_irqsave(&vq->vq_lock, flags);
  392. needs_kick = virtqueue_kick_prepare(vq->vq);
  393. spin_unlock_irqrestore(&vq->vq_lock, flags);
  394. if (needs_kick)
  395. virtqueue_notify(vq->vq);
  396. }
  397. /**
  398. * virtscsi_add_cmd - add a virtio_scsi_cmd to a virtqueue, optionally kick it
  399. * @vq : the struct virtqueue we're talking about
  400. * @cmd : command structure
  401. * @req_size : size of the request buffer
  402. * @resp_size : size of the response buffer
  403. * @kick : whether to kick the virtqueue immediately
  404. */
  405. static int virtscsi_add_cmd(struct virtio_scsi_vq *vq,
  406. struct virtio_scsi_cmd *cmd,
  407. size_t req_size, size_t resp_size,
  408. bool kick)
  409. {
  410. unsigned long flags;
  411. int err;
  412. bool needs_kick = false;
  413. spin_lock_irqsave(&vq->vq_lock, flags);
  414. err = __virtscsi_add_cmd(vq->vq, cmd, req_size, resp_size);
  415. if (!err && kick)
  416. needs_kick = virtqueue_kick_prepare(vq->vq);
  417. spin_unlock_irqrestore(&vq->vq_lock, flags);
  418. if (needs_kick)
  419. virtqueue_notify(vq->vq);
  420. return err;
  421. }
  422. static void virtio_scsi_init_hdr(struct virtio_device *vdev,
  423. struct virtio_scsi_cmd_req *cmd,
  424. struct scsi_cmnd *sc)
  425. {
  426. cmd->lun[0] = 1;
  427. cmd->lun[1] = sc->device->id;
  428. cmd->lun[2] = (sc->device->lun >> 8) | 0x40;
  429. cmd->lun[3] = sc->device->lun & 0xff;
  430. cmd->tag = cpu_to_virtio64(vdev, (unsigned long)sc);
  431. cmd->task_attr = VIRTIO_SCSI_S_SIMPLE;
  432. cmd->prio = 0;
  433. cmd->crn = 0;
  434. }
  435. #ifdef CONFIG_BLK_DEV_INTEGRITY
  436. static void virtio_scsi_init_hdr_pi(struct virtio_device *vdev,
  437. struct virtio_scsi_cmd_req_pi *cmd_pi,
  438. struct scsi_cmnd *sc)
  439. {
  440. struct request *rq = scsi_cmd_to_rq(sc);
  441. struct blk_integrity *bi;
  442. virtio_scsi_init_hdr(vdev, (struct virtio_scsi_cmd_req *)cmd_pi, sc);
  443. if (!rq || !scsi_prot_sg_count(sc))
  444. return;
  445. bi = blk_get_integrity(rq->q->disk);
  446. if (sc->sc_data_direction == DMA_TO_DEVICE)
  447. cmd_pi->pi_bytesout = cpu_to_virtio32(vdev,
  448. bio_integrity_bytes(bi,
  449. blk_rq_sectors(rq)));
  450. else if (sc->sc_data_direction == DMA_FROM_DEVICE)
  451. cmd_pi->pi_bytesin = cpu_to_virtio32(vdev,
  452. bio_integrity_bytes(bi,
  453. blk_rq_sectors(rq)));
  454. }
  455. #endif
  456. static struct virtio_scsi_vq *virtscsi_pick_vq_mq(struct virtio_scsi *vscsi,
  457. struct scsi_cmnd *sc)
  458. {
  459. u32 tag = blk_mq_unique_tag(scsi_cmd_to_rq(sc));
  460. u16 hwq = blk_mq_unique_tag_to_hwq(tag);
  461. return &vscsi->req_vqs[hwq];
  462. }
  463. static int virtscsi_queuecommand(struct Scsi_Host *shost,
  464. struct scsi_cmnd *sc)
  465. {
  466. struct virtio_scsi *vscsi = shost_priv(shost);
  467. struct virtio_scsi_vq *req_vq = virtscsi_pick_vq_mq(vscsi, sc);
  468. struct virtio_scsi_cmd *cmd = scsi_cmd_priv(sc);
  469. bool kick;
  470. unsigned long flags;
  471. int req_size;
  472. int ret;
  473. BUG_ON(scsi_sg_count(sc) > shost->sg_tablesize);
  474. /* TODO: check feature bit and fail if unsupported? */
  475. BUG_ON(sc->sc_data_direction == DMA_BIDIRECTIONAL);
  476. dev_dbg(&sc->device->sdev_gendev,
  477. "cmd %p CDB: %#02x\n", sc, sc->cmnd[0]);
  478. cmd->sc = sc;
  479. BUG_ON(sc->cmd_len > VIRTIO_SCSI_CDB_SIZE);
  480. #ifdef CONFIG_BLK_DEV_INTEGRITY
  481. if (virtio_has_feature(vscsi->vdev, VIRTIO_SCSI_F_T10_PI)) {
  482. virtio_scsi_init_hdr_pi(vscsi->vdev, &cmd->req.cmd_pi, sc);
  483. memcpy(cmd->req.cmd_pi.cdb, sc->cmnd, sc->cmd_len);
  484. req_size = sizeof(cmd->req.cmd_pi);
  485. } else
  486. #endif
  487. {
  488. virtio_scsi_init_hdr(vscsi->vdev, &cmd->req.cmd, sc);
  489. memcpy(cmd->req.cmd.cdb, sc->cmnd, sc->cmd_len);
  490. req_size = sizeof(cmd->req.cmd);
  491. }
  492. kick = (sc->flags & SCMD_LAST) != 0;
  493. ret = virtscsi_add_cmd(req_vq, cmd, req_size, sizeof(cmd->resp.cmd), kick);
  494. if (ret == -EIO) {
  495. cmd->resp.cmd.response = VIRTIO_SCSI_S_BAD_TARGET;
  496. spin_lock_irqsave(&req_vq->vq_lock, flags);
  497. virtscsi_complete_cmd(vscsi, cmd);
  498. spin_unlock_irqrestore(&req_vq->vq_lock, flags);
  499. } else if (ret != 0) {
  500. return SCSI_MLQUEUE_HOST_BUSY;
  501. }
  502. return 0;
  503. }
  504. static int virtscsi_tmf(struct virtio_scsi *vscsi, struct virtio_scsi_cmd *cmd)
  505. {
  506. DECLARE_COMPLETION_ONSTACK(comp);
  507. int ret = FAILED;
  508. cmd->comp = &comp;
  509. if (virtscsi_add_cmd(&vscsi->ctrl_vq, cmd,
  510. sizeof cmd->req.tmf, sizeof cmd->resp.tmf, true) < 0)
  511. goto out;
  512. wait_for_completion(&comp);
  513. if (cmd->resp.tmf.response == VIRTIO_SCSI_S_OK ||
  514. cmd->resp.tmf.response == VIRTIO_SCSI_S_FUNCTION_SUCCEEDED)
  515. ret = SUCCESS;
  516. /*
  517. * The spec guarantees that all requests related to the TMF have
  518. * been completed, but the callback might not have run yet if
  519. * we're using independent interrupts (e.g. MSI). Poll the
  520. * virtqueues once.
  521. *
  522. * In the abort case, scsi_done() will do nothing, because the
  523. * command timed out and hence SCMD_STATE_COMPLETE has been set.
  524. */
  525. virtscsi_poll_requests(vscsi);
  526. out:
  527. mempool_free(cmd, virtscsi_cmd_pool);
  528. return ret;
  529. }
  530. static int virtscsi_device_reset(struct scsi_cmnd *sc)
  531. {
  532. struct virtio_scsi *vscsi = shost_priv(sc->device->host);
  533. struct virtio_scsi_cmd *cmd;
  534. sdev_printk(KERN_INFO, sc->device, "device reset\n");
  535. cmd = mempool_alloc(virtscsi_cmd_pool, GFP_NOIO);
  536. if (!cmd)
  537. return FAILED;
  538. memset(cmd, 0, sizeof(*cmd));
  539. cmd->req.tmf = (struct virtio_scsi_ctrl_tmf_req){
  540. .type = VIRTIO_SCSI_T_TMF,
  541. .subtype = cpu_to_virtio32(vscsi->vdev,
  542. VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET),
  543. .lun[0] = 1,
  544. .lun[1] = sc->device->id,
  545. .lun[2] = (sc->device->lun >> 8) | 0x40,
  546. .lun[3] = sc->device->lun & 0xff,
  547. };
  548. return virtscsi_tmf(vscsi, cmd);
  549. }
  550. static int virtscsi_device_alloc(struct scsi_device *sdevice)
  551. {
  552. /*
  553. * Passed through SCSI targets (e.g. with qemu's 'scsi-block')
  554. * may have transfer limits which come from the host SCSI
  555. * controller or something on the host side other than the
  556. * target itself.
  557. *
  558. * To make this work properly, the hypervisor can adjust the
  559. * target's VPD information to advertise these limits. But
  560. * for that to work, the guest has to look at the VPD pages,
  561. * which we won't do by default if it is an SPC-2 device, even
  562. * if it does actually support it.
  563. *
  564. * So, set the blist to always try to read the VPD pages.
  565. */
  566. sdevice->sdev_bflags = BLIST_TRY_VPD_PAGES;
  567. return 0;
  568. }
  569. /**
  570. * virtscsi_change_queue_depth() - Change a virtscsi target's queue depth
  571. * @sdev: Virtscsi target whose queue depth to change
  572. * @qdepth: New queue depth
  573. */
  574. static int virtscsi_change_queue_depth(struct scsi_device *sdev, int qdepth)
  575. {
  576. struct Scsi_Host *shost = sdev->host;
  577. int max_depth = shost->cmd_per_lun;
  578. return scsi_change_queue_depth(sdev, min(max_depth, qdepth));
  579. }
  580. static int virtscsi_abort(struct scsi_cmnd *sc)
  581. {
  582. struct virtio_scsi *vscsi = shost_priv(sc->device->host);
  583. struct virtio_scsi_cmd *cmd;
  584. scmd_printk(KERN_INFO, sc, "abort\n");
  585. cmd = mempool_alloc(virtscsi_cmd_pool, GFP_NOIO);
  586. if (!cmd)
  587. return FAILED;
  588. memset(cmd, 0, sizeof(*cmd));
  589. cmd->req.tmf = (struct virtio_scsi_ctrl_tmf_req){
  590. .type = VIRTIO_SCSI_T_TMF,
  591. .subtype = VIRTIO_SCSI_T_TMF_ABORT_TASK,
  592. .lun[0] = 1,
  593. .lun[1] = sc->device->id,
  594. .lun[2] = (sc->device->lun >> 8) | 0x40,
  595. .lun[3] = sc->device->lun & 0xff,
  596. .tag = cpu_to_virtio64(vscsi->vdev, (unsigned long)sc),
  597. };
  598. return virtscsi_tmf(vscsi, cmd);
  599. }
  600. static void virtscsi_map_queues(struct Scsi_Host *shost)
  601. {
  602. struct virtio_scsi *vscsi = shost_priv(shost);
  603. struct blk_mq_queue_map *qmap = &shost->tag_set.map[HCTX_TYPE_DEFAULT];
  604. blk_mq_virtio_map_queues(qmap, vscsi->vdev, 2);
  605. }
  606. static void virtscsi_commit_rqs(struct Scsi_Host *shost, u16 hwq)
  607. {
  608. struct virtio_scsi *vscsi = shost_priv(shost);
  609. virtscsi_kick_vq(&vscsi->req_vqs[hwq]);
  610. }
  611. /*
  612. * The host guarantees to respond to each command, although I/O
  613. * latencies might be higher than on bare metal. Reset the timer
  614. * unconditionally to give the host a chance to perform EH.
  615. */
  616. static enum scsi_timeout_action virtscsi_eh_timed_out(struct scsi_cmnd *scmnd)
  617. {
  618. return SCSI_EH_RESET_TIMER;
  619. }
  620. static struct scsi_host_template virtscsi_host_template = {
  621. .module = THIS_MODULE,
  622. .name = "Virtio SCSI HBA",
  623. .proc_name = "virtio_scsi",
  624. .this_id = -1,
  625. .cmd_size = sizeof(struct virtio_scsi_cmd),
  626. .queuecommand = virtscsi_queuecommand,
  627. .commit_rqs = virtscsi_commit_rqs,
  628. .change_queue_depth = virtscsi_change_queue_depth,
  629. .eh_abort_handler = virtscsi_abort,
  630. .eh_device_reset_handler = virtscsi_device_reset,
  631. .eh_timed_out = virtscsi_eh_timed_out,
  632. .slave_alloc = virtscsi_device_alloc,
  633. .dma_boundary = UINT_MAX,
  634. .map_queues = virtscsi_map_queues,
  635. .track_queue_depth = 1,
  636. };
  637. #define virtscsi_config_get(vdev, fld) \
  638. ({ \
  639. __virtio_native_type(struct virtio_scsi_config, fld) __val; \
  640. virtio_cread(vdev, struct virtio_scsi_config, fld, &__val); \
  641. __val; \
  642. })
  643. #define virtscsi_config_set(vdev, fld, val) \
  644. do { \
  645. __virtio_native_type(struct virtio_scsi_config, fld) __val = (val); \
  646. virtio_cwrite(vdev, struct virtio_scsi_config, fld, &__val); \
  647. } while(0)
  648. static void virtscsi_init_vq(struct virtio_scsi_vq *virtscsi_vq,
  649. struct virtqueue *vq)
  650. {
  651. spin_lock_init(&virtscsi_vq->vq_lock);
  652. virtscsi_vq->vq = vq;
  653. }
  654. static void virtscsi_remove_vqs(struct virtio_device *vdev)
  655. {
  656. /* Stop all the virtqueues. */
  657. virtio_reset_device(vdev);
  658. vdev->config->del_vqs(vdev);
  659. }
  660. static int virtscsi_init(struct virtio_device *vdev,
  661. struct virtio_scsi *vscsi)
  662. {
  663. int err;
  664. u32 i;
  665. u32 num_vqs;
  666. vq_callback_t **callbacks;
  667. const char **names;
  668. struct virtqueue **vqs;
  669. struct irq_affinity desc = { .pre_vectors = 2 };
  670. num_vqs = vscsi->num_queues + VIRTIO_SCSI_VQ_BASE;
  671. vqs = kmalloc_array(num_vqs, sizeof(struct virtqueue *), GFP_KERNEL);
  672. callbacks = kmalloc_array(num_vqs, sizeof(vq_callback_t *),
  673. GFP_KERNEL);
  674. names = kmalloc_array(num_vqs, sizeof(char *), GFP_KERNEL);
  675. if (!callbacks || !vqs || !names) {
  676. err = -ENOMEM;
  677. goto out;
  678. }
  679. callbacks[0] = virtscsi_ctrl_done;
  680. callbacks[1] = virtscsi_event_done;
  681. names[0] = "control";
  682. names[1] = "event";
  683. for (i = VIRTIO_SCSI_VQ_BASE; i < num_vqs; i++) {
  684. callbacks[i] = virtscsi_req_done;
  685. names[i] = "request";
  686. }
  687. /* Discover virtqueues and write information to configuration. */
  688. err = virtio_find_vqs(vdev, num_vqs, vqs, callbacks, names, &desc);
  689. if (err)
  690. goto out;
  691. virtscsi_init_vq(&vscsi->ctrl_vq, vqs[0]);
  692. virtscsi_init_vq(&vscsi->event_vq, vqs[1]);
  693. for (i = VIRTIO_SCSI_VQ_BASE; i < num_vqs; i++)
  694. virtscsi_init_vq(&vscsi->req_vqs[i - VIRTIO_SCSI_VQ_BASE],
  695. vqs[i]);
  696. virtscsi_config_set(vdev, cdb_size, VIRTIO_SCSI_CDB_SIZE);
  697. virtscsi_config_set(vdev, sense_size, VIRTIO_SCSI_SENSE_SIZE);
  698. err = 0;
  699. out:
  700. kfree(names);
  701. kfree(callbacks);
  702. kfree(vqs);
  703. if (err)
  704. virtscsi_remove_vqs(vdev);
  705. return err;
  706. }
  707. static int virtscsi_probe(struct virtio_device *vdev)
  708. {
  709. struct Scsi_Host *shost;
  710. struct virtio_scsi *vscsi;
  711. int err;
  712. u32 sg_elems, num_targets;
  713. u32 cmd_per_lun;
  714. u32 num_queues;
  715. if (!vdev->config->get) {
  716. dev_err(&vdev->dev, "%s failure: config access disabled\n",
  717. __func__);
  718. return -EINVAL;
  719. }
  720. /* We need to know how many queues before we allocate. */
  721. num_queues = virtscsi_config_get(vdev, num_queues) ? : 1;
  722. num_queues = min_t(unsigned int, nr_cpu_ids, num_queues);
  723. num_targets = virtscsi_config_get(vdev, max_target) + 1;
  724. shost = scsi_host_alloc(&virtscsi_host_template,
  725. struct_size(vscsi, req_vqs, num_queues));
  726. if (!shost)
  727. return -ENOMEM;
  728. sg_elems = virtscsi_config_get(vdev, seg_max) ?: 1;
  729. shost->sg_tablesize = sg_elems;
  730. vscsi = shost_priv(shost);
  731. vscsi->vdev = vdev;
  732. vscsi->num_queues = num_queues;
  733. vdev->priv = shost;
  734. err = virtscsi_init(vdev, vscsi);
  735. if (err)
  736. goto virtscsi_init_failed;
  737. shost->can_queue = virtqueue_get_vring_size(vscsi->req_vqs[0].vq);
  738. cmd_per_lun = virtscsi_config_get(vdev, cmd_per_lun) ?: 1;
  739. shost->cmd_per_lun = min_t(u32, cmd_per_lun, shost->can_queue);
  740. shost->max_sectors = virtscsi_config_get(vdev, max_sectors) ?: 0xFFFF;
  741. /* LUNs > 256 are reported with format 1, so they go in the range
  742. * 16640-32767.
  743. */
  744. shost->max_lun = virtscsi_config_get(vdev, max_lun) + 1 + 0x4000;
  745. shost->max_id = num_targets;
  746. shost->max_channel = 0;
  747. shost->max_cmd_len = VIRTIO_SCSI_CDB_SIZE;
  748. shost->nr_hw_queues = num_queues;
  749. #ifdef CONFIG_BLK_DEV_INTEGRITY
  750. if (virtio_has_feature(vdev, VIRTIO_SCSI_F_T10_PI)) {
  751. int host_prot;
  752. host_prot = SHOST_DIF_TYPE1_PROTECTION | SHOST_DIF_TYPE2_PROTECTION |
  753. SHOST_DIF_TYPE3_PROTECTION | SHOST_DIX_TYPE1_PROTECTION |
  754. SHOST_DIX_TYPE2_PROTECTION | SHOST_DIX_TYPE3_PROTECTION;
  755. scsi_host_set_prot(shost, host_prot);
  756. scsi_host_set_guard(shost, SHOST_DIX_GUARD_CRC);
  757. }
  758. #endif
  759. err = scsi_add_host(shost, &vdev->dev);
  760. if (err)
  761. goto scsi_add_host_failed;
  762. virtio_device_ready(vdev);
  763. if (virtio_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG))
  764. virtscsi_kick_event_all(vscsi);
  765. scsi_scan_host(shost);
  766. return 0;
  767. scsi_add_host_failed:
  768. vdev->config->del_vqs(vdev);
  769. virtscsi_init_failed:
  770. scsi_host_put(shost);
  771. return err;
  772. }
  773. static void virtscsi_remove(struct virtio_device *vdev)
  774. {
  775. struct Scsi_Host *shost = virtio_scsi_host(vdev);
  776. struct virtio_scsi *vscsi = shost_priv(shost);
  777. if (virtio_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG))
  778. virtscsi_cancel_event_work(vscsi);
  779. scsi_remove_host(shost);
  780. virtscsi_remove_vqs(vdev);
  781. scsi_host_put(shost);
  782. }
  783. #ifdef CONFIG_PM_SLEEP
  784. static int virtscsi_freeze(struct virtio_device *vdev)
  785. {
  786. virtscsi_remove_vqs(vdev);
  787. return 0;
  788. }
  789. static int virtscsi_restore(struct virtio_device *vdev)
  790. {
  791. struct Scsi_Host *sh = virtio_scsi_host(vdev);
  792. struct virtio_scsi *vscsi = shost_priv(sh);
  793. int err;
  794. err = virtscsi_init(vdev, vscsi);
  795. if (err)
  796. return err;
  797. virtio_device_ready(vdev);
  798. if (virtio_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG))
  799. virtscsi_kick_event_all(vscsi);
  800. return err;
  801. }
  802. #endif
  803. static struct virtio_device_id id_table[] = {
  804. { VIRTIO_ID_SCSI, VIRTIO_DEV_ANY_ID },
  805. { 0 },
  806. };
  807. static unsigned int features[] = {
  808. VIRTIO_SCSI_F_HOTPLUG,
  809. VIRTIO_SCSI_F_CHANGE,
  810. #ifdef CONFIG_BLK_DEV_INTEGRITY
  811. VIRTIO_SCSI_F_T10_PI,
  812. #endif
  813. };
  814. static struct virtio_driver virtio_scsi_driver = {
  815. .feature_table = features,
  816. .feature_table_size = ARRAY_SIZE(features),
  817. .driver.name = KBUILD_MODNAME,
  818. .driver.owner = THIS_MODULE,
  819. .id_table = id_table,
  820. .probe = virtscsi_probe,
  821. #ifdef CONFIG_PM_SLEEP
  822. .freeze = virtscsi_freeze,
  823. .restore = virtscsi_restore,
  824. #endif
  825. .remove = virtscsi_remove,
  826. };
  827. static int __init virtio_scsi_init(void)
  828. {
  829. int ret = -ENOMEM;
  830. virtscsi_cmd_cache = KMEM_CACHE(virtio_scsi_cmd, 0);
  831. if (!virtscsi_cmd_cache) {
  832. pr_err("kmem_cache_create() for virtscsi_cmd_cache failed\n");
  833. goto error;
  834. }
  835. virtscsi_cmd_pool =
  836. mempool_create_slab_pool(VIRTIO_SCSI_MEMPOOL_SZ,
  837. virtscsi_cmd_cache);
  838. if (!virtscsi_cmd_pool) {
  839. pr_err("mempool_create() for virtscsi_cmd_pool failed\n");
  840. goto error;
  841. }
  842. ret = register_virtio_driver(&virtio_scsi_driver);
  843. if (ret < 0)
  844. goto error;
  845. return 0;
  846. error:
  847. mempool_destroy(virtscsi_cmd_pool);
  848. virtscsi_cmd_pool = NULL;
  849. kmem_cache_destroy(virtscsi_cmd_cache);
  850. virtscsi_cmd_cache = NULL;
  851. return ret;
  852. }
  853. static void __exit virtio_scsi_fini(void)
  854. {
  855. unregister_virtio_driver(&virtio_scsi_driver);
  856. mempool_destroy(virtscsi_cmd_pool);
  857. kmem_cache_destroy(virtscsi_cmd_cache);
  858. }
  859. module_init(virtio_scsi_init);
  860. module_exit(virtio_scsi_fini);
  861. MODULE_DEVICE_TABLE(virtio, id_table);
  862. MODULE_DESCRIPTION("Virtio SCSI HBA driver");
  863. MODULE_LICENSE("GPL");