vmw_pvscsi.c 43 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621
  1. /*
  2. * Linux driver for VMware's para-virtualized SCSI HBA.
  3. *
  4. * Copyright (C) 2008-2014, VMware, Inc. All Rights Reserved.
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms of the GNU General Public License as published by the
  8. * Free Software Foundation; version 2 of the License and no later version.
  9. *
  10. * This program is distributed in the hope that it will be useful, but
  11. * WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  13. * NON INFRINGEMENT. See the GNU General Public License for more
  14. * details.
  15. *
  16. * You should have received a copy of the GNU General Public License
  17. * along with this program; if not, write to the Free Software
  18. * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19. *
  20. */
  21. #include <linux/kernel.h>
  22. #include <linux/module.h>
  23. #include <linux/interrupt.h>
  24. #include <linux/slab.h>
  25. #include <linux/workqueue.h>
  26. #include <linux/pci.h>
  27. #include <scsi/scsi.h>
  28. #include <scsi/scsi_host.h>
  29. #include <scsi/scsi_cmnd.h>
  30. #include <scsi/scsi_device.h>
  31. #include <scsi/scsi_tcq.h>
  32. #include "vmw_pvscsi.h"
  33. #define PVSCSI_LINUX_DRIVER_DESC "VMware PVSCSI driver"
  34. MODULE_DESCRIPTION(PVSCSI_LINUX_DRIVER_DESC);
  35. MODULE_AUTHOR("VMware, Inc.");
  36. MODULE_LICENSE("GPL");
  37. MODULE_VERSION(PVSCSI_DRIVER_VERSION_STRING);
  38. #define PVSCSI_DEFAULT_NUM_PAGES_PER_RING 8
  39. #define PVSCSI_DEFAULT_NUM_PAGES_MSG_RING 1
  40. #define PVSCSI_DEFAULT_QUEUE_DEPTH 254
  41. #define SGL_SIZE PAGE_SIZE
  42. struct pvscsi_sg_list {
  43. struct PVSCSISGElement sge[PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT];
  44. };
  45. struct pvscsi_ctx {
  46. /*
  47. * The index of the context in cmd_map serves as the context ID for a
  48. * 1-to-1 mapping completions back to requests.
  49. */
  50. struct scsi_cmnd *cmd;
  51. struct pvscsi_sg_list *sgl;
  52. struct list_head list;
  53. dma_addr_t dataPA;
  54. dma_addr_t sensePA;
  55. dma_addr_t sglPA;
  56. struct completion *abort_cmp;
  57. };
  58. struct pvscsi_adapter {
  59. char *mmioBase;
  60. u8 rev;
  61. bool use_msg;
  62. bool use_req_threshold;
  63. spinlock_t hw_lock;
  64. struct workqueue_struct *workqueue;
  65. struct work_struct work;
  66. struct PVSCSIRingReqDesc *req_ring;
  67. unsigned req_pages;
  68. unsigned req_depth;
  69. dma_addr_t reqRingPA;
  70. struct PVSCSIRingCmpDesc *cmp_ring;
  71. unsigned cmp_pages;
  72. dma_addr_t cmpRingPA;
  73. struct PVSCSIRingMsgDesc *msg_ring;
  74. unsigned msg_pages;
  75. dma_addr_t msgRingPA;
  76. struct PVSCSIRingsState *rings_state;
  77. dma_addr_t ringStatePA;
  78. struct pci_dev *dev;
  79. struct Scsi_Host *host;
  80. struct list_head cmd_pool;
  81. struct pvscsi_ctx *cmd_map;
  82. };
  83. /* Command line parameters */
  84. static int pvscsi_ring_pages;
  85. static int pvscsi_msg_ring_pages = PVSCSI_DEFAULT_NUM_PAGES_MSG_RING;
  86. static int pvscsi_cmd_per_lun = PVSCSI_DEFAULT_QUEUE_DEPTH;
  87. static bool pvscsi_disable_msi;
  88. static bool pvscsi_disable_msix;
  89. static bool pvscsi_use_msg = true;
  90. static bool pvscsi_use_req_threshold = true;
  91. #define PVSCSI_RW (S_IRUSR | S_IWUSR)
  92. module_param_named(ring_pages, pvscsi_ring_pages, int, PVSCSI_RW);
  93. MODULE_PARM_DESC(ring_pages, "Number of pages per req/cmp ring - (default="
  94. __stringify(PVSCSI_DEFAULT_NUM_PAGES_PER_RING)
  95. "[up to 16 targets],"
  96. __stringify(PVSCSI_SETUP_RINGS_MAX_NUM_PAGES)
  97. "[for 16+ targets])");
  98. module_param_named(msg_ring_pages, pvscsi_msg_ring_pages, int, PVSCSI_RW);
  99. MODULE_PARM_DESC(msg_ring_pages, "Number of pages for the msg ring - (default="
  100. __stringify(PVSCSI_DEFAULT_NUM_PAGES_MSG_RING) ")");
  101. module_param_named(cmd_per_lun, pvscsi_cmd_per_lun, int, PVSCSI_RW);
  102. MODULE_PARM_DESC(cmd_per_lun, "Maximum commands per lun - (default="
  103. __stringify(PVSCSI_DEFAULT_QUEUE_DEPTH) ")");
  104. module_param_named(disable_msi, pvscsi_disable_msi, bool, PVSCSI_RW);
  105. MODULE_PARM_DESC(disable_msi, "Disable MSI use in driver - (default=0)");
  106. module_param_named(disable_msix, pvscsi_disable_msix, bool, PVSCSI_RW);
  107. MODULE_PARM_DESC(disable_msix, "Disable MSI-X use in driver - (default=0)");
  108. module_param_named(use_msg, pvscsi_use_msg, bool, PVSCSI_RW);
  109. MODULE_PARM_DESC(use_msg, "Use msg ring when available - (default=1)");
  110. module_param_named(use_req_threshold, pvscsi_use_req_threshold,
  111. bool, PVSCSI_RW);
  112. MODULE_PARM_DESC(use_req_threshold, "Use driver-based request coalescing if configured - (default=1)");
  113. static const struct pci_device_id pvscsi_pci_tbl[] = {
  114. { PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_PVSCSI) },
  115. { 0 }
  116. };
  117. MODULE_DEVICE_TABLE(pci, pvscsi_pci_tbl);
  118. static struct device *
  119. pvscsi_dev(const struct pvscsi_adapter *adapter)
  120. {
  121. return &(adapter->dev->dev);
  122. }
  123. static struct pvscsi_ctx *
  124. pvscsi_find_context(const struct pvscsi_adapter *adapter, struct scsi_cmnd *cmd)
  125. {
  126. struct pvscsi_ctx *ctx, *end;
  127. end = &adapter->cmd_map[adapter->req_depth];
  128. for (ctx = adapter->cmd_map; ctx < end; ctx++)
  129. if (ctx->cmd == cmd)
  130. return ctx;
  131. return NULL;
  132. }
  133. static struct pvscsi_ctx *
  134. pvscsi_acquire_context(struct pvscsi_adapter *adapter, struct scsi_cmnd *cmd)
  135. {
  136. struct pvscsi_ctx *ctx;
  137. if (list_empty(&adapter->cmd_pool))
  138. return NULL;
  139. ctx = list_first_entry(&adapter->cmd_pool, struct pvscsi_ctx, list);
  140. ctx->cmd = cmd;
  141. list_del(&ctx->list);
  142. return ctx;
  143. }
  144. static void pvscsi_release_context(struct pvscsi_adapter *adapter,
  145. struct pvscsi_ctx *ctx)
  146. {
  147. ctx->cmd = NULL;
  148. ctx->abort_cmp = NULL;
  149. list_add(&ctx->list, &adapter->cmd_pool);
  150. }
  151. /*
  152. * Map a pvscsi_ctx struct to a context ID field value; we map to a simple
  153. * non-zero integer. ctx always points to an entry in cmd_map array, hence
  154. * the return value is always >=1.
  155. */
  156. static u64 pvscsi_map_context(const struct pvscsi_adapter *adapter,
  157. const struct pvscsi_ctx *ctx)
  158. {
  159. return ctx - adapter->cmd_map + 1;
  160. }
  161. static struct pvscsi_ctx *
  162. pvscsi_get_context(const struct pvscsi_adapter *adapter, u64 context)
  163. {
  164. return &adapter->cmd_map[context - 1];
  165. }
  166. static void pvscsi_reg_write(const struct pvscsi_adapter *adapter,
  167. u32 offset, u32 val)
  168. {
  169. writel(val, adapter->mmioBase + offset);
  170. }
  171. static u32 pvscsi_reg_read(const struct pvscsi_adapter *adapter, u32 offset)
  172. {
  173. return readl(adapter->mmioBase + offset);
  174. }
  175. static u32 pvscsi_read_intr_status(const struct pvscsi_adapter *adapter)
  176. {
  177. return pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_INTR_STATUS);
  178. }
  179. static void pvscsi_write_intr_status(const struct pvscsi_adapter *adapter,
  180. u32 val)
  181. {
  182. pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_STATUS, val);
  183. }
  184. static void pvscsi_unmask_intr(const struct pvscsi_adapter *adapter)
  185. {
  186. u32 intr_bits;
  187. intr_bits = PVSCSI_INTR_CMPL_MASK;
  188. if (adapter->use_msg)
  189. intr_bits |= PVSCSI_INTR_MSG_MASK;
  190. pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_MASK, intr_bits);
  191. }
  192. static void pvscsi_mask_intr(const struct pvscsi_adapter *adapter)
  193. {
  194. pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_MASK, 0);
  195. }
  196. static void pvscsi_write_cmd_desc(const struct pvscsi_adapter *adapter,
  197. u32 cmd, const void *desc, size_t len)
  198. {
  199. const u32 *ptr = desc;
  200. size_t i;
  201. len /= sizeof(*ptr);
  202. pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND, cmd);
  203. for (i = 0; i < len; i++)
  204. pvscsi_reg_write(adapter,
  205. PVSCSI_REG_OFFSET_COMMAND_DATA, ptr[i]);
  206. }
  207. static void pvscsi_abort_cmd(const struct pvscsi_adapter *adapter,
  208. const struct pvscsi_ctx *ctx)
  209. {
  210. struct PVSCSICmdDescAbortCmd cmd = { 0 };
  211. cmd.target = ctx->cmd->device->id;
  212. cmd.context = pvscsi_map_context(adapter, ctx);
  213. pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_ABORT_CMD, &cmd, sizeof(cmd));
  214. }
  215. static void pvscsi_kick_rw_io(const struct pvscsi_adapter *adapter)
  216. {
  217. pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_KICK_RW_IO, 0);
  218. }
  219. static void pvscsi_process_request_ring(const struct pvscsi_adapter *adapter)
  220. {
  221. pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_KICK_NON_RW_IO, 0);
  222. }
  223. static int scsi_is_rw(unsigned char op)
  224. {
  225. return op == READ_6 || op == WRITE_6 ||
  226. op == READ_10 || op == WRITE_10 ||
  227. op == READ_12 || op == WRITE_12 ||
  228. op == READ_16 || op == WRITE_16;
  229. }
  230. static void pvscsi_kick_io(const struct pvscsi_adapter *adapter,
  231. unsigned char op)
  232. {
  233. if (scsi_is_rw(op)) {
  234. struct PVSCSIRingsState *s = adapter->rings_state;
  235. if (!adapter->use_req_threshold ||
  236. s->reqProdIdx - s->reqConsIdx >= s->reqCallThreshold)
  237. pvscsi_kick_rw_io(adapter);
  238. } else {
  239. pvscsi_process_request_ring(adapter);
  240. }
  241. }
  242. static void ll_adapter_reset(const struct pvscsi_adapter *adapter)
  243. {
  244. dev_dbg(pvscsi_dev(adapter), "Adapter Reset on %p\n", adapter);
  245. pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_ADAPTER_RESET, NULL, 0);
  246. }
  247. static void ll_bus_reset(const struct pvscsi_adapter *adapter)
  248. {
  249. dev_dbg(pvscsi_dev(adapter), "Resetting bus on %p\n", adapter);
  250. pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_RESET_BUS, NULL, 0);
  251. }
  252. static void ll_device_reset(const struct pvscsi_adapter *adapter, u32 target)
  253. {
  254. struct PVSCSICmdDescResetDevice cmd = { 0 };
  255. dev_dbg(pvscsi_dev(adapter), "Resetting device: target=%u\n", target);
  256. cmd.target = target;
  257. pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_RESET_DEVICE,
  258. &cmd, sizeof(cmd));
  259. }
  260. static void pvscsi_create_sg(struct pvscsi_ctx *ctx,
  261. struct scatterlist *sg, unsigned count)
  262. {
  263. unsigned i;
  264. struct PVSCSISGElement *sge;
  265. BUG_ON(count > PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT);
  266. sge = &ctx->sgl->sge[0];
  267. for (i = 0; i < count; i++, sg = sg_next(sg)) {
  268. sge[i].addr = sg_dma_address(sg);
  269. sge[i].length = sg_dma_len(sg);
  270. sge[i].flags = 0;
  271. }
  272. }
  273. /*
  274. * Map all data buffers for a command into PCI space and
  275. * setup the scatter/gather list if needed.
  276. */
  277. static int pvscsi_map_buffers(struct pvscsi_adapter *adapter,
  278. struct pvscsi_ctx *ctx, struct scsi_cmnd *cmd,
  279. struct PVSCSIRingReqDesc *e)
  280. {
  281. unsigned count;
  282. unsigned bufflen = scsi_bufflen(cmd);
  283. struct scatterlist *sg;
  284. e->dataLen = bufflen;
  285. e->dataAddr = 0;
  286. if (bufflen == 0)
  287. return 0;
  288. sg = scsi_sglist(cmd);
  289. count = scsi_sg_count(cmd);
  290. if (count != 0) {
  291. int segs = scsi_dma_map(cmd);
  292. if (segs == -ENOMEM) {
  293. scmd_printk(KERN_DEBUG, cmd,
  294. "vmw_pvscsi: Failed to map cmd sglist for DMA.\n");
  295. return -ENOMEM;
  296. } else if (segs > 1) {
  297. pvscsi_create_sg(ctx, sg, segs);
  298. e->flags |= PVSCSI_FLAG_CMD_WITH_SG_LIST;
  299. ctx->sglPA = dma_map_single(&adapter->dev->dev,
  300. ctx->sgl, SGL_SIZE, DMA_TO_DEVICE);
  301. if (dma_mapping_error(&adapter->dev->dev, ctx->sglPA)) {
  302. scmd_printk(KERN_ERR, cmd,
  303. "vmw_pvscsi: Failed to map ctx sglist for DMA.\n");
  304. scsi_dma_unmap(cmd);
  305. ctx->sglPA = 0;
  306. return -ENOMEM;
  307. }
  308. e->dataAddr = ctx->sglPA;
  309. } else
  310. e->dataAddr = sg_dma_address(sg);
  311. } else {
  312. /*
  313. * In case there is no S/G list, scsi_sglist points
  314. * directly to the buffer.
  315. */
  316. ctx->dataPA = dma_map_single(&adapter->dev->dev, sg, bufflen,
  317. cmd->sc_data_direction);
  318. if (dma_mapping_error(&adapter->dev->dev, ctx->dataPA)) {
  319. scmd_printk(KERN_DEBUG, cmd,
  320. "vmw_pvscsi: Failed to map direct data buffer for DMA.\n");
  321. return -ENOMEM;
  322. }
  323. e->dataAddr = ctx->dataPA;
  324. }
  325. return 0;
  326. }
  327. /*
  328. * The device incorrectly doesn't clear the first byte of the sense
  329. * buffer in some cases. We have to do it ourselves.
  330. * Otherwise we run into trouble when SWIOTLB is forced.
  331. */
  332. static void pvscsi_patch_sense(struct scsi_cmnd *cmd)
  333. {
  334. if (cmd->sense_buffer)
  335. cmd->sense_buffer[0] = 0;
  336. }
  337. static void pvscsi_unmap_buffers(const struct pvscsi_adapter *adapter,
  338. struct pvscsi_ctx *ctx)
  339. {
  340. struct scsi_cmnd *cmd;
  341. unsigned bufflen;
  342. cmd = ctx->cmd;
  343. bufflen = scsi_bufflen(cmd);
  344. if (bufflen != 0) {
  345. unsigned count = scsi_sg_count(cmd);
  346. if (count != 0) {
  347. scsi_dma_unmap(cmd);
  348. if (ctx->sglPA) {
  349. dma_unmap_single(&adapter->dev->dev, ctx->sglPA,
  350. SGL_SIZE, DMA_TO_DEVICE);
  351. ctx->sglPA = 0;
  352. }
  353. } else
  354. dma_unmap_single(&adapter->dev->dev, ctx->dataPA,
  355. bufflen, cmd->sc_data_direction);
  356. }
  357. if (cmd->sense_buffer)
  358. dma_unmap_single(&adapter->dev->dev, ctx->sensePA,
  359. SCSI_SENSE_BUFFERSIZE, DMA_FROM_DEVICE);
  360. }
  361. static int pvscsi_allocate_rings(struct pvscsi_adapter *adapter)
  362. {
  363. adapter->rings_state = dma_alloc_coherent(&adapter->dev->dev, PAGE_SIZE,
  364. &adapter->ringStatePA, GFP_KERNEL);
  365. if (!adapter->rings_state)
  366. return -ENOMEM;
  367. adapter->req_pages = min(PVSCSI_MAX_NUM_PAGES_REQ_RING,
  368. pvscsi_ring_pages);
  369. adapter->req_depth = adapter->req_pages
  370. * PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE;
  371. adapter->req_ring = dma_alloc_coherent(&adapter->dev->dev,
  372. adapter->req_pages * PAGE_SIZE, &adapter->reqRingPA,
  373. GFP_KERNEL);
  374. if (!adapter->req_ring)
  375. return -ENOMEM;
  376. adapter->cmp_pages = min(PVSCSI_MAX_NUM_PAGES_CMP_RING,
  377. pvscsi_ring_pages);
  378. adapter->cmp_ring = dma_alloc_coherent(&adapter->dev->dev,
  379. adapter->cmp_pages * PAGE_SIZE, &adapter->cmpRingPA,
  380. GFP_KERNEL);
  381. if (!adapter->cmp_ring)
  382. return -ENOMEM;
  383. BUG_ON(!IS_ALIGNED(adapter->ringStatePA, PAGE_SIZE));
  384. BUG_ON(!IS_ALIGNED(adapter->reqRingPA, PAGE_SIZE));
  385. BUG_ON(!IS_ALIGNED(adapter->cmpRingPA, PAGE_SIZE));
  386. if (!adapter->use_msg)
  387. return 0;
  388. adapter->msg_pages = min(PVSCSI_MAX_NUM_PAGES_MSG_RING,
  389. pvscsi_msg_ring_pages);
  390. adapter->msg_ring = dma_alloc_coherent(&adapter->dev->dev,
  391. adapter->msg_pages * PAGE_SIZE, &adapter->msgRingPA,
  392. GFP_KERNEL);
  393. if (!adapter->msg_ring)
  394. return -ENOMEM;
  395. BUG_ON(!IS_ALIGNED(adapter->msgRingPA, PAGE_SIZE));
  396. return 0;
  397. }
  398. static void pvscsi_setup_all_rings(const struct pvscsi_adapter *adapter)
  399. {
  400. struct PVSCSICmdDescSetupRings cmd = { 0 };
  401. dma_addr_t base;
  402. unsigned i;
  403. cmd.ringsStatePPN = adapter->ringStatePA >> PAGE_SHIFT;
  404. cmd.reqRingNumPages = adapter->req_pages;
  405. cmd.cmpRingNumPages = adapter->cmp_pages;
  406. base = adapter->reqRingPA;
  407. for (i = 0; i < adapter->req_pages; i++) {
  408. cmd.reqRingPPNs[i] = base >> PAGE_SHIFT;
  409. base += PAGE_SIZE;
  410. }
  411. base = adapter->cmpRingPA;
  412. for (i = 0; i < adapter->cmp_pages; i++) {
  413. cmd.cmpRingPPNs[i] = base >> PAGE_SHIFT;
  414. base += PAGE_SIZE;
  415. }
  416. memset(adapter->rings_state, 0, PAGE_SIZE);
  417. memset(adapter->req_ring, 0, adapter->req_pages * PAGE_SIZE);
  418. memset(adapter->cmp_ring, 0, adapter->cmp_pages * PAGE_SIZE);
  419. pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_SETUP_RINGS,
  420. &cmd, sizeof(cmd));
  421. if (adapter->use_msg) {
  422. struct PVSCSICmdDescSetupMsgRing cmd_msg = { 0 };
  423. cmd_msg.numPages = adapter->msg_pages;
  424. base = adapter->msgRingPA;
  425. for (i = 0; i < adapter->msg_pages; i++) {
  426. cmd_msg.ringPPNs[i] = base >> PAGE_SHIFT;
  427. base += PAGE_SIZE;
  428. }
  429. memset(adapter->msg_ring, 0, adapter->msg_pages * PAGE_SIZE);
  430. pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_SETUP_MSG_RING,
  431. &cmd_msg, sizeof(cmd_msg));
  432. }
  433. }
  434. static int pvscsi_change_queue_depth(struct scsi_device *sdev, int qdepth)
  435. {
  436. if (!sdev->tagged_supported)
  437. qdepth = 1;
  438. return scsi_change_queue_depth(sdev, qdepth);
  439. }
  440. /*
  441. * Pull a completion descriptor off and pass the completion back
  442. * to the SCSI mid layer.
  443. */
  444. static void pvscsi_complete_request(struct pvscsi_adapter *adapter,
  445. const struct PVSCSIRingCmpDesc *e)
  446. {
  447. struct pvscsi_ctx *ctx;
  448. struct scsi_cmnd *cmd;
  449. struct completion *abort_cmp;
  450. u32 btstat = e->hostStatus;
  451. u32 sdstat = e->scsiStatus;
  452. ctx = pvscsi_get_context(adapter, e->context);
  453. cmd = ctx->cmd;
  454. abort_cmp = ctx->abort_cmp;
  455. pvscsi_unmap_buffers(adapter, ctx);
  456. if (sdstat != SAM_STAT_CHECK_CONDITION)
  457. pvscsi_patch_sense(cmd);
  458. pvscsi_release_context(adapter, ctx);
  459. if (abort_cmp) {
  460. /*
  461. * The command was requested to be aborted. Just signal that
  462. * the request completed and swallow the actual cmd completion
  463. * here. The abort handler will post a completion for this
  464. * command indicating that it got successfully aborted.
  465. */
  466. complete(abort_cmp);
  467. return;
  468. }
  469. cmd->result = 0;
  470. if (sdstat != SAM_STAT_GOOD &&
  471. (btstat == BTSTAT_SUCCESS ||
  472. btstat == BTSTAT_LINKED_COMMAND_COMPLETED ||
  473. btstat == BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG)) {
  474. if (sdstat == SAM_STAT_COMMAND_TERMINATED) {
  475. cmd->result = (DID_RESET << 16);
  476. } else {
  477. cmd->result = (DID_OK << 16) | sdstat;
  478. }
  479. } else
  480. switch (btstat) {
  481. case BTSTAT_SUCCESS:
  482. case BTSTAT_LINKED_COMMAND_COMPLETED:
  483. case BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG:
  484. /*
  485. * Commands like INQUIRY may transfer less data than
  486. * requested by the initiator via bufflen. Set residual
  487. * count to make upper layer aware of the actual amount
  488. * of data returned. There are cases when controller
  489. * returns zero dataLen with non zero data - do not set
  490. * residual count in that case.
  491. */
  492. if (e->dataLen && (e->dataLen < scsi_bufflen(cmd)))
  493. scsi_set_resid(cmd, scsi_bufflen(cmd) - e->dataLen);
  494. cmd->result = (DID_OK << 16);
  495. break;
  496. case BTSTAT_DATARUN:
  497. case BTSTAT_DATA_UNDERRUN:
  498. /* Report residual data in underruns */
  499. scsi_set_resid(cmd, scsi_bufflen(cmd) - e->dataLen);
  500. cmd->result = (DID_ERROR << 16);
  501. break;
  502. case BTSTAT_SELTIMEO:
  503. /* Our emulation returns this for non-connected devs */
  504. cmd->result = (DID_BAD_TARGET << 16);
  505. break;
  506. case BTSTAT_LUNMISMATCH:
  507. case BTSTAT_TAGREJECT:
  508. case BTSTAT_BADMSG:
  509. case BTSTAT_HAHARDWARE:
  510. case BTSTAT_INVPHASE:
  511. case BTSTAT_HATIMEOUT:
  512. case BTSTAT_NORESPONSE:
  513. case BTSTAT_DISCONNECT:
  514. case BTSTAT_HASOFTWARE:
  515. case BTSTAT_BUSFREE:
  516. case BTSTAT_SENSFAILED:
  517. cmd->result |= (DID_ERROR << 16);
  518. break;
  519. case BTSTAT_SENTRST:
  520. case BTSTAT_RECVRST:
  521. case BTSTAT_BUSRESET:
  522. cmd->result = (DID_RESET << 16);
  523. break;
  524. case BTSTAT_ABORTQUEUE:
  525. cmd->result = (DID_BUS_BUSY << 16);
  526. break;
  527. case BTSTAT_SCSIPARITY:
  528. cmd->result = (DID_PARITY << 16);
  529. break;
  530. default:
  531. cmd->result = (DID_ERROR << 16);
  532. scmd_printk(KERN_DEBUG, cmd,
  533. "Unknown completion status: 0x%x\n",
  534. btstat);
  535. }
  536. dev_dbg(&cmd->device->sdev_gendev,
  537. "cmd=%p %x ctx=%p result=0x%x status=0x%x,%x\n",
  538. cmd, cmd->cmnd[0], ctx, cmd->result, btstat, sdstat);
  539. scsi_done(cmd);
  540. }
  541. /*
  542. * barrier usage : Since the PVSCSI device is emulated, there could be cases
  543. * where we may want to serialize some accesses between the driver and the
  544. * emulation layer. We use compiler barriers instead of the more expensive
  545. * memory barriers because PVSCSI is only supported on X86 which has strong
  546. * memory access ordering.
  547. */
  548. static void pvscsi_process_completion_ring(struct pvscsi_adapter *adapter)
  549. {
  550. struct PVSCSIRingsState *s = adapter->rings_state;
  551. struct PVSCSIRingCmpDesc *ring = adapter->cmp_ring;
  552. u32 cmp_entries = s->cmpNumEntriesLog2;
  553. while (s->cmpConsIdx != s->cmpProdIdx) {
  554. struct PVSCSIRingCmpDesc *e = ring + (s->cmpConsIdx &
  555. MASK(cmp_entries));
  556. /*
  557. * This barrier() ensures that *e is not dereferenced while
  558. * the device emulation still writes data into the slot.
  559. * Since the device emulation advances s->cmpProdIdx only after
  560. * updating the slot we want to check it first.
  561. */
  562. barrier();
  563. pvscsi_complete_request(adapter, e);
  564. /*
  565. * This barrier() ensures that compiler doesn't reorder write
  566. * to s->cmpConsIdx before the read of (*e) inside
  567. * pvscsi_complete_request. Otherwise, device emulation may
  568. * overwrite *e before we had a chance to read it.
  569. */
  570. barrier();
  571. s->cmpConsIdx++;
  572. }
  573. }
  574. /*
  575. * Translate a Linux SCSI request into a request ring entry.
  576. */
  577. static int pvscsi_queue_ring(struct pvscsi_adapter *adapter,
  578. struct pvscsi_ctx *ctx, struct scsi_cmnd *cmd)
  579. {
  580. struct PVSCSIRingsState *s;
  581. struct PVSCSIRingReqDesc *e;
  582. struct scsi_device *sdev;
  583. u32 req_entries;
  584. s = adapter->rings_state;
  585. sdev = cmd->device;
  586. req_entries = s->reqNumEntriesLog2;
  587. /*
  588. * If this condition holds, we might have room on the request ring, but
  589. * we might not have room on the completion ring for the response.
  590. * However, we have already ruled out this possibility - we would not
  591. * have successfully allocated a context if it were true, since we only
  592. * have one context per request entry. Check for it anyway, since it
  593. * would be a serious bug.
  594. */
  595. if (s->reqProdIdx - s->cmpConsIdx >= 1 << req_entries) {
  596. scmd_printk(KERN_ERR, cmd, "vmw_pvscsi: "
  597. "ring full: reqProdIdx=%d cmpConsIdx=%d\n",
  598. s->reqProdIdx, s->cmpConsIdx);
  599. return -1;
  600. }
  601. e = adapter->req_ring + (s->reqProdIdx & MASK(req_entries));
  602. e->bus = sdev->channel;
  603. e->target = sdev->id;
  604. memset(e->lun, 0, sizeof(e->lun));
  605. e->lun[1] = sdev->lun;
  606. if (cmd->sense_buffer) {
  607. ctx->sensePA = dma_map_single(&adapter->dev->dev,
  608. cmd->sense_buffer, SCSI_SENSE_BUFFERSIZE,
  609. DMA_FROM_DEVICE);
  610. if (dma_mapping_error(&adapter->dev->dev, ctx->sensePA)) {
  611. scmd_printk(KERN_DEBUG, cmd,
  612. "vmw_pvscsi: Failed to map sense buffer for DMA.\n");
  613. ctx->sensePA = 0;
  614. return -ENOMEM;
  615. }
  616. e->senseAddr = ctx->sensePA;
  617. e->senseLen = SCSI_SENSE_BUFFERSIZE;
  618. } else {
  619. e->senseLen = 0;
  620. e->senseAddr = 0;
  621. }
  622. e->cdbLen = cmd->cmd_len;
  623. e->vcpuHint = smp_processor_id();
  624. memcpy(e->cdb, cmd->cmnd, e->cdbLen);
  625. e->tag = SIMPLE_QUEUE_TAG;
  626. if (cmd->sc_data_direction == DMA_FROM_DEVICE)
  627. e->flags = PVSCSI_FLAG_CMD_DIR_TOHOST;
  628. else if (cmd->sc_data_direction == DMA_TO_DEVICE)
  629. e->flags = PVSCSI_FLAG_CMD_DIR_TODEVICE;
  630. else if (cmd->sc_data_direction == DMA_NONE)
  631. e->flags = PVSCSI_FLAG_CMD_DIR_NONE;
  632. else
  633. e->flags = 0;
  634. if (pvscsi_map_buffers(adapter, ctx, cmd, e) != 0) {
  635. if (cmd->sense_buffer) {
  636. dma_unmap_single(&adapter->dev->dev, ctx->sensePA,
  637. SCSI_SENSE_BUFFERSIZE,
  638. DMA_FROM_DEVICE);
  639. ctx->sensePA = 0;
  640. }
  641. return -ENOMEM;
  642. }
  643. e->context = pvscsi_map_context(adapter, ctx);
  644. barrier();
  645. s->reqProdIdx++;
  646. return 0;
  647. }
  648. static int pvscsi_queue_lck(struct scsi_cmnd *cmd)
  649. {
  650. struct Scsi_Host *host = cmd->device->host;
  651. struct pvscsi_adapter *adapter = shost_priv(host);
  652. struct pvscsi_ctx *ctx;
  653. unsigned long flags;
  654. unsigned char op;
  655. spin_lock_irqsave(&adapter->hw_lock, flags);
  656. ctx = pvscsi_acquire_context(adapter, cmd);
  657. if (!ctx || pvscsi_queue_ring(adapter, ctx, cmd) != 0) {
  658. if (ctx)
  659. pvscsi_release_context(adapter, ctx);
  660. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  661. return SCSI_MLQUEUE_HOST_BUSY;
  662. }
  663. op = cmd->cmnd[0];
  664. dev_dbg(&cmd->device->sdev_gendev,
  665. "queued cmd %p, ctx %p, op=%x\n", cmd, ctx, op);
  666. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  667. pvscsi_kick_io(adapter, op);
  668. return 0;
  669. }
  670. static DEF_SCSI_QCMD(pvscsi_queue)
  671. static int pvscsi_abort(struct scsi_cmnd *cmd)
  672. {
  673. struct pvscsi_adapter *adapter = shost_priv(cmd->device->host);
  674. struct pvscsi_ctx *ctx;
  675. unsigned long flags;
  676. int result = SUCCESS;
  677. DECLARE_COMPLETION_ONSTACK(abort_cmp);
  678. int done;
  679. scmd_printk(KERN_DEBUG, cmd, "task abort on host %u, %p\n",
  680. adapter->host->host_no, cmd);
  681. spin_lock_irqsave(&adapter->hw_lock, flags);
  682. /*
  683. * Poll the completion ring first - we might be trying to abort
  684. * a command that is waiting to be dispatched in the completion ring.
  685. */
  686. pvscsi_process_completion_ring(adapter);
  687. /*
  688. * If there is no context for the command, it either already succeeded
  689. * or else was never properly issued. Not our problem.
  690. */
  691. ctx = pvscsi_find_context(adapter, cmd);
  692. if (!ctx) {
  693. scmd_printk(KERN_DEBUG, cmd, "Failed to abort cmd %p\n", cmd);
  694. goto out;
  695. }
  696. /*
  697. * Mark that the command has been requested to be aborted and issue
  698. * the abort.
  699. */
  700. ctx->abort_cmp = &abort_cmp;
  701. pvscsi_abort_cmd(adapter, ctx);
  702. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  703. /* Wait for 2 secs for the completion. */
  704. done = wait_for_completion_timeout(&abort_cmp, msecs_to_jiffies(2000));
  705. spin_lock_irqsave(&adapter->hw_lock, flags);
  706. if (!done) {
  707. /*
  708. * Failed to abort the command, unmark the fact that it
  709. * was requested to be aborted.
  710. */
  711. ctx->abort_cmp = NULL;
  712. result = FAILED;
  713. scmd_printk(KERN_DEBUG, cmd,
  714. "Failed to get completion for aborted cmd %p\n",
  715. cmd);
  716. goto out;
  717. }
  718. /*
  719. * Successfully aborted the command.
  720. */
  721. cmd->result = (DID_ABORT << 16);
  722. scsi_done(cmd);
  723. out:
  724. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  725. return result;
  726. }
  727. /*
  728. * Abort all outstanding requests. This is only safe to use if the completion
  729. * ring will never be walked again or the device has been reset, because it
  730. * destroys the 1-1 mapping between context field passed to emulation and our
  731. * request structure.
  732. */
  733. static void pvscsi_reset_all(struct pvscsi_adapter *adapter)
  734. {
  735. unsigned i;
  736. for (i = 0; i < adapter->req_depth; i++) {
  737. struct pvscsi_ctx *ctx = &adapter->cmd_map[i];
  738. struct scsi_cmnd *cmd = ctx->cmd;
  739. if (cmd) {
  740. scmd_printk(KERN_ERR, cmd,
  741. "Forced reset on cmd %p\n", cmd);
  742. pvscsi_unmap_buffers(adapter, ctx);
  743. pvscsi_patch_sense(cmd);
  744. pvscsi_release_context(adapter, ctx);
  745. cmd->result = (DID_RESET << 16);
  746. scsi_done(cmd);
  747. }
  748. }
  749. }
  750. static int pvscsi_host_reset(struct scsi_cmnd *cmd)
  751. {
  752. struct Scsi_Host *host = cmd->device->host;
  753. struct pvscsi_adapter *adapter = shost_priv(host);
  754. unsigned long flags;
  755. bool use_msg;
  756. scmd_printk(KERN_INFO, cmd, "SCSI Host reset\n");
  757. spin_lock_irqsave(&adapter->hw_lock, flags);
  758. use_msg = adapter->use_msg;
  759. if (use_msg) {
  760. adapter->use_msg = false;
  761. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  762. /*
  763. * Now that we know that the ISR won't add more work on the
  764. * workqueue we can safely flush any outstanding work.
  765. */
  766. flush_workqueue(adapter->workqueue);
  767. spin_lock_irqsave(&adapter->hw_lock, flags);
  768. }
  769. /*
  770. * We're going to tear down the entire ring structure and set it back
  771. * up, so stalling new requests until all completions are flushed and
  772. * the rings are back in place.
  773. */
  774. pvscsi_process_request_ring(adapter);
  775. ll_adapter_reset(adapter);
  776. /*
  777. * Now process any completions. Note we do this AFTER adapter reset,
  778. * which is strange, but stops races where completions get posted
  779. * between processing the ring and issuing the reset. The backend will
  780. * not touch the ring memory after reset, so the immediately pre-reset
  781. * completion ring state is still valid.
  782. */
  783. pvscsi_process_completion_ring(adapter);
  784. pvscsi_reset_all(adapter);
  785. adapter->use_msg = use_msg;
  786. pvscsi_setup_all_rings(adapter);
  787. pvscsi_unmask_intr(adapter);
  788. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  789. return SUCCESS;
  790. }
  791. static int pvscsi_bus_reset(struct scsi_cmnd *cmd)
  792. {
  793. struct Scsi_Host *host = cmd->device->host;
  794. struct pvscsi_adapter *adapter = shost_priv(host);
  795. unsigned long flags;
  796. scmd_printk(KERN_INFO, cmd, "SCSI Bus reset\n");
  797. /*
  798. * We don't want to queue new requests for this bus after
  799. * flushing all pending requests to emulation, since new
  800. * requests could then sneak in during this bus reset phase,
  801. * so take the lock now.
  802. */
  803. spin_lock_irqsave(&adapter->hw_lock, flags);
  804. pvscsi_process_request_ring(adapter);
  805. ll_bus_reset(adapter);
  806. pvscsi_process_completion_ring(adapter);
  807. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  808. return SUCCESS;
  809. }
  810. static int pvscsi_device_reset(struct scsi_cmnd *cmd)
  811. {
  812. struct Scsi_Host *host = cmd->device->host;
  813. struct pvscsi_adapter *adapter = shost_priv(host);
  814. unsigned long flags;
  815. scmd_printk(KERN_INFO, cmd, "SCSI device reset on scsi%u:%u\n",
  816. host->host_no, cmd->device->id);
  817. /*
  818. * We don't want to queue new requests for this device after flushing
  819. * all pending requests to emulation, since new requests could then
  820. * sneak in during this device reset phase, so take the lock now.
  821. */
  822. spin_lock_irqsave(&adapter->hw_lock, flags);
  823. pvscsi_process_request_ring(adapter);
  824. ll_device_reset(adapter, cmd->device->id);
  825. pvscsi_process_completion_ring(adapter);
  826. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  827. return SUCCESS;
  828. }
  829. static struct scsi_host_template pvscsi_template;
  830. static const char *pvscsi_info(struct Scsi_Host *host)
  831. {
  832. struct pvscsi_adapter *adapter = shost_priv(host);
  833. static char buf[256];
  834. sprintf(buf, "VMware PVSCSI storage adapter rev %d, req/cmp/msg rings: "
  835. "%u/%u/%u pages, cmd_per_lun=%u", adapter->rev,
  836. adapter->req_pages, adapter->cmp_pages, adapter->msg_pages,
  837. pvscsi_template.cmd_per_lun);
  838. return buf;
  839. }
  840. static struct scsi_host_template pvscsi_template = {
  841. .module = THIS_MODULE,
  842. .name = "VMware PVSCSI Host Adapter",
  843. .proc_name = "vmw_pvscsi",
  844. .info = pvscsi_info,
  845. .queuecommand = pvscsi_queue,
  846. .this_id = -1,
  847. .sg_tablesize = PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT,
  848. .dma_boundary = UINT_MAX,
  849. .max_sectors = 0xffff,
  850. .change_queue_depth = pvscsi_change_queue_depth,
  851. .eh_abort_handler = pvscsi_abort,
  852. .eh_device_reset_handler = pvscsi_device_reset,
  853. .eh_bus_reset_handler = pvscsi_bus_reset,
  854. .eh_host_reset_handler = pvscsi_host_reset,
  855. };
  856. static void pvscsi_process_msg(const struct pvscsi_adapter *adapter,
  857. const struct PVSCSIRingMsgDesc *e)
  858. {
  859. struct PVSCSIRingsState *s = adapter->rings_state;
  860. struct Scsi_Host *host = adapter->host;
  861. struct scsi_device *sdev;
  862. printk(KERN_INFO "vmw_pvscsi: msg type: 0x%x - MSG RING: %u/%u (%u) \n",
  863. e->type, s->msgProdIdx, s->msgConsIdx, s->msgNumEntriesLog2);
  864. BUILD_BUG_ON(PVSCSI_MSG_LAST != 2);
  865. if (e->type == PVSCSI_MSG_DEV_ADDED) {
  866. struct PVSCSIMsgDescDevStatusChanged *desc;
  867. desc = (struct PVSCSIMsgDescDevStatusChanged *)e;
  868. printk(KERN_INFO
  869. "vmw_pvscsi: msg: device added at scsi%u:%u:%u\n",
  870. desc->bus, desc->target, desc->lun[1]);
  871. if (!scsi_host_get(host))
  872. return;
  873. sdev = scsi_device_lookup(host, desc->bus, desc->target,
  874. desc->lun[1]);
  875. if (sdev) {
  876. printk(KERN_INFO "vmw_pvscsi: device already exists\n");
  877. scsi_device_put(sdev);
  878. } else
  879. scsi_add_device(adapter->host, desc->bus,
  880. desc->target, desc->lun[1]);
  881. scsi_host_put(host);
  882. } else if (e->type == PVSCSI_MSG_DEV_REMOVED) {
  883. struct PVSCSIMsgDescDevStatusChanged *desc;
  884. desc = (struct PVSCSIMsgDescDevStatusChanged *)e;
  885. printk(KERN_INFO
  886. "vmw_pvscsi: msg: device removed at scsi%u:%u:%u\n",
  887. desc->bus, desc->target, desc->lun[1]);
  888. if (!scsi_host_get(host))
  889. return;
  890. sdev = scsi_device_lookup(host, desc->bus, desc->target,
  891. desc->lun[1]);
  892. if (sdev) {
  893. scsi_remove_device(sdev);
  894. scsi_device_put(sdev);
  895. } else
  896. printk(KERN_INFO
  897. "vmw_pvscsi: failed to lookup scsi%u:%u:%u\n",
  898. desc->bus, desc->target, desc->lun[1]);
  899. scsi_host_put(host);
  900. }
  901. }
  902. static int pvscsi_msg_pending(const struct pvscsi_adapter *adapter)
  903. {
  904. struct PVSCSIRingsState *s = adapter->rings_state;
  905. return s->msgProdIdx != s->msgConsIdx;
  906. }
  907. static void pvscsi_process_msg_ring(const struct pvscsi_adapter *adapter)
  908. {
  909. struct PVSCSIRingsState *s = adapter->rings_state;
  910. struct PVSCSIRingMsgDesc *ring = adapter->msg_ring;
  911. u32 msg_entries = s->msgNumEntriesLog2;
  912. while (pvscsi_msg_pending(adapter)) {
  913. struct PVSCSIRingMsgDesc *e = ring + (s->msgConsIdx &
  914. MASK(msg_entries));
  915. barrier();
  916. pvscsi_process_msg(adapter, e);
  917. barrier();
  918. s->msgConsIdx++;
  919. }
  920. }
  921. static void pvscsi_msg_workqueue_handler(struct work_struct *data)
  922. {
  923. struct pvscsi_adapter *adapter;
  924. adapter = container_of(data, struct pvscsi_adapter, work);
  925. pvscsi_process_msg_ring(adapter);
  926. }
  927. static int pvscsi_setup_msg_workqueue(struct pvscsi_adapter *adapter)
  928. {
  929. char name[32];
  930. if (!pvscsi_use_msg)
  931. return 0;
  932. pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND,
  933. PVSCSI_CMD_SETUP_MSG_RING);
  934. if (pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_COMMAND_STATUS) == -1)
  935. return 0;
  936. snprintf(name, sizeof(name),
  937. "vmw_pvscsi_wq_%u", adapter->host->host_no);
  938. adapter->workqueue = create_singlethread_workqueue(name);
  939. if (!adapter->workqueue) {
  940. printk(KERN_ERR "vmw_pvscsi: failed to create work queue\n");
  941. return 0;
  942. }
  943. INIT_WORK(&adapter->work, pvscsi_msg_workqueue_handler);
  944. return 1;
  945. }
  946. static bool pvscsi_setup_req_threshold(struct pvscsi_adapter *adapter,
  947. bool enable)
  948. {
  949. u32 val;
  950. if (!pvscsi_use_req_threshold)
  951. return false;
  952. pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND,
  953. PVSCSI_CMD_SETUP_REQCALLTHRESHOLD);
  954. val = pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_COMMAND_STATUS);
  955. if (val == -1) {
  956. printk(KERN_INFO "vmw_pvscsi: device does not support req_threshold\n");
  957. return false;
  958. } else {
  959. struct PVSCSICmdDescSetupReqCall cmd_msg = { 0 };
  960. cmd_msg.enable = enable;
  961. printk(KERN_INFO
  962. "vmw_pvscsi: %sabling reqCallThreshold\n",
  963. enable ? "en" : "dis");
  964. pvscsi_write_cmd_desc(adapter,
  965. PVSCSI_CMD_SETUP_REQCALLTHRESHOLD,
  966. &cmd_msg, sizeof(cmd_msg));
  967. return pvscsi_reg_read(adapter,
  968. PVSCSI_REG_OFFSET_COMMAND_STATUS) != 0;
  969. }
  970. }
  971. static irqreturn_t pvscsi_isr(int irq, void *devp)
  972. {
  973. struct pvscsi_adapter *adapter = devp;
  974. unsigned long flags;
  975. spin_lock_irqsave(&adapter->hw_lock, flags);
  976. pvscsi_process_completion_ring(adapter);
  977. if (adapter->use_msg && pvscsi_msg_pending(adapter))
  978. queue_work(adapter->workqueue, &adapter->work);
  979. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  980. return IRQ_HANDLED;
  981. }
  982. static irqreturn_t pvscsi_shared_isr(int irq, void *devp)
  983. {
  984. struct pvscsi_adapter *adapter = devp;
  985. u32 val = pvscsi_read_intr_status(adapter);
  986. if (!(val & PVSCSI_INTR_ALL_SUPPORTED))
  987. return IRQ_NONE;
  988. pvscsi_write_intr_status(devp, val);
  989. return pvscsi_isr(irq, devp);
  990. }
  991. static void pvscsi_free_sgls(const struct pvscsi_adapter *adapter)
  992. {
  993. struct pvscsi_ctx *ctx = adapter->cmd_map;
  994. unsigned i;
  995. for (i = 0; i < adapter->req_depth; ++i, ++ctx)
  996. free_pages((unsigned long)ctx->sgl, get_order(SGL_SIZE));
  997. }
  998. static void pvscsi_shutdown_intr(struct pvscsi_adapter *adapter)
  999. {
  1000. free_irq(pci_irq_vector(adapter->dev, 0), adapter);
  1001. pci_free_irq_vectors(adapter->dev);
  1002. }
  1003. static void pvscsi_release_resources(struct pvscsi_adapter *adapter)
  1004. {
  1005. if (adapter->workqueue)
  1006. destroy_workqueue(adapter->workqueue);
  1007. if (adapter->mmioBase)
  1008. pci_iounmap(adapter->dev, adapter->mmioBase);
  1009. pci_release_regions(adapter->dev);
  1010. if (adapter->cmd_map) {
  1011. pvscsi_free_sgls(adapter);
  1012. kfree(adapter->cmd_map);
  1013. }
  1014. if (adapter->rings_state)
  1015. dma_free_coherent(&adapter->dev->dev, PAGE_SIZE,
  1016. adapter->rings_state, adapter->ringStatePA);
  1017. if (adapter->req_ring)
  1018. dma_free_coherent(&adapter->dev->dev,
  1019. adapter->req_pages * PAGE_SIZE,
  1020. adapter->req_ring, adapter->reqRingPA);
  1021. if (adapter->cmp_ring)
  1022. dma_free_coherent(&adapter->dev->dev,
  1023. adapter->cmp_pages * PAGE_SIZE,
  1024. adapter->cmp_ring, adapter->cmpRingPA);
  1025. if (adapter->msg_ring)
  1026. dma_free_coherent(&adapter->dev->dev,
  1027. adapter->msg_pages * PAGE_SIZE,
  1028. adapter->msg_ring, adapter->msgRingPA);
  1029. }
  1030. /*
  1031. * Allocate scatter gather lists.
  1032. *
  1033. * These are statically allocated. Trying to be clever was not worth it.
  1034. *
  1035. * Dynamic allocation can fail, and we can't go deep into the memory
  1036. * allocator, since we're a SCSI driver, and trying too hard to allocate
  1037. * memory might generate disk I/O. We also don't want to fail disk I/O
  1038. * in that case because we can't get an allocation - the I/O could be
  1039. * trying to swap out data to free memory. Since that is pathological,
  1040. * just use a statically allocated scatter list.
  1041. *
  1042. */
  1043. static int pvscsi_allocate_sg(struct pvscsi_adapter *adapter)
  1044. {
  1045. struct pvscsi_ctx *ctx;
  1046. int i;
  1047. ctx = adapter->cmd_map;
  1048. BUILD_BUG_ON(sizeof(struct pvscsi_sg_list) > SGL_SIZE);
  1049. for (i = 0; i < adapter->req_depth; ++i, ++ctx) {
  1050. ctx->sgl = (void *)__get_free_pages(GFP_KERNEL,
  1051. get_order(SGL_SIZE));
  1052. ctx->sglPA = 0;
  1053. BUG_ON(!IS_ALIGNED(((unsigned long)ctx->sgl), PAGE_SIZE));
  1054. if (!ctx->sgl) {
  1055. for (; i >= 0; --i, --ctx) {
  1056. free_pages((unsigned long)ctx->sgl,
  1057. get_order(SGL_SIZE));
  1058. ctx->sgl = NULL;
  1059. }
  1060. return -ENOMEM;
  1061. }
  1062. }
  1063. return 0;
  1064. }
  1065. /*
  1066. * Query the device, fetch the config info and return the
  1067. * maximum number of targets on the adapter. In case of
  1068. * failure due to any reason return default i.e. 16.
  1069. */
  1070. static u32 pvscsi_get_max_targets(struct pvscsi_adapter *adapter)
  1071. {
  1072. struct PVSCSICmdDescConfigCmd cmd;
  1073. struct PVSCSIConfigPageHeader *header;
  1074. struct device *dev;
  1075. dma_addr_t configPagePA;
  1076. void *config_page;
  1077. u32 numPhys = 16;
  1078. dev = pvscsi_dev(adapter);
  1079. config_page = dma_alloc_coherent(&adapter->dev->dev, PAGE_SIZE,
  1080. &configPagePA, GFP_KERNEL);
  1081. if (!config_page) {
  1082. dev_warn(dev, "vmw_pvscsi: failed to allocate memory for config page\n");
  1083. goto exit;
  1084. }
  1085. BUG_ON(configPagePA & ~PAGE_MASK);
  1086. /* Fetch config info from the device. */
  1087. cmd.configPageAddress = ((u64)PVSCSI_CONFIG_CONTROLLER_ADDRESS) << 32;
  1088. cmd.configPageNum = PVSCSI_CONFIG_PAGE_CONTROLLER;
  1089. cmd.cmpAddr = configPagePA;
  1090. cmd._pad = 0;
  1091. /*
  1092. * Mark the completion page header with error values. If the device
  1093. * completes the command successfully, it sets the status values to
  1094. * indicate success.
  1095. */
  1096. header = config_page;
  1097. header->hostStatus = BTSTAT_INVPARAM;
  1098. header->scsiStatus = SDSTAT_CHECK;
  1099. pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_CONFIG, &cmd, sizeof cmd);
  1100. if (header->hostStatus == BTSTAT_SUCCESS &&
  1101. header->scsiStatus == SDSTAT_GOOD) {
  1102. struct PVSCSIConfigPageController *config;
  1103. config = config_page;
  1104. numPhys = config->numPhys;
  1105. } else
  1106. dev_warn(dev, "vmw_pvscsi: PVSCSI_CMD_CONFIG failed. hostStatus = 0x%x, scsiStatus = 0x%x\n",
  1107. header->hostStatus, header->scsiStatus);
  1108. dma_free_coherent(&adapter->dev->dev, PAGE_SIZE, config_page,
  1109. configPagePA);
  1110. exit:
  1111. return numPhys;
  1112. }
  1113. static int pvscsi_probe(struct pci_dev *pdev, const struct pci_device_id *id)
  1114. {
  1115. unsigned int irq_flag = PCI_IRQ_MSIX | PCI_IRQ_MSI | PCI_IRQ_LEGACY;
  1116. struct pvscsi_adapter *adapter;
  1117. struct pvscsi_adapter adapter_temp;
  1118. struct Scsi_Host *host = NULL;
  1119. unsigned int i;
  1120. int error;
  1121. u32 max_id;
  1122. error = -ENODEV;
  1123. if (pci_enable_device(pdev))
  1124. return error;
  1125. if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
  1126. printk(KERN_INFO "vmw_pvscsi: using 64bit dma\n");
  1127. } else if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) {
  1128. printk(KERN_INFO "vmw_pvscsi: using 32bit dma\n");
  1129. } else {
  1130. printk(KERN_ERR "vmw_pvscsi: failed to set DMA mask\n");
  1131. goto out_disable_device;
  1132. }
  1133. /*
  1134. * Let's use a temp pvscsi_adapter struct until we find the number of
  1135. * targets on the adapter, after that we will switch to the real
  1136. * allocated struct.
  1137. */
  1138. adapter = &adapter_temp;
  1139. memset(adapter, 0, sizeof(*adapter));
  1140. adapter->dev = pdev;
  1141. adapter->rev = pdev->revision;
  1142. if (pci_request_regions(pdev, "vmw_pvscsi")) {
  1143. printk(KERN_ERR "vmw_pvscsi: pci memory selection failed\n");
  1144. goto out_disable_device;
  1145. }
  1146. for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
  1147. if ((pci_resource_flags(pdev, i) & PCI_BASE_ADDRESS_SPACE_IO))
  1148. continue;
  1149. if (pci_resource_len(pdev, i) < PVSCSI_MEM_SPACE_SIZE)
  1150. continue;
  1151. break;
  1152. }
  1153. if (i == DEVICE_COUNT_RESOURCE) {
  1154. printk(KERN_ERR
  1155. "vmw_pvscsi: adapter has no suitable MMIO region\n");
  1156. goto out_release_resources_and_disable;
  1157. }
  1158. adapter->mmioBase = pci_iomap(pdev, i, PVSCSI_MEM_SPACE_SIZE);
  1159. if (!adapter->mmioBase) {
  1160. printk(KERN_ERR
  1161. "vmw_pvscsi: can't iomap for BAR %d memsize %lu\n",
  1162. i, PVSCSI_MEM_SPACE_SIZE);
  1163. goto out_release_resources_and_disable;
  1164. }
  1165. pci_set_master(pdev);
  1166. /*
  1167. * Ask the device for max number of targets before deciding the
  1168. * default pvscsi_ring_pages value.
  1169. */
  1170. max_id = pvscsi_get_max_targets(adapter);
  1171. printk(KERN_INFO "vmw_pvscsi: max_id: %u\n", max_id);
  1172. if (pvscsi_ring_pages == 0)
  1173. /*
  1174. * Set the right default value. Up to 16 it is 8, above it is
  1175. * max.
  1176. */
  1177. pvscsi_ring_pages = (max_id > 16) ?
  1178. PVSCSI_SETUP_RINGS_MAX_NUM_PAGES :
  1179. PVSCSI_DEFAULT_NUM_PAGES_PER_RING;
  1180. printk(KERN_INFO
  1181. "vmw_pvscsi: setting ring_pages to %d\n",
  1182. pvscsi_ring_pages);
  1183. pvscsi_template.can_queue =
  1184. min(PVSCSI_MAX_NUM_PAGES_REQ_RING, pvscsi_ring_pages) *
  1185. PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE;
  1186. pvscsi_template.cmd_per_lun =
  1187. min(pvscsi_template.can_queue, pvscsi_cmd_per_lun);
  1188. host = scsi_host_alloc(&pvscsi_template, sizeof(struct pvscsi_adapter));
  1189. if (!host) {
  1190. printk(KERN_ERR "vmw_pvscsi: failed to allocate host\n");
  1191. goto out_release_resources_and_disable;
  1192. }
  1193. /*
  1194. * Let's use the real pvscsi_adapter struct here onwards.
  1195. */
  1196. adapter = shost_priv(host);
  1197. memset(adapter, 0, sizeof(*adapter));
  1198. adapter->dev = pdev;
  1199. adapter->host = host;
  1200. /*
  1201. * Copy back what we already have to the allocated adapter struct.
  1202. */
  1203. adapter->rev = adapter_temp.rev;
  1204. adapter->mmioBase = adapter_temp.mmioBase;
  1205. spin_lock_init(&adapter->hw_lock);
  1206. host->max_channel = 0;
  1207. host->max_lun = 1;
  1208. host->max_cmd_len = 16;
  1209. host->max_id = max_id;
  1210. pci_set_drvdata(pdev, host);
  1211. ll_adapter_reset(adapter);
  1212. adapter->use_msg = pvscsi_setup_msg_workqueue(adapter);
  1213. error = pvscsi_allocate_rings(adapter);
  1214. if (error) {
  1215. printk(KERN_ERR "vmw_pvscsi: unable to allocate ring memory\n");
  1216. goto out_release_resources;
  1217. }
  1218. /*
  1219. * From this point on we should reset the adapter if anything goes
  1220. * wrong.
  1221. */
  1222. pvscsi_setup_all_rings(adapter);
  1223. adapter->cmd_map = kcalloc(adapter->req_depth,
  1224. sizeof(struct pvscsi_ctx), GFP_KERNEL);
  1225. if (!adapter->cmd_map) {
  1226. printk(KERN_ERR "vmw_pvscsi: failed to allocate memory.\n");
  1227. error = -ENOMEM;
  1228. goto out_reset_adapter;
  1229. }
  1230. INIT_LIST_HEAD(&adapter->cmd_pool);
  1231. for (i = 0; i < adapter->req_depth; i++) {
  1232. struct pvscsi_ctx *ctx = adapter->cmd_map + i;
  1233. list_add(&ctx->list, &adapter->cmd_pool);
  1234. }
  1235. error = pvscsi_allocate_sg(adapter);
  1236. if (error) {
  1237. printk(KERN_ERR "vmw_pvscsi: unable to allocate s/g table\n");
  1238. goto out_reset_adapter;
  1239. }
  1240. if (pvscsi_disable_msix)
  1241. irq_flag &= ~PCI_IRQ_MSIX;
  1242. if (pvscsi_disable_msi)
  1243. irq_flag &= ~PCI_IRQ_MSI;
  1244. error = pci_alloc_irq_vectors(adapter->dev, 1, 1, irq_flag);
  1245. if (error < 0)
  1246. goto out_reset_adapter;
  1247. adapter->use_req_threshold = pvscsi_setup_req_threshold(adapter, true);
  1248. printk(KERN_DEBUG "vmw_pvscsi: driver-based request coalescing %sabled\n",
  1249. adapter->use_req_threshold ? "en" : "dis");
  1250. if (adapter->dev->msix_enabled || adapter->dev->msi_enabled) {
  1251. printk(KERN_INFO "vmw_pvscsi: using MSI%s\n",
  1252. adapter->dev->msix_enabled ? "-X" : "");
  1253. error = request_irq(pci_irq_vector(pdev, 0), pvscsi_isr,
  1254. 0, "vmw_pvscsi", adapter);
  1255. } else {
  1256. printk(KERN_INFO "vmw_pvscsi: using INTx\n");
  1257. error = request_irq(pci_irq_vector(pdev, 0), pvscsi_shared_isr,
  1258. IRQF_SHARED, "vmw_pvscsi", adapter);
  1259. }
  1260. if (error) {
  1261. printk(KERN_ERR
  1262. "vmw_pvscsi: unable to request IRQ: %d\n", error);
  1263. goto out_reset_adapter;
  1264. }
  1265. error = scsi_add_host(host, &pdev->dev);
  1266. if (error) {
  1267. printk(KERN_ERR
  1268. "vmw_pvscsi: scsi_add_host failed: %d\n", error);
  1269. goto out_reset_adapter;
  1270. }
  1271. dev_info(&pdev->dev, "VMware PVSCSI rev %d host #%u\n",
  1272. adapter->rev, host->host_no);
  1273. pvscsi_unmask_intr(adapter);
  1274. scsi_scan_host(host);
  1275. return 0;
  1276. out_reset_adapter:
  1277. ll_adapter_reset(adapter);
  1278. out_release_resources:
  1279. pvscsi_shutdown_intr(adapter);
  1280. pvscsi_release_resources(adapter);
  1281. scsi_host_put(host);
  1282. out_disable_device:
  1283. pci_disable_device(pdev);
  1284. return error;
  1285. out_release_resources_and_disable:
  1286. pvscsi_shutdown_intr(adapter);
  1287. pvscsi_release_resources(adapter);
  1288. goto out_disable_device;
  1289. }
  1290. static void __pvscsi_shutdown(struct pvscsi_adapter *adapter)
  1291. {
  1292. pvscsi_mask_intr(adapter);
  1293. if (adapter->workqueue)
  1294. flush_workqueue(adapter->workqueue);
  1295. pvscsi_shutdown_intr(adapter);
  1296. pvscsi_process_request_ring(adapter);
  1297. pvscsi_process_completion_ring(adapter);
  1298. ll_adapter_reset(adapter);
  1299. }
  1300. static void pvscsi_shutdown(struct pci_dev *dev)
  1301. {
  1302. struct Scsi_Host *host = pci_get_drvdata(dev);
  1303. struct pvscsi_adapter *adapter = shost_priv(host);
  1304. __pvscsi_shutdown(adapter);
  1305. }
  1306. static void pvscsi_remove(struct pci_dev *pdev)
  1307. {
  1308. struct Scsi_Host *host = pci_get_drvdata(pdev);
  1309. struct pvscsi_adapter *adapter = shost_priv(host);
  1310. scsi_remove_host(host);
  1311. __pvscsi_shutdown(adapter);
  1312. pvscsi_release_resources(adapter);
  1313. scsi_host_put(host);
  1314. pci_disable_device(pdev);
  1315. }
  1316. static struct pci_driver pvscsi_pci_driver = {
  1317. .name = "vmw_pvscsi",
  1318. .id_table = pvscsi_pci_tbl,
  1319. .probe = pvscsi_probe,
  1320. .remove = pvscsi_remove,
  1321. .shutdown = pvscsi_shutdown,
  1322. };
  1323. static int __init pvscsi_init(void)
  1324. {
  1325. pr_info("%s - version %s\n",
  1326. PVSCSI_LINUX_DRIVER_DESC, PVSCSI_DRIVER_VERSION_STRING);
  1327. return pci_register_driver(&pvscsi_pci_driver);
  1328. }
  1329. static void __exit pvscsi_exit(void)
  1330. {
  1331. pci_unregister_driver(&pvscsi_pci_driver);
  1332. }
  1333. module_init(pvscsi_init);
  1334. module_exit(pvscsi_exit);