loop.c 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * NVMe over Fabrics loopback device.
  4. * Copyright (c) 2015-2016 HGST, a Western Digital Company.
  5. */
  6. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  7. #include <linux/scatterlist.h>
  8. #include <linux/blk-mq.h>
  9. #include <linux/nvme.h>
  10. #include <linux/module.h>
  11. #include <linux/parser.h>
  12. #include "nvmet.h"
  13. #include "../host/nvme.h"
  14. #include "../host/fabrics.h"
  15. #define NVME_LOOP_MAX_SEGMENTS 256
  16. struct nvme_loop_iod {
  17. struct nvme_request nvme_req;
  18. struct nvme_command cmd;
  19. struct nvme_completion cqe;
  20. struct nvmet_req req;
  21. struct nvme_loop_queue *queue;
  22. struct work_struct work;
  23. struct sg_table sg_table;
  24. struct scatterlist first_sgl[];
  25. };
  26. struct nvme_loop_ctrl {
  27. struct nvme_loop_queue *queues;
  28. struct blk_mq_tag_set admin_tag_set;
  29. struct list_head list;
  30. struct blk_mq_tag_set tag_set;
  31. struct nvme_loop_iod async_event_iod;
  32. struct nvme_ctrl ctrl;
  33. struct nvmet_port *port;
  34. };
  35. static inline struct nvme_loop_ctrl *to_loop_ctrl(struct nvme_ctrl *ctrl)
  36. {
  37. return container_of(ctrl, struct nvme_loop_ctrl, ctrl);
  38. }
  39. enum nvme_loop_queue_flags {
  40. NVME_LOOP_Q_LIVE = 0,
  41. };
  42. struct nvme_loop_queue {
  43. struct nvmet_cq nvme_cq;
  44. struct nvmet_sq nvme_sq;
  45. struct nvme_loop_ctrl *ctrl;
  46. unsigned long flags;
  47. };
  48. static LIST_HEAD(nvme_loop_ports);
  49. static DEFINE_MUTEX(nvme_loop_ports_mutex);
  50. static LIST_HEAD(nvme_loop_ctrl_list);
  51. static DEFINE_MUTEX(nvme_loop_ctrl_mutex);
  52. static void nvme_loop_queue_response(struct nvmet_req *nvme_req);
  53. static void nvme_loop_delete_ctrl(struct nvmet_ctrl *ctrl);
  54. static const struct nvmet_fabrics_ops nvme_loop_ops;
  55. static inline int nvme_loop_queue_idx(struct nvme_loop_queue *queue)
  56. {
  57. return queue - queue->ctrl->queues;
  58. }
  59. static void nvme_loop_complete_rq(struct request *req)
  60. {
  61. struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req);
  62. sg_free_table_chained(&iod->sg_table, NVME_INLINE_SG_CNT);
  63. nvme_complete_rq(req);
  64. }
  65. static struct blk_mq_tags *nvme_loop_tagset(struct nvme_loop_queue *queue)
  66. {
  67. u32 queue_idx = nvme_loop_queue_idx(queue);
  68. if (queue_idx == 0)
  69. return queue->ctrl->admin_tag_set.tags[queue_idx];
  70. return queue->ctrl->tag_set.tags[queue_idx - 1];
  71. }
  72. static void nvme_loop_queue_response(struct nvmet_req *req)
  73. {
  74. struct nvme_loop_queue *queue =
  75. container_of(req->sq, struct nvme_loop_queue, nvme_sq);
  76. struct nvme_completion *cqe = req->cqe;
  77. /*
  78. * AEN requests are special as they don't time out and can
  79. * survive any kind of queue freeze and often don't respond to
  80. * aborts. We don't even bother to allocate a struct request
  81. * for them but rather special case them here.
  82. */
  83. if (unlikely(nvme_is_aen_req(nvme_loop_queue_idx(queue),
  84. cqe->command_id))) {
  85. nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
  86. &cqe->result);
  87. } else {
  88. struct request *rq;
  89. rq = nvme_find_rq(nvme_loop_tagset(queue), cqe->command_id);
  90. if (!rq) {
  91. dev_err(queue->ctrl->ctrl.device,
  92. "got bad command_id %#x on queue %d\n",
  93. cqe->command_id, nvme_loop_queue_idx(queue));
  94. return;
  95. }
  96. if (!nvme_try_complete_req(rq, cqe->status, cqe->result))
  97. nvme_loop_complete_rq(rq);
  98. }
  99. }
  100. static void nvme_loop_execute_work(struct work_struct *work)
  101. {
  102. struct nvme_loop_iod *iod =
  103. container_of(work, struct nvme_loop_iod, work);
  104. iod->req.execute(&iod->req);
  105. }
  106. static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,
  107. const struct blk_mq_queue_data *bd)
  108. {
  109. struct nvme_ns *ns = hctx->queue->queuedata;
  110. struct nvme_loop_queue *queue = hctx->driver_data;
  111. struct request *req = bd->rq;
  112. struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req);
  113. bool queue_ready = test_bit(NVME_LOOP_Q_LIVE, &queue->flags);
  114. blk_status_t ret;
  115. if (!nvme_check_ready(&queue->ctrl->ctrl, req, queue_ready))
  116. return nvme_fail_nonready_command(&queue->ctrl->ctrl, req);
  117. ret = nvme_setup_cmd(ns, req);
  118. if (ret)
  119. return ret;
  120. blk_mq_start_request(req);
  121. iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
  122. iod->req.port = queue->ctrl->port;
  123. if (!nvmet_req_init(&iod->req, &queue->nvme_cq,
  124. &queue->nvme_sq, &nvme_loop_ops))
  125. return BLK_STS_OK;
  126. if (blk_rq_nr_phys_segments(req)) {
  127. iod->sg_table.sgl = iod->first_sgl;
  128. if (sg_alloc_table_chained(&iod->sg_table,
  129. blk_rq_nr_phys_segments(req),
  130. iod->sg_table.sgl, NVME_INLINE_SG_CNT)) {
  131. nvme_cleanup_cmd(req);
  132. return BLK_STS_RESOURCE;
  133. }
  134. iod->req.sg = iod->sg_table.sgl;
  135. iod->req.sg_cnt = blk_rq_map_sg(req->q, req, iod->sg_table.sgl);
  136. iod->req.transfer_len = blk_rq_payload_bytes(req);
  137. }
  138. queue_work(nvmet_wq, &iod->work);
  139. return BLK_STS_OK;
  140. }
  141. static void nvme_loop_submit_async_event(struct nvme_ctrl *arg)
  142. {
  143. struct nvme_loop_ctrl *ctrl = to_loop_ctrl(arg);
  144. struct nvme_loop_queue *queue = &ctrl->queues[0];
  145. struct nvme_loop_iod *iod = &ctrl->async_event_iod;
  146. memset(&iod->cmd, 0, sizeof(iod->cmd));
  147. iod->cmd.common.opcode = nvme_admin_async_event;
  148. iod->cmd.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
  149. iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
  150. if (!nvmet_req_init(&iod->req, &queue->nvme_cq, &queue->nvme_sq,
  151. &nvme_loop_ops)) {
  152. dev_err(ctrl->ctrl.device, "failed async event work\n");
  153. return;
  154. }
  155. queue_work(nvmet_wq, &iod->work);
  156. }
  157. static int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl,
  158. struct nvme_loop_iod *iod, unsigned int queue_idx)
  159. {
  160. iod->req.cmd = &iod->cmd;
  161. iod->req.cqe = &iod->cqe;
  162. iod->queue = &ctrl->queues[queue_idx];
  163. INIT_WORK(&iod->work, nvme_loop_execute_work);
  164. return 0;
  165. }
  166. static int nvme_loop_init_request(struct blk_mq_tag_set *set,
  167. struct request *req, unsigned int hctx_idx,
  168. unsigned int numa_node)
  169. {
  170. struct nvme_loop_ctrl *ctrl = to_loop_ctrl(set->driver_data);
  171. struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req);
  172. nvme_req(req)->ctrl = &ctrl->ctrl;
  173. nvme_req(req)->cmd = &iod->cmd;
  174. return nvme_loop_init_iod(ctrl, blk_mq_rq_to_pdu(req),
  175. (set == &ctrl->tag_set) ? hctx_idx + 1 : 0);
  176. }
  177. static struct lock_class_key loop_hctx_fq_lock_key;
  178. static int nvme_loop_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
  179. unsigned int hctx_idx)
  180. {
  181. struct nvme_loop_ctrl *ctrl = to_loop_ctrl(data);
  182. struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1];
  183. BUG_ON(hctx_idx >= ctrl->ctrl.queue_count);
  184. /*
  185. * flush_end_io() can be called recursively for us, so use our own
  186. * lock class key for avoiding lockdep possible recursive locking,
  187. * then we can remove the dynamically allocated lock class for each
  188. * flush queue, that way may cause horrible boot delay.
  189. */
  190. blk_mq_hctx_set_fq_lock_class(hctx, &loop_hctx_fq_lock_key);
  191. hctx->driver_data = queue;
  192. return 0;
  193. }
  194. static int nvme_loop_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
  195. unsigned int hctx_idx)
  196. {
  197. struct nvme_loop_ctrl *ctrl = to_loop_ctrl(data);
  198. struct nvme_loop_queue *queue = &ctrl->queues[0];
  199. BUG_ON(hctx_idx != 0);
  200. hctx->driver_data = queue;
  201. return 0;
  202. }
  203. static const struct blk_mq_ops nvme_loop_mq_ops = {
  204. .queue_rq = nvme_loop_queue_rq,
  205. .complete = nvme_loop_complete_rq,
  206. .init_request = nvme_loop_init_request,
  207. .init_hctx = nvme_loop_init_hctx,
  208. };
  209. static const struct blk_mq_ops nvme_loop_admin_mq_ops = {
  210. .queue_rq = nvme_loop_queue_rq,
  211. .complete = nvme_loop_complete_rq,
  212. .init_request = nvme_loop_init_request,
  213. .init_hctx = nvme_loop_init_admin_hctx,
  214. };
  215. static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl)
  216. {
  217. if (!test_and_clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags))
  218. return;
  219. nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);
  220. nvme_remove_admin_tag_set(&ctrl->ctrl);
  221. }
  222. static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl)
  223. {
  224. struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl);
  225. if (list_empty(&ctrl->list))
  226. goto free_ctrl;
  227. mutex_lock(&nvme_loop_ctrl_mutex);
  228. list_del(&ctrl->list);
  229. mutex_unlock(&nvme_loop_ctrl_mutex);
  230. if (nctrl->tagset)
  231. nvme_remove_io_tag_set(nctrl);
  232. kfree(ctrl->queues);
  233. nvmf_free_options(nctrl->opts);
  234. free_ctrl:
  235. kfree(ctrl);
  236. }
  237. static void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl)
  238. {
  239. int i;
  240. for (i = 1; i < ctrl->ctrl.queue_count; i++) {
  241. clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags);
  242. nvmet_sq_destroy(&ctrl->queues[i].nvme_sq);
  243. }
  244. ctrl->ctrl.queue_count = 1;
  245. }
  246. static int nvme_loop_init_io_queues(struct nvme_loop_ctrl *ctrl)
  247. {
  248. struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
  249. unsigned int nr_io_queues;
  250. int ret, i;
  251. nr_io_queues = min(opts->nr_io_queues, num_online_cpus());
  252. ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
  253. if (ret || !nr_io_queues)
  254. return ret;
  255. dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues);
  256. for (i = 1; i <= nr_io_queues; i++) {
  257. ctrl->queues[i].ctrl = ctrl;
  258. ret = nvmet_sq_init(&ctrl->queues[i].nvme_sq);
  259. if (ret)
  260. goto out_destroy_queues;
  261. ctrl->ctrl.queue_count++;
  262. }
  263. return 0;
  264. out_destroy_queues:
  265. nvme_loop_destroy_io_queues(ctrl);
  266. return ret;
  267. }
  268. static int nvme_loop_connect_io_queues(struct nvme_loop_ctrl *ctrl)
  269. {
  270. int i, ret;
  271. for (i = 1; i < ctrl->ctrl.queue_count; i++) {
  272. ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
  273. if (ret)
  274. return ret;
  275. set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags);
  276. }
  277. return 0;
  278. }
  279. static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl)
  280. {
  281. int error;
  282. ctrl->queues[0].ctrl = ctrl;
  283. error = nvmet_sq_init(&ctrl->queues[0].nvme_sq);
  284. if (error)
  285. return error;
  286. ctrl->ctrl.queue_count = 1;
  287. error = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set,
  288. &nvme_loop_admin_mq_ops,
  289. sizeof(struct nvme_loop_iod) +
  290. NVME_INLINE_SG_CNT * sizeof(struct scatterlist));
  291. if (error)
  292. goto out_free_sq;
  293. /* reset stopped state for the fresh admin queue */
  294. clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->ctrl.flags);
  295. error = nvmf_connect_admin_queue(&ctrl->ctrl);
  296. if (error)
  297. goto out_cleanup_tagset;
  298. set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);
  299. error = nvme_enable_ctrl(&ctrl->ctrl);
  300. if (error)
  301. goto out_cleanup_tagset;
  302. ctrl->ctrl.max_hw_sectors =
  303. (NVME_LOOP_MAX_SEGMENTS - 1) << (PAGE_SHIFT - 9);
  304. nvme_start_admin_queue(&ctrl->ctrl);
  305. error = nvme_init_ctrl_finish(&ctrl->ctrl);
  306. if (error)
  307. goto out_cleanup_tagset;
  308. return 0;
  309. out_cleanup_tagset:
  310. clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);
  311. nvme_remove_admin_tag_set(&ctrl->ctrl);
  312. out_free_sq:
  313. nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);
  314. return error;
  315. }
  316. static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl)
  317. {
  318. if (ctrl->ctrl.queue_count > 1) {
  319. nvme_stop_queues(&ctrl->ctrl);
  320. nvme_cancel_tagset(&ctrl->ctrl);
  321. nvme_loop_destroy_io_queues(ctrl);
  322. }
  323. nvme_stop_admin_queue(&ctrl->ctrl);
  324. if (ctrl->ctrl.state == NVME_CTRL_LIVE)
  325. nvme_shutdown_ctrl(&ctrl->ctrl);
  326. nvme_cancel_admin_tagset(&ctrl->ctrl);
  327. nvme_loop_destroy_admin_queue(ctrl);
  328. }
  329. static void nvme_loop_delete_ctrl_host(struct nvme_ctrl *ctrl)
  330. {
  331. nvme_loop_shutdown_ctrl(to_loop_ctrl(ctrl));
  332. }
  333. static void nvme_loop_delete_ctrl(struct nvmet_ctrl *nctrl)
  334. {
  335. struct nvme_loop_ctrl *ctrl;
  336. mutex_lock(&nvme_loop_ctrl_mutex);
  337. list_for_each_entry(ctrl, &nvme_loop_ctrl_list, list) {
  338. if (ctrl->ctrl.cntlid == nctrl->cntlid)
  339. nvme_delete_ctrl(&ctrl->ctrl);
  340. }
  341. mutex_unlock(&nvme_loop_ctrl_mutex);
  342. }
  343. static void nvme_loop_reset_ctrl_work(struct work_struct *work)
  344. {
  345. struct nvme_loop_ctrl *ctrl =
  346. container_of(work, struct nvme_loop_ctrl, ctrl.reset_work);
  347. int ret;
  348. nvme_stop_ctrl(&ctrl->ctrl);
  349. nvme_loop_shutdown_ctrl(ctrl);
  350. if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
  351. if (ctrl->ctrl.state != NVME_CTRL_DELETING &&
  352. ctrl->ctrl.state != NVME_CTRL_DELETING_NOIO)
  353. /* state change failure for non-deleted ctrl? */
  354. WARN_ON_ONCE(1);
  355. return;
  356. }
  357. ret = nvme_loop_configure_admin_queue(ctrl);
  358. if (ret)
  359. goto out_disable;
  360. ret = nvme_loop_init_io_queues(ctrl);
  361. if (ret)
  362. goto out_destroy_admin;
  363. ret = nvme_loop_connect_io_queues(ctrl);
  364. if (ret)
  365. goto out_destroy_io;
  366. blk_mq_update_nr_hw_queues(&ctrl->tag_set,
  367. ctrl->ctrl.queue_count - 1);
  368. if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE))
  369. WARN_ON_ONCE(1);
  370. nvme_start_ctrl(&ctrl->ctrl);
  371. return;
  372. out_destroy_io:
  373. nvme_loop_destroy_io_queues(ctrl);
  374. out_destroy_admin:
  375. nvme_loop_destroy_admin_queue(ctrl);
  376. out_disable:
  377. dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");
  378. nvme_uninit_ctrl(&ctrl->ctrl);
  379. }
  380. static const struct nvme_ctrl_ops nvme_loop_ctrl_ops = {
  381. .name = "loop",
  382. .module = THIS_MODULE,
  383. .flags = NVME_F_FABRICS,
  384. .reg_read32 = nvmf_reg_read32,
  385. .reg_read64 = nvmf_reg_read64,
  386. .reg_write32 = nvmf_reg_write32,
  387. .free_ctrl = nvme_loop_free_ctrl,
  388. .submit_async_event = nvme_loop_submit_async_event,
  389. .delete_ctrl = nvme_loop_delete_ctrl_host,
  390. .get_address = nvmf_get_address,
  391. };
  392. static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl)
  393. {
  394. int ret;
  395. ret = nvme_loop_init_io_queues(ctrl);
  396. if (ret)
  397. return ret;
  398. ret = nvme_alloc_io_tag_set(&ctrl->ctrl, &ctrl->tag_set,
  399. &nvme_loop_mq_ops, 1,
  400. sizeof(struct nvme_loop_iod) +
  401. NVME_INLINE_SG_CNT * sizeof(struct scatterlist));
  402. if (ret)
  403. goto out_destroy_queues;
  404. ret = nvme_loop_connect_io_queues(ctrl);
  405. if (ret)
  406. goto out_cleanup_tagset;
  407. return 0;
  408. out_cleanup_tagset:
  409. nvme_remove_io_tag_set(&ctrl->ctrl);
  410. out_destroy_queues:
  411. nvme_loop_destroy_io_queues(ctrl);
  412. return ret;
  413. }
  414. static struct nvmet_port *nvme_loop_find_port(struct nvme_ctrl *ctrl)
  415. {
  416. struct nvmet_port *p, *found = NULL;
  417. mutex_lock(&nvme_loop_ports_mutex);
  418. list_for_each_entry(p, &nvme_loop_ports, entry) {
  419. /* if no transport address is specified use the first port */
  420. if ((ctrl->opts->mask & NVMF_OPT_TRADDR) &&
  421. strcmp(ctrl->opts->traddr, p->disc_addr.traddr))
  422. continue;
  423. found = p;
  424. break;
  425. }
  426. mutex_unlock(&nvme_loop_ports_mutex);
  427. return found;
  428. }
  429. static struct nvme_ctrl *nvme_loop_create_ctrl(struct device *dev,
  430. struct nvmf_ctrl_options *opts)
  431. {
  432. struct nvme_loop_ctrl *ctrl;
  433. int ret;
  434. ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
  435. if (!ctrl)
  436. return ERR_PTR(-ENOMEM);
  437. ctrl->ctrl.opts = opts;
  438. INIT_LIST_HEAD(&ctrl->list);
  439. INIT_WORK(&ctrl->ctrl.reset_work, nvme_loop_reset_ctrl_work);
  440. ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_loop_ctrl_ops,
  441. 0 /* no quirks, we're perfect! */);
  442. if (ret) {
  443. kfree(ctrl);
  444. goto out;
  445. }
  446. if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING))
  447. WARN_ON_ONCE(1);
  448. ret = -ENOMEM;
  449. ctrl->ctrl.kato = opts->kato;
  450. ctrl->port = nvme_loop_find_port(&ctrl->ctrl);
  451. ctrl->queues = kcalloc(opts->nr_io_queues + 1, sizeof(*ctrl->queues),
  452. GFP_KERNEL);
  453. if (!ctrl->queues)
  454. goto out_uninit_ctrl;
  455. ret = nvme_loop_configure_admin_queue(ctrl);
  456. if (ret)
  457. goto out_free_queues;
  458. if (opts->queue_size > ctrl->ctrl.maxcmd) {
  459. /* warn if maxcmd is lower than queue_size */
  460. dev_warn(ctrl->ctrl.device,
  461. "queue_size %zu > ctrl maxcmd %u, clamping down\n",
  462. opts->queue_size, ctrl->ctrl.maxcmd);
  463. opts->queue_size = ctrl->ctrl.maxcmd;
  464. }
  465. ctrl->ctrl.sqsize = opts->queue_size - 1;
  466. if (opts->nr_io_queues) {
  467. ret = nvme_loop_create_io_queues(ctrl);
  468. if (ret)
  469. goto out_remove_admin_queue;
  470. }
  471. nvme_loop_init_iod(ctrl, &ctrl->async_event_iod, 0);
  472. dev_info(ctrl->ctrl.device,
  473. "new ctrl: \"%s\"\n", ctrl->ctrl.opts->subsysnqn);
  474. if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE))
  475. WARN_ON_ONCE(1);
  476. mutex_lock(&nvme_loop_ctrl_mutex);
  477. list_add_tail(&ctrl->list, &nvme_loop_ctrl_list);
  478. mutex_unlock(&nvme_loop_ctrl_mutex);
  479. nvme_start_ctrl(&ctrl->ctrl);
  480. return &ctrl->ctrl;
  481. out_remove_admin_queue:
  482. nvme_loop_destroy_admin_queue(ctrl);
  483. out_free_queues:
  484. kfree(ctrl->queues);
  485. out_uninit_ctrl:
  486. nvme_uninit_ctrl(&ctrl->ctrl);
  487. nvme_put_ctrl(&ctrl->ctrl);
  488. out:
  489. if (ret > 0)
  490. ret = -EIO;
  491. return ERR_PTR(ret);
  492. }
  493. static int nvme_loop_add_port(struct nvmet_port *port)
  494. {
  495. mutex_lock(&nvme_loop_ports_mutex);
  496. list_add_tail(&port->entry, &nvme_loop_ports);
  497. mutex_unlock(&nvme_loop_ports_mutex);
  498. return 0;
  499. }
  500. static void nvme_loop_remove_port(struct nvmet_port *port)
  501. {
  502. mutex_lock(&nvme_loop_ports_mutex);
  503. list_del_init(&port->entry);
  504. mutex_unlock(&nvme_loop_ports_mutex);
  505. /*
  506. * Ensure any ctrls that are in the process of being
  507. * deleted are in fact deleted before we return
  508. * and free the port. This is to prevent active
  509. * ctrls from using a port after it's freed.
  510. */
  511. flush_workqueue(nvme_delete_wq);
  512. }
  513. static const struct nvmet_fabrics_ops nvme_loop_ops = {
  514. .owner = THIS_MODULE,
  515. .type = NVMF_TRTYPE_LOOP,
  516. .add_port = nvme_loop_add_port,
  517. .remove_port = nvme_loop_remove_port,
  518. .queue_response = nvme_loop_queue_response,
  519. .delete_ctrl = nvme_loop_delete_ctrl,
  520. };
  521. static struct nvmf_transport_ops nvme_loop_transport = {
  522. .name = "loop",
  523. .module = THIS_MODULE,
  524. .create_ctrl = nvme_loop_create_ctrl,
  525. .allowed_opts = NVMF_OPT_TRADDR,
  526. };
  527. static int __init nvme_loop_init_module(void)
  528. {
  529. int ret;
  530. ret = nvmet_register_transport(&nvme_loop_ops);
  531. if (ret)
  532. return ret;
  533. ret = nvmf_register_transport(&nvme_loop_transport);
  534. if (ret)
  535. nvmet_unregister_transport(&nvme_loop_ops);
  536. return ret;
  537. }
  538. static void __exit nvme_loop_cleanup_module(void)
  539. {
  540. struct nvme_loop_ctrl *ctrl, *next;
  541. nvmf_unregister_transport(&nvme_loop_transport);
  542. nvmet_unregister_transport(&nvme_loop_ops);
  543. mutex_lock(&nvme_loop_ctrl_mutex);
  544. list_for_each_entry_safe(ctrl, next, &nvme_loop_ctrl_list, list)
  545. nvme_delete_ctrl(&ctrl->ctrl);
  546. mutex_unlock(&nvme_loop_ctrl_mutex);
  547. flush_workqueue(nvme_delete_wq);
  548. }
  549. module_init(nvme_loop_init_module);
  550. module_exit(nvme_loop_cleanup_module);
  551. MODULE_LICENSE("GPL v2");
  552. MODULE_ALIAS("nvmet-transport-254"); /* 254 == NVMF_TRTYPE_LOOP */