dm-mpath.c 56 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256
  1. /*
  2. * Copyright (C) 2003 Sistina Software Limited.
  3. * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
  4. *
  5. * This file is released under the GPL.
  6. */
  7. #include <linux/device-mapper.h>
  8. #include "dm-rq.h"
  9. #include "dm-bio-record.h"
  10. #include "dm-path-selector.h"
  11. #include "dm-uevent.h"
  12. #include <linux/blkdev.h>
  13. #include <linux/ctype.h>
  14. #include <linux/init.h>
  15. #include <linux/mempool.h>
  16. #include <linux/module.h>
  17. #include <linux/pagemap.h>
  18. #include <linux/slab.h>
  19. #include <linux/time.h>
  20. #include <linux/timer.h>
  21. #include <linux/workqueue.h>
  22. #include <linux/delay.h>
  23. #include <scsi/scsi_dh.h>
  24. #include <linux/atomic.h>
  25. #include <linux/blk-mq.h>
  26. #define DM_MSG_PREFIX "multipath"
  27. #define DM_PG_INIT_DELAY_MSECS 2000
  28. #define DM_PG_INIT_DELAY_DEFAULT ((unsigned int) -1)
  29. #define QUEUE_IF_NO_PATH_TIMEOUT_DEFAULT 0
  30. static unsigned long queue_if_no_path_timeout_secs = QUEUE_IF_NO_PATH_TIMEOUT_DEFAULT;
  31. /* Path properties */
  32. struct pgpath {
  33. struct list_head list;
  34. struct priority_group *pg; /* Owning PG */
  35. unsigned int fail_count; /* Cumulative failure count */
  36. struct dm_path path;
  37. struct delayed_work activate_path;
  38. bool is_active:1; /* Path status */
  39. };
  40. #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
  41. /*
  42. * Paths are grouped into Priority Groups and numbered from 1 upwards.
  43. * Each has a path selector which controls which path gets used.
  44. */
  45. struct priority_group {
  46. struct list_head list;
  47. struct multipath *m; /* Owning multipath instance */
  48. struct path_selector ps;
  49. unsigned int pg_num; /* Reference number */
  50. unsigned int nr_pgpaths; /* Number of paths in PG */
  51. struct list_head pgpaths;
  52. bool bypassed:1; /* Temporarily bypass this PG? */
  53. };
  54. /* Multipath context */
  55. struct multipath {
  56. unsigned long flags; /* Multipath state flags */
  57. spinlock_t lock;
  58. enum dm_queue_mode queue_mode;
  59. struct pgpath *current_pgpath;
  60. struct priority_group *current_pg;
  61. struct priority_group *next_pg; /* Switch to this PG if set */
  62. atomic_t nr_valid_paths; /* Total number of usable paths */
  63. unsigned int nr_priority_groups;
  64. struct list_head priority_groups;
  65. const char *hw_handler_name;
  66. char *hw_handler_params;
  67. wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */
  68. unsigned int pg_init_retries; /* Number of times to retry pg_init */
  69. unsigned int pg_init_delay_msecs; /* Number of msecs before pg_init retry */
  70. atomic_t pg_init_in_progress; /* Only one pg_init allowed at once */
  71. atomic_t pg_init_count; /* Number of times pg_init called */
  72. struct mutex work_mutex;
  73. struct work_struct trigger_event;
  74. struct dm_target *ti;
  75. struct work_struct process_queued_bios;
  76. struct bio_list queued_bios;
  77. struct timer_list nopath_timer; /* Timeout for queue_if_no_path */
  78. };
  79. /*
  80. * Context information attached to each io we process.
  81. */
  82. struct dm_mpath_io {
  83. struct pgpath *pgpath;
  84. size_t nr_bytes;
  85. u64 start_time_ns;
  86. };
  87. typedef int (*action_fn) (struct pgpath *pgpath);
  88. static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
  89. static void trigger_event(struct work_struct *work);
  90. static void activate_or_offline_path(struct pgpath *pgpath);
  91. static void activate_path_work(struct work_struct *work);
  92. static void process_queued_bios(struct work_struct *work);
  93. static void queue_if_no_path_timeout_work(struct timer_list *t);
  94. /*-----------------------------------------------
  95. * Multipath state flags.
  96. *-----------------------------------------------*/
  97. #define MPATHF_QUEUE_IO 0 /* Must we queue all I/O? */
  98. #define MPATHF_QUEUE_IF_NO_PATH 1 /* Queue I/O if last path fails? */
  99. #define MPATHF_SAVED_QUEUE_IF_NO_PATH 2 /* Saved state during suspension */
  100. #define MPATHF_RETAIN_ATTACHED_HW_HANDLER 3 /* If there's already a hw_handler present, don't change it. */
  101. #define MPATHF_PG_INIT_DISABLED 4 /* pg_init is not currently allowed */
  102. #define MPATHF_PG_INIT_REQUIRED 5 /* pg_init needs calling? */
  103. #define MPATHF_PG_INIT_DELAY_RETRY 6 /* Delay pg_init retry? */
  104. static bool mpath_double_check_test_bit(int MPATHF_bit, struct multipath *m)
  105. {
  106. bool r = test_bit(MPATHF_bit, &m->flags);
  107. if (r) {
  108. unsigned long flags;
  109. spin_lock_irqsave(&m->lock, flags);
  110. r = test_bit(MPATHF_bit, &m->flags);
  111. spin_unlock_irqrestore(&m->lock, flags);
  112. }
  113. return r;
  114. }
  115. /*-----------------------------------------------
  116. * Allocation routines
  117. *-----------------------------------------------*/
  118. static struct pgpath *alloc_pgpath(void)
  119. {
  120. struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
  121. if (!pgpath)
  122. return NULL;
  123. pgpath->is_active = true;
  124. return pgpath;
  125. }
  126. static void free_pgpath(struct pgpath *pgpath)
  127. {
  128. kfree(pgpath);
  129. }
  130. static struct priority_group *alloc_priority_group(void)
  131. {
  132. struct priority_group *pg;
  133. pg = kzalloc(sizeof(*pg), GFP_KERNEL);
  134. if (pg)
  135. INIT_LIST_HEAD(&pg->pgpaths);
  136. return pg;
  137. }
  138. static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
  139. {
  140. struct pgpath *pgpath, *tmp;
  141. list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
  142. list_del(&pgpath->list);
  143. dm_put_device(ti, pgpath->path.dev);
  144. free_pgpath(pgpath);
  145. }
  146. }
  147. static void free_priority_group(struct priority_group *pg,
  148. struct dm_target *ti)
  149. {
  150. struct path_selector *ps = &pg->ps;
  151. if (ps->type) {
  152. ps->type->destroy(ps);
  153. dm_put_path_selector(ps->type);
  154. }
  155. free_pgpaths(&pg->pgpaths, ti);
  156. kfree(pg);
  157. }
  158. static struct multipath *alloc_multipath(struct dm_target *ti)
  159. {
  160. struct multipath *m;
  161. m = kzalloc(sizeof(*m), GFP_KERNEL);
  162. if (m) {
  163. INIT_LIST_HEAD(&m->priority_groups);
  164. spin_lock_init(&m->lock);
  165. atomic_set(&m->nr_valid_paths, 0);
  166. INIT_WORK(&m->trigger_event, trigger_event);
  167. mutex_init(&m->work_mutex);
  168. m->queue_mode = DM_TYPE_NONE;
  169. m->ti = ti;
  170. ti->private = m;
  171. timer_setup(&m->nopath_timer, queue_if_no_path_timeout_work, 0);
  172. }
  173. return m;
  174. }
  175. static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m)
  176. {
  177. if (m->queue_mode == DM_TYPE_NONE) {
  178. m->queue_mode = DM_TYPE_REQUEST_BASED;
  179. } else if (m->queue_mode == DM_TYPE_BIO_BASED) {
  180. INIT_WORK(&m->process_queued_bios, process_queued_bios);
  181. /*
  182. * bio-based doesn't support any direct scsi_dh management;
  183. * it just discovers if a scsi_dh is attached.
  184. */
  185. set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
  186. }
  187. dm_table_set_type(ti->table, m->queue_mode);
  188. /*
  189. * Init fields that are only used when a scsi_dh is attached
  190. * - must do this unconditionally (really doesn't hurt non-SCSI uses)
  191. */
  192. set_bit(MPATHF_QUEUE_IO, &m->flags);
  193. atomic_set(&m->pg_init_in_progress, 0);
  194. atomic_set(&m->pg_init_count, 0);
  195. m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
  196. init_waitqueue_head(&m->pg_init_wait);
  197. return 0;
  198. }
  199. static void free_multipath(struct multipath *m)
  200. {
  201. struct priority_group *pg, *tmp;
  202. list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
  203. list_del(&pg->list);
  204. free_priority_group(pg, m->ti);
  205. }
  206. kfree(m->hw_handler_name);
  207. kfree(m->hw_handler_params);
  208. mutex_destroy(&m->work_mutex);
  209. kfree(m);
  210. }
  211. static struct dm_mpath_io *get_mpio(union map_info *info)
  212. {
  213. return info->ptr;
  214. }
  215. static size_t multipath_per_bio_data_size(void)
  216. {
  217. return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details);
  218. }
  219. static struct dm_mpath_io *get_mpio_from_bio(struct bio *bio)
  220. {
  221. return dm_per_bio_data(bio, multipath_per_bio_data_size());
  222. }
  223. static struct dm_bio_details *get_bio_details_from_mpio(struct dm_mpath_io *mpio)
  224. {
  225. /* dm_bio_details is immediately after the dm_mpath_io in bio's per-bio-data */
  226. void *bio_details = mpio + 1;
  227. return bio_details;
  228. }
  229. static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p)
  230. {
  231. struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
  232. struct dm_bio_details *bio_details = get_bio_details_from_mpio(mpio);
  233. mpio->nr_bytes = bio->bi_iter.bi_size;
  234. mpio->pgpath = NULL;
  235. mpio->start_time_ns = 0;
  236. *mpio_p = mpio;
  237. dm_bio_record(bio_details, bio);
  238. }
  239. /*-----------------------------------------------
  240. * Path selection
  241. *-----------------------------------------------*/
  242. static int __pg_init_all_paths(struct multipath *m)
  243. {
  244. struct pgpath *pgpath;
  245. unsigned long pg_init_delay = 0;
  246. lockdep_assert_held(&m->lock);
  247. if (atomic_read(&m->pg_init_in_progress) || test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
  248. return 0;
  249. atomic_inc(&m->pg_init_count);
  250. clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
  251. /* Check here to reset pg_init_required */
  252. if (!m->current_pg)
  253. return 0;
  254. if (test_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags))
  255. pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ?
  256. m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS);
  257. list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) {
  258. /* Skip failed paths */
  259. if (!pgpath->is_active)
  260. continue;
  261. if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path,
  262. pg_init_delay))
  263. atomic_inc(&m->pg_init_in_progress);
  264. }
  265. return atomic_read(&m->pg_init_in_progress);
  266. }
  267. static int pg_init_all_paths(struct multipath *m)
  268. {
  269. int ret;
  270. unsigned long flags;
  271. spin_lock_irqsave(&m->lock, flags);
  272. ret = __pg_init_all_paths(m);
  273. spin_unlock_irqrestore(&m->lock, flags);
  274. return ret;
  275. }
  276. static void __switch_pg(struct multipath *m, struct priority_group *pg)
  277. {
  278. lockdep_assert_held(&m->lock);
  279. m->current_pg = pg;
  280. /* Must we initialise the PG first, and queue I/O till it's ready? */
  281. if (m->hw_handler_name) {
  282. set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
  283. set_bit(MPATHF_QUEUE_IO, &m->flags);
  284. } else {
  285. clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
  286. clear_bit(MPATHF_QUEUE_IO, &m->flags);
  287. }
  288. atomic_set(&m->pg_init_count, 0);
  289. }
  290. static struct pgpath *choose_path_in_pg(struct multipath *m,
  291. struct priority_group *pg,
  292. size_t nr_bytes)
  293. {
  294. unsigned long flags;
  295. struct dm_path *path;
  296. struct pgpath *pgpath;
  297. path = pg->ps.type->select_path(&pg->ps, nr_bytes);
  298. if (!path)
  299. return ERR_PTR(-ENXIO);
  300. pgpath = path_to_pgpath(path);
  301. if (unlikely(READ_ONCE(m->current_pg) != pg)) {
  302. /* Only update current_pgpath if pg changed */
  303. spin_lock_irqsave(&m->lock, flags);
  304. m->current_pgpath = pgpath;
  305. __switch_pg(m, pg);
  306. spin_unlock_irqrestore(&m->lock, flags);
  307. }
  308. return pgpath;
  309. }
  310. static struct pgpath *choose_pgpath(struct multipath *m, size_t nr_bytes)
  311. {
  312. unsigned long flags;
  313. struct priority_group *pg;
  314. struct pgpath *pgpath;
  315. unsigned int bypassed = 1;
  316. if (!atomic_read(&m->nr_valid_paths)) {
  317. spin_lock_irqsave(&m->lock, flags);
  318. clear_bit(MPATHF_QUEUE_IO, &m->flags);
  319. spin_unlock_irqrestore(&m->lock, flags);
  320. goto failed;
  321. }
  322. /* Were we instructed to switch PG? */
  323. if (READ_ONCE(m->next_pg)) {
  324. spin_lock_irqsave(&m->lock, flags);
  325. pg = m->next_pg;
  326. if (!pg) {
  327. spin_unlock_irqrestore(&m->lock, flags);
  328. goto check_current_pg;
  329. }
  330. m->next_pg = NULL;
  331. spin_unlock_irqrestore(&m->lock, flags);
  332. pgpath = choose_path_in_pg(m, pg, nr_bytes);
  333. if (!IS_ERR_OR_NULL(pgpath))
  334. return pgpath;
  335. }
  336. /* Don't change PG until it has no remaining paths */
  337. check_current_pg:
  338. pg = READ_ONCE(m->current_pg);
  339. if (pg) {
  340. pgpath = choose_path_in_pg(m, pg, nr_bytes);
  341. if (!IS_ERR_OR_NULL(pgpath))
  342. return pgpath;
  343. }
  344. /*
  345. * Loop through priority groups until we find a valid path.
  346. * First time we skip PGs marked 'bypassed'.
  347. * Second time we only try the ones we skipped, but set
  348. * pg_init_delay_retry so we do not hammer controllers.
  349. */
  350. do {
  351. list_for_each_entry(pg, &m->priority_groups, list) {
  352. if (pg->bypassed == !!bypassed)
  353. continue;
  354. pgpath = choose_path_in_pg(m, pg, nr_bytes);
  355. if (!IS_ERR_OR_NULL(pgpath)) {
  356. if (!bypassed) {
  357. spin_lock_irqsave(&m->lock, flags);
  358. set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
  359. spin_unlock_irqrestore(&m->lock, flags);
  360. }
  361. return pgpath;
  362. }
  363. }
  364. } while (bypassed--);
  365. failed:
  366. spin_lock_irqsave(&m->lock, flags);
  367. m->current_pgpath = NULL;
  368. m->current_pg = NULL;
  369. spin_unlock_irqrestore(&m->lock, flags);
  370. return NULL;
  371. }
  372. /*
  373. * dm_report_EIO() is a macro instead of a function to make pr_debug_ratelimited()
  374. * report the function name and line number of the function from which
  375. * it has been invoked.
  376. */
  377. #define dm_report_EIO(m) \
  378. do { \
  379. DMDEBUG_LIMIT("%s: returning EIO; QIFNP = %d; SQIFNP = %d; DNFS = %d", \
  380. dm_table_device_name((m)->ti->table), \
  381. test_bit(MPATHF_QUEUE_IF_NO_PATH, &(m)->flags), \
  382. test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &(m)->flags), \
  383. dm_noflush_suspending((m)->ti)); \
  384. } while (0)
  385. /*
  386. * Check whether bios must be queued in the device-mapper core rather
  387. * than here in the target.
  388. */
  389. static bool __must_push_back(struct multipath *m)
  390. {
  391. return dm_noflush_suspending(m->ti);
  392. }
  393. static bool must_push_back_rq(struct multipath *m)
  394. {
  395. unsigned long flags;
  396. bool ret;
  397. spin_lock_irqsave(&m->lock, flags);
  398. ret = (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) || __must_push_back(m));
  399. spin_unlock_irqrestore(&m->lock, flags);
  400. return ret;
  401. }
  402. /*
  403. * Map cloned requests (request-based multipath)
  404. */
  405. static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
  406. union map_info *map_context,
  407. struct request **__clone)
  408. {
  409. struct multipath *m = ti->private;
  410. size_t nr_bytes = blk_rq_bytes(rq);
  411. struct pgpath *pgpath;
  412. struct block_device *bdev;
  413. struct dm_mpath_io *mpio = get_mpio(map_context);
  414. struct request_queue *q;
  415. struct request *clone;
  416. /* Do we need to select a new pgpath? */
  417. pgpath = READ_ONCE(m->current_pgpath);
  418. if (!pgpath || !mpath_double_check_test_bit(MPATHF_QUEUE_IO, m))
  419. pgpath = choose_pgpath(m, nr_bytes);
  420. if (!pgpath) {
  421. if (must_push_back_rq(m))
  422. return DM_MAPIO_DELAY_REQUEUE;
  423. dm_report_EIO(m); /* Failed */
  424. return DM_MAPIO_KILL;
  425. } else if (mpath_double_check_test_bit(MPATHF_QUEUE_IO, m) ||
  426. mpath_double_check_test_bit(MPATHF_PG_INIT_REQUIRED, m)) {
  427. pg_init_all_paths(m);
  428. return DM_MAPIO_DELAY_REQUEUE;
  429. }
  430. mpio->pgpath = pgpath;
  431. mpio->nr_bytes = nr_bytes;
  432. bdev = pgpath->path.dev->bdev;
  433. q = bdev_get_queue(bdev);
  434. clone = blk_mq_alloc_request(q, rq->cmd_flags | REQ_NOMERGE,
  435. BLK_MQ_REQ_NOWAIT);
  436. if (IS_ERR(clone)) {
  437. /* EBUSY, ENODEV or EWOULDBLOCK: requeue */
  438. if (blk_queue_dying(q)) {
  439. atomic_inc(&m->pg_init_in_progress);
  440. activate_or_offline_path(pgpath);
  441. return DM_MAPIO_DELAY_REQUEUE;
  442. }
  443. /*
  444. * blk-mq's SCHED_RESTART can cover this requeue, so we
  445. * needn't deal with it by DELAY_REQUEUE. More importantly,
  446. * we have to return DM_MAPIO_REQUEUE so that blk-mq can
  447. * get the queue busy feedback (via BLK_STS_RESOURCE),
  448. * otherwise I/O merging can suffer.
  449. */
  450. return DM_MAPIO_REQUEUE;
  451. }
  452. clone->bio = clone->biotail = NULL;
  453. clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
  454. *__clone = clone;
  455. if (pgpath->pg->ps.type->start_io)
  456. pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
  457. &pgpath->path,
  458. nr_bytes);
  459. return DM_MAPIO_REMAPPED;
  460. }
  461. static void multipath_release_clone(struct request *clone,
  462. union map_info *map_context)
  463. {
  464. if (unlikely(map_context)) {
  465. /*
  466. * non-NULL map_context means caller is still map
  467. * method; must undo multipath_clone_and_map()
  468. */
  469. struct dm_mpath_io *mpio = get_mpio(map_context);
  470. struct pgpath *pgpath = mpio->pgpath;
  471. if (pgpath && pgpath->pg->ps.type->end_io)
  472. pgpath->pg->ps.type->end_io(&pgpath->pg->ps,
  473. &pgpath->path,
  474. mpio->nr_bytes,
  475. clone->io_start_time_ns);
  476. }
  477. blk_mq_free_request(clone);
  478. }
  479. /*
  480. * Map cloned bios (bio-based multipath)
  481. */
  482. static void __multipath_queue_bio(struct multipath *m, struct bio *bio)
  483. {
  484. /* Queue for the daemon to resubmit */
  485. bio_list_add(&m->queued_bios, bio);
  486. if (!test_bit(MPATHF_QUEUE_IO, &m->flags))
  487. queue_work(kmultipathd, &m->process_queued_bios);
  488. }
  489. static void multipath_queue_bio(struct multipath *m, struct bio *bio)
  490. {
  491. unsigned long flags;
  492. spin_lock_irqsave(&m->lock, flags);
  493. __multipath_queue_bio(m, bio);
  494. spin_unlock_irqrestore(&m->lock, flags);
  495. }
  496. static struct pgpath *__map_bio(struct multipath *m, struct bio *bio)
  497. {
  498. struct pgpath *pgpath;
  499. unsigned long flags;
  500. /* Do we need to select a new pgpath? */
  501. pgpath = READ_ONCE(m->current_pgpath);
  502. if (!pgpath || !mpath_double_check_test_bit(MPATHF_QUEUE_IO, m))
  503. pgpath = choose_pgpath(m, bio->bi_iter.bi_size);
  504. if (!pgpath) {
  505. spin_lock_irqsave(&m->lock, flags);
  506. if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
  507. __multipath_queue_bio(m, bio);
  508. pgpath = ERR_PTR(-EAGAIN);
  509. }
  510. spin_unlock_irqrestore(&m->lock, flags);
  511. } else if (mpath_double_check_test_bit(MPATHF_QUEUE_IO, m) ||
  512. mpath_double_check_test_bit(MPATHF_PG_INIT_REQUIRED, m)) {
  513. multipath_queue_bio(m, bio);
  514. pg_init_all_paths(m);
  515. return ERR_PTR(-EAGAIN);
  516. }
  517. return pgpath;
  518. }
  519. static int __multipath_map_bio(struct multipath *m, struct bio *bio,
  520. struct dm_mpath_io *mpio)
  521. {
  522. struct pgpath *pgpath = __map_bio(m, bio);
  523. if (IS_ERR(pgpath))
  524. return DM_MAPIO_SUBMITTED;
  525. if (!pgpath) {
  526. if (__must_push_back(m))
  527. return DM_MAPIO_REQUEUE;
  528. dm_report_EIO(m);
  529. return DM_MAPIO_KILL;
  530. }
  531. mpio->pgpath = pgpath;
  532. if (dm_ps_use_hr_timer(pgpath->pg->ps.type))
  533. mpio->start_time_ns = ktime_get_ns();
  534. bio->bi_status = 0;
  535. bio_set_dev(bio, pgpath->path.dev->bdev);
  536. bio->bi_opf |= REQ_FAILFAST_TRANSPORT;
  537. if (pgpath->pg->ps.type->start_io)
  538. pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
  539. &pgpath->path,
  540. mpio->nr_bytes);
  541. return DM_MAPIO_REMAPPED;
  542. }
  543. static int multipath_map_bio(struct dm_target *ti, struct bio *bio)
  544. {
  545. struct multipath *m = ti->private;
  546. struct dm_mpath_io *mpio = NULL;
  547. multipath_init_per_bio_data(bio, &mpio);
  548. return __multipath_map_bio(m, bio, mpio);
  549. }
  550. static void process_queued_io_list(struct multipath *m)
  551. {
  552. if (m->queue_mode == DM_TYPE_REQUEST_BASED)
  553. dm_mq_kick_requeue_list(dm_table_get_md(m->ti->table));
  554. else if (m->queue_mode == DM_TYPE_BIO_BASED)
  555. queue_work(kmultipathd, &m->process_queued_bios);
  556. }
  557. static void process_queued_bios(struct work_struct *work)
  558. {
  559. int r;
  560. unsigned long flags;
  561. struct bio *bio;
  562. struct bio_list bios;
  563. struct blk_plug plug;
  564. struct multipath *m =
  565. container_of(work, struct multipath, process_queued_bios);
  566. bio_list_init(&bios);
  567. spin_lock_irqsave(&m->lock, flags);
  568. if (bio_list_empty(&m->queued_bios)) {
  569. spin_unlock_irqrestore(&m->lock, flags);
  570. return;
  571. }
  572. bio_list_merge(&bios, &m->queued_bios);
  573. bio_list_init(&m->queued_bios);
  574. spin_unlock_irqrestore(&m->lock, flags);
  575. blk_start_plug(&plug);
  576. while ((bio = bio_list_pop(&bios))) {
  577. struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
  578. dm_bio_restore(get_bio_details_from_mpio(mpio), bio);
  579. r = __multipath_map_bio(m, bio, mpio);
  580. switch (r) {
  581. case DM_MAPIO_KILL:
  582. bio->bi_status = BLK_STS_IOERR;
  583. bio_endio(bio);
  584. break;
  585. case DM_MAPIO_REQUEUE:
  586. bio->bi_status = BLK_STS_DM_REQUEUE;
  587. bio_endio(bio);
  588. break;
  589. case DM_MAPIO_REMAPPED:
  590. submit_bio_noacct(bio);
  591. break;
  592. case DM_MAPIO_SUBMITTED:
  593. break;
  594. default:
  595. WARN_ONCE(true, "__multipath_map_bio() returned %d\n", r);
  596. }
  597. }
  598. blk_finish_plug(&plug);
  599. }
  600. /*
  601. * If we run out of usable paths, should we queue I/O or error it?
  602. */
  603. static int queue_if_no_path(struct multipath *m, bool queue_if_no_path,
  604. bool save_old_value, const char *caller)
  605. {
  606. unsigned long flags;
  607. bool queue_if_no_path_bit, saved_queue_if_no_path_bit;
  608. const char *dm_dev_name = dm_table_device_name(m->ti->table);
  609. DMDEBUG("%s: %s caller=%s queue_if_no_path=%d save_old_value=%d",
  610. dm_dev_name, __func__, caller, queue_if_no_path, save_old_value);
  611. spin_lock_irqsave(&m->lock, flags);
  612. queue_if_no_path_bit = test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
  613. saved_queue_if_no_path_bit = test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
  614. if (save_old_value) {
  615. if (unlikely(!queue_if_no_path_bit && saved_queue_if_no_path_bit)) {
  616. DMERR("%s: QIFNP disabled but saved as enabled, saving again loses state, not saving!",
  617. dm_dev_name);
  618. } else
  619. assign_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags, queue_if_no_path_bit);
  620. } else if (!queue_if_no_path && saved_queue_if_no_path_bit) {
  621. /* due to "fail_if_no_path" message, need to honor it. */
  622. clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
  623. }
  624. assign_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags, queue_if_no_path);
  625. DMDEBUG("%s: after %s changes; QIFNP = %d; SQIFNP = %d; DNFS = %d",
  626. dm_dev_name, __func__,
  627. test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags),
  628. test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags),
  629. dm_noflush_suspending(m->ti));
  630. spin_unlock_irqrestore(&m->lock, flags);
  631. if (!queue_if_no_path) {
  632. dm_table_run_md_queue_async(m->ti->table);
  633. process_queued_io_list(m);
  634. }
  635. return 0;
  636. }
  637. /*
  638. * If the queue_if_no_path timeout fires, turn off queue_if_no_path and
  639. * process any queued I/O.
  640. */
  641. static void queue_if_no_path_timeout_work(struct timer_list *t)
  642. {
  643. struct multipath *m = from_timer(m, t, nopath_timer);
  644. DMWARN("queue_if_no_path timeout on %s, failing queued IO",
  645. dm_table_device_name(m->ti->table));
  646. queue_if_no_path(m, false, false, __func__);
  647. }
  648. /*
  649. * Enable the queue_if_no_path timeout if necessary.
  650. * Called with m->lock held.
  651. */
  652. static void enable_nopath_timeout(struct multipath *m)
  653. {
  654. unsigned long queue_if_no_path_timeout =
  655. READ_ONCE(queue_if_no_path_timeout_secs) * HZ;
  656. lockdep_assert_held(&m->lock);
  657. if (queue_if_no_path_timeout > 0 &&
  658. atomic_read(&m->nr_valid_paths) == 0 &&
  659. test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
  660. mod_timer(&m->nopath_timer,
  661. jiffies + queue_if_no_path_timeout);
  662. }
  663. }
  664. static void disable_nopath_timeout(struct multipath *m)
  665. {
  666. del_timer_sync(&m->nopath_timer);
  667. }
  668. /*
  669. * An event is triggered whenever a path is taken out of use.
  670. * Includes path failure and PG bypass.
  671. */
  672. static void trigger_event(struct work_struct *work)
  673. {
  674. struct multipath *m =
  675. container_of(work, struct multipath, trigger_event);
  676. dm_table_event(m->ti->table);
  677. }
  678. /*-----------------------------------------------------------------
  679. * Constructor/argument parsing:
  680. * <#multipath feature args> [<arg>]*
  681. * <#hw_handler args> [hw_handler [<arg>]*]
  682. * <#priority groups>
  683. * <initial priority group>
  684. * [<selector> <#selector args> [<arg>]*
  685. * <#paths> <#per-path selector args>
  686. * [<path> [<arg>]* ]+ ]+
  687. *---------------------------------------------------------------*/
  688. static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg,
  689. struct dm_target *ti)
  690. {
  691. int r;
  692. struct path_selector_type *pst;
  693. unsigned int ps_argc;
  694. static const struct dm_arg _args[] = {
  695. {0, 1024, "invalid number of path selector args"},
  696. };
  697. pst = dm_get_path_selector(dm_shift_arg(as));
  698. if (!pst) {
  699. ti->error = "unknown path selector type";
  700. return -EINVAL;
  701. }
  702. r = dm_read_arg_group(_args, as, &ps_argc, &ti->error);
  703. if (r) {
  704. dm_put_path_selector(pst);
  705. return -EINVAL;
  706. }
  707. r = pst->create(&pg->ps, ps_argc, as->argv);
  708. if (r) {
  709. dm_put_path_selector(pst);
  710. ti->error = "path selector constructor failed";
  711. return r;
  712. }
  713. pg->ps.type = pst;
  714. dm_consume_args(as, ps_argc);
  715. return 0;
  716. }
  717. static int setup_scsi_dh(struct block_device *bdev, struct multipath *m,
  718. const char **attached_handler_name, char **error)
  719. {
  720. struct request_queue *q = bdev_get_queue(bdev);
  721. int r;
  722. if (mpath_double_check_test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, m)) {
  723. retain:
  724. if (*attached_handler_name) {
  725. /*
  726. * Clear any hw_handler_params associated with a
  727. * handler that isn't already attached.
  728. */
  729. if (m->hw_handler_name && strcmp(*attached_handler_name, m->hw_handler_name)) {
  730. kfree(m->hw_handler_params);
  731. m->hw_handler_params = NULL;
  732. }
  733. /*
  734. * Reset hw_handler_name to match the attached handler
  735. *
  736. * NB. This modifies the table line to show the actual
  737. * handler instead of the original table passed in.
  738. */
  739. kfree(m->hw_handler_name);
  740. m->hw_handler_name = *attached_handler_name;
  741. *attached_handler_name = NULL;
  742. }
  743. }
  744. if (m->hw_handler_name) {
  745. r = scsi_dh_attach(q, m->hw_handler_name);
  746. if (r == -EBUSY) {
  747. DMINFO("retaining handler on device %pg", bdev);
  748. goto retain;
  749. }
  750. if (r < 0) {
  751. *error = "error attaching hardware handler";
  752. return r;
  753. }
  754. if (m->hw_handler_params) {
  755. r = scsi_dh_set_params(q, m->hw_handler_params);
  756. if (r < 0) {
  757. *error = "unable to set hardware handler parameters";
  758. return r;
  759. }
  760. }
  761. }
  762. return 0;
  763. }
  764. static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps,
  765. struct dm_target *ti)
  766. {
  767. int r;
  768. struct pgpath *p;
  769. struct multipath *m = ti->private;
  770. struct request_queue *q;
  771. const char *attached_handler_name = NULL;
  772. /* we need at least a path arg */
  773. if (as->argc < 1) {
  774. ti->error = "no device given";
  775. return ERR_PTR(-EINVAL);
  776. }
  777. p = alloc_pgpath();
  778. if (!p)
  779. return ERR_PTR(-ENOMEM);
  780. r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
  781. &p->path.dev);
  782. if (r) {
  783. ti->error = "error getting device";
  784. goto bad;
  785. }
  786. q = bdev_get_queue(p->path.dev->bdev);
  787. attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL);
  788. if (attached_handler_name || m->hw_handler_name) {
  789. INIT_DELAYED_WORK(&p->activate_path, activate_path_work);
  790. r = setup_scsi_dh(p->path.dev->bdev, m, &attached_handler_name, &ti->error);
  791. kfree(attached_handler_name);
  792. if (r) {
  793. dm_put_device(ti, p->path.dev);
  794. goto bad;
  795. }
  796. }
  797. r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
  798. if (r) {
  799. dm_put_device(ti, p->path.dev);
  800. goto bad;
  801. }
  802. return p;
  803. bad:
  804. free_pgpath(p);
  805. return ERR_PTR(r);
  806. }
  807. static struct priority_group *parse_priority_group(struct dm_arg_set *as,
  808. struct multipath *m)
  809. {
  810. static const struct dm_arg _args[] = {
  811. {1, 1024, "invalid number of paths"},
  812. {0, 1024, "invalid number of selector args"}
  813. };
  814. int r;
  815. unsigned int i, nr_selector_args, nr_args;
  816. struct priority_group *pg;
  817. struct dm_target *ti = m->ti;
  818. if (as->argc < 2) {
  819. as->argc = 0;
  820. ti->error = "not enough priority group arguments";
  821. return ERR_PTR(-EINVAL);
  822. }
  823. pg = alloc_priority_group();
  824. if (!pg) {
  825. ti->error = "couldn't allocate priority group";
  826. return ERR_PTR(-ENOMEM);
  827. }
  828. pg->m = m;
  829. r = parse_path_selector(as, pg, ti);
  830. if (r)
  831. goto bad;
  832. /*
  833. * read the paths
  834. */
  835. r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error);
  836. if (r)
  837. goto bad;
  838. r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error);
  839. if (r)
  840. goto bad;
  841. nr_args = 1 + nr_selector_args;
  842. for (i = 0; i < pg->nr_pgpaths; i++) {
  843. struct pgpath *pgpath;
  844. struct dm_arg_set path_args;
  845. if (as->argc < nr_args) {
  846. ti->error = "not enough path parameters";
  847. r = -EINVAL;
  848. goto bad;
  849. }
  850. path_args.argc = nr_args;
  851. path_args.argv = as->argv;
  852. pgpath = parse_path(&path_args, &pg->ps, ti);
  853. if (IS_ERR(pgpath)) {
  854. r = PTR_ERR(pgpath);
  855. goto bad;
  856. }
  857. pgpath->pg = pg;
  858. list_add_tail(&pgpath->list, &pg->pgpaths);
  859. dm_consume_args(as, nr_args);
  860. }
  861. return pg;
  862. bad:
  863. free_priority_group(pg, ti);
  864. return ERR_PTR(r);
  865. }
  866. static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m)
  867. {
  868. unsigned int hw_argc;
  869. int ret;
  870. struct dm_target *ti = m->ti;
  871. static const struct dm_arg _args[] = {
  872. {0, 1024, "invalid number of hardware handler args"},
  873. };
  874. if (dm_read_arg_group(_args, as, &hw_argc, &ti->error))
  875. return -EINVAL;
  876. if (!hw_argc)
  877. return 0;
  878. if (m->queue_mode == DM_TYPE_BIO_BASED) {
  879. dm_consume_args(as, hw_argc);
  880. DMERR("bio-based multipath doesn't allow hardware handler args");
  881. return 0;
  882. }
  883. m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL);
  884. if (!m->hw_handler_name)
  885. return -EINVAL;
  886. if (hw_argc > 1) {
  887. char *p;
  888. int i, j, len = 4;
  889. for (i = 0; i <= hw_argc - 2; i++)
  890. len += strlen(as->argv[i]) + 1;
  891. p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
  892. if (!p) {
  893. ti->error = "memory allocation failed";
  894. ret = -ENOMEM;
  895. goto fail;
  896. }
  897. j = sprintf(p, "%d", hw_argc - 1);
  898. for (i = 0, p += j + 1; i <= hw_argc - 2; i++, p += j + 1)
  899. j = sprintf(p, "%s", as->argv[i]);
  900. }
  901. dm_consume_args(as, hw_argc - 1);
  902. return 0;
  903. fail:
  904. kfree(m->hw_handler_name);
  905. m->hw_handler_name = NULL;
  906. return ret;
  907. }
  908. static int parse_features(struct dm_arg_set *as, struct multipath *m)
  909. {
  910. int r;
  911. unsigned int argc;
  912. struct dm_target *ti = m->ti;
  913. const char *arg_name;
  914. static const struct dm_arg _args[] = {
  915. {0, 8, "invalid number of feature args"},
  916. {1, 50, "pg_init_retries must be between 1 and 50"},
  917. {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"},
  918. };
  919. r = dm_read_arg_group(_args, as, &argc, &ti->error);
  920. if (r)
  921. return -EINVAL;
  922. if (!argc)
  923. return 0;
  924. do {
  925. arg_name = dm_shift_arg(as);
  926. argc--;
  927. if (!strcasecmp(arg_name, "queue_if_no_path")) {
  928. r = queue_if_no_path(m, true, false, __func__);
  929. continue;
  930. }
  931. if (!strcasecmp(arg_name, "retain_attached_hw_handler")) {
  932. set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
  933. continue;
  934. }
  935. if (!strcasecmp(arg_name, "pg_init_retries") &&
  936. (argc >= 1)) {
  937. r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error);
  938. argc--;
  939. continue;
  940. }
  941. if (!strcasecmp(arg_name, "pg_init_delay_msecs") &&
  942. (argc >= 1)) {
  943. r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error);
  944. argc--;
  945. continue;
  946. }
  947. if (!strcasecmp(arg_name, "queue_mode") &&
  948. (argc >= 1)) {
  949. const char *queue_mode_name = dm_shift_arg(as);
  950. if (!strcasecmp(queue_mode_name, "bio"))
  951. m->queue_mode = DM_TYPE_BIO_BASED;
  952. else if (!strcasecmp(queue_mode_name, "rq") ||
  953. !strcasecmp(queue_mode_name, "mq"))
  954. m->queue_mode = DM_TYPE_REQUEST_BASED;
  955. else {
  956. ti->error = "Unknown 'queue_mode' requested";
  957. r = -EINVAL;
  958. }
  959. argc--;
  960. continue;
  961. }
  962. ti->error = "Unrecognised multipath feature request";
  963. r = -EINVAL;
  964. } while (argc && !r);
  965. return r;
  966. }
  967. static int multipath_ctr(struct dm_target *ti, unsigned int argc, char **argv)
  968. {
  969. /* target arguments */
  970. static const struct dm_arg _args[] = {
  971. {0, 1024, "invalid number of priority groups"},
  972. {0, 1024, "invalid initial priority group number"},
  973. };
  974. int r;
  975. struct multipath *m;
  976. struct dm_arg_set as;
  977. unsigned int pg_count = 0;
  978. unsigned int next_pg_num;
  979. unsigned long flags;
  980. as.argc = argc;
  981. as.argv = argv;
  982. m = alloc_multipath(ti);
  983. if (!m) {
  984. ti->error = "can't allocate multipath";
  985. return -EINVAL;
  986. }
  987. r = parse_features(&as, m);
  988. if (r)
  989. goto bad;
  990. r = alloc_multipath_stage2(ti, m);
  991. if (r)
  992. goto bad;
  993. r = parse_hw_handler(&as, m);
  994. if (r)
  995. goto bad;
  996. r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error);
  997. if (r)
  998. goto bad;
  999. r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error);
  1000. if (r)
  1001. goto bad;
  1002. if ((!m->nr_priority_groups && next_pg_num) ||
  1003. (m->nr_priority_groups && !next_pg_num)) {
  1004. ti->error = "invalid initial priority group";
  1005. r = -EINVAL;
  1006. goto bad;
  1007. }
  1008. /* parse the priority groups */
  1009. while (as.argc) {
  1010. struct priority_group *pg;
  1011. unsigned int nr_valid_paths = atomic_read(&m->nr_valid_paths);
  1012. pg = parse_priority_group(&as, m);
  1013. if (IS_ERR(pg)) {
  1014. r = PTR_ERR(pg);
  1015. goto bad;
  1016. }
  1017. nr_valid_paths += pg->nr_pgpaths;
  1018. atomic_set(&m->nr_valid_paths, nr_valid_paths);
  1019. list_add_tail(&pg->list, &m->priority_groups);
  1020. pg_count++;
  1021. pg->pg_num = pg_count;
  1022. if (!--next_pg_num)
  1023. m->next_pg = pg;
  1024. }
  1025. if (pg_count != m->nr_priority_groups) {
  1026. ti->error = "priority group count mismatch";
  1027. r = -EINVAL;
  1028. goto bad;
  1029. }
  1030. spin_lock_irqsave(&m->lock, flags);
  1031. enable_nopath_timeout(m);
  1032. spin_unlock_irqrestore(&m->lock, flags);
  1033. ti->num_flush_bios = 1;
  1034. ti->num_discard_bios = 1;
  1035. ti->num_write_zeroes_bios = 1;
  1036. if (m->queue_mode == DM_TYPE_BIO_BASED)
  1037. ti->per_io_data_size = multipath_per_bio_data_size();
  1038. else
  1039. ti->per_io_data_size = sizeof(struct dm_mpath_io);
  1040. return 0;
  1041. bad:
  1042. free_multipath(m);
  1043. return r;
  1044. }
  1045. static void multipath_wait_for_pg_init_completion(struct multipath *m)
  1046. {
  1047. DEFINE_WAIT(wait);
  1048. while (1) {
  1049. prepare_to_wait(&m->pg_init_wait, &wait, TASK_UNINTERRUPTIBLE);
  1050. if (!atomic_read(&m->pg_init_in_progress))
  1051. break;
  1052. io_schedule();
  1053. }
  1054. finish_wait(&m->pg_init_wait, &wait);
  1055. }
  1056. static void flush_multipath_work(struct multipath *m)
  1057. {
  1058. if (m->hw_handler_name) {
  1059. unsigned long flags;
  1060. if (!atomic_read(&m->pg_init_in_progress))
  1061. goto skip;
  1062. spin_lock_irqsave(&m->lock, flags);
  1063. if (atomic_read(&m->pg_init_in_progress) &&
  1064. !test_and_set_bit(MPATHF_PG_INIT_DISABLED, &m->flags)) {
  1065. spin_unlock_irqrestore(&m->lock, flags);
  1066. flush_workqueue(kmpath_handlerd);
  1067. multipath_wait_for_pg_init_completion(m);
  1068. spin_lock_irqsave(&m->lock, flags);
  1069. clear_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
  1070. }
  1071. spin_unlock_irqrestore(&m->lock, flags);
  1072. }
  1073. skip:
  1074. if (m->queue_mode == DM_TYPE_BIO_BASED)
  1075. flush_work(&m->process_queued_bios);
  1076. flush_work(&m->trigger_event);
  1077. }
  1078. static void multipath_dtr(struct dm_target *ti)
  1079. {
  1080. struct multipath *m = ti->private;
  1081. disable_nopath_timeout(m);
  1082. flush_multipath_work(m);
  1083. free_multipath(m);
  1084. }
  1085. /*
  1086. * Take a path out of use.
  1087. */
  1088. static int fail_path(struct pgpath *pgpath)
  1089. {
  1090. unsigned long flags;
  1091. struct multipath *m = pgpath->pg->m;
  1092. spin_lock_irqsave(&m->lock, flags);
  1093. if (!pgpath->is_active)
  1094. goto out;
  1095. DMWARN("%s: Failing path %s.",
  1096. dm_table_device_name(m->ti->table),
  1097. pgpath->path.dev->name);
  1098. pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
  1099. pgpath->is_active = false;
  1100. pgpath->fail_count++;
  1101. atomic_dec(&m->nr_valid_paths);
  1102. if (pgpath == m->current_pgpath)
  1103. m->current_pgpath = NULL;
  1104. dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti,
  1105. pgpath->path.dev->name, atomic_read(&m->nr_valid_paths));
  1106. schedule_work(&m->trigger_event);
  1107. enable_nopath_timeout(m);
  1108. out:
  1109. spin_unlock_irqrestore(&m->lock, flags);
  1110. return 0;
  1111. }
  1112. /*
  1113. * Reinstate a previously-failed path
  1114. */
  1115. static int reinstate_path(struct pgpath *pgpath)
  1116. {
  1117. int r = 0, run_queue = 0;
  1118. unsigned long flags;
  1119. struct multipath *m = pgpath->pg->m;
  1120. unsigned int nr_valid_paths;
  1121. spin_lock_irqsave(&m->lock, flags);
  1122. if (pgpath->is_active)
  1123. goto out;
  1124. DMWARN("%s: Reinstating path %s.",
  1125. dm_table_device_name(m->ti->table),
  1126. pgpath->path.dev->name);
  1127. r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
  1128. if (r)
  1129. goto out;
  1130. pgpath->is_active = true;
  1131. nr_valid_paths = atomic_inc_return(&m->nr_valid_paths);
  1132. if (nr_valid_paths == 1) {
  1133. m->current_pgpath = NULL;
  1134. run_queue = 1;
  1135. } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
  1136. if (queue_work(kmpath_handlerd, &pgpath->activate_path.work))
  1137. atomic_inc(&m->pg_init_in_progress);
  1138. }
  1139. dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
  1140. pgpath->path.dev->name, nr_valid_paths);
  1141. schedule_work(&m->trigger_event);
  1142. out:
  1143. spin_unlock_irqrestore(&m->lock, flags);
  1144. if (run_queue) {
  1145. dm_table_run_md_queue_async(m->ti->table);
  1146. process_queued_io_list(m);
  1147. }
  1148. if (pgpath->is_active)
  1149. disable_nopath_timeout(m);
  1150. return r;
  1151. }
  1152. /*
  1153. * Fail or reinstate all paths that match the provided struct dm_dev.
  1154. */
  1155. static int action_dev(struct multipath *m, struct dm_dev *dev,
  1156. action_fn action)
  1157. {
  1158. int r = -EINVAL;
  1159. struct pgpath *pgpath;
  1160. struct priority_group *pg;
  1161. list_for_each_entry(pg, &m->priority_groups, list) {
  1162. list_for_each_entry(pgpath, &pg->pgpaths, list) {
  1163. if (pgpath->path.dev == dev)
  1164. r = action(pgpath);
  1165. }
  1166. }
  1167. return r;
  1168. }
  1169. /*
  1170. * Temporarily try to avoid having to use the specified PG
  1171. */
  1172. static void bypass_pg(struct multipath *m, struct priority_group *pg,
  1173. bool bypassed)
  1174. {
  1175. unsigned long flags;
  1176. spin_lock_irqsave(&m->lock, flags);
  1177. pg->bypassed = bypassed;
  1178. m->current_pgpath = NULL;
  1179. m->current_pg = NULL;
  1180. spin_unlock_irqrestore(&m->lock, flags);
  1181. schedule_work(&m->trigger_event);
  1182. }
  1183. /*
  1184. * Switch to using the specified PG from the next I/O that gets mapped
  1185. */
  1186. static int switch_pg_num(struct multipath *m, const char *pgstr)
  1187. {
  1188. struct priority_group *pg;
  1189. unsigned int pgnum;
  1190. unsigned long flags;
  1191. char dummy;
  1192. if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
  1193. !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
  1194. DMWARN("invalid PG number supplied to switch_pg_num");
  1195. return -EINVAL;
  1196. }
  1197. spin_lock_irqsave(&m->lock, flags);
  1198. list_for_each_entry(pg, &m->priority_groups, list) {
  1199. pg->bypassed = false;
  1200. if (--pgnum)
  1201. continue;
  1202. m->current_pgpath = NULL;
  1203. m->current_pg = NULL;
  1204. m->next_pg = pg;
  1205. }
  1206. spin_unlock_irqrestore(&m->lock, flags);
  1207. schedule_work(&m->trigger_event);
  1208. return 0;
  1209. }
  1210. /*
  1211. * Set/clear bypassed status of a PG.
  1212. * PGs are numbered upwards from 1 in the order they were declared.
  1213. */
  1214. static int bypass_pg_num(struct multipath *m, const char *pgstr, bool bypassed)
  1215. {
  1216. struct priority_group *pg;
  1217. unsigned int pgnum;
  1218. char dummy;
  1219. if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
  1220. !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
  1221. DMWARN("invalid PG number supplied to bypass_pg");
  1222. return -EINVAL;
  1223. }
  1224. list_for_each_entry(pg, &m->priority_groups, list) {
  1225. if (!--pgnum)
  1226. break;
  1227. }
  1228. bypass_pg(m, pg, bypassed);
  1229. return 0;
  1230. }
  1231. /*
  1232. * Should we retry pg_init immediately?
  1233. */
  1234. static bool pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
  1235. {
  1236. unsigned long flags;
  1237. bool limit_reached = false;
  1238. spin_lock_irqsave(&m->lock, flags);
  1239. if (atomic_read(&m->pg_init_count) <= m->pg_init_retries &&
  1240. !test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
  1241. set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
  1242. else
  1243. limit_reached = true;
  1244. spin_unlock_irqrestore(&m->lock, flags);
  1245. return limit_reached;
  1246. }
  1247. static void pg_init_done(void *data, int errors)
  1248. {
  1249. struct pgpath *pgpath = data;
  1250. struct priority_group *pg = pgpath->pg;
  1251. struct multipath *m = pg->m;
  1252. unsigned long flags;
  1253. bool delay_retry = false;
  1254. /* device or driver problems */
  1255. switch (errors) {
  1256. case SCSI_DH_OK:
  1257. break;
  1258. case SCSI_DH_NOSYS:
  1259. if (!m->hw_handler_name) {
  1260. errors = 0;
  1261. break;
  1262. }
  1263. DMERR("Could not failover the device: Handler scsi_dh_%s "
  1264. "Error %d.", m->hw_handler_name, errors);
  1265. /*
  1266. * Fail path for now, so we do not ping pong
  1267. */
  1268. fail_path(pgpath);
  1269. break;
  1270. case SCSI_DH_DEV_TEMP_BUSY:
  1271. /*
  1272. * Probably doing something like FW upgrade on the
  1273. * controller so try the other pg.
  1274. */
  1275. bypass_pg(m, pg, true);
  1276. break;
  1277. case SCSI_DH_RETRY:
  1278. /* Wait before retrying. */
  1279. delay_retry = true;
  1280. fallthrough;
  1281. case SCSI_DH_IMM_RETRY:
  1282. case SCSI_DH_RES_TEMP_UNAVAIL:
  1283. if (pg_init_limit_reached(m, pgpath))
  1284. fail_path(pgpath);
  1285. errors = 0;
  1286. break;
  1287. case SCSI_DH_DEV_OFFLINED:
  1288. default:
  1289. /*
  1290. * We probably do not want to fail the path for a device
  1291. * error, but this is what the old dm did. In future
  1292. * patches we can do more advanced handling.
  1293. */
  1294. fail_path(pgpath);
  1295. }
  1296. spin_lock_irqsave(&m->lock, flags);
  1297. if (errors) {
  1298. if (pgpath == m->current_pgpath) {
  1299. DMERR("Could not failover device. Error %d.", errors);
  1300. m->current_pgpath = NULL;
  1301. m->current_pg = NULL;
  1302. }
  1303. } else if (!test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
  1304. pg->bypassed = false;
  1305. if (atomic_dec_return(&m->pg_init_in_progress) > 0)
  1306. /* Activations of other paths are still on going */
  1307. goto out;
  1308. if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
  1309. if (delay_retry)
  1310. set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
  1311. else
  1312. clear_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
  1313. if (__pg_init_all_paths(m))
  1314. goto out;
  1315. }
  1316. clear_bit(MPATHF_QUEUE_IO, &m->flags);
  1317. process_queued_io_list(m);
  1318. /*
  1319. * Wake up any thread waiting to suspend.
  1320. */
  1321. wake_up(&m->pg_init_wait);
  1322. out:
  1323. spin_unlock_irqrestore(&m->lock, flags);
  1324. }
  1325. static void activate_or_offline_path(struct pgpath *pgpath)
  1326. {
  1327. struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
  1328. if (pgpath->is_active && !blk_queue_dying(q))
  1329. scsi_dh_activate(q, pg_init_done, pgpath);
  1330. else
  1331. pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED);
  1332. }
  1333. static void activate_path_work(struct work_struct *work)
  1334. {
  1335. struct pgpath *pgpath =
  1336. container_of(work, struct pgpath, activate_path.work);
  1337. activate_or_offline_path(pgpath);
  1338. }
  1339. static int multipath_end_io(struct dm_target *ti, struct request *clone,
  1340. blk_status_t error, union map_info *map_context)
  1341. {
  1342. struct dm_mpath_io *mpio = get_mpio(map_context);
  1343. struct pgpath *pgpath = mpio->pgpath;
  1344. int r = DM_ENDIO_DONE;
  1345. /*
  1346. * We don't queue any clone request inside the multipath target
  1347. * during end I/O handling, since those clone requests don't have
  1348. * bio clones. If we queue them inside the multipath target,
  1349. * we need to make bio clones, that requires memory allocation.
  1350. * (See drivers/md/dm-rq.c:end_clone_bio() about why the clone requests
  1351. * don't have bio clones.)
  1352. * Instead of queueing the clone request here, we queue the original
  1353. * request into dm core, which will remake a clone request and
  1354. * clone bios for it and resubmit it later.
  1355. */
  1356. if (error && blk_path_error(error)) {
  1357. struct multipath *m = ti->private;
  1358. if (error == BLK_STS_RESOURCE)
  1359. r = DM_ENDIO_DELAY_REQUEUE;
  1360. else
  1361. r = DM_ENDIO_REQUEUE;
  1362. if (pgpath)
  1363. fail_path(pgpath);
  1364. if (!atomic_read(&m->nr_valid_paths) &&
  1365. !must_push_back_rq(m)) {
  1366. if (error == BLK_STS_IOERR)
  1367. dm_report_EIO(m);
  1368. /* complete with the original error */
  1369. r = DM_ENDIO_DONE;
  1370. }
  1371. }
  1372. if (pgpath) {
  1373. struct path_selector *ps = &pgpath->pg->ps;
  1374. if (ps->type->end_io)
  1375. ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes,
  1376. clone->io_start_time_ns);
  1377. }
  1378. return r;
  1379. }
  1380. static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone,
  1381. blk_status_t *error)
  1382. {
  1383. struct multipath *m = ti->private;
  1384. struct dm_mpath_io *mpio = get_mpio_from_bio(clone);
  1385. struct pgpath *pgpath = mpio->pgpath;
  1386. unsigned long flags;
  1387. int r = DM_ENDIO_DONE;
  1388. if (!*error || !blk_path_error(*error))
  1389. goto done;
  1390. if (pgpath)
  1391. fail_path(pgpath);
  1392. if (!atomic_read(&m->nr_valid_paths)) {
  1393. spin_lock_irqsave(&m->lock, flags);
  1394. if (!test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
  1395. if (__must_push_back(m)) {
  1396. r = DM_ENDIO_REQUEUE;
  1397. } else {
  1398. dm_report_EIO(m);
  1399. *error = BLK_STS_IOERR;
  1400. }
  1401. spin_unlock_irqrestore(&m->lock, flags);
  1402. goto done;
  1403. }
  1404. spin_unlock_irqrestore(&m->lock, flags);
  1405. }
  1406. multipath_queue_bio(m, clone);
  1407. r = DM_ENDIO_INCOMPLETE;
  1408. done:
  1409. if (pgpath) {
  1410. struct path_selector *ps = &pgpath->pg->ps;
  1411. if (ps->type->end_io)
  1412. ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes,
  1413. (mpio->start_time_ns ?:
  1414. dm_start_time_ns_from_clone(clone)));
  1415. }
  1416. return r;
  1417. }
  1418. /*
  1419. * Suspend with flush can't complete until all the I/O is processed
  1420. * so if the last path fails we must error any remaining I/O.
  1421. * - Note that if the freeze_bdev fails while suspending, the
  1422. * queue_if_no_path state is lost - userspace should reset it.
  1423. * Otherwise, during noflush suspend, queue_if_no_path will not change.
  1424. */
  1425. static void multipath_presuspend(struct dm_target *ti)
  1426. {
  1427. struct multipath *m = ti->private;
  1428. /* FIXME: bio-based shouldn't need to always disable queue_if_no_path */
  1429. if (m->queue_mode == DM_TYPE_BIO_BASED || !dm_noflush_suspending(m->ti))
  1430. queue_if_no_path(m, false, true, __func__);
  1431. }
  1432. static void multipath_postsuspend(struct dm_target *ti)
  1433. {
  1434. struct multipath *m = ti->private;
  1435. mutex_lock(&m->work_mutex);
  1436. flush_multipath_work(m);
  1437. mutex_unlock(&m->work_mutex);
  1438. }
  1439. /*
  1440. * Restore the queue_if_no_path setting.
  1441. */
  1442. static void multipath_resume(struct dm_target *ti)
  1443. {
  1444. struct multipath *m = ti->private;
  1445. unsigned long flags;
  1446. spin_lock_irqsave(&m->lock, flags);
  1447. if (test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags)) {
  1448. set_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
  1449. clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
  1450. }
  1451. DMDEBUG("%s: %s finished; QIFNP = %d; SQIFNP = %d",
  1452. dm_table_device_name(m->ti->table), __func__,
  1453. test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags),
  1454. test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags));
  1455. spin_unlock_irqrestore(&m->lock, flags);
  1456. }
  1457. /*
  1458. * Info output has the following format:
  1459. * num_multipath_feature_args [multipath_feature_args]*
  1460. * num_handler_status_args [handler_status_args]*
  1461. * num_groups init_group_number
  1462. * [A|D|E num_ps_status_args [ps_status_args]*
  1463. * num_paths num_selector_args
  1464. * [path_dev A|F fail_count [selector_args]* ]+ ]+
  1465. *
  1466. * Table output has the following format (identical to the constructor string):
  1467. * num_feature_args [features_args]*
  1468. * num_handler_args hw_handler [hw_handler_args]*
  1469. * num_groups init_group_number
  1470. * [priority selector-name num_ps_args [ps_args]*
  1471. * num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
  1472. */
  1473. static void multipath_status(struct dm_target *ti, status_type_t type,
  1474. unsigned int status_flags, char *result, unsigned int maxlen)
  1475. {
  1476. int sz = 0, pg_counter, pgpath_counter;
  1477. unsigned long flags;
  1478. struct multipath *m = ti->private;
  1479. struct priority_group *pg;
  1480. struct pgpath *p;
  1481. unsigned int pg_num;
  1482. char state;
  1483. spin_lock_irqsave(&m->lock, flags);
  1484. /* Features */
  1485. if (type == STATUSTYPE_INFO)
  1486. DMEMIT("2 %u %u ", test_bit(MPATHF_QUEUE_IO, &m->flags),
  1487. atomic_read(&m->pg_init_count));
  1488. else {
  1489. DMEMIT("%u ", test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) +
  1490. (m->pg_init_retries > 0) * 2 +
  1491. (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 +
  1492. test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) +
  1493. (m->queue_mode != DM_TYPE_REQUEST_BASED) * 2);
  1494. if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
  1495. DMEMIT("queue_if_no_path ");
  1496. if (m->pg_init_retries)
  1497. DMEMIT("pg_init_retries %u ", m->pg_init_retries);
  1498. if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT)
  1499. DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs);
  1500. if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags))
  1501. DMEMIT("retain_attached_hw_handler ");
  1502. if (m->queue_mode != DM_TYPE_REQUEST_BASED) {
  1503. switch(m->queue_mode) {
  1504. case DM_TYPE_BIO_BASED:
  1505. DMEMIT("queue_mode bio ");
  1506. break;
  1507. default:
  1508. WARN_ON_ONCE(true);
  1509. break;
  1510. }
  1511. }
  1512. }
  1513. if (!m->hw_handler_name || type == STATUSTYPE_INFO)
  1514. DMEMIT("0 ");
  1515. else
  1516. DMEMIT("1 %s ", m->hw_handler_name);
  1517. DMEMIT("%u ", m->nr_priority_groups);
  1518. if (m->next_pg)
  1519. pg_num = m->next_pg->pg_num;
  1520. else if (m->current_pg)
  1521. pg_num = m->current_pg->pg_num;
  1522. else
  1523. pg_num = (m->nr_priority_groups ? 1 : 0);
  1524. DMEMIT("%u ", pg_num);
  1525. switch (type) {
  1526. case STATUSTYPE_INFO:
  1527. list_for_each_entry(pg, &m->priority_groups, list) {
  1528. if (pg->bypassed)
  1529. state = 'D'; /* Disabled */
  1530. else if (pg == m->current_pg)
  1531. state = 'A'; /* Currently Active */
  1532. else
  1533. state = 'E'; /* Enabled */
  1534. DMEMIT("%c ", state);
  1535. if (pg->ps.type->status)
  1536. sz += pg->ps.type->status(&pg->ps, NULL, type,
  1537. result + sz,
  1538. maxlen - sz);
  1539. else
  1540. DMEMIT("0 ");
  1541. DMEMIT("%u %u ", pg->nr_pgpaths,
  1542. pg->ps.type->info_args);
  1543. list_for_each_entry(p, &pg->pgpaths, list) {
  1544. DMEMIT("%s %s %u ", p->path.dev->name,
  1545. p->is_active ? "A" : "F",
  1546. p->fail_count);
  1547. if (pg->ps.type->status)
  1548. sz += pg->ps.type->status(&pg->ps,
  1549. &p->path, type, result + sz,
  1550. maxlen - sz);
  1551. }
  1552. }
  1553. break;
  1554. case STATUSTYPE_TABLE:
  1555. list_for_each_entry(pg, &m->priority_groups, list) {
  1556. DMEMIT("%s ", pg->ps.type->name);
  1557. if (pg->ps.type->status)
  1558. sz += pg->ps.type->status(&pg->ps, NULL, type,
  1559. result + sz,
  1560. maxlen - sz);
  1561. else
  1562. DMEMIT("0 ");
  1563. DMEMIT("%u %u ", pg->nr_pgpaths,
  1564. pg->ps.type->table_args);
  1565. list_for_each_entry(p, &pg->pgpaths, list) {
  1566. DMEMIT("%s ", p->path.dev->name);
  1567. if (pg->ps.type->status)
  1568. sz += pg->ps.type->status(&pg->ps,
  1569. &p->path, type, result + sz,
  1570. maxlen - sz);
  1571. }
  1572. }
  1573. break;
  1574. case STATUSTYPE_IMA:
  1575. sz = 0; /*reset the result pointer*/
  1576. DMEMIT_TARGET_NAME_VERSION(ti->type);
  1577. DMEMIT(",nr_priority_groups=%u", m->nr_priority_groups);
  1578. pg_counter = 0;
  1579. list_for_each_entry(pg, &m->priority_groups, list) {
  1580. if (pg->bypassed)
  1581. state = 'D'; /* Disabled */
  1582. else if (pg == m->current_pg)
  1583. state = 'A'; /* Currently Active */
  1584. else
  1585. state = 'E'; /* Enabled */
  1586. DMEMIT(",pg_state_%d=%c", pg_counter, state);
  1587. DMEMIT(",nr_pgpaths_%d=%u", pg_counter, pg->nr_pgpaths);
  1588. DMEMIT(",path_selector_name_%d=%s", pg_counter, pg->ps.type->name);
  1589. pgpath_counter = 0;
  1590. list_for_each_entry(p, &pg->pgpaths, list) {
  1591. DMEMIT(",path_name_%d_%d=%s,is_active_%d_%d=%c,fail_count_%d_%d=%u",
  1592. pg_counter, pgpath_counter, p->path.dev->name,
  1593. pg_counter, pgpath_counter, p->is_active ? 'A' : 'F',
  1594. pg_counter, pgpath_counter, p->fail_count);
  1595. if (pg->ps.type->status) {
  1596. DMEMIT(",path_selector_status_%d_%d=",
  1597. pg_counter, pgpath_counter);
  1598. sz += pg->ps.type->status(&pg->ps, &p->path,
  1599. type, result + sz,
  1600. maxlen - sz);
  1601. }
  1602. pgpath_counter++;
  1603. }
  1604. pg_counter++;
  1605. }
  1606. DMEMIT(";");
  1607. break;
  1608. }
  1609. spin_unlock_irqrestore(&m->lock, flags);
  1610. }
  1611. static int multipath_message(struct dm_target *ti, unsigned int argc, char **argv,
  1612. char *result, unsigned int maxlen)
  1613. {
  1614. int r = -EINVAL;
  1615. struct dm_dev *dev;
  1616. struct multipath *m = ti->private;
  1617. action_fn action;
  1618. unsigned long flags;
  1619. mutex_lock(&m->work_mutex);
  1620. if (dm_suspended(ti)) {
  1621. r = -EBUSY;
  1622. goto out;
  1623. }
  1624. if (argc == 1) {
  1625. if (!strcasecmp(argv[0], "queue_if_no_path")) {
  1626. r = queue_if_no_path(m, true, false, __func__);
  1627. spin_lock_irqsave(&m->lock, flags);
  1628. enable_nopath_timeout(m);
  1629. spin_unlock_irqrestore(&m->lock, flags);
  1630. goto out;
  1631. } else if (!strcasecmp(argv[0], "fail_if_no_path")) {
  1632. r = queue_if_no_path(m, false, false, __func__);
  1633. disable_nopath_timeout(m);
  1634. goto out;
  1635. }
  1636. }
  1637. if (argc != 2) {
  1638. DMWARN("Invalid multipath message arguments. Expected 2 arguments, got %d.", argc);
  1639. goto out;
  1640. }
  1641. if (!strcasecmp(argv[0], "disable_group")) {
  1642. r = bypass_pg_num(m, argv[1], true);
  1643. goto out;
  1644. } else if (!strcasecmp(argv[0], "enable_group")) {
  1645. r = bypass_pg_num(m, argv[1], false);
  1646. goto out;
  1647. } else if (!strcasecmp(argv[0], "switch_group")) {
  1648. r = switch_pg_num(m, argv[1]);
  1649. goto out;
  1650. } else if (!strcasecmp(argv[0], "reinstate_path"))
  1651. action = reinstate_path;
  1652. else if (!strcasecmp(argv[0], "fail_path"))
  1653. action = fail_path;
  1654. else {
  1655. DMWARN("Unrecognised multipath message received: %s", argv[0]);
  1656. goto out;
  1657. }
  1658. r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev);
  1659. if (r) {
  1660. DMWARN("message: error getting device %s",
  1661. argv[1]);
  1662. goto out;
  1663. }
  1664. r = action_dev(m, dev, action);
  1665. dm_put_device(ti, dev);
  1666. out:
  1667. mutex_unlock(&m->work_mutex);
  1668. return r;
  1669. }
  1670. static int multipath_prepare_ioctl(struct dm_target *ti,
  1671. struct block_device **bdev)
  1672. {
  1673. struct multipath *m = ti->private;
  1674. struct pgpath *pgpath;
  1675. unsigned long flags;
  1676. int r;
  1677. pgpath = READ_ONCE(m->current_pgpath);
  1678. if (!pgpath || !mpath_double_check_test_bit(MPATHF_QUEUE_IO, m))
  1679. pgpath = choose_pgpath(m, 0);
  1680. if (pgpath) {
  1681. if (!mpath_double_check_test_bit(MPATHF_QUEUE_IO, m)) {
  1682. *bdev = pgpath->path.dev->bdev;
  1683. r = 0;
  1684. } else {
  1685. /* pg_init has not started or completed */
  1686. r = -ENOTCONN;
  1687. }
  1688. } else {
  1689. /* No path is available */
  1690. r = -EIO;
  1691. spin_lock_irqsave(&m->lock, flags);
  1692. if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
  1693. r = -ENOTCONN;
  1694. spin_unlock_irqrestore(&m->lock, flags);
  1695. }
  1696. if (r == -ENOTCONN) {
  1697. if (!READ_ONCE(m->current_pg)) {
  1698. /* Path status changed, redo selection */
  1699. (void) choose_pgpath(m, 0);
  1700. }
  1701. spin_lock_irqsave(&m->lock, flags);
  1702. if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
  1703. (void) __pg_init_all_paths(m);
  1704. spin_unlock_irqrestore(&m->lock, flags);
  1705. dm_table_run_md_queue_async(m->ti->table);
  1706. process_queued_io_list(m);
  1707. }
  1708. /*
  1709. * Only pass ioctls through if the device sizes match exactly.
  1710. */
  1711. if (!r && ti->len != bdev_nr_sectors((*bdev)))
  1712. return 1;
  1713. return r;
  1714. }
  1715. static int multipath_iterate_devices(struct dm_target *ti,
  1716. iterate_devices_callout_fn fn, void *data)
  1717. {
  1718. struct multipath *m = ti->private;
  1719. struct priority_group *pg;
  1720. struct pgpath *p;
  1721. int ret = 0;
  1722. list_for_each_entry(pg, &m->priority_groups, list) {
  1723. list_for_each_entry(p, &pg->pgpaths, list) {
  1724. ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
  1725. if (ret)
  1726. goto out;
  1727. }
  1728. }
  1729. out:
  1730. return ret;
  1731. }
  1732. static int pgpath_busy(struct pgpath *pgpath)
  1733. {
  1734. struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
  1735. return blk_lld_busy(q);
  1736. }
  1737. /*
  1738. * We return "busy", only when we can map I/Os but underlying devices
  1739. * are busy (so even if we map I/Os now, the I/Os will wait on
  1740. * the underlying queue).
  1741. * In other words, if we want to kill I/Os or queue them inside us
  1742. * due to map unavailability, we don't return "busy". Otherwise,
  1743. * dm core won't give us the I/Os and we can't do what we want.
  1744. */
  1745. static int multipath_busy(struct dm_target *ti)
  1746. {
  1747. bool busy = false, has_active = false;
  1748. struct multipath *m = ti->private;
  1749. struct priority_group *pg, *next_pg;
  1750. struct pgpath *pgpath;
  1751. /* pg_init in progress */
  1752. if (atomic_read(&m->pg_init_in_progress))
  1753. return true;
  1754. /* no paths available, for blk-mq: rely on IO mapping to delay requeue */
  1755. if (!atomic_read(&m->nr_valid_paths)) {
  1756. unsigned long flags;
  1757. spin_lock_irqsave(&m->lock, flags);
  1758. if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
  1759. spin_unlock_irqrestore(&m->lock, flags);
  1760. return (m->queue_mode != DM_TYPE_REQUEST_BASED);
  1761. }
  1762. spin_unlock_irqrestore(&m->lock, flags);
  1763. }
  1764. /* Guess which priority_group will be used at next mapping time */
  1765. pg = READ_ONCE(m->current_pg);
  1766. next_pg = READ_ONCE(m->next_pg);
  1767. if (unlikely(!READ_ONCE(m->current_pgpath) && next_pg))
  1768. pg = next_pg;
  1769. if (!pg) {
  1770. /*
  1771. * We don't know which pg will be used at next mapping time.
  1772. * We don't call choose_pgpath() here to avoid to trigger
  1773. * pg_init just by busy checking.
  1774. * So we don't know whether underlying devices we will be using
  1775. * at next mapping time are busy or not. Just try mapping.
  1776. */
  1777. return busy;
  1778. }
  1779. /*
  1780. * If there is one non-busy active path at least, the path selector
  1781. * will be able to select it. So we consider such a pg as not busy.
  1782. */
  1783. busy = true;
  1784. list_for_each_entry(pgpath, &pg->pgpaths, list) {
  1785. if (pgpath->is_active) {
  1786. has_active = true;
  1787. if (!pgpath_busy(pgpath)) {
  1788. busy = false;
  1789. break;
  1790. }
  1791. }
  1792. }
  1793. if (!has_active) {
  1794. /*
  1795. * No active path in this pg, so this pg won't be used and
  1796. * the current_pg will be changed at next mapping time.
  1797. * We need to try mapping to determine it.
  1798. */
  1799. busy = false;
  1800. }
  1801. return busy;
  1802. }
  1803. /*-----------------------------------------------------------------
  1804. * Module setup
  1805. *---------------------------------------------------------------*/
  1806. static struct target_type multipath_target = {
  1807. .name = "multipath",
  1808. .version = {1, 14, 0},
  1809. .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE |
  1810. DM_TARGET_PASSES_INTEGRITY,
  1811. .module = THIS_MODULE,
  1812. .ctr = multipath_ctr,
  1813. .dtr = multipath_dtr,
  1814. .clone_and_map_rq = multipath_clone_and_map,
  1815. .release_clone_rq = multipath_release_clone,
  1816. .rq_end_io = multipath_end_io,
  1817. .map = multipath_map_bio,
  1818. .end_io = multipath_end_io_bio,
  1819. .presuspend = multipath_presuspend,
  1820. .postsuspend = multipath_postsuspend,
  1821. .resume = multipath_resume,
  1822. .status = multipath_status,
  1823. .message = multipath_message,
  1824. .prepare_ioctl = multipath_prepare_ioctl,
  1825. .iterate_devices = multipath_iterate_devices,
  1826. .busy = multipath_busy,
  1827. };
  1828. static int __init dm_multipath_init(void)
  1829. {
  1830. int r;
  1831. kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0);
  1832. if (!kmultipathd) {
  1833. DMERR("failed to create workqueue kmpathd");
  1834. r = -ENOMEM;
  1835. goto bad_alloc_kmultipathd;
  1836. }
  1837. /*
  1838. * A separate workqueue is used to handle the device handlers
  1839. * to avoid overloading existing workqueue. Overloading the
  1840. * old workqueue would also create a bottleneck in the
  1841. * path of the storage hardware device activation.
  1842. */
  1843. kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd",
  1844. WQ_MEM_RECLAIM);
  1845. if (!kmpath_handlerd) {
  1846. DMERR("failed to create workqueue kmpath_handlerd");
  1847. r = -ENOMEM;
  1848. goto bad_alloc_kmpath_handlerd;
  1849. }
  1850. r = dm_register_target(&multipath_target);
  1851. if (r < 0) {
  1852. DMERR("request-based register failed %d", r);
  1853. r = -EINVAL;
  1854. goto bad_register_target;
  1855. }
  1856. return 0;
  1857. bad_register_target:
  1858. destroy_workqueue(kmpath_handlerd);
  1859. bad_alloc_kmpath_handlerd:
  1860. destroy_workqueue(kmultipathd);
  1861. bad_alloc_kmultipathd:
  1862. return r;
  1863. }
  1864. static void __exit dm_multipath_exit(void)
  1865. {
  1866. destroy_workqueue(kmpath_handlerd);
  1867. destroy_workqueue(kmultipathd);
  1868. dm_unregister_target(&multipath_target);
  1869. }
  1870. module_init(dm_multipath_init);
  1871. module_exit(dm_multipath_exit);
  1872. module_param_named(queue_if_no_path_timeout_secs,
  1873. queue_if_no_path_timeout_secs, ulong, S_IRUGO | S_IWUSR);
  1874. MODULE_PARM_DESC(queue_if_no_path_timeout_secs, "No available paths queue IO timeout in seconds");
  1875. MODULE_DESCRIPTION(DM_NAME " multipath target");
  1876. MODULE_AUTHOR("Sistina Software <[email protected]>");
  1877. MODULE_LICENSE("GPL");