dm-log-writes.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960
  1. /*
  2. * Copyright (C) 2014 Facebook. All rights reserved.
  3. *
  4. * This file is released under the GPL.
  5. */
  6. #include <linux/device-mapper.h>
  7. #include <linux/module.h>
  8. #include <linux/init.h>
  9. #include <linux/blkdev.h>
  10. #include <linux/bio.h>
  11. #include <linux/dax.h>
  12. #include <linux/slab.h>
  13. #include <linux/kthread.h>
  14. #include <linux/freezer.h>
  15. #include <linux/uio.h>
  16. #define DM_MSG_PREFIX "log-writes"
  17. /*
  18. * This target will sequentially log all writes to the target device onto the
  19. * log device. This is helpful for replaying writes to check for fs consistency
  20. * at all times. This target provides a mechanism to mark specific events to
  21. * check data at a later time. So for example you would:
  22. *
  23. * write data
  24. * fsync
  25. * dmsetup message /dev/whatever mark mymark
  26. * unmount /mnt/test
  27. *
  28. * Then replay the log up to mymark and check the contents of the replay to
  29. * verify it matches what was written.
  30. *
  31. * We log writes only after they have been flushed, this makes the log describe
  32. * close to the order in which the data hits the actual disk, not its cache. So
  33. * for example the following sequence (W means write, C means complete)
  34. *
  35. * Wa,Wb,Wc,Cc,Ca,FLUSH,FUAd,Cb,CFLUSH,CFUAd
  36. *
  37. * Would result in the log looking like this:
  38. *
  39. * c,a,b,flush,fuad,<other writes>,<next flush>
  40. *
  41. * This is meant to help expose problems where file systems do not properly wait
  42. * on data being written before invoking a FLUSH. FUA bypasses cache so once it
  43. * completes it is added to the log as it should be on disk.
  44. *
  45. * We treat DISCARDs as if they don't bypass cache so that they are logged in
  46. * order of completion along with the normal writes. If we didn't do it this
  47. * way we would process all the discards first and then write all the data, when
  48. * in fact we want to do the data and the discard in the order that they
  49. * completed.
  50. */
  51. #define LOG_FLUSH_FLAG (1 << 0)
  52. #define LOG_FUA_FLAG (1 << 1)
  53. #define LOG_DISCARD_FLAG (1 << 2)
  54. #define LOG_MARK_FLAG (1 << 3)
  55. #define LOG_METADATA_FLAG (1 << 4)
  56. #define WRITE_LOG_VERSION 1ULL
  57. #define WRITE_LOG_MAGIC 0x6a736677736872ULL
  58. #define WRITE_LOG_SUPER_SECTOR 0
  59. /*
  60. * The disk format for this is braindead simple.
  61. *
  62. * At byte 0 we have our super, followed by the following sequence for
  63. * nr_entries:
  64. *
  65. * [ 1 sector ][ entry->nr_sectors ]
  66. * [log_write_entry][ data written ]
  67. *
  68. * The log_write_entry takes up a full sector so we can have arbitrary length
  69. * marks and it leaves us room for extra content in the future.
  70. */
  71. /*
  72. * Basic info about the log for userspace.
  73. */
  74. struct log_write_super {
  75. __le64 magic;
  76. __le64 version;
  77. __le64 nr_entries;
  78. __le32 sectorsize;
  79. };
  80. /*
  81. * sector - the sector we wrote.
  82. * nr_sectors - the number of sectors we wrote.
  83. * flags - flags for this log entry.
  84. * data_len - the size of the data in this log entry, this is for private log
  85. * entry stuff, the MARK data provided by userspace for example.
  86. */
  87. struct log_write_entry {
  88. __le64 sector;
  89. __le64 nr_sectors;
  90. __le64 flags;
  91. __le64 data_len;
  92. };
  93. struct log_writes_c {
  94. struct dm_dev *dev;
  95. struct dm_dev *logdev;
  96. u64 logged_entries;
  97. u32 sectorsize;
  98. u32 sectorshift;
  99. atomic_t io_blocks;
  100. atomic_t pending_blocks;
  101. sector_t next_sector;
  102. sector_t end_sector;
  103. bool logging_enabled;
  104. bool device_supports_discard;
  105. spinlock_t blocks_lock;
  106. struct list_head unflushed_blocks;
  107. struct list_head logging_blocks;
  108. wait_queue_head_t wait;
  109. struct task_struct *log_kthread;
  110. struct completion super_done;
  111. };
  112. struct pending_block {
  113. int vec_cnt;
  114. u64 flags;
  115. sector_t sector;
  116. sector_t nr_sectors;
  117. char *data;
  118. u32 datalen;
  119. struct list_head list;
  120. struct bio_vec vecs[];
  121. };
  122. struct per_bio_data {
  123. struct pending_block *block;
  124. };
  125. static inline sector_t bio_to_dev_sectors(struct log_writes_c *lc,
  126. sector_t sectors)
  127. {
  128. return sectors >> (lc->sectorshift - SECTOR_SHIFT);
  129. }
  130. static inline sector_t dev_to_bio_sectors(struct log_writes_c *lc,
  131. sector_t sectors)
  132. {
  133. return sectors << (lc->sectorshift - SECTOR_SHIFT);
  134. }
  135. static void put_pending_block(struct log_writes_c *lc)
  136. {
  137. if (atomic_dec_and_test(&lc->pending_blocks)) {
  138. smp_mb__after_atomic();
  139. if (waitqueue_active(&lc->wait))
  140. wake_up(&lc->wait);
  141. }
  142. }
  143. static void put_io_block(struct log_writes_c *lc)
  144. {
  145. if (atomic_dec_and_test(&lc->io_blocks)) {
  146. smp_mb__after_atomic();
  147. if (waitqueue_active(&lc->wait))
  148. wake_up(&lc->wait);
  149. }
  150. }
  151. static void log_end_io(struct bio *bio)
  152. {
  153. struct log_writes_c *lc = bio->bi_private;
  154. if (bio->bi_status) {
  155. unsigned long flags;
  156. DMERR("Error writing log block, error=%d", bio->bi_status);
  157. spin_lock_irqsave(&lc->blocks_lock, flags);
  158. lc->logging_enabled = false;
  159. spin_unlock_irqrestore(&lc->blocks_lock, flags);
  160. }
  161. bio_free_pages(bio);
  162. put_io_block(lc);
  163. bio_put(bio);
  164. }
  165. static void log_end_super(struct bio *bio)
  166. {
  167. struct log_writes_c *lc = bio->bi_private;
  168. complete(&lc->super_done);
  169. log_end_io(bio);
  170. }
  171. /*
  172. * Meant to be called if there is an error, it will free all the pages
  173. * associated with the block.
  174. */
  175. static void free_pending_block(struct log_writes_c *lc,
  176. struct pending_block *block)
  177. {
  178. int i;
  179. for (i = 0; i < block->vec_cnt; i++) {
  180. if (block->vecs[i].bv_page)
  181. __free_page(block->vecs[i].bv_page);
  182. }
  183. kfree(block->data);
  184. kfree(block);
  185. put_pending_block(lc);
  186. }
  187. static int write_metadata(struct log_writes_c *lc, void *entry,
  188. size_t entrylen, void *data, size_t datalen,
  189. sector_t sector)
  190. {
  191. struct bio *bio;
  192. struct page *page;
  193. void *ptr;
  194. size_t ret;
  195. bio = bio_alloc(lc->logdev->bdev, 1, REQ_OP_WRITE, GFP_KERNEL);
  196. bio->bi_iter.bi_size = 0;
  197. bio->bi_iter.bi_sector = sector;
  198. bio->bi_end_io = (sector == WRITE_LOG_SUPER_SECTOR) ?
  199. log_end_super : log_end_io;
  200. bio->bi_private = lc;
  201. page = alloc_page(GFP_KERNEL);
  202. if (!page) {
  203. DMERR("Couldn't alloc log page");
  204. bio_put(bio);
  205. goto error;
  206. }
  207. ptr = kmap_atomic(page);
  208. memcpy(ptr, entry, entrylen);
  209. if (datalen)
  210. memcpy(ptr + entrylen, data, datalen);
  211. memset(ptr + entrylen + datalen, 0,
  212. lc->sectorsize - entrylen - datalen);
  213. kunmap_atomic(ptr);
  214. ret = bio_add_page(bio, page, lc->sectorsize, 0);
  215. if (ret != lc->sectorsize) {
  216. DMERR("Couldn't add page to the log block");
  217. goto error_bio;
  218. }
  219. submit_bio(bio);
  220. return 0;
  221. error_bio:
  222. bio_put(bio);
  223. __free_page(page);
  224. error:
  225. put_io_block(lc);
  226. return -1;
  227. }
  228. static int write_inline_data(struct log_writes_c *lc, void *entry,
  229. size_t entrylen, void *data, size_t datalen,
  230. sector_t sector)
  231. {
  232. int bio_pages, pg_datalen, pg_sectorlen, i;
  233. struct page *page;
  234. struct bio *bio;
  235. size_t ret;
  236. void *ptr;
  237. while (datalen) {
  238. bio_pages = bio_max_segs(DIV_ROUND_UP(datalen, PAGE_SIZE));
  239. atomic_inc(&lc->io_blocks);
  240. bio = bio_alloc(lc->logdev->bdev, bio_pages, REQ_OP_WRITE,
  241. GFP_KERNEL);
  242. bio->bi_iter.bi_size = 0;
  243. bio->bi_iter.bi_sector = sector;
  244. bio->bi_end_io = log_end_io;
  245. bio->bi_private = lc;
  246. for (i = 0; i < bio_pages; i++) {
  247. pg_datalen = min_t(int, datalen, PAGE_SIZE);
  248. pg_sectorlen = ALIGN(pg_datalen, lc->sectorsize);
  249. page = alloc_page(GFP_KERNEL);
  250. if (!page) {
  251. DMERR("Couldn't alloc inline data page");
  252. goto error_bio;
  253. }
  254. ptr = kmap_atomic(page);
  255. memcpy(ptr, data, pg_datalen);
  256. if (pg_sectorlen > pg_datalen)
  257. memset(ptr + pg_datalen, 0, pg_sectorlen - pg_datalen);
  258. kunmap_atomic(ptr);
  259. ret = bio_add_page(bio, page, pg_sectorlen, 0);
  260. if (ret != pg_sectorlen) {
  261. DMERR("Couldn't add page of inline data");
  262. __free_page(page);
  263. goto error_bio;
  264. }
  265. datalen -= pg_datalen;
  266. data += pg_datalen;
  267. }
  268. submit_bio(bio);
  269. sector += bio_pages * PAGE_SECTORS;
  270. }
  271. return 0;
  272. error_bio:
  273. bio_free_pages(bio);
  274. bio_put(bio);
  275. put_io_block(lc);
  276. return -1;
  277. }
  278. static int log_one_block(struct log_writes_c *lc,
  279. struct pending_block *block, sector_t sector)
  280. {
  281. struct bio *bio;
  282. struct log_write_entry entry;
  283. size_t metadatalen, ret;
  284. int i;
  285. entry.sector = cpu_to_le64(block->sector);
  286. entry.nr_sectors = cpu_to_le64(block->nr_sectors);
  287. entry.flags = cpu_to_le64(block->flags);
  288. entry.data_len = cpu_to_le64(block->datalen);
  289. metadatalen = (block->flags & LOG_MARK_FLAG) ? block->datalen : 0;
  290. if (write_metadata(lc, &entry, sizeof(entry), block->data,
  291. metadatalen, sector)) {
  292. free_pending_block(lc, block);
  293. return -1;
  294. }
  295. sector += dev_to_bio_sectors(lc, 1);
  296. if (block->datalen && metadatalen == 0) {
  297. if (write_inline_data(lc, &entry, sizeof(entry), block->data,
  298. block->datalen, sector)) {
  299. free_pending_block(lc, block);
  300. return -1;
  301. }
  302. /* we don't support both inline data & bio data */
  303. goto out;
  304. }
  305. if (!block->vec_cnt)
  306. goto out;
  307. atomic_inc(&lc->io_blocks);
  308. bio = bio_alloc(lc->logdev->bdev, bio_max_segs(block->vec_cnt),
  309. REQ_OP_WRITE, GFP_KERNEL);
  310. bio->bi_iter.bi_size = 0;
  311. bio->bi_iter.bi_sector = sector;
  312. bio->bi_end_io = log_end_io;
  313. bio->bi_private = lc;
  314. for (i = 0; i < block->vec_cnt; i++) {
  315. /*
  316. * The page offset is always 0 because we allocate a new page
  317. * for every bvec in the original bio for simplicity sake.
  318. */
  319. ret = bio_add_page(bio, block->vecs[i].bv_page,
  320. block->vecs[i].bv_len, 0);
  321. if (ret != block->vecs[i].bv_len) {
  322. atomic_inc(&lc->io_blocks);
  323. submit_bio(bio);
  324. bio = bio_alloc(lc->logdev->bdev,
  325. bio_max_segs(block->vec_cnt - i),
  326. REQ_OP_WRITE, GFP_KERNEL);
  327. bio->bi_iter.bi_size = 0;
  328. bio->bi_iter.bi_sector = sector;
  329. bio->bi_end_io = log_end_io;
  330. bio->bi_private = lc;
  331. ret = bio_add_page(bio, block->vecs[i].bv_page,
  332. block->vecs[i].bv_len, 0);
  333. if (ret != block->vecs[i].bv_len) {
  334. DMERR("Couldn't add page on new bio?");
  335. bio_put(bio);
  336. goto error;
  337. }
  338. }
  339. sector += block->vecs[i].bv_len >> SECTOR_SHIFT;
  340. }
  341. submit_bio(bio);
  342. out:
  343. kfree(block->data);
  344. kfree(block);
  345. put_pending_block(lc);
  346. return 0;
  347. error:
  348. free_pending_block(lc, block);
  349. put_io_block(lc);
  350. return -1;
  351. }
  352. static int log_super(struct log_writes_c *lc)
  353. {
  354. struct log_write_super super;
  355. super.magic = cpu_to_le64(WRITE_LOG_MAGIC);
  356. super.version = cpu_to_le64(WRITE_LOG_VERSION);
  357. super.nr_entries = cpu_to_le64(lc->logged_entries);
  358. super.sectorsize = cpu_to_le32(lc->sectorsize);
  359. if (write_metadata(lc, &super, sizeof(super), NULL, 0,
  360. WRITE_LOG_SUPER_SECTOR)) {
  361. DMERR("Couldn't write super");
  362. return -1;
  363. }
  364. /*
  365. * Super sector should be writen in-order, otherwise the
  366. * nr_entries could be rewritten incorrectly by an old bio.
  367. */
  368. wait_for_completion_io(&lc->super_done);
  369. return 0;
  370. }
  371. static inline sector_t logdev_last_sector(struct log_writes_c *lc)
  372. {
  373. return bdev_nr_sectors(lc->logdev->bdev);
  374. }
  375. static int log_writes_kthread(void *arg)
  376. {
  377. struct log_writes_c *lc = (struct log_writes_c *)arg;
  378. sector_t sector = 0;
  379. while (!kthread_should_stop()) {
  380. bool super = false;
  381. bool logging_enabled;
  382. struct pending_block *block = NULL;
  383. int ret;
  384. spin_lock_irq(&lc->blocks_lock);
  385. if (!list_empty(&lc->logging_blocks)) {
  386. block = list_first_entry(&lc->logging_blocks,
  387. struct pending_block, list);
  388. list_del_init(&block->list);
  389. if (!lc->logging_enabled)
  390. goto next;
  391. sector = lc->next_sector;
  392. if (!(block->flags & LOG_DISCARD_FLAG))
  393. lc->next_sector += dev_to_bio_sectors(lc, block->nr_sectors);
  394. lc->next_sector += dev_to_bio_sectors(lc, 1);
  395. /*
  396. * Apparently the size of the device may not be known
  397. * right away, so handle this properly.
  398. */
  399. if (!lc->end_sector)
  400. lc->end_sector = logdev_last_sector(lc);
  401. if (lc->end_sector &&
  402. lc->next_sector >= lc->end_sector) {
  403. DMERR("Ran out of space on the logdev");
  404. lc->logging_enabled = false;
  405. goto next;
  406. }
  407. lc->logged_entries++;
  408. atomic_inc(&lc->io_blocks);
  409. super = (block->flags & (LOG_FUA_FLAG | LOG_MARK_FLAG));
  410. if (super)
  411. atomic_inc(&lc->io_blocks);
  412. }
  413. next:
  414. logging_enabled = lc->logging_enabled;
  415. spin_unlock_irq(&lc->blocks_lock);
  416. if (block) {
  417. if (logging_enabled) {
  418. ret = log_one_block(lc, block, sector);
  419. if (!ret && super)
  420. ret = log_super(lc);
  421. if (ret) {
  422. spin_lock_irq(&lc->blocks_lock);
  423. lc->logging_enabled = false;
  424. spin_unlock_irq(&lc->blocks_lock);
  425. }
  426. } else
  427. free_pending_block(lc, block);
  428. continue;
  429. }
  430. if (!try_to_freeze()) {
  431. set_current_state(TASK_INTERRUPTIBLE);
  432. if (!kthread_should_stop() &&
  433. list_empty(&lc->logging_blocks))
  434. schedule();
  435. __set_current_state(TASK_RUNNING);
  436. }
  437. }
  438. return 0;
  439. }
  440. /*
  441. * Construct a log-writes mapping:
  442. * log-writes <dev_path> <log_dev_path>
  443. */
  444. static int log_writes_ctr(struct dm_target *ti, unsigned int argc, char **argv)
  445. {
  446. struct log_writes_c *lc;
  447. struct dm_arg_set as;
  448. const char *devname, *logdevname;
  449. int ret;
  450. as.argc = argc;
  451. as.argv = argv;
  452. if (argc < 2) {
  453. ti->error = "Invalid argument count";
  454. return -EINVAL;
  455. }
  456. lc = kzalloc(sizeof(struct log_writes_c), GFP_KERNEL);
  457. if (!lc) {
  458. ti->error = "Cannot allocate context";
  459. return -ENOMEM;
  460. }
  461. spin_lock_init(&lc->blocks_lock);
  462. INIT_LIST_HEAD(&lc->unflushed_blocks);
  463. INIT_LIST_HEAD(&lc->logging_blocks);
  464. init_waitqueue_head(&lc->wait);
  465. init_completion(&lc->super_done);
  466. atomic_set(&lc->io_blocks, 0);
  467. atomic_set(&lc->pending_blocks, 0);
  468. devname = dm_shift_arg(&as);
  469. ret = dm_get_device(ti, devname, dm_table_get_mode(ti->table), &lc->dev);
  470. if (ret) {
  471. ti->error = "Device lookup failed";
  472. goto bad;
  473. }
  474. logdevname = dm_shift_arg(&as);
  475. ret = dm_get_device(ti, logdevname, dm_table_get_mode(ti->table),
  476. &lc->logdev);
  477. if (ret) {
  478. ti->error = "Log device lookup failed";
  479. dm_put_device(ti, lc->dev);
  480. goto bad;
  481. }
  482. lc->sectorsize = bdev_logical_block_size(lc->dev->bdev);
  483. lc->sectorshift = ilog2(lc->sectorsize);
  484. lc->log_kthread = kthread_run(log_writes_kthread, lc, "log-write");
  485. if (IS_ERR(lc->log_kthread)) {
  486. ret = PTR_ERR(lc->log_kthread);
  487. ti->error = "Couldn't alloc kthread";
  488. dm_put_device(ti, lc->dev);
  489. dm_put_device(ti, lc->logdev);
  490. goto bad;
  491. }
  492. /*
  493. * next_sector is in 512b sectors to correspond to what bi_sector expects.
  494. * The super starts at sector 0, and the next_sector is the next logical
  495. * one based on the sectorsize of the device.
  496. */
  497. lc->next_sector = lc->sectorsize >> SECTOR_SHIFT;
  498. lc->logging_enabled = true;
  499. lc->end_sector = logdev_last_sector(lc);
  500. lc->device_supports_discard = true;
  501. ti->num_flush_bios = 1;
  502. ti->flush_supported = true;
  503. ti->num_discard_bios = 1;
  504. ti->discards_supported = true;
  505. ti->per_io_data_size = sizeof(struct per_bio_data);
  506. ti->private = lc;
  507. return 0;
  508. bad:
  509. kfree(lc);
  510. return ret;
  511. }
  512. static int log_mark(struct log_writes_c *lc, char *data)
  513. {
  514. struct pending_block *block;
  515. size_t maxsize = lc->sectorsize - sizeof(struct log_write_entry);
  516. block = kzalloc(sizeof(struct pending_block), GFP_KERNEL);
  517. if (!block) {
  518. DMERR("Error allocating pending block");
  519. return -ENOMEM;
  520. }
  521. block->data = kstrndup(data, maxsize - 1, GFP_KERNEL);
  522. if (!block->data) {
  523. DMERR("Error copying mark data");
  524. kfree(block);
  525. return -ENOMEM;
  526. }
  527. atomic_inc(&lc->pending_blocks);
  528. block->datalen = strlen(block->data);
  529. block->flags |= LOG_MARK_FLAG;
  530. spin_lock_irq(&lc->blocks_lock);
  531. list_add_tail(&block->list, &lc->logging_blocks);
  532. spin_unlock_irq(&lc->blocks_lock);
  533. wake_up_process(lc->log_kthread);
  534. return 0;
  535. }
  536. static void log_writes_dtr(struct dm_target *ti)
  537. {
  538. struct log_writes_c *lc = ti->private;
  539. spin_lock_irq(&lc->blocks_lock);
  540. list_splice_init(&lc->unflushed_blocks, &lc->logging_blocks);
  541. spin_unlock_irq(&lc->blocks_lock);
  542. /*
  543. * This is just nice to have since it'll update the super to include the
  544. * unflushed blocks, if it fails we don't really care.
  545. */
  546. log_mark(lc, "dm-log-writes-end");
  547. wake_up_process(lc->log_kthread);
  548. wait_event(lc->wait, !atomic_read(&lc->io_blocks) &&
  549. !atomic_read(&lc->pending_blocks));
  550. kthread_stop(lc->log_kthread);
  551. WARN_ON(!list_empty(&lc->logging_blocks));
  552. WARN_ON(!list_empty(&lc->unflushed_blocks));
  553. dm_put_device(ti, lc->dev);
  554. dm_put_device(ti, lc->logdev);
  555. kfree(lc);
  556. }
  557. static void normal_map_bio(struct dm_target *ti, struct bio *bio)
  558. {
  559. struct log_writes_c *lc = ti->private;
  560. bio_set_dev(bio, lc->dev->bdev);
  561. }
  562. static int log_writes_map(struct dm_target *ti, struct bio *bio)
  563. {
  564. struct log_writes_c *lc = ti->private;
  565. struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data));
  566. struct pending_block *block;
  567. struct bvec_iter iter;
  568. struct bio_vec bv;
  569. size_t alloc_size;
  570. int i = 0;
  571. bool flush_bio = (bio->bi_opf & REQ_PREFLUSH);
  572. bool fua_bio = (bio->bi_opf & REQ_FUA);
  573. bool discard_bio = (bio_op(bio) == REQ_OP_DISCARD);
  574. bool meta_bio = (bio->bi_opf & REQ_META);
  575. pb->block = NULL;
  576. /* Don't bother doing anything if logging has been disabled */
  577. if (!lc->logging_enabled)
  578. goto map_bio;
  579. /*
  580. * Map reads as normal.
  581. */
  582. if (bio_data_dir(bio) == READ)
  583. goto map_bio;
  584. /* No sectors and not a flush? Don't care */
  585. if (!bio_sectors(bio) && !flush_bio)
  586. goto map_bio;
  587. /*
  588. * Discards will have bi_size set but there's no actual data, so just
  589. * allocate the size of the pending block.
  590. */
  591. if (discard_bio)
  592. alloc_size = sizeof(struct pending_block);
  593. else
  594. alloc_size = struct_size(block, vecs, bio_segments(bio));
  595. block = kzalloc(alloc_size, GFP_NOIO);
  596. if (!block) {
  597. DMERR("Error allocating pending block");
  598. spin_lock_irq(&lc->blocks_lock);
  599. lc->logging_enabled = false;
  600. spin_unlock_irq(&lc->blocks_lock);
  601. return DM_MAPIO_KILL;
  602. }
  603. INIT_LIST_HEAD(&block->list);
  604. pb->block = block;
  605. atomic_inc(&lc->pending_blocks);
  606. if (flush_bio)
  607. block->flags |= LOG_FLUSH_FLAG;
  608. if (fua_bio)
  609. block->flags |= LOG_FUA_FLAG;
  610. if (discard_bio)
  611. block->flags |= LOG_DISCARD_FLAG;
  612. if (meta_bio)
  613. block->flags |= LOG_METADATA_FLAG;
  614. block->sector = bio_to_dev_sectors(lc, bio->bi_iter.bi_sector);
  615. block->nr_sectors = bio_to_dev_sectors(lc, bio_sectors(bio));
  616. /* We don't need the data, just submit */
  617. if (discard_bio) {
  618. WARN_ON(flush_bio || fua_bio);
  619. if (lc->device_supports_discard)
  620. goto map_bio;
  621. bio_endio(bio);
  622. return DM_MAPIO_SUBMITTED;
  623. }
  624. /* Flush bio, splice the unflushed blocks onto this list and submit */
  625. if (flush_bio && !bio_sectors(bio)) {
  626. spin_lock_irq(&lc->blocks_lock);
  627. list_splice_init(&lc->unflushed_blocks, &block->list);
  628. spin_unlock_irq(&lc->blocks_lock);
  629. goto map_bio;
  630. }
  631. /*
  632. * We will write this bio somewhere else way later so we need to copy
  633. * the actual contents into new pages so we know the data will always be
  634. * there.
  635. *
  636. * We do this because this could be a bio from O_DIRECT in which case we
  637. * can't just hold onto the page until some later point, we have to
  638. * manually copy the contents.
  639. */
  640. bio_for_each_segment(bv, bio, iter) {
  641. struct page *page;
  642. void *dst;
  643. page = alloc_page(GFP_NOIO);
  644. if (!page) {
  645. DMERR("Error allocing page");
  646. free_pending_block(lc, block);
  647. spin_lock_irq(&lc->blocks_lock);
  648. lc->logging_enabled = false;
  649. spin_unlock_irq(&lc->blocks_lock);
  650. return DM_MAPIO_KILL;
  651. }
  652. dst = kmap_atomic(page);
  653. memcpy_from_bvec(dst, &bv);
  654. kunmap_atomic(dst);
  655. block->vecs[i].bv_page = page;
  656. block->vecs[i].bv_len = bv.bv_len;
  657. block->vec_cnt++;
  658. i++;
  659. }
  660. /* Had a flush with data in it, weird */
  661. if (flush_bio) {
  662. spin_lock_irq(&lc->blocks_lock);
  663. list_splice_init(&lc->unflushed_blocks, &block->list);
  664. spin_unlock_irq(&lc->blocks_lock);
  665. }
  666. map_bio:
  667. normal_map_bio(ti, bio);
  668. return DM_MAPIO_REMAPPED;
  669. }
  670. static int normal_end_io(struct dm_target *ti, struct bio *bio,
  671. blk_status_t *error)
  672. {
  673. struct log_writes_c *lc = ti->private;
  674. struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data));
  675. if (bio_data_dir(bio) == WRITE && pb->block) {
  676. struct pending_block *block = pb->block;
  677. unsigned long flags;
  678. spin_lock_irqsave(&lc->blocks_lock, flags);
  679. if (block->flags & LOG_FLUSH_FLAG) {
  680. list_splice_tail_init(&block->list, &lc->logging_blocks);
  681. list_add_tail(&block->list, &lc->logging_blocks);
  682. wake_up_process(lc->log_kthread);
  683. } else if (block->flags & LOG_FUA_FLAG) {
  684. list_add_tail(&block->list, &lc->logging_blocks);
  685. wake_up_process(lc->log_kthread);
  686. } else
  687. list_add_tail(&block->list, &lc->unflushed_blocks);
  688. spin_unlock_irqrestore(&lc->blocks_lock, flags);
  689. }
  690. return DM_ENDIO_DONE;
  691. }
  692. /*
  693. * INFO format: <logged entries> <highest allocated sector>
  694. */
  695. static void log_writes_status(struct dm_target *ti, status_type_t type,
  696. unsigned int status_flags, char *result,
  697. unsigned int maxlen)
  698. {
  699. unsigned int sz = 0;
  700. struct log_writes_c *lc = ti->private;
  701. switch (type) {
  702. case STATUSTYPE_INFO:
  703. DMEMIT("%llu %llu", lc->logged_entries,
  704. (unsigned long long)lc->next_sector - 1);
  705. if (!lc->logging_enabled)
  706. DMEMIT(" logging_disabled");
  707. break;
  708. case STATUSTYPE_TABLE:
  709. DMEMIT("%s %s", lc->dev->name, lc->logdev->name);
  710. break;
  711. case STATUSTYPE_IMA:
  712. *result = '\0';
  713. break;
  714. }
  715. }
  716. static int log_writes_prepare_ioctl(struct dm_target *ti,
  717. struct block_device **bdev)
  718. {
  719. struct log_writes_c *lc = ti->private;
  720. struct dm_dev *dev = lc->dev;
  721. *bdev = dev->bdev;
  722. /*
  723. * Only pass ioctls through if the device sizes match exactly.
  724. */
  725. if (ti->len != bdev_nr_sectors(dev->bdev))
  726. return 1;
  727. return 0;
  728. }
  729. static int log_writes_iterate_devices(struct dm_target *ti,
  730. iterate_devices_callout_fn fn,
  731. void *data)
  732. {
  733. struct log_writes_c *lc = ti->private;
  734. return fn(ti, lc->dev, 0, ti->len, data);
  735. }
  736. /*
  737. * Messages supported:
  738. * mark <mark data> - specify the marked data.
  739. */
  740. static int log_writes_message(struct dm_target *ti, unsigned int argc, char **argv,
  741. char *result, unsigned int maxlen)
  742. {
  743. int r = -EINVAL;
  744. struct log_writes_c *lc = ti->private;
  745. if (argc != 2) {
  746. DMWARN("Invalid log-writes message arguments, expect 2 arguments, got %d", argc);
  747. return r;
  748. }
  749. if (!strcasecmp(argv[0], "mark"))
  750. r = log_mark(lc, argv[1]);
  751. else
  752. DMWARN("Unrecognised log writes target message received: %s", argv[0]);
  753. return r;
  754. }
  755. static void log_writes_io_hints(struct dm_target *ti, struct queue_limits *limits)
  756. {
  757. struct log_writes_c *lc = ti->private;
  758. if (!bdev_max_discard_sectors(lc->dev->bdev)) {
  759. lc->device_supports_discard = false;
  760. limits->discard_granularity = lc->sectorsize;
  761. limits->max_discard_sectors = (UINT_MAX >> SECTOR_SHIFT);
  762. }
  763. limits->logical_block_size = bdev_logical_block_size(lc->dev->bdev);
  764. limits->physical_block_size = bdev_physical_block_size(lc->dev->bdev);
  765. limits->io_min = limits->physical_block_size;
  766. limits->dma_alignment = limits->logical_block_size - 1;
  767. }
  768. #if IS_ENABLED(CONFIG_FS_DAX)
  769. static struct dax_device *log_writes_dax_pgoff(struct dm_target *ti,
  770. pgoff_t *pgoff)
  771. {
  772. struct log_writes_c *lc = ti->private;
  773. *pgoff += (get_start_sect(lc->dev->bdev) >> PAGE_SECTORS_SHIFT);
  774. return lc->dev->dax_dev;
  775. }
  776. static long log_writes_dax_direct_access(struct dm_target *ti, pgoff_t pgoff,
  777. long nr_pages, enum dax_access_mode mode, void **kaddr,
  778. pfn_t *pfn)
  779. {
  780. struct dax_device *dax_dev = log_writes_dax_pgoff(ti, &pgoff);
  781. return dax_direct_access(dax_dev, pgoff, nr_pages, mode, kaddr, pfn);
  782. }
  783. static int log_writes_dax_zero_page_range(struct dm_target *ti, pgoff_t pgoff,
  784. size_t nr_pages)
  785. {
  786. struct dax_device *dax_dev = log_writes_dax_pgoff(ti, &pgoff);
  787. return dax_zero_page_range(dax_dev, pgoff, nr_pages << PAGE_SHIFT);
  788. }
  789. static size_t log_writes_dax_recovery_write(struct dm_target *ti,
  790. pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i)
  791. {
  792. struct dax_device *dax_dev = log_writes_dax_pgoff(ti, &pgoff);
  793. return dax_recovery_write(dax_dev, pgoff, addr, bytes, i);
  794. }
  795. #else
  796. #define log_writes_dax_direct_access NULL
  797. #define log_writes_dax_zero_page_range NULL
  798. #define log_writes_dax_recovery_write NULL
  799. #endif
  800. static struct target_type log_writes_target = {
  801. .name = "log-writes",
  802. .version = {1, 1, 0},
  803. .module = THIS_MODULE,
  804. .ctr = log_writes_ctr,
  805. .dtr = log_writes_dtr,
  806. .map = log_writes_map,
  807. .end_io = normal_end_io,
  808. .status = log_writes_status,
  809. .prepare_ioctl = log_writes_prepare_ioctl,
  810. .message = log_writes_message,
  811. .iterate_devices = log_writes_iterate_devices,
  812. .io_hints = log_writes_io_hints,
  813. .direct_access = log_writes_dax_direct_access,
  814. .dax_zero_page_range = log_writes_dax_zero_page_range,
  815. .dax_recovery_write = log_writes_dax_recovery_write,
  816. };
  817. static int __init dm_log_writes_init(void)
  818. {
  819. int r = dm_register_target(&log_writes_target);
  820. if (r < 0)
  821. DMERR("register failed %d", r);
  822. return r;
  823. }
  824. static void __exit dm_log_writes_exit(void)
  825. {
  826. dm_unregister_target(&log_writes_target);
  827. }
  828. module_init(dm_log_writes_init);
  829. module_exit(dm_log_writes_exit);
  830. MODULE_DESCRIPTION(DM_NAME " log writes target");
  831. MODULE_AUTHOR("Josef Bacik <[email protected]>");
  832. MODULE_LICENSE("GPL");