dm-era-target.c 39 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. #include "dm.h"
  3. #include "persistent-data/dm-transaction-manager.h"
  4. #include "persistent-data/dm-bitset.h"
  5. #include "persistent-data/dm-space-map.h"
  6. #include <linux/dm-io.h>
  7. #include <linux/dm-kcopyd.h>
  8. #include <linux/init.h>
  9. #include <linux/mempool.h>
  10. #include <linux/module.h>
  11. #include <linux/slab.h>
  12. #include <linux/vmalloc.h>
  13. #define DM_MSG_PREFIX "era"
  14. #define SUPERBLOCK_LOCATION 0
  15. #define SUPERBLOCK_MAGIC 2126579579
  16. #define SUPERBLOCK_CSUM_XOR 146538381
  17. #define MIN_ERA_VERSION 1
  18. #define MAX_ERA_VERSION 1
  19. #define INVALID_WRITESET_ROOT SUPERBLOCK_LOCATION
  20. #define MIN_BLOCK_SIZE 8
  21. /*----------------------------------------------------------------
  22. * Writeset
  23. *--------------------------------------------------------------*/
  24. struct writeset_metadata {
  25. uint32_t nr_bits;
  26. dm_block_t root;
  27. };
  28. struct writeset {
  29. struct writeset_metadata md;
  30. /*
  31. * An in core copy of the bits to save constantly doing look ups on
  32. * disk.
  33. */
  34. unsigned long *bits;
  35. };
  36. /*
  37. * This does not free off the on disk bitset as this will normally be done
  38. * after digesting into the era array.
  39. */
  40. static void writeset_free(struct writeset *ws)
  41. {
  42. vfree(ws->bits);
  43. ws->bits = NULL;
  44. }
  45. static int setup_on_disk_bitset(struct dm_disk_bitset *info,
  46. unsigned int nr_bits, dm_block_t *root)
  47. {
  48. int r;
  49. r = dm_bitset_empty(info, root);
  50. if (r)
  51. return r;
  52. return dm_bitset_resize(info, *root, 0, nr_bits, false, root);
  53. }
  54. static size_t bitset_size(unsigned int nr_bits)
  55. {
  56. return sizeof(unsigned long) * dm_div_up(nr_bits, BITS_PER_LONG);
  57. }
  58. /*
  59. * Allocates memory for the in core bitset.
  60. */
  61. static int writeset_alloc(struct writeset *ws, dm_block_t nr_blocks)
  62. {
  63. ws->bits = vzalloc(bitset_size(nr_blocks));
  64. if (!ws->bits) {
  65. DMERR("%s: couldn't allocate in memory bitset", __func__);
  66. return -ENOMEM;
  67. }
  68. return 0;
  69. }
  70. /*
  71. * Wipes the in-core bitset, and creates a new on disk bitset.
  72. */
  73. static int writeset_init(struct dm_disk_bitset *info, struct writeset *ws,
  74. dm_block_t nr_blocks)
  75. {
  76. int r;
  77. memset(ws->bits, 0, bitset_size(nr_blocks));
  78. ws->md.nr_bits = nr_blocks;
  79. r = setup_on_disk_bitset(info, ws->md.nr_bits, &ws->md.root);
  80. if (r) {
  81. DMERR("%s: setup_on_disk_bitset failed", __func__);
  82. return r;
  83. }
  84. return 0;
  85. }
  86. static bool writeset_marked(struct writeset *ws, dm_block_t block)
  87. {
  88. return test_bit(block, ws->bits);
  89. }
  90. static int writeset_marked_on_disk(struct dm_disk_bitset *info,
  91. struct writeset_metadata *m, dm_block_t block,
  92. bool *result)
  93. {
  94. dm_block_t old = m->root;
  95. /*
  96. * The bitset was flushed when it was archived, so we know there'll
  97. * be no change to the root.
  98. */
  99. int r = dm_bitset_test_bit(info, m->root, block, &m->root, result);
  100. if (r) {
  101. DMERR("%s: dm_bitset_test_bit failed", __func__);
  102. return r;
  103. }
  104. BUG_ON(m->root != old);
  105. return r;
  106. }
  107. /*
  108. * Returns < 0 on error, 0 if the bit wasn't previously set, 1 if it was.
  109. */
  110. static int writeset_test_and_set(struct dm_disk_bitset *info,
  111. struct writeset *ws, uint32_t block)
  112. {
  113. int r;
  114. if (!test_bit(block, ws->bits)) {
  115. r = dm_bitset_set_bit(info, ws->md.root, block, &ws->md.root);
  116. if (r) {
  117. /* FIXME: fail mode */
  118. return r;
  119. }
  120. return 0;
  121. }
  122. return 1;
  123. }
  124. /*----------------------------------------------------------------
  125. * On disk metadata layout
  126. *--------------------------------------------------------------*/
  127. #define SPACE_MAP_ROOT_SIZE 128
  128. #define UUID_LEN 16
  129. struct writeset_disk {
  130. __le32 nr_bits;
  131. __le64 root;
  132. } __packed;
  133. struct superblock_disk {
  134. __le32 csum;
  135. __le32 flags;
  136. __le64 blocknr;
  137. __u8 uuid[UUID_LEN];
  138. __le64 magic;
  139. __le32 version;
  140. __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
  141. __le32 data_block_size;
  142. __le32 metadata_block_size;
  143. __le32 nr_blocks;
  144. __le32 current_era;
  145. struct writeset_disk current_writeset;
  146. /*
  147. * Only these two fields are valid within the metadata snapshot.
  148. */
  149. __le64 writeset_tree_root;
  150. __le64 era_array_root;
  151. __le64 metadata_snap;
  152. } __packed;
  153. /*----------------------------------------------------------------
  154. * Superblock validation
  155. *--------------------------------------------------------------*/
  156. static void sb_prepare_for_write(struct dm_block_validator *v,
  157. struct dm_block *b,
  158. size_t sb_block_size)
  159. {
  160. struct superblock_disk *disk = dm_block_data(b);
  161. disk->blocknr = cpu_to_le64(dm_block_location(b));
  162. disk->csum = cpu_to_le32(dm_bm_checksum(&disk->flags,
  163. sb_block_size - sizeof(__le32),
  164. SUPERBLOCK_CSUM_XOR));
  165. }
  166. static int check_metadata_version(struct superblock_disk *disk)
  167. {
  168. uint32_t metadata_version = le32_to_cpu(disk->version);
  169. if (metadata_version < MIN_ERA_VERSION || metadata_version > MAX_ERA_VERSION) {
  170. DMERR("Era metadata version %u found, but only versions between %u and %u supported.",
  171. metadata_version, MIN_ERA_VERSION, MAX_ERA_VERSION);
  172. return -EINVAL;
  173. }
  174. return 0;
  175. }
  176. static int sb_check(struct dm_block_validator *v,
  177. struct dm_block *b,
  178. size_t sb_block_size)
  179. {
  180. struct superblock_disk *disk = dm_block_data(b);
  181. __le32 csum_le;
  182. if (dm_block_location(b) != le64_to_cpu(disk->blocknr)) {
  183. DMERR("sb_check failed: blocknr %llu: wanted %llu",
  184. le64_to_cpu(disk->blocknr),
  185. (unsigned long long)dm_block_location(b));
  186. return -ENOTBLK;
  187. }
  188. if (le64_to_cpu(disk->magic) != SUPERBLOCK_MAGIC) {
  189. DMERR("sb_check failed: magic %llu: wanted %llu",
  190. le64_to_cpu(disk->magic),
  191. (unsigned long long) SUPERBLOCK_MAGIC);
  192. return -EILSEQ;
  193. }
  194. csum_le = cpu_to_le32(dm_bm_checksum(&disk->flags,
  195. sb_block_size - sizeof(__le32),
  196. SUPERBLOCK_CSUM_XOR));
  197. if (csum_le != disk->csum) {
  198. DMERR("sb_check failed: csum %u: wanted %u",
  199. le32_to_cpu(csum_le), le32_to_cpu(disk->csum));
  200. return -EILSEQ;
  201. }
  202. return check_metadata_version(disk);
  203. }
  204. static struct dm_block_validator sb_validator = {
  205. .name = "superblock",
  206. .prepare_for_write = sb_prepare_for_write,
  207. .check = sb_check
  208. };
  209. /*----------------------------------------------------------------
  210. * Low level metadata handling
  211. *--------------------------------------------------------------*/
  212. #define DM_ERA_METADATA_BLOCK_SIZE 4096
  213. #define ERA_MAX_CONCURRENT_LOCKS 5
  214. struct era_metadata {
  215. struct block_device *bdev;
  216. struct dm_block_manager *bm;
  217. struct dm_space_map *sm;
  218. struct dm_transaction_manager *tm;
  219. dm_block_t block_size;
  220. uint32_t nr_blocks;
  221. uint32_t current_era;
  222. /*
  223. * We preallocate 2 writesets. When an era rolls over we
  224. * switch between them. This means the allocation is done at
  225. * preresume time, rather than on the io path.
  226. */
  227. struct writeset writesets[2];
  228. struct writeset *current_writeset;
  229. dm_block_t writeset_tree_root;
  230. dm_block_t era_array_root;
  231. struct dm_disk_bitset bitset_info;
  232. struct dm_btree_info writeset_tree_info;
  233. struct dm_array_info era_array_info;
  234. dm_block_t metadata_snap;
  235. /*
  236. * A flag that is set whenever a writeset has been archived.
  237. */
  238. bool archived_writesets;
  239. /*
  240. * Reading the space map root can fail, so we read it into this
  241. * buffer before the superblock is locked and updated.
  242. */
  243. __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
  244. };
  245. static int superblock_read_lock(struct era_metadata *md,
  246. struct dm_block **sblock)
  247. {
  248. return dm_bm_read_lock(md->bm, SUPERBLOCK_LOCATION,
  249. &sb_validator, sblock);
  250. }
  251. static int superblock_lock_zero(struct era_metadata *md,
  252. struct dm_block **sblock)
  253. {
  254. return dm_bm_write_lock_zero(md->bm, SUPERBLOCK_LOCATION,
  255. &sb_validator, sblock);
  256. }
  257. static int superblock_lock(struct era_metadata *md,
  258. struct dm_block **sblock)
  259. {
  260. return dm_bm_write_lock(md->bm, SUPERBLOCK_LOCATION,
  261. &sb_validator, sblock);
  262. }
  263. /* FIXME: duplication with cache and thin */
  264. static int superblock_all_zeroes(struct dm_block_manager *bm, bool *result)
  265. {
  266. int r;
  267. unsigned int i;
  268. struct dm_block *b;
  269. __le64 *data_le, zero = cpu_to_le64(0);
  270. unsigned int sb_block_size = dm_bm_block_size(bm) / sizeof(__le64);
  271. /*
  272. * We can't use a validator here - it may be all zeroes.
  273. */
  274. r = dm_bm_read_lock(bm, SUPERBLOCK_LOCATION, NULL, &b);
  275. if (r)
  276. return r;
  277. data_le = dm_block_data(b);
  278. *result = true;
  279. for (i = 0; i < sb_block_size; i++) {
  280. if (data_le[i] != zero) {
  281. *result = false;
  282. break;
  283. }
  284. }
  285. dm_bm_unlock(b);
  286. return 0;
  287. }
  288. /*----------------------------------------------------------------*/
  289. static void ws_pack(const struct writeset_metadata *core, struct writeset_disk *disk)
  290. {
  291. disk->nr_bits = cpu_to_le32(core->nr_bits);
  292. disk->root = cpu_to_le64(core->root);
  293. }
  294. static void ws_unpack(const struct writeset_disk *disk, struct writeset_metadata *core)
  295. {
  296. core->nr_bits = le32_to_cpu(disk->nr_bits);
  297. core->root = le64_to_cpu(disk->root);
  298. }
  299. static void ws_inc(void *context, const void *value, unsigned int count)
  300. {
  301. struct era_metadata *md = context;
  302. struct writeset_disk ws_d;
  303. dm_block_t b;
  304. unsigned int i;
  305. for (i = 0; i < count; i++) {
  306. memcpy(&ws_d, value + (i * sizeof(ws_d)), sizeof(ws_d));
  307. b = le64_to_cpu(ws_d.root);
  308. dm_tm_inc(md->tm, b);
  309. }
  310. }
  311. static void ws_dec(void *context, const void *value, unsigned int count)
  312. {
  313. struct era_metadata *md = context;
  314. struct writeset_disk ws_d;
  315. dm_block_t b;
  316. unsigned int i;
  317. for (i = 0; i < count; i++) {
  318. memcpy(&ws_d, value + (i * sizeof(ws_d)), sizeof(ws_d));
  319. b = le64_to_cpu(ws_d.root);
  320. dm_bitset_del(&md->bitset_info, b);
  321. }
  322. }
  323. static int ws_eq(void *context, const void *value1, const void *value2)
  324. {
  325. return !memcmp(value1, value2, sizeof(struct writeset_disk));
  326. }
  327. /*----------------------------------------------------------------*/
  328. static void setup_writeset_tree_info(struct era_metadata *md)
  329. {
  330. struct dm_btree_value_type *vt = &md->writeset_tree_info.value_type;
  331. md->writeset_tree_info.tm = md->tm;
  332. md->writeset_tree_info.levels = 1;
  333. vt->context = md;
  334. vt->size = sizeof(struct writeset_disk);
  335. vt->inc = ws_inc;
  336. vt->dec = ws_dec;
  337. vt->equal = ws_eq;
  338. }
  339. static void setup_era_array_info(struct era_metadata *md)
  340. {
  341. struct dm_btree_value_type vt;
  342. vt.context = NULL;
  343. vt.size = sizeof(__le32);
  344. vt.inc = NULL;
  345. vt.dec = NULL;
  346. vt.equal = NULL;
  347. dm_array_info_init(&md->era_array_info, md->tm, &vt);
  348. }
  349. static void setup_infos(struct era_metadata *md)
  350. {
  351. dm_disk_bitset_init(md->tm, &md->bitset_info);
  352. setup_writeset_tree_info(md);
  353. setup_era_array_info(md);
  354. }
  355. /*----------------------------------------------------------------*/
  356. static int create_fresh_metadata(struct era_metadata *md)
  357. {
  358. int r;
  359. r = dm_tm_create_with_sm(md->bm, SUPERBLOCK_LOCATION,
  360. &md->tm, &md->sm);
  361. if (r < 0) {
  362. DMERR("dm_tm_create_with_sm failed");
  363. return r;
  364. }
  365. setup_infos(md);
  366. r = dm_btree_empty(&md->writeset_tree_info, &md->writeset_tree_root);
  367. if (r) {
  368. DMERR("couldn't create new writeset tree");
  369. goto bad;
  370. }
  371. r = dm_array_empty(&md->era_array_info, &md->era_array_root);
  372. if (r) {
  373. DMERR("couldn't create era array");
  374. goto bad;
  375. }
  376. return 0;
  377. bad:
  378. dm_sm_destroy(md->sm);
  379. dm_tm_destroy(md->tm);
  380. return r;
  381. }
  382. static int save_sm_root(struct era_metadata *md)
  383. {
  384. int r;
  385. size_t metadata_len;
  386. r = dm_sm_root_size(md->sm, &metadata_len);
  387. if (r < 0)
  388. return r;
  389. return dm_sm_copy_root(md->sm, &md->metadata_space_map_root,
  390. metadata_len);
  391. }
  392. static void copy_sm_root(struct era_metadata *md, struct superblock_disk *disk)
  393. {
  394. memcpy(&disk->metadata_space_map_root,
  395. &md->metadata_space_map_root,
  396. sizeof(md->metadata_space_map_root));
  397. }
  398. /*
  399. * Writes a superblock, including the static fields that don't get updated
  400. * with every commit (possible optimisation here). 'md' should be fully
  401. * constructed when this is called.
  402. */
  403. static void prepare_superblock(struct era_metadata *md, struct superblock_disk *disk)
  404. {
  405. disk->magic = cpu_to_le64(SUPERBLOCK_MAGIC);
  406. disk->flags = cpu_to_le32(0ul);
  407. /* FIXME: can't keep blanking the uuid (uuid is currently unused though) */
  408. memset(disk->uuid, 0, sizeof(disk->uuid));
  409. disk->version = cpu_to_le32(MAX_ERA_VERSION);
  410. copy_sm_root(md, disk);
  411. disk->data_block_size = cpu_to_le32(md->block_size);
  412. disk->metadata_block_size = cpu_to_le32(DM_ERA_METADATA_BLOCK_SIZE >> SECTOR_SHIFT);
  413. disk->nr_blocks = cpu_to_le32(md->nr_blocks);
  414. disk->current_era = cpu_to_le32(md->current_era);
  415. ws_pack(&md->current_writeset->md, &disk->current_writeset);
  416. disk->writeset_tree_root = cpu_to_le64(md->writeset_tree_root);
  417. disk->era_array_root = cpu_to_le64(md->era_array_root);
  418. disk->metadata_snap = cpu_to_le64(md->metadata_snap);
  419. }
  420. static int write_superblock(struct era_metadata *md)
  421. {
  422. int r;
  423. struct dm_block *sblock;
  424. struct superblock_disk *disk;
  425. r = save_sm_root(md);
  426. if (r) {
  427. DMERR("%s: save_sm_root failed", __func__);
  428. return r;
  429. }
  430. r = superblock_lock_zero(md, &sblock);
  431. if (r)
  432. return r;
  433. disk = dm_block_data(sblock);
  434. prepare_superblock(md, disk);
  435. return dm_tm_commit(md->tm, sblock);
  436. }
  437. /*
  438. * Assumes block_size and the infos are set.
  439. */
  440. static int format_metadata(struct era_metadata *md)
  441. {
  442. int r;
  443. r = create_fresh_metadata(md);
  444. if (r)
  445. return r;
  446. r = write_superblock(md);
  447. if (r) {
  448. dm_sm_destroy(md->sm);
  449. dm_tm_destroy(md->tm);
  450. return r;
  451. }
  452. return 0;
  453. }
  454. static int open_metadata(struct era_metadata *md)
  455. {
  456. int r;
  457. struct dm_block *sblock;
  458. struct superblock_disk *disk;
  459. r = superblock_read_lock(md, &sblock);
  460. if (r) {
  461. DMERR("couldn't read_lock superblock");
  462. return r;
  463. }
  464. disk = dm_block_data(sblock);
  465. /* Verify the data block size hasn't changed */
  466. if (le32_to_cpu(disk->data_block_size) != md->block_size) {
  467. DMERR("changing the data block size (from %u to %llu) is not supported",
  468. le32_to_cpu(disk->data_block_size), md->block_size);
  469. r = -EINVAL;
  470. goto bad;
  471. }
  472. r = dm_tm_open_with_sm(md->bm, SUPERBLOCK_LOCATION,
  473. disk->metadata_space_map_root,
  474. sizeof(disk->metadata_space_map_root),
  475. &md->tm, &md->sm);
  476. if (r) {
  477. DMERR("dm_tm_open_with_sm failed");
  478. goto bad;
  479. }
  480. setup_infos(md);
  481. md->nr_blocks = le32_to_cpu(disk->nr_blocks);
  482. md->current_era = le32_to_cpu(disk->current_era);
  483. ws_unpack(&disk->current_writeset, &md->current_writeset->md);
  484. md->writeset_tree_root = le64_to_cpu(disk->writeset_tree_root);
  485. md->era_array_root = le64_to_cpu(disk->era_array_root);
  486. md->metadata_snap = le64_to_cpu(disk->metadata_snap);
  487. md->archived_writesets = true;
  488. dm_bm_unlock(sblock);
  489. return 0;
  490. bad:
  491. dm_bm_unlock(sblock);
  492. return r;
  493. }
  494. static int open_or_format_metadata(struct era_metadata *md,
  495. bool may_format)
  496. {
  497. int r;
  498. bool unformatted = false;
  499. r = superblock_all_zeroes(md->bm, &unformatted);
  500. if (r)
  501. return r;
  502. if (unformatted)
  503. return may_format ? format_metadata(md) : -EPERM;
  504. return open_metadata(md);
  505. }
  506. static int create_persistent_data_objects(struct era_metadata *md,
  507. bool may_format)
  508. {
  509. int r;
  510. md->bm = dm_block_manager_create(md->bdev, DM_ERA_METADATA_BLOCK_SIZE,
  511. ERA_MAX_CONCURRENT_LOCKS);
  512. if (IS_ERR(md->bm)) {
  513. DMERR("could not create block manager");
  514. return PTR_ERR(md->bm);
  515. }
  516. r = open_or_format_metadata(md, may_format);
  517. if (r)
  518. dm_block_manager_destroy(md->bm);
  519. return r;
  520. }
  521. static void destroy_persistent_data_objects(struct era_metadata *md)
  522. {
  523. dm_sm_destroy(md->sm);
  524. dm_tm_destroy(md->tm);
  525. dm_block_manager_destroy(md->bm);
  526. }
  527. /*
  528. * This waits until all era_map threads have picked up the new filter.
  529. */
  530. static void swap_writeset(struct era_metadata *md, struct writeset *new_writeset)
  531. {
  532. rcu_assign_pointer(md->current_writeset, new_writeset);
  533. synchronize_rcu();
  534. }
  535. /*----------------------------------------------------------------
  536. * Writesets get 'digested' into the main era array.
  537. *
  538. * We're using a coroutine here so the worker thread can do the digestion,
  539. * thus avoiding synchronisation of the metadata. Digesting a whole
  540. * writeset in one go would cause too much latency.
  541. *--------------------------------------------------------------*/
  542. struct digest {
  543. uint32_t era;
  544. unsigned int nr_bits, current_bit;
  545. struct writeset_metadata writeset;
  546. __le32 value;
  547. struct dm_disk_bitset info;
  548. int (*step)(struct era_metadata *, struct digest *);
  549. };
  550. static int metadata_digest_lookup_writeset(struct era_metadata *md,
  551. struct digest *d);
  552. static int metadata_digest_remove_writeset(struct era_metadata *md,
  553. struct digest *d)
  554. {
  555. int r;
  556. uint64_t key = d->era;
  557. r = dm_btree_remove(&md->writeset_tree_info, md->writeset_tree_root,
  558. &key, &md->writeset_tree_root);
  559. if (r) {
  560. DMERR("%s: dm_btree_remove failed", __func__);
  561. return r;
  562. }
  563. d->step = metadata_digest_lookup_writeset;
  564. return 0;
  565. }
  566. #define INSERTS_PER_STEP 100
  567. static int metadata_digest_transcribe_writeset(struct era_metadata *md,
  568. struct digest *d)
  569. {
  570. int r;
  571. bool marked;
  572. unsigned int b, e = min(d->current_bit + INSERTS_PER_STEP, d->nr_bits);
  573. for (b = d->current_bit; b < e; b++) {
  574. r = writeset_marked_on_disk(&d->info, &d->writeset, b, &marked);
  575. if (r) {
  576. DMERR("%s: writeset_marked_on_disk failed", __func__);
  577. return r;
  578. }
  579. if (!marked)
  580. continue;
  581. __dm_bless_for_disk(&d->value);
  582. r = dm_array_set_value(&md->era_array_info, md->era_array_root,
  583. b, &d->value, &md->era_array_root);
  584. if (r) {
  585. DMERR("%s: dm_array_set_value failed", __func__);
  586. return r;
  587. }
  588. }
  589. if (b == d->nr_bits)
  590. d->step = metadata_digest_remove_writeset;
  591. else
  592. d->current_bit = b;
  593. return 0;
  594. }
  595. static int metadata_digest_lookup_writeset(struct era_metadata *md,
  596. struct digest *d)
  597. {
  598. int r;
  599. uint64_t key;
  600. struct writeset_disk disk;
  601. r = dm_btree_find_lowest_key(&md->writeset_tree_info,
  602. md->writeset_tree_root, &key);
  603. if (r < 0)
  604. return r;
  605. d->era = key;
  606. r = dm_btree_lookup(&md->writeset_tree_info,
  607. md->writeset_tree_root, &key, &disk);
  608. if (r) {
  609. if (r == -ENODATA) {
  610. d->step = NULL;
  611. return 0;
  612. }
  613. DMERR("%s: dm_btree_lookup failed", __func__);
  614. return r;
  615. }
  616. ws_unpack(&disk, &d->writeset);
  617. d->value = cpu_to_le32(key);
  618. /*
  619. * We initialise another bitset info to avoid any caching side effects
  620. * with the previous one.
  621. */
  622. dm_disk_bitset_init(md->tm, &d->info);
  623. d->nr_bits = min(d->writeset.nr_bits, md->nr_blocks);
  624. d->current_bit = 0;
  625. d->step = metadata_digest_transcribe_writeset;
  626. return 0;
  627. }
  628. static int metadata_digest_start(struct era_metadata *md, struct digest *d)
  629. {
  630. if (d->step)
  631. return 0;
  632. memset(d, 0, sizeof(*d));
  633. d->step = metadata_digest_lookup_writeset;
  634. return 0;
  635. }
  636. /*----------------------------------------------------------------
  637. * High level metadata interface. Target methods should use these, and not
  638. * the lower level ones.
  639. *--------------------------------------------------------------*/
  640. static struct era_metadata *metadata_open(struct block_device *bdev,
  641. sector_t block_size,
  642. bool may_format)
  643. {
  644. int r;
  645. struct era_metadata *md = kzalloc(sizeof(*md), GFP_KERNEL);
  646. if (!md)
  647. return NULL;
  648. md->bdev = bdev;
  649. md->block_size = block_size;
  650. md->writesets[0].md.root = INVALID_WRITESET_ROOT;
  651. md->writesets[1].md.root = INVALID_WRITESET_ROOT;
  652. md->current_writeset = &md->writesets[0];
  653. r = create_persistent_data_objects(md, may_format);
  654. if (r) {
  655. kfree(md);
  656. return ERR_PTR(r);
  657. }
  658. return md;
  659. }
  660. static void metadata_close(struct era_metadata *md)
  661. {
  662. writeset_free(&md->writesets[0]);
  663. writeset_free(&md->writesets[1]);
  664. destroy_persistent_data_objects(md);
  665. kfree(md);
  666. }
  667. static bool valid_nr_blocks(dm_block_t n)
  668. {
  669. /*
  670. * dm_bitset restricts us to 2^32. test_bit & co. restrict us
  671. * further to 2^31 - 1
  672. */
  673. return n < (1ull << 31);
  674. }
  675. static int metadata_resize(struct era_metadata *md, void *arg)
  676. {
  677. int r;
  678. dm_block_t *new_size = arg;
  679. __le32 value;
  680. if (!valid_nr_blocks(*new_size)) {
  681. DMERR("Invalid number of origin blocks %llu",
  682. (unsigned long long) *new_size);
  683. return -EINVAL;
  684. }
  685. writeset_free(&md->writesets[0]);
  686. writeset_free(&md->writesets[1]);
  687. r = writeset_alloc(&md->writesets[0], *new_size);
  688. if (r) {
  689. DMERR("%s: writeset_alloc failed for writeset 0", __func__);
  690. return r;
  691. }
  692. r = writeset_alloc(&md->writesets[1], *new_size);
  693. if (r) {
  694. DMERR("%s: writeset_alloc failed for writeset 1", __func__);
  695. writeset_free(&md->writesets[0]);
  696. return r;
  697. }
  698. value = cpu_to_le32(0u);
  699. __dm_bless_for_disk(&value);
  700. r = dm_array_resize(&md->era_array_info, md->era_array_root,
  701. md->nr_blocks, *new_size,
  702. &value, &md->era_array_root);
  703. if (r) {
  704. DMERR("%s: dm_array_resize failed", __func__);
  705. writeset_free(&md->writesets[0]);
  706. writeset_free(&md->writesets[1]);
  707. return r;
  708. }
  709. md->nr_blocks = *new_size;
  710. return 0;
  711. }
  712. static int metadata_era_archive(struct era_metadata *md)
  713. {
  714. int r;
  715. uint64_t keys[1];
  716. struct writeset_disk value;
  717. r = dm_bitset_flush(&md->bitset_info, md->current_writeset->md.root,
  718. &md->current_writeset->md.root);
  719. if (r) {
  720. DMERR("%s: dm_bitset_flush failed", __func__);
  721. return r;
  722. }
  723. ws_pack(&md->current_writeset->md, &value);
  724. keys[0] = md->current_era;
  725. __dm_bless_for_disk(&value);
  726. r = dm_btree_insert(&md->writeset_tree_info, md->writeset_tree_root,
  727. keys, &value, &md->writeset_tree_root);
  728. if (r) {
  729. DMERR("%s: couldn't insert writeset into btree", __func__);
  730. /* FIXME: fail mode */
  731. return r;
  732. }
  733. md->current_writeset->md.root = INVALID_WRITESET_ROOT;
  734. md->archived_writesets = true;
  735. return 0;
  736. }
  737. static struct writeset *next_writeset(struct era_metadata *md)
  738. {
  739. return (md->current_writeset == &md->writesets[0]) ?
  740. &md->writesets[1] : &md->writesets[0];
  741. }
  742. static int metadata_new_era(struct era_metadata *md)
  743. {
  744. int r;
  745. struct writeset *new_writeset = next_writeset(md);
  746. r = writeset_init(&md->bitset_info, new_writeset, md->nr_blocks);
  747. if (r) {
  748. DMERR("%s: writeset_init failed", __func__);
  749. return r;
  750. }
  751. swap_writeset(md, new_writeset);
  752. md->current_era++;
  753. return 0;
  754. }
  755. static int metadata_era_rollover(struct era_metadata *md)
  756. {
  757. int r;
  758. if (md->current_writeset->md.root != INVALID_WRITESET_ROOT) {
  759. r = metadata_era_archive(md);
  760. if (r) {
  761. DMERR("%s: metadata_archive_era failed", __func__);
  762. /* FIXME: fail mode? */
  763. return r;
  764. }
  765. }
  766. r = metadata_new_era(md);
  767. if (r) {
  768. DMERR("%s: new era failed", __func__);
  769. /* FIXME: fail mode */
  770. return r;
  771. }
  772. return 0;
  773. }
  774. static bool metadata_current_marked(struct era_metadata *md, dm_block_t block)
  775. {
  776. bool r;
  777. struct writeset *ws;
  778. rcu_read_lock();
  779. ws = rcu_dereference(md->current_writeset);
  780. r = writeset_marked(ws, block);
  781. rcu_read_unlock();
  782. return r;
  783. }
  784. static int metadata_commit(struct era_metadata *md)
  785. {
  786. int r;
  787. struct dm_block *sblock;
  788. if (md->current_writeset->md.root != INVALID_WRITESET_ROOT) {
  789. r = dm_bitset_flush(&md->bitset_info, md->current_writeset->md.root,
  790. &md->current_writeset->md.root);
  791. if (r) {
  792. DMERR("%s: bitset flush failed", __func__);
  793. return r;
  794. }
  795. }
  796. r = dm_tm_pre_commit(md->tm);
  797. if (r) {
  798. DMERR("%s: pre commit failed", __func__);
  799. return r;
  800. }
  801. r = save_sm_root(md);
  802. if (r) {
  803. DMERR("%s: save_sm_root failed", __func__);
  804. return r;
  805. }
  806. r = superblock_lock(md, &sblock);
  807. if (r) {
  808. DMERR("%s: superblock lock failed", __func__);
  809. return r;
  810. }
  811. prepare_superblock(md, dm_block_data(sblock));
  812. return dm_tm_commit(md->tm, sblock);
  813. }
  814. static int metadata_checkpoint(struct era_metadata *md)
  815. {
  816. /*
  817. * For now we just rollover, but later I want to put a check in to
  818. * avoid this if the filter is still pretty fresh.
  819. */
  820. return metadata_era_rollover(md);
  821. }
  822. /*
  823. * Metadata snapshots allow userland to access era data.
  824. */
  825. static int metadata_take_snap(struct era_metadata *md)
  826. {
  827. int r, inc;
  828. struct dm_block *clone;
  829. if (md->metadata_snap != SUPERBLOCK_LOCATION) {
  830. DMERR("%s: metadata snapshot already exists", __func__);
  831. return -EINVAL;
  832. }
  833. r = metadata_era_rollover(md);
  834. if (r) {
  835. DMERR("%s: era rollover failed", __func__);
  836. return r;
  837. }
  838. r = metadata_commit(md);
  839. if (r) {
  840. DMERR("%s: pre commit failed", __func__);
  841. return r;
  842. }
  843. r = dm_sm_inc_block(md->sm, SUPERBLOCK_LOCATION);
  844. if (r) {
  845. DMERR("%s: couldn't increment superblock", __func__);
  846. return r;
  847. }
  848. r = dm_tm_shadow_block(md->tm, SUPERBLOCK_LOCATION,
  849. &sb_validator, &clone, &inc);
  850. if (r) {
  851. DMERR("%s: couldn't shadow superblock", __func__);
  852. dm_sm_dec_block(md->sm, SUPERBLOCK_LOCATION);
  853. return r;
  854. }
  855. BUG_ON(!inc);
  856. r = dm_sm_inc_block(md->sm, md->writeset_tree_root);
  857. if (r) {
  858. DMERR("%s: couldn't inc writeset tree root", __func__);
  859. dm_tm_unlock(md->tm, clone);
  860. return r;
  861. }
  862. r = dm_sm_inc_block(md->sm, md->era_array_root);
  863. if (r) {
  864. DMERR("%s: couldn't inc era tree root", __func__);
  865. dm_sm_dec_block(md->sm, md->writeset_tree_root);
  866. dm_tm_unlock(md->tm, clone);
  867. return r;
  868. }
  869. md->metadata_snap = dm_block_location(clone);
  870. dm_tm_unlock(md->tm, clone);
  871. return 0;
  872. }
  873. static int metadata_drop_snap(struct era_metadata *md)
  874. {
  875. int r;
  876. dm_block_t location;
  877. struct dm_block *clone;
  878. struct superblock_disk *disk;
  879. if (md->metadata_snap == SUPERBLOCK_LOCATION) {
  880. DMERR("%s: no snap to drop", __func__);
  881. return -EINVAL;
  882. }
  883. r = dm_tm_read_lock(md->tm, md->metadata_snap, &sb_validator, &clone);
  884. if (r) {
  885. DMERR("%s: couldn't read lock superblock clone", __func__);
  886. return r;
  887. }
  888. /*
  889. * Whatever happens now we'll commit with no record of the metadata
  890. * snap.
  891. */
  892. md->metadata_snap = SUPERBLOCK_LOCATION;
  893. disk = dm_block_data(clone);
  894. r = dm_btree_del(&md->writeset_tree_info,
  895. le64_to_cpu(disk->writeset_tree_root));
  896. if (r) {
  897. DMERR("%s: error deleting writeset tree clone", __func__);
  898. dm_tm_unlock(md->tm, clone);
  899. return r;
  900. }
  901. r = dm_array_del(&md->era_array_info, le64_to_cpu(disk->era_array_root));
  902. if (r) {
  903. DMERR("%s: error deleting era array clone", __func__);
  904. dm_tm_unlock(md->tm, clone);
  905. return r;
  906. }
  907. location = dm_block_location(clone);
  908. dm_tm_unlock(md->tm, clone);
  909. return dm_sm_dec_block(md->sm, location);
  910. }
  911. struct metadata_stats {
  912. dm_block_t used;
  913. dm_block_t total;
  914. dm_block_t snap;
  915. uint32_t era;
  916. };
  917. static int metadata_get_stats(struct era_metadata *md, void *ptr)
  918. {
  919. int r;
  920. struct metadata_stats *s = ptr;
  921. dm_block_t nr_free, nr_total;
  922. r = dm_sm_get_nr_free(md->sm, &nr_free);
  923. if (r) {
  924. DMERR("dm_sm_get_nr_free returned %d", r);
  925. return r;
  926. }
  927. r = dm_sm_get_nr_blocks(md->sm, &nr_total);
  928. if (r) {
  929. DMERR("dm_pool_get_metadata_dev_size returned %d", r);
  930. return r;
  931. }
  932. s->used = nr_total - nr_free;
  933. s->total = nr_total;
  934. s->snap = md->metadata_snap;
  935. s->era = md->current_era;
  936. return 0;
  937. }
  938. /*----------------------------------------------------------------*/
  939. struct era {
  940. struct dm_target *ti;
  941. struct dm_dev *metadata_dev;
  942. struct dm_dev *origin_dev;
  943. dm_block_t nr_blocks;
  944. uint32_t sectors_per_block;
  945. int sectors_per_block_shift;
  946. struct era_metadata *md;
  947. struct workqueue_struct *wq;
  948. struct work_struct worker;
  949. spinlock_t deferred_lock;
  950. struct bio_list deferred_bios;
  951. spinlock_t rpc_lock;
  952. struct list_head rpc_calls;
  953. struct digest digest;
  954. atomic_t suspended;
  955. };
  956. struct rpc {
  957. struct list_head list;
  958. int (*fn0)(struct era_metadata *);
  959. int (*fn1)(struct era_metadata *, void *);
  960. void *arg;
  961. int result;
  962. struct completion complete;
  963. };
  964. /*----------------------------------------------------------------
  965. * Remapping.
  966. *---------------------------------------------------------------*/
  967. static bool block_size_is_power_of_two(struct era *era)
  968. {
  969. return era->sectors_per_block_shift >= 0;
  970. }
  971. static dm_block_t get_block(struct era *era, struct bio *bio)
  972. {
  973. sector_t block_nr = bio->bi_iter.bi_sector;
  974. if (!block_size_is_power_of_two(era))
  975. (void) sector_div(block_nr, era->sectors_per_block);
  976. else
  977. block_nr >>= era->sectors_per_block_shift;
  978. return block_nr;
  979. }
  980. static void remap_to_origin(struct era *era, struct bio *bio)
  981. {
  982. bio_set_dev(bio, era->origin_dev->bdev);
  983. }
  984. /*----------------------------------------------------------------
  985. * Worker thread
  986. *--------------------------------------------------------------*/
  987. static void wake_worker(struct era *era)
  988. {
  989. if (!atomic_read(&era->suspended))
  990. queue_work(era->wq, &era->worker);
  991. }
  992. static void process_old_eras(struct era *era)
  993. {
  994. int r;
  995. if (!era->digest.step)
  996. return;
  997. r = era->digest.step(era->md, &era->digest);
  998. if (r < 0) {
  999. DMERR("%s: digest step failed, stopping digestion", __func__);
  1000. era->digest.step = NULL;
  1001. } else if (era->digest.step)
  1002. wake_worker(era);
  1003. }
  1004. static void process_deferred_bios(struct era *era)
  1005. {
  1006. int r;
  1007. struct bio_list deferred_bios, marked_bios;
  1008. struct bio *bio;
  1009. struct blk_plug plug;
  1010. bool commit_needed = false;
  1011. bool failed = false;
  1012. struct writeset *ws = era->md->current_writeset;
  1013. bio_list_init(&deferred_bios);
  1014. bio_list_init(&marked_bios);
  1015. spin_lock(&era->deferred_lock);
  1016. bio_list_merge(&deferred_bios, &era->deferred_bios);
  1017. bio_list_init(&era->deferred_bios);
  1018. spin_unlock(&era->deferred_lock);
  1019. if (bio_list_empty(&deferred_bios))
  1020. return;
  1021. while ((bio = bio_list_pop(&deferred_bios))) {
  1022. r = writeset_test_and_set(&era->md->bitset_info, ws,
  1023. get_block(era, bio));
  1024. if (r < 0) {
  1025. /*
  1026. * This is bad news, we need to rollback.
  1027. * FIXME: finish.
  1028. */
  1029. failed = true;
  1030. } else if (r == 0)
  1031. commit_needed = true;
  1032. bio_list_add(&marked_bios, bio);
  1033. }
  1034. if (commit_needed) {
  1035. r = metadata_commit(era->md);
  1036. if (r)
  1037. failed = true;
  1038. }
  1039. if (failed)
  1040. while ((bio = bio_list_pop(&marked_bios)))
  1041. bio_io_error(bio);
  1042. else {
  1043. blk_start_plug(&plug);
  1044. while ((bio = bio_list_pop(&marked_bios))) {
  1045. /*
  1046. * Only update the in-core writeset if the on-disk one
  1047. * was updated too.
  1048. */
  1049. if (commit_needed)
  1050. set_bit(get_block(era, bio), ws->bits);
  1051. submit_bio_noacct(bio);
  1052. }
  1053. blk_finish_plug(&plug);
  1054. }
  1055. }
  1056. static void process_rpc_calls(struct era *era)
  1057. {
  1058. int r;
  1059. bool need_commit = false;
  1060. struct list_head calls;
  1061. struct rpc *rpc, *tmp;
  1062. INIT_LIST_HEAD(&calls);
  1063. spin_lock(&era->rpc_lock);
  1064. list_splice_init(&era->rpc_calls, &calls);
  1065. spin_unlock(&era->rpc_lock);
  1066. list_for_each_entry_safe(rpc, tmp, &calls, list) {
  1067. rpc->result = rpc->fn0 ? rpc->fn0(era->md) : rpc->fn1(era->md, rpc->arg);
  1068. need_commit = true;
  1069. }
  1070. if (need_commit) {
  1071. r = metadata_commit(era->md);
  1072. if (r)
  1073. list_for_each_entry_safe(rpc, tmp, &calls, list)
  1074. rpc->result = r;
  1075. }
  1076. list_for_each_entry_safe(rpc, tmp, &calls, list)
  1077. complete(&rpc->complete);
  1078. }
  1079. static void kick_off_digest(struct era *era)
  1080. {
  1081. if (era->md->archived_writesets) {
  1082. era->md->archived_writesets = false;
  1083. metadata_digest_start(era->md, &era->digest);
  1084. }
  1085. }
  1086. static void do_work(struct work_struct *ws)
  1087. {
  1088. struct era *era = container_of(ws, struct era, worker);
  1089. kick_off_digest(era);
  1090. process_old_eras(era);
  1091. process_deferred_bios(era);
  1092. process_rpc_calls(era);
  1093. }
  1094. static void defer_bio(struct era *era, struct bio *bio)
  1095. {
  1096. spin_lock(&era->deferred_lock);
  1097. bio_list_add(&era->deferred_bios, bio);
  1098. spin_unlock(&era->deferred_lock);
  1099. wake_worker(era);
  1100. }
  1101. /*
  1102. * Make an rpc call to the worker to change the metadata.
  1103. */
  1104. static int perform_rpc(struct era *era, struct rpc *rpc)
  1105. {
  1106. rpc->result = 0;
  1107. init_completion(&rpc->complete);
  1108. spin_lock(&era->rpc_lock);
  1109. list_add(&rpc->list, &era->rpc_calls);
  1110. spin_unlock(&era->rpc_lock);
  1111. wake_worker(era);
  1112. wait_for_completion(&rpc->complete);
  1113. return rpc->result;
  1114. }
  1115. static int in_worker0(struct era *era, int (*fn)(struct era_metadata *))
  1116. {
  1117. struct rpc rpc;
  1118. rpc.fn0 = fn;
  1119. rpc.fn1 = NULL;
  1120. return perform_rpc(era, &rpc);
  1121. }
  1122. static int in_worker1(struct era *era,
  1123. int (*fn)(struct era_metadata *, void *), void *arg)
  1124. {
  1125. struct rpc rpc;
  1126. rpc.fn0 = NULL;
  1127. rpc.fn1 = fn;
  1128. rpc.arg = arg;
  1129. return perform_rpc(era, &rpc);
  1130. }
  1131. static void start_worker(struct era *era)
  1132. {
  1133. atomic_set(&era->suspended, 0);
  1134. }
  1135. static void stop_worker(struct era *era)
  1136. {
  1137. atomic_set(&era->suspended, 1);
  1138. drain_workqueue(era->wq);
  1139. }
  1140. /*----------------------------------------------------------------
  1141. * Target methods
  1142. *--------------------------------------------------------------*/
  1143. static void era_destroy(struct era *era)
  1144. {
  1145. if (era->md)
  1146. metadata_close(era->md);
  1147. if (era->wq)
  1148. destroy_workqueue(era->wq);
  1149. if (era->origin_dev)
  1150. dm_put_device(era->ti, era->origin_dev);
  1151. if (era->metadata_dev)
  1152. dm_put_device(era->ti, era->metadata_dev);
  1153. kfree(era);
  1154. }
  1155. static dm_block_t calc_nr_blocks(struct era *era)
  1156. {
  1157. return dm_sector_div_up(era->ti->len, era->sectors_per_block);
  1158. }
  1159. static bool valid_block_size(dm_block_t block_size)
  1160. {
  1161. bool greater_than_zero = block_size > 0;
  1162. bool multiple_of_min_block_size = (block_size & (MIN_BLOCK_SIZE - 1)) == 0;
  1163. return greater_than_zero && multiple_of_min_block_size;
  1164. }
  1165. /*
  1166. * <metadata dev> <data dev> <data block size (sectors)>
  1167. */
  1168. static int era_ctr(struct dm_target *ti, unsigned int argc, char **argv)
  1169. {
  1170. int r;
  1171. char dummy;
  1172. struct era *era;
  1173. struct era_metadata *md;
  1174. if (argc != 3) {
  1175. ti->error = "Invalid argument count";
  1176. return -EINVAL;
  1177. }
  1178. era = kzalloc(sizeof(*era), GFP_KERNEL);
  1179. if (!era) {
  1180. ti->error = "Error allocating era structure";
  1181. return -ENOMEM;
  1182. }
  1183. era->ti = ti;
  1184. r = dm_get_device(ti, argv[0], FMODE_READ | FMODE_WRITE, &era->metadata_dev);
  1185. if (r) {
  1186. ti->error = "Error opening metadata device";
  1187. era_destroy(era);
  1188. return -EINVAL;
  1189. }
  1190. r = dm_get_device(ti, argv[1], FMODE_READ | FMODE_WRITE, &era->origin_dev);
  1191. if (r) {
  1192. ti->error = "Error opening data device";
  1193. era_destroy(era);
  1194. return -EINVAL;
  1195. }
  1196. r = sscanf(argv[2], "%u%c", &era->sectors_per_block, &dummy);
  1197. if (r != 1) {
  1198. ti->error = "Error parsing block size";
  1199. era_destroy(era);
  1200. return -EINVAL;
  1201. }
  1202. r = dm_set_target_max_io_len(ti, era->sectors_per_block);
  1203. if (r) {
  1204. ti->error = "could not set max io len";
  1205. era_destroy(era);
  1206. return -EINVAL;
  1207. }
  1208. if (!valid_block_size(era->sectors_per_block)) {
  1209. ti->error = "Invalid block size";
  1210. era_destroy(era);
  1211. return -EINVAL;
  1212. }
  1213. if (era->sectors_per_block & (era->sectors_per_block - 1))
  1214. era->sectors_per_block_shift = -1;
  1215. else
  1216. era->sectors_per_block_shift = __ffs(era->sectors_per_block);
  1217. md = metadata_open(era->metadata_dev->bdev, era->sectors_per_block, true);
  1218. if (IS_ERR(md)) {
  1219. ti->error = "Error reading metadata";
  1220. era_destroy(era);
  1221. return PTR_ERR(md);
  1222. }
  1223. era->md = md;
  1224. era->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
  1225. if (!era->wq) {
  1226. ti->error = "could not create workqueue for metadata object";
  1227. era_destroy(era);
  1228. return -ENOMEM;
  1229. }
  1230. INIT_WORK(&era->worker, do_work);
  1231. spin_lock_init(&era->deferred_lock);
  1232. bio_list_init(&era->deferred_bios);
  1233. spin_lock_init(&era->rpc_lock);
  1234. INIT_LIST_HEAD(&era->rpc_calls);
  1235. ti->private = era;
  1236. ti->num_flush_bios = 1;
  1237. ti->flush_supported = true;
  1238. ti->num_discard_bios = 1;
  1239. return 0;
  1240. }
  1241. static void era_dtr(struct dm_target *ti)
  1242. {
  1243. era_destroy(ti->private);
  1244. }
  1245. static int era_map(struct dm_target *ti, struct bio *bio)
  1246. {
  1247. struct era *era = ti->private;
  1248. dm_block_t block = get_block(era, bio);
  1249. /*
  1250. * All bios get remapped to the origin device. We do this now, but
  1251. * it may not get issued until later. Depending on whether the
  1252. * block is marked in this era.
  1253. */
  1254. remap_to_origin(era, bio);
  1255. /*
  1256. * REQ_PREFLUSH bios carry no data, so we're not interested in them.
  1257. */
  1258. if (!(bio->bi_opf & REQ_PREFLUSH) &&
  1259. (bio_data_dir(bio) == WRITE) &&
  1260. !metadata_current_marked(era->md, block)) {
  1261. defer_bio(era, bio);
  1262. return DM_MAPIO_SUBMITTED;
  1263. }
  1264. return DM_MAPIO_REMAPPED;
  1265. }
  1266. static void era_postsuspend(struct dm_target *ti)
  1267. {
  1268. int r;
  1269. struct era *era = ti->private;
  1270. r = in_worker0(era, metadata_era_archive);
  1271. if (r) {
  1272. DMERR("%s: couldn't archive current era", __func__);
  1273. /* FIXME: fail mode */
  1274. }
  1275. stop_worker(era);
  1276. r = metadata_commit(era->md);
  1277. if (r) {
  1278. DMERR("%s: metadata_commit failed", __func__);
  1279. /* FIXME: fail mode */
  1280. }
  1281. }
  1282. static int era_preresume(struct dm_target *ti)
  1283. {
  1284. int r;
  1285. struct era *era = ti->private;
  1286. dm_block_t new_size = calc_nr_blocks(era);
  1287. if (era->nr_blocks != new_size) {
  1288. r = metadata_resize(era->md, &new_size);
  1289. if (r) {
  1290. DMERR("%s: metadata_resize failed", __func__);
  1291. return r;
  1292. }
  1293. r = metadata_commit(era->md);
  1294. if (r) {
  1295. DMERR("%s: metadata_commit failed", __func__);
  1296. return r;
  1297. }
  1298. era->nr_blocks = new_size;
  1299. }
  1300. start_worker(era);
  1301. r = in_worker0(era, metadata_era_rollover);
  1302. if (r) {
  1303. DMERR("%s: metadata_era_rollover failed", __func__);
  1304. return r;
  1305. }
  1306. return 0;
  1307. }
  1308. /*
  1309. * Status format:
  1310. *
  1311. * <metadata block size> <#used metadata blocks>/<#total metadata blocks>
  1312. * <current era> <held metadata root | '-'>
  1313. */
  1314. static void era_status(struct dm_target *ti, status_type_t type,
  1315. unsigned int status_flags, char *result, unsigned int maxlen)
  1316. {
  1317. int r;
  1318. struct era *era = ti->private;
  1319. ssize_t sz = 0;
  1320. struct metadata_stats stats;
  1321. char buf[BDEVNAME_SIZE];
  1322. switch (type) {
  1323. case STATUSTYPE_INFO:
  1324. r = in_worker1(era, metadata_get_stats, &stats);
  1325. if (r)
  1326. goto err;
  1327. DMEMIT("%u %llu/%llu %u",
  1328. (unsigned int) (DM_ERA_METADATA_BLOCK_SIZE >> SECTOR_SHIFT),
  1329. (unsigned long long) stats.used,
  1330. (unsigned long long) stats.total,
  1331. (unsigned int) stats.era);
  1332. if (stats.snap != SUPERBLOCK_LOCATION)
  1333. DMEMIT(" %llu", stats.snap);
  1334. else
  1335. DMEMIT(" -");
  1336. break;
  1337. case STATUSTYPE_TABLE:
  1338. format_dev_t(buf, era->metadata_dev->bdev->bd_dev);
  1339. DMEMIT("%s ", buf);
  1340. format_dev_t(buf, era->origin_dev->bdev->bd_dev);
  1341. DMEMIT("%s %u", buf, era->sectors_per_block);
  1342. break;
  1343. case STATUSTYPE_IMA:
  1344. *result = '\0';
  1345. break;
  1346. }
  1347. return;
  1348. err:
  1349. DMEMIT("Error");
  1350. }
  1351. static int era_message(struct dm_target *ti, unsigned int argc, char **argv,
  1352. char *result, unsigned int maxlen)
  1353. {
  1354. struct era *era = ti->private;
  1355. if (argc != 1) {
  1356. DMERR("incorrect number of message arguments");
  1357. return -EINVAL;
  1358. }
  1359. if (!strcasecmp(argv[0], "checkpoint"))
  1360. return in_worker0(era, metadata_checkpoint);
  1361. if (!strcasecmp(argv[0], "take_metadata_snap"))
  1362. return in_worker0(era, metadata_take_snap);
  1363. if (!strcasecmp(argv[0], "drop_metadata_snap"))
  1364. return in_worker0(era, metadata_drop_snap);
  1365. DMERR("unsupported message '%s'", argv[0]);
  1366. return -EINVAL;
  1367. }
  1368. static sector_t get_dev_size(struct dm_dev *dev)
  1369. {
  1370. return bdev_nr_sectors(dev->bdev);
  1371. }
  1372. static int era_iterate_devices(struct dm_target *ti,
  1373. iterate_devices_callout_fn fn, void *data)
  1374. {
  1375. struct era *era = ti->private;
  1376. return fn(ti, era->origin_dev, 0, get_dev_size(era->origin_dev), data);
  1377. }
  1378. static void era_io_hints(struct dm_target *ti, struct queue_limits *limits)
  1379. {
  1380. struct era *era = ti->private;
  1381. uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
  1382. /*
  1383. * If the system-determined stacked limits are compatible with the
  1384. * era device's blocksize (io_opt is a factor) do not override them.
  1385. */
  1386. if (io_opt_sectors < era->sectors_per_block ||
  1387. do_div(io_opt_sectors, era->sectors_per_block)) {
  1388. blk_limits_io_min(limits, 0);
  1389. blk_limits_io_opt(limits, era->sectors_per_block << SECTOR_SHIFT);
  1390. }
  1391. }
  1392. /*----------------------------------------------------------------*/
  1393. static struct target_type era_target = {
  1394. .name = "era",
  1395. .version = {1, 0, 0},
  1396. .module = THIS_MODULE,
  1397. .ctr = era_ctr,
  1398. .dtr = era_dtr,
  1399. .map = era_map,
  1400. .postsuspend = era_postsuspend,
  1401. .preresume = era_preresume,
  1402. .status = era_status,
  1403. .message = era_message,
  1404. .iterate_devices = era_iterate_devices,
  1405. .io_hints = era_io_hints
  1406. };
  1407. static int __init dm_era_init(void)
  1408. {
  1409. int r;
  1410. r = dm_register_target(&era_target);
  1411. if (r) {
  1412. DMERR("era target registration failed: %d", r);
  1413. return r;
  1414. }
  1415. return 0;
  1416. }
  1417. static void __exit dm_era_exit(void)
  1418. {
  1419. dm_unregister_target(&era_target);
  1420. }
  1421. module_init(dm_era_init);
  1422. module_exit(dm_era_exit);
  1423. MODULE_DESCRIPTION(DM_NAME " era target");
  1424. MODULE_AUTHOR("Joe Thornber <[email protected]>");
  1425. MODULE_LICENSE("GPL");