dm-snap.c 68 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866
  1. /*
  2. * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
  3. *
  4. * This file is released under the GPL.
  5. */
  6. #include <linux/blkdev.h>
  7. #include <linux/device-mapper.h>
  8. #include <linux/delay.h>
  9. #include <linux/fs.h>
  10. #include <linux/init.h>
  11. #include <linux/kdev_t.h>
  12. #include <linux/list.h>
  13. #include <linux/list_bl.h>
  14. #include <linux/mempool.h>
  15. #include <linux/module.h>
  16. #include <linux/slab.h>
  17. #include <linux/vmalloc.h>
  18. #include <linux/log2.h>
  19. #include <linux/dm-kcopyd.h>
  20. #include "dm.h"
  21. #include "dm-exception-store.h"
  22. #define DM_MSG_PREFIX "snapshots"
  23. static const char dm_snapshot_merge_target_name[] = "snapshot-merge";
  24. #define dm_target_is_snapshot_merge(ti) \
  25. ((ti)->type->name == dm_snapshot_merge_target_name)
  26. /*
  27. * The size of the mempool used to track chunks in use.
  28. */
  29. #define MIN_IOS 256
  30. #define DM_TRACKED_CHUNK_HASH_SIZE 16
  31. #define DM_TRACKED_CHUNK_HASH(x) ((unsigned long)(x) & \
  32. (DM_TRACKED_CHUNK_HASH_SIZE - 1))
  33. struct dm_exception_table {
  34. uint32_t hash_mask;
  35. unsigned int hash_shift;
  36. struct hlist_bl_head *table;
  37. };
  38. struct dm_snapshot {
  39. struct rw_semaphore lock;
  40. struct dm_dev *origin;
  41. struct dm_dev *cow;
  42. struct dm_target *ti;
  43. /* List of snapshots per Origin */
  44. struct list_head list;
  45. /*
  46. * You can't use a snapshot if this is 0 (e.g. if full).
  47. * A snapshot-merge target never clears this.
  48. */
  49. int valid;
  50. /*
  51. * The snapshot overflowed because of a write to the snapshot device.
  52. * We don't have to invalidate the snapshot in this case, but we need
  53. * to prevent further writes.
  54. */
  55. int snapshot_overflowed;
  56. /* Origin writes don't trigger exceptions until this is set */
  57. int active;
  58. atomic_t pending_exceptions_count;
  59. spinlock_t pe_allocation_lock;
  60. /* Protected by "pe_allocation_lock" */
  61. sector_t exception_start_sequence;
  62. /* Protected by kcopyd single-threaded callback */
  63. sector_t exception_complete_sequence;
  64. /*
  65. * A list of pending exceptions that completed out of order.
  66. * Protected by kcopyd single-threaded callback.
  67. */
  68. struct rb_root out_of_order_tree;
  69. mempool_t pending_pool;
  70. struct dm_exception_table pending;
  71. struct dm_exception_table complete;
  72. /*
  73. * pe_lock protects all pending_exception operations and access
  74. * as well as the snapshot_bios list.
  75. */
  76. spinlock_t pe_lock;
  77. /* Chunks with outstanding reads */
  78. spinlock_t tracked_chunk_lock;
  79. struct hlist_head tracked_chunk_hash[DM_TRACKED_CHUNK_HASH_SIZE];
  80. /* The on disk metadata handler */
  81. struct dm_exception_store *store;
  82. unsigned int in_progress;
  83. struct wait_queue_head in_progress_wait;
  84. struct dm_kcopyd_client *kcopyd_client;
  85. /* Wait for events based on state_bits */
  86. unsigned long state_bits;
  87. /* Range of chunks currently being merged. */
  88. chunk_t first_merging_chunk;
  89. int num_merging_chunks;
  90. /*
  91. * The merge operation failed if this flag is set.
  92. * Failure modes are handled as follows:
  93. * - I/O error reading the header
  94. * => don't load the target; abort.
  95. * - Header does not have "valid" flag set
  96. * => use the origin; forget about the snapshot.
  97. * - I/O error when reading exceptions
  98. * => don't load the target; abort.
  99. * (We can't use the intermediate origin state.)
  100. * - I/O error while merging
  101. * => stop merging; set merge_failed; process I/O normally.
  102. */
  103. bool merge_failed:1;
  104. bool discard_zeroes_cow:1;
  105. bool discard_passdown_origin:1;
  106. /*
  107. * Incoming bios that overlap with chunks being merged must wait
  108. * for them to be committed.
  109. */
  110. struct bio_list bios_queued_during_merge;
  111. };
  112. /*
  113. * state_bits:
  114. * RUNNING_MERGE - Merge operation is in progress.
  115. * SHUTDOWN_MERGE - Set to signal that merge needs to be stopped;
  116. * cleared afterwards.
  117. */
  118. #define RUNNING_MERGE 0
  119. #define SHUTDOWN_MERGE 1
  120. /*
  121. * Maximum number of chunks being copied on write.
  122. *
  123. * The value was decided experimentally as a trade-off between memory
  124. * consumption, stalling the kernel's workqueues and maintaining a high enough
  125. * throughput.
  126. */
  127. #define DEFAULT_COW_THRESHOLD 2048
  128. static unsigned int cow_threshold = DEFAULT_COW_THRESHOLD;
  129. module_param_named(snapshot_cow_threshold, cow_threshold, uint, 0644);
  130. MODULE_PARM_DESC(snapshot_cow_threshold, "Maximum number of chunks being copied on write");
  131. DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle,
  132. "A percentage of time allocated for copy on write");
  133. struct dm_dev *dm_snap_origin(struct dm_snapshot *s)
  134. {
  135. return s->origin;
  136. }
  137. EXPORT_SYMBOL(dm_snap_origin);
  138. struct dm_dev *dm_snap_cow(struct dm_snapshot *s)
  139. {
  140. return s->cow;
  141. }
  142. EXPORT_SYMBOL(dm_snap_cow);
  143. static sector_t chunk_to_sector(struct dm_exception_store *store,
  144. chunk_t chunk)
  145. {
  146. return chunk << store->chunk_shift;
  147. }
  148. static int bdev_equal(struct block_device *lhs, struct block_device *rhs)
  149. {
  150. /*
  151. * There is only ever one instance of a particular block
  152. * device so we can compare pointers safely.
  153. */
  154. return lhs == rhs;
  155. }
  156. struct dm_snap_pending_exception {
  157. struct dm_exception e;
  158. /*
  159. * Origin buffers waiting for this to complete are held
  160. * in a bio list
  161. */
  162. struct bio_list origin_bios;
  163. struct bio_list snapshot_bios;
  164. /* Pointer back to snapshot context */
  165. struct dm_snapshot *snap;
  166. /*
  167. * 1 indicates the exception has already been sent to
  168. * kcopyd.
  169. */
  170. int started;
  171. /* There was copying error. */
  172. int copy_error;
  173. /* A sequence number, it is used for in-order completion. */
  174. sector_t exception_sequence;
  175. struct rb_node out_of_order_node;
  176. /*
  177. * For writing a complete chunk, bypassing the copy.
  178. */
  179. struct bio *full_bio;
  180. bio_end_io_t *full_bio_end_io;
  181. };
  182. /*
  183. * Hash table mapping origin volumes to lists of snapshots and
  184. * a lock to protect it
  185. */
  186. static struct kmem_cache *exception_cache;
  187. static struct kmem_cache *pending_cache;
  188. struct dm_snap_tracked_chunk {
  189. struct hlist_node node;
  190. chunk_t chunk;
  191. };
  192. static void init_tracked_chunk(struct bio *bio)
  193. {
  194. struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
  195. INIT_HLIST_NODE(&c->node);
  196. }
  197. static bool is_bio_tracked(struct bio *bio)
  198. {
  199. struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
  200. return !hlist_unhashed(&c->node);
  201. }
  202. static void track_chunk(struct dm_snapshot *s, struct bio *bio, chunk_t chunk)
  203. {
  204. struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
  205. c->chunk = chunk;
  206. spin_lock_irq(&s->tracked_chunk_lock);
  207. hlist_add_head(&c->node,
  208. &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)]);
  209. spin_unlock_irq(&s->tracked_chunk_lock);
  210. }
  211. static void stop_tracking_chunk(struct dm_snapshot *s, struct bio *bio)
  212. {
  213. struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
  214. unsigned long flags;
  215. spin_lock_irqsave(&s->tracked_chunk_lock, flags);
  216. hlist_del(&c->node);
  217. spin_unlock_irqrestore(&s->tracked_chunk_lock, flags);
  218. }
  219. static int __chunk_is_tracked(struct dm_snapshot *s, chunk_t chunk)
  220. {
  221. struct dm_snap_tracked_chunk *c;
  222. int found = 0;
  223. spin_lock_irq(&s->tracked_chunk_lock);
  224. hlist_for_each_entry(c,
  225. &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)], node) {
  226. if (c->chunk == chunk) {
  227. found = 1;
  228. break;
  229. }
  230. }
  231. spin_unlock_irq(&s->tracked_chunk_lock);
  232. return found;
  233. }
  234. /*
  235. * This conflicting I/O is extremely improbable in the caller,
  236. * so msleep(1) is sufficient and there is no need for a wait queue.
  237. */
  238. static void __check_for_conflicting_io(struct dm_snapshot *s, chunk_t chunk)
  239. {
  240. while (__chunk_is_tracked(s, chunk))
  241. msleep(1);
  242. }
  243. /*
  244. * One of these per registered origin, held in the snapshot_origins hash
  245. */
  246. struct origin {
  247. /* The origin device */
  248. struct block_device *bdev;
  249. struct list_head hash_list;
  250. /* List of snapshots for this origin */
  251. struct list_head snapshots;
  252. };
  253. /*
  254. * This structure is allocated for each origin target
  255. */
  256. struct dm_origin {
  257. struct dm_dev *dev;
  258. struct dm_target *ti;
  259. unsigned int split_boundary;
  260. struct list_head hash_list;
  261. };
  262. /*
  263. * Size of the hash table for origin volumes. If we make this
  264. * the size of the minors list then it should be nearly perfect
  265. */
  266. #define ORIGIN_HASH_SIZE 256
  267. #define ORIGIN_MASK 0xFF
  268. static struct list_head *_origins;
  269. static struct list_head *_dm_origins;
  270. static struct rw_semaphore _origins_lock;
  271. static DECLARE_WAIT_QUEUE_HEAD(_pending_exceptions_done);
  272. static DEFINE_SPINLOCK(_pending_exceptions_done_spinlock);
  273. static uint64_t _pending_exceptions_done_count;
  274. static int init_origin_hash(void)
  275. {
  276. int i;
  277. _origins = kmalloc_array(ORIGIN_HASH_SIZE, sizeof(struct list_head),
  278. GFP_KERNEL);
  279. if (!_origins) {
  280. DMERR("unable to allocate memory for _origins");
  281. return -ENOMEM;
  282. }
  283. for (i = 0; i < ORIGIN_HASH_SIZE; i++)
  284. INIT_LIST_HEAD(_origins + i);
  285. _dm_origins = kmalloc_array(ORIGIN_HASH_SIZE,
  286. sizeof(struct list_head),
  287. GFP_KERNEL);
  288. if (!_dm_origins) {
  289. DMERR("unable to allocate memory for _dm_origins");
  290. kfree(_origins);
  291. return -ENOMEM;
  292. }
  293. for (i = 0; i < ORIGIN_HASH_SIZE; i++)
  294. INIT_LIST_HEAD(_dm_origins + i);
  295. init_rwsem(&_origins_lock);
  296. return 0;
  297. }
  298. static void exit_origin_hash(void)
  299. {
  300. kfree(_origins);
  301. kfree(_dm_origins);
  302. }
  303. static unsigned int origin_hash(struct block_device *bdev)
  304. {
  305. return bdev->bd_dev & ORIGIN_MASK;
  306. }
  307. static struct origin *__lookup_origin(struct block_device *origin)
  308. {
  309. struct list_head *ol;
  310. struct origin *o;
  311. ol = &_origins[origin_hash(origin)];
  312. list_for_each_entry(o, ol, hash_list)
  313. if (bdev_equal(o->bdev, origin))
  314. return o;
  315. return NULL;
  316. }
  317. static void __insert_origin(struct origin *o)
  318. {
  319. struct list_head *sl = &_origins[origin_hash(o->bdev)];
  320. list_add_tail(&o->hash_list, sl);
  321. }
  322. static struct dm_origin *__lookup_dm_origin(struct block_device *origin)
  323. {
  324. struct list_head *ol;
  325. struct dm_origin *o;
  326. ol = &_dm_origins[origin_hash(origin)];
  327. list_for_each_entry(o, ol, hash_list)
  328. if (bdev_equal(o->dev->bdev, origin))
  329. return o;
  330. return NULL;
  331. }
  332. static void __insert_dm_origin(struct dm_origin *o)
  333. {
  334. struct list_head *sl = &_dm_origins[origin_hash(o->dev->bdev)];
  335. list_add_tail(&o->hash_list, sl);
  336. }
  337. static void __remove_dm_origin(struct dm_origin *o)
  338. {
  339. list_del(&o->hash_list);
  340. }
  341. /*
  342. * _origins_lock must be held when calling this function.
  343. * Returns number of snapshots registered using the supplied cow device, plus:
  344. * snap_src - a snapshot suitable for use as a source of exception handover
  345. * snap_dest - a snapshot capable of receiving exception handover.
  346. * snap_merge - an existing snapshot-merge target linked to the same origin.
  347. * There can be at most one snapshot-merge target. The parameter is optional.
  348. *
  349. * Possible return values and states of snap_src and snap_dest.
  350. * 0: NULL, NULL - first new snapshot
  351. * 1: snap_src, NULL - normal snapshot
  352. * 2: snap_src, snap_dest - waiting for handover
  353. * 2: snap_src, NULL - handed over, waiting for old to be deleted
  354. * 1: NULL, snap_dest - source got destroyed without handover
  355. */
  356. static int __find_snapshots_sharing_cow(struct dm_snapshot *snap,
  357. struct dm_snapshot **snap_src,
  358. struct dm_snapshot **snap_dest,
  359. struct dm_snapshot **snap_merge)
  360. {
  361. struct dm_snapshot *s;
  362. struct origin *o;
  363. int count = 0;
  364. int active;
  365. o = __lookup_origin(snap->origin->bdev);
  366. if (!o)
  367. goto out;
  368. list_for_each_entry(s, &o->snapshots, list) {
  369. if (dm_target_is_snapshot_merge(s->ti) && snap_merge)
  370. *snap_merge = s;
  371. if (!bdev_equal(s->cow->bdev, snap->cow->bdev))
  372. continue;
  373. down_read(&s->lock);
  374. active = s->active;
  375. up_read(&s->lock);
  376. if (active) {
  377. if (snap_src)
  378. *snap_src = s;
  379. } else if (snap_dest)
  380. *snap_dest = s;
  381. count++;
  382. }
  383. out:
  384. return count;
  385. }
  386. /*
  387. * On success, returns 1 if this snapshot is a handover destination,
  388. * otherwise returns 0.
  389. */
  390. static int __validate_exception_handover(struct dm_snapshot *snap)
  391. {
  392. struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
  393. struct dm_snapshot *snap_merge = NULL;
  394. /* Does snapshot need exceptions handed over to it? */
  395. if ((__find_snapshots_sharing_cow(snap, &snap_src, &snap_dest,
  396. &snap_merge) == 2) ||
  397. snap_dest) {
  398. snap->ti->error = "Snapshot cow pairing for exception table handover failed";
  399. return -EINVAL;
  400. }
  401. /*
  402. * If no snap_src was found, snap cannot become a handover
  403. * destination.
  404. */
  405. if (!snap_src)
  406. return 0;
  407. /*
  408. * Non-snapshot-merge handover?
  409. */
  410. if (!dm_target_is_snapshot_merge(snap->ti))
  411. return 1;
  412. /*
  413. * Do not allow more than one merging snapshot.
  414. */
  415. if (snap_merge) {
  416. snap->ti->error = "A snapshot is already merging.";
  417. return -EINVAL;
  418. }
  419. if (!snap_src->store->type->prepare_merge ||
  420. !snap_src->store->type->commit_merge) {
  421. snap->ti->error = "Snapshot exception store does not support snapshot-merge.";
  422. return -EINVAL;
  423. }
  424. return 1;
  425. }
  426. static void __insert_snapshot(struct origin *o, struct dm_snapshot *s)
  427. {
  428. struct dm_snapshot *l;
  429. /* Sort the list according to chunk size, largest-first smallest-last */
  430. list_for_each_entry(l, &o->snapshots, list)
  431. if (l->store->chunk_size < s->store->chunk_size)
  432. break;
  433. list_add_tail(&s->list, &l->list);
  434. }
  435. /*
  436. * Make a note of the snapshot and its origin so we can look it
  437. * up when the origin has a write on it.
  438. *
  439. * Also validate snapshot exception store handovers.
  440. * On success, returns 1 if this registration is a handover destination,
  441. * otherwise returns 0.
  442. */
  443. static int register_snapshot(struct dm_snapshot *snap)
  444. {
  445. struct origin *o, *new_o = NULL;
  446. struct block_device *bdev = snap->origin->bdev;
  447. int r = 0;
  448. new_o = kmalloc(sizeof(*new_o), GFP_KERNEL);
  449. if (!new_o)
  450. return -ENOMEM;
  451. down_write(&_origins_lock);
  452. r = __validate_exception_handover(snap);
  453. if (r < 0) {
  454. kfree(new_o);
  455. goto out;
  456. }
  457. o = __lookup_origin(bdev);
  458. if (o)
  459. kfree(new_o);
  460. else {
  461. /* New origin */
  462. o = new_o;
  463. /* Initialise the struct */
  464. INIT_LIST_HEAD(&o->snapshots);
  465. o->bdev = bdev;
  466. __insert_origin(o);
  467. }
  468. __insert_snapshot(o, snap);
  469. out:
  470. up_write(&_origins_lock);
  471. return r;
  472. }
  473. /*
  474. * Move snapshot to correct place in list according to chunk size.
  475. */
  476. static void reregister_snapshot(struct dm_snapshot *s)
  477. {
  478. struct block_device *bdev = s->origin->bdev;
  479. down_write(&_origins_lock);
  480. list_del(&s->list);
  481. __insert_snapshot(__lookup_origin(bdev), s);
  482. up_write(&_origins_lock);
  483. }
  484. static void unregister_snapshot(struct dm_snapshot *s)
  485. {
  486. struct origin *o;
  487. down_write(&_origins_lock);
  488. o = __lookup_origin(s->origin->bdev);
  489. list_del(&s->list);
  490. if (o && list_empty(&o->snapshots)) {
  491. list_del(&o->hash_list);
  492. kfree(o);
  493. }
  494. up_write(&_origins_lock);
  495. }
  496. /*
  497. * Implementation of the exception hash tables.
  498. * The lowest hash_shift bits of the chunk number are ignored, allowing
  499. * some consecutive chunks to be grouped together.
  500. */
  501. static uint32_t exception_hash(struct dm_exception_table *et, chunk_t chunk);
  502. /* Lock to protect access to the completed and pending exception hash tables. */
  503. struct dm_exception_table_lock {
  504. struct hlist_bl_head *complete_slot;
  505. struct hlist_bl_head *pending_slot;
  506. };
  507. static void dm_exception_table_lock_init(struct dm_snapshot *s, chunk_t chunk,
  508. struct dm_exception_table_lock *lock)
  509. {
  510. struct dm_exception_table *complete = &s->complete;
  511. struct dm_exception_table *pending = &s->pending;
  512. lock->complete_slot = &complete->table[exception_hash(complete, chunk)];
  513. lock->pending_slot = &pending->table[exception_hash(pending, chunk)];
  514. }
  515. static void dm_exception_table_lock(struct dm_exception_table_lock *lock)
  516. {
  517. hlist_bl_lock(lock->complete_slot);
  518. hlist_bl_lock(lock->pending_slot);
  519. }
  520. static void dm_exception_table_unlock(struct dm_exception_table_lock *lock)
  521. {
  522. hlist_bl_unlock(lock->pending_slot);
  523. hlist_bl_unlock(lock->complete_slot);
  524. }
  525. static int dm_exception_table_init(struct dm_exception_table *et,
  526. uint32_t size, unsigned int hash_shift)
  527. {
  528. unsigned int i;
  529. et->hash_shift = hash_shift;
  530. et->hash_mask = size - 1;
  531. et->table = kvmalloc_array(size, sizeof(struct hlist_bl_head),
  532. GFP_KERNEL);
  533. if (!et->table)
  534. return -ENOMEM;
  535. for (i = 0; i < size; i++)
  536. INIT_HLIST_BL_HEAD(et->table + i);
  537. return 0;
  538. }
  539. static void dm_exception_table_exit(struct dm_exception_table *et,
  540. struct kmem_cache *mem)
  541. {
  542. struct hlist_bl_head *slot;
  543. struct dm_exception *ex;
  544. struct hlist_bl_node *pos, *n;
  545. int i, size;
  546. size = et->hash_mask + 1;
  547. for (i = 0; i < size; i++) {
  548. slot = et->table + i;
  549. hlist_bl_for_each_entry_safe(ex, pos, n, slot, hash_list)
  550. kmem_cache_free(mem, ex);
  551. }
  552. kvfree(et->table);
  553. }
  554. static uint32_t exception_hash(struct dm_exception_table *et, chunk_t chunk)
  555. {
  556. return (chunk >> et->hash_shift) & et->hash_mask;
  557. }
  558. static void dm_remove_exception(struct dm_exception *e)
  559. {
  560. hlist_bl_del(&e->hash_list);
  561. }
  562. /*
  563. * Return the exception data for a sector, or NULL if not
  564. * remapped.
  565. */
  566. static struct dm_exception *dm_lookup_exception(struct dm_exception_table *et,
  567. chunk_t chunk)
  568. {
  569. struct hlist_bl_head *slot;
  570. struct hlist_bl_node *pos;
  571. struct dm_exception *e;
  572. slot = &et->table[exception_hash(et, chunk)];
  573. hlist_bl_for_each_entry(e, pos, slot, hash_list)
  574. if (chunk >= e->old_chunk &&
  575. chunk <= e->old_chunk + dm_consecutive_chunk_count(e))
  576. return e;
  577. return NULL;
  578. }
  579. static struct dm_exception *alloc_completed_exception(gfp_t gfp)
  580. {
  581. struct dm_exception *e;
  582. e = kmem_cache_alloc(exception_cache, gfp);
  583. if (!e && gfp == GFP_NOIO)
  584. e = kmem_cache_alloc(exception_cache, GFP_ATOMIC);
  585. return e;
  586. }
  587. static void free_completed_exception(struct dm_exception *e)
  588. {
  589. kmem_cache_free(exception_cache, e);
  590. }
  591. static struct dm_snap_pending_exception *alloc_pending_exception(struct dm_snapshot *s)
  592. {
  593. struct dm_snap_pending_exception *pe = mempool_alloc(&s->pending_pool,
  594. GFP_NOIO);
  595. atomic_inc(&s->pending_exceptions_count);
  596. pe->snap = s;
  597. return pe;
  598. }
  599. static void free_pending_exception(struct dm_snap_pending_exception *pe)
  600. {
  601. struct dm_snapshot *s = pe->snap;
  602. mempool_free(pe, &s->pending_pool);
  603. smp_mb__before_atomic();
  604. atomic_dec(&s->pending_exceptions_count);
  605. }
  606. static void dm_insert_exception(struct dm_exception_table *eh,
  607. struct dm_exception *new_e)
  608. {
  609. struct hlist_bl_head *l;
  610. struct hlist_bl_node *pos;
  611. struct dm_exception *e = NULL;
  612. l = &eh->table[exception_hash(eh, new_e->old_chunk)];
  613. /* Add immediately if this table doesn't support consecutive chunks */
  614. if (!eh->hash_shift)
  615. goto out;
  616. /* List is ordered by old_chunk */
  617. hlist_bl_for_each_entry(e, pos, l, hash_list) {
  618. /* Insert after an existing chunk? */
  619. if (new_e->old_chunk == (e->old_chunk +
  620. dm_consecutive_chunk_count(e) + 1) &&
  621. new_e->new_chunk == (dm_chunk_number(e->new_chunk) +
  622. dm_consecutive_chunk_count(e) + 1)) {
  623. dm_consecutive_chunk_count_inc(e);
  624. free_completed_exception(new_e);
  625. return;
  626. }
  627. /* Insert before an existing chunk? */
  628. if (new_e->old_chunk == (e->old_chunk - 1) &&
  629. new_e->new_chunk == (dm_chunk_number(e->new_chunk) - 1)) {
  630. dm_consecutive_chunk_count_inc(e);
  631. e->old_chunk--;
  632. e->new_chunk--;
  633. free_completed_exception(new_e);
  634. return;
  635. }
  636. if (new_e->old_chunk < e->old_chunk)
  637. break;
  638. }
  639. out:
  640. if (!e) {
  641. /*
  642. * Either the table doesn't support consecutive chunks or slot
  643. * l is empty.
  644. */
  645. hlist_bl_add_head(&new_e->hash_list, l);
  646. } else if (new_e->old_chunk < e->old_chunk) {
  647. /* Add before an existing exception */
  648. hlist_bl_add_before(&new_e->hash_list, &e->hash_list);
  649. } else {
  650. /* Add to l's tail: e is the last exception in this slot */
  651. hlist_bl_add_behind(&new_e->hash_list, &e->hash_list);
  652. }
  653. }
  654. /*
  655. * Callback used by the exception stores to load exceptions when
  656. * initialising.
  657. */
  658. static int dm_add_exception(void *context, chunk_t old, chunk_t new)
  659. {
  660. struct dm_exception_table_lock lock;
  661. struct dm_snapshot *s = context;
  662. struct dm_exception *e;
  663. e = alloc_completed_exception(GFP_KERNEL);
  664. if (!e)
  665. return -ENOMEM;
  666. e->old_chunk = old;
  667. /* Consecutive_count is implicitly initialised to zero */
  668. e->new_chunk = new;
  669. /*
  670. * Although there is no need to lock access to the exception tables
  671. * here, if we don't then hlist_bl_add_head(), called by
  672. * dm_insert_exception(), will complain about accessing the
  673. * corresponding list without locking it first.
  674. */
  675. dm_exception_table_lock_init(s, old, &lock);
  676. dm_exception_table_lock(&lock);
  677. dm_insert_exception(&s->complete, e);
  678. dm_exception_table_unlock(&lock);
  679. return 0;
  680. }
  681. /*
  682. * Return a minimum chunk size of all snapshots that have the specified origin.
  683. * Return zero if the origin has no snapshots.
  684. */
  685. static uint32_t __minimum_chunk_size(struct origin *o)
  686. {
  687. struct dm_snapshot *snap;
  688. unsigned int chunk_size = rounddown_pow_of_two(UINT_MAX);
  689. if (o)
  690. list_for_each_entry(snap, &o->snapshots, list)
  691. chunk_size = min_not_zero(chunk_size,
  692. snap->store->chunk_size);
  693. return (uint32_t) chunk_size;
  694. }
  695. /*
  696. * Hard coded magic.
  697. */
  698. static int calc_max_buckets(void)
  699. {
  700. /* use a fixed size of 2MB */
  701. unsigned long mem = 2 * 1024 * 1024;
  702. mem /= sizeof(struct hlist_bl_head);
  703. return mem;
  704. }
  705. /*
  706. * Allocate room for a suitable hash table.
  707. */
  708. static int init_hash_tables(struct dm_snapshot *s)
  709. {
  710. sector_t hash_size, cow_dev_size, max_buckets;
  711. /*
  712. * Calculate based on the size of the original volume or
  713. * the COW volume...
  714. */
  715. cow_dev_size = get_dev_size(s->cow->bdev);
  716. max_buckets = calc_max_buckets();
  717. hash_size = cow_dev_size >> s->store->chunk_shift;
  718. hash_size = min(hash_size, max_buckets);
  719. if (hash_size < 64)
  720. hash_size = 64;
  721. hash_size = rounddown_pow_of_two(hash_size);
  722. if (dm_exception_table_init(&s->complete, hash_size,
  723. DM_CHUNK_CONSECUTIVE_BITS))
  724. return -ENOMEM;
  725. /*
  726. * Allocate hash table for in-flight exceptions
  727. * Make this smaller than the real hash table
  728. */
  729. hash_size >>= 3;
  730. if (hash_size < 64)
  731. hash_size = 64;
  732. if (dm_exception_table_init(&s->pending, hash_size, 0)) {
  733. dm_exception_table_exit(&s->complete, exception_cache);
  734. return -ENOMEM;
  735. }
  736. return 0;
  737. }
  738. static void merge_shutdown(struct dm_snapshot *s)
  739. {
  740. clear_bit_unlock(RUNNING_MERGE, &s->state_bits);
  741. smp_mb__after_atomic();
  742. wake_up_bit(&s->state_bits, RUNNING_MERGE);
  743. }
  744. static struct bio *__release_queued_bios_after_merge(struct dm_snapshot *s)
  745. {
  746. s->first_merging_chunk = 0;
  747. s->num_merging_chunks = 0;
  748. return bio_list_get(&s->bios_queued_during_merge);
  749. }
  750. /*
  751. * Remove one chunk from the index of completed exceptions.
  752. */
  753. static int __remove_single_exception_chunk(struct dm_snapshot *s,
  754. chunk_t old_chunk)
  755. {
  756. struct dm_exception *e;
  757. e = dm_lookup_exception(&s->complete, old_chunk);
  758. if (!e) {
  759. DMERR("Corruption detected: exception for block %llu is on disk but not in memory",
  760. (unsigned long long)old_chunk);
  761. return -EINVAL;
  762. }
  763. /*
  764. * If this is the only chunk using this exception, remove exception.
  765. */
  766. if (!dm_consecutive_chunk_count(e)) {
  767. dm_remove_exception(e);
  768. free_completed_exception(e);
  769. return 0;
  770. }
  771. /*
  772. * The chunk may be either at the beginning or the end of a
  773. * group of consecutive chunks - never in the middle. We are
  774. * removing chunks in the opposite order to that in which they
  775. * were added, so this should always be true.
  776. * Decrement the consecutive chunk counter and adjust the
  777. * starting point if necessary.
  778. */
  779. if (old_chunk == e->old_chunk) {
  780. e->old_chunk++;
  781. e->new_chunk++;
  782. } else if (old_chunk != e->old_chunk +
  783. dm_consecutive_chunk_count(e)) {
  784. DMERR("Attempt to merge block %llu from the middle of a chunk range [%llu - %llu]",
  785. (unsigned long long)old_chunk,
  786. (unsigned long long)e->old_chunk,
  787. (unsigned long long)
  788. e->old_chunk + dm_consecutive_chunk_count(e));
  789. return -EINVAL;
  790. }
  791. dm_consecutive_chunk_count_dec(e);
  792. return 0;
  793. }
  794. static void flush_bios(struct bio *bio);
  795. static int remove_single_exception_chunk(struct dm_snapshot *s)
  796. {
  797. struct bio *b = NULL;
  798. int r;
  799. chunk_t old_chunk = s->first_merging_chunk + s->num_merging_chunks - 1;
  800. down_write(&s->lock);
  801. /*
  802. * Process chunks (and associated exceptions) in reverse order
  803. * so that dm_consecutive_chunk_count_dec() accounting works.
  804. */
  805. do {
  806. r = __remove_single_exception_chunk(s, old_chunk);
  807. if (r)
  808. goto out;
  809. } while (old_chunk-- > s->first_merging_chunk);
  810. b = __release_queued_bios_after_merge(s);
  811. out:
  812. up_write(&s->lock);
  813. if (b)
  814. flush_bios(b);
  815. return r;
  816. }
  817. static int origin_write_extent(struct dm_snapshot *merging_snap,
  818. sector_t sector, unsigned int chunk_size);
  819. static void merge_callback(int read_err, unsigned long write_err,
  820. void *context);
  821. static uint64_t read_pending_exceptions_done_count(void)
  822. {
  823. uint64_t pending_exceptions_done;
  824. spin_lock(&_pending_exceptions_done_spinlock);
  825. pending_exceptions_done = _pending_exceptions_done_count;
  826. spin_unlock(&_pending_exceptions_done_spinlock);
  827. return pending_exceptions_done;
  828. }
  829. static void increment_pending_exceptions_done_count(void)
  830. {
  831. spin_lock(&_pending_exceptions_done_spinlock);
  832. _pending_exceptions_done_count++;
  833. spin_unlock(&_pending_exceptions_done_spinlock);
  834. wake_up_all(&_pending_exceptions_done);
  835. }
  836. static void snapshot_merge_next_chunks(struct dm_snapshot *s)
  837. {
  838. int i, linear_chunks;
  839. chunk_t old_chunk, new_chunk;
  840. struct dm_io_region src, dest;
  841. sector_t io_size;
  842. uint64_t previous_count;
  843. BUG_ON(!test_bit(RUNNING_MERGE, &s->state_bits));
  844. if (unlikely(test_bit(SHUTDOWN_MERGE, &s->state_bits)))
  845. goto shut;
  846. /*
  847. * valid flag never changes during merge, so no lock required.
  848. */
  849. if (!s->valid) {
  850. DMERR("Snapshot is invalid: can't merge");
  851. goto shut;
  852. }
  853. linear_chunks = s->store->type->prepare_merge(s->store, &old_chunk,
  854. &new_chunk);
  855. if (linear_chunks <= 0) {
  856. if (linear_chunks < 0) {
  857. DMERR("Read error in exception store: shutting down merge");
  858. down_write(&s->lock);
  859. s->merge_failed = true;
  860. up_write(&s->lock);
  861. }
  862. goto shut;
  863. }
  864. /* Adjust old_chunk and new_chunk to reflect start of linear region */
  865. old_chunk = old_chunk + 1 - linear_chunks;
  866. new_chunk = new_chunk + 1 - linear_chunks;
  867. /*
  868. * Use one (potentially large) I/O to copy all 'linear_chunks'
  869. * from the exception store to the origin
  870. */
  871. io_size = linear_chunks * s->store->chunk_size;
  872. dest.bdev = s->origin->bdev;
  873. dest.sector = chunk_to_sector(s->store, old_chunk);
  874. dest.count = min(io_size, get_dev_size(dest.bdev) - dest.sector);
  875. src.bdev = s->cow->bdev;
  876. src.sector = chunk_to_sector(s->store, new_chunk);
  877. src.count = dest.count;
  878. /*
  879. * Reallocate any exceptions needed in other snapshots then
  880. * wait for the pending exceptions to complete.
  881. * Each time any pending exception (globally on the system)
  882. * completes we are woken and repeat the process to find out
  883. * if we can proceed. While this may not seem a particularly
  884. * efficient algorithm, it is not expected to have any
  885. * significant impact on performance.
  886. */
  887. previous_count = read_pending_exceptions_done_count();
  888. while (origin_write_extent(s, dest.sector, io_size)) {
  889. wait_event(_pending_exceptions_done,
  890. (read_pending_exceptions_done_count() !=
  891. previous_count));
  892. /* Retry after the wait, until all exceptions are done. */
  893. previous_count = read_pending_exceptions_done_count();
  894. }
  895. down_write(&s->lock);
  896. s->first_merging_chunk = old_chunk;
  897. s->num_merging_chunks = linear_chunks;
  898. up_write(&s->lock);
  899. /* Wait until writes to all 'linear_chunks' drain */
  900. for (i = 0; i < linear_chunks; i++)
  901. __check_for_conflicting_io(s, old_chunk + i);
  902. dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, merge_callback, s);
  903. return;
  904. shut:
  905. merge_shutdown(s);
  906. }
  907. static void error_bios(struct bio *bio);
  908. static void merge_callback(int read_err, unsigned long write_err, void *context)
  909. {
  910. struct dm_snapshot *s = context;
  911. struct bio *b = NULL;
  912. if (read_err || write_err) {
  913. if (read_err)
  914. DMERR("Read error: shutting down merge.");
  915. else
  916. DMERR("Write error: shutting down merge.");
  917. goto shut;
  918. }
  919. if (blkdev_issue_flush(s->origin->bdev) < 0) {
  920. DMERR("Flush after merge failed: shutting down merge");
  921. goto shut;
  922. }
  923. if (s->store->type->commit_merge(s->store,
  924. s->num_merging_chunks) < 0) {
  925. DMERR("Write error in exception store: shutting down merge");
  926. goto shut;
  927. }
  928. if (remove_single_exception_chunk(s) < 0)
  929. goto shut;
  930. snapshot_merge_next_chunks(s);
  931. return;
  932. shut:
  933. down_write(&s->lock);
  934. s->merge_failed = true;
  935. b = __release_queued_bios_after_merge(s);
  936. up_write(&s->lock);
  937. error_bios(b);
  938. merge_shutdown(s);
  939. }
  940. static void start_merge(struct dm_snapshot *s)
  941. {
  942. if (!test_and_set_bit(RUNNING_MERGE, &s->state_bits))
  943. snapshot_merge_next_chunks(s);
  944. }
  945. /*
  946. * Stop the merging process and wait until it finishes.
  947. */
  948. static void stop_merge(struct dm_snapshot *s)
  949. {
  950. set_bit(SHUTDOWN_MERGE, &s->state_bits);
  951. wait_on_bit(&s->state_bits, RUNNING_MERGE, TASK_UNINTERRUPTIBLE);
  952. clear_bit(SHUTDOWN_MERGE, &s->state_bits);
  953. }
  954. static int parse_snapshot_features(struct dm_arg_set *as, struct dm_snapshot *s,
  955. struct dm_target *ti)
  956. {
  957. int r;
  958. unsigned int argc;
  959. const char *arg_name;
  960. static const struct dm_arg _args[] = {
  961. {0, 2, "Invalid number of feature arguments"},
  962. };
  963. /*
  964. * No feature arguments supplied.
  965. */
  966. if (!as->argc)
  967. return 0;
  968. r = dm_read_arg_group(_args, as, &argc, &ti->error);
  969. if (r)
  970. return -EINVAL;
  971. while (argc && !r) {
  972. arg_name = dm_shift_arg(as);
  973. argc--;
  974. if (!strcasecmp(arg_name, "discard_zeroes_cow"))
  975. s->discard_zeroes_cow = true;
  976. else if (!strcasecmp(arg_name, "discard_passdown_origin"))
  977. s->discard_passdown_origin = true;
  978. else {
  979. ti->error = "Unrecognised feature requested";
  980. r = -EINVAL;
  981. break;
  982. }
  983. }
  984. if (!s->discard_zeroes_cow && s->discard_passdown_origin) {
  985. /*
  986. * TODO: really these are disjoint.. but ti->num_discard_bios
  987. * and dm_bio_get_target_bio_nr() require rigid constraints.
  988. */
  989. ti->error = "discard_passdown_origin feature depends on discard_zeroes_cow";
  990. r = -EINVAL;
  991. }
  992. return r;
  993. }
  994. /*
  995. * Construct a snapshot mapping:
  996. * <origin_dev> <COW-dev> <p|po|n> <chunk-size> [<# feature args> [<arg>]*]
  997. */
  998. static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv)
  999. {
  1000. struct dm_snapshot *s;
  1001. struct dm_arg_set as;
  1002. int i;
  1003. int r = -EINVAL;
  1004. char *origin_path, *cow_path;
  1005. dev_t origin_dev, cow_dev;
  1006. unsigned int args_used, num_flush_bios = 1;
  1007. fmode_t origin_mode = FMODE_READ;
  1008. if (argc < 4) {
  1009. ti->error = "requires 4 or more arguments";
  1010. r = -EINVAL;
  1011. goto bad;
  1012. }
  1013. if (dm_target_is_snapshot_merge(ti)) {
  1014. num_flush_bios = 2;
  1015. origin_mode = FMODE_WRITE;
  1016. }
  1017. s = kzalloc(sizeof(*s), GFP_KERNEL);
  1018. if (!s) {
  1019. ti->error = "Cannot allocate private snapshot structure";
  1020. r = -ENOMEM;
  1021. goto bad;
  1022. }
  1023. as.argc = argc;
  1024. as.argv = argv;
  1025. dm_consume_args(&as, 4);
  1026. r = parse_snapshot_features(&as, s, ti);
  1027. if (r)
  1028. goto bad_features;
  1029. origin_path = argv[0];
  1030. argv++;
  1031. argc--;
  1032. r = dm_get_device(ti, origin_path, origin_mode, &s->origin);
  1033. if (r) {
  1034. ti->error = "Cannot get origin device";
  1035. goto bad_origin;
  1036. }
  1037. origin_dev = s->origin->bdev->bd_dev;
  1038. cow_path = argv[0];
  1039. argv++;
  1040. argc--;
  1041. cow_dev = dm_get_dev_t(cow_path);
  1042. if (cow_dev && cow_dev == origin_dev) {
  1043. ti->error = "COW device cannot be the same as origin device";
  1044. r = -EINVAL;
  1045. goto bad_cow;
  1046. }
  1047. r = dm_get_device(ti, cow_path, dm_table_get_mode(ti->table), &s->cow);
  1048. if (r) {
  1049. ti->error = "Cannot get COW device";
  1050. goto bad_cow;
  1051. }
  1052. r = dm_exception_store_create(ti, argc, argv, s, &args_used, &s->store);
  1053. if (r) {
  1054. ti->error = "Couldn't create exception store";
  1055. r = -EINVAL;
  1056. goto bad_store;
  1057. }
  1058. argv += args_used;
  1059. argc -= args_used;
  1060. s->ti = ti;
  1061. s->valid = 1;
  1062. s->snapshot_overflowed = 0;
  1063. s->active = 0;
  1064. atomic_set(&s->pending_exceptions_count, 0);
  1065. spin_lock_init(&s->pe_allocation_lock);
  1066. s->exception_start_sequence = 0;
  1067. s->exception_complete_sequence = 0;
  1068. s->out_of_order_tree = RB_ROOT;
  1069. init_rwsem(&s->lock);
  1070. INIT_LIST_HEAD(&s->list);
  1071. spin_lock_init(&s->pe_lock);
  1072. s->state_bits = 0;
  1073. s->merge_failed = false;
  1074. s->first_merging_chunk = 0;
  1075. s->num_merging_chunks = 0;
  1076. bio_list_init(&s->bios_queued_during_merge);
  1077. /* Allocate hash table for COW data */
  1078. if (init_hash_tables(s)) {
  1079. ti->error = "Unable to allocate hash table space";
  1080. r = -ENOMEM;
  1081. goto bad_hash_tables;
  1082. }
  1083. init_waitqueue_head(&s->in_progress_wait);
  1084. s->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle);
  1085. if (IS_ERR(s->kcopyd_client)) {
  1086. r = PTR_ERR(s->kcopyd_client);
  1087. ti->error = "Could not create kcopyd client";
  1088. goto bad_kcopyd;
  1089. }
  1090. r = mempool_init_slab_pool(&s->pending_pool, MIN_IOS, pending_cache);
  1091. if (r) {
  1092. ti->error = "Could not allocate mempool for pending exceptions";
  1093. goto bad_pending_pool;
  1094. }
  1095. for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++)
  1096. INIT_HLIST_HEAD(&s->tracked_chunk_hash[i]);
  1097. spin_lock_init(&s->tracked_chunk_lock);
  1098. ti->private = s;
  1099. ti->num_flush_bios = num_flush_bios;
  1100. if (s->discard_zeroes_cow)
  1101. ti->num_discard_bios = (s->discard_passdown_origin ? 2 : 1);
  1102. ti->per_io_data_size = sizeof(struct dm_snap_tracked_chunk);
  1103. /* Add snapshot to the list of snapshots for this origin */
  1104. /* Exceptions aren't triggered till snapshot_resume() is called */
  1105. r = register_snapshot(s);
  1106. if (r == -ENOMEM) {
  1107. ti->error = "Snapshot origin struct allocation failed";
  1108. goto bad_load_and_register;
  1109. } else if (r < 0) {
  1110. /* invalid handover, register_snapshot has set ti->error */
  1111. goto bad_load_and_register;
  1112. }
  1113. /*
  1114. * Metadata must only be loaded into one table at once, so skip this
  1115. * if metadata will be handed over during resume.
  1116. * Chunk size will be set during the handover - set it to zero to
  1117. * ensure it's ignored.
  1118. */
  1119. if (r > 0) {
  1120. s->store->chunk_size = 0;
  1121. return 0;
  1122. }
  1123. r = s->store->type->read_metadata(s->store, dm_add_exception,
  1124. (void *)s);
  1125. if (r < 0) {
  1126. ti->error = "Failed to read snapshot metadata";
  1127. goto bad_read_metadata;
  1128. } else if (r > 0) {
  1129. s->valid = 0;
  1130. DMWARN("Snapshot is marked invalid.");
  1131. }
  1132. if (!s->store->chunk_size) {
  1133. ti->error = "Chunk size not set";
  1134. r = -EINVAL;
  1135. goto bad_read_metadata;
  1136. }
  1137. r = dm_set_target_max_io_len(ti, s->store->chunk_size);
  1138. if (r)
  1139. goto bad_read_metadata;
  1140. return 0;
  1141. bad_read_metadata:
  1142. unregister_snapshot(s);
  1143. bad_load_and_register:
  1144. mempool_exit(&s->pending_pool);
  1145. bad_pending_pool:
  1146. dm_kcopyd_client_destroy(s->kcopyd_client);
  1147. bad_kcopyd:
  1148. dm_exception_table_exit(&s->pending, pending_cache);
  1149. dm_exception_table_exit(&s->complete, exception_cache);
  1150. bad_hash_tables:
  1151. dm_exception_store_destroy(s->store);
  1152. bad_store:
  1153. dm_put_device(ti, s->cow);
  1154. bad_cow:
  1155. dm_put_device(ti, s->origin);
  1156. bad_origin:
  1157. bad_features:
  1158. kfree(s);
  1159. bad:
  1160. return r;
  1161. }
  1162. static void __free_exceptions(struct dm_snapshot *s)
  1163. {
  1164. dm_kcopyd_client_destroy(s->kcopyd_client);
  1165. s->kcopyd_client = NULL;
  1166. dm_exception_table_exit(&s->pending, pending_cache);
  1167. dm_exception_table_exit(&s->complete, exception_cache);
  1168. }
  1169. static void __handover_exceptions(struct dm_snapshot *snap_src,
  1170. struct dm_snapshot *snap_dest)
  1171. {
  1172. union {
  1173. struct dm_exception_table table_swap;
  1174. struct dm_exception_store *store_swap;
  1175. } u;
  1176. /*
  1177. * Swap all snapshot context information between the two instances.
  1178. */
  1179. u.table_swap = snap_dest->complete;
  1180. snap_dest->complete = snap_src->complete;
  1181. snap_src->complete = u.table_swap;
  1182. u.store_swap = snap_dest->store;
  1183. snap_dest->store = snap_src->store;
  1184. snap_dest->store->userspace_supports_overflow = u.store_swap->userspace_supports_overflow;
  1185. snap_src->store = u.store_swap;
  1186. snap_dest->store->snap = snap_dest;
  1187. snap_src->store->snap = snap_src;
  1188. snap_dest->ti->max_io_len = snap_dest->store->chunk_size;
  1189. snap_dest->valid = snap_src->valid;
  1190. snap_dest->snapshot_overflowed = snap_src->snapshot_overflowed;
  1191. /*
  1192. * Set source invalid to ensure it receives no further I/O.
  1193. */
  1194. snap_src->valid = 0;
  1195. }
  1196. static void snapshot_dtr(struct dm_target *ti)
  1197. {
  1198. #ifdef CONFIG_DM_DEBUG
  1199. int i;
  1200. #endif
  1201. struct dm_snapshot *s = ti->private;
  1202. struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
  1203. down_read(&_origins_lock);
  1204. /* Check whether exception handover must be cancelled */
  1205. (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
  1206. if (snap_src && snap_dest && (s == snap_src)) {
  1207. down_write(&snap_dest->lock);
  1208. snap_dest->valid = 0;
  1209. up_write(&snap_dest->lock);
  1210. DMERR("Cancelling snapshot handover.");
  1211. }
  1212. up_read(&_origins_lock);
  1213. if (dm_target_is_snapshot_merge(ti))
  1214. stop_merge(s);
  1215. /* Prevent further origin writes from using this snapshot. */
  1216. /* After this returns there can be no new kcopyd jobs. */
  1217. unregister_snapshot(s);
  1218. while (atomic_read(&s->pending_exceptions_count))
  1219. msleep(1);
  1220. /*
  1221. * Ensure instructions in mempool_exit aren't reordered
  1222. * before atomic_read.
  1223. */
  1224. smp_mb();
  1225. #ifdef CONFIG_DM_DEBUG
  1226. for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++)
  1227. BUG_ON(!hlist_empty(&s->tracked_chunk_hash[i]));
  1228. #endif
  1229. __free_exceptions(s);
  1230. mempool_exit(&s->pending_pool);
  1231. dm_exception_store_destroy(s->store);
  1232. dm_put_device(ti, s->cow);
  1233. dm_put_device(ti, s->origin);
  1234. WARN_ON(s->in_progress);
  1235. kfree(s);
  1236. }
  1237. static void account_start_copy(struct dm_snapshot *s)
  1238. {
  1239. spin_lock(&s->in_progress_wait.lock);
  1240. s->in_progress++;
  1241. spin_unlock(&s->in_progress_wait.lock);
  1242. }
  1243. static void account_end_copy(struct dm_snapshot *s)
  1244. {
  1245. spin_lock(&s->in_progress_wait.lock);
  1246. BUG_ON(!s->in_progress);
  1247. s->in_progress--;
  1248. if (likely(s->in_progress <= cow_threshold) &&
  1249. unlikely(waitqueue_active(&s->in_progress_wait)))
  1250. wake_up_locked(&s->in_progress_wait);
  1251. spin_unlock(&s->in_progress_wait.lock);
  1252. }
  1253. static bool wait_for_in_progress(struct dm_snapshot *s, bool unlock_origins)
  1254. {
  1255. if (unlikely(s->in_progress > cow_threshold)) {
  1256. spin_lock(&s->in_progress_wait.lock);
  1257. if (likely(s->in_progress > cow_threshold)) {
  1258. /*
  1259. * NOTE: this throttle doesn't account for whether
  1260. * the caller is servicing an IO that will trigger a COW
  1261. * so excess throttling may result for chunks not required
  1262. * to be COW'd. But if cow_threshold was reached, extra
  1263. * throttling is unlikely to negatively impact performance.
  1264. */
  1265. DECLARE_WAITQUEUE(wait, current);
  1266. __add_wait_queue(&s->in_progress_wait, &wait);
  1267. __set_current_state(TASK_UNINTERRUPTIBLE);
  1268. spin_unlock(&s->in_progress_wait.lock);
  1269. if (unlock_origins)
  1270. up_read(&_origins_lock);
  1271. io_schedule();
  1272. remove_wait_queue(&s->in_progress_wait, &wait);
  1273. return false;
  1274. }
  1275. spin_unlock(&s->in_progress_wait.lock);
  1276. }
  1277. return true;
  1278. }
  1279. /*
  1280. * Flush a list of buffers.
  1281. */
  1282. static void flush_bios(struct bio *bio)
  1283. {
  1284. struct bio *n;
  1285. while (bio) {
  1286. n = bio->bi_next;
  1287. bio->bi_next = NULL;
  1288. submit_bio_noacct(bio);
  1289. bio = n;
  1290. }
  1291. }
  1292. static int do_origin(struct dm_dev *origin, struct bio *bio, bool limit);
  1293. /*
  1294. * Flush a list of buffers.
  1295. */
  1296. static void retry_origin_bios(struct dm_snapshot *s, struct bio *bio)
  1297. {
  1298. struct bio *n;
  1299. int r;
  1300. while (bio) {
  1301. n = bio->bi_next;
  1302. bio->bi_next = NULL;
  1303. r = do_origin(s->origin, bio, false);
  1304. if (r == DM_MAPIO_REMAPPED)
  1305. submit_bio_noacct(bio);
  1306. bio = n;
  1307. }
  1308. }
  1309. /*
  1310. * Error a list of buffers.
  1311. */
  1312. static void error_bios(struct bio *bio)
  1313. {
  1314. struct bio *n;
  1315. while (bio) {
  1316. n = bio->bi_next;
  1317. bio->bi_next = NULL;
  1318. bio_io_error(bio);
  1319. bio = n;
  1320. }
  1321. }
  1322. static void __invalidate_snapshot(struct dm_snapshot *s, int err)
  1323. {
  1324. if (!s->valid)
  1325. return;
  1326. if (err == -EIO)
  1327. DMERR("Invalidating snapshot: Error reading/writing.");
  1328. else if (err == -ENOMEM)
  1329. DMERR("Invalidating snapshot: Unable to allocate exception.");
  1330. if (s->store->type->drop_snapshot)
  1331. s->store->type->drop_snapshot(s->store);
  1332. s->valid = 0;
  1333. dm_table_event(s->ti->table);
  1334. }
  1335. static void invalidate_snapshot(struct dm_snapshot *s, int err)
  1336. {
  1337. down_write(&s->lock);
  1338. __invalidate_snapshot(s, err);
  1339. up_write(&s->lock);
  1340. }
  1341. static void pending_complete(void *context, int success)
  1342. {
  1343. struct dm_snap_pending_exception *pe = context;
  1344. struct dm_exception *e;
  1345. struct dm_snapshot *s = pe->snap;
  1346. struct bio *origin_bios = NULL;
  1347. struct bio *snapshot_bios = NULL;
  1348. struct bio *full_bio = NULL;
  1349. struct dm_exception_table_lock lock;
  1350. int error = 0;
  1351. dm_exception_table_lock_init(s, pe->e.old_chunk, &lock);
  1352. if (!success) {
  1353. /* Read/write error - snapshot is unusable */
  1354. invalidate_snapshot(s, -EIO);
  1355. error = 1;
  1356. dm_exception_table_lock(&lock);
  1357. goto out;
  1358. }
  1359. e = alloc_completed_exception(GFP_NOIO);
  1360. if (!e) {
  1361. invalidate_snapshot(s, -ENOMEM);
  1362. error = 1;
  1363. dm_exception_table_lock(&lock);
  1364. goto out;
  1365. }
  1366. *e = pe->e;
  1367. down_read(&s->lock);
  1368. dm_exception_table_lock(&lock);
  1369. if (!s->valid) {
  1370. up_read(&s->lock);
  1371. free_completed_exception(e);
  1372. error = 1;
  1373. goto out;
  1374. }
  1375. /*
  1376. * Add a proper exception. After inserting the completed exception all
  1377. * subsequent snapshot reads to this chunk will be redirected to the
  1378. * COW device. This ensures that we do not starve. Moreover, as long
  1379. * as the pending exception exists, neither origin writes nor snapshot
  1380. * merging can overwrite the chunk in origin.
  1381. */
  1382. dm_insert_exception(&s->complete, e);
  1383. up_read(&s->lock);
  1384. /* Wait for conflicting reads to drain */
  1385. if (__chunk_is_tracked(s, pe->e.old_chunk)) {
  1386. dm_exception_table_unlock(&lock);
  1387. __check_for_conflicting_io(s, pe->e.old_chunk);
  1388. dm_exception_table_lock(&lock);
  1389. }
  1390. out:
  1391. /* Remove the in-flight exception from the list */
  1392. dm_remove_exception(&pe->e);
  1393. dm_exception_table_unlock(&lock);
  1394. snapshot_bios = bio_list_get(&pe->snapshot_bios);
  1395. origin_bios = bio_list_get(&pe->origin_bios);
  1396. full_bio = pe->full_bio;
  1397. if (full_bio)
  1398. full_bio->bi_end_io = pe->full_bio_end_io;
  1399. increment_pending_exceptions_done_count();
  1400. /* Submit any pending write bios */
  1401. if (error) {
  1402. if (full_bio)
  1403. bio_io_error(full_bio);
  1404. error_bios(snapshot_bios);
  1405. } else {
  1406. if (full_bio)
  1407. bio_endio(full_bio);
  1408. flush_bios(snapshot_bios);
  1409. }
  1410. retry_origin_bios(s, origin_bios);
  1411. free_pending_exception(pe);
  1412. }
  1413. static void complete_exception(struct dm_snap_pending_exception *pe)
  1414. {
  1415. struct dm_snapshot *s = pe->snap;
  1416. /* Update the metadata if we are persistent */
  1417. s->store->type->commit_exception(s->store, &pe->e, !pe->copy_error,
  1418. pending_complete, pe);
  1419. }
  1420. /*
  1421. * Called when the copy I/O has finished. kcopyd actually runs
  1422. * this code so don't block.
  1423. */
  1424. static void copy_callback(int read_err, unsigned long write_err, void *context)
  1425. {
  1426. struct dm_snap_pending_exception *pe = context;
  1427. struct dm_snapshot *s = pe->snap;
  1428. pe->copy_error = read_err || write_err;
  1429. if (pe->exception_sequence == s->exception_complete_sequence) {
  1430. struct rb_node *next;
  1431. s->exception_complete_sequence++;
  1432. complete_exception(pe);
  1433. next = rb_first(&s->out_of_order_tree);
  1434. while (next) {
  1435. pe = rb_entry(next, struct dm_snap_pending_exception,
  1436. out_of_order_node);
  1437. if (pe->exception_sequence != s->exception_complete_sequence)
  1438. break;
  1439. next = rb_next(next);
  1440. s->exception_complete_sequence++;
  1441. rb_erase(&pe->out_of_order_node, &s->out_of_order_tree);
  1442. complete_exception(pe);
  1443. cond_resched();
  1444. }
  1445. } else {
  1446. struct rb_node *parent = NULL;
  1447. struct rb_node **p = &s->out_of_order_tree.rb_node;
  1448. struct dm_snap_pending_exception *pe2;
  1449. while (*p) {
  1450. pe2 = rb_entry(*p, struct dm_snap_pending_exception, out_of_order_node);
  1451. parent = *p;
  1452. BUG_ON(pe->exception_sequence == pe2->exception_sequence);
  1453. if (pe->exception_sequence < pe2->exception_sequence)
  1454. p = &((*p)->rb_left);
  1455. else
  1456. p = &((*p)->rb_right);
  1457. }
  1458. rb_link_node(&pe->out_of_order_node, parent, p);
  1459. rb_insert_color(&pe->out_of_order_node, &s->out_of_order_tree);
  1460. }
  1461. account_end_copy(s);
  1462. }
  1463. /*
  1464. * Dispatches the copy operation to kcopyd.
  1465. */
  1466. static void start_copy(struct dm_snap_pending_exception *pe)
  1467. {
  1468. struct dm_snapshot *s = pe->snap;
  1469. struct dm_io_region src, dest;
  1470. struct block_device *bdev = s->origin->bdev;
  1471. sector_t dev_size;
  1472. dev_size = get_dev_size(bdev);
  1473. src.bdev = bdev;
  1474. src.sector = chunk_to_sector(s->store, pe->e.old_chunk);
  1475. src.count = min((sector_t)s->store->chunk_size, dev_size - src.sector);
  1476. dest.bdev = s->cow->bdev;
  1477. dest.sector = chunk_to_sector(s->store, pe->e.new_chunk);
  1478. dest.count = src.count;
  1479. /* Hand over to kcopyd */
  1480. account_start_copy(s);
  1481. dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe);
  1482. }
  1483. static void full_bio_end_io(struct bio *bio)
  1484. {
  1485. void *callback_data = bio->bi_private;
  1486. dm_kcopyd_do_callback(callback_data, 0, bio->bi_status ? 1 : 0);
  1487. }
  1488. static void start_full_bio(struct dm_snap_pending_exception *pe,
  1489. struct bio *bio)
  1490. {
  1491. struct dm_snapshot *s = pe->snap;
  1492. void *callback_data;
  1493. pe->full_bio = bio;
  1494. pe->full_bio_end_io = bio->bi_end_io;
  1495. account_start_copy(s);
  1496. callback_data = dm_kcopyd_prepare_callback(s->kcopyd_client,
  1497. copy_callback, pe);
  1498. bio->bi_end_io = full_bio_end_io;
  1499. bio->bi_private = callback_data;
  1500. submit_bio_noacct(bio);
  1501. }
  1502. static struct dm_snap_pending_exception *
  1503. __lookup_pending_exception(struct dm_snapshot *s, chunk_t chunk)
  1504. {
  1505. struct dm_exception *e = dm_lookup_exception(&s->pending, chunk);
  1506. if (!e)
  1507. return NULL;
  1508. return container_of(e, struct dm_snap_pending_exception, e);
  1509. }
  1510. /*
  1511. * Inserts a pending exception into the pending table.
  1512. *
  1513. * NOTE: a write lock must be held on the chunk's pending exception table slot
  1514. * before calling this.
  1515. */
  1516. static struct dm_snap_pending_exception *
  1517. __insert_pending_exception(struct dm_snapshot *s,
  1518. struct dm_snap_pending_exception *pe, chunk_t chunk)
  1519. {
  1520. pe->e.old_chunk = chunk;
  1521. bio_list_init(&pe->origin_bios);
  1522. bio_list_init(&pe->snapshot_bios);
  1523. pe->started = 0;
  1524. pe->full_bio = NULL;
  1525. spin_lock(&s->pe_allocation_lock);
  1526. if (s->store->type->prepare_exception(s->store, &pe->e)) {
  1527. spin_unlock(&s->pe_allocation_lock);
  1528. free_pending_exception(pe);
  1529. return NULL;
  1530. }
  1531. pe->exception_sequence = s->exception_start_sequence++;
  1532. spin_unlock(&s->pe_allocation_lock);
  1533. dm_insert_exception(&s->pending, &pe->e);
  1534. return pe;
  1535. }
  1536. /*
  1537. * Looks to see if this snapshot already has a pending exception
  1538. * for this chunk, otherwise it allocates a new one and inserts
  1539. * it into the pending table.
  1540. *
  1541. * NOTE: a write lock must be held on the chunk's pending exception table slot
  1542. * before calling this.
  1543. */
  1544. static struct dm_snap_pending_exception *
  1545. __find_pending_exception(struct dm_snapshot *s,
  1546. struct dm_snap_pending_exception *pe, chunk_t chunk)
  1547. {
  1548. struct dm_snap_pending_exception *pe2;
  1549. pe2 = __lookup_pending_exception(s, chunk);
  1550. if (pe2) {
  1551. free_pending_exception(pe);
  1552. return pe2;
  1553. }
  1554. return __insert_pending_exception(s, pe, chunk);
  1555. }
  1556. static void remap_exception(struct dm_snapshot *s, struct dm_exception *e,
  1557. struct bio *bio, chunk_t chunk)
  1558. {
  1559. bio_set_dev(bio, s->cow->bdev);
  1560. bio->bi_iter.bi_sector =
  1561. chunk_to_sector(s->store, dm_chunk_number(e->new_chunk) +
  1562. (chunk - e->old_chunk)) +
  1563. (bio->bi_iter.bi_sector & s->store->chunk_mask);
  1564. }
  1565. static void zero_callback(int read_err, unsigned long write_err, void *context)
  1566. {
  1567. struct bio *bio = context;
  1568. struct dm_snapshot *s = bio->bi_private;
  1569. account_end_copy(s);
  1570. bio->bi_status = write_err ? BLK_STS_IOERR : 0;
  1571. bio_endio(bio);
  1572. }
  1573. static void zero_exception(struct dm_snapshot *s, struct dm_exception *e,
  1574. struct bio *bio, chunk_t chunk)
  1575. {
  1576. struct dm_io_region dest;
  1577. dest.bdev = s->cow->bdev;
  1578. dest.sector = bio->bi_iter.bi_sector;
  1579. dest.count = s->store->chunk_size;
  1580. account_start_copy(s);
  1581. WARN_ON_ONCE(bio->bi_private);
  1582. bio->bi_private = s;
  1583. dm_kcopyd_zero(s->kcopyd_client, 1, &dest, 0, zero_callback, bio);
  1584. }
  1585. static bool io_overlaps_chunk(struct dm_snapshot *s, struct bio *bio)
  1586. {
  1587. return bio->bi_iter.bi_size ==
  1588. (s->store->chunk_size << SECTOR_SHIFT);
  1589. }
  1590. static int snapshot_map(struct dm_target *ti, struct bio *bio)
  1591. {
  1592. struct dm_exception *e;
  1593. struct dm_snapshot *s = ti->private;
  1594. int r = DM_MAPIO_REMAPPED;
  1595. chunk_t chunk;
  1596. struct dm_snap_pending_exception *pe = NULL;
  1597. struct dm_exception_table_lock lock;
  1598. init_tracked_chunk(bio);
  1599. if (bio->bi_opf & REQ_PREFLUSH) {
  1600. bio_set_dev(bio, s->cow->bdev);
  1601. return DM_MAPIO_REMAPPED;
  1602. }
  1603. chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector);
  1604. dm_exception_table_lock_init(s, chunk, &lock);
  1605. /* Full snapshots are not usable */
  1606. /* To get here the table must be live so s->active is always set. */
  1607. if (!s->valid)
  1608. return DM_MAPIO_KILL;
  1609. if (bio_data_dir(bio) == WRITE) {
  1610. while (unlikely(!wait_for_in_progress(s, false)))
  1611. ; /* wait_for_in_progress() has slept */
  1612. }
  1613. down_read(&s->lock);
  1614. dm_exception_table_lock(&lock);
  1615. if (!s->valid || (unlikely(s->snapshot_overflowed) &&
  1616. bio_data_dir(bio) == WRITE)) {
  1617. r = DM_MAPIO_KILL;
  1618. goto out_unlock;
  1619. }
  1620. if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) {
  1621. if (s->discard_passdown_origin && dm_bio_get_target_bio_nr(bio)) {
  1622. /*
  1623. * passdown discard to origin (without triggering
  1624. * snapshot exceptions via do_origin; doing so would
  1625. * defeat the goal of freeing space in origin that is
  1626. * implied by the "discard_passdown_origin" feature)
  1627. */
  1628. bio_set_dev(bio, s->origin->bdev);
  1629. track_chunk(s, bio, chunk);
  1630. goto out_unlock;
  1631. }
  1632. /* discard to snapshot (target_bio_nr == 0) zeroes exceptions */
  1633. }
  1634. /* If the block is already remapped - use that, else remap it */
  1635. e = dm_lookup_exception(&s->complete, chunk);
  1636. if (e) {
  1637. remap_exception(s, e, bio, chunk);
  1638. if (unlikely(bio_op(bio) == REQ_OP_DISCARD) &&
  1639. io_overlaps_chunk(s, bio)) {
  1640. dm_exception_table_unlock(&lock);
  1641. up_read(&s->lock);
  1642. zero_exception(s, e, bio, chunk);
  1643. r = DM_MAPIO_SUBMITTED; /* discard is not issued */
  1644. goto out;
  1645. }
  1646. goto out_unlock;
  1647. }
  1648. if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) {
  1649. /*
  1650. * If no exception exists, complete discard immediately
  1651. * otherwise it'll trigger copy-out.
  1652. */
  1653. bio_endio(bio);
  1654. r = DM_MAPIO_SUBMITTED;
  1655. goto out_unlock;
  1656. }
  1657. /*
  1658. * Write to snapshot - higher level takes care of RW/RO
  1659. * flags so we should only get this if we are
  1660. * writable.
  1661. */
  1662. if (bio_data_dir(bio) == WRITE) {
  1663. pe = __lookup_pending_exception(s, chunk);
  1664. if (!pe) {
  1665. dm_exception_table_unlock(&lock);
  1666. pe = alloc_pending_exception(s);
  1667. dm_exception_table_lock(&lock);
  1668. e = dm_lookup_exception(&s->complete, chunk);
  1669. if (e) {
  1670. free_pending_exception(pe);
  1671. remap_exception(s, e, bio, chunk);
  1672. goto out_unlock;
  1673. }
  1674. pe = __find_pending_exception(s, pe, chunk);
  1675. if (!pe) {
  1676. dm_exception_table_unlock(&lock);
  1677. up_read(&s->lock);
  1678. down_write(&s->lock);
  1679. if (s->store->userspace_supports_overflow) {
  1680. if (s->valid && !s->snapshot_overflowed) {
  1681. s->snapshot_overflowed = 1;
  1682. DMERR("Snapshot overflowed: Unable to allocate exception.");
  1683. }
  1684. } else
  1685. __invalidate_snapshot(s, -ENOMEM);
  1686. up_write(&s->lock);
  1687. r = DM_MAPIO_KILL;
  1688. goto out;
  1689. }
  1690. }
  1691. remap_exception(s, &pe->e, bio, chunk);
  1692. r = DM_MAPIO_SUBMITTED;
  1693. if (!pe->started && io_overlaps_chunk(s, bio)) {
  1694. pe->started = 1;
  1695. dm_exception_table_unlock(&lock);
  1696. up_read(&s->lock);
  1697. start_full_bio(pe, bio);
  1698. goto out;
  1699. }
  1700. bio_list_add(&pe->snapshot_bios, bio);
  1701. if (!pe->started) {
  1702. /* this is protected by the exception table lock */
  1703. pe->started = 1;
  1704. dm_exception_table_unlock(&lock);
  1705. up_read(&s->lock);
  1706. start_copy(pe);
  1707. goto out;
  1708. }
  1709. } else {
  1710. bio_set_dev(bio, s->origin->bdev);
  1711. track_chunk(s, bio, chunk);
  1712. }
  1713. out_unlock:
  1714. dm_exception_table_unlock(&lock);
  1715. up_read(&s->lock);
  1716. out:
  1717. return r;
  1718. }
  1719. /*
  1720. * A snapshot-merge target behaves like a combination of a snapshot
  1721. * target and a snapshot-origin target. It only generates new
  1722. * exceptions in other snapshots and not in the one that is being
  1723. * merged.
  1724. *
  1725. * For each chunk, if there is an existing exception, it is used to
  1726. * redirect I/O to the cow device. Otherwise I/O is sent to the origin,
  1727. * which in turn might generate exceptions in other snapshots.
  1728. * If merging is currently taking place on the chunk in question, the
  1729. * I/O is deferred by adding it to s->bios_queued_during_merge.
  1730. */
  1731. static int snapshot_merge_map(struct dm_target *ti, struct bio *bio)
  1732. {
  1733. struct dm_exception *e;
  1734. struct dm_snapshot *s = ti->private;
  1735. int r = DM_MAPIO_REMAPPED;
  1736. chunk_t chunk;
  1737. init_tracked_chunk(bio);
  1738. if (bio->bi_opf & REQ_PREFLUSH) {
  1739. if (!dm_bio_get_target_bio_nr(bio))
  1740. bio_set_dev(bio, s->origin->bdev);
  1741. else
  1742. bio_set_dev(bio, s->cow->bdev);
  1743. return DM_MAPIO_REMAPPED;
  1744. }
  1745. if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) {
  1746. /* Once merging, discards no longer effect change */
  1747. bio_endio(bio);
  1748. return DM_MAPIO_SUBMITTED;
  1749. }
  1750. chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector);
  1751. down_write(&s->lock);
  1752. /* Full merging snapshots are redirected to the origin */
  1753. if (!s->valid)
  1754. goto redirect_to_origin;
  1755. /* If the block is already remapped - use that */
  1756. e = dm_lookup_exception(&s->complete, chunk);
  1757. if (e) {
  1758. /* Queue writes overlapping with chunks being merged */
  1759. if (bio_data_dir(bio) == WRITE &&
  1760. chunk >= s->first_merging_chunk &&
  1761. chunk < (s->first_merging_chunk +
  1762. s->num_merging_chunks)) {
  1763. bio_set_dev(bio, s->origin->bdev);
  1764. bio_list_add(&s->bios_queued_during_merge, bio);
  1765. r = DM_MAPIO_SUBMITTED;
  1766. goto out_unlock;
  1767. }
  1768. remap_exception(s, e, bio, chunk);
  1769. if (bio_data_dir(bio) == WRITE)
  1770. track_chunk(s, bio, chunk);
  1771. goto out_unlock;
  1772. }
  1773. redirect_to_origin:
  1774. bio_set_dev(bio, s->origin->bdev);
  1775. if (bio_data_dir(bio) == WRITE) {
  1776. up_write(&s->lock);
  1777. return do_origin(s->origin, bio, false);
  1778. }
  1779. out_unlock:
  1780. up_write(&s->lock);
  1781. return r;
  1782. }
  1783. static int snapshot_end_io(struct dm_target *ti, struct bio *bio,
  1784. blk_status_t *error)
  1785. {
  1786. struct dm_snapshot *s = ti->private;
  1787. if (is_bio_tracked(bio))
  1788. stop_tracking_chunk(s, bio);
  1789. return DM_ENDIO_DONE;
  1790. }
  1791. static void snapshot_merge_presuspend(struct dm_target *ti)
  1792. {
  1793. struct dm_snapshot *s = ti->private;
  1794. stop_merge(s);
  1795. }
  1796. static int snapshot_preresume(struct dm_target *ti)
  1797. {
  1798. int r = 0;
  1799. struct dm_snapshot *s = ti->private;
  1800. struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
  1801. down_read(&_origins_lock);
  1802. (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
  1803. if (snap_src && snap_dest) {
  1804. down_read(&snap_src->lock);
  1805. if (s == snap_src) {
  1806. DMERR("Unable to resume snapshot source until handover completes.");
  1807. r = -EINVAL;
  1808. } else if (!dm_suspended(snap_src->ti)) {
  1809. DMERR("Unable to perform snapshot handover until source is suspended.");
  1810. r = -EINVAL;
  1811. }
  1812. up_read(&snap_src->lock);
  1813. }
  1814. up_read(&_origins_lock);
  1815. return r;
  1816. }
  1817. static void snapshot_resume(struct dm_target *ti)
  1818. {
  1819. struct dm_snapshot *s = ti->private;
  1820. struct dm_snapshot *snap_src = NULL, *snap_dest = NULL, *snap_merging = NULL;
  1821. struct dm_origin *o;
  1822. struct mapped_device *origin_md = NULL;
  1823. bool must_restart_merging = false;
  1824. down_read(&_origins_lock);
  1825. o = __lookup_dm_origin(s->origin->bdev);
  1826. if (o)
  1827. origin_md = dm_table_get_md(o->ti->table);
  1828. if (!origin_md) {
  1829. (void) __find_snapshots_sharing_cow(s, NULL, NULL, &snap_merging);
  1830. if (snap_merging)
  1831. origin_md = dm_table_get_md(snap_merging->ti->table);
  1832. }
  1833. if (origin_md == dm_table_get_md(ti->table))
  1834. origin_md = NULL;
  1835. if (origin_md) {
  1836. if (dm_hold(origin_md))
  1837. origin_md = NULL;
  1838. }
  1839. up_read(&_origins_lock);
  1840. if (origin_md) {
  1841. dm_internal_suspend_fast(origin_md);
  1842. if (snap_merging && test_bit(RUNNING_MERGE, &snap_merging->state_bits)) {
  1843. must_restart_merging = true;
  1844. stop_merge(snap_merging);
  1845. }
  1846. }
  1847. down_read(&_origins_lock);
  1848. (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
  1849. if (snap_src && snap_dest) {
  1850. down_write(&snap_src->lock);
  1851. down_write_nested(&snap_dest->lock, SINGLE_DEPTH_NESTING);
  1852. __handover_exceptions(snap_src, snap_dest);
  1853. up_write(&snap_dest->lock);
  1854. up_write(&snap_src->lock);
  1855. }
  1856. up_read(&_origins_lock);
  1857. if (origin_md) {
  1858. if (must_restart_merging)
  1859. start_merge(snap_merging);
  1860. dm_internal_resume_fast(origin_md);
  1861. dm_put(origin_md);
  1862. }
  1863. /* Now we have correct chunk size, reregister */
  1864. reregister_snapshot(s);
  1865. down_write(&s->lock);
  1866. s->active = 1;
  1867. up_write(&s->lock);
  1868. }
  1869. static uint32_t get_origin_minimum_chunksize(struct block_device *bdev)
  1870. {
  1871. uint32_t min_chunksize;
  1872. down_read(&_origins_lock);
  1873. min_chunksize = __minimum_chunk_size(__lookup_origin(bdev));
  1874. up_read(&_origins_lock);
  1875. return min_chunksize;
  1876. }
  1877. static void snapshot_merge_resume(struct dm_target *ti)
  1878. {
  1879. struct dm_snapshot *s = ti->private;
  1880. /*
  1881. * Handover exceptions from existing snapshot.
  1882. */
  1883. snapshot_resume(ti);
  1884. /*
  1885. * snapshot-merge acts as an origin, so set ti->max_io_len
  1886. */
  1887. ti->max_io_len = get_origin_minimum_chunksize(s->origin->bdev);
  1888. start_merge(s);
  1889. }
  1890. static void snapshot_status(struct dm_target *ti, status_type_t type,
  1891. unsigned int status_flags, char *result, unsigned int maxlen)
  1892. {
  1893. unsigned int sz = 0;
  1894. struct dm_snapshot *snap = ti->private;
  1895. unsigned int num_features;
  1896. switch (type) {
  1897. case STATUSTYPE_INFO:
  1898. down_write(&snap->lock);
  1899. if (!snap->valid)
  1900. DMEMIT("Invalid");
  1901. else if (snap->merge_failed)
  1902. DMEMIT("Merge failed");
  1903. else if (snap->snapshot_overflowed)
  1904. DMEMIT("Overflow");
  1905. else {
  1906. if (snap->store->type->usage) {
  1907. sector_t total_sectors, sectors_allocated,
  1908. metadata_sectors;
  1909. snap->store->type->usage(snap->store,
  1910. &total_sectors,
  1911. &sectors_allocated,
  1912. &metadata_sectors);
  1913. DMEMIT("%llu/%llu %llu",
  1914. (unsigned long long)sectors_allocated,
  1915. (unsigned long long)total_sectors,
  1916. (unsigned long long)metadata_sectors);
  1917. }
  1918. else
  1919. DMEMIT("Unknown");
  1920. }
  1921. up_write(&snap->lock);
  1922. break;
  1923. case STATUSTYPE_TABLE:
  1924. /*
  1925. * kdevname returns a static pointer so we need
  1926. * to make private copies if the output is to
  1927. * make sense.
  1928. */
  1929. DMEMIT("%s %s", snap->origin->name, snap->cow->name);
  1930. sz += snap->store->type->status(snap->store, type, result + sz,
  1931. maxlen - sz);
  1932. num_features = snap->discard_zeroes_cow + snap->discard_passdown_origin;
  1933. if (num_features) {
  1934. DMEMIT(" %u", num_features);
  1935. if (snap->discard_zeroes_cow)
  1936. DMEMIT(" discard_zeroes_cow");
  1937. if (snap->discard_passdown_origin)
  1938. DMEMIT(" discard_passdown_origin");
  1939. }
  1940. break;
  1941. case STATUSTYPE_IMA:
  1942. DMEMIT_TARGET_NAME_VERSION(ti->type);
  1943. DMEMIT(",snap_origin_name=%s", snap->origin->name);
  1944. DMEMIT(",snap_cow_name=%s", snap->cow->name);
  1945. DMEMIT(",snap_valid=%c", snap->valid ? 'y' : 'n');
  1946. DMEMIT(",snap_merge_failed=%c", snap->merge_failed ? 'y' : 'n');
  1947. DMEMIT(",snapshot_overflowed=%c", snap->snapshot_overflowed ? 'y' : 'n');
  1948. DMEMIT(";");
  1949. break;
  1950. }
  1951. }
  1952. static int snapshot_iterate_devices(struct dm_target *ti,
  1953. iterate_devices_callout_fn fn, void *data)
  1954. {
  1955. struct dm_snapshot *snap = ti->private;
  1956. int r;
  1957. r = fn(ti, snap->origin, 0, ti->len, data);
  1958. if (!r)
  1959. r = fn(ti, snap->cow, 0, get_dev_size(snap->cow->bdev), data);
  1960. return r;
  1961. }
  1962. static void snapshot_io_hints(struct dm_target *ti, struct queue_limits *limits)
  1963. {
  1964. struct dm_snapshot *snap = ti->private;
  1965. if (snap->discard_zeroes_cow) {
  1966. struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
  1967. down_read(&_origins_lock);
  1968. (void) __find_snapshots_sharing_cow(snap, &snap_src, &snap_dest, NULL);
  1969. if (snap_src && snap_dest)
  1970. snap = snap_src;
  1971. /* All discards are split on chunk_size boundary */
  1972. limits->discard_granularity = snap->store->chunk_size;
  1973. limits->max_discard_sectors = snap->store->chunk_size;
  1974. up_read(&_origins_lock);
  1975. }
  1976. }
  1977. /*-----------------------------------------------------------------
  1978. * Origin methods
  1979. *---------------------------------------------------------------*/
  1980. /*
  1981. * If no exceptions need creating, DM_MAPIO_REMAPPED is returned and any
  1982. * supplied bio was ignored. The caller may submit it immediately.
  1983. * (No remapping actually occurs as the origin is always a direct linear
  1984. * map.)
  1985. *
  1986. * If further exceptions are required, DM_MAPIO_SUBMITTED is returned
  1987. * and any supplied bio is added to a list to be submitted once all
  1988. * the necessary exceptions exist.
  1989. */
  1990. static int __origin_write(struct list_head *snapshots, sector_t sector,
  1991. struct bio *bio)
  1992. {
  1993. int r = DM_MAPIO_REMAPPED;
  1994. struct dm_snapshot *snap;
  1995. struct dm_exception *e;
  1996. struct dm_snap_pending_exception *pe, *pe2;
  1997. struct dm_snap_pending_exception *pe_to_start_now = NULL;
  1998. struct dm_snap_pending_exception *pe_to_start_last = NULL;
  1999. struct dm_exception_table_lock lock;
  2000. chunk_t chunk;
  2001. /* Do all the snapshots on this origin */
  2002. list_for_each_entry(snap, snapshots, list) {
  2003. /*
  2004. * Don't make new exceptions in a merging snapshot
  2005. * because it has effectively been deleted
  2006. */
  2007. if (dm_target_is_snapshot_merge(snap->ti))
  2008. continue;
  2009. /* Nothing to do if writing beyond end of snapshot */
  2010. if (sector >= dm_table_get_size(snap->ti->table))
  2011. continue;
  2012. /*
  2013. * Remember, different snapshots can have
  2014. * different chunk sizes.
  2015. */
  2016. chunk = sector_to_chunk(snap->store, sector);
  2017. dm_exception_table_lock_init(snap, chunk, &lock);
  2018. down_read(&snap->lock);
  2019. dm_exception_table_lock(&lock);
  2020. /* Only deal with valid and active snapshots */
  2021. if (!snap->valid || !snap->active)
  2022. goto next_snapshot;
  2023. pe = __lookup_pending_exception(snap, chunk);
  2024. if (!pe) {
  2025. /*
  2026. * Check exception table to see if block is already
  2027. * remapped in this snapshot and trigger an exception
  2028. * if not.
  2029. */
  2030. e = dm_lookup_exception(&snap->complete, chunk);
  2031. if (e)
  2032. goto next_snapshot;
  2033. dm_exception_table_unlock(&lock);
  2034. pe = alloc_pending_exception(snap);
  2035. dm_exception_table_lock(&lock);
  2036. pe2 = __lookup_pending_exception(snap, chunk);
  2037. if (!pe2) {
  2038. e = dm_lookup_exception(&snap->complete, chunk);
  2039. if (e) {
  2040. free_pending_exception(pe);
  2041. goto next_snapshot;
  2042. }
  2043. pe = __insert_pending_exception(snap, pe, chunk);
  2044. if (!pe) {
  2045. dm_exception_table_unlock(&lock);
  2046. up_read(&snap->lock);
  2047. invalidate_snapshot(snap, -ENOMEM);
  2048. continue;
  2049. }
  2050. } else {
  2051. free_pending_exception(pe);
  2052. pe = pe2;
  2053. }
  2054. }
  2055. r = DM_MAPIO_SUBMITTED;
  2056. /*
  2057. * If an origin bio was supplied, queue it to wait for the
  2058. * completion of this exception, and start this one last,
  2059. * at the end of the function.
  2060. */
  2061. if (bio) {
  2062. bio_list_add(&pe->origin_bios, bio);
  2063. bio = NULL;
  2064. if (!pe->started) {
  2065. pe->started = 1;
  2066. pe_to_start_last = pe;
  2067. }
  2068. }
  2069. if (!pe->started) {
  2070. pe->started = 1;
  2071. pe_to_start_now = pe;
  2072. }
  2073. next_snapshot:
  2074. dm_exception_table_unlock(&lock);
  2075. up_read(&snap->lock);
  2076. if (pe_to_start_now) {
  2077. start_copy(pe_to_start_now);
  2078. pe_to_start_now = NULL;
  2079. }
  2080. }
  2081. /*
  2082. * Submit the exception against which the bio is queued last,
  2083. * to give the other exceptions a head start.
  2084. */
  2085. if (pe_to_start_last)
  2086. start_copy(pe_to_start_last);
  2087. return r;
  2088. }
  2089. /*
  2090. * Called on a write from the origin driver.
  2091. */
  2092. static int do_origin(struct dm_dev *origin, struct bio *bio, bool limit)
  2093. {
  2094. struct origin *o;
  2095. int r = DM_MAPIO_REMAPPED;
  2096. again:
  2097. down_read(&_origins_lock);
  2098. o = __lookup_origin(origin->bdev);
  2099. if (o) {
  2100. if (limit) {
  2101. struct dm_snapshot *s;
  2102. list_for_each_entry(s, &o->snapshots, list)
  2103. if (unlikely(!wait_for_in_progress(s, true)))
  2104. goto again;
  2105. }
  2106. r = __origin_write(&o->snapshots, bio->bi_iter.bi_sector, bio);
  2107. }
  2108. up_read(&_origins_lock);
  2109. return r;
  2110. }
  2111. /*
  2112. * Trigger exceptions in all non-merging snapshots.
  2113. *
  2114. * The chunk size of the merging snapshot may be larger than the chunk
  2115. * size of some other snapshot so we may need to reallocate multiple
  2116. * chunks in other snapshots.
  2117. *
  2118. * We scan all the overlapping exceptions in the other snapshots.
  2119. * Returns 1 if anything was reallocated and must be waited for,
  2120. * otherwise returns 0.
  2121. *
  2122. * size must be a multiple of merging_snap's chunk_size.
  2123. */
  2124. static int origin_write_extent(struct dm_snapshot *merging_snap,
  2125. sector_t sector, unsigned int size)
  2126. {
  2127. int must_wait = 0;
  2128. sector_t n;
  2129. struct origin *o;
  2130. /*
  2131. * The origin's __minimum_chunk_size() got stored in max_io_len
  2132. * by snapshot_merge_resume().
  2133. */
  2134. down_read(&_origins_lock);
  2135. o = __lookup_origin(merging_snap->origin->bdev);
  2136. for (n = 0; n < size; n += merging_snap->ti->max_io_len)
  2137. if (__origin_write(&o->snapshots, sector + n, NULL) ==
  2138. DM_MAPIO_SUBMITTED)
  2139. must_wait = 1;
  2140. up_read(&_origins_lock);
  2141. return must_wait;
  2142. }
  2143. /*
  2144. * Origin: maps a linear range of a device, with hooks for snapshotting.
  2145. */
  2146. /*
  2147. * Construct an origin mapping: <dev_path>
  2148. * The context for an origin is merely a 'struct dm_dev *'
  2149. * pointing to the real device.
  2150. */
  2151. static int origin_ctr(struct dm_target *ti, unsigned int argc, char **argv)
  2152. {
  2153. int r;
  2154. struct dm_origin *o;
  2155. if (argc != 1) {
  2156. ti->error = "origin: incorrect number of arguments";
  2157. return -EINVAL;
  2158. }
  2159. o = kmalloc(sizeof(struct dm_origin), GFP_KERNEL);
  2160. if (!o) {
  2161. ti->error = "Cannot allocate private origin structure";
  2162. r = -ENOMEM;
  2163. goto bad_alloc;
  2164. }
  2165. r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &o->dev);
  2166. if (r) {
  2167. ti->error = "Cannot get target device";
  2168. goto bad_open;
  2169. }
  2170. o->ti = ti;
  2171. ti->private = o;
  2172. ti->num_flush_bios = 1;
  2173. return 0;
  2174. bad_open:
  2175. kfree(o);
  2176. bad_alloc:
  2177. return r;
  2178. }
  2179. static void origin_dtr(struct dm_target *ti)
  2180. {
  2181. struct dm_origin *o = ti->private;
  2182. dm_put_device(ti, o->dev);
  2183. kfree(o);
  2184. }
  2185. static int origin_map(struct dm_target *ti, struct bio *bio)
  2186. {
  2187. struct dm_origin *o = ti->private;
  2188. unsigned int available_sectors;
  2189. bio_set_dev(bio, o->dev->bdev);
  2190. if (unlikely(bio->bi_opf & REQ_PREFLUSH))
  2191. return DM_MAPIO_REMAPPED;
  2192. if (bio_data_dir(bio) != WRITE)
  2193. return DM_MAPIO_REMAPPED;
  2194. available_sectors = o->split_boundary -
  2195. ((unsigned int)bio->bi_iter.bi_sector & (o->split_boundary - 1));
  2196. if (bio_sectors(bio) > available_sectors)
  2197. dm_accept_partial_bio(bio, available_sectors);
  2198. /* Only tell snapshots if this is a write */
  2199. return do_origin(o->dev, bio, true);
  2200. }
  2201. /*
  2202. * Set the target "max_io_len" field to the minimum of all the snapshots'
  2203. * chunk sizes.
  2204. */
  2205. static void origin_resume(struct dm_target *ti)
  2206. {
  2207. struct dm_origin *o = ti->private;
  2208. o->split_boundary = get_origin_minimum_chunksize(o->dev->bdev);
  2209. down_write(&_origins_lock);
  2210. __insert_dm_origin(o);
  2211. up_write(&_origins_lock);
  2212. }
  2213. static void origin_postsuspend(struct dm_target *ti)
  2214. {
  2215. struct dm_origin *o = ti->private;
  2216. down_write(&_origins_lock);
  2217. __remove_dm_origin(o);
  2218. up_write(&_origins_lock);
  2219. }
  2220. static void origin_status(struct dm_target *ti, status_type_t type,
  2221. unsigned int status_flags, char *result, unsigned int maxlen)
  2222. {
  2223. struct dm_origin *o = ti->private;
  2224. switch (type) {
  2225. case STATUSTYPE_INFO:
  2226. result[0] = '\0';
  2227. break;
  2228. case STATUSTYPE_TABLE:
  2229. snprintf(result, maxlen, "%s", o->dev->name);
  2230. break;
  2231. case STATUSTYPE_IMA:
  2232. result[0] = '\0';
  2233. break;
  2234. }
  2235. }
  2236. static int origin_iterate_devices(struct dm_target *ti,
  2237. iterate_devices_callout_fn fn, void *data)
  2238. {
  2239. struct dm_origin *o = ti->private;
  2240. return fn(ti, o->dev, 0, ti->len, data);
  2241. }
  2242. static struct target_type origin_target = {
  2243. .name = "snapshot-origin",
  2244. .version = {1, 9, 0},
  2245. .module = THIS_MODULE,
  2246. .ctr = origin_ctr,
  2247. .dtr = origin_dtr,
  2248. .map = origin_map,
  2249. .resume = origin_resume,
  2250. .postsuspend = origin_postsuspend,
  2251. .status = origin_status,
  2252. .iterate_devices = origin_iterate_devices,
  2253. };
  2254. static struct target_type snapshot_target = {
  2255. .name = "snapshot",
  2256. .version = {1, 16, 0},
  2257. .module = THIS_MODULE,
  2258. .ctr = snapshot_ctr,
  2259. .dtr = snapshot_dtr,
  2260. .map = snapshot_map,
  2261. .end_io = snapshot_end_io,
  2262. .preresume = snapshot_preresume,
  2263. .resume = snapshot_resume,
  2264. .status = snapshot_status,
  2265. .iterate_devices = snapshot_iterate_devices,
  2266. .io_hints = snapshot_io_hints,
  2267. };
  2268. static struct target_type merge_target = {
  2269. .name = dm_snapshot_merge_target_name,
  2270. .version = {1, 5, 0},
  2271. .module = THIS_MODULE,
  2272. .ctr = snapshot_ctr,
  2273. .dtr = snapshot_dtr,
  2274. .map = snapshot_merge_map,
  2275. .end_io = snapshot_end_io,
  2276. .presuspend = snapshot_merge_presuspend,
  2277. .preresume = snapshot_preresume,
  2278. .resume = snapshot_merge_resume,
  2279. .status = snapshot_status,
  2280. .iterate_devices = snapshot_iterate_devices,
  2281. .io_hints = snapshot_io_hints,
  2282. };
  2283. static int __init dm_snapshot_init(void)
  2284. {
  2285. int r;
  2286. r = dm_exception_store_init();
  2287. if (r) {
  2288. DMERR("Failed to initialize exception stores");
  2289. return r;
  2290. }
  2291. r = init_origin_hash();
  2292. if (r) {
  2293. DMERR("init_origin_hash failed.");
  2294. goto bad_origin_hash;
  2295. }
  2296. exception_cache = KMEM_CACHE(dm_exception, 0);
  2297. if (!exception_cache) {
  2298. DMERR("Couldn't create exception cache.");
  2299. r = -ENOMEM;
  2300. goto bad_exception_cache;
  2301. }
  2302. pending_cache = KMEM_CACHE(dm_snap_pending_exception, 0);
  2303. if (!pending_cache) {
  2304. DMERR("Couldn't create pending cache.");
  2305. r = -ENOMEM;
  2306. goto bad_pending_cache;
  2307. }
  2308. r = dm_register_target(&snapshot_target);
  2309. if (r < 0) {
  2310. DMERR("snapshot target register failed %d", r);
  2311. goto bad_register_snapshot_target;
  2312. }
  2313. r = dm_register_target(&origin_target);
  2314. if (r < 0) {
  2315. DMERR("Origin target register failed %d", r);
  2316. goto bad_register_origin_target;
  2317. }
  2318. r = dm_register_target(&merge_target);
  2319. if (r < 0) {
  2320. DMERR("Merge target register failed %d", r);
  2321. goto bad_register_merge_target;
  2322. }
  2323. return 0;
  2324. bad_register_merge_target:
  2325. dm_unregister_target(&origin_target);
  2326. bad_register_origin_target:
  2327. dm_unregister_target(&snapshot_target);
  2328. bad_register_snapshot_target:
  2329. kmem_cache_destroy(pending_cache);
  2330. bad_pending_cache:
  2331. kmem_cache_destroy(exception_cache);
  2332. bad_exception_cache:
  2333. exit_origin_hash();
  2334. bad_origin_hash:
  2335. dm_exception_store_exit();
  2336. return r;
  2337. }
  2338. static void __exit dm_snapshot_exit(void)
  2339. {
  2340. dm_unregister_target(&snapshot_target);
  2341. dm_unregister_target(&origin_target);
  2342. dm_unregister_target(&merge_target);
  2343. exit_origin_hash();
  2344. kmem_cache_destroy(pending_cache);
  2345. kmem_cache_destroy(exception_cache);
  2346. dm_exception_store_exit();
  2347. }
  2348. /* Module hooks */
  2349. module_init(dm_snapshot_init);
  2350. module_exit(dm_snapshot_exit);
  2351. MODULE_DESCRIPTION(DM_NAME " snapshot target");
  2352. MODULE_AUTHOR("Joe Thornber");
  2353. MODULE_LICENSE("GPL");
  2354. MODULE_ALIAS("dm-snapshot-origin");
  2355. MODULE_ALIAS("dm-snapshot-merge");