bus.c 36 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499
  1. // SPDX-License-Identifier: GPL-2.0
  2. /* Copyright(c) 2017-2018 Intel Corporation. All rights reserved. */
  3. #include <linux/memremap.h>
  4. #include <linux/device.h>
  5. #include <linux/mutex.h>
  6. #include <linux/list.h>
  7. #include <linux/slab.h>
  8. #include <linux/dax.h>
  9. #include <linux/io.h>
  10. #include "dax-private.h"
  11. #include "bus.h"
  12. static DEFINE_MUTEX(dax_bus_lock);
  13. #define DAX_NAME_LEN 30
  14. struct dax_id {
  15. struct list_head list;
  16. char dev_name[DAX_NAME_LEN];
  17. };
  18. static int dax_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
  19. {
  20. /*
  21. * We only ever expect to handle device-dax instances, i.e. the
  22. * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
  23. */
  24. return add_uevent_var(env, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT, 0);
  25. }
  26. static struct dax_device_driver *to_dax_drv(struct device_driver *drv)
  27. {
  28. return container_of(drv, struct dax_device_driver, drv);
  29. }
  30. static struct dax_id *__dax_match_id(struct dax_device_driver *dax_drv,
  31. const char *dev_name)
  32. {
  33. struct dax_id *dax_id;
  34. lockdep_assert_held(&dax_bus_lock);
  35. list_for_each_entry(dax_id, &dax_drv->ids, list)
  36. if (sysfs_streq(dax_id->dev_name, dev_name))
  37. return dax_id;
  38. return NULL;
  39. }
  40. static int dax_match_id(struct dax_device_driver *dax_drv, struct device *dev)
  41. {
  42. int match;
  43. mutex_lock(&dax_bus_lock);
  44. match = !!__dax_match_id(dax_drv, dev_name(dev));
  45. mutex_unlock(&dax_bus_lock);
  46. return match;
  47. }
  48. enum id_action {
  49. ID_REMOVE,
  50. ID_ADD,
  51. };
  52. static ssize_t do_id_store(struct device_driver *drv, const char *buf,
  53. size_t count, enum id_action action)
  54. {
  55. struct dax_device_driver *dax_drv = to_dax_drv(drv);
  56. unsigned int region_id, id;
  57. char devname[DAX_NAME_LEN];
  58. struct dax_id *dax_id;
  59. ssize_t rc = count;
  60. int fields;
  61. fields = sscanf(buf, "dax%d.%d", &region_id, &id);
  62. if (fields != 2)
  63. return -EINVAL;
  64. sprintf(devname, "dax%d.%d", region_id, id);
  65. if (!sysfs_streq(buf, devname))
  66. return -EINVAL;
  67. mutex_lock(&dax_bus_lock);
  68. dax_id = __dax_match_id(dax_drv, buf);
  69. if (!dax_id) {
  70. if (action == ID_ADD) {
  71. dax_id = kzalloc(sizeof(*dax_id), GFP_KERNEL);
  72. if (dax_id) {
  73. strncpy(dax_id->dev_name, buf, DAX_NAME_LEN);
  74. list_add(&dax_id->list, &dax_drv->ids);
  75. } else
  76. rc = -ENOMEM;
  77. }
  78. } else if (action == ID_REMOVE) {
  79. list_del(&dax_id->list);
  80. kfree(dax_id);
  81. }
  82. mutex_unlock(&dax_bus_lock);
  83. if (rc < 0)
  84. return rc;
  85. if (action == ID_ADD)
  86. rc = driver_attach(drv);
  87. if (rc)
  88. return rc;
  89. return count;
  90. }
  91. static ssize_t new_id_store(struct device_driver *drv, const char *buf,
  92. size_t count)
  93. {
  94. return do_id_store(drv, buf, count, ID_ADD);
  95. }
  96. static DRIVER_ATTR_WO(new_id);
  97. static ssize_t remove_id_store(struct device_driver *drv, const char *buf,
  98. size_t count)
  99. {
  100. return do_id_store(drv, buf, count, ID_REMOVE);
  101. }
  102. static DRIVER_ATTR_WO(remove_id);
  103. static struct attribute *dax_drv_attrs[] = {
  104. &driver_attr_new_id.attr,
  105. &driver_attr_remove_id.attr,
  106. NULL,
  107. };
  108. ATTRIBUTE_GROUPS(dax_drv);
  109. static int dax_bus_match(struct device *dev, struct device_driver *drv);
  110. /*
  111. * Static dax regions are regions created by an external subsystem
  112. * nvdimm where a single range is assigned. Its boundaries are by the external
  113. * subsystem and are usually limited to one physical memory range. For example,
  114. * for PMEM it is usually defined by NVDIMM Namespace boundaries (i.e. a
  115. * single contiguous range)
  116. *
  117. * On dynamic dax regions, the assigned region can be partitioned by dax core
  118. * into multiple subdivisions. A subdivision is represented into one
  119. * /dev/daxN.M device composed by one or more potentially discontiguous ranges.
  120. *
  121. * When allocating a dax region, drivers must set whether it's static
  122. * (IORESOURCE_DAX_STATIC). On static dax devices, the @pgmap is pre-assigned
  123. * to dax core when calling devm_create_dev_dax(), whereas in dynamic dax
  124. * devices it is NULL but afterwards allocated by dax core on device ->probe().
  125. * Care is needed to make sure that dynamic dax devices are torn down with a
  126. * cleared @pgmap field (see kill_dev_dax()).
  127. */
  128. static bool is_static(struct dax_region *dax_region)
  129. {
  130. return (dax_region->res.flags & IORESOURCE_DAX_STATIC) != 0;
  131. }
  132. bool static_dev_dax(struct dev_dax *dev_dax)
  133. {
  134. return is_static(dev_dax->region);
  135. }
  136. EXPORT_SYMBOL_GPL(static_dev_dax);
  137. static u64 dev_dax_size(struct dev_dax *dev_dax)
  138. {
  139. u64 size = 0;
  140. int i;
  141. device_lock_assert(&dev_dax->dev);
  142. for (i = 0; i < dev_dax->nr_range; i++)
  143. size += range_len(&dev_dax->ranges[i].range);
  144. return size;
  145. }
  146. static int dax_bus_probe(struct device *dev)
  147. {
  148. struct dax_device_driver *dax_drv = to_dax_drv(dev->driver);
  149. struct dev_dax *dev_dax = to_dev_dax(dev);
  150. struct dax_region *dax_region = dev_dax->region;
  151. int rc;
  152. if (dev_dax_size(dev_dax) == 0 || dev_dax->id < 0)
  153. return -ENXIO;
  154. rc = dax_drv->probe(dev_dax);
  155. if (rc || is_static(dax_region))
  156. return rc;
  157. /*
  158. * Track new seed creation only after successful probe of the
  159. * previous seed.
  160. */
  161. if (dax_region->seed == dev)
  162. dax_region->seed = NULL;
  163. return 0;
  164. }
  165. static void dax_bus_remove(struct device *dev)
  166. {
  167. struct dax_device_driver *dax_drv = to_dax_drv(dev->driver);
  168. struct dev_dax *dev_dax = to_dev_dax(dev);
  169. if (dax_drv->remove)
  170. dax_drv->remove(dev_dax);
  171. }
  172. static struct bus_type dax_bus_type = {
  173. .name = "dax",
  174. .uevent = dax_bus_uevent,
  175. .match = dax_bus_match,
  176. .probe = dax_bus_probe,
  177. .remove = dax_bus_remove,
  178. .drv_groups = dax_drv_groups,
  179. };
  180. static int dax_bus_match(struct device *dev, struct device_driver *drv)
  181. {
  182. struct dax_device_driver *dax_drv = to_dax_drv(drv);
  183. /*
  184. * All but the 'device-dax' driver, which has 'match_always'
  185. * set, requires an exact id match.
  186. */
  187. if (dax_drv->match_always)
  188. return 1;
  189. return dax_match_id(dax_drv, dev);
  190. }
  191. /*
  192. * Rely on the fact that drvdata is set before the attributes are
  193. * registered, and that the attributes are unregistered before drvdata
  194. * is cleared to assume that drvdata is always valid.
  195. */
  196. static ssize_t id_show(struct device *dev,
  197. struct device_attribute *attr, char *buf)
  198. {
  199. struct dax_region *dax_region = dev_get_drvdata(dev);
  200. return sprintf(buf, "%d\n", dax_region->id);
  201. }
  202. static DEVICE_ATTR_RO(id);
  203. static ssize_t region_size_show(struct device *dev,
  204. struct device_attribute *attr, char *buf)
  205. {
  206. struct dax_region *dax_region = dev_get_drvdata(dev);
  207. return sprintf(buf, "%llu\n", (unsigned long long)
  208. resource_size(&dax_region->res));
  209. }
  210. static struct device_attribute dev_attr_region_size = __ATTR(size, 0444,
  211. region_size_show, NULL);
  212. static ssize_t region_align_show(struct device *dev,
  213. struct device_attribute *attr, char *buf)
  214. {
  215. struct dax_region *dax_region = dev_get_drvdata(dev);
  216. return sprintf(buf, "%u\n", dax_region->align);
  217. }
  218. static struct device_attribute dev_attr_region_align =
  219. __ATTR(align, 0400, region_align_show, NULL);
  220. #define for_each_dax_region_resource(dax_region, res) \
  221. for (res = (dax_region)->res.child; res; res = res->sibling)
  222. static unsigned long long dax_region_avail_size(struct dax_region *dax_region)
  223. {
  224. resource_size_t size = resource_size(&dax_region->res);
  225. struct resource *res;
  226. device_lock_assert(dax_region->dev);
  227. for_each_dax_region_resource(dax_region, res)
  228. size -= resource_size(res);
  229. return size;
  230. }
  231. static ssize_t available_size_show(struct device *dev,
  232. struct device_attribute *attr, char *buf)
  233. {
  234. struct dax_region *dax_region = dev_get_drvdata(dev);
  235. unsigned long long size;
  236. device_lock(dev);
  237. size = dax_region_avail_size(dax_region);
  238. device_unlock(dev);
  239. return sprintf(buf, "%llu\n", size);
  240. }
  241. static DEVICE_ATTR_RO(available_size);
  242. static ssize_t seed_show(struct device *dev,
  243. struct device_attribute *attr, char *buf)
  244. {
  245. struct dax_region *dax_region = dev_get_drvdata(dev);
  246. struct device *seed;
  247. ssize_t rc;
  248. if (is_static(dax_region))
  249. return -EINVAL;
  250. device_lock(dev);
  251. seed = dax_region->seed;
  252. rc = sprintf(buf, "%s\n", seed ? dev_name(seed) : "");
  253. device_unlock(dev);
  254. return rc;
  255. }
  256. static DEVICE_ATTR_RO(seed);
  257. static ssize_t create_show(struct device *dev,
  258. struct device_attribute *attr, char *buf)
  259. {
  260. struct dax_region *dax_region = dev_get_drvdata(dev);
  261. struct device *youngest;
  262. ssize_t rc;
  263. if (is_static(dax_region))
  264. return -EINVAL;
  265. device_lock(dev);
  266. youngest = dax_region->youngest;
  267. rc = sprintf(buf, "%s\n", youngest ? dev_name(youngest) : "");
  268. device_unlock(dev);
  269. return rc;
  270. }
  271. static ssize_t create_store(struct device *dev, struct device_attribute *attr,
  272. const char *buf, size_t len)
  273. {
  274. struct dax_region *dax_region = dev_get_drvdata(dev);
  275. unsigned long long avail;
  276. ssize_t rc;
  277. int val;
  278. if (is_static(dax_region))
  279. return -EINVAL;
  280. rc = kstrtoint(buf, 0, &val);
  281. if (rc)
  282. return rc;
  283. if (val != 1)
  284. return -EINVAL;
  285. device_lock(dev);
  286. avail = dax_region_avail_size(dax_region);
  287. if (avail == 0)
  288. rc = -ENOSPC;
  289. else {
  290. struct dev_dax_data data = {
  291. .dax_region = dax_region,
  292. .size = 0,
  293. .id = -1,
  294. };
  295. struct dev_dax *dev_dax = devm_create_dev_dax(&data);
  296. if (IS_ERR(dev_dax))
  297. rc = PTR_ERR(dev_dax);
  298. else {
  299. /*
  300. * In support of crafting multiple new devices
  301. * simultaneously multiple seeds can be created,
  302. * but only the first one that has not been
  303. * successfully bound is tracked as the region
  304. * seed.
  305. */
  306. if (!dax_region->seed)
  307. dax_region->seed = &dev_dax->dev;
  308. dax_region->youngest = &dev_dax->dev;
  309. rc = len;
  310. }
  311. }
  312. device_unlock(dev);
  313. return rc;
  314. }
  315. static DEVICE_ATTR_RW(create);
  316. void kill_dev_dax(struct dev_dax *dev_dax)
  317. {
  318. struct dax_device *dax_dev = dev_dax->dax_dev;
  319. struct inode *inode = dax_inode(dax_dev);
  320. kill_dax(dax_dev);
  321. unmap_mapping_range(inode->i_mapping, 0, 0, 1);
  322. /*
  323. * Dynamic dax region have the pgmap allocated via dev_kzalloc()
  324. * and thus freed by devm. Clear the pgmap to not have stale pgmap
  325. * ranges on probe() from previous reconfigurations of region devices.
  326. */
  327. if (!static_dev_dax(dev_dax))
  328. dev_dax->pgmap = NULL;
  329. }
  330. EXPORT_SYMBOL_GPL(kill_dev_dax);
  331. static void trim_dev_dax_range(struct dev_dax *dev_dax)
  332. {
  333. int i = dev_dax->nr_range - 1;
  334. struct range *range = &dev_dax->ranges[i].range;
  335. struct dax_region *dax_region = dev_dax->region;
  336. device_lock_assert(dax_region->dev);
  337. dev_dbg(&dev_dax->dev, "delete range[%d]: %#llx:%#llx\n", i,
  338. (unsigned long long)range->start,
  339. (unsigned long long)range->end);
  340. __release_region(&dax_region->res, range->start, range_len(range));
  341. if (--dev_dax->nr_range == 0) {
  342. kfree(dev_dax->ranges);
  343. dev_dax->ranges = NULL;
  344. }
  345. }
  346. static void free_dev_dax_ranges(struct dev_dax *dev_dax)
  347. {
  348. while (dev_dax->nr_range)
  349. trim_dev_dax_range(dev_dax);
  350. }
  351. static void unregister_dev_dax(void *dev)
  352. {
  353. struct dev_dax *dev_dax = to_dev_dax(dev);
  354. dev_dbg(dev, "%s\n", __func__);
  355. kill_dev_dax(dev_dax);
  356. device_del(dev);
  357. free_dev_dax_ranges(dev_dax);
  358. put_device(dev);
  359. }
  360. static void dax_region_free(struct kref *kref)
  361. {
  362. struct dax_region *dax_region;
  363. dax_region = container_of(kref, struct dax_region, kref);
  364. kfree(dax_region);
  365. }
  366. void dax_region_put(struct dax_region *dax_region)
  367. {
  368. kref_put(&dax_region->kref, dax_region_free);
  369. }
  370. EXPORT_SYMBOL_GPL(dax_region_put);
  371. /* a return value >= 0 indicates this invocation invalidated the id */
  372. static int __free_dev_dax_id(struct dev_dax *dev_dax)
  373. {
  374. struct device *dev = &dev_dax->dev;
  375. struct dax_region *dax_region;
  376. int rc = dev_dax->id;
  377. device_lock_assert(dev);
  378. if (!dev_dax->dyn_id || dev_dax->id < 0)
  379. return -1;
  380. dax_region = dev_dax->region;
  381. ida_free(&dax_region->ida, dev_dax->id);
  382. dax_region_put(dax_region);
  383. dev_dax->id = -1;
  384. return rc;
  385. }
  386. static int free_dev_dax_id(struct dev_dax *dev_dax)
  387. {
  388. struct device *dev = &dev_dax->dev;
  389. int rc;
  390. device_lock(dev);
  391. rc = __free_dev_dax_id(dev_dax);
  392. device_unlock(dev);
  393. return rc;
  394. }
  395. static int alloc_dev_dax_id(struct dev_dax *dev_dax)
  396. {
  397. struct dax_region *dax_region = dev_dax->region;
  398. int id;
  399. id = ida_alloc(&dax_region->ida, GFP_KERNEL);
  400. if (id < 0)
  401. return id;
  402. kref_get(&dax_region->kref);
  403. dev_dax->dyn_id = true;
  404. dev_dax->id = id;
  405. return id;
  406. }
  407. static ssize_t delete_store(struct device *dev, struct device_attribute *attr,
  408. const char *buf, size_t len)
  409. {
  410. struct dax_region *dax_region = dev_get_drvdata(dev);
  411. struct dev_dax *dev_dax;
  412. struct device *victim;
  413. bool do_del = false;
  414. int rc;
  415. if (is_static(dax_region))
  416. return -EINVAL;
  417. victim = device_find_child_by_name(dax_region->dev, buf);
  418. if (!victim)
  419. return -ENXIO;
  420. device_lock(dev);
  421. device_lock(victim);
  422. dev_dax = to_dev_dax(victim);
  423. if (victim->driver || dev_dax_size(dev_dax))
  424. rc = -EBUSY;
  425. else {
  426. /*
  427. * Invalidate the device so it does not become active
  428. * again, but always preserve device-id-0 so that
  429. * /sys/bus/dax/ is guaranteed to be populated while any
  430. * dax_region is registered.
  431. */
  432. if (dev_dax->id > 0) {
  433. do_del = __free_dev_dax_id(dev_dax) >= 0;
  434. rc = len;
  435. if (dax_region->seed == victim)
  436. dax_region->seed = NULL;
  437. if (dax_region->youngest == victim)
  438. dax_region->youngest = NULL;
  439. } else
  440. rc = -EBUSY;
  441. }
  442. device_unlock(victim);
  443. /* won the race to invalidate the device, clean it up */
  444. if (do_del)
  445. devm_release_action(dev, unregister_dev_dax, victim);
  446. device_unlock(dev);
  447. put_device(victim);
  448. return rc;
  449. }
  450. static DEVICE_ATTR_WO(delete);
  451. static umode_t dax_region_visible(struct kobject *kobj, struct attribute *a,
  452. int n)
  453. {
  454. struct device *dev = container_of(kobj, struct device, kobj);
  455. struct dax_region *dax_region = dev_get_drvdata(dev);
  456. if (is_static(dax_region))
  457. if (a == &dev_attr_available_size.attr
  458. || a == &dev_attr_create.attr
  459. || a == &dev_attr_seed.attr
  460. || a == &dev_attr_delete.attr)
  461. return 0;
  462. return a->mode;
  463. }
  464. static struct attribute *dax_region_attributes[] = {
  465. &dev_attr_available_size.attr,
  466. &dev_attr_region_size.attr,
  467. &dev_attr_region_align.attr,
  468. &dev_attr_create.attr,
  469. &dev_attr_seed.attr,
  470. &dev_attr_delete.attr,
  471. &dev_attr_id.attr,
  472. NULL,
  473. };
  474. static const struct attribute_group dax_region_attribute_group = {
  475. .name = "dax_region",
  476. .attrs = dax_region_attributes,
  477. .is_visible = dax_region_visible,
  478. };
  479. static const struct attribute_group *dax_region_attribute_groups[] = {
  480. &dax_region_attribute_group,
  481. NULL,
  482. };
  483. static void dax_region_unregister(void *region)
  484. {
  485. struct dax_region *dax_region = region;
  486. sysfs_remove_groups(&dax_region->dev->kobj,
  487. dax_region_attribute_groups);
  488. dax_region_put(dax_region);
  489. }
  490. struct dax_region *alloc_dax_region(struct device *parent, int region_id,
  491. struct range *range, int target_node, unsigned int align,
  492. unsigned long flags)
  493. {
  494. struct dax_region *dax_region;
  495. /*
  496. * The DAX core assumes that it can store its private data in
  497. * parent->driver_data. This WARN is a reminder / safeguard for
  498. * developers of device-dax drivers.
  499. */
  500. if (dev_get_drvdata(parent)) {
  501. dev_WARN(parent, "dax core failed to setup private data\n");
  502. return NULL;
  503. }
  504. if (!IS_ALIGNED(range->start, align)
  505. || !IS_ALIGNED(range_len(range), align))
  506. return NULL;
  507. dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL);
  508. if (!dax_region)
  509. return NULL;
  510. dev_set_drvdata(parent, dax_region);
  511. kref_init(&dax_region->kref);
  512. dax_region->id = region_id;
  513. dax_region->align = align;
  514. dax_region->dev = parent;
  515. dax_region->target_node = target_node;
  516. ida_init(&dax_region->ida);
  517. dax_region->res = (struct resource) {
  518. .start = range->start,
  519. .end = range->end,
  520. .flags = IORESOURCE_MEM | flags,
  521. };
  522. if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) {
  523. kfree(dax_region);
  524. return NULL;
  525. }
  526. kref_get(&dax_region->kref);
  527. if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region))
  528. return NULL;
  529. return dax_region;
  530. }
  531. EXPORT_SYMBOL_GPL(alloc_dax_region);
  532. static void dax_mapping_release(struct device *dev)
  533. {
  534. struct dax_mapping *mapping = to_dax_mapping(dev);
  535. struct device *parent = dev->parent;
  536. struct dev_dax *dev_dax = to_dev_dax(parent);
  537. ida_free(&dev_dax->ida, mapping->id);
  538. kfree(mapping);
  539. put_device(parent);
  540. }
  541. static void unregister_dax_mapping(void *data)
  542. {
  543. struct device *dev = data;
  544. struct dax_mapping *mapping = to_dax_mapping(dev);
  545. struct dev_dax *dev_dax = to_dev_dax(dev->parent);
  546. struct dax_region *dax_region = dev_dax->region;
  547. dev_dbg(dev, "%s\n", __func__);
  548. device_lock_assert(dax_region->dev);
  549. dev_dax->ranges[mapping->range_id].mapping = NULL;
  550. mapping->range_id = -1;
  551. device_del(dev);
  552. put_device(dev);
  553. }
  554. static struct dev_dax_range *get_dax_range(struct device *dev)
  555. {
  556. struct dax_mapping *mapping = to_dax_mapping(dev);
  557. struct dev_dax *dev_dax = to_dev_dax(dev->parent);
  558. struct dax_region *dax_region = dev_dax->region;
  559. device_lock(dax_region->dev);
  560. if (mapping->range_id < 0) {
  561. device_unlock(dax_region->dev);
  562. return NULL;
  563. }
  564. return &dev_dax->ranges[mapping->range_id];
  565. }
  566. static void put_dax_range(struct dev_dax_range *dax_range)
  567. {
  568. struct dax_mapping *mapping = dax_range->mapping;
  569. struct dev_dax *dev_dax = to_dev_dax(mapping->dev.parent);
  570. struct dax_region *dax_region = dev_dax->region;
  571. device_unlock(dax_region->dev);
  572. }
  573. static ssize_t start_show(struct device *dev,
  574. struct device_attribute *attr, char *buf)
  575. {
  576. struct dev_dax_range *dax_range;
  577. ssize_t rc;
  578. dax_range = get_dax_range(dev);
  579. if (!dax_range)
  580. return -ENXIO;
  581. rc = sprintf(buf, "%#llx\n", dax_range->range.start);
  582. put_dax_range(dax_range);
  583. return rc;
  584. }
  585. static DEVICE_ATTR(start, 0400, start_show, NULL);
  586. static ssize_t end_show(struct device *dev,
  587. struct device_attribute *attr, char *buf)
  588. {
  589. struct dev_dax_range *dax_range;
  590. ssize_t rc;
  591. dax_range = get_dax_range(dev);
  592. if (!dax_range)
  593. return -ENXIO;
  594. rc = sprintf(buf, "%#llx\n", dax_range->range.end);
  595. put_dax_range(dax_range);
  596. return rc;
  597. }
  598. static DEVICE_ATTR(end, 0400, end_show, NULL);
  599. static ssize_t pgoff_show(struct device *dev,
  600. struct device_attribute *attr, char *buf)
  601. {
  602. struct dev_dax_range *dax_range;
  603. ssize_t rc;
  604. dax_range = get_dax_range(dev);
  605. if (!dax_range)
  606. return -ENXIO;
  607. rc = sprintf(buf, "%#lx\n", dax_range->pgoff);
  608. put_dax_range(dax_range);
  609. return rc;
  610. }
  611. static DEVICE_ATTR(page_offset, 0400, pgoff_show, NULL);
  612. static struct attribute *dax_mapping_attributes[] = {
  613. &dev_attr_start.attr,
  614. &dev_attr_end.attr,
  615. &dev_attr_page_offset.attr,
  616. NULL,
  617. };
  618. static const struct attribute_group dax_mapping_attribute_group = {
  619. .attrs = dax_mapping_attributes,
  620. };
  621. static const struct attribute_group *dax_mapping_attribute_groups[] = {
  622. &dax_mapping_attribute_group,
  623. NULL,
  624. };
  625. static struct device_type dax_mapping_type = {
  626. .release = dax_mapping_release,
  627. .groups = dax_mapping_attribute_groups,
  628. };
  629. static int devm_register_dax_mapping(struct dev_dax *dev_dax, int range_id)
  630. {
  631. struct dax_region *dax_region = dev_dax->region;
  632. struct dax_mapping *mapping;
  633. struct device *dev;
  634. int rc;
  635. device_lock_assert(dax_region->dev);
  636. if (dev_WARN_ONCE(&dev_dax->dev, !dax_region->dev->driver,
  637. "region disabled\n"))
  638. return -ENXIO;
  639. mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
  640. if (!mapping)
  641. return -ENOMEM;
  642. mapping->range_id = range_id;
  643. mapping->id = ida_alloc(&dev_dax->ida, GFP_KERNEL);
  644. if (mapping->id < 0) {
  645. kfree(mapping);
  646. return -ENOMEM;
  647. }
  648. dev_dax->ranges[range_id].mapping = mapping;
  649. dev = &mapping->dev;
  650. device_initialize(dev);
  651. dev->parent = &dev_dax->dev;
  652. get_device(dev->parent);
  653. dev->type = &dax_mapping_type;
  654. dev_set_name(dev, "mapping%d", mapping->id);
  655. rc = device_add(dev);
  656. if (rc) {
  657. put_device(dev);
  658. return rc;
  659. }
  660. rc = devm_add_action_or_reset(dax_region->dev, unregister_dax_mapping,
  661. dev);
  662. if (rc)
  663. return rc;
  664. return 0;
  665. }
  666. static int alloc_dev_dax_range(struct dev_dax *dev_dax, u64 start,
  667. resource_size_t size)
  668. {
  669. struct dax_region *dax_region = dev_dax->region;
  670. struct resource *res = &dax_region->res;
  671. struct device *dev = &dev_dax->dev;
  672. struct dev_dax_range *ranges;
  673. unsigned long pgoff = 0;
  674. struct resource *alloc;
  675. int i, rc;
  676. device_lock_assert(dax_region->dev);
  677. /* handle the seed alloc special case */
  678. if (!size) {
  679. if (dev_WARN_ONCE(dev, dev_dax->nr_range,
  680. "0-size allocation must be first\n"))
  681. return -EBUSY;
  682. /* nr_range == 0 is elsewhere special cased as 0-size device */
  683. return 0;
  684. }
  685. alloc = __request_region(res, start, size, dev_name(dev), 0);
  686. if (!alloc)
  687. return -ENOMEM;
  688. ranges = krealloc(dev_dax->ranges, sizeof(*ranges)
  689. * (dev_dax->nr_range + 1), GFP_KERNEL);
  690. if (!ranges) {
  691. __release_region(res, alloc->start, resource_size(alloc));
  692. return -ENOMEM;
  693. }
  694. for (i = 0; i < dev_dax->nr_range; i++)
  695. pgoff += PHYS_PFN(range_len(&ranges[i].range));
  696. dev_dax->ranges = ranges;
  697. ranges[dev_dax->nr_range++] = (struct dev_dax_range) {
  698. .pgoff = pgoff,
  699. .range = {
  700. .start = alloc->start,
  701. .end = alloc->end,
  702. },
  703. };
  704. dev_dbg(dev, "alloc range[%d]: %pa:%pa\n", dev_dax->nr_range - 1,
  705. &alloc->start, &alloc->end);
  706. /*
  707. * A dev_dax instance must be registered before mapping device
  708. * children can be added. Defer to devm_create_dev_dax() to add
  709. * the initial mapping device.
  710. */
  711. if (!device_is_registered(&dev_dax->dev))
  712. return 0;
  713. rc = devm_register_dax_mapping(dev_dax, dev_dax->nr_range - 1);
  714. if (rc)
  715. trim_dev_dax_range(dev_dax);
  716. return rc;
  717. }
  718. static int adjust_dev_dax_range(struct dev_dax *dev_dax, struct resource *res, resource_size_t size)
  719. {
  720. int last_range = dev_dax->nr_range - 1;
  721. struct dev_dax_range *dax_range = &dev_dax->ranges[last_range];
  722. struct dax_region *dax_region = dev_dax->region;
  723. bool is_shrink = resource_size(res) > size;
  724. struct range *range = &dax_range->range;
  725. struct device *dev = &dev_dax->dev;
  726. int rc;
  727. device_lock_assert(dax_region->dev);
  728. if (dev_WARN_ONCE(dev, !size, "deletion is handled by dev_dax_shrink\n"))
  729. return -EINVAL;
  730. rc = adjust_resource(res, range->start, size);
  731. if (rc)
  732. return rc;
  733. *range = (struct range) {
  734. .start = range->start,
  735. .end = range->start + size - 1,
  736. };
  737. dev_dbg(dev, "%s range[%d]: %#llx:%#llx\n", is_shrink ? "shrink" : "extend",
  738. last_range, (unsigned long long) range->start,
  739. (unsigned long long) range->end);
  740. return 0;
  741. }
  742. static ssize_t size_show(struct device *dev,
  743. struct device_attribute *attr, char *buf)
  744. {
  745. struct dev_dax *dev_dax = to_dev_dax(dev);
  746. unsigned long long size;
  747. device_lock(dev);
  748. size = dev_dax_size(dev_dax);
  749. device_unlock(dev);
  750. return sprintf(buf, "%llu\n", size);
  751. }
  752. static bool alloc_is_aligned(struct dev_dax *dev_dax, resource_size_t size)
  753. {
  754. /*
  755. * The minimum mapping granularity for a device instance is a
  756. * single subsection, unless the arch says otherwise.
  757. */
  758. return IS_ALIGNED(size, max_t(unsigned long, dev_dax->align, memremap_compat_align()));
  759. }
  760. static int dev_dax_shrink(struct dev_dax *dev_dax, resource_size_t size)
  761. {
  762. resource_size_t to_shrink = dev_dax_size(dev_dax) - size;
  763. struct dax_region *dax_region = dev_dax->region;
  764. struct device *dev = &dev_dax->dev;
  765. int i;
  766. for (i = dev_dax->nr_range - 1; i >= 0; i--) {
  767. struct range *range = &dev_dax->ranges[i].range;
  768. struct dax_mapping *mapping = dev_dax->ranges[i].mapping;
  769. struct resource *adjust = NULL, *res;
  770. resource_size_t shrink;
  771. shrink = min_t(u64, to_shrink, range_len(range));
  772. if (shrink >= range_len(range)) {
  773. devm_release_action(dax_region->dev,
  774. unregister_dax_mapping, &mapping->dev);
  775. trim_dev_dax_range(dev_dax);
  776. to_shrink -= shrink;
  777. if (!to_shrink)
  778. break;
  779. continue;
  780. }
  781. for_each_dax_region_resource(dax_region, res)
  782. if (strcmp(res->name, dev_name(dev)) == 0
  783. && res->start == range->start) {
  784. adjust = res;
  785. break;
  786. }
  787. if (dev_WARN_ONCE(dev, !adjust || i != dev_dax->nr_range - 1,
  788. "failed to find matching resource\n"))
  789. return -ENXIO;
  790. return adjust_dev_dax_range(dev_dax, adjust, range_len(range)
  791. - shrink);
  792. }
  793. return 0;
  794. }
  795. /*
  796. * Only allow adjustments that preserve the relative pgoff of existing
  797. * allocations. I.e. the dev_dax->ranges array is ordered by increasing pgoff.
  798. */
  799. static bool adjust_ok(struct dev_dax *dev_dax, struct resource *res)
  800. {
  801. struct dev_dax_range *last;
  802. int i;
  803. if (dev_dax->nr_range == 0)
  804. return false;
  805. if (strcmp(res->name, dev_name(&dev_dax->dev)) != 0)
  806. return false;
  807. last = &dev_dax->ranges[dev_dax->nr_range - 1];
  808. if (last->range.start != res->start || last->range.end != res->end)
  809. return false;
  810. for (i = 0; i < dev_dax->nr_range - 1; i++) {
  811. struct dev_dax_range *dax_range = &dev_dax->ranges[i];
  812. if (dax_range->pgoff > last->pgoff)
  813. return false;
  814. }
  815. return true;
  816. }
  817. static ssize_t dev_dax_resize(struct dax_region *dax_region,
  818. struct dev_dax *dev_dax, resource_size_t size)
  819. {
  820. resource_size_t avail = dax_region_avail_size(dax_region), to_alloc;
  821. resource_size_t dev_size = dev_dax_size(dev_dax);
  822. struct resource *region_res = &dax_region->res;
  823. struct device *dev = &dev_dax->dev;
  824. struct resource *res, *first;
  825. resource_size_t alloc = 0;
  826. int rc;
  827. if (dev->driver)
  828. return -EBUSY;
  829. if (size == dev_size)
  830. return 0;
  831. if (size > dev_size && size - dev_size > avail)
  832. return -ENOSPC;
  833. if (size < dev_size)
  834. return dev_dax_shrink(dev_dax, size);
  835. to_alloc = size - dev_size;
  836. if (dev_WARN_ONCE(dev, !alloc_is_aligned(dev_dax, to_alloc),
  837. "resize of %pa misaligned\n", &to_alloc))
  838. return -ENXIO;
  839. /*
  840. * Expand the device into the unused portion of the region. This
  841. * may involve adjusting the end of an existing resource, or
  842. * allocating a new resource.
  843. */
  844. retry:
  845. first = region_res->child;
  846. if (!first)
  847. return alloc_dev_dax_range(dev_dax, dax_region->res.start, to_alloc);
  848. rc = -ENOSPC;
  849. for (res = first; res; res = res->sibling) {
  850. struct resource *next = res->sibling;
  851. /* space at the beginning of the region */
  852. if (res == first && res->start > dax_region->res.start) {
  853. alloc = min(res->start - dax_region->res.start, to_alloc);
  854. rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, alloc);
  855. break;
  856. }
  857. alloc = 0;
  858. /* space between allocations */
  859. if (next && next->start > res->end + 1)
  860. alloc = min(next->start - (res->end + 1), to_alloc);
  861. /* space at the end of the region */
  862. if (!alloc && !next && res->end < region_res->end)
  863. alloc = min(region_res->end - res->end, to_alloc);
  864. if (!alloc)
  865. continue;
  866. if (adjust_ok(dev_dax, res)) {
  867. rc = adjust_dev_dax_range(dev_dax, res, resource_size(res) + alloc);
  868. break;
  869. }
  870. rc = alloc_dev_dax_range(dev_dax, res->end + 1, alloc);
  871. break;
  872. }
  873. if (rc)
  874. return rc;
  875. to_alloc -= alloc;
  876. if (to_alloc)
  877. goto retry;
  878. return 0;
  879. }
  880. static ssize_t size_store(struct device *dev, struct device_attribute *attr,
  881. const char *buf, size_t len)
  882. {
  883. ssize_t rc;
  884. unsigned long long val;
  885. struct dev_dax *dev_dax = to_dev_dax(dev);
  886. struct dax_region *dax_region = dev_dax->region;
  887. rc = kstrtoull(buf, 0, &val);
  888. if (rc)
  889. return rc;
  890. if (!alloc_is_aligned(dev_dax, val)) {
  891. dev_dbg(dev, "%s: size: %lld misaligned\n", __func__, val);
  892. return -EINVAL;
  893. }
  894. device_lock(dax_region->dev);
  895. if (!dax_region->dev->driver) {
  896. device_unlock(dax_region->dev);
  897. return -ENXIO;
  898. }
  899. device_lock(dev);
  900. rc = dev_dax_resize(dax_region, dev_dax, val);
  901. device_unlock(dev);
  902. device_unlock(dax_region->dev);
  903. return rc == 0 ? len : rc;
  904. }
  905. static DEVICE_ATTR_RW(size);
  906. static ssize_t range_parse(const char *opt, size_t len, struct range *range)
  907. {
  908. unsigned long long addr = 0;
  909. char *start, *end, *str;
  910. ssize_t rc = -EINVAL;
  911. str = kstrdup(opt, GFP_KERNEL);
  912. if (!str)
  913. return rc;
  914. end = str;
  915. start = strsep(&end, "-");
  916. if (!start || !end)
  917. goto err;
  918. rc = kstrtoull(start, 16, &addr);
  919. if (rc)
  920. goto err;
  921. range->start = addr;
  922. rc = kstrtoull(end, 16, &addr);
  923. if (rc)
  924. goto err;
  925. range->end = addr;
  926. err:
  927. kfree(str);
  928. return rc;
  929. }
  930. static ssize_t mapping_store(struct device *dev, struct device_attribute *attr,
  931. const char *buf, size_t len)
  932. {
  933. struct dev_dax *dev_dax = to_dev_dax(dev);
  934. struct dax_region *dax_region = dev_dax->region;
  935. size_t to_alloc;
  936. struct range r;
  937. ssize_t rc;
  938. rc = range_parse(buf, len, &r);
  939. if (rc)
  940. return rc;
  941. rc = -ENXIO;
  942. device_lock(dax_region->dev);
  943. if (!dax_region->dev->driver) {
  944. device_unlock(dax_region->dev);
  945. return rc;
  946. }
  947. device_lock(dev);
  948. to_alloc = range_len(&r);
  949. if (alloc_is_aligned(dev_dax, to_alloc))
  950. rc = alloc_dev_dax_range(dev_dax, r.start, to_alloc);
  951. device_unlock(dev);
  952. device_unlock(dax_region->dev);
  953. return rc == 0 ? len : rc;
  954. }
  955. static DEVICE_ATTR_WO(mapping);
  956. static ssize_t align_show(struct device *dev,
  957. struct device_attribute *attr, char *buf)
  958. {
  959. struct dev_dax *dev_dax = to_dev_dax(dev);
  960. return sprintf(buf, "%d\n", dev_dax->align);
  961. }
  962. static ssize_t dev_dax_validate_align(struct dev_dax *dev_dax)
  963. {
  964. struct device *dev = &dev_dax->dev;
  965. int i;
  966. for (i = 0; i < dev_dax->nr_range; i++) {
  967. size_t len = range_len(&dev_dax->ranges[i].range);
  968. if (!alloc_is_aligned(dev_dax, len)) {
  969. dev_dbg(dev, "%s: align %u invalid for range %d\n",
  970. __func__, dev_dax->align, i);
  971. return -EINVAL;
  972. }
  973. }
  974. return 0;
  975. }
  976. static ssize_t align_store(struct device *dev, struct device_attribute *attr,
  977. const char *buf, size_t len)
  978. {
  979. struct dev_dax *dev_dax = to_dev_dax(dev);
  980. struct dax_region *dax_region = dev_dax->region;
  981. unsigned long val, align_save;
  982. ssize_t rc;
  983. rc = kstrtoul(buf, 0, &val);
  984. if (rc)
  985. return -ENXIO;
  986. if (!dax_align_valid(val))
  987. return -EINVAL;
  988. device_lock(dax_region->dev);
  989. if (!dax_region->dev->driver) {
  990. device_unlock(dax_region->dev);
  991. return -ENXIO;
  992. }
  993. device_lock(dev);
  994. if (dev->driver) {
  995. rc = -EBUSY;
  996. goto out_unlock;
  997. }
  998. align_save = dev_dax->align;
  999. dev_dax->align = val;
  1000. rc = dev_dax_validate_align(dev_dax);
  1001. if (rc)
  1002. dev_dax->align = align_save;
  1003. out_unlock:
  1004. device_unlock(dev);
  1005. device_unlock(dax_region->dev);
  1006. return rc == 0 ? len : rc;
  1007. }
  1008. static DEVICE_ATTR_RW(align);
  1009. static int dev_dax_target_node(struct dev_dax *dev_dax)
  1010. {
  1011. struct dax_region *dax_region = dev_dax->region;
  1012. return dax_region->target_node;
  1013. }
  1014. static ssize_t target_node_show(struct device *dev,
  1015. struct device_attribute *attr, char *buf)
  1016. {
  1017. struct dev_dax *dev_dax = to_dev_dax(dev);
  1018. return sprintf(buf, "%d\n", dev_dax_target_node(dev_dax));
  1019. }
  1020. static DEVICE_ATTR_RO(target_node);
  1021. static ssize_t resource_show(struct device *dev,
  1022. struct device_attribute *attr, char *buf)
  1023. {
  1024. struct dev_dax *dev_dax = to_dev_dax(dev);
  1025. struct dax_region *dax_region = dev_dax->region;
  1026. unsigned long long start;
  1027. if (dev_dax->nr_range < 1)
  1028. start = dax_region->res.start;
  1029. else
  1030. start = dev_dax->ranges[0].range.start;
  1031. return sprintf(buf, "%#llx\n", start);
  1032. }
  1033. static DEVICE_ATTR(resource, 0400, resource_show, NULL);
  1034. static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
  1035. char *buf)
  1036. {
  1037. /*
  1038. * We only ever expect to handle device-dax instances, i.e. the
  1039. * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
  1040. */
  1041. return sprintf(buf, DAX_DEVICE_MODALIAS_FMT "\n", 0);
  1042. }
  1043. static DEVICE_ATTR_RO(modalias);
  1044. static ssize_t numa_node_show(struct device *dev,
  1045. struct device_attribute *attr, char *buf)
  1046. {
  1047. return sprintf(buf, "%d\n", dev_to_node(dev));
  1048. }
  1049. static DEVICE_ATTR_RO(numa_node);
  1050. static umode_t dev_dax_visible(struct kobject *kobj, struct attribute *a, int n)
  1051. {
  1052. struct device *dev = container_of(kobj, struct device, kobj);
  1053. struct dev_dax *dev_dax = to_dev_dax(dev);
  1054. struct dax_region *dax_region = dev_dax->region;
  1055. if (a == &dev_attr_target_node.attr && dev_dax_target_node(dev_dax) < 0)
  1056. return 0;
  1057. if (a == &dev_attr_numa_node.attr && !IS_ENABLED(CONFIG_NUMA))
  1058. return 0;
  1059. if (a == &dev_attr_mapping.attr && is_static(dax_region))
  1060. return 0;
  1061. if ((a == &dev_attr_align.attr ||
  1062. a == &dev_attr_size.attr) && is_static(dax_region))
  1063. return 0444;
  1064. return a->mode;
  1065. }
  1066. static struct attribute *dev_dax_attributes[] = {
  1067. &dev_attr_modalias.attr,
  1068. &dev_attr_size.attr,
  1069. &dev_attr_mapping.attr,
  1070. &dev_attr_target_node.attr,
  1071. &dev_attr_align.attr,
  1072. &dev_attr_resource.attr,
  1073. &dev_attr_numa_node.attr,
  1074. NULL,
  1075. };
  1076. static const struct attribute_group dev_dax_attribute_group = {
  1077. .attrs = dev_dax_attributes,
  1078. .is_visible = dev_dax_visible,
  1079. };
  1080. static const struct attribute_group *dax_attribute_groups[] = {
  1081. &dev_dax_attribute_group,
  1082. NULL,
  1083. };
  1084. static void dev_dax_release(struct device *dev)
  1085. {
  1086. struct dev_dax *dev_dax = to_dev_dax(dev);
  1087. struct dax_device *dax_dev = dev_dax->dax_dev;
  1088. put_dax(dax_dev);
  1089. free_dev_dax_id(dev_dax);
  1090. kfree(dev_dax->pgmap);
  1091. kfree(dev_dax);
  1092. }
  1093. static const struct device_type dev_dax_type = {
  1094. .release = dev_dax_release,
  1095. .groups = dax_attribute_groups,
  1096. };
  1097. struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data)
  1098. {
  1099. struct dax_region *dax_region = data->dax_region;
  1100. struct device *parent = dax_region->dev;
  1101. struct dax_device *dax_dev;
  1102. struct dev_dax *dev_dax;
  1103. struct inode *inode;
  1104. struct device *dev;
  1105. int rc;
  1106. dev_dax = kzalloc(sizeof(*dev_dax), GFP_KERNEL);
  1107. if (!dev_dax)
  1108. return ERR_PTR(-ENOMEM);
  1109. dev_dax->region = dax_region;
  1110. if (is_static(dax_region)) {
  1111. if (dev_WARN_ONCE(parent, data->id < 0,
  1112. "dynamic id specified to static region\n")) {
  1113. rc = -EINVAL;
  1114. goto err_id;
  1115. }
  1116. dev_dax->id = data->id;
  1117. } else {
  1118. if (dev_WARN_ONCE(parent, data->id >= 0,
  1119. "static id specified to dynamic region\n")) {
  1120. rc = -EINVAL;
  1121. goto err_id;
  1122. }
  1123. rc = alloc_dev_dax_id(dev_dax);
  1124. if (rc < 0)
  1125. goto err_id;
  1126. }
  1127. dev = &dev_dax->dev;
  1128. device_initialize(dev);
  1129. dev_set_name(dev, "dax%d.%d", dax_region->id, dev_dax->id);
  1130. rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, data->size);
  1131. if (rc)
  1132. goto err_range;
  1133. if (data->pgmap) {
  1134. dev_WARN_ONCE(parent, !is_static(dax_region),
  1135. "custom dev_pagemap requires a static dax_region\n");
  1136. dev_dax->pgmap = kmemdup(data->pgmap,
  1137. sizeof(struct dev_pagemap), GFP_KERNEL);
  1138. if (!dev_dax->pgmap) {
  1139. rc = -ENOMEM;
  1140. goto err_pgmap;
  1141. }
  1142. }
  1143. /*
  1144. * No dax_operations since there is no access to this device outside of
  1145. * mmap of the resulting character device.
  1146. */
  1147. dax_dev = alloc_dax(dev_dax, NULL);
  1148. if (IS_ERR(dax_dev)) {
  1149. rc = PTR_ERR(dax_dev);
  1150. goto err_alloc_dax;
  1151. }
  1152. set_dax_synchronous(dax_dev);
  1153. set_dax_nocache(dax_dev);
  1154. set_dax_nomc(dax_dev);
  1155. /* a device_dax instance is dead while the driver is not attached */
  1156. kill_dax(dax_dev);
  1157. dev_dax->dax_dev = dax_dev;
  1158. dev_dax->target_node = dax_region->target_node;
  1159. dev_dax->align = dax_region->align;
  1160. ida_init(&dev_dax->ida);
  1161. inode = dax_inode(dax_dev);
  1162. dev->devt = inode->i_rdev;
  1163. dev->bus = &dax_bus_type;
  1164. dev->parent = parent;
  1165. dev->type = &dev_dax_type;
  1166. rc = device_add(dev);
  1167. if (rc) {
  1168. kill_dev_dax(dev_dax);
  1169. put_device(dev);
  1170. return ERR_PTR(rc);
  1171. }
  1172. rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev);
  1173. if (rc)
  1174. return ERR_PTR(rc);
  1175. /* register mapping device for the initial allocation range */
  1176. if (dev_dax->nr_range && range_len(&dev_dax->ranges[0].range)) {
  1177. rc = devm_register_dax_mapping(dev_dax, 0);
  1178. if (rc)
  1179. return ERR_PTR(rc);
  1180. }
  1181. return dev_dax;
  1182. err_alloc_dax:
  1183. kfree(dev_dax->pgmap);
  1184. err_pgmap:
  1185. free_dev_dax_ranges(dev_dax);
  1186. err_range:
  1187. free_dev_dax_id(dev_dax);
  1188. err_id:
  1189. kfree(dev_dax);
  1190. return ERR_PTR(rc);
  1191. }
  1192. EXPORT_SYMBOL_GPL(devm_create_dev_dax);
  1193. static int match_always_count;
  1194. int __dax_driver_register(struct dax_device_driver *dax_drv,
  1195. struct module *module, const char *mod_name)
  1196. {
  1197. struct device_driver *drv = &dax_drv->drv;
  1198. int rc = 0;
  1199. /*
  1200. * dax_bus_probe() calls dax_drv->probe() unconditionally.
  1201. * So better be safe than sorry and ensure it is provided.
  1202. */
  1203. if (!dax_drv->probe)
  1204. return -EINVAL;
  1205. INIT_LIST_HEAD(&dax_drv->ids);
  1206. drv->owner = module;
  1207. drv->name = mod_name;
  1208. drv->mod_name = mod_name;
  1209. drv->bus = &dax_bus_type;
  1210. /* there can only be one default driver */
  1211. mutex_lock(&dax_bus_lock);
  1212. match_always_count += dax_drv->match_always;
  1213. if (match_always_count > 1) {
  1214. match_always_count--;
  1215. WARN_ON(1);
  1216. rc = -EINVAL;
  1217. }
  1218. mutex_unlock(&dax_bus_lock);
  1219. if (rc)
  1220. return rc;
  1221. rc = driver_register(drv);
  1222. if (rc && dax_drv->match_always) {
  1223. mutex_lock(&dax_bus_lock);
  1224. match_always_count -= dax_drv->match_always;
  1225. mutex_unlock(&dax_bus_lock);
  1226. }
  1227. return rc;
  1228. }
  1229. EXPORT_SYMBOL_GPL(__dax_driver_register);
  1230. void dax_driver_unregister(struct dax_device_driver *dax_drv)
  1231. {
  1232. struct device_driver *drv = &dax_drv->drv;
  1233. struct dax_id *dax_id, *_id;
  1234. mutex_lock(&dax_bus_lock);
  1235. match_always_count -= dax_drv->match_always;
  1236. list_for_each_entry_safe(dax_id, _id, &dax_drv->ids, list) {
  1237. list_del(&dax_id->list);
  1238. kfree(dax_id);
  1239. }
  1240. mutex_unlock(&dax_bus_lock);
  1241. driver_unregister(drv);
  1242. }
  1243. EXPORT_SYMBOL_GPL(dax_driver_unregister);
  1244. int __init dax_bus_init(void)
  1245. {
  1246. return bus_register(&dax_bus_type);
  1247. }
  1248. void __exit dax_bus_exit(void)
  1249. {
  1250. bus_unregister(&dax_bus_type);
  1251. }