iommu.c 85 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
  4. * Author: Joerg Roedel <[email protected]>
  5. */
  6. #define pr_fmt(fmt) "iommu: " fmt
  7. #include <linux/amba/bus.h>
  8. #include <linux/device.h>
  9. #include <linux/kernel.h>
  10. #include <linux/bits.h>
  11. #include <linux/bug.h>
  12. #include <linux/types.h>
  13. #include <linux/init.h>
  14. #include <linux/export.h>
  15. #include <linux/slab.h>
  16. #include <linux/errno.h>
  17. #include <linux/host1x_context_bus.h>
  18. #include <linux/iommu.h>
  19. #include <linux/idr.h>
  20. #include <linux/err.h>
  21. #include <linux/pci.h>
  22. #include <linux/pci-ats.h>
  23. #include <linux/bitops.h>
  24. #include <linux/platform_device.h>
  25. #include <linux/property.h>
  26. #include <linux/fsl/mc.h>
  27. #include <linux/module.h>
  28. #include <linux/cc_platform.h>
  29. #include <trace/events/iommu.h>
  30. #include <linux/sched/mm.h>
  31. #include <trace/hooks/iommu.h>
  32. #include "dma-iommu.h"
  33. #include "iommu-sva.h"
  34. static struct kset *iommu_group_kset;
  35. static DEFINE_IDA(iommu_group_ida);
  36. static unsigned int iommu_def_domain_type __read_mostly;
  37. static bool iommu_dma_strict __read_mostly = IS_ENABLED(CONFIG_IOMMU_DEFAULT_DMA_STRICT);
  38. static u32 iommu_cmd_line __read_mostly;
  39. struct iommu_group {
  40. struct kobject kobj;
  41. struct kobject *devices_kobj;
  42. struct list_head devices;
  43. struct xarray pasid_array;
  44. struct mutex mutex;
  45. void *iommu_data;
  46. void (*iommu_data_release)(void *iommu_data);
  47. char *name;
  48. int id;
  49. struct iommu_domain *default_domain;
  50. struct iommu_domain *blocking_domain;
  51. struct iommu_domain *domain;
  52. struct list_head entry;
  53. unsigned int owner_cnt;
  54. void *owner;
  55. };
  56. struct group_device {
  57. struct list_head list;
  58. struct device *dev;
  59. char *name;
  60. };
  61. struct iommu_group_attribute {
  62. struct attribute attr;
  63. ssize_t (*show)(struct iommu_group *group, char *buf);
  64. ssize_t (*store)(struct iommu_group *group,
  65. const char *buf, size_t count);
  66. };
  67. static const char * const iommu_group_resv_type_string[] = {
  68. [IOMMU_RESV_DIRECT] = "direct",
  69. [IOMMU_RESV_DIRECT_RELAXABLE] = "direct-relaxable",
  70. [IOMMU_RESV_RESERVED] = "reserved",
  71. [IOMMU_RESV_MSI] = "msi",
  72. [IOMMU_RESV_SW_MSI] = "msi",
  73. };
  74. #define IOMMU_CMD_LINE_DMA_API BIT(0)
  75. #define IOMMU_CMD_LINE_STRICT BIT(1)
  76. static int iommu_bus_notifier(struct notifier_block *nb,
  77. unsigned long action, void *data);
  78. static int iommu_alloc_default_domain(struct iommu_group *group,
  79. struct device *dev);
  80. static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
  81. unsigned type);
  82. static int __iommu_attach_device(struct iommu_domain *domain,
  83. struct device *dev);
  84. static int __iommu_attach_group(struct iommu_domain *domain,
  85. struct iommu_group *group);
  86. static int __iommu_group_set_domain(struct iommu_group *group,
  87. struct iommu_domain *new_domain);
  88. static int iommu_create_device_direct_mappings(struct iommu_group *group,
  89. struct device *dev);
  90. static struct iommu_group *iommu_group_get_for_dev(struct device *dev);
  91. static ssize_t iommu_group_store_type(struct iommu_group *group,
  92. const char *buf, size_t count);
  93. #define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \
  94. struct iommu_group_attribute iommu_group_attr_##_name = \
  95. __ATTR(_name, _mode, _show, _store)
  96. #define to_iommu_group_attr(_attr) \
  97. container_of(_attr, struct iommu_group_attribute, attr)
  98. #define to_iommu_group(_kobj) \
  99. container_of(_kobj, struct iommu_group, kobj)
  100. static LIST_HEAD(iommu_device_list);
  101. static DEFINE_SPINLOCK(iommu_device_lock);
  102. static struct bus_type * const iommu_buses[] = {
  103. &platform_bus_type,
  104. #ifdef CONFIG_PCI
  105. &pci_bus_type,
  106. #endif
  107. #ifdef CONFIG_ARM_AMBA
  108. &amba_bustype,
  109. #endif
  110. #ifdef CONFIG_FSL_MC_BUS
  111. &fsl_mc_bus_type,
  112. #endif
  113. #ifdef CONFIG_TEGRA_HOST1X_CONTEXT_BUS
  114. &host1x_context_device_bus_type,
  115. #endif
  116. };
  117. /*
  118. * Use a function instead of an array here because the domain-type is a
  119. * bit-field, so an array would waste memory.
  120. */
  121. static const char *iommu_domain_type_str(unsigned int t)
  122. {
  123. switch (t) {
  124. case IOMMU_DOMAIN_BLOCKED:
  125. return "Blocked";
  126. case IOMMU_DOMAIN_IDENTITY:
  127. return "Passthrough";
  128. case IOMMU_DOMAIN_UNMANAGED:
  129. return "Unmanaged";
  130. case IOMMU_DOMAIN_DMA:
  131. case IOMMU_DOMAIN_DMA_FQ:
  132. return "Translated";
  133. default:
  134. return "Unknown";
  135. }
  136. }
  137. static int __init iommu_subsys_init(void)
  138. {
  139. struct notifier_block *nb;
  140. if (!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API)) {
  141. if (IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH))
  142. iommu_set_default_passthrough(false);
  143. else
  144. iommu_set_default_translated(false);
  145. if (iommu_default_passthrough() && cc_platform_has(CC_ATTR_MEM_ENCRYPT)) {
  146. pr_info("Memory encryption detected - Disabling default IOMMU Passthrough\n");
  147. iommu_set_default_translated(false);
  148. }
  149. }
  150. if (!iommu_default_passthrough() && !iommu_dma_strict)
  151. iommu_def_domain_type = IOMMU_DOMAIN_DMA_FQ;
  152. pr_info("Default domain type: %s %s\n",
  153. iommu_domain_type_str(iommu_def_domain_type),
  154. (iommu_cmd_line & IOMMU_CMD_LINE_DMA_API) ?
  155. "(set via kernel command line)" : "");
  156. if (!iommu_default_passthrough())
  157. pr_info("DMA domain TLB invalidation policy: %s mode %s\n",
  158. iommu_dma_strict ? "strict" : "lazy",
  159. (iommu_cmd_line & IOMMU_CMD_LINE_STRICT) ?
  160. "(set via kernel command line)" : "");
  161. nb = kcalloc(ARRAY_SIZE(iommu_buses), sizeof(*nb), GFP_KERNEL);
  162. if (!nb)
  163. return -ENOMEM;
  164. for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) {
  165. nb[i].notifier_call = iommu_bus_notifier;
  166. bus_register_notifier(iommu_buses[i], &nb[i]);
  167. }
  168. return 0;
  169. }
  170. subsys_initcall(iommu_subsys_init);
  171. static int remove_iommu_group(struct device *dev, void *data)
  172. {
  173. if (dev->iommu && dev->iommu->iommu_dev == data)
  174. iommu_release_device(dev);
  175. return 0;
  176. }
  177. /**
  178. * iommu_device_register() - Register an IOMMU hardware instance
  179. * @iommu: IOMMU handle for the instance
  180. * @ops: IOMMU ops to associate with the instance
  181. * @hwdev: (optional) actual instance device, used for fwnode lookup
  182. *
  183. * Return: 0 on success, or an error.
  184. */
  185. int iommu_device_register(struct iommu_device *iommu,
  186. const struct iommu_ops *ops, struct device *hwdev)
  187. {
  188. int err = 0;
  189. /* We need to be able to take module references appropriately */
  190. if (WARN_ON(is_module_address((unsigned long)ops) && !ops->owner))
  191. return -EINVAL;
  192. /*
  193. * Temporarily enforce global restriction to a single driver. This was
  194. * already the de-facto behaviour, since any possible combination of
  195. * existing drivers would compete for at least the PCI or platform bus.
  196. */
  197. if (iommu_buses[0]->iommu_ops && iommu_buses[0]->iommu_ops != ops
  198. && !trace_android_vh_bus_iommu_probe_enabled())
  199. return -EBUSY;
  200. iommu->ops = ops;
  201. if (hwdev)
  202. iommu->fwnode = dev_fwnode(hwdev);
  203. spin_lock(&iommu_device_lock);
  204. list_add_tail(&iommu->list, &iommu_device_list);
  205. spin_unlock(&iommu_device_lock);
  206. for (int i = 0; i < ARRAY_SIZE(iommu_buses) && !err; i++) {
  207. bool skip = false;
  208. trace_android_vh_bus_iommu_probe(iommu, iommu_buses[i], &skip);
  209. if (skip)
  210. continue;
  211. iommu_buses[i]->iommu_ops = ops;
  212. err = bus_iommu_probe(iommu_buses[i]);
  213. }
  214. if (err)
  215. iommu_device_unregister(iommu);
  216. return err;
  217. }
  218. EXPORT_SYMBOL_GPL(iommu_device_register);
  219. void iommu_device_unregister(struct iommu_device *iommu)
  220. {
  221. for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++)
  222. bus_for_each_dev(iommu_buses[i], NULL, iommu, remove_iommu_group);
  223. spin_lock(&iommu_device_lock);
  224. list_del(&iommu->list);
  225. spin_unlock(&iommu_device_lock);
  226. }
  227. EXPORT_SYMBOL_GPL(iommu_device_unregister);
  228. static struct dev_iommu *dev_iommu_get(struct device *dev)
  229. {
  230. struct dev_iommu *param = dev->iommu;
  231. if (param)
  232. return param;
  233. param = kzalloc(sizeof(*param), GFP_KERNEL);
  234. if (!param)
  235. return NULL;
  236. mutex_init(&param->lock);
  237. dev->iommu = param;
  238. return param;
  239. }
  240. static void dev_iommu_free(struct device *dev)
  241. {
  242. struct dev_iommu *param = dev->iommu;
  243. struct iommu_fwspec *fwspec = param->fwspec;
  244. WRITE_ONCE(param->fwspec, NULL);
  245. smp_rmb();
  246. WRITE_ONCE(dev->iommu, NULL);
  247. if (fwspec) {
  248. fwnode_handle_put(fwspec->iommu_fwnode);
  249. kfree(fwspec);
  250. }
  251. kfree(param);
  252. }
  253. static u32 dev_iommu_get_max_pasids(struct device *dev)
  254. {
  255. u32 max_pasids = 0, bits = 0;
  256. int ret;
  257. if (dev_is_pci(dev)) {
  258. ret = pci_max_pasids(to_pci_dev(dev));
  259. if (ret > 0)
  260. max_pasids = ret;
  261. } else {
  262. ret = device_property_read_u32(dev, "pasid-num-bits", &bits);
  263. if (!ret)
  264. max_pasids = 1UL << bits;
  265. }
  266. return min_t(u32, max_pasids, dev->iommu->iommu_dev->max_pasids);
  267. }
  268. DEFINE_MUTEX(iommu_probe_device_lock);
  269. static int __iommu_probe_device(struct device *dev, struct list_head *group_list)
  270. {
  271. const struct iommu_ops *ops = dev->bus->iommu_ops;
  272. struct iommu_device *iommu_dev;
  273. struct iommu_group *group;
  274. int ret;
  275. if (!ops)
  276. return -ENODEV;
  277. /*
  278. * Serialise to avoid races between IOMMU drivers registering in
  279. * parallel and/or the "replay" calls from ACPI/OF code via client
  280. * driver probe. Once the latter have been cleaned up we should
  281. * probably be able to use device_lock() here to minimise the scope,
  282. * but for now enforcing a simple global ordering is fine.
  283. */
  284. lockdep_assert_held(&iommu_probe_device_lock);
  285. if (!dev_iommu_get(dev)) {
  286. ret = -ENOMEM;
  287. goto err_out;
  288. }
  289. if (!try_module_get(ops->owner)) {
  290. ret = -EINVAL;
  291. goto err_free;
  292. }
  293. iommu_dev = ops->probe_device(dev);
  294. if (IS_ERR(iommu_dev)) {
  295. ret = PTR_ERR(iommu_dev);
  296. goto out_module_put;
  297. }
  298. dev->iommu->iommu_dev = iommu_dev;
  299. dev->iommu->max_pasids = dev_iommu_get_max_pasids(dev);
  300. group = iommu_group_get_for_dev(dev);
  301. if (IS_ERR(group)) {
  302. ret = PTR_ERR(group);
  303. goto out_release;
  304. }
  305. mutex_lock(&group->mutex);
  306. if (group_list && !group->default_domain && list_empty(&group->entry))
  307. list_add_tail(&group->entry, group_list);
  308. mutex_unlock(&group->mutex);
  309. iommu_group_put(group);
  310. iommu_device_link(iommu_dev, dev);
  311. return 0;
  312. out_release:
  313. if (ops->release_device)
  314. ops->release_device(dev);
  315. out_module_put:
  316. module_put(ops->owner);
  317. err_free:
  318. dev_iommu_free(dev);
  319. err_out:
  320. return ret;
  321. }
  322. int iommu_probe_device(struct device *dev)
  323. {
  324. const struct iommu_ops *ops;
  325. struct iommu_group *group;
  326. int ret;
  327. mutex_lock(&iommu_probe_device_lock);
  328. ret = __iommu_probe_device(dev, NULL);
  329. mutex_unlock(&iommu_probe_device_lock);
  330. if (ret)
  331. goto err_out;
  332. group = iommu_group_get(dev);
  333. if (!group) {
  334. ret = -ENODEV;
  335. goto err_release;
  336. }
  337. /*
  338. * Try to allocate a default domain - needs support from the
  339. * IOMMU driver. There are still some drivers which don't
  340. * support default domains, so the return value is not yet
  341. * checked.
  342. */
  343. mutex_lock(&group->mutex);
  344. iommu_alloc_default_domain(group, dev);
  345. /*
  346. * If device joined an existing group which has been claimed, don't
  347. * attach the default domain.
  348. */
  349. if (group->default_domain && !group->owner) {
  350. ret = __iommu_attach_device(group->default_domain, dev);
  351. if (ret) {
  352. mutex_unlock(&group->mutex);
  353. iommu_group_put(group);
  354. goto err_release;
  355. }
  356. }
  357. iommu_create_device_direct_mappings(group, dev);
  358. mutex_unlock(&group->mutex);
  359. iommu_group_put(group);
  360. ops = dev_iommu_ops(dev);
  361. if (ops->probe_finalize)
  362. ops->probe_finalize(dev);
  363. return 0;
  364. err_release:
  365. iommu_release_device(dev);
  366. err_out:
  367. return ret;
  368. }
  369. void iommu_release_device(struct device *dev)
  370. {
  371. const struct iommu_ops *ops;
  372. if (!dev->iommu)
  373. return;
  374. iommu_device_unlink(dev->iommu->iommu_dev, dev);
  375. ops = dev_iommu_ops(dev);
  376. if (ops->release_device)
  377. ops->release_device(dev);
  378. iommu_group_remove_device(dev);
  379. module_put(ops->owner);
  380. dev_iommu_free(dev);
  381. }
  382. static int __init iommu_set_def_domain_type(char *str)
  383. {
  384. bool pt;
  385. int ret;
  386. ret = kstrtobool(str, &pt);
  387. if (ret)
  388. return ret;
  389. if (pt)
  390. iommu_set_default_passthrough(true);
  391. else
  392. iommu_set_default_translated(true);
  393. return 0;
  394. }
  395. early_param("iommu.passthrough", iommu_set_def_domain_type);
  396. static int __init iommu_dma_setup(char *str)
  397. {
  398. int ret = kstrtobool(str, &iommu_dma_strict);
  399. if (!ret)
  400. iommu_cmd_line |= IOMMU_CMD_LINE_STRICT;
  401. return ret;
  402. }
  403. early_param("iommu.strict", iommu_dma_setup);
  404. void iommu_set_dma_strict(void)
  405. {
  406. iommu_dma_strict = true;
  407. if (iommu_def_domain_type == IOMMU_DOMAIN_DMA_FQ)
  408. iommu_def_domain_type = IOMMU_DOMAIN_DMA;
  409. }
  410. static ssize_t iommu_group_attr_show(struct kobject *kobj,
  411. struct attribute *__attr, char *buf)
  412. {
  413. struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
  414. struct iommu_group *group = to_iommu_group(kobj);
  415. ssize_t ret = -EIO;
  416. if (attr->show)
  417. ret = attr->show(group, buf);
  418. return ret;
  419. }
  420. static ssize_t iommu_group_attr_store(struct kobject *kobj,
  421. struct attribute *__attr,
  422. const char *buf, size_t count)
  423. {
  424. struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
  425. struct iommu_group *group = to_iommu_group(kobj);
  426. ssize_t ret = -EIO;
  427. if (attr->store)
  428. ret = attr->store(group, buf, count);
  429. return ret;
  430. }
  431. static const struct sysfs_ops iommu_group_sysfs_ops = {
  432. .show = iommu_group_attr_show,
  433. .store = iommu_group_attr_store,
  434. };
  435. static int iommu_group_create_file(struct iommu_group *group,
  436. struct iommu_group_attribute *attr)
  437. {
  438. return sysfs_create_file(&group->kobj, &attr->attr);
  439. }
  440. static void iommu_group_remove_file(struct iommu_group *group,
  441. struct iommu_group_attribute *attr)
  442. {
  443. sysfs_remove_file(&group->kobj, &attr->attr);
  444. }
  445. static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf)
  446. {
  447. return sprintf(buf, "%s\n", group->name);
  448. }
  449. /**
  450. * iommu_insert_resv_region - Insert a new region in the
  451. * list of reserved regions.
  452. * @new: new region to insert
  453. * @regions: list of regions
  454. *
  455. * Elements are sorted by start address and overlapping segments
  456. * of the same type are merged.
  457. */
  458. static int iommu_insert_resv_region(struct iommu_resv_region *new,
  459. struct list_head *regions)
  460. {
  461. struct iommu_resv_region *iter, *tmp, *nr, *top;
  462. LIST_HEAD(stack);
  463. nr = iommu_alloc_resv_region(new->start, new->length,
  464. new->prot, new->type, GFP_KERNEL);
  465. if (!nr)
  466. return -ENOMEM;
  467. /* First add the new element based on start address sorting */
  468. list_for_each_entry(iter, regions, list) {
  469. if (nr->start < iter->start ||
  470. (nr->start == iter->start && nr->type <= iter->type))
  471. break;
  472. }
  473. list_add_tail(&nr->list, &iter->list);
  474. /* Merge overlapping segments of type nr->type in @regions, if any */
  475. list_for_each_entry_safe(iter, tmp, regions, list) {
  476. phys_addr_t top_end, iter_end = iter->start + iter->length - 1;
  477. /* no merge needed on elements of different types than @new */
  478. if (iter->type != new->type) {
  479. list_move_tail(&iter->list, &stack);
  480. continue;
  481. }
  482. /* look for the last stack element of same type as @iter */
  483. list_for_each_entry_reverse(top, &stack, list)
  484. if (top->type == iter->type)
  485. goto check_overlap;
  486. list_move_tail(&iter->list, &stack);
  487. continue;
  488. check_overlap:
  489. top_end = top->start + top->length - 1;
  490. if (iter->start > top_end + 1) {
  491. list_move_tail(&iter->list, &stack);
  492. } else {
  493. top->length = max(top_end, iter_end) - top->start + 1;
  494. list_del(&iter->list);
  495. kfree(iter);
  496. }
  497. }
  498. list_splice(&stack, regions);
  499. return 0;
  500. }
  501. static int
  502. iommu_insert_device_resv_regions(struct list_head *dev_resv_regions,
  503. struct list_head *group_resv_regions)
  504. {
  505. struct iommu_resv_region *entry;
  506. int ret = 0;
  507. list_for_each_entry(entry, dev_resv_regions, list) {
  508. ret = iommu_insert_resv_region(entry, group_resv_regions);
  509. if (ret)
  510. break;
  511. }
  512. return ret;
  513. }
  514. int iommu_get_group_resv_regions(struct iommu_group *group,
  515. struct list_head *head)
  516. {
  517. struct group_device *device;
  518. int ret = 0;
  519. mutex_lock(&group->mutex);
  520. list_for_each_entry(device, &group->devices, list) {
  521. struct list_head dev_resv_regions;
  522. /*
  523. * Non-API groups still expose reserved_regions in sysfs,
  524. * so filter out calls that get here that way.
  525. */
  526. if (!device->dev->iommu)
  527. break;
  528. INIT_LIST_HEAD(&dev_resv_regions);
  529. iommu_get_resv_regions(device->dev, &dev_resv_regions);
  530. ret = iommu_insert_device_resv_regions(&dev_resv_regions, head);
  531. iommu_put_resv_regions(device->dev, &dev_resv_regions);
  532. if (ret)
  533. break;
  534. }
  535. mutex_unlock(&group->mutex);
  536. return ret;
  537. }
  538. EXPORT_SYMBOL_GPL(iommu_get_group_resv_regions);
  539. static ssize_t iommu_group_show_resv_regions(struct iommu_group *group,
  540. char *buf)
  541. {
  542. struct iommu_resv_region *region, *next;
  543. struct list_head group_resv_regions;
  544. char *str = buf;
  545. INIT_LIST_HEAD(&group_resv_regions);
  546. iommu_get_group_resv_regions(group, &group_resv_regions);
  547. list_for_each_entry_safe(region, next, &group_resv_regions, list) {
  548. str += sprintf(str, "0x%016llx 0x%016llx %s\n",
  549. (long long int)region->start,
  550. (long long int)(region->start +
  551. region->length - 1),
  552. iommu_group_resv_type_string[region->type]);
  553. kfree(region);
  554. }
  555. return (str - buf);
  556. }
  557. static ssize_t iommu_group_show_type(struct iommu_group *group,
  558. char *buf)
  559. {
  560. char *type = "unknown\n";
  561. mutex_lock(&group->mutex);
  562. if (group->default_domain) {
  563. switch (group->default_domain->type) {
  564. case IOMMU_DOMAIN_BLOCKED:
  565. type = "blocked\n";
  566. break;
  567. case IOMMU_DOMAIN_IDENTITY:
  568. type = "identity\n";
  569. break;
  570. case IOMMU_DOMAIN_UNMANAGED:
  571. type = "unmanaged\n";
  572. break;
  573. case IOMMU_DOMAIN_DMA:
  574. type = "DMA\n";
  575. break;
  576. case IOMMU_DOMAIN_DMA_FQ:
  577. type = "DMA-FQ\n";
  578. break;
  579. }
  580. }
  581. mutex_unlock(&group->mutex);
  582. strcpy(buf, type);
  583. return strlen(type);
  584. }
  585. static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL);
  586. static IOMMU_GROUP_ATTR(reserved_regions, 0444,
  587. iommu_group_show_resv_regions, NULL);
  588. static IOMMU_GROUP_ATTR(type, 0644, iommu_group_show_type,
  589. iommu_group_store_type);
  590. static void iommu_group_release(struct kobject *kobj)
  591. {
  592. struct iommu_group *group = to_iommu_group(kobj);
  593. pr_debug("Releasing group %d\n", group->id);
  594. if (group->iommu_data_release)
  595. group->iommu_data_release(group->iommu_data);
  596. ida_free(&iommu_group_ida, group->id);
  597. if (group->default_domain)
  598. iommu_domain_free(group->default_domain);
  599. if (group->blocking_domain)
  600. iommu_domain_free(group->blocking_domain);
  601. kfree(group->name);
  602. kfree(group);
  603. }
  604. static struct kobj_type iommu_group_ktype = {
  605. .sysfs_ops = &iommu_group_sysfs_ops,
  606. .release = iommu_group_release,
  607. };
  608. /**
  609. * iommu_group_alloc - Allocate a new group
  610. *
  611. * This function is called by an iommu driver to allocate a new iommu
  612. * group. The iommu group represents the minimum granularity of the iommu.
  613. * Upon successful return, the caller holds a reference to the supplied
  614. * group in order to hold the group until devices are added. Use
  615. * iommu_group_put() to release this extra reference count, allowing the
  616. * group to be automatically reclaimed once it has no devices or external
  617. * references.
  618. */
  619. struct iommu_group *iommu_group_alloc(void)
  620. {
  621. struct iommu_group *group;
  622. int ret;
  623. group = kzalloc(sizeof(*group), GFP_KERNEL);
  624. if (!group)
  625. return ERR_PTR(-ENOMEM);
  626. group->kobj.kset = iommu_group_kset;
  627. mutex_init(&group->mutex);
  628. INIT_LIST_HEAD(&group->devices);
  629. INIT_LIST_HEAD(&group->entry);
  630. xa_init(&group->pasid_array);
  631. ret = ida_alloc(&iommu_group_ida, GFP_KERNEL);
  632. if (ret < 0) {
  633. kfree(group);
  634. return ERR_PTR(ret);
  635. }
  636. group->id = ret;
  637. ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype,
  638. NULL, "%d", group->id);
  639. if (ret) {
  640. kobject_put(&group->kobj);
  641. return ERR_PTR(ret);
  642. }
  643. group->devices_kobj = kobject_create_and_add("devices", &group->kobj);
  644. if (!group->devices_kobj) {
  645. kobject_put(&group->kobj); /* triggers .release & free */
  646. return ERR_PTR(-ENOMEM);
  647. }
  648. /*
  649. * The devices_kobj holds a reference on the group kobject, so
  650. * as long as that exists so will the group. We can therefore
  651. * use the devices_kobj for reference counting.
  652. */
  653. kobject_put(&group->kobj);
  654. ret = iommu_group_create_file(group,
  655. &iommu_group_attr_reserved_regions);
  656. if (ret) {
  657. kobject_put(group->devices_kobj);
  658. return ERR_PTR(ret);
  659. }
  660. ret = iommu_group_create_file(group, &iommu_group_attr_type);
  661. if (ret) {
  662. kobject_put(group->devices_kobj);
  663. return ERR_PTR(ret);
  664. }
  665. pr_debug("Allocated group %d\n", group->id);
  666. return group;
  667. }
  668. EXPORT_SYMBOL_GPL(iommu_group_alloc);
  669. struct iommu_group *iommu_group_get_by_id(int id)
  670. {
  671. struct kobject *group_kobj;
  672. struct iommu_group *group;
  673. const char *name;
  674. if (!iommu_group_kset)
  675. return NULL;
  676. name = kasprintf(GFP_KERNEL, "%d", id);
  677. if (!name)
  678. return NULL;
  679. group_kobj = kset_find_obj(iommu_group_kset, name);
  680. kfree(name);
  681. if (!group_kobj)
  682. return NULL;
  683. group = container_of(group_kobj, struct iommu_group, kobj);
  684. BUG_ON(group->id != id);
  685. kobject_get(group->devices_kobj);
  686. kobject_put(&group->kobj);
  687. return group;
  688. }
  689. EXPORT_SYMBOL_GPL(iommu_group_get_by_id);
  690. /**
  691. * iommu_group_get_iommudata - retrieve iommu_data registered for a group
  692. * @group: the group
  693. *
  694. * iommu drivers can store data in the group for use when doing iommu
  695. * operations. This function provides a way to retrieve it. Caller
  696. * should hold a group reference.
  697. */
  698. void *iommu_group_get_iommudata(struct iommu_group *group)
  699. {
  700. return group->iommu_data;
  701. }
  702. EXPORT_SYMBOL_GPL(iommu_group_get_iommudata);
  703. /**
  704. * iommu_group_set_iommudata - set iommu_data for a group
  705. * @group: the group
  706. * @iommu_data: new data
  707. * @release: release function for iommu_data
  708. *
  709. * iommu drivers can store data in the group for use when doing iommu
  710. * operations. This function provides a way to set the data after
  711. * the group has been allocated. Caller should hold a group reference.
  712. */
  713. void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data,
  714. void (*release)(void *iommu_data))
  715. {
  716. group->iommu_data = iommu_data;
  717. group->iommu_data_release = release;
  718. }
  719. EXPORT_SYMBOL_GPL(iommu_group_set_iommudata);
  720. /**
  721. * iommu_group_set_name - set name for a group
  722. * @group: the group
  723. * @name: name
  724. *
  725. * Allow iommu driver to set a name for a group. When set it will
  726. * appear in a name attribute file under the group in sysfs.
  727. */
  728. int iommu_group_set_name(struct iommu_group *group, const char *name)
  729. {
  730. int ret;
  731. if (group->name) {
  732. iommu_group_remove_file(group, &iommu_group_attr_name);
  733. kfree(group->name);
  734. group->name = NULL;
  735. if (!name)
  736. return 0;
  737. }
  738. group->name = kstrdup(name, GFP_KERNEL);
  739. if (!group->name)
  740. return -ENOMEM;
  741. ret = iommu_group_create_file(group, &iommu_group_attr_name);
  742. if (ret) {
  743. kfree(group->name);
  744. group->name = NULL;
  745. return ret;
  746. }
  747. return 0;
  748. }
  749. EXPORT_SYMBOL_GPL(iommu_group_set_name);
  750. static int iommu_create_device_direct_mappings(struct iommu_group *group,
  751. struct device *dev)
  752. {
  753. struct iommu_domain *domain = group->default_domain;
  754. struct iommu_resv_region *entry;
  755. struct list_head mappings;
  756. unsigned long pg_size;
  757. int ret = 0;
  758. if (!domain || !iommu_is_dma_domain(domain))
  759. return 0;
  760. BUG_ON(!domain->pgsize_bitmap);
  761. pg_size = 1UL << __ffs(domain->pgsize_bitmap);
  762. INIT_LIST_HEAD(&mappings);
  763. iommu_get_resv_regions(dev, &mappings);
  764. /* We need to consider overlapping regions for different devices */
  765. list_for_each_entry(entry, &mappings, list) {
  766. dma_addr_t start, end, addr;
  767. size_t map_size = 0;
  768. start = ALIGN(entry->start, pg_size);
  769. end = ALIGN(entry->start + entry->length, pg_size);
  770. if (entry->type != IOMMU_RESV_DIRECT &&
  771. entry->type != IOMMU_RESV_DIRECT_RELAXABLE)
  772. continue;
  773. for (addr = start; addr <= end; addr += pg_size) {
  774. phys_addr_t phys_addr;
  775. if (addr == end)
  776. goto map_end;
  777. phys_addr = iommu_iova_to_phys(domain, addr);
  778. if (!phys_addr) {
  779. map_size += pg_size;
  780. continue;
  781. }
  782. map_end:
  783. if (map_size) {
  784. ret = iommu_map(domain, addr - map_size,
  785. addr - map_size, map_size,
  786. entry->prot);
  787. if (ret)
  788. goto out;
  789. map_size = 0;
  790. }
  791. }
  792. }
  793. iommu_flush_iotlb_all(domain);
  794. out:
  795. iommu_put_resv_regions(dev, &mappings);
  796. return ret;
  797. }
  798. static bool iommu_is_attach_deferred(struct device *dev)
  799. {
  800. const struct iommu_ops *ops = dev_iommu_ops(dev);
  801. if (ops->is_attach_deferred)
  802. return ops->is_attach_deferred(dev);
  803. return false;
  804. }
  805. /**
  806. * iommu_group_add_device - add a device to an iommu group
  807. * @group: the group into which to add the device (reference should be held)
  808. * @dev: the device
  809. *
  810. * This function is called by an iommu driver to add a device into a
  811. * group. Adding a device increments the group reference count.
  812. */
  813. int iommu_group_add_device(struct iommu_group *group, struct device *dev)
  814. {
  815. int ret, i = 0;
  816. struct group_device *device;
  817. device = kzalloc(sizeof(*device), GFP_KERNEL);
  818. if (!device)
  819. return -ENOMEM;
  820. device->dev = dev;
  821. ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group");
  822. if (ret)
  823. goto err_free_device;
  824. device->name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj));
  825. rename:
  826. if (!device->name) {
  827. ret = -ENOMEM;
  828. goto err_remove_link;
  829. }
  830. ret = sysfs_create_link_nowarn(group->devices_kobj,
  831. &dev->kobj, device->name);
  832. if (ret) {
  833. if (ret == -EEXIST && i >= 0) {
  834. /*
  835. * Account for the slim chance of collision
  836. * and append an instance to the name.
  837. */
  838. kfree(device->name);
  839. device->name = kasprintf(GFP_KERNEL, "%s.%d",
  840. kobject_name(&dev->kobj), i++);
  841. goto rename;
  842. }
  843. goto err_free_name;
  844. }
  845. kobject_get(group->devices_kobj);
  846. dev->iommu_group = group;
  847. mutex_lock(&group->mutex);
  848. list_add_tail(&device->list, &group->devices);
  849. if (group->domain && !iommu_is_attach_deferred(dev))
  850. ret = __iommu_attach_device(group->domain, dev);
  851. mutex_unlock(&group->mutex);
  852. if (ret)
  853. goto err_put_group;
  854. trace_add_device_to_group(group->id, dev);
  855. dev_info(dev, "Adding to iommu group %d\n", group->id);
  856. return 0;
  857. err_put_group:
  858. mutex_lock(&group->mutex);
  859. list_del(&device->list);
  860. mutex_unlock(&group->mutex);
  861. dev->iommu_group = NULL;
  862. kobject_put(group->devices_kobj);
  863. sysfs_remove_link(group->devices_kobj, device->name);
  864. err_free_name:
  865. kfree(device->name);
  866. err_remove_link:
  867. sysfs_remove_link(&dev->kobj, "iommu_group");
  868. err_free_device:
  869. kfree(device);
  870. dev_err(dev, "Failed to add to iommu group %d: %d\n", group->id, ret);
  871. return ret;
  872. }
  873. EXPORT_SYMBOL_GPL(iommu_group_add_device);
  874. /**
  875. * iommu_group_remove_device - remove a device from it's current group
  876. * @dev: device to be removed
  877. *
  878. * This function is called by an iommu driver to remove the device from
  879. * it's current group. This decrements the iommu group reference count.
  880. */
  881. void iommu_group_remove_device(struct device *dev)
  882. {
  883. struct iommu_group *group = dev->iommu_group;
  884. struct group_device *tmp_device, *device = NULL;
  885. if (!group)
  886. return;
  887. dev_info(dev, "Removing from iommu group %d\n", group->id);
  888. mutex_lock(&group->mutex);
  889. list_for_each_entry(tmp_device, &group->devices, list) {
  890. if (tmp_device->dev == dev) {
  891. device = tmp_device;
  892. list_del(&device->list);
  893. break;
  894. }
  895. }
  896. mutex_unlock(&group->mutex);
  897. if (!device)
  898. return;
  899. sysfs_remove_link(group->devices_kobj, device->name);
  900. sysfs_remove_link(&dev->kobj, "iommu_group");
  901. trace_remove_device_from_group(group->id, dev);
  902. kfree(device->name);
  903. kfree(device);
  904. dev->iommu_group = NULL;
  905. kobject_put(group->devices_kobj);
  906. }
  907. EXPORT_SYMBOL_GPL(iommu_group_remove_device);
  908. static int iommu_group_device_count(struct iommu_group *group)
  909. {
  910. struct group_device *entry;
  911. int ret = 0;
  912. list_for_each_entry(entry, &group->devices, list)
  913. ret++;
  914. return ret;
  915. }
  916. static int __iommu_group_for_each_dev(struct iommu_group *group, void *data,
  917. int (*fn)(struct device *, void *))
  918. {
  919. struct group_device *device;
  920. int ret = 0;
  921. list_for_each_entry(device, &group->devices, list) {
  922. ret = fn(device->dev, data);
  923. if (ret)
  924. break;
  925. }
  926. return ret;
  927. }
  928. /**
  929. * iommu_group_for_each_dev - iterate over each device in the group
  930. * @group: the group
  931. * @data: caller opaque data to be passed to callback function
  932. * @fn: caller supplied callback function
  933. *
  934. * This function is called by group users to iterate over group devices.
  935. * Callers should hold a reference count to the group during callback.
  936. * The group->mutex is held across callbacks, which will block calls to
  937. * iommu_group_add/remove_device.
  938. */
  939. int iommu_group_for_each_dev(struct iommu_group *group, void *data,
  940. int (*fn)(struct device *, void *))
  941. {
  942. int ret;
  943. mutex_lock(&group->mutex);
  944. ret = __iommu_group_for_each_dev(group, data, fn);
  945. mutex_unlock(&group->mutex);
  946. return ret;
  947. }
  948. EXPORT_SYMBOL_GPL(iommu_group_for_each_dev);
  949. /**
  950. * iommu_group_get - Return the group for a device and increment reference
  951. * @dev: get the group that this device belongs to
  952. *
  953. * This function is called by iommu drivers and users to get the group
  954. * for the specified device. If found, the group is returned and the group
  955. * reference in incremented, else NULL.
  956. */
  957. struct iommu_group *iommu_group_get(struct device *dev)
  958. {
  959. struct iommu_group *group = dev->iommu_group;
  960. if (group)
  961. kobject_get(group->devices_kobj);
  962. return group;
  963. }
  964. EXPORT_SYMBOL_GPL(iommu_group_get);
  965. /**
  966. * iommu_group_ref_get - Increment reference on a group
  967. * @group: the group to use, must not be NULL
  968. *
  969. * This function is called by iommu drivers to take additional references on an
  970. * existing group. Returns the given group for convenience.
  971. */
  972. struct iommu_group *iommu_group_ref_get(struct iommu_group *group)
  973. {
  974. kobject_get(group->devices_kobj);
  975. return group;
  976. }
  977. EXPORT_SYMBOL_GPL(iommu_group_ref_get);
  978. /**
  979. * iommu_group_put - Decrement group reference
  980. * @group: the group to use
  981. *
  982. * This function is called by iommu drivers and users to release the
  983. * iommu group. Once the reference count is zero, the group is released.
  984. */
  985. void iommu_group_put(struct iommu_group *group)
  986. {
  987. if (group)
  988. kobject_put(group->devices_kobj);
  989. }
  990. EXPORT_SYMBOL_GPL(iommu_group_put);
  991. /**
  992. * iommu_register_device_fault_handler() - Register a device fault handler
  993. * @dev: the device
  994. * @handler: the fault handler
  995. * @data: private data passed as argument to the handler
  996. *
  997. * When an IOMMU fault event is received, this handler gets called with the
  998. * fault event and data as argument. The handler should return 0 on success. If
  999. * the fault is recoverable (IOMMU_FAULT_PAGE_REQ), the consumer should also
  1000. * complete the fault by calling iommu_page_response() with one of the following
  1001. * response code:
  1002. * - IOMMU_PAGE_RESP_SUCCESS: retry the translation
  1003. * - IOMMU_PAGE_RESP_INVALID: terminate the fault
  1004. * - IOMMU_PAGE_RESP_FAILURE: terminate the fault and stop reporting
  1005. * page faults if possible.
  1006. *
  1007. * Return 0 if the fault handler was installed successfully, or an error.
  1008. */
  1009. int iommu_register_device_fault_handler(struct device *dev,
  1010. iommu_dev_fault_handler_t handler,
  1011. void *data)
  1012. {
  1013. struct dev_iommu *param = dev->iommu;
  1014. int ret = 0;
  1015. if (!param)
  1016. return -EINVAL;
  1017. mutex_lock(&param->lock);
  1018. /* Only allow one fault handler registered for each device */
  1019. if (param->fault_param) {
  1020. ret = -EBUSY;
  1021. goto done_unlock;
  1022. }
  1023. get_device(dev);
  1024. param->fault_param = kzalloc(sizeof(*param->fault_param), GFP_KERNEL);
  1025. if (!param->fault_param) {
  1026. put_device(dev);
  1027. ret = -ENOMEM;
  1028. goto done_unlock;
  1029. }
  1030. param->fault_param->handler = handler;
  1031. param->fault_param->data = data;
  1032. mutex_init(&param->fault_param->lock);
  1033. INIT_LIST_HEAD(&param->fault_param->faults);
  1034. done_unlock:
  1035. mutex_unlock(&param->lock);
  1036. return ret;
  1037. }
  1038. EXPORT_SYMBOL_GPL(iommu_register_device_fault_handler);
  1039. /**
  1040. * iommu_unregister_device_fault_handler() - Unregister the device fault handler
  1041. * @dev: the device
  1042. *
  1043. * Remove the device fault handler installed with
  1044. * iommu_register_device_fault_handler().
  1045. *
  1046. * Return 0 on success, or an error.
  1047. */
  1048. int iommu_unregister_device_fault_handler(struct device *dev)
  1049. {
  1050. struct dev_iommu *param = dev->iommu;
  1051. int ret = 0;
  1052. if (!param)
  1053. return -EINVAL;
  1054. mutex_lock(&param->lock);
  1055. if (!param->fault_param)
  1056. goto unlock;
  1057. /* we cannot unregister handler if there are pending faults */
  1058. if (!list_empty(&param->fault_param->faults)) {
  1059. ret = -EBUSY;
  1060. goto unlock;
  1061. }
  1062. kfree(param->fault_param);
  1063. param->fault_param = NULL;
  1064. put_device(dev);
  1065. unlock:
  1066. mutex_unlock(&param->lock);
  1067. return ret;
  1068. }
  1069. EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler);
  1070. /**
  1071. * iommu_report_device_fault() - Report fault event to device driver
  1072. * @dev: the device
  1073. * @evt: fault event data
  1074. *
  1075. * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ
  1076. * handler. When this function fails and the fault is recoverable, it is the
  1077. * caller's responsibility to complete the fault.
  1078. *
  1079. * Return 0 on success, or an error.
  1080. */
  1081. int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt)
  1082. {
  1083. struct dev_iommu *param = dev->iommu;
  1084. struct iommu_fault_event *evt_pending = NULL;
  1085. struct iommu_fault_param *fparam;
  1086. int ret = 0;
  1087. if (!param || !evt)
  1088. return -EINVAL;
  1089. /* we only report device fault if there is a handler registered */
  1090. mutex_lock(&param->lock);
  1091. fparam = param->fault_param;
  1092. if (!fparam || !fparam->handler) {
  1093. ret = -EINVAL;
  1094. goto done_unlock;
  1095. }
  1096. if (evt->fault.type == IOMMU_FAULT_PAGE_REQ &&
  1097. (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) {
  1098. evt_pending = kmemdup(evt, sizeof(struct iommu_fault_event),
  1099. GFP_KERNEL);
  1100. if (!evt_pending) {
  1101. ret = -ENOMEM;
  1102. goto done_unlock;
  1103. }
  1104. mutex_lock(&fparam->lock);
  1105. list_add_tail(&evt_pending->list, &fparam->faults);
  1106. mutex_unlock(&fparam->lock);
  1107. }
  1108. ret = fparam->handler(&evt->fault, fparam->data);
  1109. if (ret && evt_pending) {
  1110. mutex_lock(&fparam->lock);
  1111. list_del(&evt_pending->list);
  1112. mutex_unlock(&fparam->lock);
  1113. kfree(evt_pending);
  1114. }
  1115. done_unlock:
  1116. mutex_unlock(&param->lock);
  1117. return ret;
  1118. }
  1119. EXPORT_SYMBOL_GPL(iommu_report_device_fault);
  1120. int iommu_page_response(struct device *dev,
  1121. struct iommu_page_response *msg)
  1122. {
  1123. bool needs_pasid;
  1124. int ret = -EINVAL;
  1125. struct iommu_fault_event *evt;
  1126. struct iommu_fault_page_request *prm;
  1127. struct dev_iommu *param = dev->iommu;
  1128. const struct iommu_ops *ops = dev_iommu_ops(dev);
  1129. bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID;
  1130. if (!ops->page_response)
  1131. return -ENODEV;
  1132. if (!param || !param->fault_param)
  1133. return -EINVAL;
  1134. if (msg->version != IOMMU_PAGE_RESP_VERSION_1 ||
  1135. msg->flags & ~IOMMU_PAGE_RESP_PASID_VALID)
  1136. return -EINVAL;
  1137. /* Only send response if there is a fault report pending */
  1138. mutex_lock(&param->fault_param->lock);
  1139. if (list_empty(&param->fault_param->faults)) {
  1140. dev_warn_ratelimited(dev, "no pending PRQ, drop response\n");
  1141. goto done_unlock;
  1142. }
  1143. /*
  1144. * Check if we have a matching page request pending to respond,
  1145. * otherwise return -EINVAL
  1146. */
  1147. list_for_each_entry(evt, &param->fault_param->faults, list) {
  1148. prm = &evt->fault.prm;
  1149. if (prm->grpid != msg->grpid)
  1150. continue;
  1151. /*
  1152. * If the PASID is required, the corresponding request is
  1153. * matched using the group ID, the PASID valid bit and the PASID
  1154. * value. Otherwise only the group ID matches request and
  1155. * response.
  1156. */
  1157. needs_pasid = prm->flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID;
  1158. if (needs_pasid && (!has_pasid || msg->pasid != prm->pasid))
  1159. continue;
  1160. if (!needs_pasid && has_pasid) {
  1161. /* No big deal, just clear it. */
  1162. msg->flags &= ~IOMMU_PAGE_RESP_PASID_VALID;
  1163. msg->pasid = 0;
  1164. }
  1165. ret = ops->page_response(dev, evt, msg);
  1166. list_del(&evt->list);
  1167. kfree(evt);
  1168. break;
  1169. }
  1170. done_unlock:
  1171. mutex_unlock(&param->fault_param->lock);
  1172. return ret;
  1173. }
  1174. EXPORT_SYMBOL_GPL(iommu_page_response);
  1175. /**
  1176. * iommu_group_id - Return ID for a group
  1177. * @group: the group to ID
  1178. *
  1179. * Return the unique ID for the group matching the sysfs group number.
  1180. */
  1181. int iommu_group_id(struct iommu_group *group)
  1182. {
  1183. return group->id;
  1184. }
  1185. EXPORT_SYMBOL_GPL(iommu_group_id);
  1186. static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
  1187. unsigned long *devfns);
  1188. /*
  1189. * To consider a PCI device isolated, we require ACS to support Source
  1190. * Validation, Request Redirection, Completer Redirection, and Upstream
  1191. * Forwarding. This effectively means that devices cannot spoof their
  1192. * requester ID, requests and completions cannot be redirected, and all
  1193. * transactions are forwarded upstream, even as it passes through a
  1194. * bridge where the target device is downstream.
  1195. */
  1196. #define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
  1197. /*
  1198. * For multifunction devices which are not isolated from each other, find
  1199. * all the other non-isolated functions and look for existing groups. For
  1200. * each function, we also need to look for aliases to or from other devices
  1201. * that may already have a group.
  1202. */
  1203. static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev,
  1204. unsigned long *devfns)
  1205. {
  1206. struct pci_dev *tmp = NULL;
  1207. struct iommu_group *group;
  1208. if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS))
  1209. return NULL;
  1210. for_each_pci_dev(tmp) {
  1211. if (tmp == pdev || tmp->bus != pdev->bus ||
  1212. PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) ||
  1213. pci_acs_enabled(tmp, REQ_ACS_FLAGS))
  1214. continue;
  1215. group = get_pci_alias_group(tmp, devfns);
  1216. if (group) {
  1217. pci_dev_put(tmp);
  1218. return group;
  1219. }
  1220. }
  1221. return NULL;
  1222. }
  1223. /*
  1224. * Look for aliases to or from the given device for existing groups. DMA
  1225. * aliases are only supported on the same bus, therefore the search
  1226. * space is quite small (especially since we're really only looking at pcie
  1227. * device, and therefore only expect multiple slots on the root complex or
  1228. * downstream switch ports). It's conceivable though that a pair of
  1229. * multifunction devices could have aliases between them that would cause a
  1230. * loop. To prevent this, we use a bitmap to track where we've been.
  1231. */
  1232. static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
  1233. unsigned long *devfns)
  1234. {
  1235. struct pci_dev *tmp = NULL;
  1236. struct iommu_group *group;
  1237. if (test_and_set_bit(pdev->devfn & 0xff, devfns))
  1238. return NULL;
  1239. group = iommu_group_get(&pdev->dev);
  1240. if (group)
  1241. return group;
  1242. for_each_pci_dev(tmp) {
  1243. if (tmp == pdev || tmp->bus != pdev->bus)
  1244. continue;
  1245. /* We alias them or they alias us */
  1246. if (pci_devs_are_dma_aliases(pdev, tmp)) {
  1247. group = get_pci_alias_group(tmp, devfns);
  1248. if (group) {
  1249. pci_dev_put(tmp);
  1250. return group;
  1251. }
  1252. group = get_pci_function_alias_group(tmp, devfns);
  1253. if (group) {
  1254. pci_dev_put(tmp);
  1255. return group;
  1256. }
  1257. }
  1258. }
  1259. return NULL;
  1260. }
  1261. struct group_for_pci_data {
  1262. struct pci_dev *pdev;
  1263. struct iommu_group *group;
  1264. };
  1265. /*
  1266. * DMA alias iterator callback, return the last seen device. Stop and return
  1267. * the IOMMU group if we find one along the way.
  1268. */
  1269. static int get_pci_alias_or_group(struct pci_dev *pdev, u16 alias, void *opaque)
  1270. {
  1271. struct group_for_pci_data *data = opaque;
  1272. data->pdev = pdev;
  1273. data->group = iommu_group_get(&pdev->dev);
  1274. return data->group != NULL;
  1275. }
  1276. /*
  1277. * Generic device_group call-back function. It just allocates one
  1278. * iommu-group per device.
  1279. */
  1280. struct iommu_group *generic_device_group(struct device *dev)
  1281. {
  1282. return iommu_group_alloc();
  1283. }
  1284. EXPORT_SYMBOL_GPL(generic_device_group);
  1285. /*
  1286. * Use standard PCI bus topology, isolation features, and DMA alias quirks
  1287. * to find or create an IOMMU group for a device.
  1288. */
  1289. struct iommu_group *pci_device_group(struct device *dev)
  1290. {
  1291. struct pci_dev *pdev = to_pci_dev(dev);
  1292. struct group_for_pci_data data;
  1293. struct pci_bus *bus;
  1294. struct iommu_group *group = NULL;
  1295. u64 devfns[4] = { 0 };
  1296. if (WARN_ON(!dev_is_pci(dev)))
  1297. return ERR_PTR(-EINVAL);
  1298. /*
  1299. * Find the upstream DMA alias for the device. A device must not
  1300. * be aliased due to topology in order to have its own IOMMU group.
  1301. * If we find an alias along the way that already belongs to a
  1302. * group, use it.
  1303. */
  1304. if (pci_for_each_dma_alias(pdev, get_pci_alias_or_group, &data))
  1305. return data.group;
  1306. pdev = data.pdev;
  1307. /*
  1308. * Continue upstream from the point of minimum IOMMU granularity
  1309. * due to aliases to the point where devices are protected from
  1310. * peer-to-peer DMA by PCI ACS. Again, if we find an existing
  1311. * group, use it.
  1312. */
  1313. for (bus = pdev->bus; !pci_is_root_bus(bus); bus = bus->parent) {
  1314. if (!bus->self)
  1315. continue;
  1316. if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
  1317. break;
  1318. pdev = bus->self;
  1319. group = iommu_group_get(&pdev->dev);
  1320. if (group)
  1321. return group;
  1322. }
  1323. /*
  1324. * Look for existing groups on device aliases. If we alias another
  1325. * device or another device aliases us, use the same group.
  1326. */
  1327. group = get_pci_alias_group(pdev, (unsigned long *)devfns);
  1328. if (group)
  1329. return group;
  1330. /*
  1331. * Look for existing groups on non-isolated functions on the same
  1332. * slot and aliases of those funcions, if any. No need to clear
  1333. * the search bitmap, the tested devfns are still valid.
  1334. */
  1335. group = get_pci_function_alias_group(pdev, (unsigned long *)devfns);
  1336. if (group)
  1337. return group;
  1338. /* No shared group found, allocate new */
  1339. return iommu_group_alloc();
  1340. }
  1341. EXPORT_SYMBOL_GPL(pci_device_group);
  1342. /* Get the IOMMU group for device on fsl-mc bus */
  1343. struct iommu_group *fsl_mc_device_group(struct device *dev)
  1344. {
  1345. struct device *cont_dev = fsl_mc_cont_dev(dev);
  1346. struct iommu_group *group;
  1347. group = iommu_group_get(cont_dev);
  1348. if (!group)
  1349. group = iommu_group_alloc();
  1350. return group;
  1351. }
  1352. EXPORT_SYMBOL_GPL(fsl_mc_device_group);
  1353. static int iommu_get_def_domain_type(struct device *dev)
  1354. {
  1355. const struct iommu_ops *ops = dev_iommu_ops(dev);
  1356. if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted)
  1357. return IOMMU_DOMAIN_DMA;
  1358. if (ops->def_domain_type)
  1359. return ops->def_domain_type(dev);
  1360. return 0;
  1361. }
  1362. static int iommu_group_alloc_default_domain(struct bus_type *bus,
  1363. struct iommu_group *group,
  1364. unsigned int type)
  1365. {
  1366. struct iommu_domain *dom;
  1367. dom = __iommu_domain_alloc(bus, type);
  1368. if (!dom && type != IOMMU_DOMAIN_DMA) {
  1369. dom = __iommu_domain_alloc(bus, IOMMU_DOMAIN_DMA);
  1370. if (dom)
  1371. pr_warn("Failed to allocate default IOMMU domain of type %u for group %s - Falling back to IOMMU_DOMAIN_DMA",
  1372. type, group->name);
  1373. }
  1374. if (!dom)
  1375. return -ENOMEM;
  1376. group->default_domain = dom;
  1377. if (!group->domain)
  1378. group->domain = dom;
  1379. return 0;
  1380. }
  1381. static int iommu_alloc_default_domain(struct iommu_group *group,
  1382. struct device *dev)
  1383. {
  1384. unsigned int type;
  1385. if (group->default_domain)
  1386. return 0;
  1387. type = iommu_get_def_domain_type(dev) ? : iommu_def_domain_type;
  1388. return iommu_group_alloc_default_domain(dev->bus, group, type);
  1389. }
  1390. /**
  1391. * iommu_group_get_for_dev - Find or create the IOMMU group for a device
  1392. * @dev: target device
  1393. *
  1394. * This function is intended to be called by IOMMU drivers and extended to
  1395. * support common, bus-defined algorithms when determining or creating the
  1396. * IOMMU group for a device. On success, the caller will hold a reference
  1397. * to the returned IOMMU group, which will already include the provided
  1398. * device. The reference should be released with iommu_group_put().
  1399. */
  1400. static struct iommu_group *iommu_group_get_for_dev(struct device *dev)
  1401. {
  1402. const struct iommu_ops *ops = dev_iommu_ops(dev);
  1403. struct iommu_group *group;
  1404. int ret;
  1405. group = iommu_group_get(dev);
  1406. if (group)
  1407. return group;
  1408. group = ops->device_group(dev);
  1409. if (WARN_ON_ONCE(group == NULL))
  1410. return ERR_PTR(-EINVAL);
  1411. if (IS_ERR(group))
  1412. return group;
  1413. ret = iommu_group_add_device(group, dev);
  1414. if (ret)
  1415. goto out_put_group;
  1416. return group;
  1417. out_put_group:
  1418. iommu_group_put(group);
  1419. return ERR_PTR(ret);
  1420. }
  1421. struct iommu_domain *iommu_group_default_domain(struct iommu_group *group)
  1422. {
  1423. return group->default_domain;
  1424. }
  1425. static int probe_iommu_group(struct device *dev, void *data)
  1426. {
  1427. struct list_head *group_list = data;
  1428. struct iommu_group *group;
  1429. int ret;
  1430. /* Device is probed already if in a group */
  1431. group = iommu_group_get(dev);
  1432. if (group) {
  1433. iommu_group_put(group);
  1434. return 0;
  1435. }
  1436. mutex_lock(&iommu_probe_device_lock);
  1437. ret = __iommu_probe_device(dev, group_list);
  1438. mutex_unlock(&iommu_probe_device_lock);
  1439. if (ret == -ENODEV)
  1440. ret = 0;
  1441. return ret;
  1442. }
  1443. static int iommu_bus_notifier(struct notifier_block *nb,
  1444. unsigned long action, void *data)
  1445. {
  1446. struct device *dev = data;
  1447. if (action == BUS_NOTIFY_ADD_DEVICE) {
  1448. int ret;
  1449. ret = iommu_probe_device(dev);
  1450. return (ret) ? NOTIFY_DONE : NOTIFY_OK;
  1451. } else if (action == BUS_NOTIFY_REMOVED_DEVICE) {
  1452. iommu_release_device(dev);
  1453. return NOTIFY_OK;
  1454. }
  1455. return 0;
  1456. }
  1457. struct __group_domain_type {
  1458. struct device *dev;
  1459. unsigned int type;
  1460. };
  1461. static int probe_get_default_domain_type(struct device *dev, void *data)
  1462. {
  1463. struct __group_domain_type *gtype = data;
  1464. unsigned int type = iommu_get_def_domain_type(dev);
  1465. if (type) {
  1466. if (gtype->type && gtype->type != type) {
  1467. dev_warn(dev, "Device needs domain type %s, but device %s in the same iommu group requires type %s - using default\n",
  1468. iommu_domain_type_str(type),
  1469. dev_name(gtype->dev),
  1470. iommu_domain_type_str(gtype->type));
  1471. gtype->type = 0;
  1472. }
  1473. if (!gtype->dev) {
  1474. gtype->dev = dev;
  1475. gtype->type = type;
  1476. }
  1477. }
  1478. return 0;
  1479. }
  1480. static void probe_alloc_default_domain(struct bus_type *bus,
  1481. struct iommu_group *group)
  1482. {
  1483. struct __group_domain_type gtype;
  1484. if (group->default_domain)
  1485. return;
  1486. memset(&gtype, 0, sizeof(gtype));
  1487. /* Ask for default domain requirements of all devices in the group */
  1488. __iommu_group_for_each_dev(group, &gtype,
  1489. probe_get_default_domain_type);
  1490. if (!gtype.type)
  1491. gtype.type = iommu_def_domain_type;
  1492. iommu_group_alloc_default_domain(bus, group, gtype.type);
  1493. }
  1494. static int iommu_group_do_dma_attach(struct device *dev, void *data)
  1495. {
  1496. struct iommu_domain *domain = data;
  1497. int ret = 0;
  1498. if (!iommu_is_attach_deferred(dev))
  1499. ret = __iommu_attach_device(domain, dev);
  1500. return ret;
  1501. }
  1502. static int __iommu_group_dma_attach(struct iommu_group *group)
  1503. {
  1504. return __iommu_group_for_each_dev(group, group->default_domain,
  1505. iommu_group_do_dma_attach);
  1506. }
  1507. static int iommu_group_do_probe_finalize(struct device *dev, void *data)
  1508. {
  1509. const struct iommu_ops *ops = dev_iommu_ops(dev);
  1510. if (ops->probe_finalize)
  1511. ops->probe_finalize(dev);
  1512. return 0;
  1513. }
  1514. static void __iommu_group_dma_finalize(struct iommu_group *group)
  1515. {
  1516. __iommu_group_for_each_dev(group, group->default_domain,
  1517. iommu_group_do_probe_finalize);
  1518. }
  1519. static int iommu_do_create_direct_mappings(struct device *dev, void *data)
  1520. {
  1521. struct iommu_group *group = data;
  1522. iommu_create_device_direct_mappings(group, dev);
  1523. return 0;
  1524. }
  1525. static int iommu_group_create_direct_mappings(struct iommu_group *group)
  1526. {
  1527. return __iommu_group_for_each_dev(group, group,
  1528. iommu_do_create_direct_mappings);
  1529. }
  1530. int bus_iommu_probe(struct bus_type *bus)
  1531. {
  1532. struct iommu_group *group, *next;
  1533. LIST_HEAD(group_list);
  1534. int ret;
  1535. /*
  1536. * This code-path does not allocate the default domain when
  1537. * creating the iommu group, so do it after the groups are
  1538. * created.
  1539. */
  1540. ret = bus_for_each_dev(bus, NULL, &group_list, probe_iommu_group);
  1541. if (ret)
  1542. return ret;
  1543. list_for_each_entry_safe(group, next, &group_list, entry) {
  1544. mutex_lock(&group->mutex);
  1545. /* Remove item from the list */
  1546. list_del_init(&group->entry);
  1547. /* Try to allocate default domain */
  1548. probe_alloc_default_domain(bus, group);
  1549. if (!group->default_domain) {
  1550. mutex_unlock(&group->mutex);
  1551. continue;
  1552. }
  1553. iommu_group_create_direct_mappings(group);
  1554. ret = __iommu_group_dma_attach(group);
  1555. mutex_unlock(&group->mutex);
  1556. if (ret)
  1557. break;
  1558. __iommu_group_dma_finalize(group);
  1559. }
  1560. return ret;
  1561. }
  1562. bool iommu_present(struct bus_type *bus)
  1563. {
  1564. return bus->iommu_ops != NULL;
  1565. }
  1566. EXPORT_SYMBOL_GPL(iommu_present);
  1567. /**
  1568. * device_iommu_capable() - check for a general IOMMU capability
  1569. * @dev: device to which the capability would be relevant, if available
  1570. * @cap: IOMMU capability
  1571. *
  1572. * Return: true if an IOMMU is present and supports the given capability
  1573. * for the given device, otherwise false.
  1574. */
  1575. bool device_iommu_capable(struct device *dev, enum iommu_cap cap)
  1576. {
  1577. const struct iommu_ops *ops;
  1578. if (!dev->iommu || !dev->iommu->iommu_dev)
  1579. return false;
  1580. ops = dev_iommu_ops(dev);
  1581. if (!ops->capable)
  1582. return false;
  1583. return ops->capable(dev, cap);
  1584. }
  1585. EXPORT_SYMBOL_GPL(device_iommu_capable);
  1586. /**
  1587. * iommu_set_fault_handler() - set a fault handler for an iommu domain
  1588. * @domain: iommu domain
  1589. * @handler: fault handler
  1590. * @token: user data, will be passed back to the fault handler
  1591. *
  1592. * This function should be used by IOMMU users which want to be notified
  1593. * whenever an IOMMU fault happens.
  1594. *
  1595. * The fault handler itself should return 0 on success, and an appropriate
  1596. * error code otherwise.
  1597. */
  1598. void iommu_set_fault_handler(struct iommu_domain *domain,
  1599. iommu_fault_handler_t handler,
  1600. void *token)
  1601. {
  1602. BUG_ON(!domain);
  1603. domain->handler = handler;
  1604. domain->handler_token = token;
  1605. }
  1606. EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
  1607. static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
  1608. unsigned type)
  1609. {
  1610. struct iommu_domain *domain;
  1611. if (bus == NULL || bus->iommu_ops == NULL)
  1612. return NULL;
  1613. domain = bus->iommu_ops->domain_alloc(type);
  1614. if (!domain)
  1615. return NULL;
  1616. domain->type = type;
  1617. /*
  1618. * If not already set, assume all sizes by default; the driver
  1619. * may override this later
  1620. */
  1621. if (!domain->pgsize_bitmap)
  1622. domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap;
  1623. if (!domain->ops)
  1624. domain->ops = bus->iommu_ops->default_domain_ops;
  1625. if (iommu_is_dma_domain(domain) && iommu_get_dma_cookie(domain)) {
  1626. iommu_domain_free(domain);
  1627. domain = NULL;
  1628. }
  1629. return domain;
  1630. }
  1631. struct iommu_domain *iommu_domain_alloc(struct bus_type *bus)
  1632. {
  1633. return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED);
  1634. }
  1635. EXPORT_SYMBOL_GPL(iommu_domain_alloc);
  1636. void iommu_domain_free(struct iommu_domain *domain)
  1637. {
  1638. if (domain->type == IOMMU_DOMAIN_SVA)
  1639. mmdrop(domain->mm);
  1640. iommu_put_dma_cookie(domain);
  1641. domain->ops->free(domain);
  1642. }
  1643. EXPORT_SYMBOL_GPL(iommu_domain_free);
  1644. /*
  1645. * Put the group's domain back to the appropriate core-owned domain - either the
  1646. * standard kernel-mode DMA configuration or an all-DMA-blocked domain.
  1647. */
  1648. static void __iommu_group_set_core_domain(struct iommu_group *group)
  1649. {
  1650. struct iommu_domain *new_domain;
  1651. int ret;
  1652. if (group->owner)
  1653. new_domain = group->blocking_domain;
  1654. else
  1655. new_domain = group->default_domain;
  1656. ret = __iommu_group_set_domain(group, new_domain);
  1657. WARN(ret, "iommu driver failed to attach the default/blocking domain");
  1658. }
  1659. static int __iommu_attach_device(struct iommu_domain *domain,
  1660. struct device *dev)
  1661. {
  1662. int ret;
  1663. if (unlikely(domain->ops->attach_dev == NULL))
  1664. return -ENODEV;
  1665. ret = domain->ops->attach_dev(domain, dev);
  1666. if (!ret)
  1667. trace_attach_device_to_domain(dev);
  1668. return ret;
  1669. }
  1670. int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
  1671. {
  1672. struct iommu_group *group;
  1673. int ret;
  1674. group = iommu_group_get(dev);
  1675. if (!group)
  1676. return -ENODEV;
  1677. /*
  1678. * Lock the group to make sure the device-count doesn't
  1679. * change while we are attaching
  1680. */
  1681. mutex_lock(&group->mutex);
  1682. ret = -EINVAL;
  1683. if (iommu_group_device_count(group) != 1)
  1684. goto out_unlock;
  1685. ret = __iommu_attach_group(domain, group);
  1686. out_unlock:
  1687. mutex_unlock(&group->mutex);
  1688. iommu_group_put(group);
  1689. return ret;
  1690. }
  1691. EXPORT_SYMBOL_GPL(iommu_attach_device);
  1692. int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain)
  1693. {
  1694. if (iommu_is_attach_deferred(dev))
  1695. return __iommu_attach_device(domain, dev);
  1696. return 0;
  1697. }
  1698. static void __iommu_detach_device(struct iommu_domain *domain,
  1699. struct device *dev)
  1700. {
  1701. if (iommu_is_attach_deferred(dev))
  1702. return;
  1703. domain->ops->detach_dev(domain, dev);
  1704. trace_detach_device_from_domain(dev);
  1705. }
  1706. void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
  1707. {
  1708. struct iommu_group *group;
  1709. group = iommu_group_get(dev);
  1710. if (!group)
  1711. return;
  1712. mutex_lock(&group->mutex);
  1713. if (WARN_ON(domain != group->domain) ||
  1714. WARN_ON(iommu_group_device_count(group) != 1))
  1715. goto out_unlock;
  1716. __iommu_group_set_core_domain(group);
  1717. out_unlock:
  1718. mutex_unlock(&group->mutex);
  1719. iommu_group_put(group);
  1720. }
  1721. EXPORT_SYMBOL_GPL(iommu_detach_device);
  1722. struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
  1723. {
  1724. struct iommu_domain *domain;
  1725. struct iommu_group *group;
  1726. group = iommu_group_get(dev);
  1727. if (!group)
  1728. return NULL;
  1729. domain = group->domain;
  1730. iommu_group_put(group);
  1731. return domain;
  1732. }
  1733. EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);
  1734. /*
  1735. * For IOMMU_DOMAIN_DMA implementations which already provide their own
  1736. * guarantees that the group and its default domain are valid and correct.
  1737. */
  1738. struct iommu_domain *iommu_get_dma_domain(struct device *dev)
  1739. {
  1740. return dev->iommu_group->default_domain;
  1741. }
  1742. /*
  1743. * IOMMU groups are really the natural working unit of the IOMMU, but
  1744. * the IOMMU API works on domains and devices. Bridge that gap by
  1745. * iterating over the devices in a group. Ideally we'd have a single
  1746. * device which represents the requestor ID of the group, but we also
  1747. * allow IOMMU drivers to create policy defined minimum sets, where
  1748. * the physical hardware may be able to distiguish members, but we
  1749. * wish to group them at a higher level (ex. untrusted multi-function
  1750. * PCI devices). Thus we attach each device.
  1751. */
  1752. static int iommu_group_do_attach_device(struct device *dev, void *data)
  1753. {
  1754. struct iommu_domain *domain = data;
  1755. return __iommu_attach_device(domain, dev);
  1756. }
  1757. static int __iommu_attach_group(struct iommu_domain *domain,
  1758. struct iommu_group *group)
  1759. {
  1760. int ret;
  1761. if (group->domain && group->domain != group->default_domain &&
  1762. group->domain != group->blocking_domain)
  1763. return -EBUSY;
  1764. ret = __iommu_group_for_each_dev(group, domain,
  1765. iommu_group_do_attach_device);
  1766. if (ret == 0) {
  1767. group->domain = domain;
  1768. } else {
  1769. /*
  1770. * To recover from the case when certain device within the
  1771. * group fails to attach to the new domain, we need force
  1772. * attaching all devices back to the old domain. The old
  1773. * domain is compatible for all devices in the group,
  1774. * hence the iommu driver should always return success.
  1775. */
  1776. struct iommu_domain *old_domain = group->domain;
  1777. group->domain = NULL;
  1778. WARN(__iommu_group_set_domain(group, old_domain),
  1779. "iommu driver failed to attach a compatible domain");
  1780. }
  1781. return ret;
  1782. }
  1783. int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group)
  1784. {
  1785. int ret;
  1786. mutex_lock(&group->mutex);
  1787. ret = __iommu_attach_group(domain, group);
  1788. mutex_unlock(&group->mutex);
  1789. return ret;
  1790. }
  1791. EXPORT_SYMBOL_GPL(iommu_attach_group);
  1792. static int iommu_group_do_detach_device(struct device *dev, void *data)
  1793. {
  1794. struct iommu_domain *domain = data;
  1795. __iommu_detach_device(domain, dev);
  1796. return 0;
  1797. }
  1798. static int __iommu_group_set_domain(struct iommu_group *group,
  1799. struct iommu_domain *new_domain)
  1800. {
  1801. int ret;
  1802. if (group->domain == new_domain)
  1803. return 0;
  1804. /*
  1805. * New drivers should support default domains and so the detach_dev() op
  1806. * will never be called. Otherwise the NULL domain represents some
  1807. * platform specific behavior.
  1808. */
  1809. if (!new_domain) {
  1810. if (WARN_ON(!group->domain->ops->detach_dev))
  1811. return -EINVAL;
  1812. __iommu_group_for_each_dev(group, group->domain,
  1813. iommu_group_do_detach_device);
  1814. group->domain = NULL;
  1815. return 0;
  1816. }
  1817. /*
  1818. * Changing the domain is done by calling attach_dev() on the new
  1819. * domain. This switch does not have to be atomic and DMA can be
  1820. * discarded during the transition. DMA must only be able to access
  1821. * either new_domain or group->domain, never something else.
  1822. *
  1823. * Note that this is called in error unwind paths, attaching to a
  1824. * domain that has already been attached cannot fail.
  1825. */
  1826. ret = __iommu_group_for_each_dev(group, new_domain,
  1827. iommu_group_do_attach_device);
  1828. if (ret)
  1829. return ret;
  1830. group->domain = new_domain;
  1831. return 0;
  1832. }
  1833. void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group)
  1834. {
  1835. mutex_lock(&group->mutex);
  1836. __iommu_group_set_core_domain(group);
  1837. mutex_unlock(&group->mutex);
  1838. }
  1839. EXPORT_SYMBOL_GPL(iommu_detach_group);
  1840. phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
  1841. {
  1842. if (domain->type == IOMMU_DOMAIN_IDENTITY)
  1843. return iova;
  1844. if (domain->type == IOMMU_DOMAIN_BLOCKED)
  1845. return 0;
  1846. return domain->ops->iova_to_phys(domain, iova);
  1847. }
  1848. EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
  1849. static size_t iommu_pgsize(struct iommu_domain *domain, unsigned long iova,
  1850. phys_addr_t paddr, size_t size, size_t *count)
  1851. {
  1852. unsigned int pgsize_idx, pgsize_idx_next;
  1853. unsigned long pgsizes;
  1854. size_t offset, pgsize, pgsize_next;
  1855. unsigned long addr_merge = paddr | iova;
  1856. /* Page sizes supported by the hardware and small enough for @size */
  1857. pgsizes = domain->pgsize_bitmap & GENMASK(__fls(size), 0);
  1858. /* Constrain the page sizes further based on the maximum alignment */
  1859. if (likely(addr_merge))
  1860. pgsizes &= GENMASK(__ffs(addr_merge), 0);
  1861. /* Make sure we have at least one suitable page size */
  1862. BUG_ON(!pgsizes);
  1863. /* Pick the biggest page size remaining */
  1864. pgsize_idx = __fls(pgsizes);
  1865. pgsize = BIT(pgsize_idx);
  1866. if (!count)
  1867. return pgsize;
  1868. /* Find the next biggest support page size, if it exists */
  1869. pgsizes = domain->pgsize_bitmap & ~GENMASK(pgsize_idx, 0);
  1870. if (!pgsizes)
  1871. goto out_set_count;
  1872. pgsize_idx_next = __ffs(pgsizes);
  1873. pgsize_next = BIT(pgsize_idx_next);
  1874. /*
  1875. * There's no point trying a bigger page size unless the virtual
  1876. * and physical addresses are similarly offset within the larger page.
  1877. */
  1878. if ((iova ^ paddr) & (pgsize_next - 1))
  1879. goto out_set_count;
  1880. /* Calculate the offset to the next page size alignment boundary */
  1881. offset = pgsize_next - (addr_merge & (pgsize_next - 1));
  1882. /*
  1883. * If size is big enough to accommodate the larger page, reduce
  1884. * the number of smaller pages.
  1885. */
  1886. if (offset + pgsize_next <= size)
  1887. size = offset;
  1888. out_set_count:
  1889. *count = size >> pgsize_idx;
  1890. return pgsize;
  1891. }
  1892. static int __iommu_map_pages(struct iommu_domain *domain, unsigned long iova,
  1893. phys_addr_t paddr, size_t size, int prot,
  1894. gfp_t gfp, size_t *mapped)
  1895. {
  1896. const struct iommu_domain_ops *ops = domain->ops;
  1897. size_t pgsize, count;
  1898. int ret;
  1899. pgsize = iommu_pgsize(domain, iova, paddr, size, &count);
  1900. pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx count %zu\n",
  1901. iova, &paddr, pgsize, count);
  1902. if (ops->map_pages) {
  1903. ret = ops->map_pages(domain, iova, paddr, pgsize, count, prot,
  1904. gfp, mapped);
  1905. } else {
  1906. ret = ops->map(domain, iova, paddr, pgsize, prot, gfp);
  1907. *mapped = ret ? 0 : pgsize;
  1908. }
  1909. return ret;
  1910. }
  1911. static int __iommu_map(struct iommu_domain *domain, unsigned long iova,
  1912. phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
  1913. {
  1914. const struct iommu_domain_ops *ops = domain->ops;
  1915. unsigned long orig_iova = iova;
  1916. unsigned int min_pagesz;
  1917. size_t orig_size = size;
  1918. phys_addr_t orig_paddr = paddr;
  1919. int ret = 0;
  1920. if (unlikely(!(ops->map || ops->map_pages) ||
  1921. domain->pgsize_bitmap == 0UL))
  1922. return -ENODEV;
  1923. if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
  1924. return -EINVAL;
  1925. /* find out the minimum page size supported */
  1926. min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
  1927. /*
  1928. * both the virtual address and the physical one, as well as
  1929. * the size of the mapping, must be aligned (at least) to the
  1930. * size of the smallest page supported by the hardware
  1931. */
  1932. if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
  1933. pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n",
  1934. iova, &paddr, size, min_pagesz);
  1935. return -EINVAL;
  1936. }
  1937. pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size);
  1938. while (size) {
  1939. size_t mapped = 0;
  1940. ret = __iommu_map_pages(domain, iova, paddr, size, prot, gfp,
  1941. &mapped);
  1942. /*
  1943. * Some pages may have been mapped, even if an error occurred,
  1944. * so we should account for those so they can be unmapped.
  1945. */
  1946. size -= mapped;
  1947. if (ret)
  1948. break;
  1949. iova += mapped;
  1950. paddr += mapped;
  1951. }
  1952. /* unroll mapping in case something went wrong */
  1953. if (ret)
  1954. iommu_unmap(domain, orig_iova, orig_size - size);
  1955. else
  1956. trace_map(orig_iova, orig_paddr, orig_size);
  1957. return ret;
  1958. }
  1959. static int _iommu_map(struct iommu_domain *domain, unsigned long iova,
  1960. phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
  1961. {
  1962. const struct iommu_domain_ops *ops = domain->ops;
  1963. int ret;
  1964. ret = __iommu_map(domain, iova, paddr, size, prot, gfp);
  1965. if (ret == 0 && ops->iotlb_sync_map)
  1966. ops->iotlb_sync_map(domain, iova, size);
  1967. return ret;
  1968. }
  1969. int iommu_map(struct iommu_domain *domain, unsigned long iova,
  1970. phys_addr_t paddr, size_t size, int prot)
  1971. {
  1972. might_sleep();
  1973. return _iommu_map(domain, iova, paddr, size, prot, GFP_KERNEL);
  1974. }
  1975. EXPORT_SYMBOL_GPL(iommu_map);
  1976. int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova,
  1977. phys_addr_t paddr, size_t size, int prot)
  1978. {
  1979. return _iommu_map(domain, iova, paddr, size, prot, GFP_ATOMIC);
  1980. }
  1981. EXPORT_SYMBOL_GPL(iommu_map_atomic);
  1982. static size_t __iommu_unmap_pages(struct iommu_domain *domain,
  1983. unsigned long iova, size_t size,
  1984. struct iommu_iotlb_gather *iotlb_gather)
  1985. {
  1986. const struct iommu_domain_ops *ops = domain->ops;
  1987. size_t pgsize, count;
  1988. pgsize = iommu_pgsize(domain, iova, iova, size, &count);
  1989. return ops->unmap_pages ?
  1990. ops->unmap_pages(domain, iova, pgsize, count, iotlb_gather) :
  1991. ops->unmap(domain, iova, pgsize, iotlb_gather);
  1992. }
  1993. static size_t __iommu_unmap(struct iommu_domain *domain,
  1994. unsigned long iova, size_t size,
  1995. struct iommu_iotlb_gather *iotlb_gather)
  1996. {
  1997. const struct iommu_domain_ops *ops = domain->ops;
  1998. size_t unmapped_page, unmapped = 0;
  1999. unsigned long orig_iova = iova;
  2000. unsigned int min_pagesz;
  2001. if (unlikely(!(ops->unmap || ops->unmap_pages) ||
  2002. domain->pgsize_bitmap == 0UL))
  2003. return 0;
  2004. if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
  2005. return 0;
  2006. /* find out the minimum page size supported */
  2007. min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
  2008. /*
  2009. * The virtual address, as well as the size of the mapping, must be
  2010. * aligned (at least) to the size of the smallest page supported
  2011. * by the hardware
  2012. */
  2013. if (!IS_ALIGNED(iova | size, min_pagesz)) {
  2014. pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n",
  2015. iova, size, min_pagesz);
  2016. return 0;
  2017. }
  2018. pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size);
  2019. /*
  2020. * Keep iterating until we either unmap 'size' bytes (or more)
  2021. * or we hit an area that isn't mapped.
  2022. */
  2023. while (unmapped < size) {
  2024. unmapped_page = __iommu_unmap_pages(domain, iova,
  2025. size - unmapped,
  2026. iotlb_gather);
  2027. if (!unmapped_page)
  2028. break;
  2029. pr_debug("unmapped: iova 0x%lx size 0x%zx\n",
  2030. iova, unmapped_page);
  2031. iova += unmapped_page;
  2032. unmapped += unmapped_page;
  2033. }
  2034. trace_unmap(orig_iova, size, unmapped);
  2035. return unmapped;
  2036. }
  2037. size_t iommu_unmap(struct iommu_domain *domain,
  2038. unsigned long iova, size_t size)
  2039. {
  2040. struct iommu_iotlb_gather iotlb_gather;
  2041. size_t ret;
  2042. iommu_iotlb_gather_init(&iotlb_gather);
  2043. ret = __iommu_unmap(domain, iova, size, &iotlb_gather);
  2044. iommu_iotlb_sync(domain, &iotlb_gather);
  2045. return ret;
  2046. }
  2047. EXPORT_SYMBOL_GPL(iommu_unmap);
  2048. size_t iommu_unmap_fast(struct iommu_domain *domain,
  2049. unsigned long iova, size_t size,
  2050. struct iommu_iotlb_gather *iotlb_gather)
  2051. {
  2052. return __iommu_unmap(domain, iova, size, iotlb_gather);
  2053. }
  2054. EXPORT_SYMBOL_GPL(iommu_unmap_fast);
  2055. static ssize_t __iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
  2056. struct scatterlist *sg, unsigned int nents, int prot,
  2057. gfp_t gfp)
  2058. {
  2059. const struct iommu_domain_ops *ops = domain->ops;
  2060. size_t len = 0, mapped = 0;
  2061. phys_addr_t start;
  2062. unsigned int i = 0;
  2063. int ret;
  2064. while (i <= nents) {
  2065. phys_addr_t s_phys = sg_phys(sg);
  2066. if (len && s_phys != start + len) {
  2067. ret = __iommu_map(domain, iova + mapped, start,
  2068. len, prot, gfp);
  2069. if (ret)
  2070. goto out_err;
  2071. mapped += len;
  2072. len = 0;
  2073. }
  2074. if (sg_is_dma_bus_address(sg))
  2075. goto next;
  2076. if (len) {
  2077. len += sg->length;
  2078. } else {
  2079. len = sg->length;
  2080. start = s_phys;
  2081. }
  2082. next:
  2083. if (++i < nents)
  2084. sg = sg_next(sg);
  2085. }
  2086. if (ops->iotlb_sync_map)
  2087. ops->iotlb_sync_map(domain, iova, mapped);
  2088. return mapped;
  2089. out_err:
  2090. /* undo mappings already done */
  2091. iommu_unmap(domain, iova, mapped);
  2092. return ret;
  2093. }
  2094. ssize_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
  2095. struct scatterlist *sg, unsigned int nents, int prot)
  2096. {
  2097. might_sleep();
  2098. return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_KERNEL);
  2099. }
  2100. EXPORT_SYMBOL_GPL(iommu_map_sg);
  2101. ssize_t iommu_map_sg_atomic(struct iommu_domain *domain, unsigned long iova,
  2102. struct scatterlist *sg, unsigned int nents, int prot)
  2103. {
  2104. return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_ATOMIC);
  2105. }
  2106. /**
  2107. * report_iommu_fault() - report about an IOMMU fault to the IOMMU framework
  2108. * @domain: the iommu domain where the fault has happened
  2109. * @dev: the device where the fault has happened
  2110. * @iova: the faulting address
  2111. * @flags: mmu fault flags (e.g. IOMMU_FAULT_READ/IOMMU_FAULT_WRITE/...)
  2112. *
  2113. * This function should be called by the low-level IOMMU implementations
  2114. * whenever IOMMU faults happen, to allow high-level users, that are
  2115. * interested in such events, to know about them.
  2116. *
  2117. * This event may be useful for several possible use cases:
  2118. * - mere logging of the event
  2119. * - dynamic TLB/PTE loading
  2120. * - if restarting of the faulting device is required
  2121. *
  2122. * Returns 0 on success and an appropriate error code otherwise (if dynamic
  2123. * PTE/TLB loading will one day be supported, implementations will be able
  2124. * to tell whether it succeeded or not according to this return value).
  2125. *
  2126. * Specifically, -ENOSYS is returned if a fault handler isn't installed
  2127. * (though fault handlers can also return -ENOSYS, in case they want to
  2128. * elicit the default behavior of the IOMMU drivers).
  2129. */
  2130. int report_iommu_fault(struct iommu_domain *domain, struct device *dev,
  2131. unsigned long iova, int flags)
  2132. {
  2133. int ret = -ENOSYS;
  2134. /*
  2135. * if upper layers showed interest and installed a fault handler,
  2136. * invoke it.
  2137. */
  2138. if (domain->handler)
  2139. ret = domain->handler(domain, dev, iova, flags,
  2140. domain->handler_token);
  2141. trace_io_page_fault(dev, iova, flags);
  2142. return ret;
  2143. }
  2144. EXPORT_SYMBOL_GPL(report_iommu_fault);
  2145. static int __init iommu_init(void)
  2146. {
  2147. iommu_group_kset = kset_create_and_add("iommu_groups",
  2148. NULL, kernel_kobj);
  2149. BUG_ON(!iommu_group_kset);
  2150. iommu_debugfs_setup();
  2151. return 0;
  2152. }
  2153. core_initcall(iommu_init);
  2154. int iommu_enable_nesting(struct iommu_domain *domain)
  2155. {
  2156. if (domain->type != IOMMU_DOMAIN_UNMANAGED)
  2157. return -EINVAL;
  2158. if (!domain->ops->enable_nesting)
  2159. return -EINVAL;
  2160. return domain->ops->enable_nesting(domain);
  2161. }
  2162. EXPORT_SYMBOL_GPL(iommu_enable_nesting);
  2163. int iommu_set_pgtable_quirks(struct iommu_domain *domain,
  2164. unsigned long quirk)
  2165. {
  2166. if (domain->type != IOMMU_DOMAIN_UNMANAGED)
  2167. return -EINVAL;
  2168. if (!domain->ops->set_pgtable_quirks)
  2169. return -EINVAL;
  2170. return domain->ops->set_pgtable_quirks(domain, quirk);
  2171. }
  2172. EXPORT_SYMBOL_GPL(iommu_set_pgtable_quirks);
  2173. void iommu_get_resv_regions(struct device *dev, struct list_head *list)
  2174. {
  2175. const struct iommu_ops *ops = dev_iommu_ops(dev);
  2176. if (ops->get_resv_regions)
  2177. ops->get_resv_regions(dev, list);
  2178. }
  2179. /**
  2180. * iommu_put_resv_regions - release resered regions
  2181. * @dev: device for which to free reserved regions
  2182. * @list: reserved region list for device
  2183. *
  2184. * This releases a reserved region list acquired by iommu_get_resv_regions().
  2185. */
  2186. void iommu_put_resv_regions(struct device *dev, struct list_head *list)
  2187. {
  2188. struct iommu_resv_region *entry, *next;
  2189. list_for_each_entry_safe(entry, next, list, list) {
  2190. if (entry->free)
  2191. entry->free(dev, entry);
  2192. else
  2193. kfree(entry);
  2194. }
  2195. }
  2196. EXPORT_SYMBOL(iommu_put_resv_regions);
  2197. struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start,
  2198. size_t length, int prot,
  2199. enum iommu_resv_type type,
  2200. gfp_t gfp)
  2201. {
  2202. struct iommu_resv_region *region;
  2203. region = kzalloc(sizeof(*region), gfp);
  2204. if (!region)
  2205. return NULL;
  2206. INIT_LIST_HEAD(&region->list);
  2207. region->start = start;
  2208. region->length = length;
  2209. region->prot = prot;
  2210. region->type = type;
  2211. return region;
  2212. }
  2213. EXPORT_SYMBOL_GPL(iommu_alloc_resv_region);
  2214. void iommu_set_default_passthrough(bool cmd_line)
  2215. {
  2216. if (cmd_line)
  2217. iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API;
  2218. iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY;
  2219. }
  2220. void iommu_set_default_translated(bool cmd_line)
  2221. {
  2222. if (cmd_line)
  2223. iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API;
  2224. iommu_def_domain_type = IOMMU_DOMAIN_DMA;
  2225. }
  2226. bool iommu_default_passthrough(void)
  2227. {
  2228. return iommu_def_domain_type == IOMMU_DOMAIN_IDENTITY;
  2229. }
  2230. EXPORT_SYMBOL_GPL(iommu_default_passthrough);
  2231. const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode)
  2232. {
  2233. const struct iommu_ops *ops = NULL;
  2234. struct iommu_device *iommu;
  2235. spin_lock(&iommu_device_lock);
  2236. list_for_each_entry(iommu, &iommu_device_list, list)
  2237. if (iommu->fwnode == fwnode) {
  2238. ops = iommu->ops;
  2239. break;
  2240. }
  2241. spin_unlock(&iommu_device_lock);
  2242. return ops;
  2243. }
  2244. int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
  2245. const struct iommu_ops *ops)
  2246. {
  2247. struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
  2248. if (fwspec)
  2249. return ops == fwspec->ops ? 0 : -EINVAL;
  2250. if (!dev_iommu_get(dev))
  2251. return -ENOMEM;
  2252. /* Preallocate for the overwhelmingly common case of 1 ID */
  2253. fwspec = kzalloc(struct_size(fwspec, ids, 1), GFP_KERNEL);
  2254. if (!fwspec)
  2255. return -ENOMEM;
  2256. of_node_get(to_of_node(iommu_fwnode));
  2257. fwspec->iommu_fwnode = iommu_fwnode;
  2258. fwspec->ops = ops;
  2259. dev_iommu_fwspec_set(dev, fwspec);
  2260. return 0;
  2261. }
  2262. EXPORT_SYMBOL_GPL(iommu_fwspec_init);
  2263. void iommu_fwspec_free(struct device *dev)
  2264. {
  2265. struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
  2266. if (fwspec) {
  2267. fwnode_handle_put(fwspec->iommu_fwnode);
  2268. kfree(fwspec);
  2269. dev_iommu_fwspec_set(dev, NULL);
  2270. }
  2271. }
  2272. EXPORT_SYMBOL_GPL(iommu_fwspec_free);
  2273. int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids)
  2274. {
  2275. struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
  2276. int i, new_num;
  2277. if (!fwspec)
  2278. return -EINVAL;
  2279. new_num = fwspec->num_ids + num_ids;
  2280. if (new_num > 1) {
  2281. fwspec = krealloc(fwspec, struct_size(fwspec, ids, new_num),
  2282. GFP_KERNEL);
  2283. if (!fwspec)
  2284. return -ENOMEM;
  2285. dev_iommu_fwspec_set(dev, fwspec);
  2286. }
  2287. for (i = 0; i < num_ids; i++)
  2288. fwspec->ids[fwspec->num_ids + i] = ids[i];
  2289. fwspec->num_ids = new_num;
  2290. return 0;
  2291. }
  2292. EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids);
  2293. /*
  2294. * Per device IOMMU features.
  2295. */
  2296. int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat)
  2297. {
  2298. if (dev->iommu && dev->iommu->iommu_dev) {
  2299. const struct iommu_ops *ops = dev->iommu->iommu_dev->ops;
  2300. if (ops->dev_enable_feat)
  2301. return ops->dev_enable_feat(dev, feat);
  2302. }
  2303. return -ENODEV;
  2304. }
  2305. EXPORT_SYMBOL_GPL(iommu_dev_enable_feature);
  2306. /*
  2307. * The device drivers should do the necessary cleanups before calling this.
  2308. */
  2309. int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)
  2310. {
  2311. if (dev->iommu && dev->iommu->iommu_dev) {
  2312. const struct iommu_ops *ops = dev->iommu->iommu_dev->ops;
  2313. if (ops->dev_disable_feat)
  2314. return ops->dev_disable_feat(dev, feat);
  2315. }
  2316. return -EBUSY;
  2317. }
  2318. EXPORT_SYMBOL_GPL(iommu_dev_disable_feature);
  2319. /*
  2320. * Changes the default domain of an iommu group that has *only* one device
  2321. *
  2322. * @group: The group for which the default domain should be changed
  2323. * @prev_dev: The device in the group (this is used to make sure that the device
  2324. * hasn't changed after the caller has called this function)
  2325. * @type: The type of the new default domain that gets associated with the group
  2326. *
  2327. * Returns 0 on success and error code on failure
  2328. *
  2329. * Note:
  2330. * 1. Presently, this function is called only when user requests to change the
  2331. * group's default domain type through /sys/kernel/iommu_groups/<grp_id>/type
  2332. * Please take a closer look if intended to use for other purposes.
  2333. */
  2334. static int iommu_change_dev_def_domain(struct iommu_group *group,
  2335. struct device *prev_dev, int type)
  2336. {
  2337. struct iommu_domain *prev_dom;
  2338. struct group_device *grp_dev;
  2339. int ret, dev_def_dom;
  2340. struct device *dev;
  2341. mutex_lock(&group->mutex);
  2342. if (group->default_domain != group->domain) {
  2343. dev_err_ratelimited(prev_dev, "Group not assigned to default domain\n");
  2344. ret = -EBUSY;
  2345. goto out;
  2346. }
  2347. /*
  2348. * iommu group wasn't locked while acquiring device lock in
  2349. * iommu_group_store_type(). So, make sure that the device count hasn't
  2350. * changed while acquiring device lock.
  2351. *
  2352. * Changing default domain of an iommu group with two or more devices
  2353. * isn't supported because there could be a potential deadlock. Consider
  2354. * the following scenario. T1 is trying to acquire device locks of all
  2355. * the devices in the group and before it could acquire all of them,
  2356. * there could be another thread T2 (from different sub-system and use
  2357. * case) that has already acquired some of the device locks and might be
  2358. * waiting for T1 to release other device locks.
  2359. */
  2360. if (iommu_group_device_count(group) != 1) {
  2361. dev_err_ratelimited(prev_dev, "Cannot change default domain: Group has more than one device\n");
  2362. ret = -EINVAL;
  2363. goto out;
  2364. }
  2365. /* Since group has only one device */
  2366. grp_dev = list_first_entry(&group->devices, struct group_device, list);
  2367. dev = grp_dev->dev;
  2368. if (prev_dev != dev) {
  2369. dev_err_ratelimited(prev_dev, "Cannot change default domain: Device has been changed\n");
  2370. ret = -EBUSY;
  2371. goto out;
  2372. }
  2373. prev_dom = group->default_domain;
  2374. if (!prev_dom) {
  2375. ret = -EINVAL;
  2376. goto out;
  2377. }
  2378. dev_def_dom = iommu_get_def_domain_type(dev);
  2379. if (!type) {
  2380. /*
  2381. * If the user hasn't requested any specific type of domain and
  2382. * if the device supports both the domains, then default to the
  2383. * domain the device was booted with
  2384. */
  2385. type = dev_def_dom ? : iommu_def_domain_type;
  2386. } else if (dev_def_dom && type != dev_def_dom) {
  2387. dev_err_ratelimited(prev_dev, "Device cannot be in %s domain\n",
  2388. iommu_domain_type_str(type));
  2389. ret = -EINVAL;
  2390. goto out;
  2391. }
  2392. /*
  2393. * Switch to a new domain only if the requested domain type is different
  2394. * from the existing default domain type
  2395. */
  2396. if (prev_dom->type == type) {
  2397. ret = 0;
  2398. goto out;
  2399. }
  2400. /* We can bring up a flush queue without tearing down the domain */
  2401. if (type == IOMMU_DOMAIN_DMA_FQ && prev_dom->type == IOMMU_DOMAIN_DMA) {
  2402. ret = iommu_dma_init_fq(prev_dom);
  2403. if (!ret)
  2404. prev_dom->type = IOMMU_DOMAIN_DMA_FQ;
  2405. goto out;
  2406. }
  2407. /* Sets group->default_domain to the newly allocated domain */
  2408. ret = iommu_group_alloc_default_domain(dev->bus, group, type);
  2409. if (ret)
  2410. goto out;
  2411. ret = iommu_create_device_direct_mappings(group, dev);
  2412. if (ret)
  2413. goto free_new_domain;
  2414. ret = __iommu_attach_device(group->default_domain, dev);
  2415. if (ret)
  2416. goto free_new_domain;
  2417. group->domain = group->default_domain;
  2418. /*
  2419. * Release the mutex here because ops->probe_finalize() call-back of
  2420. * some vendor IOMMU drivers calls arm_iommu_attach_device() which
  2421. * in-turn might call back into IOMMU core code, where it tries to take
  2422. * group->mutex, resulting in a deadlock.
  2423. */
  2424. mutex_unlock(&group->mutex);
  2425. /* Make sure dma_ops is appropriatley set */
  2426. iommu_group_do_probe_finalize(dev, group->default_domain);
  2427. iommu_domain_free(prev_dom);
  2428. return 0;
  2429. free_new_domain:
  2430. iommu_domain_free(group->default_domain);
  2431. group->default_domain = prev_dom;
  2432. group->domain = prev_dom;
  2433. out:
  2434. mutex_unlock(&group->mutex);
  2435. return ret;
  2436. }
  2437. /*
  2438. * Changing the default domain through sysfs requires the users to unbind the
  2439. * drivers from the devices in the iommu group, except for a DMA -> DMA-FQ
  2440. * transition. Return failure if this isn't met.
  2441. *
  2442. * We need to consider the race between this and the device release path.
  2443. * device_lock(dev) is used here to guarantee that the device release path
  2444. * will not be entered at the same time.
  2445. */
  2446. static ssize_t iommu_group_store_type(struct iommu_group *group,
  2447. const char *buf, size_t count)
  2448. {
  2449. struct group_device *grp_dev;
  2450. struct device *dev;
  2451. int ret, req_type;
  2452. if (!capable(CAP_SYS_ADMIN) || !capable(CAP_SYS_RAWIO))
  2453. return -EACCES;
  2454. if (WARN_ON(!group) || !group->default_domain)
  2455. return -EINVAL;
  2456. if (sysfs_streq(buf, "identity"))
  2457. req_type = IOMMU_DOMAIN_IDENTITY;
  2458. else if (sysfs_streq(buf, "DMA"))
  2459. req_type = IOMMU_DOMAIN_DMA;
  2460. else if (sysfs_streq(buf, "DMA-FQ"))
  2461. req_type = IOMMU_DOMAIN_DMA_FQ;
  2462. else if (sysfs_streq(buf, "auto"))
  2463. req_type = 0;
  2464. else
  2465. return -EINVAL;
  2466. /*
  2467. * Lock/Unlock the group mutex here before device lock to
  2468. * 1. Make sure that the iommu group has only one device (this is a
  2469. * prerequisite for step 2)
  2470. * 2. Get struct *dev which is needed to lock device
  2471. */
  2472. mutex_lock(&group->mutex);
  2473. if (iommu_group_device_count(group) != 1) {
  2474. mutex_unlock(&group->mutex);
  2475. pr_err_ratelimited("Cannot change default domain: Group has more than one device\n");
  2476. return -EINVAL;
  2477. }
  2478. /* Since group has only one device */
  2479. grp_dev = list_first_entry(&group->devices, struct group_device, list);
  2480. dev = grp_dev->dev;
  2481. get_device(dev);
  2482. /*
  2483. * Don't hold the group mutex because taking group mutex first and then
  2484. * the device lock could potentially cause a deadlock as below. Assume
  2485. * two threads T1 and T2. T1 is trying to change default domain of an
  2486. * iommu group and T2 is trying to hot unplug a device or release [1] VF
  2487. * of a PCIe device which is in the same iommu group. T1 takes group
  2488. * mutex and before it could take device lock assume T2 has taken device
  2489. * lock and is yet to take group mutex. Now, both the threads will be
  2490. * waiting for the other thread to release lock. Below, lock order was
  2491. * suggested.
  2492. * device_lock(dev);
  2493. * mutex_lock(&group->mutex);
  2494. * iommu_change_dev_def_domain();
  2495. * mutex_unlock(&group->mutex);
  2496. * device_unlock(dev);
  2497. *
  2498. * [1] Typical device release path
  2499. * device_lock() from device/driver core code
  2500. * -> bus_notifier()
  2501. * -> iommu_bus_notifier()
  2502. * -> iommu_release_device()
  2503. * -> ops->release_device() vendor driver calls back iommu core code
  2504. * -> mutex_lock() from iommu core code
  2505. */
  2506. mutex_unlock(&group->mutex);
  2507. /* Check if the device in the group still has a driver bound to it */
  2508. device_lock(dev);
  2509. if (device_is_bound(dev) && !(req_type == IOMMU_DOMAIN_DMA_FQ &&
  2510. group->default_domain->type == IOMMU_DOMAIN_DMA)) {
  2511. pr_err_ratelimited("Device is still bound to driver\n");
  2512. ret = -EBUSY;
  2513. goto out;
  2514. }
  2515. ret = iommu_change_dev_def_domain(group, dev, req_type);
  2516. ret = ret ?: count;
  2517. out:
  2518. device_unlock(dev);
  2519. put_device(dev);
  2520. return ret;
  2521. }
  2522. static bool iommu_is_default_domain(struct iommu_group *group)
  2523. {
  2524. if (group->domain == group->default_domain)
  2525. return true;
  2526. /*
  2527. * If the default domain was set to identity and it is still an identity
  2528. * domain then we consider this a pass. This happens because of
  2529. * amd_iommu_init_device() replacing the default idenytity domain with an
  2530. * identity domain that has a different configuration for AMDGPU.
  2531. */
  2532. if (group->default_domain &&
  2533. group->default_domain->type == IOMMU_DOMAIN_IDENTITY &&
  2534. group->domain && group->domain->type == IOMMU_DOMAIN_IDENTITY)
  2535. return true;
  2536. return false;
  2537. }
  2538. /**
  2539. * iommu_device_use_default_domain() - Device driver wants to handle device
  2540. * DMA through the kernel DMA API.
  2541. * @dev: The device.
  2542. *
  2543. * The device driver about to bind @dev wants to do DMA through the kernel
  2544. * DMA API. Return 0 if it is allowed, otherwise an error.
  2545. */
  2546. int iommu_device_use_default_domain(struct device *dev)
  2547. {
  2548. struct iommu_group *group = iommu_group_get(dev);
  2549. int ret = 0;
  2550. if (!group)
  2551. return 0;
  2552. mutex_lock(&group->mutex);
  2553. if (group->owner_cnt) {
  2554. if (group->owner || !iommu_is_default_domain(group) ||
  2555. !xa_empty(&group->pasid_array)) {
  2556. ret = -EBUSY;
  2557. goto unlock_out;
  2558. }
  2559. }
  2560. group->owner_cnt++;
  2561. unlock_out:
  2562. mutex_unlock(&group->mutex);
  2563. iommu_group_put(group);
  2564. return ret;
  2565. }
  2566. /**
  2567. * iommu_device_unuse_default_domain() - Device driver stops handling device
  2568. * DMA through the kernel DMA API.
  2569. * @dev: The device.
  2570. *
  2571. * The device driver doesn't want to do DMA through kernel DMA API anymore.
  2572. * It must be called after iommu_device_use_default_domain().
  2573. */
  2574. void iommu_device_unuse_default_domain(struct device *dev)
  2575. {
  2576. struct iommu_group *group = iommu_group_get(dev);
  2577. if (!group)
  2578. return;
  2579. mutex_lock(&group->mutex);
  2580. if (!WARN_ON(!group->owner_cnt || !xa_empty(&group->pasid_array)))
  2581. group->owner_cnt--;
  2582. mutex_unlock(&group->mutex);
  2583. iommu_group_put(group);
  2584. }
  2585. static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
  2586. {
  2587. struct group_device *dev =
  2588. list_first_entry(&group->devices, struct group_device, list);
  2589. if (group->blocking_domain)
  2590. return 0;
  2591. group->blocking_domain =
  2592. __iommu_domain_alloc(dev->dev->bus, IOMMU_DOMAIN_BLOCKED);
  2593. if (!group->blocking_domain) {
  2594. /*
  2595. * For drivers that do not yet understand IOMMU_DOMAIN_BLOCKED
  2596. * create an empty domain instead.
  2597. */
  2598. group->blocking_domain = __iommu_domain_alloc(
  2599. dev->dev->bus, IOMMU_DOMAIN_UNMANAGED);
  2600. if (!group->blocking_domain)
  2601. return -EINVAL;
  2602. }
  2603. return 0;
  2604. }
  2605. /**
  2606. * iommu_group_claim_dma_owner() - Set DMA ownership of a group
  2607. * @group: The group.
  2608. * @owner: Caller specified pointer. Used for exclusive ownership.
  2609. *
  2610. * This is to support backward compatibility for vfio which manages
  2611. * the dma ownership in iommu_group level. New invocations on this
  2612. * interface should be prohibited.
  2613. */
  2614. int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner)
  2615. {
  2616. int ret = 0;
  2617. mutex_lock(&group->mutex);
  2618. if (group->owner_cnt) {
  2619. ret = -EPERM;
  2620. goto unlock_out;
  2621. } else {
  2622. if ((group->domain && group->domain != group->default_domain) ||
  2623. !xa_empty(&group->pasid_array)) {
  2624. ret = -EBUSY;
  2625. goto unlock_out;
  2626. }
  2627. ret = __iommu_group_alloc_blocking_domain(group);
  2628. if (ret)
  2629. goto unlock_out;
  2630. ret = __iommu_group_set_domain(group, group->blocking_domain);
  2631. if (ret)
  2632. goto unlock_out;
  2633. group->owner = owner;
  2634. }
  2635. group->owner_cnt++;
  2636. unlock_out:
  2637. mutex_unlock(&group->mutex);
  2638. return ret;
  2639. }
  2640. EXPORT_SYMBOL_GPL(iommu_group_claim_dma_owner);
  2641. /**
  2642. * iommu_group_release_dma_owner() - Release DMA ownership of a group
  2643. * @group: The group.
  2644. *
  2645. * Release the DMA ownership claimed by iommu_group_claim_dma_owner().
  2646. */
  2647. void iommu_group_release_dma_owner(struct iommu_group *group)
  2648. {
  2649. int ret;
  2650. mutex_lock(&group->mutex);
  2651. if (WARN_ON(!group->owner_cnt || !group->owner ||
  2652. !xa_empty(&group->pasid_array)))
  2653. goto unlock_out;
  2654. group->owner_cnt = 0;
  2655. group->owner = NULL;
  2656. ret = __iommu_group_set_domain(group, group->default_domain);
  2657. WARN(ret, "iommu driver failed to attach the default domain");
  2658. unlock_out:
  2659. mutex_unlock(&group->mutex);
  2660. }
  2661. EXPORT_SYMBOL_GPL(iommu_group_release_dma_owner);
  2662. /**
  2663. * iommu_group_dma_owner_claimed() - Query group dma ownership status
  2664. * @group: The group.
  2665. *
  2666. * This provides status query on a given group. It is racy and only for
  2667. * non-binding status reporting.
  2668. */
  2669. bool iommu_group_dma_owner_claimed(struct iommu_group *group)
  2670. {
  2671. unsigned int user;
  2672. mutex_lock(&group->mutex);
  2673. user = group->owner_cnt;
  2674. mutex_unlock(&group->mutex);
  2675. return user;
  2676. }
  2677. EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed);
  2678. static int __iommu_set_group_pasid(struct iommu_domain *domain,
  2679. struct iommu_group *group, ioasid_t pasid)
  2680. {
  2681. struct group_device *device;
  2682. int ret = 0;
  2683. list_for_each_entry(device, &group->devices, list) {
  2684. ret = domain->ops->set_dev_pasid(domain, device->dev, pasid);
  2685. if (ret)
  2686. break;
  2687. }
  2688. return ret;
  2689. }
  2690. static void __iommu_remove_group_pasid(struct iommu_group *group,
  2691. ioasid_t pasid)
  2692. {
  2693. struct group_device *device;
  2694. const struct iommu_ops *ops;
  2695. list_for_each_entry(device, &group->devices, list) {
  2696. ops = dev_iommu_ops(device->dev);
  2697. ops->remove_dev_pasid(device->dev, pasid);
  2698. }
  2699. }
  2700. /*
  2701. * iommu_attach_device_pasid() - Attach a domain to pasid of device
  2702. * @domain: the iommu domain.
  2703. * @dev: the attached device.
  2704. * @pasid: the pasid of the device.
  2705. *
  2706. * Return: 0 on success, or an error.
  2707. */
  2708. int iommu_attach_device_pasid(struct iommu_domain *domain,
  2709. struct device *dev, ioasid_t pasid)
  2710. {
  2711. struct iommu_group *group;
  2712. void *curr;
  2713. int ret;
  2714. if (!domain->ops->set_dev_pasid)
  2715. return -EOPNOTSUPP;
  2716. group = iommu_group_get(dev);
  2717. if (!group)
  2718. return -ENODEV;
  2719. mutex_lock(&group->mutex);
  2720. curr = xa_cmpxchg(&group->pasid_array, pasid, NULL, domain, GFP_KERNEL);
  2721. if (curr) {
  2722. ret = xa_err(curr) ? : -EBUSY;
  2723. goto out_unlock;
  2724. }
  2725. ret = __iommu_set_group_pasid(domain, group, pasid);
  2726. if (ret) {
  2727. __iommu_remove_group_pasid(group, pasid);
  2728. xa_erase(&group->pasid_array, pasid);
  2729. }
  2730. out_unlock:
  2731. mutex_unlock(&group->mutex);
  2732. iommu_group_put(group);
  2733. return ret;
  2734. }
  2735. EXPORT_SYMBOL_GPL(iommu_attach_device_pasid);
  2736. /*
  2737. * iommu_detach_device_pasid() - Detach the domain from pasid of device
  2738. * @domain: the iommu domain.
  2739. * @dev: the attached device.
  2740. * @pasid: the pasid of the device.
  2741. *
  2742. * The @domain must have been attached to @pasid of the @dev with
  2743. * iommu_attach_device_pasid().
  2744. */
  2745. void iommu_detach_device_pasid(struct iommu_domain *domain, struct device *dev,
  2746. ioasid_t pasid)
  2747. {
  2748. struct iommu_group *group = iommu_group_get(dev);
  2749. mutex_lock(&group->mutex);
  2750. __iommu_remove_group_pasid(group, pasid);
  2751. WARN_ON(xa_erase(&group->pasid_array, pasid) != domain);
  2752. mutex_unlock(&group->mutex);
  2753. iommu_group_put(group);
  2754. }
  2755. EXPORT_SYMBOL_GPL(iommu_detach_device_pasid);
  2756. /*
  2757. * iommu_get_domain_for_dev_pasid() - Retrieve domain for @pasid of @dev
  2758. * @dev: the queried device
  2759. * @pasid: the pasid of the device
  2760. * @type: matched domain type, 0 for any match
  2761. *
  2762. * This is a variant of iommu_get_domain_for_dev(). It returns the existing
  2763. * domain attached to pasid of a device. Callers must hold a lock around this
  2764. * function, and both iommu_attach/detach_dev_pasid() whenever a domain of
  2765. * type is being manipulated. This API does not internally resolve races with
  2766. * attach/detach.
  2767. *
  2768. * Return: attached domain on success, NULL otherwise.
  2769. */
  2770. struct iommu_domain *iommu_get_domain_for_dev_pasid(struct device *dev,
  2771. ioasid_t pasid,
  2772. unsigned int type)
  2773. {
  2774. struct iommu_domain *domain;
  2775. struct iommu_group *group;
  2776. group = iommu_group_get(dev);
  2777. if (!group)
  2778. return NULL;
  2779. xa_lock(&group->pasid_array);
  2780. domain = xa_load(&group->pasid_array, pasid);
  2781. if (type && domain && domain->type != type)
  2782. domain = ERR_PTR(-EBUSY);
  2783. xa_unlock(&group->pasid_array);
  2784. iommu_group_put(group);
  2785. return domain;
  2786. }
  2787. EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev_pasid);
  2788. struct iommu_domain *iommu_sva_domain_alloc(struct device *dev,
  2789. struct mm_struct *mm)
  2790. {
  2791. const struct iommu_ops *ops = dev_iommu_ops(dev);
  2792. struct iommu_domain *domain;
  2793. domain = ops->domain_alloc(IOMMU_DOMAIN_SVA);
  2794. if (!domain)
  2795. return NULL;
  2796. domain->type = IOMMU_DOMAIN_SVA;
  2797. mmgrab(mm);
  2798. domain->mm = mm;
  2799. domain->iopf_handler = iommu_sva_handle_iopf;
  2800. domain->fault_data = mm;
  2801. return domain;
  2802. }