dma-mapping.c 46 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/arch/arm/mm/dma-mapping.c
  4. *
  5. * Copyright (C) 2000-2004 Russell King
  6. *
  7. * DMA uncached mapping support.
  8. */
  9. #include <linux/module.h>
  10. #include <linux/mm.h>
  11. #include <linux/genalloc.h>
  12. #include <linux/gfp.h>
  13. #include <linux/errno.h>
  14. #include <linux/list.h>
  15. #include <linux/init.h>
  16. #include <linux/device.h>
  17. #include <linux/dma-direct.h>
  18. #include <linux/dma-map-ops.h>
  19. #include <linux/highmem.h>
  20. #include <linux/memblock.h>
  21. #include <linux/slab.h>
  22. #include <linux/iommu.h>
  23. #include <linux/io.h>
  24. #include <linux/vmalloc.h>
  25. #include <linux/sizes.h>
  26. #include <linux/cma.h>
  27. #include <asm/memory.h>
  28. #include <asm/highmem.h>
  29. #include <asm/cacheflush.h>
  30. #include <asm/tlbflush.h>
  31. #include <asm/mach/arch.h>
  32. #include <asm/dma-iommu.h>
  33. #include <asm/mach/map.h>
  34. #include <asm/system_info.h>
  35. #include <asm/xen/xen-ops.h>
  36. #include "dma.h"
  37. #include "mm.h"
  38. struct arm_dma_alloc_args {
  39. struct device *dev;
  40. size_t size;
  41. gfp_t gfp;
  42. pgprot_t prot;
  43. const void *caller;
  44. bool want_vaddr;
  45. int coherent_flag;
  46. };
  47. struct arm_dma_free_args {
  48. struct device *dev;
  49. size_t size;
  50. void *cpu_addr;
  51. struct page *page;
  52. bool want_vaddr;
  53. };
  54. #define NORMAL 0
  55. #define COHERENT 1
  56. struct arm_dma_allocator {
  57. void *(*alloc)(struct arm_dma_alloc_args *args,
  58. struct page **ret_page);
  59. void (*free)(struct arm_dma_free_args *args);
  60. };
  61. struct arm_dma_buffer {
  62. struct list_head list;
  63. void *virt;
  64. struct arm_dma_allocator *allocator;
  65. };
  66. static LIST_HEAD(arm_dma_bufs);
  67. static DEFINE_SPINLOCK(arm_dma_bufs_lock);
  68. static struct arm_dma_buffer *arm_dma_buffer_find(void *virt)
  69. {
  70. struct arm_dma_buffer *buf, *found = NULL;
  71. unsigned long flags;
  72. spin_lock_irqsave(&arm_dma_bufs_lock, flags);
  73. list_for_each_entry(buf, &arm_dma_bufs, list) {
  74. if (buf->virt == virt) {
  75. list_del(&buf->list);
  76. found = buf;
  77. break;
  78. }
  79. }
  80. spin_unlock_irqrestore(&arm_dma_bufs_lock, flags);
  81. return found;
  82. }
  83. /*
  84. * The DMA API is built upon the notion of "buffer ownership". A buffer
  85. * is either exclusively owned by the CPU (and therefore may be accessed
  86. * by it) or exclusively owned by the DMA device. These helper functions
  87. * represent the transitions between these two ownership states.
  88. *
  89. * Note, however, that on later ARMs, this notion does not work due to
  90. * speculative prefetches. We model our approach on the assumption that
  91. * the CPU does do speculative prefetches, which means we clean caches
  92. * before transfers and delay cache invalidation until transfer completion.
  93. *
  94. */
  95. static void __dma_clear_buffer(struct page *page, size_t size, int coherent_flag)
  96. {
  97. /*
  98. * Ensure that the allocated pages are zeroed, and that any data
  99. * lurking in the kernel direct-mapped region is invalidated.
  100. */
  101. if (PageHighMem(page)) {
  102. phys_addr_t base = __pfn_to_phys(page_to_pfn(page));
  103. phys_addr_t end = base + size;
  104. while (size > 0) {
  105. void *ptr = kmap_atomic(page);
  106. memset(ptr, 0, PAGE_SIZE);
  107. if (coherent_flag != COHERENT)
  108. dmac_flush_range(ptr, ptr + PAGE_SIZE);
  109. kunmap_atomic(ptr);
  110. page++;
  111. size -= PAGE_SIZE;
  112. }
  113. if (coherent_flag != COHERENT)
  114. outer_flush_range(base, end);
  115. } else {
  116. void *ptr = page_address(page);
  117. memset(ptr, 0, size);
  118. if (coherent_flag != COHERENT) {
  119. dmac_flush_range(ptr, ptr + size);
  120. outer_flush_range(__pa(ptr), __pa(ptr) + size);
  121. }
  122. }
  123. }
  124. /*
  125. * Allocate a DMA buffer for 'dev' of size 'size' using the
  126. * specified gfp mask. Note that 'size' must be page aligned.
  127. */
  128. static struct page *__dma_alloc_buffer(struct device *dev, size_t size,
  129. gfp_t gfp, int coherent_flag)
  130. {
  131. unsigned long order = get_order(size);
  132. struct page *page, *p, *e;
  133. page = alloc_pages(gfp, order);
  134. if (!page)
  135. return NULL;
  136. /*
  137. * Now split the huge page and free the excess pages
  138. */
  139. split_page(page, order);
  140. for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
  141. __free_page(p);
  142. __dma_clear_buffer(page, size, coherent_flag);
  143. return page;
  144. }
  145. /*
  146. * Free a DMA buffer. 'size' must be page aligned.
  147. */
  148. static void __dma_free_buffer(struct page *page, size_t size)
  149. {
  150. struct page *e = page + (size >> PAGE_SHIFT);
  151. while (page < e) {
  152. __free_page(page);
  153. page++;
  154. }
  155. }
  156. static void *__alloc_from_contiguous(struct device *dev, size_t size,
  157. pgprot_t prot, struct page **ret_page,
  158. const void *caller, bool want_vaddr,
  159. int coherent_flag, gfp_t gfp);
  160. static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
  161. pgprot_t prot, struct page **ret_page,
  162. const void *caller, bool want_vaddr);
  163. #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
  164. static struct gen_pool *atomic_pool __ro_after_init;
  165. static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;
  166. static int __init early_coherent_pool(char *p)
  167. {
  168. atomic_pool_size = memparse(p, &p);
  169. return 0;
  170. }
  171. early_param("coherent_pool", early_coherent_pool);
  172. /*
  173. * Initialise the coherent pool for atomic allocations.
  174. */
  175. static int __init atomic_pool_init(void)
  176. {
  177. pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL);
  178. gfp_t gfp = GFP_KERNEL | GFP_DMA;
  179. struct page *page;
  180. void *ptr;
  181. atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
  182. if (!atomic_pool)
  183. goto out;
  184. /*
  185. * The atomic pool is only used for non-coherent allocations
  186. * so we must pass NORMAL for coherent_flag.
  187. */
  188. if (dev_get_cma_area(NULL))
  189. ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot,
  190. &page, atomic_pool_init, true, NORMAL,
  191. GFP_KERNEL);
  192. else
  193. ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot,
  194. &page, atomic_pool_init, true);
  195. if (ptr) {
  196. int ret;
  197. ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr,
  198. page_to_phys(page),
  199. atomic_pool_size, -1);
  200. if (ret)
  201. goto destroy_genpool;
  202. gen_pool_set_algo(atomic_pool,
  203. gen_pool_first_fit_order_align,
  204. NULL);
  205. pr_info("DMA: preallocated %zu KiB pool for atomic coherent allocations\n",
  206. atomic_pool_size / 1024);
  207. return 0;
  208. }
  209. destroy_genpool:
  210. gen_pool_destroy(atomic_pool);
  211. atomic_pool = NULL;
  212. out:
  213. pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",
  214. atomic_pool_size / 1024);
  215. return -ENOMEM;
  216. }
  217. /*
  218. * CMA is activated by core_initcall, so we must be called after it.
  219. */
  220. postcore_initcall(atomic_pool_init);
  221. #ifdef CONFIG_CMA_AREAS
  222. struct dma_contig_early_reserve {
  223. phys_addr_t base;
  224. unsigned long size;
  225. };
  226. static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;
  227. static int dma_mmu_remap_num __initdata;
  228. void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
  229. {
  230. dma_mmu_remap[dma_mmu_remap_num].base = base;
  231. dma_mmu_remap[dma_mmu_remap_num].size = size;
  232. dma_mmu_remap_num++;
  233. }
  234. void __init dma_contiguous_remap(void)
  235. {
  236. int i;
  237. for (i = 0; i < dma_mmu_remap_num; i++) {
  238. phys_addr_t start = dma_mmu_remap[i].base;
  239. phys_addr_t end = start + dma_mmu_remap[i].size;
  240. struct map_desc map;
  241. unsigned long addr;
  242. if (end > arm_lowmem_limit)
  243. end = arm_lowmem_limit;
  244. if (start >= end)
  245. continue;
  246. map.pfn = __phys_to_pfn(start);
  247. map.virtual = __phys_to_virt(start);
  248. map.length = end - start;
  249. map.type = MT_MEMORY_DMA_READY;
  250. /*
  251. * Clear previous low-memory mapping to ensure that the
  252. * TLB does not see any conflicting entries, then flush
  253. * the TLB of the old entries before creating new mappings.
  254. *
  255. * This ensures that any speculatively loaded TLB entries
  256. * (even though they may be rare) can not cause any problems,
  257. * and ensures that this code is architecturally compliant.
  258. */
  259. for (addr = __phys_to_virt(start); addr < __phys_to_virt(end);
  260. addr += PMD_SIZE)
  261. pmd_clear(pmd_off_k(addr));
  262. flush_tlb_kernel_range(__phys_to_virt(start),
  263. __phys_to_virt(end));
  264. iotable_init(&map, 1);
  265. }
  266. }
  267. #endif
  268. static int __dma_update_pte(pte_t *pte, unsigned long addr, void *data)
  269. {
  270. struct page *page = virt_to_page((void *)addr);
  271. pgprot_t prot = *(pgprot_t *)data;
  272. set_pte_ext(pte, mk_pte(page, prot), 0);
  273. return 0;
  274. }
  275. static void __dma_remap(struct page *page, size_t size, pgprot_t prot)
  276. {
  277. unsigned long start = (unsigned long) page_address(page);
  278. unsigned end = start + size;
  279. apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot);
  280. flush_tlb_kernel_range(start, end);
  281. }
  282. static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
  283. pgprot_t prot, struct page **ret_page,
  284. const void *caller, bool want_vaddr)
  285. {
  286. struct page *page;
  287. void *ptr = NULL;
  288. /*
  289. * __alloc_remap_buffer is only called when the device is
  290. * non-coherent
  291. */
  292. page = __dma_alloc_buffer(dev, size, gfp, NORMAL);
  293. if (!page)
  294. return NULL;
  295. if (!want_vaddr)
  296. goto out;
  297. ptr = dma_common_contiguous_remap(page, size, prot, caller);
  298. if (!ptr) {
  299. __dma_free_buffer(page, size);
  300. return NULL;
  301. }
  302. out:
  303. *ret_page = page;
  304. return ptr;
  305. }
  306. static void *__alloc_from_pool(size_t size, struct page **ret_page)
  307. {
  308. unsigned long val;
  309. void *ptr = NULL;
  310. if (!atomic_pool) {
  311. WARN(1, "coherent pool not initialised!\n");
  312. return NULL;
  313. }
  314. val = gen_pool_alloc(atomic_pool, size);
  315. if (val) {
  316. phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
  317. *ret_page = phys_to_page(phys);
  318. ptr = (void *)val;
  319. }
  320. return ptr;
  321. }
  322. static bool __in_atomic_pool(void *start, size_t size)
  323. {
  324. return gen_pool_has_addr(atomic_pool, (unsigned long)start, size);
  325. }
  326. static int __free_from_pool(void *start, size_t size)
  327. {
  328. if (!__in_atomic_pool(start, size))
  329. return 0;
  330. gen_pool_free(atomic_pool, (unsigned long)start, size);
  331. return 1;
  332. }
  333. static void *__alloc_from_contiguous(struct device *dev, size_t size,
  334. pgprot_t prot, struct page **ret_page,
  335. const void *caller, bool want_vaddr,
  336. int coherent_flag, gfp_t gfp)
  337. {
  338. unsigned long order = get_order(size);
  339. size_t count = size >> PAGE_SHIFT;
  340. struct page *page;
  341. void *ptr = NULL;
  342. page = dma_alloc_from_contiguous(dev, count, order, gfp & __GFP_NOWARN);
  343. if (!page)
  344. return NULL;
  345. __dma_clear_buffer(page, size, coherent_flag);
  346. if (!want_vaddr)
  347. goto out;
  348. if (PageHighMem(page)) {
  349. ptr = dma_common_contiguous_remap(page, size, prot, caller);
  350. if (!ptr) {
  351. dma_release_from_contiguous(dev, page, count);
  352. return NULL;
  353. }
  354. } else {
  355. __dma_remap(page, size, prot);
  356. ptr = page_address(page);
  357. }
  358. out:
  359. *ret_page = page;
  360. return ptr;
  361. }
  362. static void __free_from_contiguous(struct device *dev, struct page *page,
  363. void *cpu_addr, size_t size, bool want_vaddr)
  364. {
  365. if (want_vaddr) {
  366. if (PageHighMem(page))
  367. dma_common_free_remap(cpu_addr, size);
  368. else
  369. __dma_remap(page, size, PAGE_KERNEL);
  370. }
  371. dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
  372. }
  373. static inline pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot)
  374. {
  375. prot = (attrs & DMA_ATTR_WRITE_COMBINE) ?
  376. pgprot_writecombine(prot) :
  377. pgprot_dmacoherent(prot);
  378. return prot;
  379. }
  380. static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
  381. struct page **ret_page)
  382. {
  383. struct page *page;
  384. /* __alloc_simple_buffer is only called when the device is coherent */
  385. page = __dma_alloc_buffer(dev, size, gfp, COHERENT);
  386. if (!page)
  387. return NULL;
  388. *ret_page = page;
  389. return page_address(page);
  390. }
  391. static void *simple_allocator_alloc(struct arm_dma_alloc_args *args,
  392. struct page **ret_page)
  393. {
  394. return __alloc_simple_buffer(args->dev, args->size, args->gfp,
  395. ret_page);
  396. }
  397. static void simple_allocator_free(struct arm_dma_free_args *args)
  398. {
  399. __dma_free_buffer(args->page, args->size);
  400. }
  401. static struct arm_dma_allocator simple_allocator = {
  402. .alloc = simple_allocator_alloc,
  403. .free = simple_allocator_free,
  404. };
  405. static void *cma_allocator_alloc(struct arm_dma_alloc_args *args,
  406. struct page **ret_page)
  407. {
  408. return __alloc_from_contiguous(args->dev, args->size, args->prot,
  409. ret_page, args->caller,
  410. args->want_vaddr, args->coherent_flag,
  411. args->gfp);
  412. }
  413. static void cma_allocator_free(struct arm_dma_free_args *args)
  414. {
  415. __free_from_contiguous(args->dev, args->page, args->cpu_addr,
  416. args->size, args->want_vaddr);
  417. }
  418. static struct arm_dma_allocator cma_allocator = {
  419. .alloc = cma_allocator_alloc,
  420. .free = cma_allocator_free,
  421. };
  422. static void *pool_allocator_alloc(struct arm_dma_alloc_args *args,
  423. struct page **ret_page)
  424. {
  425. return __alloc_from_pool(args->size, ret_page);
  426. }
  427. static void pool_allocator_free(struct arm_dma_free_args *args)
  428. {
  429. __free_from_pool(args->cpu_addr, args->size);
  430. }
  431. static struct arm_dma_allocator pool_allocator = {
  432. .alloc = pool_allocator_alloc,
  433. .free = pool_allocator_free,
  434. };
  435. static void *remap_allocator_alloc(struct arm_dma_alloc_args *args,
  436. struct page **ret_page)
  437. {
  438. return __alloc_remap_buffer(args->dev, args->size, args->gfp,
  439. args->prot, ret_page, args->caller,
  440. args->want_vaddr);
  441. }
  442. static void remap_allocator_free(struct arm_dma_free_args *args)
  443. {
  444. if (args->want_vaddr)
  445. dma_common_free_remap(args->cpu_addr, args->size);
  446. __dma_free_buffer(args->page, args->size);
  447. }
  448. static struct arm_dma_allocator remap_allocator = {
  449. .alloc = remap_allocator_alloc,
  450. .free = remap_allocator_free,
  451. };
  452. static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
  453. gfp_t gfp, pgprot_t prot, bool is_coherent,
  454. unsigned long attrs, const void *caller)
  455. {
  456. u64 mask = min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit);
  457. struct page *page = NULL;
  458. void *addr;
  459. bool allowblock, cma;
  460. struct arm_dma_buffer *buf;
  461. struct arm_dma_alloc_args args = {
  462. .dev = dev,
  463. .size = PAGE_ALIGN(size),
  464. .gfp = gfp,
  465. .prot = prot,
  466. .caller = caller,
  467. .want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0),
  468. .coherent_flag = is_coherent ? COHERENT : NORMAL,
  469. };
  470. #ifdef CONFIG_DMA_API_DEBUG
  471. u64 limit = (mask + 1) & ~mask;
  472. if (limit && size >= limit) {
  473. dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
  474. size, mask);
  475. return NULL;
  476. }
  477. #endif
  478. buf = kzalloc(sizeof(*buf),
  479. gfp & ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM));
  480. if (!buf)
  481. return NULL;
  482. if (mask < 0xffffffffULL)
  483. gfp |= GFP_DMA;
  484. /*
  485. * Following is a work-around (a.k.a. hack) to prevent pages
  486. * with __GFP_COMP being passed to split_page() which cannot
  487. * handle them. The real problem is that this flag probably
  488. * should be 0 on ARM as it is not supported on this
  489. * platform; see CONFIG_HUGETLBFS.
  490. */
  491. gfp &= ~(__GFP_COMP);
  492. args.gfp = gfp;
  493. *handle = DMA_MAPPING_ERROR;
  494. allowblock = gfpflags_allow_blocking(gfp);
  495. cma = allowblock ? dev_get_cma_area(dev) : NULL;
  496. if (cma)
  497. buf->allocator = &cma_allocator;
  498. else if (is_coherent)
  499. buf->allocator = &simple_allocator;
  500. else if (allowblock)
  501. buf->allocator = &remap_allocator;
  502. else
  503. buf->allocator = &pool_allocator;
  504. addr = buf->allocator->alloc(&args, &page);
  505. if (page) {
  506. unsigned long flags;
  507. *handle = phys_to_dma(dev, page_to_phys(page));
  508. buf->virt = args.want_vaddr ? addr : page;
  509. spin_lock_irqsave(&arm_dma_bufs_lock, flags);
  510. list_add(&buf->list, &arm_dma_bufs);
  511. spin_unlock_irqrestore(&arm_dma_bufs_lock, flags);
  512. } else {
  513. kfree(buf);
  514. }
  515. return args.want_vaddr ? addr : page;
  516. }
  517. /*
  518. * Free a buffer as defined by the above mapping.
  519. */
  520. static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
  521. dma_addr_t handle, unsigned long attrs,
  522. bool is_coherent)
  523. {
  524. struct page *page = phys_to_page(dma_to_phys(dev, handle));
  525. struct arm_dma_buffer *buf;
  526. struct arm_dma_free_args args = {
  527. .dev = dev,
  528. .size = PAGE_ALIGN(size),
  529. .cpu_addr = cpu_addr,
  530. .page = page,
  531. .want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0),
  532. };
  533. buf = arm_dma_buffer_find(cpu_addr);
  534. if (WARN(!buf, "Freeing invalid buffer %p\n", cpu_addr))
  535. return;
  536. buf->allocator->free(&args);
  537. kfree(buf);
  538. }
  539. static void dma_cache_maint_page(struct page *page, unsigned long offset,
  540. size_t size, enum dma_data_direction dir,
  541. void (*op)(const void *, size_t, int))
  542. {
  543. unsigned long pfn;
  544. size_t left = size;
  545. pfn = page_to_pfn(page) + offset / PAGE_SIZE;
  546. offset %= PAGE_SIZE;
  547. /*
  548. * A single sg entry may refer to multiple physically contiguous
  549. * pages. But we still need to process highmem pages individually.
  550. * If highmem is not configured then the bulk of this loop gets
  551. * optimized out.
  552. */
  553. do {
  554. size_t len = left;
  555. void *vaddr;
  556. page = pfn_to_page(pfn);
  557. if (PageHighMem(page)) {
  558. if (len + offset > PAGE_SIZE)
  559. len = PAGE_SIZE - offset;
  560. if (cache_is_vipt_nonaliasing()) {
  561. vaddr = kmap_atomic(page);
  562. op(vaddr + offset, len, dir);
  563. kunmap_atomic(vaddr);
  564. } else {
  565. vaddr = kmap_high_get(page);
  566. if (vaddr) {
  567. op(vaddr + offset, len, dir);
  568. kunmap_high(page);
  569. }
  570. }
  571. } else {
  572. vaddr = page_address(page) + offset;
  573. op(vaddr, len, dir);
  574. }
  575. offset = 0;
  576. pfn++;
  577. left -= len;
  578. } while (left);
  579. }
  580. /*
  581. * Make an area consistent for devices.
  582. * Note: Drivers should NOT use this function directly.
  583. * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
  584. */
  585. static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
  586. size_t size, enum dma_data_direction dir)
  587. {
  588. phys_addr_t paddr;
  589. dma_cache_maint_page(page, off, size, dir, dmac_map_area);
  590. paddr = page_to_phys(page) + off;
  591. if (dir == DMA_FROM_DEVICE) {
  592. outer_inv_range(paddr, paddr + size);
  593. } else {
  594. outer_clean_range(paddr, paddr + size);
  595. }
  596. /* FIXME: non-speculating: flush on bidirectional mappings? */
  597. }
  598. static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
  599. size_t size, enum dma_data_direction dir)
  600. {
  601. phys_addr_t paddr = page_to_phys(page) + off;
  602. /* FIXME: non-speculating: not required */
  603. /* in any case, don't bother invalidating if DMA to device */
  604. if (dir != DMA_TO_DEVICE) {
  605. outer_inv_range(paddr, paddr + size);
  606. dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
  607. }
  608. /*
  609. * Mark the D-cache clean for these pages to avoid extra flushing.
  610. */
  611. if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) {
  612. unsigned long pfn;
  613. size_t left = size;
  614. pfn = page_to_pfn(page) + off / PAGE_SIZE;
  615. off %= PAGE_SIZE;
  616. if (off) {
  617. pfn++;
  618. left -= PAGE_SIZE - off;
  619. }
  620. while (left >= PAGE_SIZE) {
  621. page = pfn_to_page(pfn++);
  622. set_bit(PG_dcache_clean, &page->flags);
  623. left -= PAGE_SIZE;
  624. }
  625. }
  626. }
  627. #ifdef CONFIG_ARM_DMA_USE_IOMMU
  628. static int __dma_info_to_prot(enum dma_data_direction dir, unsigned long attrs)
  629. {
  630. int prot = 0;
  631. if (attrs & DMA_ATTR_PRIVILEGED)
  632. prot |= IOMMU_PRIV;
  633. switch (dir) {
  634. case DMA_BIDIRECTIONAL:
  635. return prot | IOMMU_READ | IOMMU_WRITE;
  636. case DMA_TO_DEVICE:
  637. return prot | IOMMU_READ;
  638. case DMA_FROM_DEVICE:
  639. return prot | IOMMU_WRITE;
  640. default:
  641. return prot;
  642. }
  643. }
  644. /* IOMMU */
  645. static int extend_iommu_mapping(struct dma_iommu_mapping *mapping);
  646. static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
  647. size_t size)
  648. {
  649. unsigned int order = get_order(size);
  650. unsigned int align = 0;
  651. unsigned int count, start;
  652. size_t mapping_size = mapping->bits << PAGE_SHIFT;
  653. unsigned long flags;
  654. dma_addr_t iova;
  655. int i;
  656. if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT)
  657. order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT;
  658. count = PAGE_ALIGN(size) >> PAGE_SHIFT;
  659. align = (1 << order) - 1;
  660. spin_lock_irqsave(&mapping->lock, flags);
  661. for (i = 0; i < mapping->nr_bitmaps; i++) {
  662. start = bitmap_find_next_zero_area(mapping->bitmaps[i],
  663. mapping->bits, 0, count, align);
  664. if (start > mapping->bits)
  665. continue;
  666. bitmap_set(mapping->bitmaps[i], start, count);
  667. break;
  668. }
  669. /*
  670. * No unused range found. Try to extend the existing mapping
  671. * and perform a second attempt to reserve an IO virtual
  672. * address range of size bytes.
  673. */
  674. if (i == mapping->nr_bitmaps) {
  675. if (extend_iommu_mapping(mapping)) {
  676. spin_unlock_irqrestore(&mapping->lock, flags);
  677. return DMA_MAPPING_ERROR;
  678. }
  679. start = bitmap_find_next_zero_area(mapping->bitmaps[i],
  680. mapping->bits, 0, count, align);
  681. if (start > mapping->bits) {
  682. spin_unlock_irqrestore(&mapping->lock, flags);
  683. return DMA_MAPPING_ERROR;
  684. }
  685. bitmap_set(mapping->bitmaps[i], start, count);
  686. }
  687. spin_unlock_irqrestore(&mapping->lock, flags);
  688. iova = mapping->base + (mapping_size * i);
  689. iova += start << PAGE_SHIFT;
  690. return iova;
  691. }
  692. static inline void __free_iova(struct dma_iommu_mapping *mapping,
  693. dma_addr_t addr, size_t size)
  694. {
  695. unsigned int start, count;
  696. size_t mapping_size = mapping->bits << PAGE_SHIFT;
  697. unsigned long flags;
  698. dma_addr_t bitmap_base;
  699. u32 bitmap_index;
  700. if (!size)
  701. return;
  702. bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size;
  703. BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions);
  704. bitmap_base = mapping->base + mapping_size * bitmap_index;
  705. start = (addr - bitmap_base) >> PAGE_SHIFT;
  706. if (addr + size > bitmap_base + mapping_size) {
  707. /*
  708. * The address range to be freed reaches into the iova
  709. * range of the next bitmap. This should not happen as
  710. * we don't allow this in __alloc_iova (at the
  711. * moment).
  712. */
  713. BUG();
  714. } else
  715. count = size >> PAGE_SHIFT;
  716. spin_lock_irqsave(&mapping->lock, flags);
  717. bitmap_clear(mapping->bitmaps[bitmap_index], start, count);
  718. spin_unlock_irqrestore(&mapping->lock, flags);
  719. }
  720. /* We'll try 2M, 1M, 64K, and finally 4K; array must end with 0! */
  721. static const int iommu_order_array[] = { 9, 8, 4, 0 };
  722. static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,
  723. gfp_t gfp, unsigned long attrs,
  724. int coherent_flag)
  725. {
  726. struct page **pages;
  727. int count = size >> PAGE_SHIFT;
  728. int array_size = count * sizeof(struct page *);
  729. int i = 0;
  730. int order_idx = 0;
  731. if (array_size <= PAGE_SIZE)
  732. pages = kzalloc(array_size, GFP_KERNEL);
  733. else
  734. pages = vzalloc(array_size);
  735. if (!pages)
  736. return NULL;
  737. if (attrs & DMA_ATTR_FORCE_CONTIGUOUS)
  738. {
  739. unsigned long order = get_order(size);
  740. struct page *page;
  741. page = dma_alloc_from_contiguous(dev, count, order,
  742. gfp & __GFP_NOWARN);
  743. if (!page)
  744. goto error;
  745. __dma_clear_buffer(page, size, coherent_flag);
  746. for (i = 0; i < count; i++)
  747. pages[i] = page + i;
  748. return pages;
  749. }
  750. /* Go straight to 4K chunks if caller says it's OK. */
  751. if (attrs & DMA_ATTR_ALLOC_SINGLE_PAGES)
  752. order_idx = ARRAY_SIZE(iommu_order_array) - 1;
  753. /*
  754. * IOMMU can map any pages, so himem can also be used here
  755. */
  756. gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
  757. while (count) {
  758. int j, order;
  759. order = iommu_order_array[order_idx];
  760. /* Drop down when we get small */
  761. if (__fls(count) < order) {
  762. order_idx++;
  763. continue;
  764. }
  765. if (order) {
  766. /* See if it's easy to allocate a high-order chunk */
  767. pages[i] = alloc_pages(gfp | __GFP_NORETRY, order);
  768. /* Go down a notch at first sign of pressure */
  769. if (!pages[i]) {
  770. order_idx++;
  771. continue;
  772. }
  773. } else {
  774. pages[i] = alloc_pages(gfp, 0);
  775. if (!pages[i])
  776. goto error;
  777. }
  778. if (order) {
  779. split_page(pages[i], order);
  780. j = 1 << order;
  781. while (--j)
  782. pages[i + j] = pages[i] + j;
  783. }
  784. __dma_clear_buffer(pages[i], PAGE_SIZE << order, coherent_flag);
  785. i += 1 << order;
  786. count -= 1 << order;
  787. }
  788. return pages;
  789. error:
  790. while (i--)
  791. if (pages[i])
  792. __free_pages(pages[i], 0);
  793. kvfree(pages);
  794. return NULL;
  795. }
  796. static int __iommu_free_buffer(struct device *dev, struct page **pages,
  797. size_t size, unsigned long attrs)
  798. {
  799. int count = size >> PAGE_SHIFT;
  800. int i;
  801. if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {
  802. dma_release_from_contiguous(dev, pages[0], count);
  803. } else {
  804. for (i = 0; i < count; i++)
  805. if (pages[i])
  806. __free_pages(pages[i], 0);
  807. }
  808. kvfree(pages);
  809. return 0;
  810. }
  811. /*
  812. * Create a mapping in device IO address space for specified pages
  813. */
  814. static dma_addr_t
  815. __iommu_create_mapping(struct device *dev, struct page **pages, size_t size,
  816. unsigned long attrs)
  817. {
  818. struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
  819. unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
  820. dma_addr_t dma_addr, iova;
  821. int i;
  822. dma_addr = __alloc_iova(mapping, size);
  823. if (dma_addr == DMA_MAPPING_ERROR)
  824. return dma_addr;
  825. iova = dma_addr;
  826. for (i = 0; i < count; ) {
  827. int ret;
  828. unsigned int next_pfn = page_to_pfn(pages[i]) + 1;
  829. phys_addr_t phys = page_to_phys(pages[i]);
  830. unsigned int len, j;
  831. for (j = i + 1; j < count; j++, next_pfn++)
  832. if (page_to_pfn(pages[j]) != next_pfn)
  833. break;
  834. len = (j - i) << PAGE_SHIFT;
  835. ret = iommu_map(mapping->domain, iova, phys, len,
  836. __dma_info_to_prot(DMA_BIDIRECTIONAL, attrs));
  837. if (ret < 0)
  838. goto fail;
  839. iova += len;
  840. i = j;
  841. }
  842. return dma_addr;
  843. fail:
  844. iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);
  845. __free_iova(mapping, dma_addr, size);
  846. return DMA_MAPPING_ERROR;
  847. }
  848. static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
  849. {
  850. struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
  851. /*
  852. * add optional in-page offset from iova to size and align
  853. * result to page size
  854. */
  855. size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);
  856. iova &= PAGE_MASK;
  857. iommu_unmap(mapping->domain, iova, size);
  858. __free_iova(mapping, iova, size);
  859. return 0;
  860. }
  861. static struct page **__atomic_get_pages(void *addr)
  862. {
  863. struct page *page;
  864. phys_addr_t phys;
  865. phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr);
  866. page = phys_to_page(phys);
  867. return (struct page **)page;
  868. }
  869. static struct page **__iommu_get_pages(void *cpu_addr, unsigned long attrs)
  870. {
  871. if (__in_atomic_pool(cpu_addr, PAGE_SIZE))
  872. return __atomic_get_pages(cpu_addr);
  873. if (attrs & DMA_ATTR_NO_KERNEL_MAPPING)
  874. return cpu_addr;
  875. return dma_common_find_pages(cpu_addr);
  876. }
  877. static void *__iommu_alloc_simple(struct device *dev, size_t size, gfp_t gfp,
  878. dma_addr_t *handle, int coherent_flag,
  879. unsigned long attrs)
  880. {
  881. struct page *page;
  882. void *addr;
  883. if (coherent_flag == COHERENT)
  884. addr = __alloc_simple_buffer(dev, size, gfp, &page);
  885. else
  886. addr = __alloc_from_pool(size, &page);
  887. if (!addr)
  888. return NULL;
  889. *handle = __iommu_create_mapping(dev, &page, size, attrs);
  890. if (*handle == DMA_MAPPING_ERROR)
  891. goto err_mapping;
  892. return addr;
  893. err_mapping:
  894. __free_from_pool(addr, size);
  895. return NULL;
  896. }
  897. static void __iommu_free_atomic(struct device *dev, void *cpu_addr,
  898. dma_addr_t handle, size_t size, int coherent_flag)
  899. {
  900. __iommu_remove_mapping(dev, handle, size);
  901. if (coherent_flag == COHERENT)
  902. __dma_free_buffer(virt_to_page(cpu_addr), size);
  903. else
  904. __free_from_pool(cpu_addr, size);
  905. }
  906. static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
  907. dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
  908. {
  909. pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
  910. struct page **pages;
  911. void *addr = NULL;
  912. int coherent_flag = dev->dma_coherent ? COHERENT : NORMAL;
  913. *handle = DMA_MAPPING_ERROR;
  914. size = PAGE_ALIGN(size);
  915. if (coherent_flag == COHERENT || !gfpflags_allow_blocking(gfp))
  916. return __iommu_alloc_simple(dev, size, gfp, handle,
  917. coherent_flag, attrs);
  918. /*
  919. * Following is a work-around (a.k.a. hack) to prevent pages
  920. * with __GFP_COMP being passed to split_page() which cannot
  921. * handle them. The real problem is that this flag probably
  922. * should be 0 on ARM as it is not supported on this
  923. * platform; see CONFIG_HUGETLBFS.
  924. */
  925. gfp &= ~(__GFP_COMP);
  926. pages = __iommu_alloc_buffer(dev, size, gfp, attrs, coherent_flag);
  927. if (!pages)
  928. return NULL;
  929. *handle = __iommu_create_mapping(dev, pages, size, attrs);
  930. if (*handle == DMA_MAPPING_ERROR)
  931. goto err_buffer;
  932. if (attrs & DMA_ATTR_NO_KERNEL_MAPPING)
  933. return pages;
  934. addr = dma_common_pages_remap(pages, size, prot,
  935. __builtin_return_address(0));
  936. if (!addr)
  937. goto err_mapping;
  938. return addr;
  939. err_mapping:
  940. __iommu_remove_mapping(dev, *handle, size);
  941. err_buffer:
  942. __iommu_free_buffer(dev, pages, size, attrs);
  943. return NULL;
  944. }
  945. static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
  946. void *cpu_addr, dma_addr_t dma_addr, size_t size,
  947. unsigned long attrs)
  948. {
  949. struct page **pages = __iommu_get_pages(cpu_addr, attrs);
  950. unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
  951. int err;
  952. if (!pages)
  953. return -ENXIO;
  954. if (vma->vm_pgoff >= nr_pages)
  955. return -ENXIO;
  956. if (!dev->dma_coherent)
  957. vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
  958. err = vm_map_pages(vma, pages, nr_pages);
  959. if (err)
  960. pr_err("Remapping memory failed: %d\n", err);
  961. return err;
  962. }
  963. /*
  964. * free a page as defined by the above mapping.
  965. * Must not be called with IRQs disabled.
  966. */
  967. static void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
  968. dma_addr_t handle, unsigned long attrs)
  969. {
  970. int coherent_flag = dev->dma_coherent ? COHERENT : NORMAL;
  971. struct page **pages;
  972. size = PAGE_ALIGN(size);
  973. if (coherent_flag == COHERENT || __in_atomic_pool(cpu_addr, size)) {
  974. __iommu_free_atomic(dev, cpu_addr, handle, size, coherent_flag);
  975. return;
  976. }
  977. pages = __iommu_get_pages(cpu_addr, attrs);
  978. if (!pages) {
  979. WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
  980. return;
  981. }
  982. if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0)
  983. dma_common_free_remap(cpu_addr, size);
  984. __iommu_remove_mapping(dev, handle, size);
  985. __iommu_free_buffer(dev, pages, size, attrs);
  986. }
  987. static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
  988. void *cpu_addr, dma_addr_t dma_addr,
  989. size_t size, unsigned long attrs)
  990. {
  991. unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
  992. struct page **pages = __iommu_get_pages(cpu_addr, attrs);
  993. if (!pages)
  994. return -ENXIO;
  995. return sg_alloc_table_from_pages(sgt, pages, count, 0, size,
  996. GFP_KERNEL);
  997. }
  998. /*
  999. * Map a part of the scatter-gather list into contiguous io address space
  1000. */
  1001. static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
  1002. size_t size, dma_addr_t *handle,
  1003. enum dma_data_direction dir, unsigned long attrs)
  1004. {
  1005. struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
  1006. dma_addr_t iova, iova_base;
  1007. int ret = 0;
  1008. unsigned int count;
  1009. struct scatterlist *s;
  1010. int prot;
  1011. size = PAGE_ALIGN(size);
  1012. *handle = DMA_MAPPING_ERROR;
  1013. iova_base = iova = __alloc_iova(mapping, size);
  1014. if (iova == DMA_MAPPING_ERROR)
  1015. return -ENOMEM;
  1016. for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {
  1017. phys_addr_t phys = page_to_phys(sg_page(s));
  1018. unsigned int len = PAGE_ALIGN(s->offset + s->length);
  1019. if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
  1020. __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
  1021. prot = __dma_info_to_prot(dir, attrs);
  1022. ret = iommu_map(mapping->domain, iova, phys, len, prot);
  1023. if (ret < 0)
  1024. goto fail;
  1025. count += len >> PAGE_SHIFT;
  1026. iova += len;
  1027. }
  1028. *handle = iova_base;
  1029. return 0;
  1030. fail:
  1031. iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);
  1032. __free_iova(mapping, iova_base, size);
  1033. return ret;
  1034. }
  1035. /**
  1036. * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA
  1037. * @dev: valid struct device pointer
  1038. * @sg: list of buffers
  1039. * @nents: number of buffers to map
  1040. * @dir: DMA transfer direction
  1041. *
  1042. * Map a set of buffers described by scatterlist in streaming mode for DMA.
  1043. * The scatter gather list elements are merged together (if possible) and
  1044. * tagged with the appropriate dma address and length. They are obtained via
  1045. * sg_dma_{address,length}.
  1046. */
  1047. static int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg,
  1048. int nents, enum dma_data_direction dir, unsigned long attrs)
  1049. {
  1050. struct scatterlist *s = sg, *dma = sg, *start = sg;
  1051. int i, count = 0, ret;
  1052. unsigned int offset = s->offset;
  1053. unsigned int size = s->offset + s->length;
  1054. unsigned int max = dma_get_max_seg_size(dev);
  1055. for (i = 1; i < nents; i++) {
  1056. s = sg_next(s);
  1057. s->dma_length = 0;
  1058. if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {
  1059. ret = __map_sg_chunk(dev, start, size,
  1060. &dma->dma_address, dir, attrs);
  1061. if (ret < 0)
  1062. goto bad_mapping;
  1063. dma->dma_address += offset;
  1064. dma->dma_length = size - offset;
  1065. size = offset = s->offset;
  1066. start = s;
  1067. dma = sg_next(dma);
  1068. count += 1;
  1069. }
  1070. size += s->length;
  1071. }
  1072. ret = __map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs);
  1073. if (ret < 0)
  1074. goto bad_mapping;
  1075. dma->dma_address += offset;
  1076. dma->dma_length = size - offset;
  1077. return count+1;
  1078. bad_mapping:
  1079. for_each_sg(sg, s, count, i)
  1080. __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));
  1081. if (ret == -ENOMEM)
  1082. return ret;
  1083. return -EINVAL;
  1084. }
  1085. /**
  1086. * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
  1087. * @dev: valid struct device pointer
  1088. * @sg: list of buffers
  1089. * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
  1090. * @dir: DMA transfer direction (same as was passed to dma_map_sg)
  1091. *
  1092. * Unmap a set of streaming mode DMA translations. Again, CPU access
  1093. * rules concerning calls here are the same as for dma_unmap_single().
  1094. */
  1095. static void arm_iommu_unmap_sg(struct device *dev,
  1096. struct scatterlist *sg, int nents,
  1097. enum dma_data_direction dir,
  1098. unsigned long attrs)
  1099. {
  1100. struct scatterlist *s;
  1101. int i;
  1102. for_each_sg(sg, s, nents, i) {
  1103. if (sg_dma_len(s))
  1104. __iommu_remove_mapping(dev, sg_dma_address(s),
  1105. sg_dma_len(s));
  1106. if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
  1107. __dma_page_dev_to_cpu(sg_page(s), s->offset,
  1108. s->length, dir);
  1109. }
  1110. }
  1111. /**
  1112. * arm_iommu_sync_sg_for_cpu
  1113. * @dev: valid struct device pointer
  1114. * @sg: list of buffers
  1115. * @nents: number of buffers to map (returned from dma_map_sg)
  1116. * @dir: DMA transfer direction (same as was passed to dma_map_sg)
  1117. */
  1118. static void arm_iommu_sync_sg_for_cpu(struct device *dev,
  1119. struct scatterlist *sg,
  1120. int nents, enum dma_data_direction dir)
  1121. {
  1122. struct scatterlist *s;
  1123. int i;
  1124. if (dev->dma_coherent)
  1125. return;
  1126. for_each_sg(sg, s, nents, i)
  1127. __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
  1128. }
  1129. /**
  1130. * arm_iommu_sync_sg_for_device
  1131. * @dev: valid struct device pointer
  1132. * @sg: list of buffers
  1133. * @nents: number of buffers to map (returned from dma_map_sg)
  1134. * @dir: DMA transfer direction (same as was passed to dma_map_sg)
  1135. */
  1136. static void arm_iommu_sync_sg_for_device(struct device *dev,
  1137. struct scatterlist *sg,
  1138. int nents, enum dma_data_direction dir)
  1139. {
  1140. struct scatterlist *s;
  1141. int i;
  1142. if (dev->dma_coherent)
  1143. return;
  1144. for_each_sg(sg, s, nents, i)
  1145. __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
  1146. }
  1147. /**
  1148. * arm_iommu_map_page
  1149. * @dev: valid struct device pointer
  1150. * @page: page that buffer resides in
  1151. * @offset: offset into page for start of buffer
  1152. * @size: size of buffer to map
  1153. * @dir: DMA transfer direction
  1154. *
  1155. * IOMMU aware version of arm_dma_map_page()
  1156. */
  1157. static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
  1158. unsigned long offset, size_t size, enum dma_data_direction dir,
  1159. unsigned long attrs)
  1160. {
  1161. struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
  1162. dma_addr_t dma_addr;
  1163. int ret, prot, len = PAGE_ALIGN(size + offset);
  1164. if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
  1165. __dma_page_cpu_to_dev(page, offset, size, dir);
  1166. dma_addr = __alloc_iova(mapping, len);
  1167. if (dma_addr == DMA_MAPPING_ERROR)
  1168. return dma_addr;
  1169. prot = __dma_info_to_prot(dir, attrs);
  1170. ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot);
  1171. if (ret < 0)
  1172. goto fail;
  1173. return dma_addr + offset;
  1174. fail:
  1175. __free_iova(mapping, dma_addr, len);
  1176. return DMA_MAPPING_ERROR;
  1177. }
  1178. /**
  1179. * arm_iommu_unmap_page
  1180. * @dev: valid struct device pointer
  1181. * @handle: DMA address of buffer
  1182. * @size: size of buffer (same as passed to dma_map_page)
  1183. * @dir: DMA transfer direction (same as passed to dma_map_page)
  1184. *
  1185. * IOMMU aware version of arm_dma_unmap_page()
  1186. */
  1187. static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
  1188. size_t size, enum dma_data_direction dir, unsigned long attrs)
  1189. {
  1190. struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
  1191. dma_addr_t iova = handle & PAGE_MASK;
  1192. struct page *page;
  1193. int offset = handle & ~PAGE_MASK;
  1194. int len = PAGE_ALIGN(size + offset);
  1195. if (!iova)
  1196. return;
  1197. if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
  1198. page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
  1199. __dma_page_dev_to_cpu(page, offset, size, dir);
  1200. }
  1201. iommu_unmap(mapping->domain, iova, len);
  1202. __free_iova(mapping, iova, len);
  1203. }
  1204. /**
  1205. * arm_iommu_map_resource - map a device resource for DMA
  1206. * @dev: valid struct device pointer
  1207. * @phys_addr: physical address of resource
  1208. * @size: size of resource to map
  1209. * @dir: DMA transfer direction
  1210. */
  1211. static dma_addr_t arm_iommu_map_resource(struct device *dev,
  1212. phys_addr_t phys_addr, size_t size,
  1213. enum dma_data_direction dir, unsigned long attrs)
  1214. {
  1215. struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
  1216. dma_addr_t dma_addr;
  1217. int ret, prot;
  1218. phys_addr_t addr = phys_addr & PAGE_MASK;
  1219. unsigned int offset = phys_addr & ~PAGE_MASK;
  1220. size_t len = PAGE_ALIGN(size + offset);
  1221. dma_addr = __alloc_iova(mapping, len);
  1222. if (dma_addr == DMA_MAPPING_ERROR)
  1223. return dma_addr;
  1224. prot = __dma_info_to_prot(dir, attrs) | IOMMU_MMIO;
  1225. ret = iommu_map(mapping->domain, dma_addr, addr, len, prot);
  1226. if (ret < 0)
  1227. goto fail;
  1228. return dma_addr + offset;
  1229. fail:
  1230. __free_iova(mapping, dma_addr, len);
  1231. return DMA_MAPPING_ERROR;
  1232. }
  1233. /**
  1234. * arm_iommu_unmap_resource - unmap a device DMA resource
  1235. * @dev: valid struct device pointer
  1236. * @dma_handle: DMA address to resource
  1237. * @size: size of resource to map
  1238. * @dir: DMA transfer direction
  1239. */
  1240. static void arm_iommu_unmap_resource(struct device *dev, dma_addr_t dma_handle,
  1241. size_t size, enum dma_data_direction dir,
  1242. unsigned long attrs)
  1243. {
  1244. struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
  1245. dma_addr_t iova = dma_handle & PAGE_MASK;
  1246. unsigned int offset = dma_handle & ~PAGE_MASK;
  1247. size_t len = PAGE_ALIGN(size + offset);
  1248. if (!iova)
  1249. return;
  1250. iommu_unmap(mapping->domain, iova, len);
  1251. __free_iova(mapping, iova, len);
  1252. }
  1253. static void arm_iommu_sync_single_for_cpu(struct device *dev,
  1254. dma_addr_t handle, size_t size, enum dma_data_direction dir)
  1255. {
  1256. struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
  1257. dma_addr_t iova = handle & PAGE_MASK;
  1258. struct page *page;
  1259. unsigned int offset = handle & ~PAGE_MASK;
  1260. if (dev->dma_coherent || !iova)
  1261. return;
  1262. page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
  1263. __dma_page_dev_to_cpu(page, offset, size, dir);
  1264. }
  1265. static void arm_iommu_sync_single_for_device(struct device *dev,
  1266. dma_addr_t handle, size_t size, enum dma_data_direction dir)
  1267. {
  1268. struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
  1269. dma_addr_t iova = handle & PAGE_MASK;
  1270. struct page *page;
  1271. unsigned int offset = handle & ~PAGE_MASK;
  1272. if (dev->dma_coherent || !iova)
  1273. return;
  1274. page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
  1275. __dma_page_cpu_to_dev(page, offset, size, dir);
  1276. }
  1277. static const struct dma_map_ops iommu_ops = {
  1278. .alloc = arm_iommu_alloc_attrs,
  1279. .free = arm_iommu_free_attrs,
  1280. .mmap = arm_iommu_mmap_attrs,
  1281. .get_sgtable = arm_iommu_get_sgtable,
  1282. .map_page = arm_iommu_map_page,
  1283. .unmap_page = arm_iommu_unmap_page,
  1284. .sync_single_for_cpu = arm_iommu_sync_single_for_cpu,
  1285. .sync_single_for_device = arm_iommu_sync_single_for_device,
  1286. .map_sg = arm_iommu_map_sg,
  1287. .unmap_sg = arm_iommu_unmap_sg,
  1288. .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu,
  1289. .sync_sg_for_device = arm_iommu_sync_sg_for_device,
  1290. .map_resource = arm_iommu_map_resource,
  1291. .unmap_resource = arm_iommu_unmap_resource,
  1292. };
  1293. /**
  1294. * arm_iommu_create_mapping
  1295. * @bus: pointer to the bus holding the client device (for IOMMU calls)
  1296. * @base: start address of the valid IO address space
  1297. * @size: maximum size of the valid IO address space
  1298. *
  1299. * Creates a mapping structure which holds information about used/unused
  1300. * IO address ranges, which is required to perform memory allocation and
  1301. * mapping with IOMMU aware functions.
  1302. *
  1303. * The client device need to be attached to the mapping with
  1304. * arm_iommu_attach_device function.
  1305. */
  1306. struct dma_iommu_mapping *
  1307. arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, u64 size)
  1308. {
  1309. unsigned int bits = size >> PAGE_SHIFT;
  1310. unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long);
  1311. struct dma_iommu_mapping *mapping;
  1312. int extensions = 1;
  1313. int err = -ENOMEM;
  1314. /* currently only 32-bit DMA address space is supported */
  1315. if (size > DMA_BIT_MASK(32) + 1)
  1316. return ERR_PTR(-ERANGE);
  1317. if (!bitmap_size)
  1318. return ERR_PTR(-EINVAL);
  1319. if (bitmap_size > PAGE_SIZE) {
  1320. extensions = bitmap_size / PAGE_SIZE;
  1321. bitmap_size = PAGE_SIZE;
  1322. }
  1323. mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
  1324. if (!mapping)
  1325. goto err;
  1326. mapping->bitmap_size = bitmap_size;
  1327. mapping->bitmaps = kcalloc(extensions, sizeof(unsigned long *),
  1328. GFP_KERNEL);
  1329. if (!mapping->bitmaps)
  1330. goto err2;
  1331. mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL);
  1332. if (!mapping->bitmaps[0])
  1333. goto err3;
  1334. mapping->nr_bitmaps = 1;
  1335. mapping->extensions = extensions;
  1336. mapping->base = base;
  1337. mapping->bits = BITS_PER_BYTE * bitmap_size;
  1338. spin_lock_init(&mapping->lock);
  1339. mapping->domain = iommu_domain_alloc(bus);
  1340. if (!mapping->domain)
  1341. goto err4;
  1342. kref_init(&mapping->kref);
  1343. return mapping;
  1344. err4:
  1345. kfree(mapping->bitmaps[0]);
  1346. err3:
  1347. kfree(mapping->bitmaps);
  1348. err2:
  1349. kfree(mapping);
  1350. err:
  1351. return ERR_PTR(err);
  1352. }
  1353. EXPORT_SYMBOL_GPL(arm_iommu_create_mapping);
  1354. static void release_iommu_mapping(struct kref *kref)
  1355. {
  1356. int i;
  1357. struct dma_iommu_mapping *mapping =
  1358. container_of(kref, struct dma_iommu_mapping, kref);
  1359. iommu_domain_free(mapping->domain);
  1360. for (i = 0; i < mapping->nr_bitmaps; i++)
  1361. kfree(mapping->bitmaps[i]);
  1362. kfree(mapping->bitmaps);
  1363. kfree(mapping);
  1364. }
  1365. static int extend_iommu_mapping(struct dma_iommu_mapping *mapping)
  1366. {
  1367. int next_bitmap;
  1368. if (mapping->nr_bitmaps >= mapping->extensions)
  1369. return -EINVAL;
  1370. next_bitmap = mapping->nr_bitmaps;
  1371. mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size,
  1372. GFP_ATOMIC);
  1373. if (!mapping->bitmaps[next_bitmap])
  1374. return -ENOMEM;
  1375. mapping->nr_bitmaps++;
  1376. return 0;
  1377. }
  1378. void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
  1379. {
  1380. if (mapping)
  1381. kref_put(&mapping->kref, release_iommu_mapping);
  1382. }
  1383. EXPORT_SYMBOL_GPL(arm_iommu_release_mapping);
  1384. static int __arm_iommu_attach_device(struct device *dev,
  1385. struct dma_iommu_mapping *mapping)
  1386. {
  1387. int err;
  1388. err = iommu_attach_device(mapping->domain, dev);
  1389. if (err)
  1390. return err;
  1391. kref_get(&mapping->kref);
  1392. to_dma_iommu_mapping(dev) = mapping;
  1393. pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
  1394. return 0;
  1395. }
  1396. /**
  1397. * arm_iommu_attach_device
  1398. * @dev: valid struct device pointer
  1399. * @mapping: io address space mapping structure (returned from
  1400. * arm_iommu_create_mapping)
  1401. *
  1402. * Attaches specified io address space mapping to the provided device.
  1403. * This replaces the dma operations (dma_map_ops pointer) with the
  1404. * IOMMU aware version.
  1405. *
  1406. * More than one client might be attached to the same io address space
  1407. * mapping.
  1408. */
  1409. int arm_iommu_attach_device(struct device *dev,
  1410. struct dma_iommu_mapping *mapping)
  1411. {
  1412. int err;
  1413. err = __arm_iommu_attach_device(dev, mapping);
  1414. if (err)
  1415. return err;
  1416. set_dma_ops(dev, &iommu_ops);
  1417. return 0;
  1418. }
  1419. EXPORT_SYMBOL_GPL(arm_iommu_attach_device);
  1420. /**
  1421. * arm_iommu_detach_device
  1422. * @dev: valid struct device pointer
  1423. *
  1424. * Detaches the provided device from a previously attached map.
  1425. * This overwrites the dma_ops pointer with appropriate non-IOMMU ops.
  1426. */
  1427. void arm_iommu_detach_device(struct device *dev)
  1428. {
  1429. struct dma_iommu_mapping *mapping;
  1430. mapping = to_dma_iommu_mapping(dev);
  1431. if (!mapping) {
  1432. dev_warn(dev, "Not attached\n");
  1433. return;
  1434. }
  1435. iommu_detach_device(mapping->domain, dev);
  1436. kref_put(&mapping->kref, release_iommu_mapping);
  1437. to_dma_iommu_mapping(dev) = NULL;
  1438. set_dma_ops(dev, NULL);
  1439. pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));
  1440. }
  1441. EXPORT_SYMBOL_GPL(arm_iommu_detach_device);
  1442. static void arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
  1443. const struct iommu_ops *iommu, bool coherent)
  1444. {
  1445. struct dma_iommu_mapping *mapping;
  1446. mapping = arm_iommu_create_mapping(dev->bus, dma_base, size);
  1447. if (IS_ERR(mapping)) {
  1448. pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n",
  1449. size, dev_name(dev));
  1450. return;
  1451. }
  1452. if (__arm_iommu_attach_device(dev, mapping)) {
  1453. pr_warn("Failed to attached device %s to IOMMU_mapping\n",
  1454. dev_name(dev));
  1455. arm_iommu_release_mapping(mapping);
  1456. return;
  1457. }
  1458. set_dma_ops(dev, &iommu_ops);
  1459. }
  1460. static void arm_teardown_iommu_dma_ops(struct device *dev)
  1461. {
  1462. struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
  1463. if (!mapping)
  1464. return;
  1465. arm_iommu_detach_device(dev);
  1466. arm_iommu_release_mapping(mapping);
  1467. }
  1468. #else
  1469. static void arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
  1470. const struct iommu_ops *iommu, bool coherent)
  1471. {
  1472. }
  1473. static void arm_teardown_iommu_dma_ops(struct device *dev) { }
  1474. #endif /* CONFIG_ARM_DMA_USE_IOMMU */
  1475. void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
  1476. const struct iommu_ops *iommu, bool coherent)
  1477. {
  1478. /*
  1479. * Due to legacy code that sets the ->dma_coherent flag from a bus
  1480. * notifier we can't just assign coherent to the ->dma_coherent flag
  1481. * here, but instead have to make sure we only set but never clear it
  1482. * for now.
  1483. */
  1484. if (coherent)
  1485. dev->dma_coherent = true;
  1486. /*
  1487. * Don't override the dma_ops if they have already been set. Ideally
  1488. * this should be the only location where dma_ops are set, remove this
  1489. * check when all other callers of set_dma_ops will have disappeared.
  1490. */
  1491. if (dev->dma_ops)
  1492. return;
  1493. if (iommu)
  1494. arm_setup_iommu_dma_ops(dev, dma_base, size, iommu, coherent);
  1495. xen_setup_dma_ops(dev);
  1496. dev->archdata.dma_ops_setup = true;
  1497. }
  1498. void arch_teardown_dma_ops(struct device *dev)
  1499. {
  1500. if (!dev->archdata.dma_ops_setup)
  1501. return;
  1502. arm_teardown_iommu_dma_ops(dev);
  1503. /* Let arch_setup_dma_ops() start again from scratch upon re-probe */
  1504. set_dma_ops(dev, NULL);
  1505. }
  1506. void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
  1507. enum dma_data_direction dir)
  1508. {
  1509. __dma_page_cpu_to_dev(phys_to_page(paddr), paddr & (PAGE_SIZE - 1),
  1510. size, dir);
  1511. }
  1512. void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
  1513. enum dma_data_direction dir)
  1514. {
  1515. __dma_page_dev_to_cpu(phys_to_page(paddr), paddr & (PAGE_SIZE - 1),
  1516. size, dir);
  1517. }
  1518. void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
  1519. gfp_t gfp, unsigned long attrs)
  1520. {
  1521. return __dma_alloc(dev, size, dma_handle, gfp,
  1522. __get_dma_pgprot(attrs, PAGE_KERNEL), false,
  1523. attrs, __builtin_return_address(0));
  1524. }
  1525. void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
  1526. dma_addr_t dma_handle, unsigned long attrs)
  1527. {
  1528. __arm_dma_free(dev, size, cpu_addr, dma_handle, attrs, false);
  1529. }