apple-admac.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Driver for Audio DMA Controller (ADMAC) on t8103 (M1) and other Apple chips
  4. *
  5. * Copyright (C) The Asahi Linux Contributors
  6. */
  7. #include <linux/bits.h>
  8. #include <linux/bitfield.h>
  9. #include <linux/device.h>
  10. #include <linux/init.h>
  11. #include <linux/module.h>
  12. #include <linux/of_device.h>
  13. #include <linux/of_dma.h>
  14. #include <linux/reset.h>
  15. #include <linux/spinlock.h>
  16. #include <linux/interrupt.h>
  17. #include "dmaengine.h"
  18. #define NCHANNELS_MAX 64
  19. #define IRQ_NOUTPUTS 4
  20. /*
  21. * For allocation purposes we split the cache
  22. * memory into blocks of fixed size (given in bytes).
  23. */
  24. #define SRAM_BLOCK 2048
  25. #define RING_WRITE_SLOT GENMASK(1, 0)
  26. #define RING_READ_SLOT GENMASK(5, 4)
  27. #define RING_FULL BIT(9)
  28. #define RING_EMPTY BIT(8)
  29. #define RING_ERR BIT(10)
  30. #define STATUS_DESC_DONE BIT(0)
  31. #define STATUS_ERR BIT(6)
  32. #define FLAG_DESC_NOTIFY BIT(16)
  33. #define REG_TX_START 0x0000
  34. #define REG_TX_STOP 0x0004
  35. #define REG_RX_START 0x0008
  36. #define REG_RX_STOP 0x000c
  37. #define REG_IMPRINT 0x0090
  38. #define REG_TX_SRAM_SIZE 0x0094
  39. #define REG_RX_SRAM_SIZE 0x0098
  40. #define REG_CHAN_CTL(ch) (0x8000 + (ch) * 0x200)
  41. #define REG_CHAN_CTL_RST_RINGS BIT(0)
  42. #define REG_DESC_RING(ch) (0x8070 + (ch) * 0x200)
  43. #define REG_REPORT_RING(ch) (0x8074 + (ch) * 0x200)
  44. #define REG_RESIDUE(ch) (0x8064 + (ch) * 0x200)
  45. #define REG_BUS_WIDTH(ch) (0x8040 + (ch) * 0x200)
  46. #define BUS_WIDTH_8BIT 0x00
  47. #define BUS_WIDTH_16BIT 0x01
  48. #define BUS_WIDTH_32BIT 0x02
  49. #define BUS_WIDTH_FRAME_2_WORDS 0x10
  50. #define BUS_WIDTH_FRAME_4_WORDS 0x20
  51. #define REG_CHAN_SRAM_CARVEOUT(ch) (0x8050 + (ch) * 0x200)
  52. #define CHAN_SRAM_CARVEOUT_SIZE GENMASK(31, 16)
  53. #define CHAN_SRAM_CARVEOUT_BASE GENMASK(15, 0)
  54. #define REG_CHAN_FIFOCTL(ch) (0x8054 + (ch) * 0x200)
  55. #define CHAN_FIFOCTL_LIMIT GENMASK(31, 16)
  56. #define CHAN_FIFOCTL_THRESHOLD GENMASK(15, 0)
  57. #define REG_DESC_WRITE(ch) (0x10000 + ((ch) / 2) * 0x4 + ((ch) & 1) * 0x4000)
  58. #define REG_REPORT_READ(ch) (0x10100 + ((ch) / 2) * 0x4 + ((ch) & 1) * 0x4000)
  59. #define REG_TX_INTSTATE(idx) (0x0030 + (idx) * 4)
  60. #define REG_RX_INTSTATE(idx) (0x0040 + (idx) * 4)
  61. #define REG_GLOBAL_INTSTATE(idx) (0x0050 + (idx) * 4)
  62. #define REG_CHAN_INTSTATUS(ch, idx) (0x8010 + (ch) * 0x200 + (idx) * 4)
  63. #define REG_CHAN_INTMASK(ch, idx) (0x8020 + (ch) * 0x200 + (idx) * 4)
  64. struct admac_data;
  65. struct admac_tx;
  66. struct admac_chan {
  67. unsigned int no;
  68. struct admac_data *host;
  69. struct dma_chan chan;
  70. struct tasklet_struct tasklet;
  71. u32 carveout;
  72. spinlock_t lock;
  73. struct admac_tx *current_tx;
  74. int nperiod_acks;
  75. /*
  76. * We maintain a 'submitted' and 'issued' list mainly for interface
  77. * correctness. Typical use of the driver (per channel) will be
  78. * prepping, submitting and issuing a single cyclic transaction which
  79. * will stay current until terminate_all is called.
  80. */
  81. struct list_head submitted;
  82. struct list_head issued;
  83. struct list_head to_free;
  84. };
  85. struct admac_sram {
  86. u32 size;
  87. /*
  88. * SRAM_CARVEOUT has 16-bit fields, so the SRAM cannot be larger than
  89. * 64K and a 32-bit bitfield over 2K blocks covers it.
  90. */
  91. u32 allocated;
  92. };
  93. struct admac_data {
  94. struct dma_device dma;
  95. struct device *dev;
  96. __iomem void *base;
  97. struct reset_control *rstc;
  98. struct mutex cache_alloc_lock;
  99. struct admac_sram txcache, rxcache;
  100. int irq;
  101. int irq_index;
  102. int nchannels;
  103. struct admac_chan channels[];
  104. };
  105. struct admac_tx {
  106. struct dma_async_tx_descriptor tx;
  107. bool cyclic;
  108. dma_addr_t buf_addr;
  109. dma_addr_t buf_end;
  110. size_t buf_len;
  111. size_t period_len;
  112. size_t submitted_pos;
  113. size_t reclaimed_pos;
  114. struct list_head node;
  115. };
  116. static int admac_alloc_sram_carveout(struct admac_data *ad,
  117. enum dma_transfer_direction dir,
  118. u32 *out)
  119. {
  120. struct admac_sram *sram;
  121. int i, ret = 0, nblocks;
  122. if (dir == DMA_MEM_TO_DEV)
  123. sram = &ad->txcache;
  124. else
  125. sram = &ad->rxcache;
  126. mutex_lock(&ad->cache_alloc_lock);
  127. nblocks = sram->size / SRAM_BLOCK;
  128. for (i = 0; i < nblocks; i++)
  129. if (!(sram->allocated & BIT(i)))
  130. break;
  131. if (i < nblocks) {
  132. *out = FIELD_PREP(CHAN_SRAM_CARVEOUT_BASE, i * SRAM_BLOCK) |
  133. FIELD_PREP(CHAN_SRAM_CARVEOUT_SIZE, SRAM_BLOCK);
  134. sram->allocated |= BIT(i);
  135. } else {
  136. ret = -EBUSY;
  137. }
  138. mutex_unlock(&ad->cache_alloc_lock);
  139. return ret;
  140. }
  141. static void admac_free_sram_carveout(struct admac_data *ad,
  142. enum dma_transfer_direction dir,
  143. u32 carveout)
  144. {
  145. struct admac_sram *sram;
  146. u32 base = FIELD_GET(CHAN_SRAM_CARVEOUT_BASE, carveout);
  147. int i;
  148. if (dir == DMA_MEM_TO_DEV)
  149. sram = &ad->txcache;
  150. else
  151. sram = &ad->rxcache;
  152. if (WARN_ON(base >= sram->size))
  153. return;
  154. mutex_lock(&ad->cache_alloc_lock);
  155. i = base / SRAM_BLOCK;
  156. sram->allocated &= ~BIT(i);
  157. mutex_unlock(&ad->cache_alloc_lock);
  158. }
  159. static void admac_modify(struct admac_data *ad, int reg, u32 mask, u32 val)
  160. {
  161. void __iomem *addr = ad->base + reg;
  162. u32 curr = readl_relaxed(addr);
  163. writel_relaxed((curr & ~mask) | (val & mask), addr);
  164. }
  165. static struct admac_chan *to_admac_chan(struct dma_chan *chan)
  166. {
  167. return container_of(chan, struct admac_chan, chan);
  168. }
  169. static struct admac_tx *to_admac_tx(struct dma_async_tx_descriptor *tx)
  170. {
  171. return container_of(tx, struct admac_tx, tx);
  172. }
  173. static enum dma_transfer_direction admac_chan_direction(int channo)
  174. {
  175. /* Channel directions are hardwired */
  176. return (channo & 1) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV;
  177. }
  178. static dma_cookie_t admac_tx_submit(struct dma_async_tx_descriptor *tx)
  179. {
  180. struct admac_tx *adtx = to_admac_tx(tx);
  181. struct admac_chan *adchan = to_admac_chan(tx->chan);
  182. unsigned long flags;
  183. dma_cookie_t cookie;
  184. spin_lock_irqsave(&adchan->lock, flags);
  185. cookie = dma_cookie_assign(tx);
  186. list_add_tail(&adtx->node, &adchan->submitted);
  187. spin_unlock_irqrestore(&adchan->lock, flags);
  188. return cookie;
  189. }
  190. static int admac_desc_free(struct dma_async_tx_descriptor *tx)
  191. {
  192. kfree(to_admac_tx(tx));
  193. return 0;
  194. }
  195. static struct dma_async_tx_descriptor *admac_prep_dma_cyclic(
  196. struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
  197. size_t period_len, enum dma_transfer_direction direction,
  198. unsigned long flags)
  199. {
  200. struct admac_chan *adchan = container_of(chan, struct admac_chan, chan);
  201. struct admac_tx *adtx;
  202. if (direction != admac_chan_direction(adchan->no))
  203. return NULL;
  204. adtx = kzalloc(sizeof(*adtx), GFP_NOWAIT);
  205. if (!adtx)
  206. return NULL;
  207. adtx->cyclic = true;
  208. adtx->buf_addr = buf_addr;
  209. adtx->buf_len = buf_len;
  210. adtx->buf_end = buf_addr + buf_len;
  211. adtx->period_len = period_len;
  212. adtx->submitted_pos = 0;
  213. adtx->reclaimed_pos = 0;
  214. dma_async_tx_descriptor_init(&adtx->tx, chan);
  215. adtx->tx.tx_submit = admac_tx_submit;
  216. adtx->tx.desc_free = admac_desc_free;
  217. return &adtx->tx;
  218. }
  219. /*
  220. * Write one hardware descriptor for a dmaengine cyclic transaction.
  221. */
  222. static void admac_cyclic_write_one_desc(struct admac_data *ad, int channo,
  223. struct admac_tx *tx)
  224. {
  225. dma_addr_t addr;
  226. addr = tx->buf_addr + (tx->submitted_pos % tx->buf_len);
  227. /* If happens means we have buggy code */
  228. WARN_ON_ONCE(addr + tx->period_len > tx->buf_end);
  229. dev_dbg(ad->dev, "ch%d descriptor: addr=0x%pad len=0x%zx flags=0x%lx\n",
  230. channo, &addr, tx->period_len, FLAG_DESC_NOTIFY);
  231. writel_relaxed(lower_32_bits(addr), ad->base + REG_DESC_WRITE(channo));
  232. writel_relaxed(upper_32_bits(addr), ad->base + REG_DESC_WRITE(channo));
  233. writel_relaxed(tx->period_len, ad->base + REG_DESC_WRITE(channo));
  234. writel_relaxed(FLAG_DESC_NOTIFY, ad->base + REG_DESC_WRITE(channo));
  235. tx->submitted_pos += tx->period_len;
  236. tx->submitted_pos %= 2 * tx->buf_len;
  237. }
  238. /*
  239. * Write all the hardware descriptors for a dmaengine cyclic
  240. * transaction there is space for.
  241. */
  242. static void admac_cyclic_write_desc(struct admac_data *ad, int channo,
  243. struct admac_tx *tx)
  244. {
  245. int i;
  246. for (i = 0; i < 4; i++) {
  247. if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_FULL)
  248. break;
  249. admac_cyclic_write_one_desc(ad, channo, tx);
  250. }
  251. }
  252. static int admac_ring_noccupied_slots(int ringval)
  253. {
  254. int wrslot = FIELD_GET(RING_WRITE_SLOT, ringval);
  255. int rdslot = FIELD_GET(RING_READ_SLOT, ringval);
  256. if (wrslot != rdslot) {
  257. return (wrslot + 4 - rdslot) % 4;
  258. } else {
  259. WARN_ON((ringval & (RING_FULL | RING_EMPTY)) == 0);
  260. if (ringval & RING_FULL)
  261. return 4;
  262. else
  263. return 0;
  264. }
  265. }
  266. /*
  267. * Read from hardware the residue of a cyclic dmaengine transaction.
  268. */
  269. static u32 admac_cyclic_read_residue(struct admac_data *ad, int channo,
  270. struct admac_tx *adtx)
  271. {
  272. u32 ring1, ring2;
  273. u32 residue1, residue2;
  274. int nreports;
  275. size_t pos;
  276. ring1 = readl_relaxed(ad->base + REG_REPORT_RING(channo));
  277. residue1 = readl_relaxed(ad->base + REG_RESIDUE(channo));
  278. ring2 = readl_relaxed(ad->base + REG_REPORT_RING(channo));
  279. residue2 = readl_relaxed(ad->base + REG_RESIDUE(channo));
  280. if (residue2 > residue1) {
  281. /*
  282. * Controller must have loaded next descriptor between
  283. * the two residue reads
  284. */
  285. nreports = admac_ring_noccupied_slots(ring1) + 1;
  286. } else {
  287. /* No descriptor load between the two reads, ring2 is safe to use */
  288. nreports = admac_ring_noccupied_slots(ring2);
  289. }
  290. pos = adtx->reclaimed_pos + adtx->period_len * (nreports + 1) - residue2;
  291. return adtx->buf_len - pos % adtx->buf_len;
  292. }
  293. static enum dma_status admac_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
  294. struct dma_tx_state *txstate)
  295. {
  296. struct admac_chan *adchan = to_admac_chan(chan);
  297. struct admac_data *ad = adchan->host;
  298. struct admac_tx *adtx;
  299. enum dma_status ret;
  300. size_t residue;
  301. unsigned long flags;
  302. ret = dma_cookie_status(chan, cookie, txstate);
  303. if (ret == DMA_COMPLETE || !txstate)
  304. return ret;
  305. spin_lock_irqsave(&adchan->lock, flags);
  306. adtx = adchan->current_tx;
  307. if (adtx && adtx->tx.cookie == cookie) {
  308. ret = DMA_IN_PROGRESS;
  309. residue = admac_cyclic_read_residue(ad, adchan->no, adtx);
  310. } else {
  311. ret = DMA_IN_PROGRESS;
  312. residue = 0;
  313. list_for_each_entry(adtx, &adchan->issued, node) {
  314. if (adtx->tx.cookie == cookie) {
  315. residue = adtx->buf_len;
  316. break;
  317. }
  318. }
  319. }
  320. spin_unlock_irqrestore(&adchan->lock, flags);
  321. dma_set_residue(txstate, residue);
  322. return ret;
  323. }
  324. static void admac_start_chan(struct admac_chan *adchan)
  325. {
  326. struct admac_data *ad = adchan->host;
  327. u32 startbit = 1 << (adchan->no / 2);
  328. writel_relaxed(STATUS_DESC_DONE | STATUS_ERR,
  329. ad->base + REG_CHAN_INTSTATUS(adchan->no, ad->irq_index));
  330. writel_relaxed(STATUS_DESC_DONE | STATUS_ERR,
  331. ad->base + REG_CHAN_INTMASK(adchan->no, ad->irq_index));
  332. switch (admac_chan_direction(adchan->no)) {
  333. case DMA_MEM_TO_DEV:
  334. writel_relaxed(startbit, ad->base + REG_TX_START);
  335. break;
  336. case DMA_DEV_TO_MEM:
  337. writel_relaxed(startbit, ad->base + REG_RX_START);
  338. break;
  339. default:
  340. break;
  341. }
  342. dev_dbg(adchan->host->dev, "ch%d start\n", adchan->no);
  343. }
  344. static void admac_stop_chan(struct admac_chan *adchan)
  345. {
  346. struct admac_data *ad = adchan->host;
  347. u32 stopbit = 1 << (adchan->no / 2);
  348. switch (admac_chan_direction(adchan->no)) {
  349. case DMA_MEM_TO_DEV:
  350. writel_relaxed(stopbit, ad->base + REG_TX_STOP);
  351. break;
  352. case DMA_DEV_TO_MEM:
  353. writel_relaxed(stopbit, ad->base + REG_RX_STOP);
  354. break;
  355. default:
  356. break;
  357. }
  358. dev_dbg(adchan->host->dev, "ch%d stop\n", adchan->no);
  359. }
  360. static void admac_reset_rings(struct admac_chan *adchan)
  361. {
  362. struct admac_data *ad = adchan->host;
  363. writel_relaxed(REG_CHAN_CTL_RST_RINGS,
  364. ad->base + REG_CHAN_CTL(adchan->no));
  365. writel_relaxed(0, ad->base + REG_CHAN_CTL(adchan->no));
  366. }
  367. static void admac_start_current_tx(struct admac_chan *adchan)
  368. {
  369. struct admac_data *ad = adchan->host;
  370. int ch = adchan->no;
  371. admac_reset_rings(adchan);
  372. writel_relaxed(0, ad->base + REG_CHAN_CTL(ch));
  373. admac_cyclic_write_one_desc(ad, ch, adchan->current_tx);
  374. admac_start_chan(adchan);
  375. admac_cyclic_write_desc(ad, ch, adchan->current_tx);
  376. }
  377. static void admac_issue_pending(struct dma_chan *chan)
  378. {
  379. struct admac_chan *adchan = to_admac_chan(chan);
  380. struct admac_tx *tx;
  381. unsigned long flags;
  382. spin_lock_irqsave(&adchan->lock, flags);
  383. list_splice_tail_init(&adchan->submitted, &adchan->issued);
  384. if (!list_empty(&adchan->issued) && !adchan->current_tx) {
  385. tx = list_first_entry(&adchan->issued, struct admac_tx, node);
  386. list_del(&tx->node);
  387. adchan->current_tx = tx;
  388. adchan->nperiod_acks = 0;
  389. admac_start_current_tx(adchan);
  390. }
  391. spin_unlock_irqrestore(&adchan->lock, flags);
  392. }
  393. static int admac_pause(struct dma_chan *chan)
  394. {
  395. struct admac_chan *adchan = to_admac_chan(chan);
  396. admac_stop_chan(adchan);
  397. return 0;
  398. }
  399. static int admac_resume(struct dma_chan *chan)
  400. {
  401. struct admac_chan *adchan = to_admac_chan(chan);
  402. admac_start_chan(adchan);
  403. return 0;
  404. }
  405. static int admac_terminate_all(struct dma_chan *chan)
  406. {
  407. struct admac_chan *adchan = to_admac_chan(chan);
  408. unsigned long flags;
  409. spin_lock_irqsave(&adchan->lock, flags);
  410. admac_stop_chan(adchan);
  411. admac_reset_rings(adchan);
  412. if (adchan->current_tx) {
  413. list_add_tail(&adchan->current_tx->node, &adchan->to_free);
  414. adchan->current_tx = NULL;
  415. }
  416. /*
  417. * Descriptors can only be freed after the tasklet
  418. * has been killed (in admac_synchronize).
  419. */
  420. list_splice_tail_init(&adchan->submitted, &adchan->to_free);
  421. list_splice_tail_init(&adchan->issued, &adchan->to_free);
  422. spin_unlock_irqrestore(&adchan->lock, flags);
  423. return 0;
  424. }
  425. static void admac_synchronize(struct dma_chan *chan)
  426. {
  427. struct admac_chan *adchan = to_admac_chan(chan);
  428. struct admac_tx *adtx, *_adtx;
  429. unsigned long flags;
  430. LIST_HEAD(head);
  431. spin_lock_irqsave(&adchan->lock, flags);
  432. list_splice_tail_init(&adchan->to_free, &head);
  433. spin_unlock_irqrestore(&adchan->lock, flags);
  434. tasklet_kill(&adchan->tasklet);
  435. list_for_each_entry_safe(adtx, _adtx, &head, node) {
  436. list_del(&adtx->node);
  437. admac_desc_free(&adtx->tx);
  438. }
  439. }
  440. static int admac_alloc_chan_resources(struct dma_chan *chan)
  441. {
  442. struct admac_chan *adchan = to_admac_chan(chan);
  443. struct admac_data *ad = adchan->host;
  444. int ret;
  445. dma_cookie_init(&adchan->chan);
  446. ret = admac_alloc_sram_carveout(ad, admac_chan_direction(adchan->no),
  447. &adchan->carveout);
  448. if (ret < 0)
  449. return ret;
  450. writel_relaxed(adchan->carveout,
  451. ad->base + REG_CHAN_SRAM_CARVEOUT(adchan->no));
  452. return 0;
  453. }
  454. static void admac_free_chan_resources(struct dma_chan *chan)
  455. {
  456. struct admac_chan *adchan = to_admac_chan(chan);
  457. admac_terminate_all(chan);
  458. admac_synchronize(chan);
  459. admac_free_sram_carveout(adchan->host, admac_chan_direction(adchan->no),
  460. adchan->carveout);
  461. }
  462. static struct dma_chan *admac_dma_of_xlate(struct of_phandle_args *dma_spec,
  463. struct of_dma *ofdma)
  464. {
  465. struct admac_data *ad = (struct admac_data *) ofdma->of_dma_data;
  466. unsigned int index;
  467. if (dma_spec->args_count != 1)
  468. return NULL;
  469. index = dma_spec->args[0];
  470. if (index >= ad->nchannels) {
  471. dev_err(ad->dev, "channel index %u out of bounds\n", index);
  472. return NULL;
  473. }
  474. return dma_get_slave_channel(&ad->channels[index].chan);
  475. }
  476. static int admac_drain_reports(struct admac_data *ad, int channo)
  477. {
  478. int count;
  479. for (count = 0; count < 4; count++) {
  480. u32 countval_hi, countval_lo, unk1, flags;
  481. if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_EMPTY)
  482. break;
  483. countval_lo = readl_relaxed(ad->base + REG_REPORT_READ(channo));
  484. countval_hi = readl_relaxed(ad->base + REG_REPORT_READ(channo));
  485. unk1 = readl_relaxed(ad->base + REG_REPORT_READ(channo));
  486. flags = readl_relaxed(ad->base + REG_REPORT_READ(channo));
  487. dev_dbg(ad->dev, "ch%d report: countval=0x%llx unk1=0x%x flags=0x%x\n",
  488. channo, ((u64) countval_hi) << 32 | countval_lo, unk1, flags);
  489. }
  490. return count;
  491. }
  492. static void admac_handle_status_err(struct admac_data *ad, int channo)
  493. {
  494. bool handled = false;
  495. if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_ERR) {
  496. writel_relaxed(RING_ERR, ad->base + REG_DESC_RING(channo));
  497. dev_err_ratelimited(ad->dev, "ch%d descriptor ring error\n", channo);
  498. handled = true;
  499. }
  500. if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_ERR) {
  501. writel_relaxed(RING_ERR, ad->base + REG_REPORT_RING(channo));
  502. dev_err_ratelimited(ad->dev, "ch%d report ring error\n", channo);
  503. handled = true;
  504. }
  505. if (unlikely(!handled)) {
  506. dev_err(ad->dev, "ch%d unknown error, masking errors as cause of IRQs\n", channo);
  507. admac_modify(ad, REG_CHAN_INTMASK(channo, ad->irq_index),
  508. STATUS_ERR, 0);
  509. }
  510. }
  511. static void admac_handle_status_desc_done(struct admac_data *ad, int channo)
  512. {
  513. struct admac_chan *adchan = &ad->channels[channo];
  514. unsigned long flags;
  515. int nreports;
  516. writel_relaxed(STATUS_DESC_DONE,
  517. ad->base + REG_CHAN_INTSTATUS(channo, ad->irq_index));
  518. spin_lock_irqsave(&adchan->lock, flags);
  519. nreports = admac_drain_reports(ad, channo);
  520. if (adchan->current_tx) {
  521. struct admac_tx *tx = adchan->current_tx;
  522. adchan->nperiod_acks += nreports;
  523. tx->reclaimed_pos += nreports * tx->period_len;
  524. tx->reclaimed_pos %= 2 * tx->buf_len;
  525. admac_cyclic_write_desc(ad, channo, tx);
  526. tasklet_schedule(&adchan->tasklet);
  527. }
  528. spin_unlock_irqrestore(&adchan->lock, flags);
  529. }
  530. static void admac_handle_chan_int(struct admac_data *ad, int no)
  531. {
  532. u32 cause = readl_relaxed(ad->base + REG_CHAN_INTSTATUS(no, ad->irq_index));
  533. if (cause & STATUS_ERR)
  534. admac_handle_status_err(ad, no);
  535. if (cause & STATUS_DESC_DONE)
  536. admac_handle_status_desc_done(ad, no);
  537. }
  538. static irqreturn_t admac_interrupt(int irq, void *devid)
  539. {
  540. struct admac_data *ad = devid;
  541. u32 rx_intstate, tx_intstate, global_intstate;
  542. int i;
  543. rx_intstate = readl_relaxed(ad->base + REG_RX_INTSTATE(ad->irq_index));
  544. tx_intstate = readl_relaxed(ad->base + REG_TX_INTSTATE(ad->irq_index));
  545. global_intstate = readl_relaxed(ad->base + REG_GLOBAL_INTSTATE(ad->irq_index));
  546. if (!tx_intstate && !rx_intstate && !global_intstate)
  547. return IRQ_NONE;
  548. for (i = 0; i < ad->nchannels; i += 2) {
  549. if (tx_intstate & 1)
  550. admac_handle_chan_int(ad, i);
  551. tx_intstate >>= 1;
  552. }
  553. for (i = 1; i < ad->nchannels; i += 2) {
  554. if (rx_intstate & 1)
  555. admac_handle_chan_int(ad, i);
  556. rx_intstate >>= 1;
  557. }
  558. if (global_intstate) {
  559. dev_warn(ad->dev, "clearing unknown global interrupt flag: %x\n",
  560. global_intstate);
  561. writel_relaxed(~(u32) 0, ad->base + REG_GLOBAL_INTSTATE(ad->irq_index));
  562. }
  563. return IRQ_HANDLED;
  564. }
  565. static void admac_chan_tasklet(struct tasklet_struct *t)
  566. {
  567. struct admac_chan *adchan = from_tasklet(adchan, t, tasklet);
  568. struct admac_tx *adtx;
  569. struct dmaengine_desc_callback cb;
  570. struct dmaengine_result tx_result;
  571. int nacks;
  572. spin_lock_irq(&adchan->lock);
  573. adtx = adchan->current_tx;
  574. nacks = adchan->nperiod_acks;
  575. adchan->nperiod_acks = 0;
  576. spin_unlock_irq(&adchan->lock);
  577. if (!adtx || !nacks)
  578. return;
  579. tx_result.result = DMA_TRANS_NOERROR;
  580. tx_result.residue = 0;
  581. dmaengine_desc_get_callback(&adtx->tx, &cb);
  582. while (nacks--)
  583. dmaengine_desc_callback_invoke(&cb, &tx_result);
  584. }
  585. static int admac_device_config(struct dma_chan *chan,
  586. struct dma_slave_config *config)
  587. {
  588. struct admac_chan *adchan = to_admac_chan(chan);
  589. struct admac_data *ad = adchan->host;
  590. bool is_tx = admac_chan_direction(adchan->no) == DMA_MEM_TO_DEV;
  591. int wordsize = 0;
  592. u32 bus_width = 0;
  593. switch (is_tx ? config->dst_addr_width : config->src_addr_width) {
  594. case DMA_SLAVE_BUSWIDTH_1_BYTE:
  595. wordsize = 1;
  596. bus_width |= BUS_WIDTH_8BIT;
  597. break;
  598. case DMA_SLAVE_BUSWIDTH_2_BYTES:
  599. wordsize = 2;
  600. bus_width |= BUS_WIDTH_16BIT;
  601. break;
  602. case DMA_SLAVE_BUSWIDTH_4_BYTES:
  603. wordsize = 4;
  604. bus_width |= BUS_WIDTH_32BIT;
  605. break;
  606. default:
  607. return -EINVAL;
  608. }
  609. /*
  610. * We take port_window_size to be the number of words in a frame.
  611. *
  612. * The controller has some means of out-of-band signalling, to the peripheral,
  613. * of words position in a frame. That's where the importance of this control
  614. * comes from.
  615. */
  616. switch (is_tx ? config->dst_port_window_size : config->src_port_window_size) {
  617. case 0 ... 1:
  618. break;
  619. case 2:
  620. bus_width |= BUS_WIDTH_FRAME_2_WORDS;
  621. break;
  622. case 4:
  623. bus_width |= BUS_WIDTH_FRAME_4_WORDS;
  624. break;
  625. default:
  626. return -EINVAL;
  627. }
  628. writel_relaxed(bus_width, ad->base + REG_BUS_WIDTH(adchan->no));
  629. /*
  630. * By FIFOCTL_LIMIT we seem to set the maximal number of bytes allowed to be
  631. * held in controller's per-channel FIFO. Transfers seem to be triggered
  632. * around the time FIFO occupancy touches FIFOCTL_THRESHOLD.
  633. *
  634. * The numbers we set are more or less arbitrary.
  635. */
  636. writel_relaxed(FIELD_PREP(CHAN_FIFOCTL_LIMIT, 0x30 * wordsize)
  637. | FIELD_PREP(CHAN_FIFOCTL_THRESHOLD, 0x18 * wordsize),
  638. ad->base + REG_CHAN_FIFOCTL(adchan->no));
  639. return 0;
  640. }
  641. static int admac_probe(struct platform_device *pdev)
  642. {
  643. struct device_node *np = pdev->dev.of_node;
  644. struct admac_data *ad;
  645. struct dma_device *dma;
  646. int nchannels;
  647. int err, irq, i;
  648. err = of_property_read_u32(np, "dma-channels", &nchannels);
  649. if (err || nchannels > NCHANNELS_MAX) {
  650. dev_err(&pdev->dev, "missing or invalid dma-channels property\n");
  651. return -EINVAL;
  652. }
  653. ad = devm_kzalloc(&pdev->dev, struct_size(ad, channels, nchannels), GFP_KERNEL);
  654. if (!ad)
  655. return -ENOMEM;
  656. platform_set_drvdata(pdev, ad);
  657. ad->dev = &pdev->dev;
  658. ad->nchannels = nchannels;
  659. mutex_init(&ad->cache_alloc_lock);
  660. /*
  661. * The controller has 4 IRQ outputs. Try them all until
  662. * we find one we can use.
  663. */
  664. for (i = 0; i < IRQ_NOUTPUTS; i++) {
  665. irq = platform_get_irq_optional(pdev, i);
  666. if (irq >= 0) {
  667. ad->irq_index = i;
  668. break;
  669. }
  670. }
  671. if (irq < 0)
  672. return dev_err_probe(&pdev->dev, irq, "no usable interrupt\n");
  673. ad->irq = irq;
  674. ad->base = devm_platform_ioremap_resource(pdev, 0);
  675. if (IS_ERR(ad->base))
  676. return dev_err_probe(&pdev->dev, PTR_ERR(ad->base),
  677. "unable to obtain MMIO resource\n");
  678. ad->rstc = devm_reset_control_get_optional_shared(&pdev->dev, NULL);
  679. if (IS_ERR(ad->rstc))
  680. return PTR_ERR(ad->rstc);
  681. dma = &ad->dma;
  682. dma_cap_set(DMA_PRIVATE, dma->cap_mask);
  683. dma_cap_set(DMA_CYCLIC, dma->cap_mask);
  684. dma->dev = &pdev->dev;
  685. dma->device_alloc_chan_resources = admac_alloc_chan_resources;
  686. dma->device_free_chan_resources = admac_free_chan_resources;
  687. dma->device_tx_status = admac_tx_status;
  688. dma->device_issue_pending = admac_issue_pending;
  689. dma->device_terminate_all = admac_terminate_all;
  690. dma->device_synchronize = admac_synchronize;
  691. dma->device_prep_dma_cyclic = admac_prep_dma_cyclic;
  692. dma->device_config = admac_device_config;
  693. dma->device_pause = admac_pause;
  694. dma->device_resume = admac_resume;
  695. dma->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);
  696. dma->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
  697. dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
  698. BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
  699. BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
  700. dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
  701. BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
  702. BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
  703. INIT_LIST_HEAD(&dma->channels);
  704. for (i = 0; i < nchannels; i++) {
  705. struct admac_chan *adchan = &ad->channels[i];
  706. adchan->host = ad;
  707. adchan->no = i;
  708. adchan->chan.device = &ad->dma;
  709. spin_lock_init(&adchan->lock);
  710. INIT_LIST_HEAD(&adchan->submitted);
  711. INIT_LIST_HEAD(&adchan->issued);
  712. INIT_LIST_HEAD(&adchan->to_free);
  713. list_add_tail(&adchan->chan.device_node, &dma->channels);
  714. tasklet_setup(&adchan->tasklet, admac_chan_tasklet);
  715. }
  716. err = reset_control_reset(ad->rstc);
  717. if (err)
  718. return dev_err_probe(&pdev->dev, err,
  719. "unable to trigger reset\n");
  720. err = request_irq(irq, admac_interrupt, 0, dev_name(&pdev->dev), ad);
  721. if (err) {
  722. dev_err_probe(&pdev->dev, err,
  723. "unable to register interrupt\n");
  724. goto free_reset;
  725. }
  726. err = dma_async_device_register(&ad->dma);
  727. if (err) {
  728. dev_err_probe(&pdev->dev, err, "failed to register DMA device\n");
  729. goto free_irq;
  730. }
  731. err = of_dma_controller_register(pdev->dev.of_node, admac_dma_of_xlate, ad);
  732. if (err) {
  733. dma_async_device_unregister(&ad->dma);
  734. dev_err_probe(&pdev->dev, err, "failed to register with OF\n");
  735. goto free_irq;
  736. }
  737. ad->txcache.size = readl_relaxed(ad->base + REG_TX_SRAM_SIZE);
  738. ad->rxcache.size = readl_relaxed(ad->base + REG_RX_SRAM_SIZE);
  739. dev_info(&pdev->dev, "Audio DMA Controller\n");
  740. dev_info(&pdev->dev, "imprint %x TX cache %u RX cache %u\n",
  741. readl_relaxed(ad->base + REG_IMPRINT), ad->txcache.size, ad->rxcache.size);
  742. return 0;
  743. free_irq:
  744. free_irq(ad->irq, ad);
  745. free_reset:
  746. reset_control_rearm(ad->rstc);
  747. return err;
  748. }
  749. static int admac_remove(struct platform_device *pdev)
  750. {
  751. struct admac_data *ad = platform_get_drvdata(pdev);
  752. of_dma_controller_free(pdev->dev.of_node);
  753. dma_async_device_unregister(&ad->dma);
  754. free_irq(ad->irq, ad);
  755. reset_control_rearm(ad->rstc);
  756. return 0;
  757. }
  758. static const struct of_device_id admac_of_match[] = {
  759. { .compatible = "apple,admac", },
  760. { }
  761. };
  762. MODULE_DEVICE_TABLE(of, admac_of_match);
  763. static struct platform_driver apple_admac_driver = {
  764. .driver = {
  765. .name = "apple-admac",
  766. .of_match_table = admac_of_match,
  767. },
  768. .probe = admac_probe,
  769. .remove = admac_remove,
  770. };
  771. module_platform_driver(apple_admac_driver);
  772. MODULE_AUTHOR("Martin Povišer <[email protected]>");
  773. MODULE_DESCRIPTION("Driver for Audio DMA Controller (ADMAC) on Apple SoCs");
  774. MODULE_LICENSE("GPL");