1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102 |
- // SPDX-License-Identifier: GPL-2.0-only
- /*
- * SA11x0 DMAengine support
- *
- * Copyright (C) 2012 Russell King
- * Derived in part from arch/arm/mach-sa1100/dma.c,
- * Copyright (C) 2000, 2001 by Nicolas Pitre
- */
- #include <linux/sched.h>
- #include <linux/device.h>
- #include <linux/dmaengine.h>
- #include <linux/init.h>
- #include <linux/interrupt.h>
- #include <linux/kernel.h>
- #include <linux/module.h>
- #include <linux/platform_device.h>
- #include <linux/slab.h>
- #include <linux/spinlock.h>
- #include "virt-dma.h"
- #define NR_PHY_CHAN 6
- #define DMA_ALIGN 3
- #define DMA_MAX_SIZE 0x1fff
- #define DMA_CHUNK_SIZE 0x1000
- #define DMA_DDAR 0x00
- #define DMA_DCSR_S 0x04
- #define DMA_DCSR_C 0x08
- #define DMA_DCSR_R 0x0c
- #define DMA_DBSA 0x10
- #define DMA_DBTA 0x14
- #define DMA_DBSB 0x18
- #define DMA_DBTB 0x1c
- #define DMA_SIZE 0x20
- #define DCSR_RUN (1 << 0)
- #define DCSR_IE (1 << 1)
- #define DCSR_ERROR (1 << 2)
- #define DCSR_DONEA (1 << 3)
- #define DCSR_STRTA (1 << 4)
- #define DCSR_DONEB (1 << 5)
- #define DCSR_STRTB (1 << 6)
- #define DCSR_BIU (1 << 7)
- #define DDAR_RW (1 << 0) /* 0 = W, 1 = R */
- #define DDAR_E (1 << 1) /* 0 = LE, 1 = BE */
- #define DDAR_BS (1 << 2) /* 0 = BS4, 1 = BS8 */
- #define DDAR_DW (1 << 3) /* 0 = 8b, 1 = 16b */
- #define DDAR_Ser0UDCTr (0x0 << 4)
- #define DDAR_Ser0UDCRc (0x1 << 4)
- #define DDAR_Ser1SDLCTr (0x2 << 4)
- #define DDAR_Ser1SDLCRc (0x3 << 4)
- #define DDAR_Ser1UARTTr (0x4 << 4)
- #define DDAR_Ser1UARTRc (0x5 << 4)
- #define DDAR_Ser2ICPTr (0x6 << 4)
- #define DDAR_Ser2ICPRc (0x7 << 4)
- #define DDAR_Ser3UARTTr (0x8 << 4)
- #define DDAR_Ser3UARTRc (0x9 << 4)
- #define DDAR_Ser4MCP0Tr (0xa << 4)
- #define DDAR_Ser4MCP0Rc (0xb << 4)
- #define DDAR_Ser4MCP1Tr (0xc << 4)
- #define DDAR_Ser4MCP1Rc (0xd << 4)
- #define DDAR_Ser4SSPTr (0xe << 4)
- #define DDAR_Ser4SSPRc (0xf << 4)
- struct sa11x0_dma_sg {
- u32 addr;
- u32 len;
- };
- struct sa11x0_dma_desc {
- struct virt_dma_desc vd;
- u32 ddar;
- size_t size;
- unsigned period;
- bool cyclic;
- unsigned sglen;
- struct sa11x0_dma_sg sg[];
- };
- struct sa11x0_dma_phy;
- struct sa11x0_dma_chan {
- struct virt_dma_chan vc;
- /* protected by c->vc.lock */
- struct sa11x0_dma_phy *phy;
- enum dma_status status;
- /* protected by d->lock */
- struct list_head node;
- u32 ddar;
- const char *name;
- };
- struct sa11x0_dma_phy {
- void __iomem *base;
- struct sa11x0_dma_dev *dev;
- unsigned num;
- struct sa11x0_dma_chan *vchan;
- /* Protected by c->vc.lock */
- unsigned sg_load;
- struct sa11x0_dma_desc *txd_load;
- unsigned sg_done;
- struct sa11x0_dma_desc *txd_done;
- u32 dbs[2];
- u32 dbt[2];
- u32 dcsr;
- };
- struct sa11x0_dma_dev {
- struct dma_device slave;
- void __iomem *base;
- spinlock_t lock;
- struct tasklet_struct task;
- struct list_head chan_pending;
- struct sa11x0_dma_phy phy[NR_PHY_CHAN];
- };
- static struct sa11x0_dma_chan *to_sa11x0_dma_chan(struct dma_chan *chan)
- {
- return container_of(chan, struct sa11x0_dma_chan, vc.chan);
- }
- static struct sa11x0_dma_dev *to_sa11x0_dma(struct dma_device *dmadev)
- {
- return container_of(dmadev, struct sa11x0_dma_dev, slave);
- }
- static struct sa11x0_dma_desc *sa11x0_dma_next_desc(struct sa11x0_dma_chan *c)
- {
- struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
- return vd ? container_of(vd, struct sa11x0_dma_desc, vd) : NULL;
- }
- static void sa11x0_dma_free_desc(struct virt_dma_desc *vd)
- {
- kfree(container_of(vd, struct sa11x0_dma_desc, vd));
- }
- static void sa11x0_dma_start_desc(struct sa11x0_dma_phy *p, struct sa11x0_dma_desc *txd)
- {
- list_del(&txd->vd.node);
- p->txd_load = txd;
- p->sg_load = 0;
- dev_vdbg(p->dev->slave.dev, "pchan %u: txd %p[%x]: starting: DDAR:%x\n",
- p->num, &txd->vd, txd->vd.tx.cookie, txd->ddar);
- }
- static void noinline sa11x0_dma_start_sg(struct sa11x0_dma_phy *p,
- struct sa11x0_dma_chan *c)
- {
- struct sa11x0_dma_desc *txd = p->txd_load;
- struct sa11x0_dma_sg *sg;
- void __iomem *base = p->base;
- unsigned dbsx, dbtx;
- u32 dcsr;
- if (!txd)
- return;
- dcsr = readl_relaxed(base + DMA_DCSR_R);
- /* Don't try to load the next transfer if both buffers are started */
- if ((dcsr & (DCSR_STRTA | DCSR_STRTB)) == (DCSR_STRTA | DCSR_STRTB))
- return;
- if (p->sg_load == txd->sglen) {
- if (!txd->cyclic) {
- struct sa11x0_dma_desc *txn = sa11x0_dma_next_desc(c);
- /*
- * We have reached the end of the current descriptor.
- * Peek at the next descriptor, and if compatible with
- * the current, start processing it.
- */
- if (txn && txn->ddar == txd->ddar) {
- txd = txn;
- sa11x0_dma_start_desc(p, txn);
- } else {
- p->txd_load = NULL;
- return;
- }
- } else {
- /* Cyclic: reset back to beginning */
- p->sg_load = 0;
- }
- }
- sg = &txd->sg[p->sg_load++];
- /* Select buffer to load according to channel status */
- if (((dcsr & (DCSR_BIU | DCSR_STRTB)) == (DCSR_BIU | DCSR_STRTB)) ||
- ((dcsr & (DCSR_BIU | DCSR_STRTA)) == 0)) {
- dbsx = DMA_DBSA;
- dbtx = DMA_DBTA;
- dcsr = DCSR_STRTA | DCSR_IE | DCSR_RUN;
- } else {
- dbsx = DMA_DBSB;
- dbtx = DMA_DBTB;
- dcsr = DCSR_STRTB | DCSR_IE | DCSR_RUN;
- }
- writel_relaxed(sg->addr, base + dbsx);
- writel_relaxed(sg->len, base + dbtx);
- writel(dcsr, base + DMA_DCSR_S);
- dev_dbg(p->dev->slave.dev, "pchan %u: load: DCSR:%02x DBS%c:%08x DBT%c:%08x\n",
- p->num, dcsr,
- 'A' + (dbsx == DMA_DBSB), sg->addr,
- 'A' + (dbtx == DMA_DBTB), sg->len);
- }
- static void noinline sa11x0_dma_complete(struct sa11x0_dma_phy *p,
- struct sa11x0_dma_chan *c)
- {
- struct sa11x0_dma_desc *txd = p->txd_done;
- if (++p->sg_done == txd->sglen) {
- if (!txd->cyclic) {
- vchan_cookie_complete(&txd->vd);
- p->sg_done = 0;
- p->txd_done = p->txd_load;
- if (!p->txd_done)
- tasklet_schedule(&p->dev->task);
- } else {
- if ((p->sg_done % txd->period) == 0)
- vchan_cyclic_callback(&txd->vd);
- /* Cyclic: reset back to beginning */
- p->sg_done = 0;
- }
- }
- sa11x0_dma_start_sg(p, c);
- }
- static irqreturn_t sa11x0_dma_irq(int irq, void *dev_id)
- {
- struct sa11x0_dma_phy *p = dev_id;
- struct sa11x0_dma_dev *d = p->dev;
- struct sa11x0_dma_chan *c;
- u32 dcsr;
- dcsr = readl_relaxed(p->base + DMA_DCSR_R);
- if (!(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB)))
- return IRQ_NONE;
- /* Clear reported status bits */
- writel_relaxed(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB),
- p->base + DMA_DCSR_C);
- dev_dbg(d->slave.dev, "pchan %u: irq: DCSR:%02x\n", p->num, dcsr);
- if (dcsr & DCSR_ERROR) {
- dev_err(d->slave.dev, "pchan %u: error. DCSR:%02x DDAR:%08x DBSA:%08x DBTA:%08x DBSB:%08x DBTB:%08x\n",
- p->num, dcsr,
- readl_relaxed(p->base + DMA_DDAR),
- readl_relaxed(p->base + DMA_DBSA),
- readl_relaxed(p->base + DMA_DBTA),
- readl_relaxed(p->base + DMA_DBSB),
- readl_relaxed(p->base + DMA_DBTB));
- }
- c = p->vchan;
- if (c) {
- unsigned long flags;
- spin_lock_irqsave(&c->vc.lock, flags);
- /*
- * Now that we're holding the lock, check that the vchan
- * really is associated with this pchan before touching the
- * hardware. This should always succeed, because we won't
- * change p->vchan or c->phy while the channel is actively
- * transferring.
- */
- if (c->phy == p) {
- if (dcsr & DCSR_DONEA)
- sa11x0_dma_complete(p, c);
- if (dcsr & DCSR_DONEB)
- sa11x0_dma_complete(p, c);
- }
- spin_unlock_irqrestore(&c->vc.lock, flags);
- }
- return IRQ_HANDLED;
- }
- static void sa11x0_dma_start_txd(struct sa11x0_dma_chan *c)
- {
- struct sa11x0_dma_desc *txd = sa11x0_dma_next_desc(c);
- /* If the issued list is empty, we have no further txds to process */
- if (txd) {
- struct sa11x0_dma_phy *p = c->phy;
- sa11x0_dma_start_desc(p, txd);
- p->txd_done = txd;
- p->sg_done = 0;
- /* The channel should not have any transfers started */
- WARN_ON(readl_relaxed(p->base + DMA_DCSR_R) &
- (DCSR_STRTA | DCSR_STRTB));
- /* Clear the run and start bits before changing DDAR */
- writel_relaxed(DCSR_RUN | DCSR_STRTA | DCSR_STRTB,
- p->base + DMA_DCSR_C);
- writel_relaxed(txd->ddar, p->base + DMA_DDAR);
- /* Try to start both buffers */
- sa11x0_dma_start_sg(p, c);
- sa11x0_dma_start_sg(p, c);
- }
- }
- static void sa11x0_dma_tasklet(struct tasklet_struct *t)
- {
- struct sa11x0_dma_dev *d = from_tasklet(d, t, task);
- struct sa11x0_dma_phy *p;
- struct sa11x0_dma_chan *c;
- unsigned pch, pch_alloc = 0;
- dev_dbg(d->slave.dev, "tasklet enter\n");
- list_for_each_entry(c, &d->slave.channels, vc.chan.device_node) {
- spin_lock_irq(&c->vc.lock);
- p = c->phy;
- if (p && !p->txd_done) {
- sa11x0_dma_start_txd(c);
- if (!p->txd_done) {
- /* No current txd associated with this channel */
- dev_dbg(d->slave.dev, "pchan %u: free\n", p->num);
- /* Mark this channel free */
- c->phy = NULL;
- p->vchan = NULL;
- }
- }
- spin_unlock_irq(&c->vc.lock);
- }
- spin_lock_irq(&d->lock);
- for (pch = 0; pch < NR_PHY_CHAN; pch++) {
- p = &d->phy[pch];
- if (p->vchan == NULL && !list_empty(&d->chan_pending)) {
- c = list_first_entry(&d->chan_pending,
- struct sa11x0_dma_chan, node);
- list_del_init(&c->node);
- pch_alloc |= 1 << pch;
- /* Mark this channel allocated */
- p->vchan = c;
- dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, &c->vc);
- }
- }
- spin_unlock_irq(&d->lock);
- for (pch = 0; pch < NR_PHY_CHAN; pch++) {
- if (pch_alloc & (1 << pch)) {
- p = &d->phy[pch];
- c = p->vchan;
- spin_lock_irq(&c->vc.lock);
- c->phy = p;
- sa11x0_dma_start_txd(c);
- spin_unlock_irq(&c->vc.lock);
- }
- }
- dev_dbg(d->slave.dev, "tasklet exit\n");
- }
- static void sa11x0_dma_free_chan_resources(struct dma_chan *chan)
- {
- struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
- struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
- unsigned long flags;
- spin_lock_irqsave(&d->lock, flags);
- list_del_init(&c->node);
- spin_unlock_irqrestore(&d->lock, flags);
- vchan_free_chan_resources(&c->vc);
- }
- static dma_addr_t sa11x0_dma_pos(struct sa11x0_dma_phy *p)
- {
- unsigned reg;
- u32 dcsr;
- dcsr = readl_relaxed(p->base + DMA_DCSR_R);
- if ((dcsr & (DCSR_BIU | DCSR_STRTA)) == DCSR_STRTA ||
- (dcsr & (DCSR_BIU | DCSR_STRTB)) == DCSR_BIU)
- reg = DMA_DBSA;
- else
- reg = DMA_DBSB;
- return readl_relaxed(p->base + reg);
- }
- static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan,
- dma_cookie_t cookie, struct dma_tx_state *state)
- {
- struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
- struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
- struct sa11x0_dma_phy *p;
- struct virt_dma_desc *vd;
- unsigned long flags;
- enum dma_status ret;
- ret = dma_cookie_status(&c->vc.chan, cookie, state);
- if (ret == DMA_COMPLETE)
- return ret;
- if (!state)
- return c->status;
- spin_lock_irqsave(&c->vc.lock, flags);
- p = c->phy;
- /*
- * If the cookie is on our issue queue, then the residue is
- * its total size.
- */
- vd = vchan_find_desc(&c->vc, cookie);
- if (vd) {
- state->residue = container_of(vd, struct sa11x0_dma_desc, vd)->size;
- } else if (!p) {
- state->residue = 0;
- } else {
- struct sa11x0_dma_desc *txd;
- size_t bytes = 0;
- if (p->txd_done && p->txd_done->vd.tx.cookie == cookie)
- txd = p->txd_done;
- else if (p->txd_load && p->txd_load->vd.tx.cookie == cookie)
- txd = p->txd_load;
- else
- txd = NULL;
- ret = c->status;
- if (txd) {
- dma_addr_t addr = sa11x0_dma_pos(p);
- unsigned i;
- dev_vdbg(d->slave.dev, "tx_status: addr:%pad\n", &addr);
- for (i = 0; i < txd->sglen; i++) {
- dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x\n",
- i, txd->sg[i].addr, txd->sg[i].len);
- if (addr >= txd->sg[i].addr &&
- addr < txd->sg[i].addr + txd->sg[i].len) {
- unsigned len;
- len = txd->sg[i].len -
- (addr - txd->sg[i].addr);
- dev_vdbg(d->slave.dev, "tx_status: [%u] +%x\n",
- i, len);
- bytes += len;
- i++;
- break;
- }
- }
- for (; i < txd->sglen; i++) {
- dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x ++\n",
- i, txd->sg[i].addr, txd->sg[i].len);
- bytes += txd->sg[i].len;
- }
- }
- state->residue = bytes;
- }
- spin_unlock_irqrestore(&c->vc.lock, flags);
- dev_vdbg(d->slave.dev, "tx_status: bytes 0x%x\n", state->residue);
- return ret;
- }
- /*
- * Move pending txds to the issued list, and re-init pending list.
- * If not already pending, add this channel to the list of pending
- * channels and trigger the tasklet to run.
- */
- static void sa11x0_dma_issue_pending(struct dma_chan *chan)
- {
- struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
- struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
- unsigned long flags;
- spin_lock_irqsave(&c->vc.lock, flags);
- if (vchan_issue_pending(&c->vc)) {
- if (!c->phy) {
- spin_lock(&d->lock);
- if (list_empty(&c->node)) {
- list_add_tail(&c->node, &d->chan_pending);
- tasklet_schedule(&d->task);
- dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc);
- }
- spin_unlock(&d->lock);
- }
- } else
- dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc);
- spin_unlock_irqrestore(&c->vc.lock, flags);
- }
- static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg(
- struct dma_chan *chan, struct scatterlist *sg, unsigned int sglen,
- enum dma_transfer_direction dir, unsigned long flags, void *context)
- {
- struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
- struct sa11x0_dma_desc *txd;
- struct scatterlist *sgent;
- unsigned i, j = sglen;
- size_t size = 0;
- /* SA11x0 channels can only operate in their native direction */
- if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
- dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
- &c->vc, c->ddar, dir);
- return NULL;
- }
- /* Do not allow zero-sized txds */
- if (sglen == 0)
- return NULL;
- for_each_sg(sg, sgent, sglen, i) {
- dma_addr_t addr = sg_dma_address(sgent);
- unsigned int len = sg_dma_len(sgent);
- if (len > DMA_MAX_SIZE)
- j += DIV_ROUND_UP(len, DMA_MAX_SIZE & ~DMA_ALIGN) - 1;
- if (addr & DMA_ALIGN) {
- dev_dbg(chan->device->dev, "vchan %p: bad buffer alignment: %pad\n",
- &c->vc, &addr);
- return NULL;
- }
- }
- txd = kzalloc(struct_size(txd, sg, j), GFP_ATOMIC);
- if (!txd) {
- dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
- return NULL;
- }
- j = 0;
- for_each_sg(sg, sgent, sglen, i) {
- dma_addr_t addr = sg_dma_address(sgent);
- unsigned len = sg_dma_len(sgent);
- size += len;
- do {
- unsigned tlen = len;
- /*
- * Check whether the transfer will fit. If not, try
- * to split the transfer up such that we end up with
- * equal chunks - but make sure that we preserve the
- * alignment. This avoids small segments.
- */
- if (tlen > DMA_MAX_SIZE) {
- unsigned mult = DIV_ROUND_UP(tlen,
- DMA_MAX_SIZE & ~DMA_ALIGN);
- tlen = (tlen / mult) & ~DMA_ALIGN;
- }
- txd->sg[j].addr = addr;
- txd->sg[j].len = tlen;
- addr += tlen;
- len -= tlen;
- j++;
- } while (len);
- }
- txd->ddar = c->ddar;
- txd->size = size;
- txd->sglen = j;
- dev_dbg(chan->device->dev, "vchan %p: txd %p: size %zu nr %u\n",
- &c->vc, &txd->vd, txd->size, txd->sglen);
- return vchan_tx_prep(&c->vc, &txd->vd, flags);
- }
- static struct dma_async_tx_descriptor *sa11x0_dma_prep_dma_cyclic(
- struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
- enum dma_transfer_direction dir, unsigned long flags)
- {
- struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
- struct sa11x0_dma_desc *txd;
- unsigned i, j, k, sglen, sgperiod;
- /* SA11x0 channels can only operate in their native direction */
- if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
- dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
- &c->vc, c->ddar, dir);
- return NULL;
- }
- sgperiod = DIV_ROUND_UP(period, DMA_MAX_SIZE & ~DMA_ALIGN);
- sglen = size * sgperiod / period;
- /* Do not allow zero-sized txds */
- if (sglen == 0)
- return NULL;
- txd = kzalloc(struct_size(txd, sg, sglen), GFP_ATOMIC);
- if (!txd) {
- dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
- return NULL;
- }
- for (i = k = 0; i < size / period; i++) {
- size_t tlen, len = period;
- for (j = 0; j < sgperiod; j++, k++) {
- tlen = len;
- if (tlen > DMA_MAX_SIZE) {
- unsigned mult = DIV_ROUND_UP(tlen, DMA_MAX_SIZE & ~DMA_ALIGN);
- tlen = (tlen / mult) & ~DMA_ALIGN;
- }
- txd->sg[k].addr = addr;
- txd->sg[k].len = tlen;
- addr += tlen;
- len -= tlen;
- }
- WARN_ON(len != 0);
- }
- WARN_ON(k != sglen);
- txd->ddar = c->ddar;
- txd->size = size;
- txd->sglen = sglen;
- txd->cyclic = 1;
- txd->period = sgperiod;
- return vchan_tx_prep(&c->vc, &txd->vd, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
- }
- static int sa11x0_dma_device_config(struct dma_chan *chan,
- struct dma_slave_config *cfg)
- {
- struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
- u32 ddar = c->ddar & ((0xf << 4) | DDAR_RW);
- dma_addr_t addr;
- enum dma_slave_buswidth width;
- u32 maxburst;
- if (ddar & DDAR_RW) {
- addr = cfg->src_addr;
- width = cfg->src_addr_width;
- maxburst = cfg->src_maxburst;
- } else {
- addr = cfg->dst_addr;
- width = cfg->dst_addr_width;
- maxburst = cfg->dst_maxburst;
- }
- if ((width != DMA_SLAVE_BUSWIDTH_1_BYTE &&
- width != DMA_SLAVE_BUSWIDTH_2_BYTES) ||
- (maxburst != 4 && maxburst != 8))
- return -EINVAL;
- if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
- ddar |= DDAR_DW;
- if (maxburst == 8)
- ddar |= DDAR_BS;
- dev_dbg(c->vc.chan.device->dev, "vchan %p: dma_slave_config addr %pad width %u burst %u\n",
- &c->vc, &addr, width, maxburst);
- c->ddar = ddar | (addr & 0xf0000000) | (addr & 0x003ffffc) << 6;
- return 0;
- }
- static int sa11x0_dma_device_pause(struct dma_chan *chan)
- {
- struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
- struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
- struct sa11x0_dma_phy *p;
- unsigned long flags;
- dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc);
- spin_lock_irqsave(&c->vc.lock, flags);
- if (c->status == DMA_IN_PROGRESS) {
- c->status = DMA_PAUSED;
- p = c->phy;
- if (p) {
- writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
- } else {
- spin_lock(&d->lock);
- list_del_init(&c->node);
- spin_unlock(&d->lock);
- }
- }
- spin_unlock_irqrestore(&c->vc.lock, flags);
- return 0;
- }
- static int sa11x0_dma_device_resume(struct dma_chan *chan)
- {
- struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
- struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
- struct sa11x0_dma_phy *p;
- unsigned long flags;
- dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc);
- spin_lock_irqsave(&c->vc.lock, flags);
- if (c->status == DMA_PAUSED) {
- c->status = DMA_IN_PROGRESS;
- p = c->phy;
- if (p) {
- writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_S);
- } else if (!list_empty(&c->vc.desc_issued)) {
- spin_lock(&d->lock);
- list_add_tail(&c->node, &d->chan_pending);
- spin_unlock(&d->lock);
- }
- }
- spin_unlock_irqrestore(&c->vc.lock, flags);
- return 0;
- }
- static int sa11x0_dma_device_terminate_all(struct dma_chan *chan)
- {
- struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
- struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
- struct sa11x0_dma_phy *p;
- LIST_HEAD(head);
- unsigned long flags;
- dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc);
- /* Clear the tx descriptor lists */
- spin_lock_irqsave(&c->vc.lock, flags);
- vchan_get_all_descriptors(&c->vc, &head);
- p = c->phy;
- if (p) {
- dev_dbg(d->slave.dev, "pchan %u: terminating\n", p->num);
- /* vchan is assigned to a pchan - stop the channel */
- writel(DCSR_RUN | DCSR_IE |
- DCSR_STRTA | DCSR_DONEA |
- DCSR_STRTB | DCSR_DONEB,
- p->base + DMA_DCSR_C);
- if (p->txd_load) {
- if (p->txd_load != p->txd_done)
- list_add_tail(&p->txd_load->vd.node, &head);
- p->txd_load = NULL;
- }
- if (p->txd_done) {
- list_add_tail(&p->txd_done->vd.node, &head);
- p->txd_done = NULL;
- }
- c->phy = NULL;
- spin_lock(&d->lock);
- p->vchan = NULL;
- spin_unlock(&d->lock);
- tasklet_schedule(&d->task);
- }
- spin_unlock_irqrestore(&c->vc.lock, flags);
- vchan_dma_desc_free_list(&c->vc, &head);
- return 0;
- }
- struct sa11x0_dma_channel_desc {
- u32 ddar;
- const char *name;
- };
- #define CD(d1, d2) { .ddar = DDAR_##d1 | d2, .name = #d1 }
- static const struct sa11x0_dma_channel_desc chan_desc[] = {
- CD(Ser0UDCTr, 0),
- CD(Ser0UDCRc, DDAR_RW),
- CD(Ser1SDLCTr, 0),
- CD(Ser1SDLCRc, DDAR_RW),
- CD(Ser1UARTTr, 0),
- CD(Ser1UARTRc, DDAR_RW),
- CD(Ser2ICPTr, 0),
- CD(Ser2ICPRc, DDAR_RW),
- CD(Ser3UARTTr, 0),
- CD(Ser3UARTRc, DDAR_RW),
- CD(Ser4MCP0Tr, 0),
- CD(Ser4MCP0Rc, DDAR_RW),
- CD(Ser4MCP1Tr, 0),
- CD(Ser4MCP1Rc, DDAR_RW),
- CD(Ser4SSPTr, 0),
- CD(Ser4SSPRc, DDAR_RW),
- };
- static const struct dma_slave_map sa11x0_dma_map[] = {
- { "sa11x0-ir", "tx", "Ser2ICPTr" },
- { "sa11x0-ir", "rx", "Ser2ICPRc" },
- { "sa11x0-ssp", "tx", "Ser4SSPTr" },
- { "sa11x0-ssp", "rx", "Ser4SSPRc" },
- };
- static bool sa11x0_dma_filter_fn(struct dma_chan *chan, void *param)
- {
- struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
- const char *p = param;
- return !strcmp(c->name, p);
- }
- static int sa11x0_dma_init_dmadev(struct dma_device *dmadev,
- struct device *dev)
- {
- unsigned i;
- INIT_LIST_HEAD(&dmadev->channels);
- dmadev->dev = dev;
- dmadev->device_free_chan_resources = sa11x0_dma_free_chan_resources;
- dmadev->device_config = sa11x0_dma_device_config;
- dmadev->device_pause = sa11x0_dma_device_pause;
- dmadev->device_resume = sa11x0_dma_device_resume;
- dmadev->device_terminate_all = sa11x0_dma_device_terminate_all;
- dmadev->device_tx_status = sa11x0_dma_tx_status;
- dmadev->device_issue_pending = sa11x0_dma_issue_pending;
- for (i = 0; i < ARRAY_SIZE(chan_desc); i++) {
- struct sa11x0_dma_chan *c;
- c = kzalloc(sizeof(*c), GFP_KERNEL);
- if (!c) {
- dev_err(dev, "no memory for channel %u\n", i);
- return -ENOMEM;
- }
- c->status = DMA_IN_PROGRESS;
- c->ddar = chan_desc[i].ddar;
- c->name = chan_desc[i].name;
- INIT_LIST_HEAD(&c->node);
- c->vc.desc_free = sa11x0_dma_free_desc;
- vchan_init(&c->vc, dmadev);
- }
- return dma_async_device_register(dmadev);
- }
- static int sa11x0_dma_request_irq(struct platform_device *pdev, int nr,
- void *data)
- {
- int irq = platform_get_irq(pdev, nr);
- if (irq <= 0)
- return -ENXIO;
- return request_irq(irq, sa11x0_dma_irq, 0, dev_name(&pdev->dev), data);
- }
- static void sa11x0_dma_free_irq(struct platform_device *pdev, int nr,
- void *data)
- {
- int irq = platform_get_irq(pdev, nr);
- if (irq > 0)
- free_irq(irq, data);
- }
- static void sa11x0_dma_free_channels(struct dma_device *dmadev)
- {
- struct sa11x0_dma_chan *c, *cn;
- list_for_each_entry_safe(c, cn, &dmadev->channels, vc.chan.device_node) {
- list_del(&c->vc.chan.device_node);
- tasklet_kill(&c->vc.task);
- kfree(c);
- }
- }
- static int sa11x0_dma_probe(struct platform_device *pdev)
- {
- struct sa11x0_dma_dev *d;
- struct resource *res;
- unsigned i;
- int ret;
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- if (!res)
- return -ENXIO;
- d = kzalloc(sizeof(*d), GFP_KERNEL);
- if (!d) {
- ret = -ENOMEM;
- goto err_alloc;
- }
- spin_lock_init(&d->lock);
- INIT_LIST_HEAD(&d->chan_pending);
- d->slave.filter.fn = sa11x0_dma_filter_fn;
- d->slave.filter.mapcnt = ARRAY_SIZE(sa11x0_dma_map);
- d->slave.filter.map = sa11x0_dma_map;
- d->base = ioremap(res->start, resource_size(res));
- if (!d->base) {
- ret = -ENOMEM;
- goto err_ioremap;
- }
- tasklet_setup(&d->task, sa11x0_dma_tasklet);
- for (i = 0; i < NR_PHY_CHAN; i++) {
- struct sa11x0_dma_phy *p = &d->phy[i];
- p->dev = d;
- p->num = i;
- p->base = d->base + i * DMA_SIZE;
- writel_relaxed(DCSR_RUN | DCSR_IE | DCSR_ERROR |
- DCSR_DONEA | DCSR_STRTA | DCSR_DONEB | DCSR_STRTB,
- p->base + DMA_DCSR_C);
- writel_relaxed(0, p->base + DMA_DDAR);
- ret = sa11x0_dma_request_irq(pdev, i, p);
- if (ret) {
- while (i) {
- i--;
- sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
- }
- goto err_irq;
- }
- }
- dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
- dma_cap_set(DMA_CYCLIC, d->slave.cap_mask);
- d->slave.device_prep_slave_sg = sa11x0_dma_prep_slave_sg;
- d->slave.device_prep_dma_cyclic = sa11x0_dma_prep_dma_cyclic;
- d->slave.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
- d->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
- d->slave.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
- BIT(DMA_SLAVE_BUSWIDTH_2_BYTES);
- d->slave.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
- BIT(DMA_SLAVE_BUSWIDTH_2_BYTES);
- ret = sa11x0_dma_init_dmadev(&d->slave, &pdev->dev);
- if (ret) {
- dev_warn(d->slave.dev, "failed to register slave async device: %d\n",
- ret);
- goto err_slave_reg;
- }
- platform_set_drvdata(pdev, d);
- return 0;
- err_slave_reg:
- sa11x0_dma_free_channels(&d->slave);
- for (i = 0; i < NR_PHY_CHAN; i++)
- sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
- err_irq:
- tasklet_kill(&d->task);
- iounmap(d->base);
- err_ioremap:
- kfree(d);
- err_alloc:
- return ret;
- }
- static int sa11x0_dma_remove(struct platform_device *pdev)
- {
- struct sa11x0_dma_dev *d = platform_get_drvdata(pdev);
- unsigned pch;
- dma_async_device_unregister(&d->slave);
- sa11x0_dma_free_channels(&d->slave);
- for (pch = 0; pch < NR_PHY_CHAN; pch++)
- sa11x0_dma_free_irq(pdev, pch, &d->phy[pch]);
- tasklet_kill(&d->task);
- iounmap(d->base);
- kfree(d);
- return 0;
- }
- static __maybe_unused int sa11x0_dma_suspend(struct device *dev)
- {
- struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
- unsigned pch;
- for (pch = 0; pch < NR_PHY_CHAN; pch++) {
- struct sa11x0_dma_phy *p = &d->phy[pch];
- u32 dcsr, saved_dcsr;
- dcsr = saved_dcsr = readl_relaxed(p->base + DMA_DCSR_R);
- if (dcsr & DCSR_RUN) {
- writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
- dcsr = readl_relaxed(p->base + DMA_DCSR_R);
- }
- saved_dcsr &= DCSR_RUN | DCSR_IE;
- if (dcsr & DCSR_BIU) {
- p->dbs[0] = readl_relaxed(p->base + DMA_DBSB);
- p->dbt[0] = readl_relaxed(p->base + DMA_DBTB);
- p->dbs[1] = readl_relaxed(p->base + DMA_DBSA);
- p->dbt[1] = readl_relaxed(p->base + DMA_DBTA);
- saved_dcsr |= (dcsr & DCSR_STRTA ? DCSR_STRTB : 0) |
- (dcsr & DCSR_STRTB ? DCSR_STRTA : 0);
- } else {
- p->dbs[0] = readl_relaxed(p->base + DMA_DBSA);
- p->dbt[0] = readl_relaxed(p->base + DMA_DBTA);
- p->dbs[1] = readl_relaxed(p->base + DMA_DBSB);
- p->dbt[1] = readl_relaxed(p->base + DMA_DBTB);
- saved_dcsr |= dcsr & (DCSR_STRTA | DCSR_STRTB);
- }
- p->dcsr = saved_dcsr;
- writel(DCSR_STRTA | DCSR_STRTB, p->base + DMA_DCSR_C);
- }
- return 0;
- }
- static __maybe_unused int sa11x0_dma_resume(struct device *dev)
- {
- struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
- unsigned pch;
- for (pch = 0; pch < NR_PHY_CHAN; pch++) {
- struct sa11x0_dma_phy *p = &d->phy[pch];
- struct sa11x0_dma_desc *txd = NULL;
- u32 dcsr = readl_relaxed(p->base + DMA_DCSR_R);
- WARN_ON(dcsr & (DCSR_BIU | DCSR_STRTA | DCSR_STRTB | DCSR_RUN));
- if (p->txd_done)
- txd = p->txd_done;
- else if (p->txd_load)
- txd = p->txd_load;
- if (!txd)
- continue;
- writel_relaxed(txd->ddar, p->base + DMA_DDAR);
- writel_relaxed(p->dbs[0], p->base + DMA_DBSA);
- writel_relaxed(p->dbt[0], p->base + DMA_DBTA);
- writel_relaxed(p->dbs[1], p->base + DMA_DBSB);
- writel_relaxed(p->dbt[1], p->base + DMA_DBTB);
- writel_relaxed(p->dcsr, p->base + DMA_DCSR_S);
- }
- return 0;
- }
- static const struct dev_pm_ops sa11x0_dma_pm_ops = {
- SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(sa11x0_dma_suspend, sa11x0_dma_resume)
- };
- static struct platform_driver sa11x0_dma_driver = {
- .driver = {
- .name = "sa11x0-dma",
- .pm = &sa11x0_dma_pm_ops,
- },
- .probe = sa11x0_dma_probe,
- .remove = sa11x0_dma_remove,
- };
- static int __init sa11x0_dma_init(void)
- {
- return platform_driver_register(&sa11x0_dma_driver);
- }
- subsys_initcall(sa11x0_dma_init);
- static void __exit sa11x0_dma_exit(void)
- {
- platform_driver_unregister(&sa11x0_dma_driver);
- }
- module_exit(sa11x0_dma_exit);
- MODULE_AUTHOR("Russell King");
- MODULE_DESCRIPTION("SA-11x0 DMA driver");
- MODULE_LICENSE("GPL v2");
- MODULE_ALIAS("platform:sa11x0-dma");
|