1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090 |
- // SPDX-License-Identifier: GPL-2.0-only
- /*
- * IMG Multi-threaded DMA Controller (MDC)
- *
- * Copyright (C) 2009,2012,2013 Imagination Technologies Ltd.
- * Copyright (C) 2014 Google, Inc.
- */
- #include <linux/clk.h>
- #include <linux/dma-mapping.h>
- #include <linux/dmaengine.h>
- #include <linux/dmapool.h>
- #include <linux/interrupt.h>
- #include <linux/io.h>
- #include <linux/irq.h>
- #include <linux/kernel.h>
- #include <linux/mfd/syscon.h>
- #include <linux/module.h>
- #include <linux/of.h>
- #include <linux/of_device.h>
- #include <linux/of_dma.h>
- #include <linux/platform_device.h>
- #include <linux/pm_runtime.h>
- #include <linux/regmap.h>
- #include <linux/slab.h>
- #include <linux/spinlock.h>
- #include "dmaengine.h"
- #include "virt-dma.h"
- #define MDC_MAX_DMA_CHANNELS 32
- #define MDC_GENERAL_CONFIG 0x000
- #define MDC_GENERAL_CONFIG_LIST_IEN BIT(31)
- #define MDC_GENERAL_CONFIG_IEN BIT(29)
- #define MDC_GENERAL_CONFIG_LEVEL_INT BIT(28)
- #define MDC_GENERAL_CONFIG_INC_W BIT(12)
- #define MDC_GENERAL_CONFIG_INC_R BIT(8)
- #define MDC_GENERAL_CONFIG_PHYSICAL_W BIT(7)
- #define MDC_GENERAL_CONFIG_WIDTH_W_SHIFT 4
- #define MDC_GENERAL_CONFIG_WIDTH_W_MASK 0x7
- #define MDC_GENERAL_CONFIG_PHYSICAL_R BIT(3)
- #define MDC_GENERAL_CONFIG_WIDTH_R_SHIFT 0
- #define MDC_GENERAL_CONFIG_WIDTH_R_MASK 0x7
- #define MDC_READ_PORT_CONFIG 0x004
- #define MDC_READ_PORT_CONFIG_STHREAD_SHIFT 28
- #define MDC_READ_PORT_CONFIG_STHREAD_MASK 0xf
- #define MDC_READ_PORT_CONFIG_RTHREAD_SHIFT 24
- #define MDC_READ_PORT_CONFIG_RTHREAD_MASK 0xf
- #define MDC_READ_PORT_CONFIG_WTHREAD_SHIFT 16
- #define MDC_READ_PORT_CONFIG_WTHREAD_MASK 0xf
- #define MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT 4
- #define MDC_READ_PORT_CONFIG_BURST_SIZE_MASK 0xff
- #define MDC_READ_PORT_CONFIG_DREQ_ENABLE BIT(1)
- #define MDC_READ_ADDRESS 0x008
- #define MDC_WRITE_ADDRESS 0x00c
- #define MDC_TRANSFER_SIZE 0x010
- #define MDC_TRANSFER_SIZE_MASK 0xffffff
- #define MDC_LIST_NODE_ADDRESS 0x014
- #define MDC_CMDS_PROCESSED 0x018
- #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT 16
- #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK 0x3f
- #define MDC_CMDS_PROCESSED_INT_ACTIVE BIT(8)
- #define MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT 0
- #define MDC_CMDS_PROCESSED_CMDS_DONE_MASK 0x3f
- #define MDC_CONTROL_AND_STATUS 0x01c
- #define MDC_CONTROL_AND_STATUS_CANCEL BIT(20)
- #define MDC_CONTROL_AND_STATUS_LIST_EN BIT(4)
- #define MDC_CONTROL_AND_STATUS_EN BIT(0)
- #define MDC_ACTIVE_TRANSFER_SIZE 0x030
- #define MDC_GLOBAL_CONFIG_A 0x900
- #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT 16
- #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK 0xff
- #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT 8
- #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK 0xff
- #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT 0
- #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK 0xff
- struct mdc_hw_list_desc {
- u32 gen_conf;
- u32 readport_conf;
- u32 read_addr;
- u32 write_addr;
- u32 xfer_size;
- u32 node_addr;
- u32 cmds_done;
- u32 ctrl_status;
- /*
- * Not part of the list descriptor, but instead used by the CPU to
- * traverse the list.
- */
- struct mdc_hw_list_desc *next_desc;
- };
- struct mdc_tx_desc {
- struct mdc_chan *chan;
- struct virt_dma_desc vd;
- dma_addr_t list_phys;
- struct mdc_hw_list_desc *list;
- bool cyclic;
- bool cmd_loaded;
- unsigned int list_len;
- unsigned int list_period_len;
- size_t list_xfer_size;
- unsigned int list_cmds_done;
- };
- struct mdc_chan {
- struct mdc_dma *mdma;
- struct virt_dma_chan vc;
- struct dma_slave_config config;
- struct mdc_tx_desc *desc;
- int irq;
- unsigned int periph;
- unsigned int thread;
- unsigned int chan_nr;
- };
- struct mdc_dma_soc_data {
- void (*enable_chan)(struct mdc_chan *mchan);
- void (*disable_chan)(struct mdc_chan *mchan);
- };
- struct mdc_dma {
- struct dma_device dma_dev;
- void __iomem *regs;
- struct clk *clk;
- struct dma_pool *desc_pool;
- struct regmap *periph_regs;
- spinlock_t lock;
- unsigned int nr_threads;
- unsigned int nr_channels;
- unsigned int bus_width;
- unsigned int max_burst_mult;
- unsigned int max_xfer_size;
- const struct mdc_dma_soc_data *soc;
- struct mdc_chan channels[MDC_MAX_DMA_CHANNELS];
- };
- static inline u32 mdc_readl(struct mdc_dma *mdma, u32 reg)
- {
- return readl(mdma->regs + reg);
- }
- static inline void mdc_writel(struct mdc_dma *mdma, u32 val, u32 reg)
- {
- writel(val, mdma->regs + reg);
- }
- static inline u32 mdc_chan_readl(struct mdc_chan *mchan, u32 reg)
- {
- return mdc_readl(mchan->mdma, mchan->chan_nr * 0x040 + reg);
- }
- static inline void mdc_chan_writel(struct mdc_chan *mchan, u32 val, u32 reg)
- {
- mdc_writel(mchan->mdma, val, mchan->chan_nr * 0x040 + reg);
- }
- static inline struct mdc_chan *to_mdc_chan(struct dma_chan *c)
- {
- return container_of(to_virt_chan(c), struct mdc_chan, vc);
- }
- static inline struct mdc_tx_desc *to_mdc_desc(struct dma_async_tx_descriptor *t)
- {
- struct virt_dma_desc *vdesc = container_of(t, struct virt_dma_desc, tx);
- return container_of(vdesc, struct mdc_tx_desc, vd);
- }
- static inline struct device *mdma2dev(struct mdc_dma *mdma)
- {
- return mdma->dma_dev.dev;
- }
- static inline unsigned int to_mdc_width(unsigned int bytes)
- {
- return ffs(bytes) - 1;
- }
- static inline void mdc_set_read_width(struct mdc_hw_list_desc *ldesc,
- unsigned int bytes)
- {
- ldesc->gen_conf |= to_mdc_width(bytes) <<
- MDC_GENERAL_CONFIG_WIDTH_R_SHIFT;
- }
- static inline void mdc_set_write_width(struct mdc_hw_list_desc *ldesc,
- unsigned int bytes)
- {
- ldesc->gen_conf |= to_mdc_width(bytes) <<
- MDC_GENERAL_CONFIG_WIDTH_W_SHIFT;
- }
- static void mdc_list_desc_config(struct mdc_chan *mchan,
- struct mdc_hw_list_desc *ldesc,
- enum dma_transfer_direction dir,
- dma_addr_t src, dma_addr_t dst, size_t len)
- {
- struct mdc_dma *mdma = mchan->mdma;
- unsigned int max_burst, burst_size;
- ldesc->gen_conf = MDC_GENERAL_CONFIG_IEN | MDC_GENERAL_CONFIG_LIST_IEN |
- MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
- MDC_GENERAL_CONFIG_PHYSICAL_R;
- ldesc->readport_conf =
- (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
- (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
- (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
- ldesc->read_addr = src;
- ldesc->write_addr = dst;
- ldesc->xfer_size = len - 1;
- ldesc->node_addr = 0;
- ldesc->cmds_done = 0;
- ldesc->ctrl_status = MDC_CONTROL_AND_STATUS_LIST_EN |
- MDC_CONTROL_AND_STATUS_EN;
- ldesc->next_desc = NULL;
- if (IS_ALIGNED(dst, mdma->bus_width) &&
- IS_ALIGNED(src, mdma->bus_width))
- max_burst = mdma->bus_width * mdma->max_burst_mult;
- else
- max_burst = mdma->bus_width * (mdma->max_burst_mult - 1);
- if (dir == DMA_MEM_TO_DEV) {
- ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R;
- ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
- mdc_set_read_width(ldesc, mdma->bus_width);
- mdc_set_write_width(ldesc, mchan->config.dst_addr_width);
- burst_size = min(max_burst, mchan->config.dst_maxburst *
- mchan->config.dst_addr_width);
- } else if (dir == DMA_DEV_TO_MEM) {
- ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_W;
- ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
- mdc_set_read_width(ldesc, mchan->config.src_addr_width);
- mdc_set_write_width(ldesc, mdma->bus_width);
- burst_size = min(max_burst, mchan->config.src_maxburst *
- mchan->config.src_addr_width);
- } else {
- ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R |
- MDC_GENERAL_CONFIG_INC_W;
- mdc_set_read_width(ldesc, mdma->bus_width);
- mdc_set_write_width(ldesc, mdma->bus_width);
- burst_size = max_burst;
- }
- ldesc->readport_conf |= (burst_size - 1) <<
- MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT;
- }
- static void mdc_list_desc_free(struct mdc_tx_desc *mdesc)
- {
- struct mdc_dma *mdma = mdesc->chan->mdma;
- struct mdc_hw_list_desc *curr, *next;
- dma_addr_t curr_phys, next_phys;
- curr = mdesc->list;
- curr_phys = mdesc->list_phys;
- while (curr) {
- next = curr->next_desc;
- next_phys = curr->node_addr;
- dma_pool_free(mdma->desc_pool, curr, curr_phys);
- curr = next;
- curr_phys = next_phys;
- }
- }
- static void mdc_desc_free(struct virt_dma_desc *vd)
- {
- struct mdc_tx_desc *mdesc = to_mdc_desc(&vd->tx);
- mdc_list_desc_free(mdesc);
- kfree(mdesc);
- }
- static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy(
- struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len,
- unsigned long flags)
- {
- struct mdc_chan *mchan = to_mdc_chan(chan);
- struct mdc_dma *mdma = mchan->mdma;
- struct mdc_tx_desc *mdesc;
- struct mdc_hw_list_desc *curr, *prev = NULL;
- dma_addr_t curr_phys;
- if (!len)
- return NULL;
- mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
- if (!mdesc)
- return NULL;
- mdesc->chan = mchan;
- mdesc->list_xfer_size = len;
- while (len > 0) {
- size_t xfer_size;
- curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, &curr_phys);
- if (!curr)
- goto free_desc;
- if (prev) {
- prev->node_addr = curr_phys;
- prev->next_desc = curr;
- } else {
- mdesc->list_phys = curr_phys;
- mdesc->list = curr;
- }
- xfer_size = min_t(size_t, mdma->max_xfer_size, len);
- mdc_list_desc_config(mchan, curr, DMA_MEM_TO_MEM, src, dest,
- xfer_size);
- prev = curr;
- mdesc->list_len++;
- src += xfer_size;
- dest += xfer_size;
- len -= xfer_size;
- }
- return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
- free_desc:
- mdc_desc_free(&mdesc->vd);
- return NULL;
- }
- static int mdc_check_slave_width(struct mdc_chan *mchan,
- enum dma_transfer_direction dir)
- {
- enum dma_slave_buswidth width;
- if (dir == DMA_MEM_TO_DEV)
- width = mchan->config.dst_addr_width;
- else
- width = mchan->config.src_addr_width;
- switch (width) {
- case DMA_SLAVE_BUSWIDTH_1_BYTE:
- case DMA_SLAVE_BUSWIDTH_2_BYTES:
- case DMA_SLAVE_BUSWIDTH_4_BYTES:
- case DMA_SLAVE_BUSWIDTH_8_BYTES:
- break;
- default:
- return -EINVAL;
- }
- if (width > mchan->mdma->bus_width)
- return -EINVAL;
- return 0;
- }
- static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic(
- struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
- size_t period_len, enum dma_transfer_direction dir,
- unsigned long flags)
- {
- struct mdc_chan *mchan = to_mdc_chan(chan);
- struct mdc_dma *mdma = mchan->mdma;
- struct mdc_tx_desc *mdesc;
- struct mdc_hw_list_desc *curr, *prev = NULL;
- dma_addr_t curr_phys;
- if (!buf_len && !period_len)
- return NULL;
- if (!is_slave_direction(dir))
- return NULL;
- if (mdc_check_slave_width(mchan, dir) < 0)
- return NULL;
- mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
- if (!mdesc)
- return NULL;
- mdesc->chan = mchan;
- mdesc->cyclic = true;
- mdesc->list_xfer_size = buf_len;
- mdesc->list_period_len = DIV_ROUND_UP(period_len,
- mdma->max_xfer_size);
- while (buf_len > 0) {
- size_t remainder = min(period_len, buf_len);
- while (remainder > 0) {
- size_t xfer_size;
- curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
- &curr_phys);
- if (!curr)
- goto free_desc;
- if (!prev) {
- mdesc->list_phys = curr_phys;
- mdesc->list = curr;
- } else {
- prev->node_addr = curr_phys;
- prev->next_desc = curr;
- }
- xfer_size = min_t(size_t, mdma->max_xfer_size,
- remainder);
- if (dir == DMA_MEM_TO_DEV) {
- mdc_list_desc_config(mchan, curr, dir,
- buf_addr,
- mchan->config.dst_addr,
- xfer_size);
- } else {
- mdc_list_desc_config(mchan, curr, dir,
- mchan->config.src_addr,
- buf_addr,
- xfer_size);
- }
- prev = curr;
- mdesc->list_len++;
- buf_addr += xfer_size;
- buf_len -= xfer_size;
- remainder -= xfer_size;
- }
- }
- prev->node_addr = mdesc->list_phys;
- return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
- free_desc:
- mdc_desc_free(&mdesc->vd);
- return NULL;
- }
- static struct dma_async_tx_descriptor *mdc_prep_slave_sg(
- struct dma_chan *chan, struct scatterlist *sgl,
- unsigned int sg_len, enum dma_transfer_direction dir,
- unsigned long flags, void *context)
- {
- struct mdc_chan *mchan = to_mdc_chan(chan);
- struct mdc_dma *mdma = mchan->mdma;
- struct mdc_tx_desc *mdesc;
- struct scatterlist *sg;
- struct mdc_hw_list_desc *curr, *prev = NULL;
- dma_addr_t curr_phys;
- unsigned int i;
- if (!sgl)
- return NULL;
- if (!is_slave_direction(dir))
- return NULL;
- if (mdc_check_slave_width(mchan, dir) < 0)
- return NULL;
- mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
- if (!mdesc)
- return NULL;
- mdesc->chan = mchan;
- for_each_sg(sgl, sg, sg_len, i) {
- dma_addr_t buf = sg_dma_address(sg);
- size_t buf_len = sg_dma_len(sg);
- while (buf_len > 0) {
- size_t xfer_size;
- curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
- &curr_phys);
- if (!curr)
- goto free_desc;
- if (!prev) {
- mdesc->list_phys = curr_phys;
- mdesc->list = curr;
- } else {
- prev->node_addr = curr_phys;
- prev->next_desc = curr;
- }
- xfer_size = min_t(size_t, mdma->max_xfer_size,
- buf_len);
- if (dir == DMA_MEM_TO_DEV) {
- mdc_list_desc_config(mchan, curr, dir, buf,
- mchan->config.dst_addr,
- xfer_size);
- } else {
- mdc_list_desc_config(mchan, curr, dir,
- mchan->config.src_addr,
- buf, xfer_size);
- }
- prev = curr;
- mdesc->list_len++;
- mdesc->list_xfer_size += xfer_size;
- buf += xfer_size;
- buf_len -= xfer_size;
- }
- }
- return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
- free_desc:
- mdc_desc_free(&mdesc->vd);
- return NULL;
- }
- static void mdc_issue_desc(struct mdc_chan *mchan)
- {
- struct mdc_dma *mdma = mchan->mdma;
- struct virt_dma_desc *vd;
- struct mdc_tx_desc *mdesc;
- u32 val;
- vd = vchan_next_desc(&mchan->vc);
- if (!vd)
- return;
- list_del(&vd->node);
- mdesc = to_mdc_desc(&vd->tx);
- mchan->desc = mdesc;
- dev_dbg(mdma2dev(mdma), "Issuing descriptor on channel %d\n",
- mchan->chan_nr);
- mdma->soc->enable_chan(mchan);
- val = mdc_chan_readl(mchan, MDC_GENERAL_CONFIG);
- val |= MDC_GENERAL_CONFIG_LIST_IEN | MDC_GENERAL_CONFIG_IEN |
- MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
- MDC_GENERAL_CONFIG_PHYSICAL_R;
- mdc_chan_writel(mchan, val, MDC_GENERAL_CONFIG);
- val = (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
- (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
- (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
- mdc_chan_writel(mchan, val, MDC_READ_PORT_CONFIG);
- mdc_chan_writel(mchan, mdesc->list_phys, MDC_LIST_NODE_ADDRESS);
- val = mdc_chan_readl(mchan, MDC_CONTROL_AND_STATUS);
- val |= MDC_CONTROL_AND_STATUS_LIST_EN;
- mdc_chan_writel(mchan, val, MDC_CONTROL_AND_STATUS);
- }
- static void mdc_issue_pending(struct dma_chan *chan)
- {
- struct mdc_chan *mchan = to_mdc_chan(chan);
- unsigned long flags;
- spin_lock_irqsave(&mchan->vc.lock, flags);
- if (vchan_issue_pending(&mchan->vc) && !mchan->desc)
- mdc_issue_desc(mchan);
- spin_unlock_irqrestore(&mchan->vc.lock, flags);
- }
- static enum dma_status mdc_tx_status(struct dma_chan *chan,
- dma_cookie_t cookie, struct dma_tx_state *txstate)
- {
- struct mdc_chan *mchan = to_mdc_chan(chan);
- struct mdc_tx_desc *mdesc;
- struct virt_dma_desc *vd;
- unsigned long flags;
- size_t bytes = 0;
- int ret;
- ret = dma_cookie_status(chan, cookie, txstate);
- if (ret == DMA_COMPLETE)
- return ret;
- if (!txstate)
- return ret;
- spin_lock_irqsave(&mchan->vc.lock, flags);
- vd = vchan_find_desc(&mchan->vc, cookie);
- if (vd) {
- mdesc = to_mdc_desc(&vd->tx);
- bytes = mdesc->list_xfer_size;
- } else if (mchan->desc && mchan->desc->vd.tx.cookie == cookie) {
- struct mdc_hw_list_desc *ldesc;
- u32 val1, val2, done, processed, residue;
- int i, cmds;
- mdesc = mchan->desc;
- /*
- * Determine the number of commands that haven't been
- * processed (handled by the IRQ handler) yet.
- */
- do {
- val1 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
- ~MDC_CMDS_PROCESSED_INT_ACTIVE;
- residue = mdc_chan_readl(mchan,
- MDC_ACTIVE_TRANSFER_SIZE);
- val2 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
- ~MDC_CMDS_PROCESSED_INT_ACTIVE;
- } while (val1 != val2);
- done = (val1 >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
- MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
- processed = (val1 >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
- MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
- cmds = (done - processed) %
- (MDC_CMDS_PROCESSED_CMDS_DONE_MASK + 1);
- /*
- * If the command loaded event hasn't been processed yet, then
- * the difference above includes an extra command.
- */
- if (!mdesc->cmd_loaded)
- cmds--;
- else
- cmds += mdesc->list_cmds_done;
- bytes = mdesc->list_xfer_size;
- ldesc = mdesc->list;
- for (i = 0; i < cmds; i++) {
- bytes -= ldesc->xfer_size + 1;
- ldesc = ldesc->next_desc;
- }
- if (ldesc) {
- if (residue != MDC_TRANSFER_SIZE_MASK)
- bytes -= ldesc->xfer_size - residue;
- else
- bytes -= ldesc->xfer_size + 1;
- }
- }
- spin_unlock_irqrestore(&mchan->vc.lock, flags);
- dma_set_residue(txstate, bytes);
- return ret;
- }
- static unsigned int mdc_get_new_events(struct mdc_chan *mchan)
- {
- u32 val, processed, done1, done2;
- unsigned int ret;
- val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
- processed = (val >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
- MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
- /*
- * CMDS_DONE may have incremented between reading CMDS_PROCESSED
- * and clearing INT_ACTIVE. Re-read CMDS_PROCESSED to ensure we
- * didn't miss a command completion.
- */
- do {
- val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
- done1 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
- MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
- val &= ~((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK <<
- MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) |
- MDC_CMDS_PROCESSED_INT_ACTIVE);
- val |= done1 << MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT;
- mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED);
- val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
- done2 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
- MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
- } while (done1 != done2);
- if (done1 >= processed)
- ret = done1 - processed;
- else
- ret = ((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK + 1) -
- processed) + done1;
- return ret;
- }
- static int mdc_terminate_all(struct dma_chan *chan)
- {
- struct mdc_chan *mchan = to_mdc_chan(chan);
- unsigned long flags;
- LIST_HEAD(head);
- spin_lock_irqsave(&mchan->vc.lock, flags);
- mdc_chan_writel(mchan, MDC_CONTROL_AND_STATUS_CANCEL,
- MDC_CONTROL_AND_STATUS);
- if (mchan->desc) {
- vchan_terminate_vdesc(&mchan->desc->vd);
- mchan->desc = NULL;
- }
- vchan_get_all_descriptors(&mchan->vc, &head);
- mdc_get_new_events(mchan);
- spin_unlock_irqrestore(&mchan->vc.lock, flags);
- vchan_dma_desc_free_list(&mchan->vc, &head);
- return 0;
- }
- static void mdc_synchronize(struct dma_chan *chan)
- {
- struct mdc_chan *mchan = to_mdc_chan(chan);
- vchan_synchronize(&mchan->vc);
- }
- static int mdc_slave_config(struct dma_chan *chan,
- struct dma_slave_config *config)
- {
- struct mdc_chan *mchan = to_mdc_chan(chan);
- unsigned long flags;
- spin_lock_irqsave(&mchan->vc.lock, flags);
- mchan->config = *config;
- spin_unlock_irqrestore(&mchan->vc.lock, flags);
- return 0;
- }
- static int mdc_alloc_chan_resources(struct dma_chan *chan)
- {
- struct mdc_chan *mchan = to_mdc_chan(chan);
- struct device *dev = mdma2dev(mchan->mdma);
- return pm_runtime_get_sync(dev);
- }
- static void mdc_free_chan_resources(struct dma_chan *chan)
- {
- struct mdc_chan *mchan = to_mdc_chan(chan);
- struct mdc_dma *mdma = mchan->mdma;
- struct device *dev = mdma2dev(mdma);
- mdc_terminate_all(chan);
- mdma->soc->disable_chan(mchan);
- pm_runtime_put(dev);
- }
- static irqreturn_t mdc_chan_irq(int irq, void *dev_id)
- {
- struct mdc_chan *mchan = (struct mdc_chan *)dev_id;
- struct mdc_tx_desc *mdesc;
- unsigned int i, new_events;
- spin_lock(&mchan->vc.lock);
- dev_dbg(mdma2dev(mchan->mdma), "IRQ on channel %d\n", mchan->chan_nr);
- new_events = mdc_get_new_events(mchan);
- if (!new_events)
- goto out;
- mdesc = mchan->desc;
- if (!mdesc) {
- dev_warn(mdma2dev(mchan->mdma),
- "IRQ with no active descriptor on channel %d\n",
- mchan->chan_nr);
- goto out;
- }
- for (i = 0; i < new_events; i++) {
- /*
- * The first interrupt in a transfer indicates that the
- * command list has been loaded, not that a command has
- * been completed.
- */
- if (!mdesc->cmd_loaded) {
- mdesc->cmd_loaded = true;
- continue;
- }
- mdesc->list_cmds_done++;
- if (mdesc->cyclic) {
- mdesc->list_cmds_done %= mdesc->list_len;
- if (mdesc->list_cmds_done % mdesc->list_period_len == 0)
- vchan_cyclic_callback(&mdesc->vd);
- } else if (mdesc->list_cmds_done == mdesc->list_len) {
- mchan->desc = NULL;
- vchan_cookie_complete(&mdesc->vd);
- mdc_issue_desc(mchan);
- break;
- }
- }
- out:
- spin_unlock(&mchan->vc.lock);
- return IRQ_HANDLED;
- }
- static struct dma_chan *mdc_of_xlate(struct of_phandle_args *dma_spec,
- struct of_dma *ofdma)
- {
- struct mdc_dma *mdma = ofdma->of_dma_data;
- struct dma_chan *chan;
- if (dma_spec->args_count != 3)
- return NULL;
- list_for_each_entry(chan, &mdma->dma_dev.channels, device_node) {
- struct mdc_chan *mchan = to_mdc_chan(chan);
- if (!(dma_spec->args[1] & BIT(mchan->chan_nr)))
- continue;
- if (dma_get_slave_channel(chan)) {
- mchan->periph = dma_spec->args[0];
- mchan->thread = dma_spec->args[2];
- return chan;
- }
- }
- return NULL;
- }
- #define PISTACHIO_CR_PERIPH_DMA_ROUTE(ch) (0x120 + 0x4 * ((ch) / 4))
- #define PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(ch) (8 * ((ch) % 4))
- #define PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK 0x3f
- static void pistachio_mdc_enable_chan(struct mdc_chan *mchan)
- {
- struct mdc_dma *mdma = mchan->mdma;
- regmap_update_bits(mdma->periph_regs,
- PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
- PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
- PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
- mchan->periph <<
- PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr));
- }
- static void pistachio_mdc_disable_chan(struct mdc_chan *mchan)
- {
- struct mdc_dma *mdma = mchan->mdma;
- regmap_update_bits(mdma->periph_regs,
- PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
- PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
- PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
- 0);
- }
- static const struct mdc_dma_soc_data pistachio_mdc_data = {
- .enable_chan = pistachio_mdc_enable_chan,
- .disable_chan = pistachio_mdc_disable_chan,
- };
- static const struct of_device_id mdc_dma_of_match[] = {
- { .compatible = "img,pistachio-mdc-dma", .data = &pistachio_mdc_data, },
- { },
- };
- MODULE_DEVICE_TABLE(of, mdc_dma_of_match);
- static int img_mdc_runtime_suspend(struct device *dev)
- {
- struct mdc_dma *mdma = dev_get_drvdata(dev);
- clk_disable_unprepare(mdma->clk);
- return 0;
- }
- static int img_mdc_runtime_resume(struct device *dev)
- {
- struct mdc_dma *mdma = dev_get_drvdata(dev);
- return clk_prepare_enable(mdma->clk);
- }
- static int mdc_dma_probe(struct platform_device *pdev)
- {
- struct mdc_dma *mdma;
- struct resource *res;
- unsigned int i;
- u32 val;
- int ret;
- mdma = devm_kzalloc(&pdev->dev, sizeof(*mdma), GFP_KERNEL);
- if (!mdma)
- return -ENOMEM;
- platform_set_drvdata(pdev, mdma);
- mdma->soc = of_device_get_match_data(&pdev->dev);
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- mdma->regs = devm_ioremap_resource(&pdev->dev, res);
- if (IS_ERR(mdma->regs))
- return PTR_ERR(mdma->regs);
- mdma->periph_regs = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
- "img,cr-periph");
- if (IS_ERR(mdma->periph_regs))
- return PTR_ERR(mdma->periph_regs);
- mdma->clk = devm_clk_get(&pdev->dev, "sys");
- if (IS_ERR(mdma->clk))
- return PTR_ERR(mdma->clk);
- dma_cap_zero(mdma->dma_dev.cap_mask);
- dma_cap_set(DMA_SLAVE, mdma->dma_dev.cap_mask);
- dma_cap_set(DMA_PRIVATE, mdma->dma_dev.cap_mask);
- dma_cap_set(DMA_CYCLIC, mdma->dma_dev.cap_mask);
- dma_cap_set(DMA_MEMCPY, mdma->dma_dev.cap_mask);
- val = mdc_readl(mdma, MDC_GLOBAL_CONFIG_A);
- mdma->nr_channels = (val >> MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT) &
- MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK;
- mdma->nr_threads =
- 1 << ((val >> MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT) &
- MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK);
- mdma->bus_width =
- (1 << ((val >> MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT) &
- MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK)) / 8;
- /*
- * Although transfer sizes of up to MDC_TRANSFER_SIZE_MASK + 1 bytes
- * are supported, this makes it possible for the value reported in
- * MDC_ACTIVE_TRANSFER_SIZE to be ambiguous - an active transfer size
- * of MDC_TRANSFER_SIZE_MASK may indicate either that 0 bytes or
- * MDC_TRANSFER_SIZE_MASK + 1 bytes are remaining. To eliminate this
- * ambiguity, restrict transfer sizes to one bus-width less than the
- * actual maximum.
- */
- mdma->max_xfer_size = MDC_TRANSFER_SIZE_MASK + 1 - mdma->bus_width;
- of_property_read_u32(pdev->dev.of_node, "dma-channels",
- &mdma->nr_channels);
- ret = of_property_read_u32(pdev->dev.of_node,
- "img,max-burst-multiplier",
- &mdma->max_burst_mult);
- if (ret)
- return ret;
- mdma->dma_dev.dev = &pdev->dev;
- mdma->dma_dev.device_prep_slave_sg = mdc_prep_slave_sg;
- mdma->dma_dev.device_prep_dma_cyclic = mdc_prep_dma_cyclic;
- mdma->dma_dev.device_prep_dma_memcpy = mdc_prep_dma_memcpy;
- mdma->dma_dev.device_alloc_chan_resources = mdc_alloc_chan_resources;
- mdma->dma_dev.device_free_chan_resources = mdc_free_chan_resources;
- mdma->dma_dev.device_tx_status = mdc_tx_status;
- mdma->dma_dev.device_issue_pending = mdc_issue_pending;
- mdma->dma_dev.device_terminate_all = mdc_terminate_all;
- mdma->dma_dev.device_synchronize = mdc_synchronize;
- mdma->dma_dev.device_config = mdc_slave_config;
- mdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
- mdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
- for (i = 1; i <= mdma->bus_width; i <<= 1) {
- mdma->dma_dev.src_addr_widths |= BIT(i);
- mdma->dma_dev.dst_addr_widths |= BIT(i);
- }
- INIT_LIST_HEAD(&mdma->dma_dev.channels);
- for (i = 0; i < mdma->nr_channels; i++) {
- struct mdc_chan *mchan = &mdma->channels[i];
- mchan->mdma = mdma;
- mchan->chan_nr = i;
- mchan->irq = platform_get_irq(pdev, i);
- if (mchan->irq < 0)
- return mchan->irq;
- ret = devm_request_irq(&pdev->dev, mchan->irq, mdc_chan_irq,
- IRQ_TYPE_LEVEL_HIGH,
- dev_name(&pdev->dev), mchan);
- if (ret < 0)
- return ret;
- mchan->vc.desc_free = mdc_desc_free;
- vchan_init(&mchan->vc, &mdma->dma_dev);
- }
- mdma->desc_pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev,
- sizeof(struct mdc_hw_list_desc),
- 4, 0);
- if (!mdma->desc_pool)
- return -ENOMEM;
- pm_runtime_enable(&pdev->dev);
- if (!pm_runtime_enabled(&pdev->dev)) {
- ret = img_mdc_runtime_resume(&pdev->dev);
- if (ret)
- return ret;
- }
- ret = dma_async_device_register(&mdma->dma_dev);
- if (ret)
- goto suspend;
- ret = of_dma_controller_register(pdev->dev.of_node, mdc_of_xlate, mdma);
- if (ret)
- goto unregister;
- dev_info(&pdev->dev, "MDC with %u channels and %u threads\n",
- mdma->nr_channels, mdma->nr_threads);
- return 0;
- unregister:
- dma_async_device_unregister(&mdma->dma_dev);
- suspend:
- if (!pm_runtime_enabled(&pdev->dev))
- img_mdc_runtime_suspend(&pdev->dev);
- pm_runtime_disable(&pdev->dev);
- return ret;
- }
- static int mdc_dma_remove(struct platform_device *pdev)
- {
- struct mdc_dma *mdma = platform_get_drvdata(pdev);
- struct mdc_chan *mchan, *next;
- of_dma_controller_free(pdev->dev.of_node);
- dma_async_device_unregister(&mdma->dma_dev);
- list_for_each_entry_safe(mchan, next, &mdma->dma_dev.channels,
- vc.chan.device_node) {
- list_del(&mchan->vc.chan.device_node);
- devm_free_irq(&pdev->dev, mchan->irq, mchan);
- tasklet_kill(&mchan->vc.task);
- }
- pm_runtime_disable(&pdev->dev);
- if (!pm_runtime_status_suspended(&pdev->dev))
- img_mdc_runtime_suspend(&pdev->dev);
- return 0;
- }
- #ifdef CONFIG_PM_SLEEP
- static int img_mdc_suspend_late(struct device *dev)
- {
- struct mdc_dma *mdma = dev_get_drvdata(dev);
- int i;
- /* Check that all channels are idle */
- for (i = 0; i < mdma->nr_channels; i++) {
- struct mdc_chan *mchan = &mdma->channels[i];
- if (unlikely(mchan->desc))
- return -EBUSY;
- }
- return pm_runtime_force_suspend(dev);
- }
- static int img_mdc_resume_early(struct device *dev)
- {
- return pm_runtime_force_resume(dev);
- }
- #endif /* CONFIG_PM_SLEEP */
- static const struct dev_pm_ops img_mdc_pm_ops = {
- SET_RUNTIME_PM_OPS(img_mdc_runtime_suspend,
- img_mdc_runtime_resume, NULL)
- SET_LATE_SYSTEM_SLEEP_PM_OPS(img_mdc_suspend_late,
- img_mdc_resume_early)
- };
- static struct platform_driver mdc_dma_driver = {
- .driver = {
- .name = "img-mdc-dma",
- .pm = &img_mdc_pm_ops,
- .of_match_table = of_match_ptr(mdc_dma_of_match),
- },
- .probe = mdc_dma_probe,
- .remove = mdc_dma_remove,
- };
- module_platform_driver(mdc_dma_driver);
- MODULE_DESCRIPTION("IMG Multi-threaded DMA Controller (MDC) driver");
- MODULE_AUTHOR("Andrew Bresticker <[email protected]>");
- MODULE_LICENSE("GPL v2");
|