|
|
|
@@ -33,6 +33,9 @@
|
|
|
|
|
#define PL330_MAX_CHAN 8
|
|
|
|
|
#define PL330_MAX_IRQS 32
|
|
|
|
|
#define PL330_MAX_PERI 32
|
|
|
|
|
#define PL330_MAX_BURST 16
|
|
|
|
|
|
|
|
|
|
#define PL330_QUIRK_BROKEN_NO_FLUSHP BIT(0)
|
|
|
|
|
|
|
|
|
|
enum pl330_cachectrl {
|
|
|
|
|
CCTRL0, /* Noncacheable and nonbufferable */
|
|
|
|
@@ -488,6 +491,17 @@ struct pl330_dmac {
|
|
|
|
|
/* Peripheral channels connected to this DMAC */
|
|
|
|
|
unsigned int num_peripherals;
|
|
|
|
|
struct dma_pl330_chan *peripherals; /* keep at end */
|
|
|
|
|
int quirks;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static struct pl330_of_quirks {
|
|
|
|
|
char *quirk;
|
|
|
|
|
int id;
|
|
|
|
|
} of_quirks[] = {
|
|
|
|
|
{
|
|
|
|
|
.quirk = "arm,pl330-broken-no-flushp",
|
|
|
|
|
.id = PL330_QUIRK_BROKEN_NO_FLUSHP,
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct dma_pl330_desc {
|
|
|
|
@@ -1137,47 +1151,67 @@ static inline int _ldst_memtomem(unsigned dry_run, u8 buf[],
|
|
|
|
|
return off;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int _ldst_devtomem(unsigned dry_run, u8 buf[],
|
|
|
|
|
const struct _xfer_spec *pxs, int cyc)
|
|
|
|
|
static inline int _ldst_devtomem(struct pl330_dmac *pl330, unsigned dry_run,
|
|
|
|
|
u8 buf[], const struct _xfer_spec *pxs,
|
|
|
|
|
int cyc)
|
|
|
|
|
{
|
|
|
|
|
int off = 0;
|
|
|
|
|
enum pl330_cond cond;
|
|
|
|
|
|
|
|
|
|
if (pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)
|
|
|
|
|
cond = BURST;
|
|
|
|
|
else
|
|
|
|
|
cond = SINGLE;
|
|
|
|
|
|
|
|
|
|
while (cyc--) {
|
|
|
|
|
off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->desc->peri);
|
|
|
|
|
off += _emit_LDP(dry_run, &buf[off], SINGLE, pxs->desc->peri);
|
|
|
|
|
off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri);
|
|
|
|
|
off += _emit_LDP(dry_run, &buf[off], cond, pxs->desc->peri);
|
|
|
|
|
off += _emit_ST(dry_run, &buf[off], ALWAYS);
|
|
|
|
|
off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri);
|
|
|
|
|
|
|
|
|
|
if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP))
|
|
|
|
|
off += _emit_FLUSHP(dry_run, &buf[off],
|
|
|
|
|
pxs->desc->peri);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return off;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int _ldst_memtodev(unsigned dry_run, u8 buf[],
|
|
|
|
|
const struct _xfer_spec *pxs, int cyc)
|
|
|
|
|
static inline int _ldst_memtodev(struct pl330_dmac *pl330,
|
|
|
|
|
unsigned dry_run, u8 buf[],
|
|
|
|
|
const struct _xfer_spec *pxs, int cyc)
|
|
|
|
|
{
|
|
|
|
|
int off = 0;
|
|
|
|
|
enum pl330_cond cond;
|
|
|
|
|
|
|
|
|
|
if (pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)
|
|
|
|
|
cond = BURST;
|
|
|
|
|
else
|
|
|
|
|
cond = SINGLE;
|
|
|
|
|
|
|
|
|
|
while (cyc--) {
|
|
|
|
|
off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->desc->peri);
|
|
|
|
|
off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri);
|
|
|
|
|
off += _emit_LD(dry_run, &buf[off], ALWAYS);
|
|
|
|
|
off += _emit_STP(dry_run, &buf[off], SINGLE, pxs->desc->peri);
|
|
|
|
|
off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri);
|
|
|
|
|
off += _emit_STP(dry_run, &buf[off], cond, pxs->desc->peri);
|
|
|
|
|
|
|
|
|
|
if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP))
|
|
|
|
|
off += _emit_FLUSHP(dry_run, &buf[off],
|
|
|
|
|
pxs->desc->peri);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return off;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int _bursts(unsigned dry_run, u8 buf[],
|
|
|
|
|
static int _bursts(struct pl330_dmac *pl330, unsigned dry_run, u8 buf[],
|
|
|
|
|
const struct _xfer_spec *pxs, int cyc)
|
|
|
|
|
{
|
|
|
|
|
int off = 0;
|
|
|
|
|
|
|
|
|
|
switch (pxs->desc->rqtype) {
|
|
|
|
|
case DMA_MEM_TO_DEV:
|
|
|
|
|
off += _ldst_memtodev(dry_run, &buf[off], pxs, cyc);
|
|
|
|
|
off += _ldst_memtodev(pl330, dry_run, &buf[off], pxs, cyc);
|
|
|
|
|
break;
|
|
|
|
|
case DMA_DEV_TO_MEM:
|
|
|
|
|
off += _ldst_devtomem(dry_run, &buf[off], pxs, cyc);
|
|
|
|
|
off += _ldst_devtomem(pl330, dry_run, &buf[off], pxs, cyc);
|
|
|
|
|
break;
|
|
|
|
|
case DMA_MEM_TO_MEM:
|
|
|
|
|
off += _ldst_memtomem(dry_run, &buf[off], pxs, cyc);
|
|
|
|
@@ -1191,7 +1225,7 @@ static int _bursts(unsigned dry_run, u8 buf[],
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Returns bytes consumed and updates bursts */
|
|
|
|
|
static inline int _loop(unsigned dry_run, u8 buf[],
|
|
|
|
|
static inline int _loop(struct pl330_dmac *pl330, unsigned dry_run, u8 buf[],
|
|
|
|
|
unsigned long *bursts, const struct _xfer_spec *pxs)
|
|
|
|
|
{
|
|
|
|
|
int cyc, cycmax, szlp, szlpend, szbrst, off;
|
|
|
|
@@ -1199,7 +1233,7 @@ static inline int _loop(unsigned dry_run, u8 buf[],
|
|
|
|
|
struct _arg_LPEND lpend;
|
|
|
|
|
|
|
|
|
|
if (*bursts == 1)
|
|
|
|
|
return _bursts(dry_run, buf, pxs, 1);
|
|
|
|
|
return _bursts(pl330, dry_run, buf, pxs, 1);
|
|
|
|
|
|
|
|
|
|
/* Max iterations possible in DMALP is 256 */
|
|
|
|
|
if (*bursts >= 256*256) {
|
|
|
|
@@ -1217,7 +1251,7 @@ static inline int _loop(unsigned dry_run, u8 buf[],
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
szlp = _emit_LP(1, buf, 0, 0);
|
|
|
|
|
szbrst = _bursts(1, buf, pxs, 1);
|
|
|
|
|
szbrst = _bursts(pl330, 1, buf, pxs, 1);
|
|
|
|
|
|
|
|
|
|
lpend.cond = ALWAYS;
|
|
|
|
|
lpend.forever = false;
|
|
|
|
@@ -1249,7 +1283,7 @@ static inline int _loop(unsigned dry_run, u8 buf[],
|
|
|
|
|
off += _emit_LP(dry_run, &buf[off], 1, lcnt1);
|
|
|
|
|
ljmp1 = off;
|
|
|
|
|
|
|
|
|
|
off += _bursts(dry_run, &buf[off], pxs, cyc);
|
|
|
|
|
off += _bursts(pl330, dry_run, &buf[off], pxs, cyc);
|
|
|
|
|
|
|
|
|
|
lpend.cond = ALWAYS;
|
|
|
|
|
lpend.forever = false;
|
|
|
|
@@ -1272,8 +1306,9 @@ static inline int _loop(unsigned dry_run, u8 buf[],
|
|
|
|
|
return off;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int _setup_loops(unsigned dry_run, u8 buf[],
|
|
|
|
|
const struct _xfer_spec *pxs)
|
|
|
|
|
static inline int _setup_loops(struct pl330_dmac *pl330,
|
|
|
|
|
unsigned dry_run, u8 buf[],
|
|
|
|
|
const struct _xfer_spec *pxs)
|
|
|
|
|
{
|
|
|
|
|
struct pl330_xfer *x = &pxs->desc->px;
|
|
|
|
|
u32 ccr = pxs->ccr;
|
|
|
|
@@ -1282,15 +1317,16 @@ static inline int _setup_loops(unsigned dry_run, u8 buf[],
|
|
|
|
|
|
|
|
|
|
while (bursts) {
|
|
|
|
|
c = bursts;
|
|
|
|
|
off += _loop(dry_run, &buf[off], &c, pxs);
|
|
|
|
|
off += _loop(pl330, dry_run, &buf[off], &c, pxs);
|
|
|
|
|
bursts -= c;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return off;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int _setup_xfer(unsigned dry_run, u8 buf[],
|
|
|
|
|
const struct _xfer_spec *pxs)
|
|
|
|
|
static inline int _setup_xfer(struct pl330_dmac *pl330,
|
|
|
|
|
unsigned dry_run, u8 buf[],
|
|
|
|
|
const struct _xfer_spec *pxs)
|
|
|
|
|
{
|
|
|
|
|
struct pl330_xfer *x = &pxs->desc->px;
|
|
|
|
|
int off = 0;
|
|
|
|
@@ -1301,7 +1337,7 @@ static inline int _setup_xfer(unsigned dry_run, u8 buf[],
|
|
|
|
|
off += _emit_MOV(dry_run, &buf[off], DAR, x->dst_addr);
|
|
|
|
|
|
|
|
|
|
/* Setup Loop(s) */
|
|
|
|
|
off += _setup_loops(dry_run, &buf[off], pxs);
|
|
|
|
|
off += _setup_loops(pl330, dry_run, &buf[off], pxs);
|
|
|
|
|
|
|
|
|
|
return off;
|
|
|
|
|
}
|
|
|
|
@@ -1310,8 +1346,9 @@ static inline int _setup_xfer(unsigned dry_run, u8 buf[],
|
|
|
|
|
* A req is a sequence of one or more xfer units.
|
|
|
|
|
* Returns the number of bytes taken to setup the MC for the req.
|
|
|
|
|
*/
|
|
|
|
|
static int _setup_req(unsigned dry_run, struct pl330_thread *thrd,
|
|
|
|
|
unsigned index, struct _xfer_spec *pxs)
|
|
|
|
|
static int _setup_req(struct pl330_dmac *pl330, unsigned dry_run,
|
|
|
|
|
struct pl330_thread *thrd, unsigned index,
|
|
|
|
|
struct _xfer_spec *pxs)
|
|
|
|
|
{
|
|
|
|
|
struct _pl330_req *req = &thrd->req[index];
|
|
|
|
|
struct pl330_xfer *x;
|
|
|
|
@@ -1328,7 +1365,7 @@ static int _setup_req(unsigned dry_run, struct pl330_thread *thrd,
|
|
|
|
|
if (x->bytes % (BRST_SIZE(pxs->ccr) * BRST_LEN(pxs->ccr)))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
off += _setup_xfer(dry_run, &buf[off], pxs);
|
|
|
|
|
off += _setup_xfer(pl330, dry_run, &buf[off], pxs);
|
|
|
|
|
|
|
|
|
|
/* DMASEV peripheral/event */
|
|
|
|
|
off += _emit_SEV(dry_run, &buf[off], thrd->ev);
|
|
|
|
@@ -1422,7 +1459,7 @@ static int pl330_submit_req(struct pl330_thread *thrd,
|
|
|
|
|
xs.desc = desc;
|
|
|
|
|
|
|
|
|
|
/* First dry run to check if req is acceptable */
|
|
|
|
|
ret = _setup_req(1, thrd, idx, &xs);
|
|
|
|
|
ret = _setup_req(pl330, 1, thrd, idx, &xs);
|
|
|
|
|
if (ret < 0)
|
|
|
|
|
goto xfer_exit;
|
|
|
|
|
|
|
|
|
@@ -1436,7 +1473,7 @@ static int pl330_submit_req(struct pl330_thread *thrd,
|
|
|
|
|
/* Hook the request */
|
|
|
|
|
thrd->lstenq = idx;
|
|
|
|
|
thrd->req[idx].desc = desc;
|
|
|
|
|
_setup_req(0, thrd, idx, &xs);
|
|
|
|
|
_setup_req(pl330, 0, thrd, idx, &xs);
|
|
|
|
|
|
|
|
|
|
ret = 0;
|
|
|
|
|
|
|
|
|
@@ -2781,6 +2818,7 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id)
|
|
|
|
|
struct resource *res;
|
|
|
|
|
int i, ret, irq;
|
|
|
|
|
int num_chan;
|
|
|
|
|
struct device_node *np = adev->dev.of_node;
|
|
|
|
|
|
|
|
|
|
pdat = dev_get_platdata(&adev->dev);
|
|
|
|
|
|
|
|
|
@@ -2800,6 +2838,11 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id)
|
|
|
|
|
|
|
|
|
|
pl330->mcbufsz = pdat ? pdat->mcbuf_sz : 0;
|
|
|
|
|
|
|
|
|
|
/* get quirk */
|
|
|
|
|
for (i = 0; i < ARRAY_SIZE(of_quirks); i++)
|
|
|
|
|
if (of_property_read_bool(np, of_quirks[i].quirk))
|
|
|
|
|
pl330->quirks |= of_quirks[i].id;
|
|
|
|
|
|
|
|
|
|
res = &adev->res;
|
|
|
|
|
pl330->base = devm_ioremap_resource(&adev->dev, res);
|
|
|
|
|
if (IS_ERR(pl330->base))
|
|
|
|
@@ -2895,6 +2938,8 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id)
|
|
|
|
|
pd->dst_addr_widths = PL330_DMA_BUSWIDTHS;
|
|
|
|
|
pd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
|
|
|
|
|
pd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
|
|
|
|
|
pd->max_burst = ((pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP) ?
|
|
|
|
|
1 : PL330_MAX_BURST);
|
|
|
|
|
|
|
|
|
|
ret = dma_async_device_register(pd);
|
|
|
|
|
if (ret) {
|
|
|
|
|