NTB: Alter MW API to support multi-ports devices

Multi-port NTB devices permit to share a memory between all accessible peers.
Memory Windows API is altered to correspondingly initialize and map memory
windows for such devices:
 ntb_mw_count(pidx); - number of inbound memory windows, which can be allocated
for shared buffer with specified peer device.
 ntb_mw_get_align(pidx, widx); - get alignment and size restriction parameters
to properly allocate inbound memory region.
 ntb_peer_mw_count(); - get number of outbound memory windows.
 ntb_peer_mw_get_addr(widx); - get mapping address of an outbound memory window

If hardware supports inbound translation configured on the local ntb port:
 ntb_mw_set_trans(pidx, widx); - set translation address of allocated inbound
memory window so a peer device could access it.
 ntb_mw_clear_trans(pidx, widx); - clear the translation address of an inbound
memory window.

If hardware supports outbound translation configured on the peer ntb port:
 ntb_peer_mw_set_trans(pidx, widx); - set translation address of a memory
window retrieved from a peer device
 ntb_peer_mw_clear_trans(pidx, widx); - clear the translation address of an
outbound memory window

Signed-off-by: Serge Semin <fancer.lancer@gmail.com>
Acked-by: Allen Hubbe <Allen.Hubbe@dell.com>
Signed-off-by: Jon Mason <jdmason@kudzu.us>
This commit is contained in:
Serge Semin
2017-01-11 03:11:33 +03:00
committed by Jon Mason
parent 4e8c11b7fd
commit 443b9a14ec
7 changed files with 340 additions and 105 deletions

View File

@@ -119,7 +119,8 @@ MODULE_VERSION(DRIVER_VERSION);
MODULE_AUTHOR(DRIVER_AUTHOR);
MODULE_DESCRIPTION(DRIVER_DESCRIPTION);
#define MAX_MWS 16
/* It is rare to have hadrware with greater than six MWs */
#define MAX_MWS 6
/* Only two-ports devices are supported */
#define PIDX NTB_DEF_PEER_IDX
@@ -670,28 +671,27 @@ static int tool_setup_mw(struct tool_ctx *tc, int idx, size_t req_size)
{
int rc;
struct tool_mw *mw = &tc->mws[idx];
phys_addr_t base;
resource_size_t size, align, align_size;
resource_size_t size, align_addr, align_size;
char buf[16];
if (mw->peer)
return 0;
rc = ntb_mw_get_range(tc->ntb, idx, &base, &size, &align,
&align_size);
rc = ntb_mw_get_align(tc->ntb, PIDX, idx, &align_addr,
&align_size, &size);
if (rc)
return rc;
mw->size = min_t(resource_size_t, req_size, size);
mw->size = round_up(mw->size, align);
mw->size = round_up(mw->size, align_addr);
mw->size = round_up(mw->size, align_size);
mw->peer = dma_alloc_coherent(&tc->ntb->pdev->dev, mw->size,
&mw->peer_dma, GFP_KERNEL);
if (!mw->peer)
if (!mw->peer || !IS_ALIGNED(mw->peer_dma, align_addr))
return -ENOMEM;
rc = ntb_mw_set_trans(tc->ntb, idx, mw->peer_dma, mw->size);
rc = ntb_mw_set_trans(tc->ntb, PIDX, idx, mw->peer_dma, mw->size);
if (rc)
goto err_free_dma;
@@ -718,7 +718,7 @@ static void tool_free_mw(struct tool_ctx *tc, int idx)
struct tool_mw *mw = &tc->mws[idx];
if (mw->peer) {
ntb_mw_clear_trans(tc->ntb, idx);
ntb_mw_clear_trans(tc->ntb, PIDX, idx);
dma_free_coherent(&tc->ntb->pdev->dev, mw->size,
mw->peer,
mw->peer_dma);
@@ -744,8 +744,9 @@ static ssize_t tool_peer_mw_trans_read(struct file *filep,
phys_addr_t base;
resource_size_t mw_size;
resource_size_t align;
resource_size_t align_addr;
resource_size_t align_size;
resource_size_t max_size;
buf_size = min_t(size_t, size, 512);
@@ -753,8 +754,9 @@ static ssize_t tool_peer_mw_trans_read(struct file *filep,
if (!buf)
return -ENOMEM;
ntb_mw_get_range(mw->tc->ntb, mw->idx,
&base, &mw_size, &align, &align_size);
ntb_mw_get_align(mw->tc->ntb, PIDX, mw->idx,
&align_addr, &align_size, &max_size);
ntb_peer_mw_get_addr(mw->tc->ntb, mw->idx, &base, &mw_size);
off += scnprintf(buf + off, buf_size - off,
"Peer MW %d Information:\n", mw->idx);
@@ -769,12 +771,16 @@ static ssize_t tool_peer_mw_trans_read(struct file *filep,
off += scnprintf(buf + off, buf_size - off,
"Alignment \t%lld\n",
(unsigned long long)align);
(unsigned long long)align_addr);
off += scnprintf(buf + off, buf_size - off,
"Size Alignment \t%lld\n",
(unsigned long long)align_size);
off += scnprintf(buf + off, buf_size - off,
"Size Max \t%lld\n",
(unsigned long long)max_size);
off += scnprintf(buf + off, buf_size - off,
"Ready \t%c\n",
(mw->peer) ? 'Y' : 'N');
@@ -829,8 +835,7 @@ static int tool_init_mw(struct tool_ctx *tc, int idx)
phys_addr_t base;
int rc;
rc = ntb_mw_get_range(tc->ntb, idx, &base, &mw->win_size,
NULL, NULL);
rc = ntb_peer_mw_get_addr(tc->ntb, idx, &base, &mw->win_size);
if (rc)
return rc;
@@ -915,6 +920,12 @@ static int tool_probe(struct ntb_client *self, struct ntb_dev *ntb)
int rc;
int i;
if (!ntb->ops->mw_set_trans) {
dev_dbg(&ntb->dev, "need inbound MW based NTB API\n");
rc = -EINVAL;
goto err_tc;
}
if (ntb_db_is_unsafe(ntb))
dev_dbg(&ntb->dev, "doorbell is unsafe\n");
@@ -933,7 +944,7 @@ static int tool_probe(struct ntb_client *self, struct ntb_dev *ntb)
tc->ntb = ntb;
init_waitqueue_head(&tc->link_wq);
tc->mw_count = min(ntb_mw_count(tc->ntb), MAX_MWS);
tc->mw_count = min(ntb_mw_count(tc->ntb, PIDX), MAX_MWS);
for (i = 0; i < tc->mw_count; i++) {
rc = tool_init_mw(tc, i);
if (rc)