IB: Refactor umem to use linear SG table
This patch refactors the IB core umem code and vendor drivers to use a linear (chained) SG table instead of chunk list. With this change the relevant code becomes clearer—no need for nested loops to build and use umem. Signed-off-by: Shachar Raindel <raindel@mellanox.com> Signed-off-by: Yishai Hadas <yishaih@mellanox.com> Signed-off-by: Roland Dreier <roland@purestorage.com>
This commit is contained in:
Yishai Hadas
committed by
Roland Dreier
Roland Dreier
szülő
cfbf8d4857
commit
eeb8461e36
@@ -2307,7 +2307,7 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
|
||||
struct nes_device *nesdev = nesvnic->nesdev;
|
||||
struct nes_adapter *nesadapter = nesdev->nesadapter;
|
||||
struct ib_mr *ibmr = ERR_PTR(-EINVAL);
|
||||
struct ib_umem_chunk *chunk;
|
||||
struct scatterlist *sg;
|
||||
struct nes_ucontext *nes_ucontext;
|
||||
struct nes_pbl *nespbl;
|
||||
struct nes_mr *nesmr;
|
||||
@@ -2315,7 +2315,7 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
|
||||
struct nes_mem_reg_req req;
|
||||
struct nes_vpbl vpbl;
|
||||
struct nes_root_vpbl root_vpbl;
|
||||
int nmap_index, page_index;
|
||||
int entry, page_index;
|
||||
int page_count = 0;
|
||||
int err, pbl_depth = 0;
|
||||
int chunk_pages;
|
||||
@@ -2330,6 +2330,7 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
|
||||
u16 pbl_count;
|
||||
u8 single_page = 1;
|
||||
u8 stag_key;
|
||||
int first_page = 1;
|
||||
|
||||
region = ib_umem_get(pd->uobject->context, start, length, acc, 0);
|
||||
if (IS_ERR(region)) {
|
||||
@@ -2380,128 +2381,125 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
|
||||
}
|
||||
nesmr->region = region;
|
||||
|
||||
list_for_each_entry(chunk, ®ion->chunk_list, list) {
|
||||
nes_debug(NES_DBG_MR, "Chunk: nents = %u, nmap = %u .\n",
|
||||
chunk->nents, chunk->nmap);
|
||||
for (nmap_index = 0; nmap_index < chunk->nmap; ++nmap_index) {
|
||||
if (sg_dma_address(&chunk->page_list[nmap_index]) & ~PAGE_MASK) {
|
||||
ib_umem_release(region);
|
||||
nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
|
||||
nes_debug(NES_DBG_MR, "Unaligned Memory Buffer: 0x%x\n",
|
||||
(unsigned int) sg_dma_address(&chunk->page_list[nmap_index]));
|
||||
ibmr = ERR_PTR(-EINVAL);
|
||||
kfree(nesmr);
|
||||
goto reg_user_mr_err;
|
||||
}
|
||||
for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) {
|
||||
if (sg_dma_address(sg) & ~PAGE_MASK) {
|
||||
ib_umem_release(region);
|
||||
nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
|
||||
nes_debug(NES_DBG_MR, "Unaligned Memory Buffer: 0x%x\n",
|
||||
(unsigned int) sg_dma_address(sg));
|
||||
ibmr = ERR_PTR(-EINVAL);
|
||||
kfree(nesmr);
|
||||
goto reg_user_mr_err;
|
||||
}
|
||||
|
||||
if (!sg_dma_len(&chunk->page_list[nmap_index])) {
|
||||
ib_umem_release(region);
|
||||
nes_free_resource(nesadapter, nesadapter->allocated_mrs,
|
||||
stag_index);
|
||||
nes_debug(NES_DBG_MR, "Invalid Buffer Size\n");
|
||||
ibmr = ERR_PTR(-EINVAL);
|
||||
kfree(nesmr);
|
||||
goto reg_user_mr_err;
|
||||
}
|
||||
if (!sg_dma_len(sg)) {
|
||||
ib_umem_release(region);
|
||||
nes_free_resource(nesadapter, nesadapter->allocated_mrs,
|
||||
stag_index);
|
||||
nes_debug(NES_DBG_MR, "Invalid Buffer Size\n");
|
||||
ibmr = ERR_PTR(-EINVAL);
|
||||
kfree(nesmr);
|
||||
goto reg_user_mr_err;
|
||||
}
|
||||
|
||||
region_length += sg_dma_len(&chunk->page_list[nmap_index]);
|
||||
chunk_pages = sg_dma_len(&chunk->page_list[nmap_index]) >> 12;
|
||||
region_length -= skip_pages << 12;
|
||||
for (page_index=skip_pages; page_index < chunk_pages; page_index++) {
|
||||
skip_pages = 0;
|
||||
if ((page_count!=0)&&(page_count<<12)-(region->offset&(4096-1))>=region->length)
|
||||
goto enough_pages;
|
||||
if ((page_count&0x01FF) == 0) {
|
||||
if (page_count >= 1024 * 512) {
|
||||
region_length += sg_dma_len(sg);
|
||||
chunk_pages = sg_dma_len(sg) >> 12;
|
||||
region_length -= skip_pages << 12;
|
||||
for (page_index = skip_pages; page_index < chunk_pages; page_index++) {
|
||||
skip_pages = 0;
|
||||
if ((page_count != 0) && (page_count<<12)-(region->offset&(4096-1)) >= region->length)
|
||||
goto enough_pages;
|
||||
if ((page_count&0x01FF) == 0) {
|
||||
if (page_count >= 1024 * 512) {
|
||||
ib_umem_release(region);
|
||||
nes_free_resource(nesadapter,
|
||||
nesadapter->allocated_mrs, stag_index);
|
||||
kfree(nesmr);
|
||||
ibmr = ERR_PTR(-E2BIG);
|
||||
goto reg_user_mr_err;
|
||||
}
|
||||
if (root_pbl_index == 1) {
|
||||
root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev,
|
||||
8192, &root_vpbl.pbl_pbase);
|
||||
nes_debug(NES_DBG_MR, "Allocating root PBL, va = %p, pa = 0x%08X\n",
|
||||
root_vpbl.pbl_vbase, (unsigned int)root_vpbl.pbl_pbase);
|
||||
if (!root_vpbl.pbl_vbase) {
|
||||
ib_umem_release(region);
|
||||
nes_free_resource(nesadapter,
|
||||
nesadapter->allocated_mrs, stag_index);
|
||||
pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
|
||||
vpbl.pbl_pbase);
|
||||
nes_free_resource(nesadapter, nesadapter->allocated_mrs,
|
||||
stag_index);
|
||||
kfree(nesmr);
|
||||
ibmr = ERR_PTR(-E2BIG);
|
||||
goto reg_user_mr_err;
|
||||
}
|
||||
if (root_pbl_index == 1) {
|
||||
root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev,
|
||||
8192, &root_vpbl.pbl_pbase);
|
||||
nes_debug(NES_DBG_MR, "Allocating root PBL, va = %p, pa = 0x%08X\n",
|
||||
root_vpbl.pbl_vbase, (unsigned int)root_vpbl.pbl_pbase);
|
||||
if (!root_vpbl.pbl_vbase) {
|
||||
ib_umem_release(region);
|
||||
pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
|
||||
vpbl.pbl_pbase);
|
||||
nes_free_resource(nesadapter, nesadapter->allocated_mrs,
|
||||
stag_index);
|
||||
kfree(nesmr);
|
||||
ibmr = ERR_PTR(-ENOMEM);
|
||||
goto reg_user_mr_err;
|
||||
}
|
||||
root_vpbl.leaf_vpbl = kzalloc(sizeof(*root_vpbl.leaf_vpbl)*1024,
|
||||
GFP_KERNEL);
|
||||
if (!root_vpbl.leaf_vpbl) {
|
||||
ib_umem_release(region);
|
||||
pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase,
|
||||
root_vpbl.pbl_pbase);
|
||||
pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
|
||||
vpbl.pbl_pbase);
|
||||
nes_free_resource(nesadapter, nesadapter->allocated_mrs,
|
||||
stag_index);
|
||||
kfree(nesmr);
|
||||
ibmr = ERR_PTR(-ENOMEM);
|
||||
goto reg_user_mr_err;
|
||||
}
|
||||
root_vpbl.pbl_vbase[0].pa_low =
|
||||
cpu_to_le32((u32)vpbl.pbl_pbase);
|
||||
root_vpbl.pbl_vbase[0].pa_high =
|
||||
cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32)));
|
||||
root_vpbl.leaf_vpbl[0] = vpbl;
|
||||
}
|
||||
vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096,
|
||||
&vpbl.pbl_pbase);
|
||||
nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%08X\n",
|
||||
vpbl.pbl_vbase, (unsigned int)vpbl.pbl_pbase);
|
||||
if (!vpbl.pbl_vbase) {
|
||||
ib_umem_release(region);
|
||||
nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
|
||||
ibmr = ERR_PTR(-ENOMEM);
|
||||
kfree(nesmr);
|
||||
goto reg_user_mr_err;
|
||||
}
|
||||
if (1 <= root_pbl_index) {
|
||||
root_vpbl.pbl_vbase[root_pbl_index].pa_low =
|
||||
cpu_to_le32((u32)vpbl.pbl_pbase);
|
||||
root_vpbl.pbl_vbase[root_pbl_index].pa_high =
|
||||
cpu_to_le32((u32)((((u64)vpbl.pbl_pbase)>>32)));
|
||||
root_vpbl.leaf_vpbl[root_pbl_index] = vpbl;
|
||||
root_vpbl.leaf_vpbl = kzalloc(sizeof(*root_vpbl.leaf_vpbl)*1024,
|
||||
GFP_KERNEL);
|
||||
if (!root_vpbl.leaf_vpbl) {
|
||||
ib_umem_release(region);
|
||||
pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase,
|
||||
root_vpbl.pbl_pbase);
|
||||
pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
|
||||
vpbl.pbl_pbase);
|
||||
nes_free_resource(nesadapter, nesadapter->allocated_mrs,
|
||||
stag_index);
|
||||
kfree(nesmr);
|
||||
ibmr = ERR_PTR(-ENOMEM);
|
||||
goto reg_user_mr_err;
|
||||
}
|
||||
root_pbl_index++;
|
||||
cur_pbl_index = 0;
|
||||
root_vpbl.pbl_vbase[0].pa_low =
|
||||
cpu_to_le32((u32)vpbl.pbl_pbase);
|
||||
root_vpbl.pbl_vbase[0].pa_high =
|
||||
cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32)));
|
||||
root_vpbl.leaf_vpbl[0] = vpbl;
|
||||
}
|
||||
if (single_page) {
|
||||
if (page_count != 0) {
|
||||
if ((last_dma_addr+4096) !=
|
||||
(sg_dma_address(&chunk->page_list[nmap_index])+
|
||||
(page_index*4096)))
|
||||
single_page = 0;
|
||||
last_dma_addr = sg_dma_address(&chunk->page_list[nmap_index])+
|
||||
(page_index*4096);
|
||||
} else {
|
||||
first_dma_addr = sg_dma_address(&chunk->page_list[nmap_index])+
|
||||
(page_index*4096);
|
||||
last_dma_addr = first_dma_addr;
|
||||
}
|
||||
vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096,
|
||||
&vpbl.pbl_pbase);
|
||||
nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%08X\n",
|
||||
vpbl.pbl_vbase, (unsigned int)vpbl.pbl_pbase);
|
||||
if (!vpbl.pbl_vbase) {
|
||||
ib_umem_release(region);
|
||||
nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
|
||||
ibmr = ERR_PTR(-ENOMEM);
|
||||
kfree(nesmr);
|
||||
goto reg_user_mr_err;
|
||||
}
|
||||
|
||||
vpbl.pbl_vbase[cur_pbl_index].pa_low =
|
||||
cpu_to_le32((u32)(sg_dma_address(&chunk->page_list[nmap_index])+
|
||||
(page_index*4096)));
|
||||
vpbl.pbl_vbase[cur_pbl_index].pa_high =
|
||||
cpu_to_le32((u32)((((u64)(sg_dma_address(&chunk->page_list[nmap_index])+
|
||||
(page_index*4096))) >> 32)));
|
||||
cur_pbl_index++;
|
||||
page_count++;
|
||||
if (1 <= root_pbl_index) {
|
||||
root_vpbl.pbl_vbase[root_pbl_index].pa_low =
|
||||
cpu_to_le32((u32)vpbl.pbl_pbase);
|
||||
root_vpbl.pbl_vbase[root_pbl_index].pa_high =
|
||||
cpu_to_le32((u32)((((u64)vpbl.pbl_pbase)>>32)));
|
||||
root_vpbl.leaf_vpbl[root_pbl_index] = vpbl;
|
||||
}
|
||||
root_pbl_index++;
|
||||
cur_pbl_index = 0;
|
||||
}
|
||||
if (single_page) {
|
||||
if (page_count != 0) {
|
||||
if ((last_dma_addr+4096) !=
|
||||
(sg_dma_address(sg)+
|
||||
(page_index*4096)))
|
||||
single_page = 0;
|
||||
last_dma_addr = sg_dma_address(sg)+
|
||||
(page_index*4096);
|
||||
} else {
|
||||
first_dma_addr = sg_dma_address(sg)+
|
||||
(page_index*4096);
|
||||
last_dma_addr = first_dma_addr;
|
||||
}
|
||||
}
|
||||
|
||||
vpbl.pbl_vbase[cur_pbl_index].pa_low =
|
||||
cpu_to_le32((u32)(sg_dma_address(sg)+
|
||||
(page_index*4096)));
|
||||
vpbl.pbl_vbase[cur_pbl_index].pa_high =
|
||||
cpu_to_le32((u32)((((u64)(sg_dma_address(sg)+
|
||||
(page_index*4096))) >> 32)));
|
||||
cur_pbl_index++;
|
||||
page_count++;
|
||||
}
|
||||
}
|
||||
|
||||
enough_pages:
|
||||
nes_debug(NES_DBG_MR, "calculating stag, stag_index=0x%08x, driver_key=0x%08x,"
|
||||
" stag_key=0x%08x\n",
|
||||
@@ -2613,25 +2611,28 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
|
||||
nespbl->pbl_size, (unsigned long) nespbl->pbl_pbase,
|
||||
(void *) nespbl->pbl_vbase, nespbl->user_base);
|
||||
|
||||
list_for_each_entry(chunk, ®ion->chunk_list, list) {
|
||||
for (nmap_index = 0; nmap_index < chunk->nmap; ++nmap_index) {
|
||||
chunk_pages = sg_dma_len(&chunk->page_list[nmap_index]) >> 12;
|
||||
chunk_pages += (sg_dma_len(&chunk->page_list[nmap_index]) & (4096-1)) ? 1 : 0;
|
||||
nespbl->page = sg_page(&chunk->page_list[0]);
|
||||
for (page_index=0; page_index<chunk_pages; page_index++) {
|
||||
((__le32 *)pbl)[0] = cpu_to_le32((u32)
|
||||
(sg_dma_address(&chunk->page_list[nmap_index])+
|
||||
(page_index*4096)));
|
||||
((__le32 *)pbl)[1] = cpu_to_le32(((u64)
|
||||
(sg_dma_address(&chunk->page_list[nmap_index])+
|
||||
(page_index*4096)))>>32);
|
||||
nes_debug(NES_DBG_MR, "pbl=%p, *pbl=0x%016llx, 0x%08x%08x\n", pbl,
|
||||
(unsigned long long)*pbl,
|
||||
le32_to_cpu(((__le32 *)pbl)[1]), le32_to_cpu(((__le32 *)pbl)[0]));
|
||||
pbl++;
|
||||
}
|
||||
for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) {
|
||||
chunk_pages = sg_dma_len(sg) >> 12;
|
||||
chunk_pages += (sg_dma_len(sg) & (4096-1)) ? 1 : 0;
|
||||
if (first_page) {
|
||||
nespbl->page = sg_page(sg);
|
||||
first_page = 0;
|
||||
}
|
||||
|
||||
for (page_index = 0; page_index < chunk_pages; page_index++) {
|
||||
((__le32 *)pbl)[0] = cpu_to_le32((u32)
|
||||
(sg_dma_address(sg)+
|
||||
(page_index*4096)));
|
||||
((__le32 *)pbl)[1] = cpu_to_le32(((u64)
|
||||
(sg_dma_address(sg)+
|
||||
(page_index*4096)))>>32);
|
||||
nes_debug(NES_DBG_MR, "pbl=%p, *pbl=0x%016llx, 0x%08x%08x\n", pbl,
|
||||
(unsigned long long)*pbl,
|
||||
le32_to_cpu(((__le32 *)pbl)[1]), le32_to_cpu(((__le32 *)pbl)[0]));
|
||||
pbl++;
|
||||
}
|
||||
}
|
||||
|
||||
if (req.reg_type == IWNES_MEMREG_TYPE_QP) {
|
||||
list_add_tail(&nespbl->list, &nes_ucontext->qp_reg_mem_list);
|
||||
} else {
|
||||
|
||||
Reference in New Issue
Block a user